diff --git a/engine/src/main/java/org/archive/crawler/framework/Engine.java b/engine/src/main/java/org/archive/crawler/framework/Engine.java index b63b22f2..0659e3d8 100644 --- a/engine/src/main/java/org/archive/crawler/framework/Engine.java +++ b/engine/src/main/java/org/archive/crawler/framework/Engine.java @@ -51,7 +51,7 @@ public class Engine { final private static Logger LOGGER = Logger.getLogger(Engine.class.getName()); - /** directory where job directores are expected */ + /** directory where job directories are expected */ protected File jobsDir; /** map of job short names -> CrawlJob instances */ protected HashMap jobConfigs = new HashMap(); @@ -69,7 +69,8 @@ public class Engine { /** * Find all job configurations in the usual place -- subdirectories - * of the jobs directory with files ending '.cxml'. + * of the jobs directory with files ending '.cxml', and from jobPathFiles + * (previously added by user) found in the jobs directory */ public void findJobConfigs() { // TODO: allow other places/paths to be scanned/added as well? @@ -83,54 +84,63 @@ public class Engine { } } - // discover any new job directories + // just in case... if (! jobsDir.exists()) { - LOGGER.log(Level.WARNING,"jobsDir has disappeared: "+jobsDir.toString()); - } else { - for (File jobFile: jobsDir.listFiles()) { - if (jobFile.isDirectory()) { - considerAsJobDirectory(jobFile); - } else if (jobFile.getName().endsWith(".jobpath")) { - considerAsJobPath(jobFile); - } - } + LOGGER.log(Level.SEVERE,"jobsDir has disappeared: "+jobsDir.toString()); + return; + } + + // discover any new job directories + for (File jobFile: jobsDir.listFiles()) { + if (jobFile.getName().endsWith(".jobpath")) { + String jobPath = getJobPathFromFile(jobFile); + jobFile = new File(jobPath); + if (jobFile.isDirectory()) { + if (!addJobDirectory(jobFile,false)) { + LOGGER.log(Level.WARNING,"invalid job dir: " + jobPath + + " specified in jobpath file: " + + jobFile.getAbsolutePath()); + } + } else { + LOGGER.log(Level.WARNING,"non-directory: " + jobPath + + " specified in jobpath file: " + + jobFile.getAbsolutePath()); + } + } + if (jobFile.isDirectory()) { + addJobDirectory(jobFile,false); + } } } - private void considerAsJobPath(File jobpathFile) { - try { - String pathToJob = FileUtils.readFileToString(jobpathFile).trim(); - File jobPathFromFile = new File(pathToJob); - if (jobPathFromFile.isDirectory()) { - if (!considerAsJobDirectory(jobPathFromFile)) { - LOGGER.log(Level.WARNING,"invalid job path: " - + "'" + jobPathFromFile.toString().trim() + "'" - + " specified in jobpathFile: " - + jobpathFile.toString()); - } - } else { - // invalid job path specified - LOGGER.log(Level.WARNING,"invalid job path: " - + "'" + jobPathFromFile.toString().trim() + "'" - + " specified in jobpathFile: " - + jobpathFile.toString()); - } - } catch (IOException e) { - // problem reading jobpathFile - LOGGER.log(Level.WARNING, - "could not read jobPath from jobpathFile: " - + jobpathFile.toString()); - e.printStackTrace(); - } + /* + * read path from ".jobpath" file (jobpathFile) and add directory to jobConfig + */ + protected String getJobPathFromFile(File jobPathFile) { + try { + String pathToJob = FileUtils.readFileToString(jobPathFile).trim(); + return pathToJob; + } catch (IOException e) { + LOGGER.log(Level.SEVERE,"error reading jobPathFile: " + + jobPathFile.toString()); + e.printStackTrace(); + return null; + } } - public boolean considerAsJobDirectory(File dir) { + /** + * adds a job directory to the Engine jobConfigs if not extant + * @param dir directory to be added + * @param userRequest calls writeJobPathFile when true + * @return true if directory successfully added to jobConfigs + */ + public boolean addJobDirectory(File dir, Boolean userRequest) { File[] candidateConfigs = dir.listFiles(new FilenameFilter() { public boolean accept(File dir, String name) { return name.endsWith(".cxml"); }}); - if(candidateConfigs==null) { - // directory did not exist or did not contain cxml + if(candidateConfigs.length == 0) { + // no CXML file found! return false; } for (File cxml : candidateConfigs) { @@ -138,12 +148,17 @@ public class Engine { CrawlJob cj = new CrawlJob(cxml); if(!jobConfigs.containsKey(cj.getShortName())) { jobConfigs.put(cj.getShortName(),cj); - LOGGER.log(Level.INFO,"added crawl job: " +cj.getShortName()); - return true; + if (userRequest) { + writeJobPathFile(dir.getAbsolutePath()); + } + LOGGER.log(Level.INFO,"added crawl job: " + cj.getShortName()); } else { - // jobConfig exists - return true; + if (userRequest) { + LOGGER.log(Level.INFO,"requested job to add already exists: " + + cj.getShortName()); + } } + return true; } catch (IllegalArgumentException iae) { LOGGER.log(Level.WARNING,"bad cxml: "+cxml,iae); } @@ -329,41 +344,57 @@ public class Engine { return getClass().getResourceAsStream(profileCxmlPath); } - public boolean createNewJobWithDefaults(String path) throws IOException { + /** + * create a new job dir and copy profile CXML into as non-profile CXML + * @param newJobDir new job directory + * @throws IOException + */ + public boolean createNewJobWithDefaults(File newJobDir) { + try { + // get crawler-beans template into string + InputStream inStream = getProfileCxmlResource(); + String defaultCxmlStr; + defaultCxmlStr = IOUtils.toString(inStream); + inStream.close(); - File newJobDir = new File(jobsDir,"/"+path); - if (newJobDir.exists()) { - throw new IOException("file exists: "+newJobDir); - } - newJobDir.mkdirs(); + // write default crawler-beans string to new job dir + newJobDir.mkdirs(); + File newJobCxml = new File(newJobDir,"crawler-beans.cxml"); + FileUtils.writeStringToFile(newJobCxml, defaultCxmlStr); - // get crawler-beans template from this package into string - InputStream inStream = getProfileCxmlResource(); - String defaultCxmlStr = IOUtils.toString(inStream); - inStream.close(); - - // write default crawler-beans string to new job config - File newJobCxml = new File(newJobDir,"crawler-beans.cxml"); - FileUtils.writeStringToFile(newJobCxml, defaultCxmlStr); + return true; - return true; + } catch (IOException e) { + LOGGER.log(Level.SEVERE,"failed to create new job: " + + newJobDir.getAbsolutePath()); + return false; + } } - public void leaveJobPathFile(String path) throws IOException { - String jobName = path.substring(path.lastIndexOf("/")+1,path.length()); + /** + * writes a .jobpath file for "added" paths + * @param path + * @throws IOException + */ + public void writeJobPathFile(String path) { + String jobName = path.substring(path.lastIndexOf(File.separatorChar)+1, + path.length()); if (jobConfigs.containsKey(jobName)) { String jobpathFileName = jobName+".jobpath"; File jobpathFile = new File(jobsDir,jobpathFileName); try { - FileUtils.writeStringToFile(jobpathFile, path+"\n"); - System.out.println("Engine.leaveJobPathFile() wrote file: " - + jobpathFileName); + if (!jobpathFile.exists()) { + FileUtils.writeStringToFile(jobpathFile, path+"\n"); + LOGGER.log(Level.INFO, "wrote jobpath file: " + + jobpathFileName); + } } catch (IOException e) { - throw new IOException("could not create jobpathFile: " - + jobpathFile.toString()); + LOGGER.log(Level.SEVERE,"failed to write jobPathFile: " + + jobpathFileName + "\n" + + e.getMessage()); } } else { - LOGGER.log(Level.SEVERE,"job: "+jobName+" not found in jobConfig!"); + LOGGER.log(Level.SEVERE,"added job: " + jobName + " not found!"); } } } \ No newline at end of file diff --git a/engine/src/main/java/org/archive/crawler/restlet/EngineResource.java b/engine/src/main/java/org/archive/crawler/restlet/EngineResource.java index 4bc0a74a..dc746ac7 100644 --- a/engine/src/main/java/org/archive/crawler/restlet/EngineResource.java +++ b/engine/src/main/java/org/archive/crawler/restlet/EngineResource.java @@ -26,13 +26,11 @@ import java.io.Writer; import java.util.ArrayList; import java.util.Collection; import java.util.Collections; -import java.util.HashMap; import java.util.LinkedHashMap; import java.util.LinkedList; import java.util.List; import java.util.Map; -import org.apache.commons.lang.StringUtils; import org.archive.crawler.framework.CrawlJob; import org.archive.crawler.framework.Engine; import org.archive.crawler.framework.CrawlController.State; @@ -92,56 +90,64 @@ public class EngineResource extends BaseResource { getEngine().findJobConfigs(); } else if ("add".equals(action)) { String path = form.getFirstValue("addpath"); - if(path==null) { - path = ""; - } - boolean added = false; - if(StringUtils.isNotBlank(path)) { - added = getEngine().considerAsJobDirectory(new File(path)); - } - if(!added) { - String msg = messageDiv("ERROR! invalid job path: '"+path+"'","ERROR"); - Flash.addFlash(getResponse(),msg,Flash.Kind.NACK); + if (path==null) { + Flash.addFlash(getResponse(), "Cannot add null path", + Flash.Kind.NACK); } else { - try { - getEngine().leaveJobPathFile(path); - String msg = messageDiv("Added job path: '"+path+"'","MESSAGE"); - Flash.addFlash(getResponse(),msg); - } catch (IOException e) { - String msg = messageDiv(e.getMessage(),"ERROR"); - Flash.addFlash(getResponse(),msg,Flash.Kind.NACK); - } + File jobFile = new File(path); + String jobName = jobFile.getName(); + if (!jobFile.isDirectory()) { + Flash.addFlash(getResponse(), "Cannot add non-directory: " + + path + "", Flash.Kind.NACK); + } else if (getEngine().getJobConfigs().containsKey(jobName)) { + Flash.addFlash(getResponse(), "Job exists: " + + jobName + "", Flash.Kind.NACK); + } else if (getEngine().addJobDirectory(new File(path),true)) { + Flash.addFlash(getResponse(), "Added crawl job: " + + "'" + path + "'", Flash.Kind.NACK); + } else { + Flash.addFlash(getResponse(), "Could not add job: " + + "'" + path + "'", Flash.Kind.NACK); + } } } else if ("create".equals(action)) { - String path = form.getFirstValue("addpath"); + String path = form.getFirstValue("createpath"); if (path==null) { - String warn = messageDiv("WARNING! null path given.","WARNING"); - Flash.addFlash(getResponse(), warn, Flash.Kind.NACK); - } else if (path.indexOf("/") != -1) { - String warn = messageDiv("WARNING! sub-directories disallowed: " + path + "","WARNING"); - Flash.addFlash(getResponse(), warn, Flash.Kind.NACK); + // protect against null path + Flash.addFlash(getResponse(), "Cannot create null path.", + Flash.Kind.NACK); + } else if (path.indexOf(File.separatorChar) != -1) { + // prevent specifying sub-directories + Flash.addFlash(getResponse(), "Sub-directories disallowed: " + + "" + path + "", Flash.Kind.NACK); } else if (getEngine().getJobConfigs().containsKey(path)) { - String warn = messageDiv("ERROR! job exists: " + path + "","ERROR"); - Flash.addFlash(getResponse(), warn, Flash.Kind.NACK); + // protect existing jobs + Flash.addFlash(getResponse(), "Job exists: " + path + "", + Flash.Kind.NACK); } else { - boolean created = false; - try { - created = getEngine().createNewJobWithDefaults(path); - } catch (IOException e) { - String err = messageDiv("ERROR! " + e.toString(),"ERROR"); - Flash.addFlash(getResponse(), err, Flash.Kind.NACK); - } - if (created) { - String msg = messageDiv("Successfully created job: " + path + "","MESSAGE"); - Flash.addFlash(getResponse(), msg, Flash.Kind.ACK); - getEngine().findJobConfigs(); - } + // try to create new job dir + File newJobDir = new File(getEngine().getJobsDir(),path); + if (newJobDir.exists()) { + // protect existing directories + Flash.addFlash(getResponse(), "Directory exists: " + + "" + path + "", Flash.Kind.NACK); + } else { + if (getEngine().createNewJobWithDefaults(newJobDir)) { + Flash.addFlash(getResponse(), "Created new crawl job: " + + "" + path + "", Flash.Kind.ACK); + getEngine().findJobConfigs(); + } else { + Flash.addFlash(getResponse(), "Failed to create new job: " + + "" + path + "", Flash.Kind.NACK); + } + } } } // default: redirect to GET self getResponse().redirectSeeOther(getRequest().getOriginalRef()); } + protected List getAvailableActions() { List actions = new LinkedList(); actions.add("rescan"); @@ -202,29 +208,6 @@ public class EngineResource extends BaseResource { return info; } - /** - * wraps a message in a styled div given messsage type - * @param msg message to be displayed - * @param type message type selector - * @return string wrapped in styled
- * TODO: put this in a sensible place, and use a stylesheet instead - */ - protected String messageDiv(String message, String type) { - HashMap colorMap = new HashMap(); - colorMap.put("ERROR","pink"); - colorMap.put("WARNING","lightyellow"); - colorMap.put("MESSAGE","lavender"); - String color; - if (colorMap.containsKey(type)) { - color = colorMap.get(type); - } else { - color = "gray"; - } - String style = "style=\"margin:1em;padding:0.2em 1em;" - + "background:" + color + ";\""; - return "
" + message + "
\n"; - } - protected void writeHtml(Writer writer) { Engine engine = getEngine(); String engineTitle = "Heritrix Engine "+engine.getHeritrixVersion(); @@ -270,7 +253,7 @@ public class EngineResource extends BaseResource { pw.println("
\n" + "Create new job directory with recommended starting configuration
\n" + "Path: " + jobsDir.getAbsolutePath() + "/\n" - + "\n" + + "\n" + "\n" + "
\n");