From efee001bbcfe2409dc3bebba6565a4264d67b440 Mon Sep 17 00:00:00 2001 From: szznax Date: Thu, 15 Oct 2009 01:03:29 +0000 Subject: [PATCH] Fix for [HER-1688] remember external job directories * Engine.java protect against disappeared jobsDir in findJobConfigs() check for .jobpath files in findJobConfigs() added considerAsJobPath() log added jobs in considerAsJobDirectory() make considerAsJobDirectory() return true when jobConfig (already) exists added leaveJobPathFile() to write .jobpath file for newly added jobs * EngineResource.java call leaveJobPathFile() when job directory added successfully added messageDiv() to make messages more uniform and prominent re-scan jobConfigs on each page load - may obviate need for "rescan" button --- .../org/archive/crawler/framework/Engine.java | 69 ++++++++++++++++--- .../crawler/restlet/EngineResource.java | 64 ++++++++++++----- 2 files changed, 107 insertions(+), 26 deletions(-) diff --git a/engine/src/main/java/org/archive/crawler/framework/Engine.java b/engine/src/main/java/org/archive/crawler/framework/Engine.java index 7b68c9c0..93de1a41 100644 --- a/engine/src/main/java/org/archive/crawler/framework/Engine.java +++ b/engine/src/main/java/org/archive/crawler/framework/Engine.java @@ -20,7 +20,6 @@ package org.archive.crawler.framework; import java.io.File; -import java.io.FileFilter; import java.io.FilenameFilter; import java.io.IOException; import java.io.InputStream; @@ -81,15 +80,47 @@ public class Engine { } // discover any new job directories - for (File dir : jobsDir.listFiles(new FileFilter(){ - public boolean accept(File pathname) { - return pathname.isDirectory(); - }})) { - considerAsJobDirectory(dir); + if (! jobsDir.exists()) { + LOGGER.log(Level.WARNING,"jobsDir has disappeared: "+jobsDir.toString()); + } else { + for (File jobFile: jobsDir.listFiles()) { + if (jobFile.isDirectory()) { + considerAsJobDirectory(jobFile); + } else if (jobFile.getName().endsWith(".jobpath")) { + considerAsJobPath(jobFile); + } + } } } - public boolean considerAsJobDirectory(File dir) { + private void considerAsJobPath(File jobpathFile) { + try { + String pathToJob = FileUtils.readFileToString(jobpathFile).trim(); + File jobPathFromFile = new File(pathToJob); + if (jobPathFromFile.isDirectory()) { + if (!considerAsJobDirectory(jobPathFromFile)) { + LOGGER.log(Level.WARNING,"invalid job path: " + + "'" + jobPathFromFile.toString().trim() + "'" + + " specified in jobpathFile: " + + jobpathFile.toString()); + } + } else { + // invalid job path specified + LOGGER.log(Level.WARNING,"invalid job path: " + + "'" + jobPathFromFile.toString().trim() + "'" + + " specified in jobpathFile: " + + jobpathFile.toString()); + } + } catch (IOException e) { + // problem reading jobpathFile + LOGGER.log(Level.WARNING, + "could not read jobPath from jobpathFile: " + + jobpathFile.toString()); + e.printStackTrace(); + } + } + + public boolean considerAsJobDirectory(File dir) { File[] candidateConfigs = dir.listFiles(new FilenameFilter() { public boolean accept(File dir, String name) { return name.endsWith(".cxml"); @@ -103,12 +134,17 @@ public class Engine { CrawlJob cj = new CrawlJob(cxml); if(!jobConfigs.containsKey(cj.getShortName())) { jobConfigs.put(cj.getShortName(),cj); + LOGGER.log(Level.INFO,"added crawl job: " +cj.getShortName()); return true; + } else { + // jobConfig exists + return true; } } catch (IllegalArgumentException iae) { LOGGER.log(Level.WARNING,"bad cxml: "+cxml,iae); } } + // path rejected for some reason return false; } @@ -295,5 +331,22 @@ public class Engine { return true; } - + public void leaveJobPathFile(String path) throws IOException { + String jobName = path.substring(path.lastIndexOf("/")+1,path.length()); + if (jobConfigs.containsKey(jobName)) { + String jobpathFileName = jobName+".jobpath"; + File jobpathFile = new File(jobsDir,jobpathFileName); + try { + FileUtils.writeStringToFile(jobpathFile, path+"\n"); + System.out.println("Engine.leaveJobPathFile() wrote file: " + + jobpathFileName); + } catch (IOException e) { + throw new IOException("could not create jobpathFile: " + + jobpathFile.toString()); + } + } else { + LOGGER.log(Level.SEVERE,"job: "+jobName+" not found in jobConfig!"); + } + } + } \ No newline at end of file diff --git a/engine/src/main/java/org/archive/crawler/restlet/EngineResource.java b/engine/src/main/java/org/archive/crawler/restlet/EngineResource.java index 74b41dd4..0cf89c28 100644 --- a/engine/src/main/java/org/archive/crawler/restlet/EngineResource.java +++ b/engine/src/main/java/org/archive/crawler/restlet/EngineResource.java @@ -25,6 +25,7 @@ import java.io.PrintWriter; import java.io.Writer; import java.util.ArrayList; import java.util.Collections; +import java.util.HashMap; import org.apache.commons.lang.StringUtils; import org.archive.crawler.framework.CrawlJob; @@ -83,39 +84,40 @@ public class EngineResource extends Resource { added = getEngine().considerAsJobDirectory(new File(path)); } if(!added) { - Flash.addFlash(getResponse(), "not a valid preexisting job directory: \""+path+"\"", Flash.Kind.NACK); + String msg = messageDiv("ERROR! invalid job path: '"+path+"'","ERROR"); + Flash.addFlash(getResponse(),msg,Flash.Kind.NACK); + } else { + try { + getEngine().leaveJobPathFile(path); + String msg = messageDiv("Added job path: '"+path+"'","MESSAGE"); + Flash.addFlash(getResponse(),msg); + } catch (IOException e) { + String msg = messageDiv(e.getMessage(),"ERROR"); + Flash.addFlash(getResponse(),msg,Flash.Kind.NACK); + } } } else if ("create".equals(action)) { - String errStyle = "style=\"margin:1em;padding:0.2em 1em;background:pink;\""; - String warnStyle = "style=\"margin:1em;padding:0.2em 1em;background:khaki;\""; - String msgStyle = "style=\"margin:1em;padding:0.2em 1em;background:lavender;\""; String path = form.getFirstValue("addpath"); if (path==null) { - String warn = "
WARNING: no job created. " - + "null path given.
\n"; + String warn = messageDiv("WARNING! null path given.","WARNING"); Flash.addFlash(getResponse(), warn, Flash.Kind.NACK); - System.err.println(warn); } else if (path.indexOf("/") != -1) { - String warn = "
WARNING: " - + "no job created. sub-directories disallowed: " - + "" + path + "
\n"; + String warn = messageDiv("WARNING! sub-directories disallowed: " + path + "","WARNING"); + Flash.addFlash(getResponse(), warn, Flash.Kind.NACK); + } else if (getEngine().getJobConfigs().containsKey(path)) { + String warn = messageDiv("ERROR! job exists: " + path + "","ERROR"); Flash.addFlash(getResponse(), warn, Flash.Kind.NACK); - System.err.println(warn); } else { boolean created = false; try { created = getEngine().createNewJobWithDefaults(path); } catch (IOException e) { - String err = "
ERROR! failed to create new job: " - + "" + path + " "+ e.toString() + "
\n"; + String err = messageDiv("ERROR! " + e.toString(),"ERROR"); Flash.addFlash(getResponse(), err, Flash.Kind.NACK); - System.err.println(err); - } if (created) { - String msg = "

Successfully created job: " - + "" + path + "

\n"; - Flash.addFlash(getResponse(), msg, Flash.Kind.NACK); + String msg = messageDiv("Successfully created job: " + path + "","MESSAGE"); + Flash.addFlash(getResponse(), msg, Flash.Kind.ACK); getEngine().findJobConfigs(); } } @@ -123,6 +125,29 @@ public class EngineResource extends Resource { // default: redirect to GET self getResponse().redirectSeeOther(getRequest().getOriginalRef()); } + + /** + * wraps a message in a styled div given messsage type + * @param msg message to be displayed + * @param type message type selector + * @return string wrapped in styled
+ * TODO: put this in a sensible place, and use a stylesheet instead + */ + protected String messageDiv(String message, String type) { + HashMap colorMap = new HashMap(); + colorMap.put("ERROR","pink"); + colorMap.put("WARNING","lightyellow"); + colorMap.put("MESSAGE","lavender"); + String color; + if (colorMap.containsKey(type)) { + color = colorMap.get(type); + } else { + color = "gray"; + } + String style = "style=\"margin:1em;padding:0.2em 1em;" + + "background:" + color + ";\""; + return "
" + message + "
\n"; + } protected void writeHtml(Writer writer) { Engine engine = getEngine(); @@ -151,6 +176,9 @@ public class EngineResource extends Resource { pw.println("Jobs Directory: "+jobsDir.getAbsolutePath()+""); ArrayList jobs = new ArrayList(); + + // re-scan job configs on each page load + engine.findJobConfigs(); jobs.addAll(engine.getJobConfigs().values()); pw.println("

Job Directories ("+jobs.size()+")");