diff --git a/engine/src/main/java/org/archive/crawler/framework/Engine.java b/engine/src/main/java/org/archive/crawler/framework/Engine.java
index 7b68c9c0..93de1a41 100644
--- a/engine/src/main/java/org/archive/crawler/framework/Engine.java
+++ b/engine/src/main/java/org/archive/crawler/framework/Engine.java
@@ -20,7 +20,6 @@
package org.archive.crawler.framework;
import java.io.File;
-import java.io.FileFilter;
import java.io.FilenameFilter;
import java.io.IOException;
import java.io.InputStream;
@@ -81,15 +80,47 @@ public class Engine {
}
// discover any new job directories
- for (File dir : jobsDir.listFiles(new FileFilter(){
- public boolean accept(File pathname) {
- return pathname.isDirectory();
- }})) {
- considerAsJobDirectory(dir);
+ if (! jobsDir.exists()) {
+ LOGGER.log(Level.WARNING,"jobsDir has disappeared: "+jobsDir.toString());
+ } else {
+ for (File jobFile: jobsDir.listFiles()) {
+ if (jobFile.isDirectory()) {
+ considerAsJobDirectory(jobFile);
+ } else if (jobFile.getName().endsWith(".jobpath")) {
+ considerAsJobPath(jobFile);
+ }
+ }
}
}
- public boolean considerAsJobDirectory(File dir) {
+ private void considerAsJobPath(File jobpathFile) {
+ try {
+ String pathToJob = FileUtils.readFileToString(jobpathFile).trim();
+ File jobPathFromFile = new File(pathToJob);
+ if (jobPathFromFile.isDirectory()) {
+ if (!considerAsJobDirectory(jobPathFromFile)) {
+ LOGGER.log(Level.WARNING,"invalid job path: "
+ + "'" + jobPathFromFile.toString().trim() + "'"
+ + " specified in jobpathFile: "
+ + jobpathFile.toString());
+ }
+ } else {
+ // invalid job path specified
+ LOGGER.log(Level.WARNING,"invalid job path: "
+ + "'" + jobPathFromFile.toString().trim() + "'"
+ + " specified in jobpathFile: "
+ + jobpathFile.toString());
+ }
+ } catch (IOException e) {
+ // problem reading jobpathFile
+ LOGGER.log(Level.WARNING,
+ "could not read jobPath from jobpathFile: "
+ + jobpathFile.toString());
+ e.printStackTrace();
+ }
+ }
+
+ public boolean considerAsJobDirectory(File dir) {
File[] candidateConfigs = dir.listFiles(new FilenameFilter() {
public boolean accept(File dir, String name) {
return name.endsWith(".cxml");
@@ -103,12 +134,17 @@ public class Engine {
CrawlJob cj = new CrawlJob(cxml);
if(!jobConfigs.containsKey(cj.getShortName())) {
jobConfigs.put(cj.getShortName(),cj);
+ LOGGER.log(Level.INFO,"added crawl job: " +cj.getShortName());
return true;
+ } else {
+ // jobConfig exists
+ return true;
}
} catch (IllegalArgumentException iae) {
LOGGER.log(Level.WARNING,"bad cxml: "+cxml,iae);
}
}
+ // path rejected for some reason
return false;
}
@@ -295,5 +331,22 @@ public class Engine {
return true;
}
-
+ public void leaveJobPathFile(String path) throws IOException {
+ String jobName = path.substring(path.lastIndexOf("/")+1,path.length());
+ if (jobConfigs.containsKey(jobName)) {
+ String jobpathFileName = jobName+".jobpath";
+ File jobpathFile = new File(jobsDir,jobpathFileName);
+ try {
+ FileUtils.writeStringToFile(jobpathFile, path+"\n");
+ System.out.println("Engine.leaveJobPathFile() wrote file: "
+ + jobpathFileName);
+ } catch (IOException e) {
+ throw new IOException("could not create jobpathFile: "
+ + jobpathFile.toString());
+ }
+ } else {
+ LOGGER.log(Level.SEVERE,"job: "+jobName+" not found in jobConfig!");
+ }
+ }
+
}
\ No newline at end of file
diff --git a/engine/src/main/java/org/archive/crawler/restlet/EngineResource.java b/engine/src/main/java/org/archive/crawler/restlet/EngineResource.java
index 74b41dd4..0cf89c28 100644
--- a/engine/src/main/java/org/archive/crawler/restlet/EngineResource.java
+++ b/engine/src/main/java/org/archive/crawler/restlet/EngineResource.java
@@ -25,6 +25,7 @@ import java.io.PrintWriter;
import java.io.Writer;
import java.util.ArrayList;
import java.util.Collections;
+import java.util.HashMap;
import org.apache.commons.lang.StringUtils;
import org.archive.crawler.framework.CrawlJob;
@@ -83,39 +84,40 @@ public class EngineResource extends Resource {
added = getEngine().considerAsJobDirectory(new File(path));
}
if(!added) {
- Flash.addFlash(getResponse(), "not a valid preexisting job directory: \""+path+"\"", Flash.Kind.NACK);
+ String msg = messageDiv("ERROR! invalid job path: '"+path+"'","ERROR");
+ Flash.addFlash(getResponse(),msg,Flash.Kind.NACK);
+ } else {
+ try {
+ getEngine().leaveJobPathFile(path);
+ String msg = messageDiv("Added job path: '"+path+"'","MESSAGE");
+ Flash.addFlash(getResponse(),msg);
+ } catch (IOException e) {
+ String msg = messageDiv(e.getMessage(),"ERROR");
+ Flash.addFlash(getResponse(),msg,Flash.Kind.NACK);
+ }
}
} else if ("create".equals(action)) {
- String errStyle = "style=\"margin:1em;padding:0.2em 1em;background:pink;\"";
- String warnStyle = "style=\"margin:1em;padding:0.2em 1em;background:khaki;\"";
- String msgStyle = "style=\"margin:1em;padding:0.2em 1em;background:lavender;\"";
String path = form.getFirstValue("addpath");
if (path==null) {
- String warn = "WARNING: no job created. "
- + "null path given.
\n";
+ String warn = messageDiv("WARNING! null path given.","WARNING");
Flash.addFlash(getResponse(), warn, Flash.Kind.NACK);
- System.err.println(warn);
} else if (path.indexOf("/") != -1) {
- String warn = "WARNING: "
- + "no job created. sub-directories disallowed: "
- + "" + path + "
\n";
+ String warn = messageDiv("WARNING! sub-directories disallowed: " + path + "","WARNING");
+ Flash.addFlash(getResponse(), warn, Flash.Kind.NACK);
+ } else if (getEngine().getJobConfigs().containsKey(path)) {
+ String warn = messageDiv("ERROR! job exists: " + path + "","ERROR");
Flash.addFlash(getResponse(), warn, Flash.Kind.NACK);
- System.err.println(warn);
} else {
boolean created = false;
try {
created = getEngine().createNewJobWithDefaults(path);
} catch (IOException e) {
- String err = "ERROR! failed to create new job: "
- + "" + path + " "+ e.toString() + "
\n";
+ String err = messageDiv("ERROR! " + e.toString(),"ERROR");
Flash.addFlash(getResponse(), err, Flash.Kind.NACK);
- System.err.println(err);
-
}
if (created) {
- String msg = "Successfully created job: "
- + "" + path + "
\n";
- Flash.addFlash(getResponse(), msg, Flash.Kind.NACK);
+ String msg = messageDiv("Successfully created job: " + path + "","MESSAGE");
+ Flash.addFlash(getResponse(), msg, Flash.Kind.ACK);
getEngine().findJobConfigs();
}
}
@@ -123,6 +125,29 @@ public class EngineResource extends Resource {
// default: redirect to GET self
getResponse().redirectSeeOther(getRequest().getOriginalRef());
}
+
+ /**
+ * wraps a message in a styled div given messsage type
+ * @param msg message to be displayed
+ * @param type message type selector
+ * @return string wrapped in styled
+ * TODO: put this in a sensible place, and use a stylesheet instead
+ */
+ protected String messageDiv(String message, String type) {
+ HashMap colorMap = new HashMap();
+ colorMap.put("ERROR","pink");
+ colorMap.put("WARNING","lightyellow");
+ colorMap.put("MESSAGE","lavender");
+ String color;
+ if (colorMap.containsKey(type)) {
+ color = colorMap.get(type);
+ } else {
+ color = "gray";
+ }
+ String style = "style=\"margin:1em;padding:0.2em 1em;"
+ + "background:" + color + ";\"";
+ return "" + message + "
\n";
+ }
protected void writeHtml(Writer writer) {
Engine engine = getEngine();
@@ -151,6 +176,9 @@ public class EngineResource extends Resource {
pw.println("Jobs Directory: "+jobsDir.getAbsolutePath()+"");
ArrayList jobs = new ArrayList();
+
+ // re-scan job configs on each page load
+ engine.findJobConfigs();
jobs.addAll(engine.getJobConfigs().values());
pw.println("