lee/heritrix3
1
0
Fork 0
mirror of https://github.com/internetarchive/heritrix3.git synced 2026-07-13 19:17:18 +00:00
Code Issues Packages Projects Releases Wiki Activity

Fix for [HER-1688] remember external job directories

Browse Source 
* Engine.java
    protect against disappeared jobsDir in findJobConfigs()
    check for .jobpath files in findJobConfigs()
    added considerAsJobPath()
    log added jobs in considerAsJobDirectory()
    make considerAsJobDirectory() return true when jobConfig (already) exists
    added leaveJobPathFile() to write .jobpath file for newly added jobs
* EngineResource.java
    call leaveJobPathFile() when job directory added successfully
    added messageDiv() to make messages more uniform and prominent
    re-scan jobConfigs on each page load - may obviate need for "rescan" button
This commit is contained in:
szznax
2009-10-15 01:03:29 +00:00
parent 8311db6a26
commit efee001bbc
2 changed files with 107 additions and 26 deletions
Download Patch File Download Diff File Expand all files Collapse all files
engine/src/main/java/org/archive/crawler/framework/Engine.java
+61 -8
View File
@@ -20,7 +20,6 @@
package org.archive.crawler.framework;
import java.io.File;
import java.io.FileFilter;
import java.io.FilenameFilter;
import java.io.IOException;
import java.io.InputStream;
@@ -81,15 +80,47 @@ public class Engine {
}
// discover any new job directories
for (File dir : jobsDir.listFiles(new FileFilter(){
public boolean accept(File pathname) {
return pathname.isDirectory();
}})) {
considerAsJobDirectory(dir);
if (! jobsDir.exists()) {
LOGGER.log(Level.WARNING,"jobsDir has disappeared: "+jobsDir.toString());
} else {
for (File jobFile: jobsDir.listFiles()) {
if (jobFile.isDirectory()) {
considerAsJobDirectory(jobFile);
} else if (jobFile.getName().endsWith(".jobpath")) {
considerAsJobPath(jobFile);
}
}
}
}
public boolean considerAsJobDirectory(File dir) {
private void considerAsJobPath(File jobpathFile) {
try {
String pathToJob = FileUtils.readFileToString(jobpathFile).trim();
File jobPathFromFile = new File(pathToJob);
if (jobPathFromFile.isDirectory()) {
if (!considerAsJobDirectory(jobPathFromFile)) {
LOGGER.log(Level.WARNING,"invalid job path: "
+ "'" + jobPathFromFile.toString().trim() + "'"
+ " specified in jobpathFile: "
+ jobpathFile.toString());
}
} else {
// invalid job path specified
LOGGER.log(Level.WARNING,"invalid job path: "
+ "'" + jobPathFromFile.toString().trim() + "'"
+ " specified in jobpathFile: "
+ jobpathFile.toString());
}
} catch (IOException e) {
// problem reading jobpathFile
LOGGER.log(Level.WARNING,
"could not read jobPath from jobpathFile: "
+ jobpathFile.toString());
e.printStackTrace();
}
}
public boolean considerAsJobDirectory(File dir) {
File[] candidateConfigs = dir.listFiles(new FilenameFilter() {
public boolean accept(File dir, String name) {
return name.endsWith(".cxml");
@@ -103,12 +134,17 @@ public class Engine {
CrawlJob cj = new CrawlJob(cxml);
if(!jobConfigs.containsKey(cj.getShortName())) {
jobConfigs.put(cj.getShortName(),cj);
LOGGER.log(Level.INFO,"added crawl job: " +cj.getShortName());
return true;
} else {
// jobConfig exists
return true;
}
} catch (IllegalArgumentException iae) {
LOGGER.log(Level.WARNING,"bad cxml: "+cxml,iae);
}
}
// path rejected for some reason
return false;
}
@@ -295,5 +331,22 @@ public class Engine {
return true;
}
public void leaveJobPathFile(String path) throws IOException {
String jobName = path.substring(path.lastIndexOf("/")+1,path.length());
if (jobConfigs.containsKey(jobName)) {
String jobpathFileName = jobName+".jobpath";
File jobpathFile = new File(jobsDir,jobpathFileName);
try {
FileUtils.writeStringToFile(jobpathFile, path+"\n");
System.out.println("Engine.leaveJobPathFile() wrote file: "
+ jobpathFileName);
} catch (IOException e) {
throw new IOException("could not create jobpathFile: "
+ jobpathFile.toString());
}
} else {
LOGGER.log(Level.SEVERE,"job: "+jobName+" not found in jobConfig!");
}
}
}
engine/src/main/java/org/archive/crawler/restlet/EngineResource.java
+46 -18
View File
@@ -25,6 +25,7 @@ import java.io.PrintWriter;
import java.io.Writer;
import java.util.ArrayList;
import java.util.Collections;
import java.util.HashMap;
import org.apache.commons.lang.StringUtils;
import org.archive.crawler.framework.CrawlJob;
@@ -83,39 +84,40 @@ public class EngineResource extends Resource {
added = getEngine().considerAsJobDirectory(new File(path));
}
if(!added) {
Flash.addFlash(getResponse(), "not a valid preexisting job directory: \""+path+"\"", Flash.Kind.NACK);
String msg = messageDiv("ERROR! invalid job path: '"+path+"'","ERROR");
Flash.addFlash(getResponse(),msg,Flash.Kind.NACK);
} else {
try {
getEngine().leaveJobPathFile(path);
String msg = messageDiv("Added job path: '"+path+"'","MESSAGE");
Flash.addFlash(getResponse(),msg);
} catch (IOException e) {
String msg = messageDiv(e.getMessage(),"ERROR");
Flash.addFlash(getResponse(),msg,Flash.Kind.NACK);
}
}
} else if ("create".equals(action)) {
String errStyle = "style=\"margin:1em;padding:0.2em 1em;background:pink;\"";
String warnStyle = "style=\"margin:1em;padding:0.2em 1em;background:khaki;\"";
String msgStyle = "style=\"margin:1em;padding:0.2em 1em;background:lavender;\"";
String path = form.getFirstValue("addpath");
if (path==null) {
String warn = "<div "+warnStyle+">WARNING: no job created. "
+ "null path given.</div>\n";
String warn = messageDiv("WARNING! null path given.","WARNING");
Flash.addFlash(getResponse(), warn, Flash.Kind.NACK);
System.err.println(warn);
} else if (path.indexOf("/") != -1) {
String warn = "<div "+warnStyle+">WARNING: "
+ "no job created. sub-directories disallowed: "
+ "<i>" + path + "</i></div>\n";
String warn = messageDiv("WARNING! sub-directories disallowed: <i>" + path + "</i>","WARNING");
Flash.addFlash(getResponse(), warn, Flash.Kind.NACK);
} else if (getEngine().getJobConfigs().containsKey(path)) {
String warn = messageDiv("ERROR! job exists: <i>" + path + "</i>","ERROR");
Flash.addFlash(getResponse(), warn, Flash.Kind.NACK);
System.err.println(warn);
} else {
boolean created = false;
try {
created = getEngine().createNewJobWithDefaults(path);
} catch (IOException e) {
String err = "<div "+errStyle+">ERROR! failed to create new job: "
+ "<i>" + path + "</i> "+ e.toString() + "</div>\n";
String err = messageDiv("ERROR! " + e.toString(),"ERROR");
Flash.addFlash(getResponse(), err, Flash.Kind.NACK);
System.err.println(err);
}
if (created) {
String msg = "<p "+msgStyle+">Successfully created job: "
+ "<i>" + path + "</i></p>\n";
Flash.addFlash(getResponse(), msg, Flash.Kind.NACK);
String msg = messageDiv("Successfully created job: <i>" + path + "</i>","MESSAGE");
Flash.addFlash(getResponse(), msg, Flash.Kind.ACK);
getEngine().findJobConfigs();
}
}
@@ -123,6 +125,29 @@ public class EngineResource extends Resource {
// default: redirect to GET self
getResponse().redirectSeeOther(getRequest().getOriginalRef());
}
/**
* wraps a message in a styled div given messsage type
* @param msg message to be displayed
* @param type message type selector
* @return string wrapped in styled <div/>
* TODO: put this in a sensible place, and use a stylesheet instead
*/
protected String messageDiv(String message, String type) {
HashMap<String,String> colorMap = new HashMap<String,String>();
colorMap.put("ERROR","pink");
colorMap.put("WARNING","lightyellow");
colorMap.put("MESSAGE","lavender");
String color;
if (colorMap.containsKey(type)) {
color = colorMap.get(type);
} else {
color = "gray";
}
String style = "style=\"margin:1em;padding:0.2em 1em;"
+ "background:" + color + ";\"";
return "<div " + style + ">" + message + "</div>\n";
}
protected void writeHtml(Writer writer) {
Engine engine = getEngine();
@@ -151,6 +176,9 @@ public class EngineResource extends Resource {
pw.println("<b>Jobs Directory</b>: <a href='jobsdir'>"+jobsDir.getAbsolutePath()+"</a></h2>");
ArrayList<CrawlJob> jobs = new ArrayList<CrawlJob>();
// re-scan job configs on each page load
engine.findJobConfigs();
jobs.addAll(engine.getJobConfigs().values());
pw.println("<form method=\'POST\'><h2>Job Directories ("+jobs.size()+")");