lee/heritrix3
1
0
Fork 0
mirror of https://github.com/internetarchive/heritrix3.git synced 2026-09-20 20:55:49 +00:00
Code Issues Packages Projects Releases Wiki Activity

tried to incorporate gojomo's suggestions of 2009-11-02:

Browse Source 
* Engine.java
  removed ERROR and WARNING strings, rely on Flash class
  log jobsDir check failure as SEVERE and return early
  collapse error-handling in considerAsJobPath() and log SEVERE
  renamed considerAsJobDirectory() to addJobDirectory()
  remove catch-and-rethrow from leaveJobPathFile() when writing,
    instead log SEVERE
  renamed leaveJobPathFile() to writeJobPathFile()
  added boolean "userRequest" arg to addJobDirectory(), and moved
    call to writeJobPathFile() inside of addJobDirectory()
  changed void considerAsJobPath(File) to String getJobPathFromFile
    and removed call to addJobDirectory
  make seperate calls to addJobDirectory() in findJobConfigs()
* EngineResource.java
  restructured "create" logic for clarity
  eliminate style code, rely on Flash class
  remove limit on "addpath" input, change to "createpath"
  removed all-caps and bangs ("!") from error messages
  removed platform-specific path separators ("/")
  call Engine.addJobDirectory() with userRequest=true in when
    form action=add
This commit is contained in:
szznax
2009-12-17 01:54:21 +00:00
parent bf2eaef6e2
commit 0cb627fff8
2 changed files with 145 additions and 131 deletions
Download Patch File Download Diff File Expand all files Collapse all files
engine/src/main/java/org/archive/crawler/framework/Engine.java
+98 -67
View File
@@ -51,7 +51,7 @@ public class Engine {
final private static Logger LOGGER =
Logger.getLogger(Engine.class.getName());
/** directory where job directores are expected */
/** directory where job directories are expected */
protected File jobsDir;
/** map of job short names -> CrawlJob instances */
protected HashMap<String,CrawlJob> jobConfigs = new HashMap<String,CrawlJob>();
@@ -69,7 +69,8 @@ public class Engine {
/**
* Find all job configurations in the usual place -- subdirectories
* of the jobs directory with files ending '.cxml'.
* of the jobs directory with files ending '.cxml', and from jobPathFiles
* (previously added by user) found in the jobs directory
*/
public void findJobConfigs() {
// TODO: allow other places/paths to be scanned/added as well?
@@ -83,54 +84,63 @@ public class Engine {
}
}
// discover any new job directories
// just in case...
if (! jobsDir.exists()) {
LOGGER.log(Level.WARNING,"jobsDir has disappeared: "+jobsDir.toString());
} else {
for (File jobFile: jobsDir.listFiles()) {
if (jobFile.isDirectory()) {
considerAsJobDirectory(jobFile);
} else if (jobFile.getName().endsWith(".jobpath")) {
considerAsJobPath(jobFile);
}
}
LOGGER.log(Level.SEVERE,"jobsDir has disappeared: "+jobsDir.toString());
return;
}
// discover any new job directories
for (File jobFile: jobsDir.listFiles()) {
if (jobFile.getName().endsWith(".jobpath")) {
String jobPath = getJobPathFromFile(jobFile);
jobFile = new File(jobPath);
if (jobFile.isDirectory()) {
if (!addJobDirectory(jobFile,false)) {
LOGGER.log(Level.WARNING,"invalid job dir: " + jobPath
+ " specified in jobpath file: "
+ jobFile.getAbsolutePath());
}
} else {
LOGGER.log(Level.WARNING,"non-directory: " + jobPath
+ " specified in jobpath file: "
+ jobFile.getAbsolutePath());
}
}
if (jobFile.isDirectory()) {
addJobDirectory(jobFile,false);
}
}
}
private void considerAsJobPath(File jobpathFile) {
try {
String pathToJob = FileUtils.readFileToString(jobpathFile).trim();
File jobPathFromFile = new File(pathToJob);
if (jobPathFromFile.isDirectory()) {
if (!considerAsJobDirectory(jobPathFromFile)) {
LOGGER.log(Level.WARNING,"invalid job path: "
+ "'" + jobPathFromFile.toString().trim() + "'"
+ " specified in jobpathFile: "
+ jobpathFile.toString());
}
} else {
// invalid job path specified
LOGGER.log(Level.WARNING,"invalid job path: "
+ "'" + jobPathFromFile.toString().trim() + "'"
+ " specified in jobpathFile: "
+ jobpathFile.toString());
}
} catch (IOException e) {
// problem reading jobpathFile
LOGGER.log(Level.WARNING,
"could not read jobPath from jobpathFile: "
+ jobpathFile.toString());
e.printStackTrace();
}
/*
* read path from ".jobpath" file (jobpathFile) and add directory to jobConfig
*/
protected String getJobPathFromFile(File jobPathFile) {
try {
String pathToJob = FileUtils.readFileToString(jobPathFile).trim();
return pathToJob;
} catch (IOException e) {
LOGGER.log(Level.SEVERE,"error reading jobPathFile: "
+ jobPathFile.toString());
e.printStackTrace();
return null;
}
}
public boolean considerAsJobDirectory(File dir) {
/**
* adds a job directory to the Engine jobConfigs if not extant
* @param dir directory to be added
* @param userRequest calls writeJobPathFile when true
* @return true if directory successfully added to jobConfigs
*/
public boolean addJobDirectory(File dir, Boolean userRequest) {
File[] candidateConfigs = dir.listFiles(new FilenameFilter() {
public boolean accept(File dir, String name) {
return name.endsWith(".cxml");
}});
if(candidateConfigs==null) {
// directory did not exist or did not contain cxml
if(candidateConfigs.length == 0) {
// no CXML file found!
return false;
}
for (File cxml : candidateConfigs) {
@@ -138,12 +148,17 @@ public class Engine {
CrawlJob cj = new CrawlJob(cxml);
if(!jobConfigs.containsKey(cj.getShortName())) {
jobConfigs.put(cj.getShortName(),cj);
LOGGER.log(Level.INFO,"added crawl job: " +cj.getShortName());
return true;
if (userRequest) {
writeJobPathFile(dir.getAbsolutePath());
}
LOGGER.log(Level.INFO,"added crawl job: " + cj.getShortName());
} else {
// jobConfig exists
return true;
if (userRequest) {
LOGGER.log(Level.INFO,"requested job to add already exists: "
+ cj.getShortName());
}
}
return true;
} catch (IllegalArgumentException iae) {
LOGGER.log(Level.WARNING,"bad cxml: "+cxml,iae);
}
@@ -329,41 +344,57 @@ public class Engine {
return getClass().getResourceAsStream(profileCxmlPath);
}
public boolean createNewJobWithDefaults(String path) throws IOException {
/**
* create a new job dir and copy profile CXML into as non-profile CXML
* @param newJobDir new job directory
* @throws IOException
*/
public boolean createNewJobWithDefaults(File newJobDir) {
try {
// get crawler-beans template into string
InputStream inStream = getProfileCxmlResource();
String defaultCxmlStr;
defaultCxmlStr = IOUtils.toString(inStream);
inStream.close();
File newJobDir = new File(jobsDir,"/"+path);
if (newJobDir.exists()) {
throw new IOException("file exists: "+newJobDir);
}
newJobDir.mkdirs();
// write default crawler-beans string to new job dir
newJobDir.mkdirs();
File newJobCxml = new File(newJobDir,"crawler-beans.cxml");
FileUtils.writeStringToFile(newJobCxml, defaultCxmlStr);
// get crawler-beans template from this package into string
InputStream inStream = getProfileCxmlResource();
String defaultCxmlStr = IOUtils.toString(inStream);
inStream.close();
// write default crawler-beans string to new job config
File newJobCxml = new File(newJobDir,"crawler-beans.cxml");
FileUtils.writeStringToFile(newJobCxml, defaultCxmlStr);
return true;
return true;
} catch (IOException e) {
LOGGER.log(Level.SEVERE,"failed to create new job: "
+ newJobDir.getAbsolutePath());
return false;
}
}
public void leaveJobPathFile(String path) throws IOException {
String jobName = path.substring(path.lastIndexOf("/")+1,path.length());
/**
* writes a .jobpath file for "added" paths
* @param path
* @throws IOException
*/
public void writeJobPathFile(String path) {
String jobName = path.substring(path.lastIndexOf(File.separatorChar)+1,
path.length());
if (jobConfigs.containsKey(jobName)) {
String jobpathFileName = jobName+".jobpath";
File jobpathFile = new File(jobsDir,jobpathFileName);
try {
FileUtils.writeStringToFile(jobpathFile, path+"\n");
System.out.println("Engine.leaveJobPathFile() wrote file: "
+ jobpathFileName);
if (!jobpathFile.exists()) {
FileUtils.writeStringToFile(jobpathFile, path+"\n");
LOGGER.log(Level.INFO, "wrote jobpath file: "
+ jobpathFileName);
}
} catch (IOException e) {
throw new IOException("could not create jobpathFile: "
+ jobpathFile.toString());
LOGGER.log(Level.SEVERE,"failed to write jobPathFile: "
+ jobpathFileName + "\n"
+ e.getMessage());
}
} else {
LOGGER.log(Level.SEVERE,"job: "+jobName+" not found in jobConfig!");
LOGGER.log(Level.SEVERE,"added job: " + jobName + " not found!");
}
}
}
engine/src/main/java/org/archive/crawler/restlet/EngineResource.java
+47 -64
View File
@@ -26,13 +26,11 @@ import java.io.Writer;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
import java.util.HashMap;
import java.util.LinkedHashMap;
import java.util.LinkedList;
import java.util.List;
import java.util.Map;
import org.apache.commons.lang.StringUtils;
import org.archive.crawler.framework.CrawlJob;
import org.archive.crawler.framework.Engine;
import org.archive.crawler.framework.CrawlController.State;
@@ -92,56 +90,64 @@ public class EngineResource extends BaseResource {
getEngine().findJobConfigs();
} else if ("add".equals(action)) {
String path = form.getFirstValue("addpath");
if(path==null) {
path = "";
}
boolean added = false;
if(StringUtils.isNotBlank(path)) {
added = getEngine().considerAsJobDirectory(new File(path));
}
if(!added) {
String msg = messageDiv("ERROR! invalid job path: '"+path+"'","ERROR");
Flash.addFlash(getResponse(),msg,Flash.Kind.NACK);
if (path==null) {
Flash.addFlash(getResponse(), "Cannot add <i>null</i> path",
Flash.Kind.NACK);
} else {
try {
getEngine().leaveJobPathFile(path);
String msg = messageDiv("Added job path: '"+path+"'","MESSAGE");
Flash.addFlash(getResponse(),msg);
} catch (IOException e) {
String msg = messageDiv(e.getMessage(),"ERROR");
Flash.addFlash(getResponse(),msg,Flash.Kind.NACK);
}
File jobFile = new File(path);
String jobName = jobFile.getName();
if (!jobFile.isDirectory()) {
Flash.addFlash(getResponse(), "Cannot add non-directory: <i>"
+ path + "</i>", Flash.Kind.NACK);
} else if (getEngine().getJobConfigs().containsKey(jobName)) {
Flash.addFlash(getResponse(), "Job exists: <i>"
+ jobName + "</i>", Flash.Kind.NACK);
} else if (getEngine().addJobDirectory(new File(path),true)) {
Flash.addFlash(getResponse(), "Added crawl job: "
+ "'" + path + "'", Flash.Kind.NACK);
} else {
Flash.addFlash(getResponse(), "Could not add job: "
+ "'" + path + "'", Flash.Kind.NACK);
}
}
} else if ("create".equals(action)) {
String path = form.getFirstValue("addpath");
String path = form.getFirstValue("createpath");
if (path==null) {
String warn = messageDiv("WARNING! null path given.","WARNING");
Flash.addFlash(getResponse(), warn, Flash.Kind.NACK);
} else if (path.indexOf("/") != -1) {
String warn = messageDiv("WARNING! sub-directories disallowed: <i>" + path + "</i>","WARNING");
Flash.addFlash(getResponse(), warn, Flash.Kind.NACK);
// protect against null path
Flash.addFlash(getResponse(), "Cannot create <i>null</i> path.",
Flash.Kind.NACK);
} else if (path.indexOf(File.separatorChar) != -1) {
// prevent specifying sub-directories
Flash.addFlash(getResponse(), "Sub-directories disallowed: "
+ "<i>" + path + "</i>", Flash.Kind.NACK);
} else if (getEngine().getJobConfigs().containsKey(path)) {
String warn = messageDiv("ERROR! job exists: <i>" + path + "</i>","ERROR");
Flash.addFlash(getResponse(), warn, Flash.Kind.NACK);
// protect existing jobs
Flash.addFlash(getResponse(), "Job exists: <i>" + path + "</i>",
Flash.Kind.NACK);
} else {
boolean created = false;
try {
created = getEngine().createNewJobWithDefaults(path);
} catch (IOException e) {
String err = messageDiv("ERROR! " + e.toString(),"ERROR");
Flash.addFlash(getResponse(), err, Flash.Kind.NACK);
}
if (created) {
String msg = messageDiv("Successfully created job: <i>" + path + "</i>","MESSAGE");
Flash.addFlash(getResponse(), msg, Flash.Kind.ACK);
getEngine().findJobConfigs();
}
// try to create new job dir
File newJobDir = new File(getEngine().getJobsDir(),path);
if (newJobDir.exists()) {
// protect existing directories
Flash.addFlash(getResponse(), "Directory exists: "
+ "<i>" + path + "</i>", Flash.Kind.NACK);
} else {
if (getEngine().createNewJobWithDefaults(newJobDir)) {
Flash.addFlash(getResponse(), "Created new crawl job: "
+ "<i>" + path + "</i>", Flash.Kind.ACK);
getEngine().findJobConfigs();
} else {
Flash.addFlash(getResponse(), "Failed to create new job: "
+ "<i>" + path + "</i>", Flash.Kind.NACK);
}
}
}
}
// default: redirect to GET self
getResponse().redirectSeeOther(getRequest().getOriginalRef());
}
protected List<String> getAvailableActions() {
List<String> actions = new LinkedList<String>();
actions.add("rescan");
@@ -202,29 +208,6 @@ public class EngineResource extends BaseResource {
return info;
}
/**
* wraps a message in a styled div given messsage type
* @param msg message to be displayed
* @param type message type selector
* @return string wrapped in styled <div/>
* TODO: put this in a sensible place, and use a stylesheet instead
*/
protected String messageDiv(String message, String type) {
HashMap<String,String> colorMap = new HashMap<String,String>();
colorMap.put("ERROR","pink");
colorMap.put("WARNING","lightyellow");
colorMap.put("MESSAGE","lavender");
String color;
if (colorMap.containsKey(type)) {
color = colorMap.get(type);
} else {
color = "gray";
}
String style = "style=\"margin:1em;padding:0.2em 1em;"
+ "background:" + color + ";\"";
return "<div " + style + ">" + message + "</div>\n";
}
protected void writeHtml(Writer writer) {
Engine engine = getEngine();
String engineTitle = "Heritrix Engine "+engine.getHeritrixVersion();
@@ -270,7 +253,7 @@ public class EngineResource extends BaseResource {
pw.println("<form method=\'POST\'>\n"
+ "Create new job directory with recommended starting configuration<br/>\n"
+ "<b>Path:</b> " + jobsDir.getAbsolutePath() + "/\n"
+ "<input size='25' name='addpath'/>\n"
+ "<input name='createpath'/>\n"
+ "<input type='submit' name='action' value='create'>\n"
+ "</form>\n");