lee/heritrix3
1
0 Fork 0
mirror of https://github.com/internetarchive/heritrix3.git synced 2026-11-09 07:24:41 +00:00
Code Issues Packages Projects Releases Wiki Activity

HER-1688 remember external job directories

Browse Source 
* EngineResource.java, Engine.java
    refactor for less branching/redundancy
This commit is contained in:
gojomo committed 2010-04-26 22:22:09 +00:00
1 parent 14b8649aa9
commit c4946d04d0
2 files changed
+46 -68
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

engine/src/main/java/org/archive/crawler/framework/Engine.java
+45 -67
View File
@@ -91,50 +91,42 @@ public class Engine {
}
// discover any new job directories
for (File jobFile: jobsDir.listFiles()) {
if (jobFile.getName().endsWith(".jobpath")) {
String jobPath = getJobPathFromFile(jobFile);
jobFile = new File(jobPath);
if (jobFile.isDirectory()) {
if (!addJobDirectory(jobFile,false)) {
LOGGER.log(Level.WARNING,"invalid job dir: " + jobPath
+ " specified in jobpath file: "
+ jobFile.getAbsolutePath());
}
} else {
LOGGER.log(Level.WARNING,"non-directory: " + jobPath
+ " specified in jobpath file: "
+ jobFile.getAbsolutePath());
}
for (File candidateFile: jobsDir.listFiles()) {
File jobFile = candidateFile;
if (candidateFile.getName().endsWith(".jobpath")) {
// convert .jobpaths to the referenced external directory
jobFile = getJobDirectoryFrom(candidateFile);
}
if (jobFile.isDirectory()) {
addJobDirectory(jobFile,false);
if(!addJobDirectory(jobFile)) {
LOGGER.log(Level.WARNING,"invalid job directory: " + jobFile
+ " where job expected from: " + candidateFile);
}
}
}
/*
* read path from ".jobpath" file (jobpathFile) and add directory to jobConfig
/**
* Return the job directory File read from the supplied ".jobpath" file,
* or null on any error.
*/
protected String getJobPathFromFile(File jobPathFile) {
protected File getJobDirectoryFrom(File jobPathFile) {
try {
String pathToJob = FileUtils.readFileToString(jobPathFile).trim();
return pathToJob;
return new File(FileUtils.readFileToString(jobPathFile).trim());
} catch (IOException e) {
LOGGER.log(Level.SEVERE,"error reading jobPathFile: "
+ jobPathFile.toString());
e.printStackTrace();
return null;
LOGGER.log(Level.SEVERE,"bad .jobpath: "+jobPathFile, e);
return null;
}
}
/**
* adds a job directory to the Engine jobConfigs if not extant
* Adds a job directory to the Engine known jobConfigs if not extant.
*
* @param dir directory to be added
* @param userRequest calls writeJobPathFile when true
* @return true if directory successfully added to jobConfigs
* @return true if directory successfully added, false for any failure
*/
public boolean addJobDirectory(File dir, Boolean userRequest) {
public boolean addJobDirectory(File dir) {
if(dir==null) {
return false;
}
File[] candidateConfigs = dir.listFiles(new FilenameFilter() {
public boolean accept(File dir, String name) {
return name.endsWith(".cxml");
@@ -143,24 +135,23 @@ public class Engine {
// no CXML file found!
return false;
}
if(jobConfigs.containsKey(dir.getName())) {
// same-name job already exists
return false;
}
for (File cxml : candidateConfigs) {
try {
CrawlJob cj = new CrawlJob(cxml);
if(!jobConfigs.containsKey(cj.getShortName())) {
jobConfigs.put(cj.getShortName(),cj);
if (userRequest) {
writeJobPathFile(dir.getAbsolutePath());
}
LOGGER.log(Level.INFO,"added crawl job: " + cj.getShortName());
} else {
if (userRequest) {
LOGGER.log(Level.INFO,"requested job to add already exists: "
+ cj.getShortName());
}
CrawlJob cj = new CrawlJob(cxml);
if(!cj.getJobDir().getParentFile().equals(getJobsDir())) {
writeJobPathFile(cj);
}
jobConfigs.put(cj.getShortName(),cj);
LOGGER.log(Level.INFO,"added crawl job: " + cj.getShortName());
return true;
} catch (IOException iae) {
LOGGER.log(Level.SEVERE,"unable to add job directory"+dir,iae);
} catch (IllegalArgumentException iae) {
LOGGER.log(Level.WARNING,"bad cxml: "+cxml,iae);
LOGGER.log(Level.SEVERE,"bad cxml: "+cxml,iae);
}
}
// path rejected for some reason
@@ -372,29 +363,16 @@ public class Engine {
}
/**
* writes a .jobpath file for "added" paths
* @param path
* @throws IOException
* Writes a .jobpath file for the new CrawlJob, whose directory is
* outside the main Engine jobs directory.
*
* @param job CrawlJob whose main directory the .jobpath should point to
* @throws IOException for any IO error
*/
public void writeJobPathFile(String path) {
String jobName = path.substring(path.lastIndexOf(File.separatorChar)+1,
path.length());
if (jobConfigs.containsKey(jobName)) {
String jobpathFileName = jobName+".jobpath";
File jobpathFile = new File(jobsDir,jobpathFileName);
try {
if (!jobpathFile.exists()) {
FileUtils.writeStringToFile(jobpathFile, path+"\n");
LOGGER.log(Level.INFO, "wrote jobpath file: "
+ jobpathFileName);
}
} catch (IOException e) {
LOGGER.log(Level.SEVERE,"failed to write jobPathFile: "
+ jobpathFileName + "\n"
+ e.getMessage());
}
} else {
LOGGER.log(Level.SEVERE,"added job: " + jobName + " not found!");
}
}
public void writeJobPathFile(CrawlJob job) throws IOException {
String jobpathFileName = job.getShortName()+".jobpath";
File jobpathFile = new File(jobsDir,jobpathFileName);
FileUtils.writeStringToFile(jobpathFile, job.getJobDir().getAbsolutePath()+"\n");
LOGGER.log(Level.INFO, "wrote jobpath file: " + jobpathFileName);
}
}
engine/src/main/java/org/archive/crawler/restlet/EngineResource.java
+1 -1
View File
@@ -102,7 +102,7 @@ public class EngineResource extends BaseResource {
} else if (getEngine().getJobConfigs().containsKey(jobName)) {
Flash.addFlash(getResponse(), "Job exists: <i>"
+ jobName + "</i>", Flash.Kind.NACK);
} else if (getEngine().addJobDirectory(new File(path),true)) {
} else if (getEngine().addJobDirectory(new File(path))) {
Flash.addFlash(getResponse(), "Added crawl job: "
+ "'" + path + "'", Flash.Kind.ACK);
} else {