lee/heritrix3
1
0
Fork 0
mirror of https://github.com/internetarchive/heritrix3.git synced 2026-09-26 07:35:39 +00:00
Code Issues Packages Projects Releases Wiki Activity

move some code into EngineModel constructor and restore reverse chronological ordering of crawl jobs

Browse Source
This commit is contained in:
Noah Levitt
2013-03-28 19:09:14 -07:00
parent 6a2b4a4a76
commit f988a26e10
3 changed files with 50 additions and 75 deletions
Download Patch File Download Diff File Expand all files Collapse all files
engine/src/main/java/org/archive/crawler/restlet/EngineResource.java
+1 -52
View File
@@ -19,29 +19,16 @@
package org.archive.crawler.restlet;
import java.io.ByteArrayOutputStream;
import java.io.File;
import java.io.IOException;
import java.io.OutputStreamWriter;
import java.io.PrintWriter;
import java.io.Writer;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
import java.util.Comparator;
import java.util.HashMap;
import java.util.LinkedHashMap;
import java.util.LinkedList;
import java.util.List;
import java.util.Map;
import org.archive.crawler.framework.CrawlJob;
import org.archive.crawler.framework.Engine;
import org.archive.crawler.framework.CrawlController.State;
import org.archive.crawler.restlet.models.CrawlJobModel;
import org.archive.crawler.restlet.models.EngineModel;
import org.archive.crawler.restlet.models.ViewModel;
import org.archive.util.FileUtils;
import org.restlet.Context;
import org.restlet.data.CharacterSet;
import org.restlet.data.Form;
@@ -219,46 +206,8 @@ public class EngineResource extends BaseResource {
if(!baseRef.endsWith("/")) {
baseRef += "/";
}
EngineModel model = new EngineModel(getEngine(), baseRef);
List<Map<String,Object>> jobList = new ArrayList<Map<String,Object>>();
model.put("jobs", jobList);
//Generate list of jobs
ArrayList<Map.Entry<String,CrawlJob>> jobConfigurations = new ArrayList<Map.Entry<String,CrawlJob>>(getEngine().getJobConfigs().entrySet());
Collections.sort(jobConfigurations, new Comparator<Map.Entry<String, CrawlJob>>() {
public int compare(Map.Entry<String, CrawlJob> cj1, Map.Entry<String, CrawlJob> cj2) {
return (cj2.getValue()).compareTo(cj1.getValue());
}
});
for(Map.Entry<String,CrawlJob> jobConfig : jobConfigurations) {
CrawlJob job = jobConfig.getValue();
HashMap<String, Object> crawlJobModel = new HashMap<String, Object>();
crawlJobModel.put("shortName",job.getShortName());
crawlJobModel.put("url",baseRef+"job/"+job.getShortName());
crawlJobModel.put("isProfile",job.isProfile());
crawlJobModel.put("launchCount",job.getLaunchCount());
crawlJobModel.put("lastLaunch",job.getLastLaunch());
crawlJobModel.put("hasApplicationContext",job.hasApplicationContext());
crawlJobModel.put("statusDescription", job.getJobStatusDescription());
crawlJobModel.put("isLaunchInfoPartial", job.isLaunchInfoPartial());
File primaryConfig = FileUtils.tryToCanonicalize(job.getPrimaryConfig());
crawlJobModel.put("primaryConfig", primaryConfig.getAbsolutePath());
crawlJobModel.put("primaryConfigUrl", baseRef + "jobdir/" + primaryConfig.getName());
if (job.getCrawlController() != null) {
crawlJobModel.put("crawlControllerState", job.getCrawlController().getState());
if (job.getCrawlController().getState() == State.FINISHED) {
crawlJobModel.put("crawlExitStatus", job.getCrawlController().getCrawlExitStatus());
}
}
crawlJobModel.put("key", jobConfig.getKey());
jobList.add(crawlJobModel);
}
return model;
return new EngineModel(getEngine(), baseRef);
}
protected void writeHtml(Writer writer) {
engine/src/main/java/org/archive/crawler/restlet/JobResource.java
+1 -23
View File
@@ -21,40 +21,23 @@ package org.archive.crawler.restlet;
import java.io.File;
import java.io.IOException;
import java.io.PrintWriter;
import java.io.Writer;
import java.util.Collections;
import java.util.HashMap;
import java.util.LinkedHashMap;
import java.util.LinkedHashSet;
import java.util.LinkedList;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.logging.Logger;
import org.apache.commons.io.filefilter.IOFileFilter;
import org.apache.commons.lang.StringEscapeUtils;
import org.apache.commons.lang.StringUtils;
import org.archive.checkpointing.Checkpoint;
import org.archive.crawler.framework.CrawlController.State;
import org.archive.crawler.framework.CrawlJob;
import org.archive.crawler.framework.Engine;
import org.archive.crawler.reporting.AlertHandler;
import org.archive.crawler.reporting.AlertThreadGroup;
import org.archive.crawler.reporting.Report;
import org.archive.crawler.restlet.models.CrawlJobModel;
import org.archive.crawler.restlet.models.EngineModel;
import org.archive.crawler.restlet.models.ViewModel;
import org.archive.spring.ConfigPath;
import org.archive.util.ArchiveUtils;
import org.archive.util.FileUtils;
import org.archive.util.TextUtils;
import org.restlet.Context;
import org.restlet.data.CharacterSet;
import org.restlet.data.Form;
import org.restlet.data.MediaType;
import org.restlet.data.Reference;
import org.restlet.data.Request;
import org.restlet.data.Response;
import org.restlet.resource.Representation;
@@ -63,7 +46,6 @@ import org.restlet.resource.Variant;
import org.restlet.resource.WriterRepresentation;
import freemarker.template.Configuration;
import freemarker.template.DefaultObjectWrapper;
import freemarker.template.ObjectWrapper;
import freemarker.template.Template;
import freemarker.template.TemplateException;
@@ -142,15 +124,11 @@ public class JobResource extends BaseResource {
* @return the nested Map data structure
*/
protected CrawlJobModel makeDataModel() {
String baseRef = getRequest().getResourceRef().getBaseRef().toString();
if (!baseRef.endsWith("/")) {
baseRef += "/";
}
Reference baseRefRef = new Reference(baseRef);
CrawlJobModel model = new CrawlJobModel(cj,baseRef);
return model;
return new CrawlJobModel(cj,baseRef);
}
protected void writeHtml(Writer writer) {
engine/src/main/java/org/archive/crawler/restlet/models/EngineModel.java
+48
View File
@@ -1,10 +1,17 @@
package org.archive.crawler.restlet.models;
import java.io.File;
import java.util.ArrayList;
import java.util.Collections;
import java.util.Comparator;
import java.util.HashMap;
import java.util.LinkedList;
import java.util.List;
import java.util.Map;
import org.archive.crawler.framework.CrawlJob;
import org.archive.crawler.framework.Engine;
import org.archive.crawler.framework.CrawlController.State;
import org.archive.util.FileUtils;
@SuppressWarnings("serial")
@@ -22,5 +29,46 @@ public class EngineModel extends HashMap<String, Object> {
actions.add("add");
actions.add("create");
this.put("availableActions", actions);
this.put("jobs", makeJobList(engine, urlBaseRef));
}
private List<Map<String, Object>> makeJobList(Engine engine,
String urlBaseRef) {
List<Map<String, Object>> jobList;
jobList = new ArrayList<Map<String,Object>>();
// Generate list of jobs
ArrayList<Map.Entry<String,CrawlJob>> jobConfigurations = new ArrayList<Map.Entry<String,CrawlJob>>(engine.getJobConfigs().entrySet());
Collections.sort(jobConfigurations, new Comparator<Map.Entry<String, CrawlJob>>() {
public int compare(Map.Entry<String, CrawlJob> cj1, Map.Entry<String, CrawlJob> cj2) {
return cj1.getValue().compareTo(cj2.getValue());
}
});
for(Map.Entry<String,CrawlJob> jobConfig : jobConfigurations) {
CrawlJob job = jobConfig.getValue();
HashMap<String, Object> crawlJobModel = new HashMap<String, Object>();
crawlJobModel.put("shortName",job.getShortName());
crawlJobModel.put("url",urlBaseRef+"job/"+job.getShortName());
crawlJobModel.put("isProfile",job.isProfile());
crawlJobModel.put("launchCount",job.getLaunchCount());
crawlJobModel.put("lastLaunch",job.getLastLaunch());
crawlJobModel.put("hasApplicationContext",job.hasApplicationContext());
crawlJobModel.put("statusDescription", job.getJobStatusDescription());
crawlJobModel.put("isLaunchInfoPartial", job.isLaunchInfoPartial());
File primaryConfig = FileUtils.tryToCanonicalize(job.getPrimaryConfig());
crawlJobModel.put("primaryConfig", primaryConfig.getAbsolutePath());
crawlJobModel.put("primaryConfigUrl", urlBaseRef + "jobdir/" + primaryConfig.getName());
if (job.getCrawlController() != null) {
crawlJobModel.put("crawlControllerState", job.getCrawlController().getState());
if (job.getCrawlController().getState() == State.FINISHED) {
crawlJobModel.put("crawlExitStatus", job.getCrawlController().getCrawlExitStatus());
}
}
crawlJobModel.put("key", jobConfig.getKey());
jobList.add(crawlJobModel);
}
return jobList;
}
}