Converted data models to be usable for conversion to HTML and XML. Updated Models and freemarker templates to match XML naming

This commit is contained in:
Adam Miller
2013-03-11 17:05:18 -07:00
parent 50e49b6399
commit 60d00e3ef3
6 changed files with 106 additions and 248 deletions
@@ -74,16 +74,7 @@ public class EngineResource extends BaseResource {
getVariants().add(new Variant(MediaType.APPLICATION_XML));
Configuration tmpltCfg = new Configuration();
tmpltCfg.setClassForTemplateLoading(this.getClass(),"");
//TODO: this is temporary, remove this try-catch block
try {
tmpltCfg.setDirectoryForTemplateLoading(new File("/0/templates/"));
} catch (IOException e) {
// TODO Auto-generated catch block
e.printStackTrace();
}
tmpltCfg.setClassForTemplateLoading(this.getClass(),"");
tmpltCfg.setObjectWrapper(new DefaultObjectWrapper());
setTemplateConfiguration(tmpltCfg);
}
@@ -99,7 +90,7 @@ public class EngineResource extends BaseResource {
if (variant.getMediaType() == MediaType.APPLICATION_XML) {
representation = new WriterRepresentation(MediaType.APPLICATION_XML) {
public void write(Writer writer) throws IOException {
XmlMarshaller.marshalDocument(writer, "engine", makePresentableMap());
XmlMarshaller.marshalDocument(writer, "engine", makeDataModel());
}
};
} else {
@@ -213,13 +204,6 @@ public class EngineResource extends BaseResource {
return builtJobs;
}
protected List<String> getAvailableActions() {
List<String> actions = new LinkedList<String>();
actions.add("rescan");
actions.add("add");
actions.add("create");
return actions;
}
/**
* Constructs a nested Map data structure with the information represented
@@ -228,53 +212,18 @@ public class EngineResource extends BaseResource {
*
* @return the nested Map data structure
*/
protected LinkedHashMap<String,Object> makePresentableMap() {
protected EngineModel makeDataModel() {
String baseRef = getRequest().getResourceRef().getBaseRef().toString();
if(!baseRef.endsWith("/")) {
baseRef += "/";
}
EngineModel model = new EngineModel(getEngine(), baseRef);
LinkedHashMap<String,Object> info = new LinkedHashMap<String,Object>();
Engine engine = getEngine();
info.put("heritrixVersion", engine.getHeritrixVersion());
File jobsDir = FileUtils.tryToCanonicalize(engine.getJobsDir());
info.put("jobsDir", jobsDir.getAbsolutePath());
info.put("jobsDirUrl", baseRef + "jobsdir/");
info.put("availableActions", getAvailableActions());
info.put("heapReport", getEngine().heapReportData());
// re-scan job configs on each page load
ArrayList<CrawlJob> jobs = new ArrayList<CrawlJob>();
jobs.addAll(engine.getJobConfigs().values());
Collections.sort(jobs);
Collection<Map<String,Object>> jobsInfo = new LinkedList<Map<String,Object>>();
for (CrawlJob cj: jobs) {
// cj.writeHtmlTo(pw,"job/");
Map<String,Object> jobInfo = new LinkedHashMap<String, Object>();
jobInfo.put("shortName", cj.getShortName());
jobInfo.put("url", baseRef + "job/" + cj.getShortName());
jobInfo.put("isProfile", cj.isProfile());
jobInfo.put("launchCount", cj.getLaunchCount());
jobInfo.put("lastLaunch", cj.getLastLaunch());
File primaryConfig = FileUtils.tryToCanonicalize(cj.getPrimaryConfig());
jobInfo.put("primaryConfig", primaryConfig.getAbsolutePath());
jobInfo.put("primaryConfigUrl", baseRef + "job/" + cj.getShortName() + "/jobdir/" + primaryConfig.getName());
if (cj.getCrawlController() != null) {
jobInfo.put("crawlControllerState", cj.getCrawlController().getState());
if (cj.getCrawlController().getState() == State.FINISHED) {
jobInfo.put("crawlExitStatus", cj.getCrawlController().getCrawlExitStatus());
}
}
jobsInfo.add(jobInfo);
}
info.put("jobs", jobsInfo);
return info;
return model;
}
protected void writeHtml(Writer writer) {
Engine engine = getEngine();
EngineModel model = makeDataModel();
String baseRef = getRequest().getResourceRef().getBaseRef().toString();
if(!baseRef.endsWith("/")) {
baseRef += "/";
@@ -285,7 +234,7 @@ public class EngineResource extends BaseResource {
viewModel.setFlashes(Flash.getFlashes(getRequest()));
viewModel.put("baseRef",baseRef);
viewModel.put("filePathSeparator", File.pathSeparator);
viewModel.put("engine", new EngineModel(engine));
viewModel.put("engine", model);
viewModel.put("cssRef", getStylesheetRef());
try {
@@ -24,6 +24,7 @@ import java.io.IOException;
import java.io.PrintWriter;
import java.io.Writer;
import java.util.Collections;
import java.util.HashMap;
import java.util.LinkedHashMap;
import java.util.LinkedHashSet;
import java.util.LinkedList;
@@ -96,16 +97,6 @@ public class JobResource extends BaseResource {
Configuration tmpltCfg = new Configuration();
tmpltCfg.setClassForTemplateLoading(this.getClass(),"");
//TODO: this is temporary, remove this try-catch block
try {
tmpltCfg.setDirectoryForTemplateLoading(new File("/0/templates/"));
} catch (IOException e) {
// TODO Auto-generated catch block
e.printStackTrace();
}
//tmpltCfg.setObjectWrapper(new DefaultObjectWrapper());
tmpltCfg.setObjectWrapper(ObjectWrapper.BEANS_WRAPPER);
setTemplateConfiguration(tmpltCfg);
}
@@ -124,8 +115,9 @@ public class JobResource extends BaseResource {
if (variant.getMediaType() == MediaType.APPLICATION_XML) {
representation = new WriterRepresentation(MediaType.APPLICATION_XML) {
public void write(Writer writer) throws IOException {
XmlMarshaller.marshalDocument(writer, "job",
makePresentableMap());
CrawlJobModel model = makeDataModel();
model.put("heapReport", getEngine().heapReportData());
XmlMarshaller.marshalDocument(writer, "job", model);
}
};
} else {
@@ -149,112 +141,16 @@ public class JobResource extends BaseResource {
*
* @return the nested Map data structure
*/
protected LinkedHashMap<String, Object> makePresentableMap() {
LinkedHashMap<String, Object> info = new LinkedHashMap<String, Object>();
protected CrawlJobModel makeDataModel() {
String baseRef = getRequest().getResourceRef().getBaseRef().toString();
if (!baseRef.endsWith("/")) {
baseRef += "/";
}
Reference baseRefRef = new Reference(baseRef);
info.put("shortName", cj.getShortName());
if (cj.getCrawlController() != null) {
info.put("crawlControllerState", cj.getCrawlController().getState());
if (cj.getCrawlController().getState() == State.FINISHED) {
info.put("crawlExitStatus", cj.getCrawlController()
.getCrawlExitStatus());
}
}
info.put("statusDescription", cj.getJobStatusDescription());
info.put("availableActions", getAvailableActions());
info.put("launchCount", cj.getLaunchCount());
info.put("lastLaunch", cj.getLastLaunch());
info.put("isProfile", cj.isProfile());
File primaryConfig = FileUtils.tryToCanonicalize(cj.getPrimaryConfig());
info.put("primaryConfig", primaryConfig.getAbsolutePath());
info.put("primaryConfigUrl",
baseRef + "jobdir/" + primaryConfig.getName());
if (cj.getJobLog().exists())
try {
List<String> logLines = new LinkedList<String>();
FileUtils.pagedLines(cj.getJobLog(), -1, -5, logLines);
info.put("jobLogTail", logLines);
} catch (IOException ioe) {
throw new RuntimeException(ioe);
}
if (cj.hasApplicationContext()) {
info.put("uriTotalsReport", cj.uriTotalsReportData());
info.put("sizeTotalsReport", cj.sizeTotalsReportData());
info.put("rateReport", cj.rateReportData());
info.put("loadReport", cj.loadReportData());
info.put("elapsedReport", cj.elapsedReportData());
info.put("threadReport", cj.threadReportData());
info.put("frontierReport", cj.frontierReportData());
info.put("heapReport", getEngine().heapReportData());
if ((cj.isRunning() || (cj.hasApplicationContext() && !cj
.isLaunchable()))
&& cj.getCrawlController().getLoggerModule()
.getCrawlLogPath().getFile().exists()) {
try {
List<String> logLines = new LinkedList<String>();
FileUtils.pagedLines(cj.getCrawlController()
.getLoggerModule().getCrawlLogPath().getFile(), -1,
-10, logLines);
info.put("crawlLogTail", logLines);
} catch (IOException ioe) {
throw new RuntimeException(ioe);
}
}
}
List<Map<String, String>> configFiles = new LinkedList<Map<String, String>>();
for (String cppp : cj.getConfigPaths().keySet()) {
Map<String, String> configFileInfo = new LinkedHashMap<String, String>();
configFileInfo.put("key", cppp);
File path = FileUtils.tryToCanonicalize(cj.getConfigPaths()
.get(cppp).getFile());
configFileInfo.put("path", path.getAbsolutePath());
Reference urlRef = new Reference(baseRefRef, getHrefPath(path, cj))
.getTargetRef();
configFileInfo.put("url", urlRef.toString());
configFiles.add(configFileInfo);
}
info.put("configFiles", configFiles);
return info;
}
protected Set<String> getAvailableActions() {
Set<String> actions = new LinkedHashSet<String>();
if (!cj.hasApplicationContext()) {
actions.add("build");
}
if (!cj.isProfile() && cj.isLaunchable()) {
actions.add("launch");
}
if (cj.isPausable()) {
actions.add("pause");
}
if (cj.isUnpausable()) {
actions.add("unpause");
}
if (cj.getCheckpointService() != null && cj.isRunning()) {
actions.add("checkpoint");
}
if (cj.isRunning()) {
actions.add("terminate");
}
if (cj.hasApplicationContext()) {
actions.add("teardown");
}
return actions;
CrawlJobModel model = new CrawlJobModel(cj,baseRef);
return model;
}
protected void writeHtml(Writer writer) {
@@ -268,8 +164,8 @@ public class JobResource extends BaseResource {
viewModel.setFlashes(Flash.getFlashes(getRequest()));
viewModel.put("baseRef",baseRef);
viewModel.put("cssRef", getStylesheetRef());
viewModel.put("job",new CrawlJobModel(cj));
viewModel.put("engine", new EngineModel(getEngine()));
viewModel.put("job", makeDataModel());
viewModel.put("engine", new EngineModel(getEngine(),baseRef));
viewModel.put("cj", cj);
try {
@@ -284,38 +180,6 @@ public class JobResource extends BaseResource {
}
/**
* Print a link to the given File
*
* @param pw
* PrintWriter
* @param f
* File
*/
protected void printLinkedFile(PrintWriter pw, File f) {
printLinkedFile(pw, f, f.toString(), null);
}
/**
* Print a link to the given File, using the given link text
*
* @param pw
* PrintWriter
* @param f
* File
*/
protected void printLinkedFile(PrintWriter pw, File f, String linktext,
String queryString) {
String relativePath = JobResource.getHrefPath(f, cj);
pw.println("<a href='" + relativePath
+ ((queryString == null) ? "" : "?" + queryString) + "'>"
+ linktext + "</a>");
if (EDIT_FILTER.accept(f)) {
pw.println("[<a href='" + relativePath
+ "?format=textedit'>edit</a>]");
}
}
/**
* Get a usable HrefPath, relative to the JobResource, for the given file.
* Assumes usual helper resources ('jobdir/', 'anypath/') at the usual
@@ -6,13 +6,16 @@ import java.io.Serializable;
import java.util.ArrayList;
import java.util.Collections;
import java.util.HashMap;
import java.util.LinkedHashSet;
import java.util.LinkedList;
import java.util.List;
import java.util.Map;
import java.util.Set;
import org.apache.commons.io.filefilter.IOFileFilter;
import org.archive.checkpointing.Checkpoint;
import org.archive.crawler.framework.CrawlJob;
import org.archive.crawler.framework.CrawlController.State;
import org.archive.crawler.reporting.Report;
import org.archive.spring.ConfigPath;
import org.archive.util.ArchiveUtils;
@@ -25,31 +28,67 @@ public class CrawlJobModel extends HashMap<String, Object> implements Serializab
public static final IOFileFilter EDIT_FILTER = FileUtils
.getRegexFileFilter(".*\\.((c?xml)|(txt))$");
public CrawlJobModel(CrawlJob crawlJob){
public CrawlJobModel(CrawlJob crawlJob, String urlBaseRef){
super();
this.crawlJob=crawlJob;
this.put("shortName",crawlJob.getShortName());
this.put("status", crawlJob.getJobStatusDescription());
if (crawlJob.getCrawlController() != null) {
this.put("crawlControllerState", crawlJob.getCrawlController().getState());
if (crawlJob.getCrawlController().getState() == State.FINISHED) {
this.put("crawlExitStatus", crawlJob.getCrawlController().getCrawlExitStatus());
}
}
this.put("statusDescription", crawlJob.getJobStatusDescription());
this.put("launchCount", crawlJob.getLaunchCount());
this.put("config", crawlJob.getPrimaryConfig());
this.put("isProfile", crawlJob.isProfile());
this.put("isLaunchInfoPartial", crawlJob.isLaunchInfoPartial());
this.put("isLaunchable", crawlJob.isLaunchable());
this.put("isPausable", crawlJob.isPausable());
this.put("isUnpausable", crawlJob.isUnpausable());
this.put("isRunning", crawlJob.isRunning());
this.put("uriTotalsReport", crawlJob.uriTotalsReport());
this.put("sizeTotalsReport", crawlJob.sizeTotalsReport());
this.put("hasApplicationContext",crawlJob.hasApplicationContext());
this.put("lastLaunch",crawlJob.getLastLaunch());
this.put("alertCount", crawlJob.getAlertCount());
this.put("rateReport", crawlJob.rateReport());
this.put("isProfile", crawlJob.isProfile());
File primaryConfig = FileUtils.tryToCanonicalize(crawlJob.getPrimaryConfig());
this.put("primaryConfig", primaryConfig.getAbsolutePath());
this.put("primaryConfigUrl", urlBaseRef + "jobdir/" + primaryConfig.getName());
this.put("url",urlBaseRef+"job/"+crawlJob.getShortName());
this.put("isLaunchInfoPartial", crawlJob.isLaunchInfoPartial());
this.put("isRunning", crawlJob.isRunning());
this.put("isLaunchable",crawlJob.isLaunchable());
this.put("uriTotalsReport", crawlJob.uriTotalsReport());
this.put("sizeTotalsReport", crawlJob.sizeTotalsReport());
this.put("rateReport", crawlJob.rateReport());
this.put("loadReport", crawlJob.loadReport());
this.put("elapsedReport", crawlJob.elapsedReport());
this.put("threadReport", crawlJob.threadReport());
this.put("frontierReport", crawlJob.frontierReport());
this.put("loadReport", crawlJob.loadReport());
this.put("configFiles",generateConfigReferencedPaths(urlBaseRef));
this.put("jobLogTail", generateJobLogTail());
this.put("crawlLogTail", generateCrawlLogTail());
this.put("hasApplicationContext",crawlJob.hasApplicationContext());
this.put("alertCount", crawlJob.getAlertCount());
Set<String> actions = new LinkedHashSet<String>();
this.put("availableActions",actions);
if (!crawlJob.hasApplicationContext())
actions.add("build");
if (!crawlJob.isProfile() && crawlJob.isLaunchable())
actions.add("launch");
if (crawlJob.isPausable())
actions.add("pause");
if (crawlJob.isUnpausable())
actions.add("unpause");
if (crawlJob.getCheckpointService() != null && crawlJob.isRunning())
actions.add("checkpoint");
if (crawlJob.isRunning())
actions.add("terminate");
if (crawlJob.hasApplicationContext())
actions.add("teardown");
this.put("key", "");
}
public String getLastLaunchTime(){
@@ -86,14 +125,9 @@ public class CrawlJobModel extends HashMap<String, Object> implements Serializab
return crawlJob.getCrawlController().getLoggerModule().getCrawlLogPath().getFile();
}
public List<File> getImportedConfigurationFilePaths(){
// List<String> configList = new ArrayList<String>();
// for( File f : crawlJob.getImportedConfigs(crawlJob.getPrimaryConfig())){
// configList.add(f);
// }
// return configList;
return crawlJob.getImportedConfigs(crawlJob.getPrimaryConfig());
}
public List<String> getJobLogTail(){
public List<String> generateJobLogTail(){
List<String> jobLog = new ArrayList<String>();
if (crawlJob.getJobLog().exists()) {
try {
@@ -105,9 +139,11 @@ public class CrawlJobModel extends HashMap<String, Object> implements Serializab
}
return jobLog;
}
public List<String> getCrawlLogTail() {
public List<String> generateCrawlLogTail() {
List<String> logLines = new LinkedList<String>();
if(crawlJob.getCrawlController().getLoggerModule().getCrawlLogPath().getFile().exists()) {
if ((crawlJob.isRunning() || (crawlJob.hasApplicationContext() && !crawlJob.isLaunchable()))
&& crawlJob.getCrawlController().getLoggerModule()
.getCrawlLogPath().getFile().exists()) {
try {
FileUtils.pagedLines(crawlJob.getCrawlController()
.getLoggerModule().getCrawlLogPath().getFile(), -1,
@@ -132,14 +168,15 @@ public class CrawlJobModel extends HashMap<String, Object> implements Serializab
}
return reports;
}
public List<Map<String,Object>> getConfigReferencedPaths(){
private List<Map<String,Object>> generateConfigReferencedPaths(String baseRef){
List<Map<String,Object>> referencedPaths = new ArrayList<Map<String,Object>>();
for (String key : crawlJob.getConfigPaths().keySet()) {
ConfigPath cp = crawlJob.getConfigPaths().get(key);
Map<String,Object> configMap = new HashMap<String,Object>();
configMap.put("key", key);
configMap.put("name", cp.getName());
configMap.put("path",cp.getFile());
configMap.put("path",FileUtils.tryToCanonicalize(cp.getFile()).getAbsolutePath());
configMap.put("url",baseRef+"engine/anypath/"+configMap.get("path"));
configMap.put("editable", EDIT_FILTER.accept(cp.getFile()));
referencedPaths.add(configMap);
}
@@ -5,6 +5,7 @@ import java.util.ArrayList;
import java.util.Collections;
import java.util.Comparator;
import java.util.HashMap;
import java.util.LinkedList;
import java.util.List;
import java.util.Map;
import java.util.Set;
@@ -16,14 +17,15 @@ import org.archive.util.FileUtils;
@SuppressWarnings("serial")
public class EngineModel extends HashMap<String, Object> {
public EngineModel(Engine engine){
public EngineModel(Engine engine, String urlBaseRef){
super();
this.put("heritrixVersion", engine.getHeritrixVersion());
this.put("heapReport", engine.heapReport());
this.put("jobsDirAbsolutePath", FileUtils.tryToCanonicalize(engine.getJobsDir()).getAbsolutePath());
this.put("jobsDir", FileUtils.tryToCanonicalize(engine.getJobsDir()).getAbsolutePath());
this.put("jobsDirUrl", urlBaseRef + "jobsdir/");
List<Map<String,Object>> jobList = new ArrayList<Map<String,Object>>();
this.put("jobList", jobList);
this.put("jobs", jobList);
//Generate list of jobs
ArrayList<Map.Entry<String,CrawlJob>> jobConfigurations = new ArrayList<Map.Entry<String,CrawlJob>>(engine.getJobConfigs().entrySet());
@@ -33,9 +35,15 @@ public class EngineModel extends HashMap<String, Object> {
}
});
for(Map.Entry<String,CrawlJob> jobConfig : jobConfigurations) {
CrawlJobModel crawlJobModel = new CrawlJobModel(jobConfig.getValue());
CrawlJobModel crawlJobModel = new CrawlJobModel(jobConfig.getValue(), urlBaseRef);
crawlJobModel.put("key", jobConfig.getKey());
jobList.add(crawlJobModel);
}
List<String> actions = new LinkedList<String>();
actions.add("rescan");
actions.add("add");
actions.add("create");
this.put("availableActions", actions);
}
}
@@ -19,19 +19,19 @@
<b>Memory: </b>
${engine.heapReport}
<br/><br/>
<b>Jobs Directory</b>: <a href='jobsdir'>${engine.jobsDirAbsolutePath}</a>
<b>Jobs Directory</b>: <a href='jobsdir'>${engine.jobsDir}</a>
<form method='POST'><h2>Job Directories (${engine.jobList?size})
<form method='POST'><h2>Job Directories (${engine.jobs?size})
<input type='submit' name='action' value='rescan'></h2>
</form>
<ul>
<#list engine.jobList as crawlJob>
<#list engine.jobs as crawlJob>
<li>
<div>
<a href="/engine/job/${crawlJob.shortName}">${crawlJob.shortName}</a>
<#if crawlJob.hasApplicationContext>
&laquo;${crawlJob.status}&raquo;
&laquo;${crawlJob.statusDescription}&raquo;
</#if>
<#if crawlJob.isLaunchInfoPartial>
<span> at least </span>
@@ -39,7 +39,7 @@ ${engine.heapReport}
${crawlJob.launchCount} launches
</div>
<div style="color:#666">
${crawlJob.config}
${crawlJob.primaryConfig}
</div>
<#if crawlJob.lastLaunch??>
<div>(last at ${crawlJob.lastLaunch})</div>
@@ -52,7 +52,7 @@ ${crawlJob.config}
<form method='POST'>
Create new job directory with recommended starting configuration<br/>
<b>Path:</b> ${engine.jobsDirAbsolutePath}${filePathSeparator}
<b>Path:</b> ${engine.jobsDir}${filePathSeparator}
<input name='createpath'/>
<input type='submit' name='action' value='create'>
</form>
@@ -72,9 +72,9 @@ will then require access to the hosting machine. You should
cleanly terminate and teardown any jobs in progress first.<div>
<form method='POST'>
<#list engine.jobList as crawlJob>
<#list engine.jobs as crawlJob>
<#if crawlJob.hasApplicationContext>
<div>Job ${crawlJob.key} still &laquo; ${crawlJob.status} &raquo;</div>
<div>Job ${crawlJob.key} still &laquo; ${crawlJob.statusDescription} &raquo;</div>
<input type='checkbox' id="ignore__${crawlJob.key}" name='ignore__${crawlJob.key}'>
<label for='ignore__${crawlJob.key}'> Ignore job '${crawlJob.key}' and exit anyway</label>
<br>
@@ -1,7 +1,7 @@
<!DOCTYPE html>
<html>
<head>
<title>${job.shortName} - ${job.status} - Job main page</title>
<title>${job.shortName} - ${job.statusDescription} - Job main page</title>
<base href='${baseRef}'/>
<link rel="stylesheet" type="text/css" href="${cssRef}">
</head>
@@ -34,14 +34,14 @@
<input type='submit' name='action' value='launch'
<#if job.isProfile> disabled='disabled' title='profiles cannot be launched'
</#if>
<#if !job.isLaunchable>
<#if !job.availableActions?seq_contains("launch")>
disabled='disabled'
</#if>
/>
</span>
<span class="bgroup">
<input <#if !job.isPausable> disabled</#if> type='submit' name='action' value='pause' />
<input <#if !job.isUnpausable> disabled</#if> type='submit' name='action' value='unpause' />
<input <#if !job.availableActions?seq_contains("pause")> disabled</#if> type='submit' name='action' value='pause' />
<input <#if !job.availableActions?seq_contains("unpause")> disabled</#if> type='submit' name='action' value='unpause' />
<input <#if !job.isRunning> disabled</#if> type='submit' name='action' value='checkpoint' />
</span>
<span class="bgroup">
@@ -75,7 +75,7 @@
<div>${line?html}</div>
</#list>
</div>
<h2>Job is ${job.status}</h2>
<h2>Job is ${job.statusDescription}</h2>
<#if job.hasApplicationContext>
<dl id="jobstats">
<dt>Totals</dt>
@@ -123,7 +123,7 @@ ${line?html}
<h2>Files</h2>
<h3>Browser <a href='jobdir'>Job Directory</a></h3>
<h3>Configuration-referenced Paths</h3>
<#assign refPaths=job.configReferencedPaths! />
<#assign refPaths=job.referen! />
<#if !refPaths?has_content >
<i>build the job to discover referenced paths</i>
<#else>