From 60d00e3ef3b7c2f3eb6e1262d13fbcfacdf1ef90 Mon Sep 17 00:00:00 2001 From: Adam Miller Date: Mon, 10 Sep 2012 17:03:49 -0700 Subject: [PATCH] Converted data models to be usable for conversion to HTML and XML. Updated Models and freemarker templates to match XML naming --- .../crawler/restlet/EngineResource.java | 65 +------- .../archive/crawler/restlet/JobResource.java | 156 ++---------------- .../crawler/restlet/models/CrawlJobModel.java | 89 +++++++--- .../crawler/restlet/models/EngineModel.java | 16 +- .../org/archive/crawler/restlet/Engine.ftl | 16 +- .../org/archive/crawler/restlet/Job.ftl | 12 +- 6 files changed, 106 insertions(+), 248 deletions(-) diff --git a/engine/src/main/java/org/archive/crawler/restlet/EngineResource.java b/engine/src/main/java/org/archive/crawler/restlet/EngineResource.java index 64947b70..35f7fcbc 100644 --- a/engine/src/main/java/org/archive/crawler/restlet/EngineResource.java +++ b/engine/src/main/java/org/archive/crawler/restlet/EngineResource.java @@ -74,16 +74,7 @@ public class EngineResource extends BaseResource { getVariants().add(new Variant(MediaType.APPLICATION_XML)); Configuration tmpltCfg = new Configuration(); - tmpltCfg.setClassForTemplateLoading(this.getClass(),""); - - //TODO: this is temporary, remove this try-catch block - try { - tmpltCfg.setDirectoryForTemplateLoading(new File("/0/templates/")); - } catch (IOException e) { - // TODO Auto-generated catch block - e.printStackTrace(); - } - + tmpltCfg.setClassForTemplateLoading(this.getClass(),""); tmpltCfg.setObjectWrapper(new DefaultObjectWrapper()); setTemplateConfiguration(tmpltCfg); } @@ -99,7 +90,7 @@ public class EngineResource extends BaseResource { if (variant.getMediaType() == MediaType.APPLICATION_XML) { representation = new WriterRepresentation(MediaType.APPLICATION_XML) { public void write(Writer writer) throws IOException { - XmlMarshaller.marshalDocument(writer, "engine", makePresentableMap()); + XmlMarshaller.marshalDocument(writer, "engine", makeDataModel()); } }; } else { @@ -213,13 +204,6 @@ public class EngineResource extends BaseResource { return builtJobs; } - protected List getAvailableActions() { - List actions = new LinkedList(); - actions.add("rescan"); - actions.add("add"); - actions.add("create"); - return actions; - } /** * Constructs a nested Map data structure with the information represented @@ -228,53 +212,18 @@ public class EngineResource extends BaseResource { * * @return the nested Map data structure */ - protected LinkedHashMap makePresentableMap() { + protected EngineModel makeDataModel() { String baseRef = getRequest().getResourceRef().getBaseRef().toString(); if(!baseRef.endsWith("/")) { baseRef += "/"; } + EngineModel model = new EngineModel(getEngine(), baseRef); - LinkedHashMap info = new LinkedHashMap(); - Engine engine = getEngine(); - info.put("heritrixVersion", engine.getHeritrixVersion()); - File jobsDir = FileUtils.tryToCanonicalize(engine.getJobsDir()); - info.put("jobsDir", jobsDir.getAbsolutePath()); - info.put("jobsDirUrl", baseRef + "jobsdir/"); - info.put("availableActions", getAvailableActions()); - info.put("heapReport", getEngine().heapReportData()); - - // re-scan job configs on each page load - ArrayList jobs = new ArrayList(); - - jobs.addAll(engine.getJobConfigs().values()); - Collections.sort(jobs); - Collection> jobsInfo = new LinkedList>(); - for (CrawlJob cj: jobs) { - // cj.writeHtmlTo(pw,"job/"); - Map jobInfo = new LinkedHashMap(); - jobInfo.put("shortName", cj.getShortName()); - jobInfo.put("url", baseRef + "job/" + cj.getShortName()); - jobInfo.put("isProfile", cj.isProfile()); - jobInfo.put("launchCount", cj.getLaunchCount()); - jobInfo.put("lastLaunch", cj.getLastLaunch()); - File primaryConfig = FileUtils.tryToCanonicalize(cj.getPrimaryConfig()); - jobInfo.put("primaryConfig", primaryConfig.getAbsolutePath()); - jobInfo.put("primaryConfigUrl", baseRef + "job/" + cj.getShortName() + "/jobdir/" + primaryConfig.getName()); - if (cj.getCrawlController() != null) { - jobInfo.put("crawlControllerState", cj.getCrawlController().getState()); - if (cj.getCrawlController().getState() == State.FINISHED) { - jobInfo.put("crawlExitStatus", cj.getCrawlController().getCrawlExitStatus()); - } - } - jobsInfo.add(jobInfo); - } - info.put("jobs", jobsInfo); - - return info; + return model; } protected void writeHtml(Writer writer) { - Engine engine = getEngine(); + EngineModel model = makeDataModel(); String baseRef = getRequest().getResourceRef().getBaseRef().toString(); if(!baseRef.endsWith("/")) { baseRef += "/"; @@ -285,7 +234,7 @@ public class EngineResource extends BaseResource { viewModel.setFlashes(Flash.getFlashes(getRequest())); viewModel.put("baseRef",baseRef); viewModel.put("filePathSeparator", File.pathSeparator); - viewModel.put("engine", new EngineModel(engine)); + viewModel.put("engine", model); viewModel.put("cssRef", getStylesheetRef()); try { diff --git a/engine/src/main/java/org/archive/crawler/restlet/JobResource.java b/engine/src/main/java/org/archive/crawler/restlet/JobResource.java index 6b7cf946..0c8c2ff7 100644 --- a/engine/src/main/java/org/archive/crawler/restlet/JobResource.java +++ b/engine/src/main/java/org/archive/crawler/restlet/JobResource.java @@ -24,6 +24,7 @@ import java.io.IOException; import java.io.PrintWriter; import java.io.Writer; import java.util.Collections; +import java.util.HashMap; import java.util.LinkedHashMap; import java.util.LinkedHashSet; import java.util.LinkedList; @@ -96,16 +97,6 @@ public class JobResource extends BaseResource { Configuration tmpltCfg = new Configuration(); tmpltCfg.setClassForTemplateLoading(this.getClass(),""); - - //TODO: this is temporary, remove this try-catch block - try { - tmpltCfg.setDirectoryForTemplateLoading(new File("/0/templates/")); - } catch (IOException e) { - // TODO Auto-generated catch block - e.printStackTrace(); - } - - //tmpltCfg.setObjectWrapper(new DefaultObjectWrapper()); tmpltCfg.setObjectWrapper(ObjectWrapper.BEANS_WRAPPER); setTemplateConfiguration(tmpltCfg); } @@ -124,8 +115,9 @@ public class JobResource extends BaseResource { if (variant.getMediaType() == MediaType.APPLICATION_XML) { representation = new WriterRepresentation(MediaType.APPLICATION_XML) { public void write(Writer writer) throws IOException { - XmlMarshaller.marshalDocument(writer, "job", - makePresentableMap()); + CrawlJobModel model = makeDataModel(); + model.put("heapReport", getEngine().heapReportData()); + XmlMarshaller.marshalDocument(writer, "job", model); } }; } else { @@ -149,112 +141,16 @@ public class JobResource extends BaseResource { * * @return the nested Map data structure */ - protected LinkedHashMap makePresentableMap() { - LinkedHashMap info = new LinkedHashMap(); + protected CrawlJobModel makeDataModel() { String baseRef = getRequest().getResourceRef().getBaseRef().toString(); if (!baseRef.endsWith("/")) { baseRef += "/"; } Reference baseRefRef = new Reference(baseRef); - - info.put("shortName", cj.getShortName()); - if (cj.getCrawlController() != null) { - info.put("crawlControllerState", cj.getCrawlController().getState()); - if (cj.getCrawlController().getState() == State.FINISHED) { - info.put("crawlExitStatus", cj.getCrawlController() - .getCrawlExitStatus()); - } - } - info.put("statusDescription", cj.getJobStatusDescription()); - info.put("availableActions", getAvailableActions()); - - info.put("launchCount", cj.getLaunchCount()); - info.put("lastLaunch", cj.getLastLaunch()); - info.put("isProfile", cj.isProfile()); - File primaryConfig = FileUtils.tryToCanonicalize(cj.getPrimaryConfig()); - info.put("primaryConfig", primaryConfig.getAbsolutePath()); - info.put("primaryConfigUrl", - baseRef + "jobdir/" + primaryConfig.getName()); - - if (cj.getJobLog().exists()) - try { - List logLines = new LinkedList(); - FileUtils.pagedLines(cj.getJobLog(), -1, -5, logLines); - info.put("jobLogTail", logLines); - } catch (IOException ioe) { - throw new RuntimeException(ioe); - } - - if (cj.hasApplicationContext()) { - info.put("uriTotalsReport", cj.uriTotalsReportData()); - info.put("sizeTotalsReport", cj.sizeTotalsReportData()); - info.put("rateReport", cj.rateReportData()); - info.put("loadReport", cj.loadReportData()); - info.put("elapsedReport", cj.elapsedReportData()); - info.put("threadReport", cj.threadReportData()); - info.put("frontierReport", cj.frontierReportData()); - info.put("heapReport", getEngine().heapReportData()); - - if ((cj.isRunning() || (cj.hasApplicationContext() && !cj - .isLaunchable())) - && cj.getCrawlController().getLoggerModule() - .getCrawlLogPath().getFile().exists()) { - try { - List logLines = new LinkedList(); - FileUtils.pagedLines(cj.getCrawlController() - .getLoggerModule().getCrawlLogPath().getFile(), -1, - -10, logLines); - info.put("crawlLogTail", logLines); - } catch (IOException ioe) { - throw new RuntimeException(ioe); - } - } - } - - List> configFiles = new LinkedList>(); - for (String cppp : cj.getConfigPaths().keySet()) { - Map configFileInfo = new LinkedHashMap(); - configFileInfo.put("key", cppp); - File path = FileUtils.tryToCanonicalize(cj.getConfigPaths() - .get(cppp).getFile()); - configFileInfo.put("path", path.getAbsolutePath()); - Reference urlRef = new Reference(baseRefRef, getHrefPath(path, cj)) - .getTargetRef(); - configFileInfo.put("url", urlRef.toString()); - configFiles.add(configFileInfo); - } - info.put("configFiles", configFiles); - - return info; - } - - protected Set getAvailableActions() { - Set actions = new LinkedHashSet(); - - if (!cj.hasApplicationContext()) { - actions.add("build"); - } - if (!cj.isProfile() && cj.isLaunchable()) { - actions.add("launch"); - } - if (cj.isPausable()) { - actions.add("pause"); - } - if (cj.isUnpausable()) { - actions.add("unpause"); - } - if (cj.getCheckpointService() != null && cj.isRunning()) { - actions.add("checkpoint"); - } - if (cj.isRunning()) { - actions.add("terminate"); - } - if (cj.hasApplicationContext()) { - actions.add("teardown"); - } - - return actions; + CrawlJobModel model = new CrawlJobModel(cj,baseRef); + + return model; } protected void writeHtml(Writer writer) { @@ -268,8 +164,8 @@ public class JobResource extends BaseResource { viewModel.setFlashes(Flash.getFlashes(getRequest())); viewModel.put("baseRef",baseRef); viewModel.put("cssRef", getStylesheetRef()); - viewModel.put("job",new CrawlJobModel(cj)); - viewModel.put("engine", new EngineModel(getEngine())); + viewModel.put("job", makeDataModel()); + viewModel.put("engine", new EngineModel(getEngine(),baseRef)); viewModel.put("cj", cj); try { @@ -284,38 +180,6 @@ public class JobResource extends BaseResource { } - /** - * Print a link to the given File - * - * @param pw - * PrintWriter - * @param f - * File - */ - protected void printLinkedFile(PrintWriter pw, File f) { - printLinkedFile(pw, f, f.toString(), null); - } - - /** - * Print a link to the given File, using the given link text - * - * @param pw - * PrintWriter - * @param f - * File - */ - protected void printLinkedFile(PrintWriter pw, File f, String linktext, - String queryString) { - String relativePath = JobResource.getHrefPath(f, cj); - pw.println("" - + linktext + ""); - if (EDIT_FILTER.accept(f)) { - pw.println("[edit]"); - } - } - /** * Get a usable HrefPath, relative to the JobResource, for the given file. * Assumes usual helper resources ('jobdir/', 'anypath/') at the usual diff --git a/engine/src/main/java/org/archive/crawler/restlet/models/CrawlJobModel.java b/engine/src/main/java/org/archive/crawler/restlet/models/CrawlJobModel.java index c81c1ee6..9c4319b6 100644 --- a/engine/src/main/java/org/archive/crawler/restlet/models/CrawlJobModel.java +++ b/engine/src/main/java/org/archive/crawler/restlet/models/CrawlJobModel.java @@ -6,13 +6,16 @@ import java.io.Serializable; import java.util.ArrayList; import java.util.Collections; import java.util.HashMap; +import java.util.LinkedHashSet; import java.util.LinkedList; import java.util.List; import java.util.Map; +import java.util.Set; import org.apache.commons.io.filefilter.IOFileFilter; import org.archive.checkpointing.Checkpoint; import org.archive.crawler.framework.CrawlJob; +import org.archive.crawler.framework.CrawlController.State; import org.archive.crawler.reporting.Report; import org.archive.spring.ConfigPath; import org.archive.util.ArchiveUtils; @@ -25,31 +28,67 @@ public class CrawlJobModel extends HashMap implements Serializab public static final IOFileFilter EDIT_FILTER = FileUtils .getRegexFileFilter(".*\\.((c?xml)|(txt))$"); - public CrawlJobModel(CrawlJob crawlJob){ + public CrawlJobModel(CrawlJob crawlJob, String urlBaseRef){ super(); this.crawlJob=crawlJob; this.put("shortName",crawlJob.getShortName()); - this.put("status", crawlJob.getJobStatusDescription()); + if (crawlJob.getCrawlController() != null) { + this.put("crawlControllerState", crawlJob.getCrawlController().getState()); + if (crawlJob.getCrawlController().getState() == State.FINISHED) { + this.put("crawlExitStatus", crawlJob.getCrawlController().getCrawlExitStatus()); + } + } + + this.put("statusDescription", crawlJob.getJobStatusDescription()); + this.put("launchCount", crawlJob.getLaunchCount()); - this.put("config", crawlJob.getPrimaryConfig()); - this.put("isProfile", crawlJob.isProfile()); - this.put("isLaunchInfoPartial", crawlJob.isLaunchInfoPartial()); - this.put("isLaunchable", crawlJob.isLaunchable()); - this.put("isPausable", crawlJob.isPausable()); - this.put("isUnpausable", crawlJob.isUnpausable()); - this.put("isRunning", crawlJob.isRunning()); - this.put("uriTotalsReport", crawlJob.uriTotalsReport()); - this.put("sizeTotalsReport", crawlJob.sizeTotalsReport()); - this.put("hasApplicationContext",crawlJob.hasApplicationContext()); this.put("lastLaunch",crawlJob.getLastLaunch()); - this.put("alertCount", crawlJob.getAlertCount()); - this.put("rateReport", crawlJob.rateReport()); + this.put("isProfile", crawlJob.isProfile()); + + File primaryConfig = FileUtils.tryToCanonicalize(crawlJob.getPrimaryConfig()); + this.put("primaryConfig", primaryConfig.getAbsolutePath()); + this.put("primaryConfigUrl", urlBaseRef + "jobdir/" + primaryConfig.getName()); + this.put("url",urlBaseRef+"job/"+crawlJob.getShortName()); + + this.put("isLaunchInfoPartial", crawlJob.isLaunchInfoPartial()); + this.put("isRunning", crawlJob.isRunning()); + this.put("isLaunchable",crawlJob.isLaunchable()); + + this.put("uriTotalsReport", crawlJob.uriTotalsReport()); + this.put("sizeTotalsReport", crawlJob.sizeTotalsReport()); + this.put("rateReport", crawlJob.rateReport()); this.put("loadReport", crawlJob.loadReport()); this.put("elapsedReport", crawlJob.elapsedReport()); this.put("threadReport", crawlJob.threadReport()); this.put("frontierReport", crawlJob.frontierReport()); - this.put("loadReport", crawlJob.loadReport()); - + + this.put("configFiles",generateConfigReferencedPaths(urlBaseRef)); + this.put("jobLogTail", generateJobLogTail()); + this.put("crawlLogTail", generateCrawlLogTail()); + + this.put("hasApplicationContext",crawlJob.hasApplicationContext()); + this.put("alertCount", crawlJob.getAlertCount()); + + Set actions = new LinkedHashSet(); + this.put("availableActions",actions); + if (!crawlJob.hasApplicationContext()) + actions.add("build"); + + if (!crawlJob.isProfile() && crawlJob.isLaunchable()) + actions.add("launch"); + if (crawlJob.isPausable()) + actions.add("pause"); + if (crawlJob.isUnpausable()) + actions.add("unpause"); + + if (crawlJob.getCheckpointService() != null && crawlJob.isRunning()) + actions.add("checkpoint"); + if (crawlJob.isRunning()) + actions.add("terminate"); + if (crawlJob.hasApplicationContext()) + actions.add("teardown"); + + this.put("key", ""); } public String getLastLaunchTime(){ @@ -86,14 +125,9 @@ public class CrawlJobModel extends HashMap implements Serializab return crawlJob.getCrawlController().getLoggerModule().getCrawlLogPath().getFile(); } public List getImportedConfigurationFilePaths(){ -// List configList = new ArrayList(); -// for( File f : crawlJob.getImportedConfigs(crawlJob.getPrimaryConfig())){ -// configList.add(f); -// } -// return configList; return crawlJob.getImportedConfigs(crawlJob.getPrimaryConfig()); } - public List getJobLogTail(){ + public List generateJobLogTail(){ List jobLog = new ArrayList(); if (crawlJob.getJobLog().exists()) { try { @@ -105,9 +139,11 @@ public class CrawlJobModel extends HashMap implements Serializab } return jobLog; } - public List getCrawlLogTail() { + public List generateCrawlLogTail() { List logLines = new LinkedList(); - if(crawlJob.getCrawlController().getLoggerModule().getCrawlLogPath().getFile().exists()) { + if ((crawlJob.isRunning() || (crawlJob.hasApplicationContext() && !crawlJob.isLaunchable())) + && crawlJob.getCrawlController().getLoggerModule() + .getCrawlLogPath().getFile().exists()) { try { FileUtils.pagedLines(crawlJob.getCrawlController() .getLoggerModule().getCrawlLogPath().getFile(), -1, @@ -132,14 +168,15 @@ public class CrawlJobModel extends HashMap implements Serializab } return reports; } - public List> getConfigReferencedPaths(){ + private List> generateConfigReferencedPaths(String baseRef){ List> referencedPaths = new ArrayList>(); for (String key : crawlJob.getConfigPaths().keySet()) { ConfigPath cp = crawlJob.getConfigPaths().get(key); Map configMap = new HashMap(); configMap.put("key", key); configMap.put("name", cp.getName()); - configMap.put("path",cp.getFile()); + configMap.put("path",FileUtils.tryToCanonicalize(cp.getFile()).getAbsolutePath()); + configMap.put("url",baseRef+"engine/anypath/"+configMap.get("path")); configMap.put("editable", EDIT_FILTER.accept(cp.getFile())); referencedPaths.add(configMap); } diff --git a/engine/src/main/java/org/archive/crawler/restlet/models/EngineModel.java b/engine/src/main/java/org/archive/crawler/restlet/models/EngineModel.java index 5527e97d..63dc7ba0 100644 --- a/engine/src/main/java/org/archive/crawler/restlet/models/EngineModel.java +++ b/engine/src/main/java/org/archive/crawler/restlet/models/EngineModel.java @@ -5,6 +5,7 @@ import java.util.ArrayList; import java.util.Collections; import java.util.Comparator; import java.util.HashMap; +import java.util.LinkedList; import java.util.List; import java.util.Map; import java.util.Set; @@ -16,14 +17,15 @@ import org.archive.util.FileUtils; @SuppressWarnings("serial") public class EngineModel extends HashMap { - public EngineModel(Engine engine){ + public EngineModel(Engine engine, String urlBaseRef){ super(); this.put("heritrixVersion", engine.getHeritrixVersion()); this.put("heapReport", engine.heapReport()); - this.put("jobsDirAbsolutePath", FileUtils.tryToCanonicalize(engine.getJobsDir()).getAbsolutePath()); + this.put("jobsDir", FileUtils.tryToCanonicalize(engine.getJobsDir()).getAbsolutePath()); + this.put("jobsDirUrl", urlBaseRef + "jobsdir/"); List> jobList = new ArrayList>(); - this.put("jobList", jobList); + this.put("jobs", jobList); //Generate list of jobs ArrayList> jobConfigurations = new ArrayList>(engine.getJobConfigs().entrySet()); @@ -33,9 +35,15 @@ public class EngineModel extends HashMap { } }); for(Map.Entry jobConfig : jobConfigurations) { - CrawlJobModel crawlJobModel = new CrawlJobModel(jobConfig.getValue()); + CrawlJobModel crawlJobModel = new CrawlJobModel(jobConfig.getValue(), urlBaseRef); crawlJobModel.put("key", jobConfig.getKey()); jobList.add(crawlJobModel); } + + List actions = new LinkedList(); + actions.add("rescan"); + actions.add("add"); + actions.add("create"); + this.put("availableActions", actions); } } diff --git a/engine/src/main/resources/org/archive/crawler/restlet/Engine.ftl b/engine/src/main/resources/org/archive/crawler/restlet/Engine.ftl index 812ee0ad..ff9277ba 100644 --- a/engine/src/main/resources/org/archive/crawler/restlet/Engine.ftl +++ b/engine/src/main/resources/org/archive/crawler/restlet/Engine.ftl @@ -19,19 +19,19 @@ Memory: ${engine.heapReport}

-Jobs Directory: ${engine.jobsDirAbsolutePath} +Jobs Directory: ${engine.jobsDir} -

Job Directories (${engine.jobList?size}) +

Job Directories (${engine.jobs?size})

    -<#list engine.jobList as crawlJob> +<#list engine.jobs as crawlJob>
  • ${crawlJob.shortName} <#if crawlJob.hasApplicationContext> - «${crawlJob.status}» + «${crawlJob.statusDescription}» <#if crawlJob.isLaunchInfoPartial> at least @@ -39,7 +39,7 @@ ${engine.heapReport} ${crawlJob.launchCount} launches
    -${crawlJob.config} +${crawlJob.primaryConfig}
    <#if crawlJob.lastLaunch??>
    (last at ${crawlJob.lastLaunch})
    @@ -52,7 +52,7 @@ ${crawlJob.config}
    Create new job directory with recommended starting configuration
    -Path: ${engine.jobsDirAbsolutePath}${filePathSeparator} +Path: ${engine.jobsDir}${filePathSeparator}
    @@ -72,9 +72,9 @@ will then require access to the hosting machine. You should cleanly terminate and teardown any jobs in progress first.
    -<#list engine.jobList as crawlJob> +<#list engine.jobs as crawlJob> <#if crawlJob.hasApplicationContext> -
    Job ${crawlJob.key} still « ${crawlJob.status} »
    +
    Job ${crawlJob.key} still « ${crawlJob.statusDescription} »

    diff --git a/engine/src/main/resources/org/archive/crawler/restlet/Job.ftl b/engine/src/main/resources/org/archive/crawler/restlet/Job.ftl index 03396496..ce68f935 100644 --- a/engine/src/main/resources/org/archive/crawler/restlet/Job.ftl +++ b/engine/src/main/resources/org/archive/crawler/restlet/Job.ftl @@ -1,7 +1,7 @@ - ${job.shortName} - ${job.status} - Job main page + ${job.shortName} - ${job.statusDescription} - Job main page @@ -34,14 +34,14 @@ disabled='disabled' title='profiles cannot be launched' - <#if !job.isLaunchable> + <#if !job.availableActions?seq_contains("launch")> disabled='disabled' /> - disabled type='submit' name='action' value='pause' /> - disabled type='submit' name='action' value='unpause' /> + disabled type='submit' name='action' value='pause' /> + disabled type='submit' name='action' value='unpause' /> disabled type='submit' name='action' value='checkpoint' /> @@ -75,7 +75,7 @@
    ${line?html}
    -

    Job is ${job.status}

    +

    Job is ${job.statusDescription}

    <#if job.hasApplicationContext>
    Totals
    @@ -123,7 +123,7 @@ ${line?html}

    Files

    Browser Job Directory

    Configuration-referenced Paths

    - <#assign refPaths=job.configReferencedPaths! /> + <#assign refPaths=job.referen! /> <#if !refPaths?has_content > build the job to discover referenced paths <#else>