diff --git a/.gitignore b/.gitignore new file mode 100644 index 00000000..c02256c8 --- /dev/null +++ b/.gitignore @@ -0,0 +1,5 @@ +bin +commons/target +modules/target +engine/target +dist/target diff --git a/commons/src/main/java/org/archive/checkpointing/Checkpoint.java b/commons/src/main/java/org/archive/checkpointing/Checkpoint.java index 2e5f2a72..47cda1ca 100644 --- a/commons/src/main/java/org/archive/checkpointing/Checkpoint.java +++ b/commons/src/main/java/org/archive/checkpointing/Checkpoint.java @@ -125,6 +125,9 @@ public class Checkpoint implements InitializingBean { public void saveJson(String beanName, JSONObject json) { try { File targetFile = new File(getCheckpointDir().getFile(),beanName); + if (LOGGER.isLoggable(Level.FINE)) { + LOGGER.fine("saving json to " + targetFile); + } FileUtils.writeStringToFile( targetFile, json.toString()); @@ -137,6 +140,9 @@ public class Checkpoint implements InitializingBean { public JSONObject loadJson(String beanName) { File sourceFile = new File(getCheckpointDir().getFile(),beanName); try { + if (LOGGER.isLoggable(Level.FINE)) { + LOGGER.fine("reading json from " + sourceFile); + } return new JSONObject(FileUtils.readFileToString(sourceFile)); } catch (JSONException e) { throw new RuntimeException(e); @@ -148,6 +154,9 @@ public class Checkpoint implements InitializingBean { public BufferedWriter saveWriter(String beanName, String extraName) throws IOException { try { File targetFile = new File(getCheckpointDir().getFile(),beanName+"-"+extraName); + if (LOGGER.isLoggable(Level.FINE)) { + LOGGER.fine("opening for writing: " + targetFile); + } return new BufferedWriter(new FileWriter(targetFile)); } catch (IOException e) { LOGGER.log(Level.SEVERE,"unable to save checkpoint writer state "+extraName+" of "+beanName,e); diff --git a/commons/src/main/java/org/archive/spring/PathSharingContext.java b/commons/src/main/java/org/archive/spring/PathSharingContext.java index f8348c70..e863773a 100644 --- a/commons/src/main/java/org/archive/spring/PathSharingContext.java +++ b/commons/src/main/java/org/archive/spring/PathSharingContext.java @@ -21,8 +21,11 @@ package org.archive.spring; import java.io.File; import java.io.IOException; +import java.net.URI; +import java.net.URISyntaxException; import java.util.HashMap; import java.util.Map.Entry; +import java.util.concurrent.ConcurrentHashMap; import java.util.logging.Level; import java.util.logging.Logger; @@ -132,12 +135,16 @@ public class PathSharingContext extends FileSystemXmlApplicationContext { protected File getConfigurationFile() { String primaryConfigurationPath = getPrimaryConfigurationPath(); - if(primaryConfigurationPath.startsWith("file:")) { + if (primaryConfigurationPath.startsWith("file:")) { // strip URI-scheme if present (as is usual) - primaryConfigurationPath = primaryConfigurationPath.substring(5); + try { + return new File(new URI(primaryConfigurationPath)); + } catch (URISyntaxException e) { + throw new RuntimeException(e); + } + } else { + return new File(primaryConfigurationPath); } - File configFile = new File(primaryConfigurationPath); - return configFile; } protected void initLaunchDir() { @@ -178,4 +185,18 @@ public class PathSharingContext extends FileSystemXmlApplicationContext { } super.initLifecycleProcessor(); } + + protected ConcurrentHashMap data; + + /** + * @return a shared map for arbitrary use during a crawl; for example, could + * be used for state persisting for the duration of the crawl, + * shared among ScriptedProcessor, scripting console, etc scripts + */ + public ConcurrentHashMap getData() { + if (data == null) { + data = new ConcurrentHashMap(); + } + return data; + } } diff --git a/commons/src/main/java/org/archive/util/ArchiveUtils.java b/commons/src/main/java/org/archive/util/ArchiveUtils.java index 7988dc5d..db3db1f0 100644 --- a/commons/src/main/java/org/archive/util/ArchiveUtils.java +++ b/commons/src/main/java/org/archive/util/ArchiveUtils.java @@ -557,6 +557,11 @@ public class ArchiveUtils { } private static String doubleToString(double val, int maxFractionDigits, int minFractionDigits) { + // NumberFormat returns U+FFFD REPLACEMENT CHARACTER for NaN which looks + // like a bug in the UI + if (Double.isNaN(val)) { + return "NaN"; + } NumberFormat f = NumberFormat.getNumberInstance(Locale.US); f.setMaximumFractionDigits(maxFractionDigits); f.setMinimumFractionDigits(minFractionDigits); diff --git a/engine/src/main/java/org/archive/crawler/framework/ActionDirectory.java b/engine/src/main/java/org/archive/crawler/framework/ActionDirectory.java index 7e492a62..56fe41dc 100644 --- a/engine/src/main/java/org/archive/crawler/framework/ActionDirectory.java +++ b/engine/src/main/java/org/archive/crawler/framework/ActionDirectory.java @@ -312,7 +312,7 @@ public class ActionDirectory implements ApplicationContextAware, Lifecycle, Runn PrintWriter rawOut = new PrintWriter(rawString); Exception ex = null; engine.put("rawOut", rawOut); - engine.put("appCtx",appCtx); + engine.put("appCtx", appCtx); // evaluate and record any exception try { diff --git a/engine/src/main/java/org/archive/crawler/framework/CheckpointService.java b/engine/src/main/java/org/archive/crawler/framework/CheckpointService.java index ef62f364..3b5d8b2e 100644 --- a/engine/src/main/java/org/archive/crawler/framework/CheckpointService.java +++ b/engine/src/main/java/org/archive/crawler/framework/CheckpointService.java @@ -40,6 +40,7 @@ import org.archive.spring.ConfigPath; import org.archive.spring.ConfigPathConfigurer; import org.archive.spring.HasValidator; import org.springframework.beans.BeansException; +import org.springframework.beans.factory.BeanFactoryUtils; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.context.ApplicationContext; import org.springframework.context.ApplicationContextAware; @@ -222,6 +223,9 @@ public class CheckpointService implements Lifecycle, ApplicationContextAware, Ha } Map toCheckpoint = appCtx.getBeansOfType(Checkpointable.class); + if (LOGGER.isLoggable(Level.FINE)) { + LOGGER.fine("checkpointing beans " + toCheckpoint); + } checkpointInProgress = new Checkpoint(); try { diff --git a/engine/src/main/java/org/archive/crawler/framework/CrawlJob.java b/engine/src/main/java/org/archive/crawler/framework/CrawlJob.java index ee9cbccd..124b6f7a 100644 --- a/engine/src/main/java/org/archive/crawler/framework/CrawlJob.java +++ b/engine/src/main/java/org/archive/crawler/framework/CrawlJob.java @@ -208,10 +208,7 @@ public class CrawlJob implements Comparable, ApplicationListener"); -// if(isRunning()) { -// pw.println("ACTIVE; "+getCrawlController().getState()+":"); -// } + pw.println("
"); pw.println(""+getShortName()+""); if(isProfile()) { pw.println("(profile)"); @@ -220,18 +217,18 @@ public class CrawlJob implements Comparable, ApplicationListener"); + pw.println(getLaunchCount() + " launches"); + pw.println("
"); + pw.println("
"); pw.println(getPrimaryConfig()); - pw.println("
"); + pw.println("
"); if(lastLaunch!=null) { - pw.println("(last at "+lastLaunch+")"); + pw.println("
(last at "+lastLaunch+")
"); } - pw.println(""); } /** @@ -290,7 +287,7 @@ public class CrawlJob implements Comparable, ApplicationListenerCrawl beans in "+cj.getShortName()+""); + pw.println(""); + pw.println(""); + pw.println(""); + pw.println("Crawl beans in "+cj.getShortName()+""); + pw.println(""); + pw.println(""); + pw.println(""); pw.println("

Crawl beans in built job "+cj.getShortName()+"

"); pw.println("Enter a bean path of the form beanName, beanName.property, beanName.property[indexOrKey], etc."); - pw.println("
"); - pw.println("
"); + pw.println("
"); + pw.println("
"); if (StringUtils.isNotBlank(beanPath)) { pw.println("

Bean path "+beanPath+"

"); @@ -222,14 +232,16 @@ public class BeanBrowseResource extends JobRelatedResource { if(bwrap.isWritableProperty(propPath) && (bwrap.getDefaultEditor(type)!=null|| type == String.class) && !Collection.class.isAssignableFrom(type)) { + pw.println("
"); pw.println(beanPath+" = "); writeObject(pw, null, target); - pw.println("edit"); - pw.println("Note: it may not be appropriate/effective to change this value in an already-built crawl context.
"); - pw.println("
"); - pw.println(""); - pw.println(beanPath + " = "); - pw.println("
"); + pw.println(" edit"); + pw.println("
"); + pw.println("
"); + pw.println("
Note: it may not be appropriate/effective to change this value in an already-built crawl context.
"); + pw.println("
"); + pw.println(beanPath + " = "); + pw.println("
"); } else { writeObject(pw, null, target, beanPath); } @@ -239,7 +251,7 @@ public class BeanBrowseResource extends JobRelatedResource { } } - pw.println("

All named crawl beansAll named crawl beans

"); pw.println("
    "); Set alreadyWritten = new HashSet(); writeNestedNames(pw, appCtx.getBean("crawlController"), getBeansRefPath(), alreadyWritten); @@ -247,5 +259,7 @@ public class BeanBrowseResource extends JobRelatedResource { writeNestedNames(pw, appCtx.getBean(name), getBeansRefPath(), alreadyWritten); } pw.println(""); + + pw.println("\n"); } } diff --git a/engine/src/main/java/org/archive/crawler/restlet/EditRepresentation.java b/engine/src/main/java/org/archive/crawler/restlet/EditRepresentation.java index c1becd33..5ccfa4e9 100644 --- a/engine/src/main/java/org/archive/crawler/restlet/EditRepresentation.java +++ b/engine/src/main/java/org/archive/crawler/restlet/EditRepresentation.java @@ -61,20 +61,26 @@ public class EditRepresentation extends CharacterRepresentation { @Override public void write(Writer writer) throws IOException { PrintWriter pw = new PrintWriter(writer); + pw.println(""); + pw.println(""); pw.println(""+fileRepresentation.getFile().getName()+""); - - Flash.renderFlashesHTML(pw, dirResource.getRequest()); - pw.println("
    "); + pw.println(""); + pw.println(""); + pw.println(""); + pw.println("
    "); // TODO: enable button on after changes made - pw.println(""); + pw.println(""); pw.println(fileRepresentation.getFile()); Reference viewRef = dirResource.getRequest().getOriginalRef().clone(); viewRef.setQuery(null); pw.println("view"); - pw.println("
    "); - pw.println(""); + Flash.renderFlashesHTML(pw, dirResource.getRequest()); + pw.println("
    "); + pw.println(""); + pw.println(""); + pw.println(""); pw.close(); } diff --git a/engine/src/main/java/org/archive/crawler/restlet/EngineResource.java b/engine/src/main/java/org/archive/crawler/restlet/EngineResource.java index 1a245207..e7b12e1f 100644 --- a/engine/src/main/java/org/archive/crawler/restlet/EngineResource.java +++ b/engine/src/main/java/org/archive/crawler/restlet/EngineResource.java @@ -145,7 +145,7 @@ public class EngineResource extends BaseResource { } } else if ("Exit Java Process".equals(action)) { boolean cancel = false; - if(!"on".equals(form.getFirstValue("I'm sure"))) { + if(!"on".equals(form.getFirstValue("im_sure"))) { Flash.addFlash( getResponse(), "You must tick \"I'm sure\" to trigger exit", @@ -251,30 +251,31 @@ public class EngineResource extends BaseResource { baseRef += "/"; } PrintWriter pw = new PrintWriter(writer); + pw.println(""); + pw.println(""); pw.println(""+engineTitle+""); pw.println(""); - pw.println(""); + pw.println(""); + pw.println("\n"); pw.println("

    "+engineTitle+"

    "); Flash.renderFlashesHTML(pw, getRequest()); - pw.println("Memory: "); - pw.println(engine.heapReport()); - pw.println("

    "); - pw.println("Jobs Directory: "+jobsDir.getAbsolutePath()+""); + pw.println("

    Memory: " + engine.heapReport() + "

    "); + pw.println("

    Jobs Directory: " + + jobsDir.getAbsolutePath() + "

    "); ArrayList jobs = new ArrayList(); jobs.addAll(engine.getJobConfigs().values()); - pw.println("

    Job Directories ("+jobs.size()+")"); + pw.println("

    Job Directories ("+jobs.size()+")"); pw.println(""); pw.println("

    "); Collections.sort(jobs); pw.println("
      "); for(CrawlJob cj: jobs) { - pw.println("
    • "); + pw.print("
    • "); cj.writeHtmlTo(pw,"job/"); pw.println("
      • "); } @@ -283,46 +284,49 @@ public class EngineResource extends BaseResource { pw.println("

      Add Job Directory

      "); // create new job with defaults - pw.println("
      \n" - + "Create new job directory with recommended starting configuration
      \n" - + "Path: " + jobsDir.getAbsolutePath() + File.separator + "\n" - + "\n" + pw.println("\n" + + "
      Create new job directory with recommended starting configuration
      \n" + + "
      Path: " + jobsDir.getAbsolutePath() + + File.separator + "\n" + "\n" - + "\n"); + + "
      \n"); - pw.println("
      "); - pw.println("Specify a path to a pre-existing job directory
      "); - pw.println("Path: " + ""); + pw.println(""); + pw.println("
      Specify a path to a pre-existing job directory
      "); + pw.println("
      Path: "); pw.println(""); - pw.println(""); + pw.println("
      "); pw.println( - "You may also compose or copy a valid job directory into " + + "

      You may also compose or copy a valid job directory into " + "the main jobs directory via outside means, then use the " + "'rescan' button above to make it appear in this interface. Or, " + "use the 'copy' functionality at the botton of any existing " + - "job's detail page."); + "job's detail page.

      "); pw.println("

      Exit Java

      "); pw.println( - "This exits the Java process running Heritrix. To restart " + + "
      This exits the Java process running Heritrix. To restart " + "will then require access to the hosting machine. You should " + - "cleanly terminate and teardown any jobs in progress first.
      "); - pw.println("
      "); + "cleanly terminate and teardown any jobs in progress first.
      "); + pw.println(""); for(Map.Entry entry : getBuiltJobs().entrySet()) { - pw.println("
      Job '"+entry.getKey()+"' still «" + pw.println("
      Job '"+entry.getKey()+"' still «" +entry.getValue().getJobStatusDescription() - +"»
      "); + +"»
      "); String checkName = "ignore__"+entry.getKey(); pw.println("
      "); + +entry.getKey()+"' and exit anyway
      "); } - pw.println("
      "); - pw.println(""); + pw.println("
      "); + pw.println(""); + pw.println(""); + pw.println("
      "); pw.println(""); pw.println(""); + pw.println(""); pw.flush(); } diff --git a/engine/src/main/java/org/archive/crawler/restlet/JobRelatedResource.java b/engine/src/main/java/org/archive/crawler/restlet/JobRelatedResource.java index 7e2d7e9f..fc65905e 100644 --- a/engine/src/main/java/org/archive/crawler/restlet/JobRelatedResource.java +++ b/engine/src/main/java/org/archive/crawler/restlet/JobRelatedResource.java @@ -23,6 +23,7 @@ import java.beans.IntrospectionException; import java.beans.PropertyDescriptor; import java.io.File; import java.io.PrintWriter; +import java.net.URLEncoder; import java.util.Arrays; import java.util.Collection; import java.util.HashSet; @@ -36,6 +37,7 @@ import java.util.Set; import org.apache.commons.lang.StringUtils; import org.archive.crawler.framework.CrawlJob; import org.archive.crawler.framework.Engine; +import org.archive.util.TextUtils; import org.restlet.Context; import org.restlet.data.Reference; import org.restlet.data.Request; @@ -156,7 +158,7 @@ public abstract class JobRelatedResource extends BaseResource { String name = getBeanToNameMap().get(obj); pw.println("
    • "+name+""); pw.println(""+obj.getClass().getName()+"
        "); - close = "
      • "; + close = "
    \n"; } if(!alreadyWritten.contains(obj)) { alreadyWritten.add(obj); @@ -183,7 +185,7 @@ public abstract class JobRelatedResource extends BaseResource { pw.println(""+ipe.getMessage()+""); } } - pw.println(close); + pw.print(close); } @@ -415,7 +417,7 @@ public abstract class JobRelatedResource extends BaseResource { String close = ""; String beanPath = beanPathPrefix; if(StringUtils.isNotBlank(field)) { - pw.write(""); + pw.write("\n"); String closeAnchor =""; if(StringUtils.isNotBlank(beanPathPrefix)) { if(beanPathPrefix.endsWith(".")) { @@ -423,21 +425,22 @@ public abstract class JobRelatedResource extends BaseResource { } else if (beanPathPrefix.endsWith("[")) { beanPath += field + "]"; } - pw.println(""); + // url-encode because brackets are not allowed here according to validator.w3.org + pw.print(""); closeAnchor = ""; } pw.write(field); pw.write(closeAnchor); - pw.write(":"); - close=""; + pw.write(":\n"); + close="\n\n"; } if(object == null) { - pw.write("null
    "); + pw.write("null"); pw.write(close); return; } if(object instanceof String) { - pw.write("\""+object+"\"
    "); + pw.write("\""+object+"\""); pw.write(close); return; } @@ -445,18 +448,18 @@ public abstract class JobRelatedResource extends BaseResource { || object instanceof File //|| BeanUtils.findEditorByConvention(object.getClass())!=null ) { - pw.write(object+"
    "); + pw.write(object.toString()); pw.write(close); return; } if(alreadyWritten.contains(object)) { - pw.println("↑
    "); + pw.println("↑"); pw.write(close); return; } alreadyWritten.add(object); // guard against repeats and cycles if(StringUtils.isNotBlank(key) && StringUtils.isNotBlank(field)) { - pw.println(""+key+"
    "); + pw.println(""+key+""); pw.write(close); return; } @@ -509,7 +512,7 @@ public abstract class JobRelatedResource extends BaseResource { } } pw.println(""); - pw.print("
    "); + pw.print(""); pw.write(close); } diff --git a/engine/src/main/java/org/archive/crawler/restlet/JobResource.java b/engine/src/main/java/org/archive/crawler/restlet/JobResource.java index e8329c91..2295a00e 100644 --- a/engine/src/main/java/org/archive/crawler/restlet/JobResource.java +++ b/engine/src/main/java/org/archive/crawler/restlet/JobResource.java @@ -59,31 +59,33 @@ import org.restlet.resource.Variant; import org.restlet.resource.WriterRepresentation; /** - * Restlet Resource representing a single local CrawlJob inside an - * Engine. + * Restlet Resource representing a single local CrawlJob inside an Engine. * * @contributor gojomo * @contributor nlevitt */ public class JobResource extends BaseResource { - public static final IOFileFilter EDIT_FILTER = - FileUtils.getRegexFileFilter(".*\\.((c?xml)|(txt))$"); + public static final IOFileFilter EDIT_FILTER = FileUtils + .getRegexFileFilter(".*\\.((c?xml)|(txt))$"); @SuppressWarnings("unused") - private static final Logger logger = Logger.getLogger(JobResource.class.getName()); + private static final Logger logger = Logger.getLogger(JobResource.class + .getName()); - protected CrawlJob cj; - - public JobResource(Context ctx, Request req, Response res) throws ResourceException { + protected CrawlJob cj; + + public JobResource(Context ctx, Request req, Response res) + throws ResourceException { super(ctx, req, res); setModifiable(true); getVariants().add(new Variant(MediaType.TEXT_HTML)); getVariants().add(new Variant(MediaType.APPLICATION_XML)); - cj = getEngine().getJob(TextUtils.urlUnescape((String)req.getAttributes().get("job"))); + cj = getEngine().getJob( + TextUtils.urlUnescape((String) req.getAttributes().get("job"))); } public Representation represent(Variant variant) throws ResourceException { - if(cj==null) { + if (cj == null) { throw new ResourceException(404); } @@ -91,17 +93,18 @@ public class JobResource extends BaseResource { if (variant.getMediaType() == MediaType.APPLICATION_XML) { representation = new WriterRepresentation(MediaType.APPLICATION_XML) { public void write(Writer writer) throws IOException { - XmlMarshaller.marshalDocument(writer, "job", makePresentableMap()); - } + XmlMarshaller.marshalDocument(writer, "job", + makePresentableMap()); + } }; } else { representation = new WriterRepresentation(MediaType.TEXT_HTML) { - public void write(Writer writer) throws IOException { - JobResource.this.writeHtml(writer); - } - }; + public void write(Writer writer) throws IOException { + JobResource.this.writeHtml(writer); + } + }; } - + // TODO: remove if not necessary in future? // honor requested charset? representation.setCharacterSet(CharacterSet.UTF_8); @@ -115,20 +118,21 @@ public class JobResource extends BaseResource { * * @return the nested Map data structure */ - protected LinkedHashMap makePresentableMap() { - LinkedHashMap info = new LinkedHashMap(); + protected LinkedHashMap makePresentableMap() { + LinkedHashMap info = new LinkedHashMap(); String baseRef = getRequest().getResourceRef().getBaseRef().toString(); if (!baseRef.endsWith("/")) { baseRef += "/"; } Reference baseRefRef = new Reference(baseRef); - + info.put("shortName", cj.getShortName()); if (cj.getCrawlController() != null) { info.put("crawlControllerState", cj.getCrawlController().getState()); if (cj.getCrawlController().getState() == State.FINISHED) { - info.put("crawlExitStatus", cj.getCrawlController().getCrawlExitStatus()); + info.put("crawlExitStatus", cj.getCrawlController() + .getCrawlExitStatus()); } } info.put("statusDescription", cj.getJobStatusDescription()); @@ -139,16 +143,18 @@ public class JobResource extends BaseResource { info.put("isProfile", cj.isProfile()); File primaryConfig = FileUtils.tryToCanonicalize(cj.getPrimaryConfig()); info.put("primaryConfig", primaryConfig.getAbsolutePath()); - info.put("primaryConfigUrl", baseRef + "jobdir/" + primaryConfig.getName()); + info.put("primaryConfigUrl", + baseRef + "jobdir/" + primaryConfig.getName()); + + if (cj.getJobLog().exists()) + try { + List logLines = new LinkedList(); + FileUtils.pagedLines(cj.getJobLog(), -1, -5, logLines); + info.put("jobLogTail", logLines); + } catch (IOException ioe) { + throw new RuntimeException(ioe); + } - if (cj.getJobLog().exists()) try { - List logLines = new LinkedList(); - FileUtils.pagedLines(cj.getJobLog(), -1, -5, logLines); - info.put("jobLogTail", logLines); - } catch (IOException ioe) { - throw new RuntimeException(ioe); - } - if (cj.hasApplicationContext()) { info.put("uriTotalsReport", cj.uriTotalsReportData()); info.put("sizeTotalsReport", cj.sizeTotalsReportData()); @@ -158,28 +164,32 @@ public class JobResource extends BaseResource { info.put("threadReport", cj.threadReportData()); info.put("frontierReport", cj.frontierReportData()); info.put("heapReport", getEngine().heapReportData()); - - if ((cj.isRunning() || (cj.hasApplicationContext() && !cj.isLaunchable())) - && cj.getCrawlController().getLoggerModule().getCrawlLogPath().getFile().exists()) { + + if ((cj.isRunning() || (cj.hasApplicationContext() && !cj + .isLaunchable())) + && cj.getCrawlController().getLoggerModule() + .getCrawlLogPath().getFile().exists()) { try { List logLines = new LinkedList(); - FileUtils.pagedLines( - cj.getCrawlController().getLoggerModule().getCrawlLogPath().getFile(), - -1, -10, logLines); + FileUtils.pagedLines(cj.getCrawlController() + .getLoggerModule().getCrawlLogPath().getFile(), -1, + -10, logLines); info.put("crawlLogTail", logLines); } catch (IOException ioe) { - throw new RuntimeException(ioe); + throw new RuntimeException(ioe); } } } - List> configFiles = new LinkedList>(); - for(String cppp: cj.getConfigPaths().keySet()) { - Map configFileInfo = new LinkedHashMap(); + List> configFiles = new LinkedList>(); + for (String cppp : cj.getConfigPaths().keySet()) { + Map configFileInfo = new LinkedHashMap(); configFileInfo.put("key", cppp); - File path = FileUtils.tryToCanonicalize(cj.getConfigPaths().get(cppp).getFile()); + File path = FileUtils.tryToCanonicalize(cj.getConfigPaths() + .get(cppp).getFile()); configFileInfo.put("path", path.getAbsolutePath()); - Reference urlRef = new Reference(baseRefRef, getHrefPath(path, cj)).getTargetRef(); + Reference urlRef = new Reference(baseRefRef, getHrefPath(path, cj)) + .getTargetRef(); configFileInfo.put("url", urlRef.toString()); configFiles.add(configFileInfo); } @@ -190,7 +200,7 @@ public class JobResource extends BaseResource { protected Set getAvailableActions() { Set actions = new LinkedHashSet(); - + if (!cj.hasApplicationContext()) { actions.add("build"); } @@ -212,249 +222,256 @@ public class JobResource extends BaseResource { if (cj.hasApplicationContext()) { actions.add("teardown"); } - + return actions; } - + protected void writeHtml(Writer writer) { - PrintWriter pw = new PrintWriter(writer); - String jobTitle = cj.getShortName() + " - " - + cj.getJobStatusDescription() - + " - Job main page"; + PrintWriter pw = new PrintWriter(writer); + String jobTitle = cj.getShortName() + " - " + + cj.getJobStatusDescription() + " - Job main page"; String baseRef = getRequest().getResourceRef().getBaseRef().toString(); - if(!baseRef.endsWith("/")) { + if (!baseRef.endsWith("/")) { baseRef += "/"; } // TODO: replace with use a templating system (FreeMarker?) - pw.println(""+jobTitle+""); - pw.println(""); - pw.println(""); + pw.println(""); + pw.println(""); + pw.println("" + jobTitle + ""); + pw.println(""); + pw.println(""); pw.println(""); - pw.print("

    Job "+cj.getShortName()+" ("); - + pw.print("

    Job " + cj.getShortName() + " ("); + if (cj.isLaunchInfoPartial()) { - pw.print("at least "); + pw.print("at least "); } pw.print(cj.getLaunchCount() + " launches"); - if(cj.getLastLaunch()!=null) { - long ago = System.currentTimeMillis() - cj.getLastLaunch().getMillis(); - pw.print(", last "+ArchiveUtils.formatMillisecondsToConventional(ago, 2)+" ago"); + if (cj.getLastLaunch() != null) { + long ago = System.currentTimeMillis() + - cj.getLastLaunch().getMillis(); + pw.print(", last " + + ArchiveUtils.formatMillisecondsToConventional(ago, 2) + + " ago"); } pw.println(")

    "); - - Flash.renderFlashesHTML(pw, getRequest()); - - if(cj.isProfile()) { - pw.print( - "

    As a profile, this job may be built for " + - "testing purposes but not launched. Use the 'copy job to' " + - "functionality at bottom to copy this profile to a " + - "launchable job.

    "); - } - - // button controls - pw.println("
    "); - // PREP, LAUNCH - pw.print(""); - pw.print("   "); - - // PAUSE, UNPAUSE, CHECKPOINT - pw.println(""); - pw.println(""); - pw.println(""); - - // TERMINATE, RESET - pw.println("   "); - pw.println("
    "); - + Flash.renderFlashesHTML(pw, getRequest()); + + if (cj.isProfile()) { + pw.print("

    As a profile, this job may be built for " + + "testing purposes but not launched. Use the 'copy job to' " + + "functionality at bottom to copy this profile to a " + + "launchable job.

    "); + } + + // button controls + pw.println(""); + // PREP, LAUNCH + pw.println("
    "); + pw.println(""); + pw.print(""); + pw.print(""); + + // PAUSE, UNPAUSE, CHECKPOINT + pw.print(""); + pw.print(""); + pw.print(""); + pw.print(""); + pw.println(""); + + // TERMINATE, RESET + pw.print("\n"); + pw.print(""); + pw.println(""); + pw.println("
    "); + // display checkpoint options - if(cj.getCheckpointService()!=null) { - Checkpoint recoveryCheckpoint = cj.getCheckpointService().getRecoveryCheckpoint(); - if(recoveryCheckpoint!=null) { - pw.println("recover from "+recoveryCheckpoint.getName()+""); - } else if (cj.getCheckpointService().hasAvailableCheckpoints() && cj.isLaunchable()) { - pw.println("select an available checkpoint before launch to recover:"); + if (cj.getCheckpointService() != null) { + Checkpoint recoveryCheckpoint = cj.getCheckpointService() + .getRecoveryCheckpoint(); + if (recoveryCheckpoint != null) { + pw.println("
    recover from " + + recoveryCheckpoint.getName() + "
    "); + } else if (cj.getCheckpointService().hasAvailableCheckpoints() + && cj.isLaunchable()) { + pw.println("
    select an available checkpoint before launch to recover:"); pw.println(""); + pw.println("
    "); } } - - pw.println("
    "); - - // configuration - pw.println("configuration: "); + pw.println(""); + + // configuration + pw.print("
    configuration: "); printLinkedFile(pw, cj.getPrimaryConfig()); - for(File f : cj.getImportedConfigs(cj.getPrimaryConfig())) { - pw.println("imported: "); - printLinkedFile(pw,f); + pw.println("
    "); + for (File f : cj.getImportedConfigs(cj.getPrimaryConfig())) { + pw.print("
    imported: "); + printLinkedFile(pw, f); + pw.println("
    "); } - -// if(cj.isXmlOk()) { -// pw.println("cxml ok
    "); -// if(cj.isContainerOk()) { -// pw.println("container ok
    "); -// if(cj.isContainerValidated()) { -// pw.println("config valid
    "); -// } else { -// pw.println("CONFIG INVALID
    "); -// } -// } else { -// pw.println("CONTAINER BAD
    "); -// } -// }else { -// // pw.println("XML NOT WELL-FORMED
    "); -// } - pw.println("

    Job Log "); - pw.println("(more)"); + pw.print("

    Job Log "); + pw.print("(more)"); pw.println("

    "); - pw.println("
    "); - if(cj.getJobLog().exists()) { + pw.println("
    "); + if (cj.getJobLog().exists()) { try { List logLines = new LinkedList(); FileUtils.pagedLines(cj.getJobLog(), -1, -5, logLines); Collections.reverse(logLines); - for(String line : logLines) { - pw.print("

    "); - StringEscapeUtils.escapeHtml(pw,line); - pw.print("

    "); + for (String line : logLines) { + pw.print("
    "); + StringEscapeUtils.escapeHtml(pw, line); + pw.println("
    "); } } catch (IOException ioe) { - throw new RuntimeException(ioe); + pw.println("
    error reading job log
    "); } } pw.println("
    "); - - pw.println("

    Job is "+cj.getJobStatusDescription()+"

    "); - if(cj.hasApplicationContext()) { - pw.println("Totals
      "); - pw.println(cj.uriTotalsReport()); - pw.println("
      "); - pw.println(cj.sizeTotalsReport()); - - pw.println("
    Alerts
      "); - pw.println(cj.getAlertCount()==0 ? "none" : cj.getAlertCount()); - if(cj.getAlertCount()>0) { - printLinkedFile( - pw, - cj.getCrawlController().getLoggerModule().getAlertsLogPath().getFile(), - "tail alert log...", - "format=paged&pos=-1&lines=-128"); + pw.println("

    Job is " + cj.getJobStatusDescription() + "

    "); + + if (cj.hasApplicationContext()) { + pw.println("
    "); + + pw.println("
    Totals
    "); + pw.println("
    " + cj.uriTotalsReport() + "
    "); + pw.println("
    " + cj.sizeTotalsReport() + "
    "); + + pw.println("
    Alerts
    "); + pw.print("
    "); + pw.println(cj.getAlertCount() == 0 ? "none" : cj + .getAlertCount()); + if (cj.getAlertCount() > 0) { + printLinkedFile(pw, cj.getCrawlController().getLoggerModule() + .getAlertsLogPath().getFile(), "tail alert log...", + "format=paged&pos=-1&lines=-128"); } - - pw.println("
    Rates
      "); - pw.println(cj.rateReport()); - - pw.println("
    Load
      "); - pw.println(cj.loadReport()); - - pw.println("
    Elapsed
      "); - pw.println(cj.elapsedReport()); - - pw.println("
    Threads
      "); - pw.println(cj.threadReport()); - - pw.println("
    Frontier
      "); - pw.println(cj.frontierReport()); - - pw.println("
    Memory
      "); - pw.println(getEngine().heapReport()); - - if ((cj.isRunning() || (cj.hasApplicationContext() && !cj.isLaunchable())) - && cj.getCrawlController().getLoggerModule().getCrawlLogPath().getFile().exists()) { + pw.println("
    "); + + pw.println("
    Rates
    "); + pw.println("
    " + cj.rateReport() + "
    "); + + pw.println("
    Load
    "); + pw.println("
    " + cj.loadReport() + "
    "); + + pw.println("
    Elapsed
    "); + pw.println("
    " + cj.elapsedReport() + "
    "); + + pw.println("
    Threads
    "); + pw.println("
    " + cj.threadReport() + " 
    "); + + pw.println("
    Frontier
    "); + pw.println("
    " + cj.frontierReport() + " 
    "); + + pw.println("
    Memory
    "); + pw.println("
    " + getEngine().heapReport() + "
    "); + + pw.println("
    "); + + if ((cj.isRunning() || (cj.hasApplicationContext() && !cj + .isLaunchable())) + && cj.getCrawlController().getLoggerModule() + .getCrawlLogPath().getFile().exists()) { // show crawl log for running or finished crawls - pw.println("

    Crawl Log"); - printLinkedFile( - pw, - cj.getCrawlController().getLoggerModule().getCrawlLogPath().getFile(), - "more", - "format=paged&pos=-1&lines=-128&reverse=y"); + pw.print("

    Crawl Log "); + printLinkedFile(pw, cj.getCrawlController().getLoggerModule() + .getCrawlLogPath().getFile(), "more", + "format=paged&pos=-1&lines=-128&reverse=y"); pw.println("

    "); pw.println("
    ");
                 try {
                     List logLines = new LinkedList();
-                    FileUtils.pagedLines(
-                            cj.getCrawlController().getLoggerModule().getCrawlLogPath().getFile(),
-                            -1, 
-                            -10, 
-                            logLines);
+                    FileUtils.pagedLines(cj.getCrawlController()
+                            .getLoggerModule().getCrawlLogPath().getFile(), -1,
+                            -10, logLines);
                     Collections.reverse(logLines);
-                    for(String line : logLines) {
-                        StringEscapeUtils.escapeHtml(pw,line);
+                    for (String line : logLines) {
+                        StringEscapeUtils.escapeHtml(pw, line);
                         pw.println();
                     }
                 } catch (IOException ioe) {
-                    throw new RuntimeException(ioe); 
+                    pw.println("--error reading crawl log--");
                 }
                 pw.println("
    "); } - + } - - if(cj.hasApplicationContext()) { + + if (cj.hasApplicationContext()) { pw.println("

    Reports

    "); - for (Report report: cj.getCrawlController().getStatisticsTracker().getReports()) { + for (Report report : cj.getCrawlController().getStatisticsTracker() + .getReports()) { if (report.getShouldReportDuringCrawl()) { String className = report.getClass().getSimpleName(); - String shortName = className.substring(0,className.length()-"Report".length()); - pw.println(""+shortName+""); + String shortName = className.substring(0, + className.length() - "Report".length()); + pw.println("" + + shortName + ""); } } } - + pw.println("

    Files

    "); pw.println("

    Browse Job Directory

    "); // specific paths from wired context pw.println("

    Configuration-referenced Paths

    "); - if(cj.getConfigPaths().isEmpty()) { + if (cj.getConfigPaths().isEmpty()) { pw.println("build the job to discover referenced paths"); } else { pw.println("
    "); - for(String cppp : cj.getConfigPaths().keySet()) { + for (String cppp : cj.getConfigPaths().keySet()) { ConfigPath cp = cj.getConfigPaths().get(cppp); - pw.println("
    "+cppp+": "+cp.getName()+"
    "); + pw.println("
    " + cppp + ": " + cp.getName() + "
    "); pw.println("
    "); - if(!StringUtils.isEmpty(cp.getPath())) { + if (!StringUtils.isEmpty(cp.getPath())) { printLinkedFile( - pw, - cp.getFile(), + pw, + cp.getFile(), cp.getFile().toString(), - cp.getPath().endsWith(".log")?"format=paged&pos=-1&lines=-128&reverse=y":null); + cp.getPath().endsWith(".log") ? "format=paged&pos=-1&lines=-128&reverse=y" + : null); } else { pw.println("unset"); } @@ -463,144 +480,158 @@ public class JobResource extends BaseResource { pw.println("
    "); } - + pw.println("

    Advanced

    "); pw.println("

    Scripting console

    "); - if(!cj.hasApplicationContext()) { + if (!cj.hasApplicationContext()) { pw.println("build the job to browse bean instances"); } else { pw.println("

    Browse beans

    "); } pw.println("

    Copy

    "); - pw.println( - "
    Copy job to " + - "" + - "" + - "
    "); - pw.println("
    "); + pw.println("
    Copy job to " + + "" + + "" + + "
    "); + // pw.println("
    "); + + pw.println(""); + pw.println(""); } /** * Print a link to the given File * - * @param pw PrintWriter - * @param f File + * @param pw + * PrintWriter + * @param f + * File */ - protected void printLinkedFile(PrintWriter pw, File f) { - printLinkedFile(pw,f,f.toString(),null); + protected void printLinkedFile(PrintWriter pw, File f) { + printLinkedFile(pw, f, f.toString(), null); } - + /** * Print a link to the given File, using the given link text * - * @param pw PrintWriter - * @param f File + * @param pw + * PrintWriter + * @param f + * File */ - protected void printLinkedFile(PrintWriter pw, File f, String linktext, String queryString) { - String relativePath = JobResource.getHrefPath(f,cj); - pw.println("" - + linktext +""); - if(EDIT_FILTER.accept(f)) { - pw.println("[edit]
    "); + protected void printLinkedFile(PrintWriter pw, File f, String linktext, + String queryString) { + String relativePath = JobResource.getHrefPath(f, cj); + pw.println("" + + linktext + ""); + if (EDIT_FILTER.accept(f)) { + pw.println("[edit]"); } } /** - * Get a usable HrefPath, relative to the JobResource, for the given - * file. Assumes usual helper resources ('jobdir/', 'anypath/') at - * the usual locations. + * Get a usable HrefPath, relative to the JobResource, for the given file. + * Assumes usual helper resources ('jobdir/', 'anypath/') at the usual + * locations. * - * @param f File to provide an href (suitable for clicking or redirection) - * @param cj CrawlJob for calculating jobdir-relative path if possible + * @param f + * File to provide an href (suitable for clicking or redirection) + * @param cj + * CrawlJob for calculating jobdir-relative path if possible * @return String path suitable as href or Location header */ public static String getHrefPath(File f, CrawlJob cj) { String jobDirRelative = cj.jobDirRelativePath(f); - if(jobDirRelative!=null) { - return "jobdir/"+jobDirRelative; + if (jobDirRelative != null) { + return "jobdir/" + jobDirRelative; } // TODO: delegate this to EngineApplication, or make // conditional on whether /anypath/ service is present? String fullPath = f.getAbsolutePath(); fullPath = fullPath.replace(File.separatorChar, '/'); - return "../../anypath/"+fullPath; + return "../../anypath/" + fullPath; } protected Engine getEngine() { - return ((EngineApplication)getApplication()).getEngine(); + return ((EngineApplication) getApplication()).getEngine(); } @Override - public void acceptRepresentation(Representation entity) throws ResourceException { + public void acceptRepresentation(Representation entity) + throws ResourceException { if (cj == null) { throw new ResourceException(404); } // copy op? Form form = null; - form = getRequest().getEntityAsForm(); + form = getRequest().getEntityAsForm(); String copyTo = form.getFirstValue("copyTo"); - if(copyTo!=null) { - copyJob(copyTo,"on".equals(form.getFirstValue("asProfile"))); + if (copyTo != null) { + copyJob(copyTo, "on".equals(form.getFirstValue("asProfile"))); return; } AlertHandler.ensureStaticInitialization(); AlertThreadGroup.setThreadLogger(cj.getJobLogger()); String action = form.getFirstValue("action"); - if("launch".equals(action)) { + if ("launch".equals(action)) { String selectedCheckpoint = form.getFirstValue("checkpoint"); - if(StringUtils.isNotEmpty(selectedCheckpoint)) { - cj.getCheckpointService().setRecoveryCheckpointByName(selectedCheckpoint); + if (StringUtils.isNotEmpty(selectedCheckpoint)) { + cj.getCheckpointService().setRecoveryCheckpointByName( + selectedCheckpoint); } - cj.launch(); - } else if("checkXML".equals(action)) { + cj.launch(); + } else if ("checkXML".equals(action)) { cj.checkXML(); - } else if("instantiate".equals(action)) { + } else if ("instantiate".equals(action)) { cj.instantiateContainer(); - } else if("build".equals(action)||"validate".equals(action)) { + } else if ("build".equals(action) || "validate".equals(action)) { cj.validateConfiguration(); - } else if("teardown".equals(action)) { - if(!cj.teardown()) { - Flash.addFlash(getResponse(), "waiting for job to finish", Flash.Kind.NACK); + } else if ("teardown".equals(action)) { + if (!cj.teardown()) { + Flash.addFlash(getResponse(), "waiting for job to finish", + Flash.Kind.NACK); } - } else if("pause".equals(action)) { + } else if ("pause".equals(action)) { cj.getCrawlController().requestCrawlPause(); - } else if("unpause".equals(action)) { + } else if ("unpause".equals(action)) { cj.getCrawlController().requestCrawlResume(); - } else if("checkpoint".equals(action)) { + } else if ("checkpoint".equals(action)) { String cp = cj.getCheckpointService().requestCrawlCheckpoint(); - if(StringUtils.isNotEmpty(cp)) { - Flash.addFlash(getResponse(), "Checkpoint "+cp+" saved",Flash.Kind.ACK); + if (StringUtils.isNotEmpty(cp)) { + Flash.addFlash(getResponse(), "Checkpoint " + cp + + " saved", Flash.Kind.ACK); } else { - Flash.addFlash(getResponse(), "Checkpoint not made -- perhaps no progress since last? (see logs)",Flash.Kind.NACK); + Flash.addFlash( + getResponse(), + "Checkpoint not made -- perhaps no progress since last? (see logs)", + Flash.Kind.NACK); } - } else if("terminate".equals(action)) { + } else if ("terminate".equals(action)) { cj.terminate(); } AlertThreadGroup.setThreadLogger(null); - + // default: redirect to GET self getResponse().redirectSeeOther(getRequest().getOriginalRef()); } - protected void copyJob(String copyTo, boolean asProfile) throws ResourceException { + protected void copyJob(String copyTo, boolean asProfile) + throws ResourceException { try { getEngine().copy(cj, copyTo, asProfile); } catch (IOException e) { - Flash.addFlash(getResponse(), "Job not copied: "+e.getMessage(), Flash.Kind.NACK); + Flash.addFlash(getResponse(), "Job not copied: " + e.getMessage(), + Flash.Kind.NACK); getResponse().redirectSeeOther(getRequest().getOriginalRef()); return; } // redirect to destination job page getResponse().redirectSeeOther(copyTo); } - - + } diff --git a/engine/src/main/java/org/archive/crawler/restlet/ScriptResource.java b/engine/src/main/java/org/archive/crawler/restlet/ScriptResource.java index 9a4bf876..82dbe5be 100644 --- a/engine/src/main/java/org/archive/crawler/restlet/ScriptResource.java +++ b/engine/src/main/java/org/archive/crawler/restlet/ScriptResource.java @@ -24,6 +24,8 @@ import java.io.PrintWriter; import java.io.StringWriter; import java.io.Writer; import java.util.Collection; +import java.util.Collections; +import java.util.Comparator; import java.util.LinkedHashMap; import java.util.LinkedList; import java.util.List; @@ -65,6 +67,13 @@ public class ScriptResource extends JobRelatedResource { static LinkedList FACTORIES = new LinkedList(); static { FACTORIES.addAll(MANAGER.getEngineFactories()); + // Sort factories alphabetically so that they appear in the UI consistently + Collections.sort(FACTORIES, new Comparator() { + @Override + public int compare(ScriptEngineFactory sef1, ScriptEngineFactory sef2) { + return sef1.getEngineName().compareTo(sef2.getEngineName()); + } + }); } String script = ""; Exception ex = null; @@ -233,8 +242,16 @@ public class ScriptResource extends JobRelatedResource { } protected void writeHtml(Writer writer) { - PrintWriter pw = new PrintWriter(writer); - pw.println("Script in "+cj.getShortName()+""); + String baseRef = getRequest().getResourceRef().getBaseRef().toString(); + if (!baseRef.endsWith("/")) baseRef += "/"; + PrintWriter pw = new PrintWriter(writer); + pw.println(""); + pw.println(""); + pw.println(""); + pw.println("Script in "+cj.getShortName()+""); + pw.println(""); + pw.println(""); + pw.println(""); pw.println("

    Execute script for job "+cj.getShortName()+"

    "); @@ -261,7 +278,7 @@ public class ScriptResource extends JobRelatedResource { } pw.println("
    "); - pw.println(""); + pw.println(""); pw.println("