From 803f9ca979feedbb66eb8a36634fb7f51cf018db Mon Sep 17 00:00:00 2001 From: Noah Levitt Date: Thu, 19 Jan 2012 13:31:57 -0800 Subject: [PATCH 01/23] HER-1984 save script state - implement by adding a map to the application context for arbitrary use, and make sure the app context is available in all scripting environments * PathSharingContext.java new member variable ConcurrentHashMap data and accessor getData() * ScriptedProcessor.java, ScriptedDecideRule.java make appCtx available to scripts; also remove unused member sharedMap * ActionDirectory.java formatting fix --- .../org/archive/spring/PathSharingContext.java | 15 +++++++++++++++ .../crawler/framework/ActionDirectory.java | 2 +- .../org/archive/modules/ScriptedProcessor.java | 6 ++---- .../modules/deciderules/ScriptedDecideRule.java | 6 ++---- 4 files changed, 20 insertions(+), 9 deletions(-) diff --git a/commons/src/main/java/org/archive/spring/PathSharingContext.java b/commons/src/main/java/org/archive/spring/PathSharingContext.java index f8348c70..3a182960 100644 --- a/commons/src/main/java/org/archive/spring/PathSharingContext.java +++ b/commons/src/main/java/org/archive/spring/PathSharingContext.java @@ -23,6 +23,7 @@ import java.io.File; import java.io.IOException; import java.util.HashMap; import java.util.Map.Entry; +import java.util.concurrent.ConcurrentHashMap; import java.util.logging.Level; import java.util.logging.Logger; @@ -178,4 +179,18 @@ public class PathSharingContext extends FileSystemXmlApplicationContext { } super.initLifecycleProcessor(); } + + protected ConcurrentHashMap data; + + /** + * @return a shared map for arbitrary use during a crawl; for example, could + * be used for state persisting for the duration of the crawl, + * shared among ScriptedProcessor, scripting console, etc scripts + */ + public ConcurrentHashMap getData() { + if (data == null) { + data = new ConcurrentHashMap(); + } + return data; + } } diff --git a/engine/src/main/java/org/archive/crawler/framework/ActionDirectory.java b/engine/src/main/java/org/archive/crawler/framework/ActionDirectory.java index 7e492a62..56fe41dc 100644 --- a/engine/src/main/java/org/archive/crawler/framework/ActionDirectory.java +++ b/engine/src/main/java/org/archive/crawler/framework/ActionDirectory.java @@ -312,7 +312,7 @@ public class ActionDirectory implements ApplicationContextAware, Lifecycle, Runn PrintWriter rawOut = new PrintWriter(rawString); Exception ex = null; engine.put("rawOut", rawOut); - engine.put("appCtx",appCtx); + engine.put("appCtx", appCtx); // evaluate and record any exception try { diff --git a/modules/src/main/java/org/archive/modules/ScriptedProcessor.java b/modules/src/main/java/org/archive/modules/ScriptedProcessor.java index 951fedb7..802de482 100644 --- a/modules/src/main/java/org/archive/modules/ScriptedProcessor.java +++ b/modules/src/main/java/org/archive/modules/ScriptedProcessor.java @@ -20,8 +20,6 @@ package org.archive.modules; import java.io.Reader; -import java.util.Map; -import java.util.concurrent.ConcurrentHashMap; import java.util.logging.Level; import java.util.logging.Logger; @@ -106,8 +104,6 @@ implements ApplicationContextAware, InitializingBean { transient protected ThreadLocal threadEngine = new ThreadLocal(); protected ScriptEngine sharedEngine; - /** map for optional use by scripts */ - public Map sharedMap = new ConcurrentHashMap(); /** * Constructor. @@ -135,12 +131,14 @@ implements ApplicationContextAware, InitializingBean { // synchronization is harmless for local thread engine, // necessary for shared engine engine.put("curi",curi); + engine.put("appCtx", appCtx); try { engine.eval("process(curi)"); } catch (ScriptException e) { logger.log(Level.WARNING,e.getMessage(),e); } finally { engine.put("curi", null); + engine.put("appCtx", null); } } } diff --git a/modules/src/main/java/org/archive/modules/deciderules/ScriptedDecideRule.java b/modules/src/main/java/org/archive/modules/deciderules/ScriptedDecideRule.java index 290f881c..448ca3c4 100644 --- a/modules/src/main/java/org/archive/modules/deciderules/ScriptedDecideRule.java +++ b/modules/src/main/java/org/archive/modules/deciderules/ScriptedDecideRule.java @@ -20,8 +20,6 @@ package org.archive.modules.deciderules; import java.io.Reader; -import java.util.Map; -import java.util.concurrent.ConcurrentHashMap; import java.util.logging.Level; import java.util.logging.Logger; @@ -106,8 +104,6 @@ implements ApplicationContextAware, InitializingBean { transient protected ThreadLocal threadEngine = new ThreadLocal(); transient protected ScriptEngine sharedEngine; - /** map for optional use by scripts */ - public Map sharedMap = new ConcurrentHashMap(); public ScriptedDecideRule() { } @@ -128,12 +124,14 @@ implements ApplicationContextAware, InitializingBean { // necessary for shared engine try { engine.put("object",uri); + engine.put("appCtx", appCtx); return (DecideResult)engine.eval("decisionFor(object)"); } catch (ScriptException e) { logger.log(Level.WARNING,e.getMessage(),e); return DecideResult.NONE; } finally { engine.put("object", null); + engine.put("appCtx", null); } } } From a652a9afd0008bdd642b9e9ce32443cc70b66274 Mon Sep 17 00:00:00 2001 From: Noah Levitt Date: Fri, 20 Jan 2012 11:30:24 -0800 Subject: [PATCH 02/23] * profile-crawler-beans.cxml commented-out default value for new CandidatesProcessor settings processErrorOutlinks --- .../org/archive/crawler/restlet/profile-crawler-beans.cxml | 1 + 1 file changed, 1 insertion(+) diff --git a/engine/src/main/resources/org/archive/crawler/restlet/profile-crawler-beans.cxml b/engine/src/main/resources/org/archive/crawler/restlet/profile-crawler-beans.cxml index eb430f87..f7842df0 100644 --- a/engine/src/main/resources/org/archive/crawler/restlet/profile-crawler-beans.cxml +++ b/engine/src/main/resources/org/archive/crawler/restlet/profile-crawler-beans.cxml @@ -354,6 +354,7 @@ http://example.example/example + From e90fc2965b53e631384e1c46873a396ddeaaeb8a Mon Sep 17 00:00:00 2001 From: Travis Wellman Date: Thu, 26 Jan 2012 00:37:19 -0800 Subject: [PATCH 03/23] Sort script engines alphabetically so that they appear consistently. Note that the name of the engine may be different than the name of the language it parses. A good example is the Rhino script engine that parses javascript. --- .../java/org/archive/crawler/restlet/ScriptResource.java | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/engine/src/main/java/org/archive/crawler/restlet/ScriptResource.java b/engine/src/main/java/org/archive/crawler/restlet/ScriptResource.java index 9a4bf876..0577e7cd 100644 --- a/engine/src/main/java/org/archive/crawler/restlet/ScriptResource.java +++ b/engine/src/main/java/org/archive/crawler/restlet/ScriptResource.java @@ -24,6 +24,8 @@ import java.io.PrintWriter; import java.io.StringWriter; import java.io.Writer; import java.util.Collection; +import java.util.Collections; +import java.util.Comparator; import java.util.LinkedHashMap; import java.util.LinkedList; import java.util.List; @@ -65,6 +67,13 @@ public class ScriptResource extends JobRelatedResource { static LinkedList FACTORIES = new LinkedList(); static { FACTORIES.addAll(MANAGER.getEngineFactories()); + // Sort factories alphabetically so that they appear in the UI consistently + Collections.sort(FACTORIES, new Comparator() { + @Override + public int compare(ScriptEngineFactory sef1, ScriptEngineFactory sef2) { + return sef1.getEngineName().compareTo(sef2.getEngineName()); + } + }); } String script = ""; Exception ex = null; From cfac7ead9162f29b3da00ed11cbf2b8c19d303f9 Mon Sep 17 00:00:00 2001 From: Noah Levitt Date: Thu, 9 Feb 2012 10:05:07 -0800 Subject: [PATCH 04/23] Fix for HER-1985 H3: SurtPrefixDecideRule forgets learned/seed-derived/seed-directive SURT prefixes in checkpoint-resume * SurtPrefixedDecideRule.java make class implement Checkpointable, save surt prefixes to json on checkpoint and load them on recover * profile-crawler-beans.cxml make main SurtPrefixedDecideRule a top-level bean so that it can be checkpointed * Checkpoint.java, CheckpointService.java add some FINE level logging during checkpointing and recovery --- .../org/archive/checkpointing/Checkpoint.java | 9 +++ .../crawler/framework/CheckpointService.java | 4 ++ .../restlet/profile-crawler-beans.cxml | 50 +++++++-------- .../surt/SurtPrefixedDecideRule.java | 61 ++++++++++++++++++- 4 files changed, 97 insertions(+), 27 deletions(-) diff --git a/commons/src/main/java/org/archive/checkpointing/Checkpoint.java b/commons/src/main/java/org/archive/checkpointing/Checkpoint.java index 2e5f2a72..47cda1ca 100644 --- a/commons/src/main/java/org/archive/checkpointing/Checkpoint.java +++ b/commons/src/main/java/org/archive/checkpointing/Checkpoint.java @@ -125,6 +125,9 @@ public class Checkpoint implements InitializingBean { public void saveJson(String beanName, JSONObject json) { try { File targetFile = new File(getCheckpointDir().getFile(),beanName); + if (LOGGER.isLoggable(Level.FINE)) { + LOGGER.fine("saving json to " + targetFile); + } FileUtils.writeStringToFile( targetFile, json.toString()); @@ -137,6 +140,9 @@ public class Checkpoint implements InitializingBean { public JSONObject loadJson(String beanName) { File sourceFile = new File(getCheckpointDir().getFile(),beanName); try { + if (LOGGER.isLoggable(Level.FINE)) { + LOGGER.fine("reading json from " + sourceFile); + } return new JSONObject(FileUtils.readFileToString(sourceFile)); } catch (JSONException e) { throw new RuntimeException(e); @@ -148,6 +154,9 @@ public class Checkpoint implements InitializingBean { public BufferedWriter saveWriter(String beanName, String extraName) throws IOException { try { File targetFile = new File(getCheckpointDir().getFile(),beanName+"-"+extraName); + if (LOGGER.isLoggable(Level.FINE)) { + LOGGER.fine("opening for writing: " + targetFile); + } return new BufferedWriter(new FileWriter(targetFile)); } catch (IOException e) { LOGGER.log(Level.SEVERE,"unable to save checkpoint writer state "+extraName+" of "+beanName,e); diff --git a/engine/src/main/java/org/archive/crawler/framework/CheckpointService.java b/engine/src/main/java/org/archive/crawler/framework/CheckpointService.java index ef62f364..3b5d8b2e 100644 --- a/engine/src/main/java/org/archive/crawler/framework/CheckpointService.java +++ b/engine/src/main/java/org/archive/crawler/framework/CheckpointService.java @@ -40,6 +40,7 @@ import org.archive.spring.ConfigPath; import org.archive.spring.ConfigPathConfigurer; import org.archive.spring.HasValidator; import org.springframework.beans.BeansException; +import org.springframework.beans.factory.BeanFactoryUtils; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.context.ApplicationContext; import org.springframework.context.ApplicationContextAware; @@ -222,6 +223,9 @@ public class CheckpointService implements Lifecycle, ApplicationContextAware, Ha } Map toCheckpoint = appCtx.getBeansOfType(Checkpointable.class); + if (LOGGER.isLoggable(Level.FINE)) { + LOGGER.fine("checkpointing beans " + toCheckpoint); + } checkpointInProgress = new Checkpoint(); try { diff --git a/engine/src/main/resources/org/archive/crawler/restlet/profile-crawler-beans.cxml b/engine/src/main/resources/org/archive/crawler/restlet/profile-crawler-beans.cxml index f7842df0..a53b476b 100644 --- a/engine/src/main/resources/org/archive/crawler/restlet/profile-crawler-beans.cxml +++ b/engine/src/main/resources/org/archive/crawler/restlet/profile-crawler-beans.cxml @@ -10,11 +10,11 @@ behavior, uncomment AND alter the shown values.) --> @@ -109,6 +109,25 @@ http://example.example/example --> + + + + + + + + + @@ -116,26 +135,9 @@ http://example.example/example - - + - - - - - - - + diff --git a/modules/src/main/java/org/archive/modules/deciderules/surt/SurtPrefixedDecideRule.java b/modules/src/main/java/org/archive/modules/deciderules/surt/SurtPrefixedDecideRule.java index ecdae30f..0f851f3f 100644 --- a/modules/src/main/java/org/archive/modules/deciderules/surt/SurtPrefixedDecideRule.java +++ b/modules/src/main/java/org/archive/modules/deciderules/surt/SurtPrefixedDecideRule.java @@ -23,10 +23,13 @@ import java.io.File; import java.io.FileWriter; import java.io.IOException; import java.io.Reader; +import java.util.logging.Level; import java.util.logging.Logger; import org.apache.commons.io.IOUtils; import org.apache.commons.lang.StringUtils; +import org.archive.checkpointing.Checkpoint; +import org.archive.checkpointing.Checkpointable; import org.archive.io.ReadSource; import org.archive.modules.CrawlURI; import org.archive.modules.deciderules.DecideResult; @@ -36,6 +39,10 @@ import org.archive.modules.seeds.SeedModule; import org.archive.net.UURI; import org.archive.spring.ConfigFile; import org.archive.util.SurtPrefixSet; +import org.json.JSONArray; +import org.json.JSONException; +import org.json.JSONObject; +import org.springframework.beans.factory.BeanNameAware; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.context.ApplicationEvent; import org.springframework.context.ApplicationListener; @@ -56,7 +63,8 @@ import org.springframework.context.event.ContextStartedEvent; * @author gojomo */ public class SurtPrefixedDecideRule extends PredicatedDecideRule implements - SeedListener, ApplicationListener { + SeedListener, ApplicationListener, Checkpointable, + BeanNameAware { private static final long serialVersionUID = 3L; private static final Logger logger = @@ -219,6 +227,9 @@ public class SurtPrefixedDecideRule extends PredicatedDecideRule implements */ protected void buildSurtPrefixSet() { if (getSurtsSource() != null) { + if (logger.isLoggable(Level.FINE)) { + logger.fine("reading surt prefixes from " + getSurtsSource()); + } Reader reader = getSurtsSource().obtainReader(); try { surtPrefixes.importFromMixed(reader, true); @@ -277,12 +288,56 @@ public class SurtPrefixedDecideRule extends PredicatedDecideRule implements } throw new IllegalArgumentException("decision must be ACCEPT or REJECT"); } - + @Override public void onApplicationEvent(ApplicationEvent event) { if (event instanceof ContextStartedEvent) { - readPrefixes(); + if (recoveryCheckpoint != null) { + JSONObject json = recoveryCheckpoint.loadJson(beanName); + try { + JSONArray jsonArray = json.getJSONArray("surtPrefixes"); + for (int i = 0; i < jsonArray.length(); i++) { + surtPrefixes.add(jsonArray.getString(i)); + } + } catch (JSONException e) { + throw new IllegalStateException(e); + } + } else { + readPrefixes(); + } } } + + // BeanNameAware + protected String beanName; + public void setBeanName(String name) { + this.beanName = name; + } + + @Override + public void startCheckpoint(Checkpoint checkpointInProgress) { + } + + @Override + public void doCheckpoint(Checkpoint checkpointInProgress) + throws IOException { + try { + JSONObject json = new JSONObject(); + json.put("surtPrefixes", surtPrefixes); + checkpointInProgress.saveJson(beanName, json); + } catch (JSONException e) { + throw new RuntimeException(e); + } + } + + @Override + public void finishCheckpoint(Checkpoint checkpointInProgress) { + } + + protected Checkpoint recoveryCheckpoint; + @Override + public void setRecoveryCheckpoint(Checkpoint recoveryCheckpoint) { + this.recoveryCheckpoint = recoveryCheckpoint; + } }//EOC From db801b3e0ffed5bf8bdb7de3877b1ed8eaba6b7b Mon Sep 17 00:00:00 2001 From: Noah Levitt Date: Fri, 10 Feb 2012 15:48:09 -0800 Subject: [PATCH 05/23] Fix problems handling crawl job paths with characters that require url-encoding, since we sometimes refer to them with file: urls * PathSharingContext.java, CrawlJob.java use java.net.URI to convert paths to and from file: urls --- .../org/archive/spring/PathSharingContext.java | 14 ++++++++++---- .../org/archive/crawler/framework/CrawlJob.java | 2 +- 2 files changed, 11 insertions(+), 5 deletions(-) diff --git a/commons/src/main/java/org/archive/spring/PathSharingContext.java b/commons/src/main/java/org/archive/spring/PathSharingContext.java index 3a182960..e863773a 100644 --- a/commons/src/main/java/org/archive/spring/PathSharingContext.java +++ b/commons/src/main/java/org/archive/spring/PathSharingContext.java @@ -21,6 +21,8 @@ package org.archive.spring; import java.io.File; import java.io.IOException; +import java.net.URI; +import java.net.URISyntaxException; import java.util.HashMap; import java.util.Map.Entry; import java.util.concurrent.ConcurrentHashMap; @@ -133,12 +135,16 @@ public class PathSharingContext extends FileSystemXmlApplicationContext { protected File getConfigurationFile() { String primaryConfigurationPath = getPrimaryConfigurationPath(); - if(primaryConfigurationPath.startsWith("file:")) { + if (primaryConfigurationPath.startsWith("file:")) { // strip URI-scheme if present (as is usual) - primaryConfigurationPath = primaryConfigurationPath.substring(5); + try { + return new File(new URI(primaryConfigurationPath)); + } catch (URISyntaxException e) { + throw new RuntimeException(e); + } + } else { + return new File(primaryConfigurationPath); } - File configFile = new File(primaryConfigurationPath); - return configFile; } protected void initLaunchDir() { diff --git a/engine/src/main/java/org/archive/crawler/framework/CrawlJob.java b/engine/src/main/java/org/archive/crawler/framework/CrawlJob.java index ee9cbccd..b15ccc8c 100644 --- a/engine/src/main/java/org/archive/crawler/framework/CrawlJob.java +++ b/engine/src/main/java/org/archive/crawler/framework/CrawlJob.java @@ -290,7 +290,7 @@ public class CrawlJob implements Comparable, ApplicationListener Date: Wed, 29 Feb 2012 11:02:39 -0800 Subject: [PATCH 06/23] * StatisticsTracker.java writeReportFile() - reduce log level from warning to info when reusing existing report (normal after crawl is finished) --- .../java/org/archive/crawler/reporting/StatisticsTracker.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/engine/src/main/java/org/archive/crawler/reporting/StatisticsTracker.java b/engine/src/main/java/org/archive/crawler/reporting/StatisticsTracker.java index 842726f2..6ffd8ef2 100644 --- a/engine/src/main/java/org/archive/crawler/reporting/StatisticsTracker.java +++ b/engine/src/main/java/org/archive/crawler/reporting/StatisticsTracker.java @@ -899,7 +899,7 @@ public class StatisticsTracker // and file exists // and force not requested // so, don't overwrite - logger.warning("reusing report: " + f.getAbsolutePath()); + logger.info("reusing report: " + f.getAbsolutePath()); return f; } From 46c335c8a309a026e33299d546b4cb96ee80410c Mon Sep 17 00:00:00 2001 From: Kenji Nagahashi Date: Wed, 11 May 2011 01:25:41 -0700 Subject: [PATCH 07/23] BeanBrowseResource: fixed malformed HTML. --- .../crawler/restlet/BeanBrowseResource.java | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/engine/src/main/java/org/archive/crawler/restlet/BeanBrowseResource.java b/engine/src/main/java/org/archive/crawler/restlet/BeanBrowseResource.java index a9bc596f..ee950764 100644 --- a/engine/src/main/java/org/archive/crawler/restlet/BeanBrowseResource.java +++ b/engine/src/main/java/org/archive/crawler/restlet/BeanBrowseResource.java @@ -195,13 +195,15 @@ public class BeanBrowseResource extends JobRelatedResource { protected void writeHtml(Writer writer) { PrintWriter pw = new PrintWriter(writer); + pw.println(""); pw.println("Crawl beans in "+cj.getShortName()+""); + pw.println(""); pw.println("

Crawl beans in built job "+cj.getShortName()+"

"); pw.println("Enter a bean path of the form beanName, beanName.property, beanName.property[indexOrKey], etc."); - pw.println("
"); - pw.println("
"); + pw.println("
"); + pw.println("
"); if (StringUtils.isNotBlank(beanPath)) { pw.println("

Bean path "+beanPath+"

"); @@ -227,9 +229,9 @@ public class BeanBrowseResource extends JobRelatedResource { pw.println("edit"); pw.println("Note: it may not be appropriate/effective to change this value in an already-built crawl context.
"); pw.println("
"); - pw.println(""); - pw.println(beanPath + " = "); - pw.println("
"); + pw.println(""); + pw.println(beanPath + " = "); + pw.println(""); } else { writeObject(pw, null, target, beanPath); } @@ -239,7 +241,7 @@ public class BeanBrowseResource extends JobRelatedResource { } } - pw.println("

All named crawl beansAll named crawl beans

"); pw.println("
    "); Set alreadyWritten = new HashSet(); writeNestedNames(pw, appCtx.getBean("crawlController"), getBeansRefPath(), alreadyWritten); @@ -247,5 +249,7 @@ public class BeanBrowseResource extends JobRelatedResource { writeNestedNames(pw, appCtx.getBean(name), getBeansRefPath(), alreadyWritten); } pw.println(""); + + pw.println("\n"); } } From adc82c1c6f72640c64a7b24687203012e6bb3be6 Mon Sep 17 00:00:00 2001 From: Kenji Nagahashi Date: Sat, 3 Mar 2012 00:46:56 -0800 Subject: [PATCH 08/23] Second round of HTML cleanup: - added missing and - fixed non-working CSS link, added CSS link to all pages - replaced layout by
    and   with structural tags with CSS - removed superfluous output, notably in BeanBrowseResource - -ify variable names in ScriptResource - added special handling of NaN in doubleToString() --- .../java/org/archive/util/ArchiveUtils.java | 3 + .../archive/crawler/framework/CrawlJob.java | 8 +- .../archive/crawler/restlet/BaseResource.java | 5 + .../crawler/restlet/BeanBrowseResource.java | 7 +- .../crawler/restlet/EngineResource.java | 42 +++--- .../crawler/restlet/JobRelatedResource.java | 4 +- .../archive/crawler/restlet/JobResource.java | 132 ++++++++++-------- .../crawler/restlet/ScriptResource.java | 25 ++-- .../org/archive/crawler/restlet/engine.css | 15 ++ 9 files changed, 149 insertions(+), 92 deletions(-) diff --git a/commons/src/main/java/org/archive/util/ArchiveUtils.java b/commons/src/main/java/org/archive/util/ArchiveUtils.java index 7988dc5d..2ecd1564 100644 --- a/commons/src/main/java/org/archive/util/ArchiveUtils.java +++ b/commons/src/main/java/org/archive/util/ArchiveUtils.java @@ -557,6 +557,9 @@ public class ArchiveUtils { } private static String doubleToString(double val, int maxFractionDigits, int minFractionDigits) { + // apparently NumberFormat generates Replacement Character (U+FFFD) for NaN in non-latin1 environment. + // following line explicitly produces infinite symbol for NaN. + if (Double.isNaN(val)) return "\u221E"; NumberFormat f = NumberFormat.getNumberInstance(Locale.US); f.setMaximumFractionDigits(maxFractionDigits); f.setMinimumFractionDigits(minFractionDigits); diff --git a/engine/src/main/java/org/archive/crawler/framework/CrawlJob.java b/engine/src/main/java/org/archive/crawler/framework/CrawlJob.java index b15ccc8c..fbef5a71 100644 --- a/engine/src/main/java/org/archive/crawler/framework/CrawlJob.java +++ b/engine/src/main/java/org/archive/crawler/framework/CrawlJob.java @@ -208,7 +208,7 @@ public class CrawlJob implements Comparable, ApplicationListener"); +// pw.println(""); // if(isRunning()) { // pw.println("ACTIVE; "+getCrawlController().getState()+":"); // } @@ -225,13 +225,13 @@ public class CrawlJob implements Comparable, ApplicationListener"); + pw.println("
    "); pw.println(getPrimaryConfig()); - pw.println("
    "); + pw.println("
    "); if(lastLaunch!=null) { pw.println("(last at "+lastLaunch+")"); } - pw.println("    "); +// pw.println(""); } /** diff --git a/engine/src/main/java/org/archive/crawler/restlet/BaseResource.java b/engine/src/main/java/org/archive/crawler/restlet/BaseResource.java index 14c004ba..a5224801 100644 --- a/engine/src/main/java/org/archive/crawler/restlet/BaseResource.java +++ b/engine/src/main/java/org/archive/crawler/restlet/BaseResource.java @@ -67,4 +67,9 @@ public abstract class BaseResource extends Resource { return super.getPreferredVariant(); } + + protected String getStylesheetRef() { + String rootRef = getRequest().getRootRef().toString(); + return rootRef + "/engine/static/engine.css"; + } } diff --git a/engine/src/main/java/org/archive/crawler/restlet/BeanBrowseResource.java b/engine/src/main/java/org/archive/crawler/restlet/BeanBrowseResource.java index ee950764..d0b5c3cc 100644 --- a/engine/src/main/java/org/archive/crawler/restlet/BeanBrowseResource.java +++ b/engine/src/main/java/org/archive/crawler/restlet/BeanBrowseResource.java @@ -194,9 +194,14 @@ public class BeanBrowseResource extends JobRelatedResource { } protected void writeHtml(Writer writer) { + String baseRef = getRequest().getResourceRef().getBaseRef().toString(); + if (!baseRef.endsWith("/")) baseRef += "/"; PrintWriter pw = new PrintWriter(writer); pw.println(""); - pw.println("Crawl beans in "+cj.getShortName()+""); + pw.println(""); + pw.println("Crawl beans in "+cj.getShortName()+""); + pw.println(""); + pw.println(""); pw.println(""); pw.println("

    Crawl beans in built job "); - pw.println(""); + pw.println(""); + pw.println("\n"); pw.println("

    "+engineTitle+"

    "); Flash.renderFlashesHTML(pw, getRequest()); - pw.println("Memory: "); + pw.println("Memory:"); pw.println(engine.heapReport()); - pw.println("

    "); - pw.println("Jobs Directory:     "+jobsDir.getAbsolutePath()+"

    "); + pw.println("

    "); + pw.println("Jobs Directory: "+jobsDir.getAbsolutePath()+""); ArrayList jobs = new ArrayList(); jobs.addAll(engine.getJobConfigs().values()); - pw.println("

    Job Directories ("+jobs.size()+")"); + pw.println("

    Job Directories ("+jobs.size()+")"); pw.println(""); pw.println("

    "); Collections.sort(jobs); pw.println("
      "); for(CrawlJob cj: jobs) { - pw.println("
    • "); + pw.print("
    • "); cj.writeHtmlTo(pw,"job/"); pw.println("
      • "); } @@ -283,16 +284,16 @@ public class EngineResource extends BaseResource { pw.println("

      Add Job Directory

      "); // create new job with defaults - pw.println("
      \n" - + "Create new job directory with recommended starting configuration
      \n" - + "Path: " + jobsDir.getAbsolutePath() + File.separator + "\n" + pw.println("\n" + + "Create new job directory with recommended starting configuration
      \n" + + "Path: " + jobsDir.getAbsolutePath() + File.separator + "\n" + "\n" + "
      \n"); - pw.println("
      "); - pw.println("Specify a path to a pre-existing job directory
      "); - pw.println("Path: " + ""); + pw.println(""); + pw.println("Specify a path to a pre-existing job directory
      "); + pw.println("Path: "); pw.println(""); pw.println("
      "); @@ -306,23 +307,24 @@ public class EngineResource extends BaseResource { pw.println("

      Exit Java

      "); pw.println( - "This exits the Java process running Heritrix. To restart " + + "
      This exits the Java process running Heritrix. To restart " + "will then require access to the hosting machine. You should " + - "cleanly terminate and teardown any jobs in progress first.
      "); - pw.println("
      "); + "cleanly terminate and teardown any jobs in progress first.
      "); + pw.println(""); for(Map.Entry entry : getBuiltJobs().entrySet()) { - pw.println("
      Job '"+entry.getKey()+"' still «" + pw.println("
      Job '"+entry.getKey()+"' still «" +entry.getValue().getJobStatusDescription() - +"»
      "); + +"»
      "); String checkName = "ignore__"+entry.getKey(); pw.println("
      "); + +entry.getKey()+"' and exit anyway
      "); } - pw.println("
      "); + pw.println(""); pw.println(""); pw.println(""); pw.println(""); + pw.println(""); pw.flush(); } diff --git a/engine/src/main/java/org/archive/crawler/restlet/JobRelatedResource.java b/engine/src/main/java/org/archive/crawler/restlet/JobRelatedResource.java index 7e2d7e9f..a3e1b711 100644 --- a/engine/src/main/java/org/archive/crawler/restlet/JobRelatedResource.java +++ b/engine/src/main/java/org/archive/crawler/restlet/JobRelatedResource.java @@ -156,7 +156,7 @@ public abstract class JobRelatedResource extends BaseResource { String name = getBeanToNameMap().get(obj); pw.println("
    • "+name+""); pw.println(""+obj.getClass().getName()+"
        "); - close = "
      • "; + close = "
    \n"; } if(!alreadyWritten.contains(obj)) { alreadyWritten.add(obj); @@ -183,7 +183,7 @@ public abstract class JobRelatedResource extends BaseResource { pw.println(""+ipe.getMessage()+""); } } - pw.println(close); + pw.print(close); } diff --git a/engine/src/main/java/org/archive/crawler/restlet/JobResource.java b/engine/src/main/java/org/archive/crawler/restlet/JobResource.java index e8329c91..dfe06d51 100644 --- a/engine/src/main/java/org/archive/crawler/restlet/JobResource.java +++ b/engine/src/main/java/org/archive/crawler/restlet/JobResource.java @@ -226,9 +226,11 @@ public class JobResource extends BaseResource { baseRef += "/"; } // TODO: replace with use a templating system (FreeMarker?) - pw.println(""+jobTitle+""); - pw.println(""); - pw.println(""); + pw.println(""); + pw.println(""+jobTitle+""); + pw.println(""); + pw.println(""); pw.println(""); pw.print("

    Job "+cj.getShortName()+" ("); @@ -255,53 +257,59 @@ public class JobResource extends BaseResource { // button controls pw.println("
    "); // PREP, LAUNCH - pw.print(""); + pw.println(""); + pw.print(""); pw.print("   "); + pw.println(">"); // PAUSE, UNPAUSE, CHECKPOINT - pw.println(""); + pw.print(""); - pw.println(""); + pw.print(""); - pw.println(""); + pw.print(""); + pw.println(" type='submit' name='action' value='checkpoint'>"); + pw.println(""); // TERMINATE, RESET - pw.println("   "); - pw.println(""); + pw.print("
    "); + pw.println(">"); + pw.println(""); // display checkpoint options if(cj.getCheckpointService()!=null) { Checkpoint recoveryCheckpoint = cj.getCheckpointService().getRecoveryCheckpoint(); if(recoveryCheckpoint!=null) { - pw.println("recover from "+recoveryCheckpoint.getName()+""); + pw.println("
    recover from "+recoveryCheckpoint.getName()+""); } else if (cj.getCheckpointService().hasAvailableCheckpoints() && cj.isLaunchable()) { - pw.println("select an available checkpoint before launch to recover:"); + pw.println("
    select an available checkpoint before launch to recover:"); pw.println("" + - "" + - "" + + "Copy job to " + + "" + + "" + ""); - pw.println("
    "); + //pw.println("
    "); + + pw.println(""); + pw.println(""); } /** @@ -508,7 +526,7 @@ public class JobResource extends BaseResource { if(EDIT_FILTER.accept(f)) { pw.println("[edit]
    "); + + "?format=textedit'>edit]"); } } diff --git a/engine/src/main/java/org/archive/crawler/restlet/ScriptResource.java b/engine/src/main/java/org/archive/crawler/restlet/ScriptResource.java index 0577e7cd..d1dbc727 100644 --- a/engine/src/main/java/org/archive/crawler/restlet/ScriptResource.java +++ b/engine/src/main/java/org/archive/crawler/restlet/ScriptResource.java @@ -242,8 +242,15 @@ public class ScriptResource extends JobRelatedResource { } protected void writeHtml(Writer writer) { + String baseRef = getRequest().getResourceRef().getBaseRef().toString(); + if (!baseRef.endsWith("/")) baseRef += "/"; PrintWriter pw = new PrintWriter(writer); - pw.println("Script in "+cj.getShortName()+""); + pw.println(""); + pw.println(""); + pw.println("Script in "+cj.getShortName()+""); + pw.println(""); + pw.println(""); + pw.println(""); pw.println("

    Execute script for job "+cj.getShortName()+"

    "); @@ -270,7 +277,7 @@ public class ScriptResource extends JobRelatedResource { } pw.println("
    "); - pw.println(""); + pw.println(""); pw.println(""); + pw.println(" edit"); + pw.println("
    "); + pw.println(""); + pw.println("
    Note: it may not be appropriate/effective to change this value in an already-built crawl context.
    "); + pw.println("
    "); pw.println(beanPath + " = "); - pw.println(""); + pw.println("
    "); } else { writeObject(pw, null, target, beanPath); } diff --git a/engine/src/main/java/org/archive/crawler/restlet/EditRepresentation.java b/engine/src/main/java/org/archive/crawler/restlet/EditRepresentation.java index c1becd33..4adc7c04 100644 --- a/engine/src/main/java/org/archive/crawler/restlet/EditRepresentation.java +++ b/engine/src/main/java/org/archive/crawler/restlet/EditRepresentation.java @@ -61,20 +61,25 @@ public class EditRepresentation extends CharacterRepresentation { @Override public void write(Writer writer) throws IOException { PrintWriter pw = new PrintWriter(writer); + pw.println(""); + pw.println(""); pw.println(""+fileRepresentation.getFile().getName()+""); - + pw.println(""); Flash.renderFlashesHTML(pw, dirResource.getRequest()); pw.println("
    "); // TODO: enable button on after changes made - pw.println(""); + pw.println("
    "); + pw.println(""); pw.println(fileRepresentation.getFile()); Reference viewRef = dirResource.getRequest().getOriginalRef().clone(); viewRef.setQuery(null); pw.println("view"); - pw.println("
    "); + pw.println("
    "); pw.println("
    "); + pw.println(""); + pw.println(""); pw.close(); } diff --git a/engine/src/main/java/org/archive/crawler/restlet/EngineResource.java b/engine/src/main/java/org/archive/crawler/restlet/EngineResource.java index 38d25546..3a483880 100644 --- a/engine/src/main/java/org/archive/crawler/restlet/EngineResource.java +++ b/engine/src/main/java/org/archive/crawler/restlet/EngineResource.java @@ -251,6 +251,7 @@ public class EngineResource extends BaseResource { baseRef += "/"; } PrintWriter pw = new PrintWriter(writer); + pw.println(""); pw.println(""); pw.println(""+engineTitle+""); pw.println(""); @@ -260,10 +261,9 @@ public class EngineResource extends BaseResource { Flash.renderFlashesHTML(pw, getRequest()); - pw.println("Memory:"); - pw.println(engine.heapReport()); - pw.println("

    "); - pw.println("Jobs Directory: "+jobsDir.getAbsolutePath()+""); + pw.println("

    Memory: " + engine.heapReport() + "

    "); + pw.println("

    Jobs Directory: " + + jobsDir.getAbsolutePath() + "

    "); ArrayList jobs = new ArrayList(); @@ -285,24 +285,24 @@ public class EngineResource extends BaseResource { // create new job with defaults pw.println("
    \n" - + "Create new job directory with recommended starting configuration
    \n" - + "Path: " + jobsDir.getAbsolutePath() + File.separator - + "\n" + + "
    Create new job directory with recommended starting configuration
    \n" + + "
    Path: " + jobsDir.getAbsolutePath() + + File.separator + "\n" + "\n" - + "\n"); + + "
    \n"); pw.println("
    "); - pw.println("Specify a path to a pre-existing job directory
    "); - pw.println("Path: "); + pw.println("
    Specify a path to a pre-existing job directory
    "); + pw.println("
    Path: "); pw.println(""); - pw.println(""); + pw.println("
    "); pw.println( - "You may also compose or copy a valid job directory into " + + "

    You may also compose or copy a valid job directory into " + "the main jobs directory via outside means, then use the " + "'rescan' button above to make it appear in this interface. Or, " + "use the 'copy' functionality at the botton of any existing " + - "job's detail page."); + "job's detail page.

    "); pw.println("

    Exit Java

    "); @@ -320,8 +320,10 @@ public class EngineResource extends BaseResource { +checkName+"'>
    "); } + pw.println("
    "); pw.println(""); - pw.println(""); + pw.println(""); + pw.println("
    "); pw.println(""); pw.println(""); pw.println(""); diff --git a/engine/src/main/java/org/archive/crawler/restlet/JobRelatedResource.java b/engine/src/main/java/org/archive/crawler/restlet/JobRelatedResource.java index a3e1b711..fc65905e 100644 --- a/engine/src/main/java/org/archive/crawler/restlet/JobRelatedResource.java +++ b/engine/src/main/java/org/archive/crawler/restlet/JobRelatedResource.java @@ -23,6 +23,7 @@ import java.beans.IntrospectionException; import java.beans.PropertyDescriptor; import java.io.File; import java.io.PrintWriter; +import java.net.URLEncoder; import java.util.Arrays; import java.util.Collection; import java.util.HashSet; @@ -36,6 +37,7 @@ import java.util.Set; import org.apache.commons.lang.StringUtils; import org.archive.crawler.framework.CrawlJob; import org.archive.crawler.framework.Engine; +import org.archive.util.TextUtils; import org.restlet.Context; import org.restlet.data.Reference; import org.restlet.data.Request; @@ -415,7 +417,7 @@ public abstract class JobRelatedResource extends BaseResource { String close = ""; String beanPath = beanPathPrefix; if(StringUtils.isNotBlank(field)) { - pw.write(""); + pw.write("\n"); String closeAnchor =""; if(StringUtils.isNotBlank(beanPathPrefix)) { if(beanPathPrefix.endsWith(".")) { @@ -423,21 +425,22 @@ public abstract class JobRelatedResource extends BaseResource { } else if (beanPathPrefix.endsWith("[")) { beanPath += field + "]"; } - pw.println(""); + // url-encode because brackets are not allowed here according to validator.w3.org + pw.print(""); closeAnchor = ""; } pw.write(field); pw.write(closeAnchor); - pw.write(":"); - close=""; + pw.write(":\n"); + close="\n\n"; } if(object == null) { - pw.write("null
    "); + pw.write("null"); pw.write(close); return; } if(object instanceof String) { - pw.write("\""+object+"\"
    "); + pw.write("\""+object+"\""); pw.write(close); return; } @@ -445,18 +448,18 @@ public abstract class JobRelatedResource extends BaseResource { || object instanceof File //|| BeanUtils.findEditorByConvention(object.getClass())!=null ) { - pw.write(object+"
    "); + pw.write(object.toString()); pw.write(close); return; } if(alreadyWritten.contains(object)) { - pw.println("↑
    "); + pw.println("↑"); pw.write(close); return; } alreadyWritten.add(object); // guard against repeats and cycles if(StringUtils.isNotBlank(key) && StringUtils.isNotBlank(field)) { - pw.println(""+key+"
    "); + pw.println(""+key+""); pw.write(close); return; } @@ -509,7 +512,7 @@ public abstract class JobRelatedResource extends BaseResource { } } pw.println(""); - pw.print("
    "); + pw.print(""); pw.write(close); } diff --git a/engine/src/main/java/org/archive/crawler/restlet/JobResource.java b/engine/src/main/java/org/archive/crawler/restlet/JobResource.java index 7125e634..2295a00e 100644 --- a/engine/src/main/java/org/archive/crawler/restlet/JobResource.java +++ b/engine/src/main/java/org/archive/crawler/restlet/JobResource.java @@ -59,31 +59,33 @@ import org.restlet.resource.Variant; import org.restlet.resource.WriterRepresentation; /** - * Restlet Resource representing a single local CrawlJob inside an - * Engine. + * Restlet Resource representing a single local CrawlJob inside an Engine. * * @contributor gojomo * @contributor nlevitt */ public class JobResource extends BaseResource { - public static final IOFileFilter EDIT_FILTER = - FileUtils.getRegexFileFilter(".*\\.((c?xml)|(txt))$"); + public static final IOFileFilter EDIT_FILTER = FileUtils + .getRegexFileFilter(".*\\.((c?xml)|(txt))$"); @SuppressWarnings("unused") - private static final Logger logger = Logger.getLogger(JobResource.class.getName()); + private static final Logger logger = Logger.getLogger(JobResource.class + .getName()); - protected CrawlJob cj; - - public JobResource(Context ctx, Request req, Response res) throws ResourceException { + protected CrawlJob cj; + + public JobResource(Context ctx, Request req, Response res) + throws ResourceException { super(ctx, req, res); setModifiable(true); getVariants().add(new Variant(MediaType.TEXT_HTML)); getVariants().add(new Variant(MediaType.APPLICATION_XML)); - cj = getEngine().getJob(TextUtils.urlUnescape((String)req.getAttributes().get("job"))); + cj = getEngine().getJob( + TextUtils.urlUnescape((String) req.getAttributes().get("job"))); } public Representation represent(Variant variant) throws ResourceException { - if(cj==null) { + if (cj == null) { throw new ResourceException(404); } @@ -91,17 +93,18 @@ public class JobResource extends BaseResource { if (variant.getMediaType() == MediaType.APPLICATION_XML) { representation = new WriterRepresentation(MediaType.APPLICATION_XML) { public void write(Writer writer) throws IOException { - XmlMarshaller.marshalDocument(writer, "job", makePresentableMap()); - } + XmlMarshaller.marshalDocument(writer, "job", + makePresentableMap()); + } }; } else { representation = new WriterRepresentation(MediaType.TEXT_HTML) { - public void write(Writer writer) throws IOException { - JobResource.this.writeHtml(writer); - } - }; + public void write(Writer writer) throws IOException { + JobResource.this.writeHtml(writer); + } + }; } - + // TODO: remove if not necessary in future? // honor requested charset? representation.setCharacterSet(CharacterSet.UTF_8); @@ -115,20 +118,21 @@ public class JobResource extends BaseResource { * * @return the nested Map data structure */ - protected LinkedHashMap makePresentableMap() { - LinkedHashMap info = new LinkedHashMap(); + protected LinkedHashMap makePresentableMap() { + LinkedHashMap info = new LinkedHashMap(); String baseRef = getRequest().getResourceRef().getBaseRef().toString(); if (!baseRef.endsWith("/")) { baseRef += "/"; } Reference baseRefRef = new Reference(baseRef); - + info.put("shortName", cj.getShortName()); if (cj.getCrawlController() != null) { info.put("crawlControllerState", cj.getCrawlController().getState()); if (cj.getCrawlController().getState() == State.FINISHED) { - info.put("crawlExitStatus", cj.getCrawlController().getCrawlExitStatus()); + info.put("crawlExitStatus", cj.getCrawlController() + .getCrawlExitStatus()); } } info.put("statusDescription", cj.getJobStatusDescription()); @@ -139,16 +143,18 @@ public class JobResource extends BaseResource { info.put("isProfile", cj.isProfile()); File primaryConfig = FileUtils.tryToCanonicalize(cj.getPrimaryConfig()); info.put("primaryConfig", primaryConfig.getAbsolutePath()); - info.put("primaryConfigUrl", baseRef + "jobdir/" + primaryConfig.getName()); + info.put("primaryConfigUrl", + baseRef + "jobdir/" + primaryConfig.getName()); + + if (cj.getJobLog().exists()) + try { + List logLines = new LinkedList(); + FileUtils.pagedLines(cj.getJobLog(), -1, -5, logLines); + info.put("jobLogTail", logLines); + } catch (IOException ioe) { + throw new RuntimeException(ioe); + } - if (cj.getJobLog().exists()) try { - List logLines = new LinkedList(); - FileUtils.pagedLines(cj.getJobLog(), -1, -5, logLines); - info.put("jobLogTail", logLines); - } catch (IOException ioe) { - throw new RuntimeException(ioe); - } - if (cj.hasApplicationContext()) { info.put("uriTotalsReport", cj.uriTotalsReportData()); info.put("sizeTotalsReport", cj.sizeTotalsReportData()); @@ -158,28 +164,32 @@ public class JobResource extends BaseResource { info.put("threadReport", cj.threadReportData()); info.put("frontierReport", cj.frontierReportData()); info.put("heapReport", getEngine().heapReportData()); - - if ((cj.isRunning() || (cj.hasApplicationContext() && !cj.isLaunchable())) - && cj.getCrawlController().getLoggerModule().getCrawlLogPath().getFile().exists()) { + + if ((cj.isRunning() || (cj.hasApplicationContext() && !cj + .isLaunchable())) + && cj.getCrawlController().getLoggerModule() + .getCrawlLogPath().getFile().exists()) { try { List logLines = new LinkedList(); - FileUtils.pagedLines( - cj.getCrawlController().getLoggerModule().getCrawlLogPath().getFile(), - -1, -10, logLines); + FileUtils.pagedLines(cj.getCrawlController() + .getLoggerModule().getCrawlLogPath().getFile(), -1, + -10, logLines); info.put("crawlLogTail", logLines); } catch (IOException ioe) { - throw new RuntimeException(ioe); + throw new RuntimeException(ioe); } } } - List> configFiles = new LinkedList>(); - for(String cppp: cj.getConfigPaths().keySet()) { - Map configFileInfo = new LinkedHashMap(); + List> configFiles = new LinkedList>(); + for (String cppp : cj.getConfigPaths().keySet()) { + Map configFileInfo = new LinkedHashMap(); configFileInfo.put("key", cppp); - File path = FileUtils.tryToCanonicalize(cj.getConfigPaths().get(cppp).getFile()); + File path = FileUtils.tryToCanonicalize(cj.getConfigPaths() + .get(cppp).getFile()); configFileInfo.put("path", path.getAbsolutePath()); - Reference urlRef = new Reference(baseRefRef, getHrefPath(path, cj)).getTargetRef(); + Reference urlRef = new Reference(baseRefRef, getHrefPath(path, cj)) + .getTargetRef(); configFileInfo.put("url", urlRef.toString()); configFiles.add(configFileInfo); } @@ -190,7 +200,7 @@ public class JobResource extends BaseResource { protected Set getAvailableActions() { Set actions = new LinkedHashSet(); - + if (!cj.hasApplicationContext()) { actions.add("build"); } @@ -212,155 +222,146 @@ public class JobResource extends BaseResource { if (cj.hasApplicationContext()) { actions.add("teardown"); } - + return actions; } - + protected void writeHtml(Writer writer) { - PrintWriter pw = new PrintWriter(writer); - String jobTitle = cj.getShortName() + " - " - + cj.getJobStatusDescription() - + " - Job main page"; + PrintWriter pw = new PrintWriter(writer); + String jobTitle = cj.getShortName() + " - " + + cj.getJobStatusDescription() + " - Job main page"; String baseRef = getRequest().getResourceRef().getBaseRef().toString(); - if(!baseRef.endsWith("/")) { + if (!baseRef.endsWith("/")) { baseRef += "/"; } // TODO: replace with use a templating system (FreeMarker?) + pw.println(""); pw.println(""); pw.println(""); - pw.println(""+jobTitle+""); - pw.println(""); - pw.println(""); + pw.println("" + jobTitle + ""); + pw.println(""); + pw.println(""); pw.println(""); - pw.print("

    Job "+cj.getShortName()+" ("); - + pw.print("

    Job " + cj.getShortName() + " ("); + if (cj.isLaunchInfoPartial()) { - pw.print("at least "); + pw.print("at least "); } pw.print(cj.getLaunchCount() + " launches"); - if(cj.getLastLaunch()!=null) { - long ago = System.currentTimeMillis() - cj.getLastLaunch().getMillis(); - pw.print(", last "+ArchiveUtils.formatMillisecondsToConventional(ago, 2)+" ago"); + if (cj.getLastLaunch() != null) { + long ago = System.currentTimeMillis() + - cj.getLastLaunch().getMillis(); + pw.print(", last " + + ArchiveUtils.formatMillisecondsToConventional(ago, 2) + + " ago"); } pw.println(")

    "); - + Flash.renderFlashesHTML(pw, getRequest()); - - if(cj.isProfile()) { - pw.print( - "

    As a profile, this job may be built for " + - "testing purposes but not launched. Use the 'copy job to' " + - "functionality at bottom to copy this profile to a " + - "launchable job.

    "); + + if (cj.isProfile()) { + pw.print("

    As a profile, this job may be built for " + + "testing purposes but not launched. Use the 'copy job to' " + + "functionality at bottom to copy this profile to a " + + "launchable job.

    "); } - + // button controls - pw.println("
    "); + pw.println(""); // PREP, LAUNCH + pw.println("
    "); pw.println(""); pw.print(""); pw.print(""); - + // PAUSE, UNPAUSE, CHECKPOINT pw.print(""); pw.print(""); pw.print(""); pw.print(""); pw.println(""); - + // TERMINATE, RESET - pw.print("\n"); pw.print(""); pw.println(""); + pw.println("
    "); - // display checkpoint options - if(cj.getCheckpointService()!=null) { - Checkpoint recoveryCheckpoint = cj.getCheckpointService().getRecoveryCheckpoint(); - if(recoveryCheckpoint!=null) { - pw.println("
    recover from "+recoveryCheckpoint.getName()+""); - } else if (cj.getCheckpointService().hasAvailableCheckpoints() && cj.isLaunchable()) { - pw.println("
    select an available checkpoint before launch to recover:"); + if (cj.getCheckpointService() != null) { + Checkpoint recoveryCheckpoint = cj.getCheckpointService() + .getRecoveryCheckpoint(); + if (recoveryCheckpoint != null) { + pw.println("
    recover from " + + recoveryCheckpoint.getName() + "
    "); + } else if (cj.getCheckpointService().hasAvailableCheckpoints() + && cj.isLaunchable()) { + pw.println("
    select an available checkpoint before launch to recover:"); pw.println(""); + pw.println("
    "); } } - - pw.println("
    "); - - // configuration + pw.println(""); + + // configuration pw.print("
    configuration: "); printLinkedFile(pw, cj.getPrimaryConfig()); pw.println("
    "); - for(File f : cj.getImportedConfigs(cj.getPrimaryConfig())) { + for (File f : cj.getImportedConfigs(cj.getPrimaryConfig())) { pw.print("
    imported: "); - printLinkedFile(pw,f); + printLinkedFile(pw, f); pw.println("
    "); } - -// if(cj.isXmlOk()) { -// pw.println("cxml ok
    "); -// if(cj.isContainerOk()) { -// pw.println("container ok
    "); -// if(cj.isContainerValidated()) { -// pw.println("config valid
    "); -// } else { -// pw.println("CONFIG INVALID
    "); -// } -// } else { -// pw.println("CONTAINER BAD
    "); -// } -// }else { -// // pw.println("XML NOT WELL-FORMED
    "); -// } pw.print("

    Job Log "); - pw.print("(more)"); + pw.print("(more)"); pw.println("

    "); pw.println("
    "); - if(cj.getJobLog().exists()) { + if (cj.getJobLog().exists()) { try { List logLines = new LinkedList(); FileUtils.pagedLines(cj.getJobLog(), -1, -5, logLines); Collections.reverse(logLines); - for(String line : logLines) { + for (String line : logLines) { pw.print("
    "); - StringEscapeUtils.escapeHtml(pw,line); + StringEscapeUtils.escapeHtml(pw, line); pw.println("
    "); } } catch (IOException ioe) { @@ -368,69 +369,66 @@ public class JobResource extends BaseResource { } } pw.println("
    "); - - pw.println("

    Job is "+cj.getJobStatusDescription()+"

    "); - if(cj.hasApplicationContext()) { + pw.println("

    Job is " + cj.getJobStatusDescription() + "

    "); + + if (cj.hasApplicationContext()) { pw.println("
    "); - + pw.println("
    Totals
    "); - pw.println("
    " + cj.uriTotalsReport() + "
    "); - pw.println(cj.sizeTotalsReport() + "
    "); - + pw.println("
    " + cj.uriTotalsReport() + "
    "); + pw.println("
    " + cj.sizeTotalsReport() + "
    "); + pw.println("
    Alerts
    "); pw.print("
    "); - pw.println(cj.getAlertCount() == 0 ? "none" : cj.getAlertCount()); + pw.println(cj.getAlertCount() == 0 ? "none" : cj + .getAlertCount()); if (cj.getAlertCount() > 0) { - printLinkedFile( - pw, - cj.getCrawlController().getLoggerModule().getAlertsLogPath().getFile(), - "tail alert log...", - "format=paged&pos=-1&lines=-128"); + printLinkedFile(pw, cj.getCrawlController().getLoggerModule() + .getAlertsLogPath().getFile(), "tail alert log...", + "format=paged&pos=-1&lines=-128"); } pw.println("
    "); - + pw.println("
    Rates
    "); pw.println("
    " + cj.rateReport() + "
    "); - + pw.println("
    Load
    "); pw.println("
    " + cj.loadReport() + "
    "); - + pw.println("
    Elapsed
    "); pw.println("
    " + cj.elapsedReport() + "
    "); - + pw.println("
    Threads
    "); pw.println("
    " + cj.threadReport() + " 
    "); - + pw.println("
    Frontier
    "); pw.println("
    " + cj.frontierReport() + " 
    "); - + pw.println("
    Memory
    "); pw.println("
    " + getEngine().heapReport() + "
    "); pw.println("
    "); - - if ((cj.isRunning() || (cj.hasApplicationContext() && !cj.isLaunchable())) - && cj.getCrawlController().getLoggerModule().getCrawlLogPath().getFile().exists()) { + + if ((cj.isRunning() || (cj.hasApplicationContext() && !cj + .isLaunchable())) + && cj.getCrawlController().getLoggerModule() + .getCrawlLogPath().getFile().exists()) { // show crawl log for running or finished crawls pw.print("

    Crawl Log "); - printLinkedFile( - pw, - cj.getCrawlController().getLoggerModule().getCrawlLogPath().getFile(), - "more", - "format=paged&pos=-1&lines=-128&reverse=y"); + printLinkedFile(pw, cj.getCrawlController().getLoggerModule() + .getCrawlLogPath().getFile(), "more", + "format=paged&pos=-1&lines=-128&reverse=y"); pw.println("

    "); pw.println("
    ");
                 try {
                     List logLines = new LinkedList();
-                    FileUtils.pagedLines(
-                            cj.getCrawlController().getLoggerModule().getCrawlLogPath().getFile(),
-                            -1, 
-                            -10, 
-                            logLines);
+                    FileUtils.pagedLines(cj.getCrawlController()
+                            .getLoggerModule().getCrawlLogPath().getFile(), -1,
+                            -10, logLines);
                     Collections.reverse(logLines);
-                    for(String line : logLines) {
-                        StringEscapeUtils.escapeHtml(pw,line);
+                    for (String line : logLines) {
+                        StringEscapeUtils.escapeHtml(pw, line);
                         pw.println();
                     }
                 } catch (IOException ioe) {
@@ -438,38 +436,42 @@ public class JobResource extends BaseResource {
                 }
                 pw.println("
    "); } - + } - - if(cj.hasApplicationContext()) { + + if (cj.hasApplicationContext()) { pw.println("

    Reports

    "); - for (Report report: cj.getCrawlController().getStatisticsTracker().getReports()) { + for (Report report : cj.getCrawlController().getStatisticsTracker() + .getReports()) { if (report.getShouldReportDuringCrawl()) { String className = report.getClass().getSimpleName(); - String shortName = className.substring(0,className.length()-"Report".length()); - pw.println(""+shortName+""); + String shortName = className.substring(0, + className.length() - "Report".length()); + pw.println("" + + shortName + ""); } } } - + pw.println("

    Files

    "); pw.println("

    Browse Job Directory

    "); // specific paths from wired context pw.println("

    Configuration-referenced Paths

    "); - if(cj.getConfigPaths().isEmpty()) { + if (cj.getConfigPaths().isEmpty()) { pw.println("build the job to discover referenced paths"); } else { pw.println("
    "); - for(String cppp : cj.getConfigPaths().keySet()) { + for (String cppp : cj.getConfigPaths().keySet()) { ConfigPath cp = cj.getConfigPaths().get(cppp); - pw.println("
    "+cppp+": "+cp.getName()+"
    "); + pw.println("
    " + cppp + ": " + cp.getName() + "
    "); pw.println("
    "); - if(!StringUtils.isEmpty(cp.getPath())) { + if (!StringUtils.isEmpty(cp.getPath())) { printLinkedFile( - pw, - cp.getFile(), + pw, + cp.getFile(), cp.getFile().toString(), - cp.getPath().endsWith(".log")?"format=paged&pos=-1&lines=-128&reverse=y":null); + cp.getPath().endsWith(".log") ? "format=paged&pos=-1&lines=-128&reverse=y" + : null); } else { pw.println("unset"); } @@ -478,24 +480,23 @@ public class JobResource extends BaseResource { pw.println("
    "); } - + pw.println("

    Advanced

    "); pw.println("

    Scripting console

    "); - if(!cj.hasApplicationContext()) { + if (!cj.hasApplicationContext()) { pw.println("build the job to browse bean instances"); } else { pw.println("

    Browse beans

    "); } pw.println("

    Copy

    "); - pw.println( - "
    Copy job to " + - "" + - "" + - "
    "); - //pw.println("
    "); - + pw.println("
    Copy job to " + + "" + + "" + + "
    "); + // pw.println("
    "); + pw.println(""); pw.println(""); } @@ -503,122 +504,134 @@ public class JobResource extends BaseResource { /** * Print a link to the given File * - * @param pw PrintWriter - * @param f File + * @param pw + * PrintWriter + * @param f + * File */ - protected void printLinkedFile(PrintWriter pw, File f) { - printLinkedFile(pw,f,f.toString(),null); + protected void printLinkedFile(PrintWriter pw, File f) { + printLinkedFile(pw, f, f.toString(), null); } - + /** * Print a link to the given File, using the given link text * - * @param pw PrintWriter - * @param f File + * @param pw + * PrintWriter + * @param f + * File */ - protected void printLinkedFile(PrintWriter pw, File f, String linktext, String queryString) { - String relativePath = JobResource.getHrefPath(f,cj); - pw.println("" - + linktext +""); - if(EDIT_FILTER.accept(f)) { - pw.println("[edit]"); + protected void printLinkedFile(PrintWriter pw, File f, String linktext, + String queryString) { + String relativePath = JobResource.getHrefPath(f, cj); + pw.println("" + + linktext + ""); + if (EDIT_FILTER.accept(f)) { + pw.println("[edit]"); } } /** - * Get a usable HrefPath, relative to the JobResource, for the given - * file. Assumes usual helper resources ('jobdir/', 'anypath/') at - * the usual locations. + * Get a usable HrefPath, relative to the JobResource, for the given file. + * Assumes usual helper resources ('jobdir/', 'anypath/') at the usual + * locations. * - * @param f File to provide an href (suitable for clicking or redirection) - * @param cj CrawlJob for calculating jobdir-relative path if possible + * @param f + * File to provide an href (suitable for clicking or redirection) + * @param cj + * CrawlJob for calculating jobdir-relative path if possible * @return String path suitable as href or Location header */ public static String getHrefPath(File f, CrawlJob cj) { String jobDirRelative = cj.jobDirRelativePath(f); - if(jobDirRelative!=null) { - return "jobdir/"+jobDirRelative; + if (jobDirRelative != null) { + return "jobdir/" + jobDirRelative; } // TODO: delegate this to EngineApplication, or make // conditional on whether /anypath/ service is present? String fullPath = f.getAbsolutePath(); fullPath = fullPath.replace(File.separatorChar, '/'); - return "../../anypath/"+fullPath; + return "../../anypath/" + fullPath; } protected Engine getEngine() { - return ((EngineApplication)getApplication()).getEngine(); + return ((EngineApplication) getApplication()).getEngine(); } @Override - public void acceptRepresentation(Representation entity) throws ResourceException { + public void acceptRepresentation(Representation entity) + throws ResourceException { if (cj == null) { throw new ResourceException(404); } // copy op? Form form = null; - form = getRequest().getEntityAsForm(); + form = getRequest().getEntityAsForm(); String copyTo = form.getFirstValue("copyTo"); - if(copyTo!=null) { - copyJob(copyTo,"on".equals(form.getFirstValue("asProfile"))); + if (copyTo != null) { + copyJob(copyTo, "on".equals(form.getFirstValue("asProfile"))); return; } AlertHandler.ensureStaticInitialization(); AlertThreadGroup.setThreadLogger(cj.getJobLogger()); String action = form.getFirstValue("action"); - if("launch".equals(action)) { + if ("launch".equals(action)) { String selectedCheckpoint = form.getFirstValue("checkpoint"); - if(StringUtils.isNotEmpty(selectedCheckpoint)) { - cj.getCheckpointService().setRecoveryCheckpointByName(selectedCheckpoint); + if (StringUtils.isNotEmpty(selectedCheckpoint)) { + cj.getCheckpointService().setRecoveryCheckpointByName( + selectedCheckpoint); } - cj.launch(); - } else if("checkXML".equals(action)) { + cj.launch(); + } else if ("checkXML".equals(action)) { cj.checkXML(); - } else if("instantiate".equals(action)) { + } else if ("instantiate".equals(action)) { cj.instantiateContainer(); - } else if("build".equals(action)||"validate".equals(action)) { + } else if ("build".equals(action) || "validate".equals(action)) { cj.validateConfiguration(); - } else if("teardown".equals(action)) { - if(!cj.teardown()) { - Flash.addFlash(getResponse(), "waiting for job to finish", Flash.Kind.NACK); + } else if ("teardown".equals(action)) { + if (!cj.teardown()) { + Flash.addFlash(getResponse(), "waiting for job to finish", + Flash.Kind.NACK); } - } else if("pause".equals(action)) { + } else if ("pause".equals(action)) { cj.getCrawlController().requestCrawlPause(); - } else if("unpause".equals(action)) { + } else if ("unpause".equals(action)) { cj.getCrawlController().requestCrawlResume(); - } else if("checkpoint".equals(action)) { + } else if ("checkpoint".equals(action)) { String cp = cj.getCheckpointService().requestCrawlCheckpoint(); - if(StringUtils.isNotEmpty(cp)) { - Flash.addFlash(getResponse(), "Checkpoint "+cp+" saved",Flash.Kind.ACK); + if (StringUtils.isNotEmpty(cp)) { + Flash.addFlash(getResponse(), "Checkpoint " + cp + + " saved", Flash.Kind.ACK); } else { - Flash.addFlash(getResponse(), "Checkpoint not made -- perhaps no progress since last? (see logs)",Flash.Kind.NACK); + Flash.addFlash( + getResponse(), + "Checkpoint not made -- perhaps no progress since last? (see logs)", + Flash.Kind.NACK); } - } else if("terminate".equals(action)) { + } else if ("terminate".equals(action)) { cj.terminate(); } AlertThreadGroup.setThreadLogger(null); - + // default: redirect to GET self getResponse().redirectSeeOther(getRequest().getOriginalRef()); } - protected void copyJob(String copyTo, boolean asProfile) throws ResourceException { + protected void copyJob(String copyTo, boolean asProfile) + throws ResourceException { try { getEngine().copy(cj, copyTo, asProfile); } catch (IOException e) { - Flash.addFlash(getResponse(), "Job not copied: "+e.getMessage(), Flash.Kind.NACK); + Flash.addFlash(getResponse(), "Job not copied: " + e.getMessage(), + Flash.Kind.NACK); getResponse().redirectSeeOther(getRequest().getOriginalRef()); return; } // redirect to destination job page getResponse().redirectSeeOther(copyTo); } - - + } diff --git a/engine/src/main/java/org/archive/crawler/restlet/ScriptResource.java b/engine/src/main/java/org/archive/crawler/restlet/ScriptResource.java index d1dbc727..82dbe5be 100644 --- a/engine/src/main/java/org/archive/crawler/restlet/ScriptResource.java +++ b/engine/src/main/java/org/archive/crawler/restlet/ScriptResource.java @@ -244,7 +244,8 @@ public class ScriptResource extends JobRelatedResource { protected void writeHtml(Writer writer) { String baseRef = getRequest().getResourceRef().getBaseRef().toString(); if (!baseRef.endsWith("/")) baseRef += "/"; - PrintWriter pw = new PrintWriter(writer); + PrintWriter pw = new PrintWriter(writer); + pw.println(""); pw.println(""); pw.println(""); pw.println("Script in "+cj.getShortName()+""); From 1a333fbd354df09daa73535b0ca3e0736babde1e Mon Sep 17 00:00:00 2001 From: Adam Miller Date: Fri, 16 Mar 2012 14:36:15 -0700 Subject: [PATCH 14/23] HER-1998 - ExtractorHTML modified to parse html inside conditional comments. Still ignores normal comments --- .../modules/extractor/ExtractorHTML.java | 2 +- .../modules/extractor/ExtractorHTMLTest.java | 28 +++++++++++++++++++ 2 files changed, 29 insertions(+), 1 deletion(-) diff --git a/modules/src/main/java/org/archive/modules/extractor/ExtractorHTML.java b/modules/src/main/java/org/archive/modules/extractor/ExtractorHTML.java index 9122cf18..d0b339c4 100644 --- a/modules/src/main/java/org/archive/modules/extractor/ExtractorHTML.java +++ b/modules/src/main/java/org/archive/modules/extractor/ExtractorHTML.java @@ -114,7 +114,7 @@ public class ExtractorHTML extends ContentExtractor implements InitializingBean "(?is)<(?:((script[^>]*+)>.*?]*+)>.*?]*+)" + // 5, 6, 7 - "|(!--.*?--))>"; // 8 + "|(!--(?!\\[if).*?--))>"; // 8 // version w/ problems with unclosed script tags // static final String RELEVANT_TAG_EXTRACTOR = diff --git a/modules/src/test/java/org/archive/modules/extractor/ExtractorHTMLTest.java b/modules/src/test/java/org/archive/modules/extractor/ExtractorHTMLTest.java index fae65d52..5fc684e8 100644 --- a/modules/src/test/java/org/archive/modules/extractor/ExtractorHTMLTest.java +++ b/modules/src/test/java/org/archive/modules/extractor/ExtractorHTMLTest.java @@ -66,6 +66,7 @@ public class ExtractorHTMLTest extends StringExtractorTestBase { " IMG", "http://www.archive.org/start/foo.gif", + }; @@ -379,5 +380,32 @@ public class ExtractorHTMLTest extends StringExtractorTestBase { assertTrue("outlinks should contain: "+expected, CollectionUtils.exists(curi.getOutLinks(),destinationsIsPredicate(expected))); } + + /** + * HER-1998 + * @throws URIException + */ + public void testConditionalComment1() throws URIException { + CrawlURI curi = new CrawlURI(UURIFactory.getInstance("http://www.example.com/")); + + CharSequence cs = + "" + + ""; + + ExtractorHTML extractor = (ExtractorHTML)makeExtractor(); + extractor.extract(curi, cs); + + Link[] links = curi.getOutLinks().toArray(new Link[0]); + Arrays.sort(links); + + String dest1 = "http://www.example.com/foo.gif"; + String dest2 = "http://www.example.com/foo.js"; + + assertEquals("outlink1 from conditional comment img src",dest1, + links[0].getDestination().toString()); + assertEquals("outlink2 from conditional comment script src",dest2, + links[1].getDestination().toString()); + + } } From ea42c0c653ca93b27088f5b0d630e16568ade490 Mon Sep 17 00:00:00 2001 From: Noah Levitt Date: Mon, 19 Mar 2012 17:30:18 -0700 Subject: [PATCH 15/23] * EngineResource.java acceptRepresentation() - "I'm sure" parameter is now "im_sure" --- .../main/java/org/archive/crawler/restlet/EngineResource.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/engine/src/main/java/org/archive/crawler/restlet/EngineResource.java b/engine/src/main/java/org/archive/crawler/restlet/EngineResource.java index 3a483880..e7b12e1f 100644 --- a/engine/src/main/java/org/archive/crawler/restlet/EngineResource.java +++ b/engine/src/main/java/org/archive/crawler/restlet/EngineResource.java @@ -145,7 +145,7 @@ public class EngineResource extends BaseResource { } } else if ("Exit Java Process".equals(action)) { boolean cancel = false; - if(!"on".equals(form.getFirstValue("I'm sure"))) { + if(!"on".equals(form.getFirstValue("im_sure"))) { Flash.addFlash( getResponse(), "You must tick \"I'm sure\" to trigger exit", From a108795d50c0e6bb4d516f79eb20729b7bffea7d Mon Sep 17 00:00:00 2001 From: Adam Miller Date: Mon, 19 Mar 2012 18:19:00 -0700 Subject: [PATCH 16/23] HER-1998 - Adding comments, adjusting test case to prevent subclasses from failing new tests --- .../modules/extractor/ExtractorHTML.java | 33 ++++++++++++------- .../modules/extractor/ExtractorHTMLTest.java | 13 ++++++-- 2 files changed, 31 insertions(+), 15 deletions(-) diff --git a/modules/src/main/java/org/archive/modules/extractor/ExtractorHTML.java b/modules/src/main/java/org/archive/modules/extractor/ExtractorHTML.java index d0b339c4..19539279 100644 --- a/modules/src/main/java/org/archive/modules/extractor/ExtractorHTML.java +++ b/modules/src/main/java/org/archive/modules/extractor/ExtractorHTML.java @@ -76,9 +76,29 @@ public class ExtractorHTML extends ContentExtractor implements InitializingBean public final static String A_META_ROBOTS = "meta-robots"; + { + setMaxElementLength(64); + } + public int getMaxElementLength() { + return (Integer) kp.get("maxElementLength"); + } + public void setMaxElementLength(int max) { + kp.put("maxElementLength",max); + } + /** * Compiled relevant tag extractor. + * + * HER-1998 - Modified part 8 to allow conditional html comments. + * Conditional HTML comment example: + * "" + * + * This technique is commonly used to reference CSS & JavaScript that are designed to deal with the quirks of a specific version of Internet Explorer. + * There is another syntax for conditional comments which already gets parsed by the regex since it doesn't start with "" + ""; - - ExtractorHTML extractor = (ExtractorHTML)makeExtractor(); + + ExtractorHTML extractor = new ExtractorHTML(); + UriErrorLoggerModule ulm = new UnitTestUriLoggerModule(); + extractor.setLoggerModule(ulm); + CrawlMetadata metadata = new CrawlMetadata(); + metadata.afterPropertiesSet(); + extractor.setMetadata(metadata); + extractor.afterPropertiesSet(); + extractor.extract(curi, cs); Link[] links = curi.getOutLinks().toArray(new Link[0]); From 335b32aecc1d21f9096e6ed8a4a0aebd25436df1 Mon Sep 17 00:00:00 2001 From: Noah Levitt Date: Tue, 20 Mar 2012 15:14:14 -0700 Subject: [PATCH 17/23] * ExtractorHTML.java some fixup on RELEVANT_TAG_EXTRACTOR javadoc comment --- .../modules/extractor/ExtractorHTML.java | 62 ++++++++++++------- 1 file changed, 38 insertions(+), 24 deletions(-) diff --git a/modules/src/main/java/org/archive/modules/extractor/ExtractorHTML.java b/modules/src/main/java/org/archive/modules/extractor/ExtractorHTML.java index 19539279..e5d4ada3 100644 --- a/modules/src/main/java/org/archive/modules/extractor/ExtractorHTML.java +++ b/modules/src/main/java/org/archive/modules/extractor/ExtractorHTML.java @@ -88,35 +88,49 @@ public class ExtractorHTML extends ContentExtractor implements InitializingBean /** - * Compiled relevant tag extractor. + * Relevant tag extractor. * - * HER-1998 - Modified part 8 to allow conditional html comments. - * Conditional HTML comment example: - * "" - * - * This technique is commonly used to reference CSS & JavaScript that are designed to deal with the quirks of a specific version of Internet Explorer. - * There is another syntax for conditional comments which already gets parsed by the regex since it doesn't start with "" + *

    + * + *

    + * This technique is commonly used to reference CSS & JavaScript that + * are designed to deal with the quirks of a specific version of Internet + * Explorer. There is another syntax for conditional comments which already + * gets parsed by the regex since it doesn't start with "<!--" Ex. + * <!if expression> HTML <!endif> + *

    + * + *

    + * https://en.wikipedia.org/wiki/Conditional_Comments + *

    */ // version w/ less unnecessary backtracking static final String RELEVANT_TAG_EXTRACTOR = From 9626b8454a594276728243a77ee85d4a7014a1b1 Mon Sep 17 00:00:00 2001 From: Noah Levitt Date: Fri, 30 Mar 2012 11:44:28 -0700 Subject: [PATCH 18/23] * EditRepresentation.java make edit page (for cxml mainly) prettier with big textarea (see Kenji's comment on HER-1993 dated 2012-03-27 18:03) --- .../crawler/restlet/EditRepresentation.java | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/engine/src/main/java/org/archive/crawler/restlet/EditRepresentation.java b/engine/src/main/java/org/archive/crawler/restlet/EditRepresentation.java index 4adc7c04..5ccfa4e9 100644 --- a/engine/src/main/java/org/archive/crawler/restlet/EditRepresentation.java +++ b/engine/src/main/java/org/archive/crawler/restlet/EditRepresentation.java @@ -64,20 +64,21 @@ public class EditRepresentation extends CharacterRepresentation { pw.println(""); pw.println(""); pw.println(""+fileRepresentation.getFile().getName()+""); - pw.println(""); - Flash.renderFlashesHTML(pw, dirResource.getRequest()); - pw.println("
    "); - // TODO: enable button on after changes made + pw.println(""); + pw.println(""); + pw.println(""); pw.println("
    "); + // TODO: enable button on after changes made pw.println(""); pw.println(fileRepresentation.getFile()); Reference viewRef = dirResource.getRequest().getOriginalRef().clone(); viewRef.setQuery(null); pw.println("view"); + Flash.renderFlashesHTML(pw, dirResource.getRequest()); pw.println("
    "); - pw.println("
    "); + pw.println(""); pw.println(""); pw.println(""); pw.close(); From f9f9e62f4c52104ea3379f560b1fe0f1cc28da11 Mon Sep 17 00:00:00 2001 From: Travis Wellman Date: Mon, 2 Apr 2012 18:16:23 -0700 Subject: [PATCH 19/23] Reject or accept matching a set of IP addresses. --- .gitignore | 7 ++ .../deciderules/IpAddressSetDecideRule.java | 87 +++++++++++++++++++ 2 files changed, 94 insertions(+) create mode 100644 .gitignore create mode 100644 modules/src/main/java/org/archive/modules/deciderules/IpAddressSetDecideRule.java diff --git a/.gitignore b/.gitignore new file mode 100644 index 00000000..53618d72 --- /dev/null +++ b/.gitignore @@ -0,0 +1,7 @@ +*.class +bin +commons/target +dist/target +engine/target +modules/target +hq/target diff --git a/modules/src/main/java/org/archive/modules/deciderules/IpAddressSetDecideRule.java b/modules/src/main/java/org/archive/modules/deciderules/IpAddressSetDecideRule.java new file mode 100644 index 00000000..61a7a905 --- /dev/null +++ b/modules/src/main/java/org/archive/modules/deciderules/IpAddressSetDecideRule.java @@ -0,0 +1,87 @@ +package org.archive.modules.deciderules; + +import static org.archive.modules.CoreAttributeConstants.A_DNS_SERVER_IP_LABEL; + +import java.net.InetAddress; +import java.util.Collections; +import java.util.Set; + +import org.archive.modules.CrawlURI; +import org.archive.modules.net.CrawlHost; +import org.archive.modules.net.ServerCache; +import org.springframework.beans.factory.annotation.Autowired; + +/** + * 
    + * <bean class="org.archive.modules.deciderules.IpAddressSetDecideRule">
+ *  <property name="ipAddresses">
+ *   <set>
+ *    <value>127.0.0.1</value>
+ *    <value>69.89.27.209</value>
+ *   </set>
+ *  </property>
+ *  <property name='decision' value='REJECT' />
+ * </bean>
+ *
    + * + * @contributor Travis Wellman + */ + +class IpAddressSetDecideRule extends PredicatedDecideRule { + +// private static final Logger LOGGER = Logger.getLogger(IpAddressSetDecideRule.class.getCanonicalName()); + private static final long serialVersionUID = -3670434739183271441L; + private Set ipAddresses; + + /** + * @return the addresses being matched + */ + public Set getIpAddresses() { + return Collections.unmodifiableSet(ipAddresses); + } + + /** + * @param ipAddresses the addresses to match + */ + public void setIpAddresses(Set ipAddresses) { + this.ipAddresses = ipAddresses; + } + + @Override + protected boolean evaluate(CrawlURI curi) { + String hostAddress = getHostAddress(curi); + System.out.println("hostAddress for "+ curi +" "+ hostAddress); + return hostAddress != null && + ipAddresses.contains(hostAddress.intern()); + } + + transient protected ServerCache serverCache; + public ServerCache getServerCache() { + return this.serverCache; + } + @Autowired + public void setServerCache(ServerCache serverCache) { + this.serverCache = serverCache; + } + + /** + * from WriterPoolProcessor + * + * @param curi CrawlURI + * @return String of IP address + */ + protected String getHostAddress(CrawlURI curi) { + // special handling for DNS URIs: want address of DNS server + if (curi.getUURI().getScheme().toLowerCase().equals("dns")) { + return (String)curi.getData().get(A_DNS_SERVER_IP_LABEL); + } + // otherwise, host referenced in URI + // TODO:FIXME: have fetcher insert exact IP contacted into curi, + // use that rather than inferred by CrawlHost lookup + CrawlHost crlh = getServerCache().getHostFor(curi.getUURI()); + if (crlh == null) { return null; } + InetAddress inetadd = crlh.getIP(); + if (inetadd == null) { return null; } + return inetadd.getHostAddress(); + } +} From 64e9cc367b7166bad1f8a76a9701f2e0b0de3d8c Mon Sep 17 00:00:00 2001 From: Travis Wellman Date: Thu, 12 Apr 2012 16:46:16 -0700 Subject: [PATCH 20/23] remove println, make class public --- .../archive/modules/deciderules/IpAddressSetDecideRule.java | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/modules/src/main/java/org/archive/modules/deciderules/IpAddressSetDecideRule.java b/modules/src/main/java/org/archive/modules/deciderules/IpAddressSetDecideRule.java index 61a7a905..53c8c682 100644 --- a/modules/src/main/java/org/archive/modules/deciderules/IpAddressSetDecideRule.java +++ b/modules/src/main/java/org/archive/modules/deciderules/IpAddressSetDecideRule.java @@ -27,7 +27,7 @@ import org.springframework.beans.factory.annotation.Autowired; * @contributor Travis Wellman */ -class IpAddressSetDecideRule extends PredicatedDecideRule { +public class IpAddressSetDecideRule extends PredicatedDecideRule { // private static final Logger LOGGER = Logger.getLogger(IpAddressSetDecideRule.class.getCanonicalName()); private static final long serialVersionUID = -3670434739183271441L; @@ -50,7 +50,6 @@ class IpAddressSetDecideRule extends PredicatedDecideRule { @Override protected boolean evaluate(CrawlURI curi) { String hostAddress = getHostAddress(curi); - System.out.println("hostAddress for "+ curi +" "+ hostAddress); return hostAddress != null && ipAddresses.contains(hostAddress.intern()); } From 57d2f4295493f5d08629c9a6265b31837d06a38d Mon Sep 17 00:00:00 2001 From: Travis Wellman Date: Thu, 12 Apr 2012 16:50:22 -0700 Subject: [PATCH 21/23] formatting --- .gitignore | 7 ------- .../modules/deciderules/IpAddressSetDecideRule.java | 8 ++++++-- 2 files changed, 6 insertions(+), 9 deletions(-) delete mode 100644 .gitignore diff --git a/.gitignore b/.gitignore deleted file mode 100644 index 53618d72..00000000 --- a/.gitignore +++ /dev/null @@ -1,7 +0,0 @@ -*.class -bin -commons/target -dist/target -engine/target -modules/target -hq/target diff --git a/modules/src/main/java/org/archive/modules/deciderules/IpAddressSetDecideRule.java b/modules/src/main/java/org/archive/modules/deciderules/IpAddressSetDecideRule.java index 53c8c682..de8546d0 100644 --- a/modules/src/main/java/org/archive/modules/deciderules/IpAddressSetDecideRule.java +++ b/modules/src/main/java/org/archive/modules/deciderules/IpAddressSetDecideRule.java @@ -78,9 +78,13 @@ public class IpAddressSetDecideRule extends PredicatedDecideRule { // TODO:FIXME: have fetcher insert exact IP contacted into curi, // use that rather than inferred by CrawlHost lookup CrawlHost crlh = getServerCache().getHostFor(curi.getUURI()); - if (crlh == null) { return null; } + if (crlh == null) { + return null; + } InetAddress inetadd = crlh.getIP(); - if (inetadd == null) { return null; } + if (inetadd == null) { + return null; + } return inetadd.getHostAddress(); } } From 54b1e245ebe6ecce250d2eaccccceeb19832aae7 Mon Sep 17 00:00:00 2001 From: Travis Wellman Date: Thu, 12 Apr 2012 17:32:57 -0700 Subject: [PATCH 22/23] gitignore for maven build artifacts --- .gitignore | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 .gitignore diff --git a/.gitignore b/.gitignore new file mode 100644 index 00000000..c02256c8 --- /dev/null +++ b/.gitignore @@ -0,0 +1,5 @@ +bin +commons/target +modules/target +engine/target +dist/target From 52eef846587bfae70d82003119d76332709a5ca8 Mon Sep 17 00:00:00 2001 From: Travis Wellman Date: Thu, 12 Apr 2012 17:34:21 -0700 Subject: [PATCH 23/23] note about preselector.recheckScope for use of IpAddressSetDecideRule --- .../archive/modules/deciderules/IpAddressSetDecideRule.java | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/modules/src/main/java/org/archive/modules/deciderules/IpAddressSetDecideRule.java b/modules/src/main/java/org/archive/modules/deciderules/IpAddressSetDecideRule.java index de8546d0..8d51fbd5 100644 --- a/modules/src/main/java/org/archive/modules/deciderules/IpAddressSetDecideRule.java +++ b/modules/src/main/java/org/archive/modules/deciderules/IpAddressSetDecideRule.java @@ -12,6 +12,11 @@ import org.archive.modules.net.ServerCache; import org.springframework.beans.factory.annotation.Autowired; /** + * IpAddressSetDecideRule must be used with + * {@link org.archive.crawler.prefetch.Preselector#setRecheckScope(boolean)} set + * to true because it relies on Heritrix' dns lookup to establish the ip address + * for a URI before it can run. + * * 
      * <bean class="org.archive.modules.deciderules.IpAddressSetDecideRule">
  *  <property name="ipAddresses">