From 1a136b21d57ba02380762a36d7f00ec82d3f22f2 Mon Sep 17 00:00:00 2001
From: szznax <szznax@daa5b2f2-a927-0410-8b2d-f5f262fa301a>
Date: Thu, 8 Oct 2009 23:55:42 +0000
Subject: [PATCH] Fix for [HER-1419] can't create job in web ui without
 profile/job to base it on in jobs dir? * Engine.java  
 createNewJobWithDefaults() to write profile-crawler-beans.cxml resoure into
 new job * EngineResource.java   added FORM for new job dir and "create"
 action * profile-crawler-beans.cxml   added resource from
 dist/src/main/conf/jobs/profile-defaults to bootstrap new jobs

---
 .../org/archive/crawler/framework/Engine.java |  23 +
 .../crawler/restlet/EngineResource.java       |  37 ++
 .../restlet/profile-crawler-beans.cxml        | 523 ++++++++++++++++++
 3 files changed, 583 insertions(+)
 create mode 100644 engine/src/main/java/org/archive/crawler/restlet/profile-crawler-beans.cxml

diff --git a/engine/src/main/java/org/archive/crawler/framework/Engine.java b/engine/src/main/java/org/archive/crawler/framework/Engine.java
index fee3ef86..7b68c9c0 100644
--- a/engine/src/main/java/org/archive/crawler/framework/Engine.java
+++ b/engine/src/main/java/org/archive/crawler/framework/Engine.java
@@ -23,12 +23,14 @@ import java.io.File;
 import java.io.FileFilter;
 import java.io.FilenameFilter;
 import java.io.IOException;
+import java.io.InputStream;
 import java.util.HashMap;
 import java.util.Map;
 import java.util.logging.Level;
 import java.util.logging.Logger;
 
 import org.apache.commons.io.FileUtils;
+import org.apache.commons.io.IOUtils;
 import org.apache.commons.io.filefilter.FileFilterUtils;
 import org.archive.util.ArchiveUtils;
 
@@ -272,5 +274,26 @@ public class Engine {
         return false; 
     }
 
+	public boolean createNewJobWithDefaults(String path) throws IOException {
+
+		File newJobDir = new File(jobsDir,"/"+path);
+		if (newJobDir.exists()) {
+		  throw new IOException("file exists: "+newJobDir);
+		}
+		newJobDir.mkdirs();
+
+		// get crawler-beans template from this package into string 
+		InputStream inStream = getClass().getResourceAsStream(
+			"/org/archive/crawler/restlet/profile-crawler-beans.cxml");
+		String defaultCxmlStr = IOUtils.toString(inStream);
+		inStream.close(); 
+		
+		// write default crawler-beans string to new job config
+		File newJobCxml = new File(newJobDir,"crawler-beans.cxml");
+		FileUtils.writeStringToFile(newJobCxml, defaultCxmlStr);
+
+		return true;
+	}
+
    
 }
\ No newline at end of file
diff --git a/engine/src/main/java/org/archive/crawler/restlet/EngineResource.java b/engine/src/main/java/org/archive/crawler/restlet/EngineResource.java
index afa7ed4b..c87a9661 100644
--- a/engine/src/main/java/org/archive/crawler/restlet/EngineResource.java
+++ b/engine/src/main/java/org/archive/crawler/restlet/EngineResource.java
@@ -85,6 +85,35 @@ public class EngineResource extends Resource {
             if(!added) {
                 Flash.addFlash(getResponse(), "not a valid preexisting job directory: \""+path+"\"", Flash.Kind.NACK);
             }
+        } else if ("create".equals(action)) {
+        	String errStyle = "style=\"margin:1em;padding:0.2em 1em;background:pink;\"";
+        	String warnStyle = errStyle;
+        	String msgStyle = "style=\"margin:1em;padding:0.2em 1em;background:khaki;\"";
+        	String path = form.getFirstValue("addpath");
+        	if (path.indexOf("/") != -1) {
+        		String warn = "<div "+warnStyle+">WARNING: "
+        			+ "no job created. sub-directories disallowed: "
+        			+ "<i>" + path + "</i></div>\n";
+                Flash.addFlash(getResponse(), warn, Flash.Kind.NACK);
+                System.err.println(warn);
+        	} else {
+        	boolean created = false;
+        	try {
+        		created = getEngine().createNewJobWithDefaults(path);
+			} catch (IOException e) {
+				String err = "<div "+errStyle+">ERROR! failed to create new job: "
+					+ "<i>" + path + "</i> "+ e.toString() + "</div>\n";
+                Flash.addFlash(getResponse(), err, Flash.Kind.NACK);
+                System.err.println(err);
+                
+			}
+			if (created) {
+				String msg = "<p "+msgStyle+">Successfully created job: " 
+					+ "<i>" + path + "</i></p>\n";
+				Flash.addFlash(getResponse(), msg, Flash.Kind.NACK);
+				getEngine().findJobConfigs();
+			}
+        	}
         }
         // default: redirect to GET self
         getResponse().redirectSeeOther(getRequest().getOriginalRef());
@@ -137,6 +166,14 @@ public class EngineResource extends Resource {
             "'rescan' button above. Or, supply a full path to another " +
             "valid job directory at the engine machine below.<br/><br/>");
         
+        // create new job with defaults
+        pw.println("<form method=\'POST\'>\n"
+        		+ "Create new job with recommended defaults<br />\n"
+        		+ jobsDir.getAbsolutePath() + "/\n"
+        		+ "<input size='16' name='addpath'/>\n"
+        		+ "<input type='submit' name='action' value='create'>\n"
+        		+ "</form>\n");
+
         pw.println("<form method=\'POST\'>");
         pw.println("Add job directory: <input size='50' name='addpath'/>");
         pw.println("<input type='submit' name='action' value='add'>");
diff --git a/engine/src/main/java/org/archive/crawler/restlet/profile-crawler-beans.cxml b/engine/src/main/java/org/archive/crawler/restlet/profile-crawler-beans.cxml
new file mode 100644
index 00000000..fa50dcc5
--- /dev/null
+++ b/engine/src/main/java/org/archive/crawler/restlet/profile-crawler-beans.cxml
@@ -0,0 +1,523 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!-- 
+  HERITRIX 3 CRAWL JOB CONFIGURATION FILE
+  
+   This is a relatively minimal configuration suitable for many crawls.
+   
+   Commented-out beans and properties are provided as an example; values
+   shown in comments reflect the actual defaults which are in effect
+   without specification. (To change from the default behavior, 
+   uncomment AND alter the shown values.)   
+ -->
+<beans xmlns="http://www.springframework.org/schema/beans"
+	     xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+         xmlns:context="http://www.springframework.org/schema/context"
+	     xmlns:aop="http://www.springframework.org/schema/aop"
+	     xmlns:tx="http://www.springframework.org/schema/tx"
+	     xsi:schemaLocation="http://www.springframework.org/schema/beans http://www.springframework.org/schema/beans/spring-beans-2.0.xsd
+           http://www.springframework.org/schema/aop http://www.springframework.org/schema/aop/spring-aop-2.0.xsd
+           http://www.springframework.org/schema/tx http://www.springframework.org/schema/tx/spring-tx-2.0.xsd
+           http://www.springframework.org/schema/context http://www.springframework.org/schema/context/spring-context-2.5.xsd">
+ 
+ <context:annotation-config/>
+
+<!-- 
+  OVERRIDES
+   Values elsewhere in the configuration may be replaced ('overridden') 
+   by a Properties map declared in a PropertiesOverrideConfigurer, 
+   using a dotted-bean-path to address individual bean properties. 
+   This allows us to collect a few of the most-often changed values
+   in an easy-to-edit format here at the beginning of the model
+   configuration.    
+ -->
+ <!-- overrides from a text property list -->
+ <bean id="simpleOverrides" class="org.springframework.beans.factory.config.PropertyOverrideConfigurer">
+  <property name="properties">
+   <value>
+# This Properties map is specified in the Java 'property list' text format
+# http://java.sun.com/javase/6/docs/api/java/util/Properties.html#load%28java.io.Reader%29
+
+metadata.operatorContactUrl=ENTER_AN_URL_WITH_YOUR_CONTACT_INFO_HERE_FOR_WEBMASTERS_AFFECTED_BY_YOUR_CRAWL
+metadata.jobName=basic
+metadata.description=Basic crawl starting with useful defaults
+
+##..more?..##
+   </value>
+  </property>
+ </bean>
+
+ <!-- overrides from declared <prop> elements, more easily allowing
+      multiline values or even declared beans -->
+ <bean id="longerOverrides" class="org.springframework.beans.factory.config.PropertyOverrideConfigurer">
+  <property name="properties">
+   <props>
+    <prop key="seeds.textSource.value">
+
+# URLS HERE
+http://example.example/example
+
+    </prop>
+   </props>
+  </property>
+ </bean>
+
+ <!-- CRAWL METADATA: including identification of crawler/operator -->
+ <bean id="metadata" class="org.archive.modules.CrawlMetadata" autowire="byName">
+       <property name="operatorContactUrl" value="[see override above]"/>
+       <property name="jobName" value="[see override above]"/>
+       <property name="description" value="[see override above]"/>
+  <!-- <property name="operator" value=""/> -->
+  <!-- <property name="operatorFrom" value=""/> -->
+  <!-- <property name="organization" value=""/> -->
+  <!-- <property name="audience" value=""/> -->
+  <!-- <property name="userAgentTemplate" 
+         value="Mozilla/5.0 (compatible; heritrix/@VERSION@ +@OPERATOR_CONTACT_URL@)"/> -->
+       
+ </bean>
+ 
+ <!-- SEEDS: crawl starting points 
+      ConfigString allows simple, inline specification of a moderate
+      number of seeds; see below comment for example of using an
+      arbitrarily-large external file. -->
+ <bean id="seeds" class="org.archive.modules.seeds.TextSeedModule">
+     <property name="textSource">
+      <bean class="org.archive.spring.ConfigString">
+       <property name="value">
+        <value>
+# [see override above]
+        </value>
+       </property>
+      </bean>
+     </property>
+<!-- <property name='sourceTagSeeds' value='false'/> -->
+ </bean>
+ 
+ <!-- SEEDS ALTERNATE APPROACH: specifying external seeds.txt file
+      Use either the above, or this, but not both. -->
+ <!-- 
+ <bean id="seeds" class="org.archive.modules.seeds.TextSeedModule">
+  <property name="textSource">
+   <bean class="org.archive.spring.ConfigFile">
+    <property name="path" value="seeds.txt" />
+   </bean>
+  </property>
+  <property name='sourceTagSeeds' value='false'/>
+ </bean>
+  -->
+ 
+ <!-- SCOPE: rules for which discovered URIs to crawl; order is very 
+      important because last decision returned other than 'NONE' wins. -->
+ <bean id="scope" class="org.archive.modules.deciderules.DecideRuleSequence">
+  <property name="rules">
+   <list>
+    <!-- Begin by REJECTing all... -->
+    <bean class="org.archive.modules.deciderules.RejectDecideRule">
+    </bean>
+    <!-- ...then ACCEPT those within configured/seed-implied SURT prefixes... -->
+    <bean class="org.archive.modules.deciderules.surt.SurtPrefixedDecideRule">
+     <!-- <property name="seedsAsSurtPrefixes" value="true" /> -->
+     <!-- <property name="alsoCheckVia" value="true" /> -->
+     <!-- <property name="surtsSourceFile" value="" /> -->
+     <!-- <property name="surtsDumpFile" value="surts.dump" /> -->
+    </bean>
+    <!-- ...but REJECT those more than a configured link-hop-count from start... -->
+    <bean class="org.archive.modules.deciderules.TooManyHopsDecideRule">
+     <!-- <property name="maxHops" value="20" /> -->
+    </bean>
+    <!-- ...but ACCEPT those more than a configured link-hop-count from start... -->
+    <bean class="org.archive.modules.deciderules.TransclusionDecideRule">
+     <!-- <property name="maxTransHops" value="2" /> -->
+     <!-- <property name="maxSpeculativeHops" value="1" /> -->
+    </bean>
+    <!-- ...but REJECT those from a configurable (initially empty) set of REJECT SURTs... -->
+    <bean class="org.archive.modules.deciderules.surt.SurtPrefixedDecideRule">
+          <property name="decision" value="REJECT"/>
+          <property name="seedsAsSurtPrefixes" value="false"/>
+          <property name="surtsDumpFile" value="negative-surts.dump" />
+     <!-- <property name="surtsSourceFile" value="" /> -->
+    </bean>
+    <!-- ...and REJECT those from a configurable (initially empty) set of URI regexes... -->
+    <bean class="org.archive.modules.deciderules.MatchesListRegexDecideRule">
+     <!-- <property name="listLogicalOr" value="true" /> -->
+     <!-- <property name="regexList">
+           <list>
+           </list>
+          </property> -->
+    </bean>
+    <!-- ...and REJECT those with suspicious repeating path-segments... -->
+    <bean class="org.archive.modules.deciderules.PathologicalPathDecideRule">
+     <!-- <property name="maxRepetitions" value="2" /> -->
+    </bean>
+    <!-- ...and REJECT those with more than threshold number of path-segments... -->
+    <bean class="org.archive.modules.deciderules.TooManyPathSegmentsDecideRule">
+     <!-- <property name="maxPathDepth" value="20" /> -->
+    </bean>
+    <!-- ...but always ACCEPT those marked as prerequisitee for another URI... -->
+    <bean class="org.archive.modules.deciderules.PrerequisiteAcceptDecideRule">
+    </bean>
+   </list>
+  </property>
+ </bean>
+ 
+ <!-- 
+   PROCESSING CHAINS
+    Much of the crawler's work is specified by the sequential 
+    application of swappable Processor modules. These Processors
+    are collected into three 'chains. The CandidateChain is applied 
+    to URIs being considered for inclusion, before a URI is enqueued
+    for collection. The FetchChain is applied to URIs when their 
+    turn for collection comes up. The DispositionChain is applied 
+    after a URI is fetched and analyzed/link-extracted.
+  -->
+  
+ <!-- CANDIDATE CHAIN --> 
+ <!-- processors declared as named beans -->
+ <bean id="candidateScoper" class="org.archive.crawler.prefetch.CandidateScoper">
+ </bean>
+ <bean id="preparer" class="org.archive.crawler.prefetch.FrontierPreparer">
+  <!-- <property name="preferenceDepthHops" value="-1" /> -->
+  <!-- <property name="preferenceEmbedHops" value="1" /> -->
+  <!-- <property name="canonicalizationPolicy"> 
+        <ref bean="canonicalizationPolicy" />
+       </property> -->
+  <!-- <property name="queueAssignmentPolicy"> 
+        <ref bean="queueAssignmentPolicy" />
+       </property> -->
+  <!-- <property name="uriPrecedencePolicy"> 
+        <ref bean="uriPrecedencePolicy" />
+       </property> -->
+  <!-- <property name="costAssignmentPolicy"> 
+        <ref bean="costAssignmentPolicy" />
+       </property> -->
+ </bean>
+ <!-- assembled into ordered CandidateChain bean -->
+ <bean id="candidateProcessors" class="org.archive.modules.CandidateChain">
+  <property name="processors">
+   <list>
+    <!-- apply scoping rules to each individual candidate URI... -->
+    <ref bean="candidateScoper"/>
+    <!-- ...then prepare those ACCEPTed for enqueuing to frontier. -->
+    <ref bean="preparer"/>
+   </list>
+  </property>
+ </bean>
+  
+ <!-- FETCH CHAIN --> 
+ <!-- processors declared as named beans -->
+ <bean id="preselector" class="org.archive.crawler.prefetch.Preselector">
+  <!-- <property name="recheckScope" value="false" /> -->
+  <!-- <property name="blockAll" value="false" /> -->
+  <!-- <property name="blockByRegex" value="" /> -->
+  <!-- <property name="allowByRegex" value="" /> -->
+ </bean>
+ <bean id="preconditions" class="org.archive.crawler.prefetch.PreconditionEnforcer">
+  <!-- <property name="ipValidityDurationSeconds" value="21600" /> -->
+  <!-- <property name="robotsValidityDurationSeconds" value="86400" /> -->
+  <!-- <property name="calculateRobotsOnly" value="false" /> -->
+ </bean>
+ <bean id="fetchDns" class="org.archive.modules.fetcher.FetchDNS">
+  <!-- <property name="acceptNonDnsResolves" value="false" /> -->
+  <!-- <property name="digestContent" value="true" /> -->
+  <!-- <property name="digestAlgorithm" value="sha1" /> -->
+ </bean>
+ <bean id="fetchHttp" class="org.archive.modules.fetcher.FetchHTTP">
+  <!-- <property name="maxLengthBytes" value="0" /> -->
+  <!-- <property name="timeoutSeconds" value="1200" /> -->
+  <!-- <property name="maxFetchKBSec" value="0" /> -->
+  <!-- <property name="defaultEncoding" value="ISO-8859-1" /> -->
+  <!-- <property name="shouldFetchBodyRule"> 
+        <bean class="org.archive.modules.deciderules.AcceptDecideRule"/>
+       </property> -->
+  <!-- <property name="soTimeoutMs" value="20000" /> -->
+  <!-- <property name="sendIfModifiedSince" value="true" /> -->
+  <!-- <property name="sendIfNoneMatch" value="true" /> -->
+  <!-- <property name="sendConnectionClose" value="true" /> -->
+  <!-- <property name="sendReferer" value="true" /> -->
+  <!-- <property name="sendRange" value="false" /> -->
+  <!-- <property name="ignoreCookies" value="false" /> -->
+  <!-- <property name="sslTrustLevel" value="OPEN" /> -->
+  <!-- <property name="acceptHeaders"> 
+        <list>
+        </list>
+       </property>
+  -->
+  <!-- <property name="httpBindAddress" value="" /> -->
+  <!-- <property name="httpProxyHost" value="" /> -->
+  <!-- <property name="httpProxyPort" value="0" /> -->
+  <!-- <property name="digestContent" value="true" /> -->
+  <!-- <property name="digestAlgorithm" value="sha1" /> -->
+ </bean>
+ <bean id="extractorHttp" class="org.archive.modules.extractor.ExtractorHTTP">
+ </bean>
+ <bean id="extractorHtml" class="org.archive.modules.extractor.ExtractorHTML">
+  <!-- <property name="extractJavascript" value="true" /> -->
+  <!-- <property name="extractValueAttributes" value="true" /> -->
+  <!-- <property name="ignoreFormActionUrls" value="false" /> -->
+  <!-- <property name="extractOnlyFormGets" value="true" /> -->
+  <!-- <property name="treatFramesAsEmbedLinks" value="true" /> -->
+  <!-- <property name="ignoreUnexpectedHtml" value="true" /> -->
+  <!-- <property name="maxElementLength" value="1024" /> -->
+  <!-- <property name="maxAttributeNameLength" value="1024" /> -->
+  <!-- <property name="maxAttributeValueLength" value="16384" /> -->
+ </bean>
+ <bean id="extractorCss" class="org.archive.modules.extractor.ExtractorCSS">
+ </bean> 
+ <bean id="extractorJs" class="org.archive.modules.extractor.ExtractorJS">
+ </bean>
+ <bean id="extractorSwf" class="org.archive.modules.extractor.ExtractorSWF">
+ </bean>    
+ <!-- assembled into ordered FetchChain bean -->
+ <bean id="fetchProcessors" class="org.archive.modules.FetchChain">
+  <property name="processors">
+   <list>
+    <!-- recheck scope, if so enabled... -->
+    <ref bean="preselector"/>
+    <!-- ...then verify or trigger prerequisite URIs fetched, allow crawling... -->
+    <ref bean="preconditions"/>
+    <!-- ...fetch if DNS URI... -->
+    <ref bean="fetchDns"/>
+    <!-- ...fetch if HTTP URI... -->
+    <ref bean="fetchHttp"/>
+    <!-- ...extract oulinks from HTTP headers... -->
+    <ref bean="extractorHttp"/>
+    <!-- ...extract oulinks from HTML content... -->
+    <ref bean="extractorHtml"/>
+    <!-- ...extract oulinks from CSS content... -->
+    <ref bean="extractorCss"/>
+    <!-- ...extract oulinks from Javascript content... -->
+    <ref bean="extractorJs"/>
+    <!-- ...extract oulinks from Flash content... -->
+    <ref bean="extractorSwf"/>
+   </list>
+  </property>
+ </bean>
+  
+ <!-- DISPOSITION CHAIN -->
+ <!-- processors declared as named beans -->
+ <bean id="warcWriter" class="org.archive.modules.writer.WARCWriterProcessor">
+  <!-- <property name="compress" value="true" /> -->
+  <!-- <property name="prefix" value="IAH" /> -->
+  <!-- <property name="suffix" value="${HOSTNAME}" /> -->
+  <!-- <property name="maxFileSizeBytes" value="1000000000" /> -->
+  <!-- <property name="poolMaxActive" value="1" /> -->
+  <!-- <property name="poolMaxWaitMs" value="300000" /> -->
+  <!-- <property name="skipIdenticalDigests" value="false" /> -->
+  <!-- <property name="maxTotalBytesToWrite" value="0" /> -->
+  <!-- <property name="directory" value="." /> -->
+  <!-- <property name="storePaths">
+        <list>
+         <value>warcs</value>
+        </list>
+       </property> -->
+  <!-- <property name="writeRequests" value="true" /> -->
+  <!-- <property name="writeMetadata" value="true" /> -->
+  <!-- <property name="writeRevisitForIdenticalDigests" value="true" /> -->
+  <!-- <property name="writeRevisitForNotModified" value="true" /> -->
+ </bean>
+ <bean id="candidates" class="org.archive.crawler.postprocessor.CandidatesProcessor">
+  <!-- <property name="seedsRedirectNewSeeds" value="true" /> -->
+ </bean>
+ <bean id="disposition" class="org.archive.crawler.postprocessor.DispositionProcessor">
+  <!-- <property name="delayFactor" value="5.0" /> -->
+  <!-- <property name="minDelayMs" value="3000" /> -->
+  <!-- <property name="respectCrawlDelayUpToSeconds" value="300" /> -->
+  <!-- <property name="maxDelayMs" value="30000" /> -->
+  <!-- <property name="maxPerHostBandwidthUsageKbSec" value="0" /> -->
+  
+ </bean>
+ <!-- assembled into ordered DispositionChain bean -->
+ <bean id="dispositionProcessors" class="org.archive.modules.DispositionChain">
+  <property name="processors">
+   <list>
+    <!-- write to aggregate archival files... -->
+    <ref bean="warcWriter"/>
+    <!-- ...send each outlink candidate URI to CandidatesChain, 
+         and enqueue those ACCEPTed to the frontier... -->
+    <ref bean="candidates"/>
+    <!-- ...then update stats, shared-structures, frontier decisions -->
+    <ref bean="disposition"/>
+   </list>
+  </property>
+ </bean>
+ 
+ <!-- CRAWLCONTROLLER: Control interface, unifying context -->
+ <bean id="crawlController" 
+   class="org.archive.crawler.framework.CrawlController">
+  <!-- <property name="maxToeThreads" value="25" /> -->
+  <!-- <property name="pauseAtStart" value="true" /> -->
+  <!-- <property name="pauseAtFinish" value="false" /> -->
+  <!-- <property name="recorderInBufferBytes" value="524288" /> -->
+  <!-- <property name="recorderOutBufferBytes" value="16384" /> -->
+  <!-- <property name="checkpointerPeriod" value="-1" /> -->
+  <!-- <property name="scratchDir" value="scratch" /> -->
+  <!-- <property name="checkpointsDir" value="checkpoints" /> -->
+ </bean>
+ 
+ <!-- FRONTIER: Record of all URIs discovered and queued-for-collection -->
+ <bean id="frontier" 
+   class="org.archive.crawler.frontier.BdbFrontier">
+  <!-- <property name="holdQueues" value="true" /> -->
+  <!-- <property name="queueTotalBudget" value="-1" /> -->
+  <!-- <property name="balanceReplenishAmount" value="3000" /> -->
+  <!-- <property name="errorPenaltyAmount" value="100" /> -->
+  <!-- <property name="precedenceFloor" value="255" /> -->
+  <!-- <property name="queuePrecedencePolicy">
+        <bean class="org.archive.crawler.frontier.precedence.BaseQueuePrecedencePolicy" />
+       </property> -->
+  <!-- <property name="snoozeLongMs" value="300000" /> -->
+  <!-- <property name="retryDelaySeconds" value="900" /> -->
+  <!-- <property name="maxRetries" value="30" /> -->
+  <!-- <property name="recoveryDir" value="logs" /> -->
+  <!-- <property name="recoveryLogEnabled" value="true" /> -->
+  <!-- <property name="maxOutlinks" value="6000" /> -->
+  <!-- <property name="outboundQueueCapacity" value="50" /> -->
+  <!-- <property name="inboundQueueMultiple" value="3" /> -->
+  <!-- <property name="dumpPendingAtClose" value="false" /> -->
+ </bean>
+ 
+ <!-- URI UNIQ FILTER: Used by frontier to remember already-included URIs --> 
+ <bean id="uriUniqFilter" 
+   class="org.archive.crawler.util.BdbUriUniqFilter">
+ </bean>
+ 
+ <!-- 
+   OPTIONAL BUT RECOMMENDED BEANS
+  -->
+  
+ <!-- ACTIONDIRECTORY: disk directory for mid-crawl operations
+      Running job will watch directory for new files with URIs, 
+      scripts, and other data to be processed during a crawl. -->
+ <bean id="actionDirectory" class="org.archive.crawler.framework.ActionDirectory">
+  <!-- <property name="actionDir" value="action" /> -->
+  <!-- <property name="initialDelaySeconds" value="10" /> -->
+  <!-- <property name="delaySeconds" value="30" /> -->
+ </bean> 
+ 
+ <!--  CRAWLLIMITENFORCER: stops crawl when it reaches configured limits -->
+ <bean id="crawlLimiter" class="org.archive.crawler.framework.CrawlLimitEnforcer">
+  <!-- <property name="maxBytesDownload" value="0" /> -->
+  <!-- <property name="maxDocumentsDownload" value="0" /> -->
+  <!-- <property name="maxTimeSeconds" value="0" /> -->
+ </bean>
+ 
+ <!-- 
+   OPTIONAL BEANS
+    Uncomment and expand as needed, or if non-default alternate 
+    implementations are preferred.
+  -->
+  
+ <!-- CANONICALIZATION POLICY -->
+ <!--
+ <bean id="canonicalizationPolicy" 
+   class="org.archive.modules.canonicalize.RulesCanonicalizationPolicy">
+  <list>
+   <bean class="org.archive.modules.canonicalize.LowercaseRule" />
+   <bean class="org.archive.modules.canonicalize.StripUserinfoRule" />
+   <bean class="org.archive.modules.canonicalize.StripWWWNRule" />
+   <bean class="org.archive.modules.canonicalize.StripSessionIDs" />
+   <bean class="org.archive.modules.canonicalize.StripSessionCFIDs" />
+   <bean class="org.archive.modules.canonicalize.FixupQueryString" />
+  </list>
+ </bean>
+ -->
+ 
+ <!-- QUEUE ASSIGNMENT POLICY -->
+ <!--
+ <bean id="queueAssignmentPolicy" 
+   class="org.archive.crawler.frontier.SurtAuthorityQueueAssignmentPolicy">
+  <property name="forceQueueAssignment" value="" />
+  <property name="deferToPrevious" value="true" />
+  <property name="parallelQueues" value="1" />
+ </bean>
+ -->
+ 
+ <!-- URI PRECEDENCE POLICY -->
+ <!--
+ <bean id="uriPrecedencePolicy" 
+   class="org.archive.crawler.frontier.precedence.CostUriPrecedencePolicy">
+ </bean>
+ -->
+ 
+ <!-- COST ASSIGNMENT POLICY -->
+ <!--
+ <bean id="costAssignmentPolicy" 
+   class="org.archive.crawler.frontier.UnitCostAssignmentPolicy">
+ </bean>
+ -->
+ 
+ <!-- CREDENTIAL STORE: HTTP authentication or FORM POST credentials -->
+ <!-- 
+ <bean id="credentialStore" 
+   class="org.archive.modules.credential.CredentialStore">
+ </bean>
+ -->
+ 
+ <!-- 
+   REQUIRED STANDARD BEANS
+    It will be very rare to replace or reconfigure the following beans.
+  -->
+
+ <!-- STATISTICSTRACKER: standard stats/reporting collector -->
+ <bean id="statisticsTracker" 
+   class="org.archive.crawler.reporting.StatisticsTracker" autowire="byName">
+  <!-- <property name="reportsDir" value="reports" /> -->
+  <!-- <property name="liveHostReportSize" value="20" /> -->
+  <!-- <property name="intervalSeconds" value="20" /> -->
+  <!-- <property name="keepSnapshotsCount" value="5" /> -->
+  <!-- <property name="liveHostReportSize" value="20" /> -->
+ </bean>
+ 
+ <!-- CRAWLERLOGGERMODULE: shared logging facility -->
+ <bean id="loggerModule" 
+   class="org.archive.crawler.reporting.CrawlerLoggerModule">
+  <!-- <property name="path" value="logs" /> -->
+  <!-- <property name="crawlLogPath" value="crawl.log" /> -->
+  <!-- <property name="alertsLogPath" value="alerts.log" /> -->
+  <!-- <property name="progressLogPath" value="progress-statistics.log" /> -->
+  <!-- <property name="uriErrorsLogPath" value="uri-errors.log" /> -->
+  <!-- <property name="runtimeErrorsLogPath" value="runtime-errors.log" /> -->
+  <!-- <property name="nonfatalErrorsLogPath" value="nonfatal-errors.log" /> -->
+ </bean>
+ 
+ <!-- SHEETOVERLAYMANAGER: manager of sheets of contextual overlays
+      Autowired to include any SheetForSurtPrefix or 
+      SheetForDecideRuled beans -->
+ <bean id="sheetOverlaysManager" autowire="byType"
+   class="org.archive.crawler.spring.SheetOverlaysManager">
+ </bean>
+
+ <!-- BDBMODULE: shared BDB-JE disk persistence manager -->
+ <bean id="bdb" 
+  class="org.archive.bdb.BdbModule">
+  <!-- <property name="dir" value="state" /> -->
+  <!-- <property name="cachePercentage" value="60" /> -->
+  <!-- <property name="useSharedCache" value="true" /> -->
+  <!-- <property name="checkpointCopyLogs" value="true" /> -->
+  <!-- <property name="expectedConcurrency" value="25" /> -->
+ </bean>
+ 
+ <!-- BDBCOOKIESTORAGE: disk-based cookie storage for FetchHTTP -->
+ <bean id="cookieStorage" 
+   class="org.archive.modules.fetcher.BdbCookieStorage">
+  <!-- <property name="cookiesLoadFile"><null/></property> -->
+  <!-- <property name="cookiesSaveFile"><null/></property> -->
+  <!-- <property name="bdb">
+        <ref bean="bdb"/>
+       </property> -->
+ </bean>
+ 
+ <!-- SERVERCACHE: shared cache of server/host info -->
+ <bean id="serverCache" 
+   class="org.archive.modules.net.BdbServerCache">
+  <!-- <property name="bdb">
+        <ref bean="bdb"/>
+       </property> -->
+ </bean>
+
+ <!-- CONFIG PATH CONFIGURER: required helper making crawl paths relative
+      to crawler-beans.cxml file, and tracking crawl files for web UI -->
+ <bean id="configPathConfigurer" 
+   class="org.archive.spring.ConfigPathConfigurer">
+ </bean>
+ 
+</beans>