mirror of
https://github.com/internetarchive/heritrix3.git
synced 2026-11-09 15:36:40 +00:00
Fix for [HER-1419] can't create job in web ui without profile/job to base it on in jobs dir?
* Engine.java createNewJobWithDefaults() to write profile-crawler-beans.cxml resoure into new job * EngineResource.java added FORM for new job dir and "create" action * profile-crawler-beans.cxml added resource from dist/src/main/conf/jobs/profile-defaults to bootstrap new jobs
This commit is contained in:
szznax committed
1 parent
58513a8940
commit
1a136b21d5
3 files changed
+583
No files matched your search
@@ -23,12 +23,14 @@ import java.io.File;
|
||||
import java.io.FileFilter;
|
||||
import java.io.FilenameFilter;
|
||||
import java.io.IOException;
|
||||
import java.io.InputStream;
|
||||
import java.util.HashMap;
|
||||
import java.util.Map;
|
||||
import java.util.logging.Level;
|
||||
import java.util.logging.Logger;
|
||||
|
||||
import org.apache.commons.io.FileUtils;
|
||||
import org.apache.commons.io.IOUtils;
|
||||
import org.apache.commons.io.filefilter.FileFilterUtils;
|
||||
import org.archive.util.ArchiveUtils;
|
||||
|
||||
@@ -272,5 +274,26 @@ public class Engine {
|
||||
return false;
|
||||
}
|
||||
|
||||
public boolean createNewJobWithDefaults(String path) throws IOException {
|
||||
|
||||
File newJobDir = new File(jobsDir,"/"+path);
|
||||
if (newJobDir.exists()) {
|
||||
throw new IOException("file exists: "+newJobDir);
|
||||
}
|
||||
newJobDir.mkdirs();
|
||||
|
||||
// get crawler-beans template from this package into string
|
||||
InputStream inStream = getClass().getResourceAsStream(
|
||||
"/org/archive/crawler/restlet/profile-crawler-beans.cxml");
|
||||
String defaultCxmlStr = IOUtils.toString(inStream);
|
||||
inStream.close();
|
||||
|
||||
// write default crawler-beans string to new job config
|
||||
File newJobCxml = new File(newJobDir,"crawler-beans.cxml");
|
||||
FileUtils.writeStringToFile(newJobCxml, defaultCxmlStr);
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
|
||||
}
|
||||
@@ -85,6 +85,35 @@ public class EngineResource extends Resource {
|
||||
if(!added) {
|
||||
Flash.addFlash(getResponse(), "not a valid preexisting job directory: \""+path+"\"", Flash.Kind.NACK);
|
||||
}
|
||||
} else if ("create".equals(action)) {
|
||||
String errStyle = "style=\"margin:1em;padding:0.2em 1em;background:pink;\"";
|
||||
String warnStyle = errStyle;
|
||||
String msgStyle = "style=\"margin:1em;padding:0.2em 1em;background:khaki;\"";
|
||||
String path = form.getFirstValue("addpath");
|
||||
if (path.indexOf("/") != -1) {
|
||||
String warn = "<div "+warnStyle+">WARNING: "
|
||||
+ "no job created. sub-directories disallowed: "
|
||||
+ "<i>" + path + "</i></div>\n";
|
||||
Flash.addFlash(getResponse(), warn, Flash.Kind.NACK);
|
||||
System.err.println(warn);
|
||||
} else {
|
||||
boolean created = false;
|
||||
try {
|
||||
created = getEngine().createNewJobWithDefaults(path);
|
||||
} catch (IOException e) {
|
||||
String err = "<div "+errStyle+">ERROR! failed to create new job: "
|
||||
+ "<i>" + path + "</i> "+ e.toString() + "</div>\n";
|
||||
Flash.addFlash(getResponse(), err, Flash.Kind.NACK);
|
||||
System.err.println(err);
|
||||
|
||||
}
|
||||
if (created) {
|
||||
String msg = "<p "+msgStyle+">Successfully created job: "
|
||||
+ "<i>" + path + "</i></p>\n";
|
||||
Flash.addFlash(getResponse(), msg, Flash.Kind.NACK);
|
||||
getEngine().findJobConfigs();
|
||||
}
|
||||
}
|
||||
}
|
||||
// default: redirect to GET self
|
||||
getResponse().redirectSeeOther(getRequest().getOriginalRef());
|
||||
@@ -137,6 +166,14 @@ public class EngineResource extends Resource {
|
||||
"'rescan' button above. Or, supply a full path to another " +
|
||||
"valid job directory at the engine machine below.<br/><br/>");
|
||||
|
||||
// create new job with defaults
|
||||
pw.println("<form method=\'POST\'>\n"
|
||||
+ "Create new job with recommended defaults<br />\n"
|
||||
+ jobsDir.getAbsolutePath() + "/\n"
|
||||
+ "<input size='16' name='addpath'/>\n"
|
||||
+ "<input type='submit' name='action' value='create'>\n"
|
||||
+ "</form>\n");
|
||||
|
||||
pw.println("<form method=\'POST\'>");
|
||||
pw.println("Add job directory: <input size='50' name='addpath'/>");
|
||||
pw.println("<input type='submit' name='action' value='add'>");
|
||||
|
||||
@@ -0,0 +1,523 @@
|
||||
<?xml version="1.0" encoding="UTF-8"?>
|
||||
<!--
|
||||
HERITRIX 3 CRAWL JOB CONFIGURATION FILE
|
||||
|
||||
This is a relatively minimal configuration suitable for many crawls.
|
||||
|
||||
Commented-out beans and properties are provided as an example; values
|
||||
shown in comments reflect the actual defaults which are in effect
|
||||
without specification. (To change from the default behavior,
|
||||
uncomment AND alter the shown values.)
|
||||
-->
|
||||
<beans xmlns="http://www.springframework.org/schema/beans"
|
||||
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
|
||||
xmlns:context="http://www.springframework.org/schema/context"
|
||||
xmlns:aop="http://www.springframework.org/schema/aop"
|
||||
xmlns:tx="http://www.springframework.org/schema/tx"
|
||||
xsi:schemaLocation="http://www.springframework.org/schema/beans http://www.springframework.org/schema/beans/spring-beans-2.0.xsd
|
||||
http://www.springframework.org/schema/aop http://www.springframework.org/schema/aop/spring-aop-2.0.xsd
|
||||
http://www.springframework.org/schema/tx http://www.springframework.org/schema/tx/spring-tx-2.0.xsd
|
||||
http://www.springframework.org/schema/context http://www.springframework.org/schema/context/spring-context-2.5.xsd">
|
||||
|
||||
<context:annotation-config/>
|
||||
|
||||
<!--
|
||||
OVERRIDES
|
||||
Values elsewhere in the configuration may be replaced ('overridden')
|
||||
by a Properties map declared in a PropertiesOverrideConfigurer,
|
||||
using a dotted-bean-path to address individual bean properties.
|
||||
This allows us to collect a few of the most-often changed values
|
||||
in an easy-to-edit format here at the beginning of the model
|
||||
configuration.
|
||||
-->
|
||||
<!-- overrides from a text property list -->
|
||||
<bean id="simpleOverrides" class="org.springframework.beans.factory.config.PropertyOverrideConfigurer">
|
||||
<property name="properties">
|
||||
<value>
|
||||
# This Properties map is specified in the Java 'property list' text format
|
||||
# http://java.sun.com/javase/6/docs/api/java/util/Properties.html#load%28java.io.Reader%29
|
||||
|
||||
metadata.operatorContactUrl=ENTER_AN_URL_WITH_YOUR_CONTACT_INFO_HERE_FOR_WEBMASTERS_AFFECTED_BY_YOUR_CRAWL
|
||||
metadata.jobName=basic
|
||||
metadata.description=Basic crawl starting with useful defaults
|
||||
|
||||
##..more?..##
|
||||
</value>
|
||||
</property>
|
||||
</bean>
|
||||
|
||||
<!-- overrides from declared <prop> elements, more easily allowing
|
||||
multiline values or even declared beans -->
|
||||
<bean id="longerOverrides" class="org.springframework.beans.factory.config.PropertyOverrideConfigurer">
|
||||
<property name="properties">
|
||||
<props>
|
||||
<prop key="seeds.textSource.value">
|
||||
|
||||
# URLS HERE
|
||||
http://example.example/example
|
||||
|
||||
</prop>
|
||||
</props>
|
||||
</property>
|
||||
</bean>
|
||||
|
||||
<!-- CRAWL METADATA: including identification of crawler/operator -->
|
||||
<bean id="metadata" class="org.archive.modules.CrawlMetadata" autowire="byName">
|
||||
<property name="operatorContactUrl" value="[see override above]"/>
|
||||
<property name="jobName" value="[see override above]"/>
|
||||
<property name="description" value="[see override above]"/>
|
||||
<!-- <property name="operator" value=""/> -->
|
||||
<!-- <property name="operatorFrom" value=""/> -->
|
||||
<!-- <property name="organization" value=""/> -->
|
||||
<!-- <property name="audience" value=""/> -->
|
||||
<!-- <property name="userAgentTemplate"
|
||||
value="Mozilla/5.0 (compatible; heritrix/@VERSION@ +@OPERATOR_CONTACT_URL@)"/> -->
|
||||
|
||||
</bean>
|
||||
|
||||
<!-- SEEDS: crawl starting points
|
||||
ConfigString allows simple, inline specification of a moderate
|
||||
number of seeds; see below comment for example of using an
|
||||
arbitrarily-large external file. -->
|
||||
<bean id="seeds" class="org.archive.modules.seeds.TextSeedModule">
|
||||
<property name="textSource">
|
||||
<bean class="org.archive.spring.ConfigString">
|
||||
<property name="value">
|
||||
<value>
|
||||
# [see override above]
|
||||
</value>
|
||||
</property>
|
||||
</bean>
|
||||
</property>
|
||||
<!-- <property name='sourceTagSeeds' value='false'/> -->
|
||||
</bean>
|
||||
|
||||
<!-- SEEDS ALTERNATE APPROACH: specifying external seeds.txt file
|
||||
Use either the above, or this, but not both. -->
|
||||
<!--
|
||||
<bean id="seeds" class="org.archive.modules.seeds.TextSeedModule">
|
||||
<property name="textSource">
|
||||
<bean class="org.archive.spring.ConfigFile">
|
||||
<property name="path" value="seeds.txt" />
|
||||
</bean>
|
||||
</property>
|
||||
<property name='sourceTagSeeds' value='false'/>
|
||||
</bean>
|
||||
-->
|
||||
|
||||
<!-- SCOPE: rules for which discovered URIs to crawl; order is very
|
||||
important because last decision returned other than 'NONE' wins. -->
|
||||
<bean id="scope" class="org.archive.modules.deciderules.DecideRuleSequence">
|
||||
<property name="rules">
|
||||
<list>
|
||||
<!-- Begin by REJECTing all... -->
|
||||
<bean class="org.archive.modules.deciderules.RejectDecideRule">
|
||||
</bean>
|
||||
<!-- ...then ACCEPT those within configured/seed-implied SURT prefixes... -->
|
||||
<bean class="org.archive.modules.deciderules.surt.SurtPrefixedDecideRule">
|
||||
<!-- <property name="seedsAsSurtPrefixes" value="true" /> -->
|
||||
<!-- <property name="alsoCheckVia" value="true" /> -->
|
||||
<!-- <property name="surtsSourceFile" value="" /> -->
|
||||
<!-- <property name="surtsDumpFile" value="surts.dump" /> -->
|
||||
</bean>
|
||||
<!-- ...but REJECT those more than a configured link-hop-count from start... -->
|
||||
<bean class="org.archive.modules.deciderules.TooManyHopsDecideRule">
|
||||
<!-- <property name="maxHops" value="20" /> -->
|
||||
</bean>
|
||||
<!-- ...but ACCEPT those more than a configured link-hop-count from start... -->
|
||||
<bean class="org.archive.modules.deciderules.TransclusionDecideRule">
|
||||
<!-- <property name="maxTransHops" value="2" /> -->
|
||||
<!-- <property name="maxSpeculativeHops" value="1" /> -->
|
||||
</bean>
|
||||
<!-- ...but REJECT those from a configurable (initially empty) set of REJECT SURTs... -->
|
||||
<bean class="org.archive.modules.deciderules.surt.SurtPrefixedDecideRule">
|
||||
<property name="decision" value="REJECT"/>
|
||||
<property name="seedsAsSurtPrefixes" value="false"/>
|
||||
<property name="surtsDumpFile" value="negative-surts.dump" />
|
||||
<!-- <property name="surtsSourceFile" value="" /> -->
|
||||
</bean>
|
||||
<!-- ...and REJECT those from a configurable (initially empty) set of URI regexes... -->
|
||||
<bean class="org.archive.modules.deciderules.MatchesListRegexDecideRule">
|
||||
<!-- <property name="listLogicalOr" value="true" /> -->
|
||||
<!-- <property name="regexList">
|
||||
<list>
|
||||
</list>
|
||||
</property> -->
|
||||
</bean>
|
||||
<!-- ...and REJECT those with suspicious repeating path-segments... -->
|
||||
<bean class="org.archive.modules.deciderules.PathologicalPathDecideRule">
|
||||
<!-- <property name="maxRepetitions" value="2" /> -->
|
||||
</bean>
|
||||
<!-- ...and REJECT those with more than threshold number of path-segments... -->
|
||||
<bean class="org.archive.modules.deciderules.TooManyPathSegmentsDecideRule">
|
||||
<!-- <property name="maxPathDepth" value="20" /> -->
|
||||
</bean>
|
||||
<!-- ...but always ACCEPT those marked as prerequisitee for another URI... -->
|
||||
<bean class="org.archive.modules.deciderules.PrerequisiteAcceptDecideRule">
|
||||
</bean>
|
||||
</list>
|
||||
</property>
|
||||
</bean>
|
||||
|
||||
<!--
|
||||
PROCESSING CHAINS
|
||||
Much of the crawler's work is specified by the sequential
|
||||
application of swappable Processor modules. These Processors
|
||||
are collected into three 'chains. The CandidateChain is applied
|
||||
to URIs being considered for inclusion, before a URI is enqueued
|
||||
for collection. The FetchChain is applied to URIs when their
|
||||
turn for collection comes up. The DispositionChain is applied
|
||||
after a URI is fetched and analyzed/link-extracted.
|
||||
-->
|
||||
|
||||
<!-- CANDIDATE CHAIN -->
|
||||
<!-- processors declared as named beans -->
|
||||
<bean id="candidateScoper" class="org.archive.crawler.prefetch.CandidateScoper">
|
||||
</bean>
|
||||
<bean id="preparer" class="org.archive.crawler.prefetch.FrontierPreparer">
|
||||
<!-- <property name="preferenceDepthHops" value="-1" /> -->
|
||||
<!-- <property name="preferenceEmbedHops" value="1" /> -->
|
||||
<!-- <property name="canonicalizationPolicy">
|
||||
<ref bean="canonicalizationPolicy" />
|
||||
</property> -->
|
||||
<!-- <property name="queueAssignmentPolicy">
|
||||
<ref bean="queueAssignmentPolicy" />
|
||||
</property> -->
|
||||
<!-- <property name="uriPrecedencePolicy">
|
||||
<ref bean="uriPrecedencePolicy" />
|
||||
</property> -->
|
||||
<!-- <property name="costAssignmentPolicy">
|
||||
<ref bean="costAssignmentPolicy" />
|
||||
</property> -->
|
||||
</bean>
|
||||
<!-- assembled into ordered CandidateChain bean -->
|
||||
<bean id="candidateProcessors" class="org.archive.modules.CandidateChain">
|
||||
<property name="processors">
|
||||
<list>
|
||||
<!-- apply scoping rules to each individual candidate URI... -->
|
||||
<ref bean="candidateScoper"/>
|
||||
<!-- ...then prepare those ACCEPTed for enqueuing to frontier. -->
|
||||
<ref bean="preparer"/>
|
||||
</list>
|
||||
</property>
|
||||
</bean>
|
||||
|
||||
<!-- FETCH CHAIN -->
|
||||
<!-- processors declared as named beans -->
|
||||
<bean id="preselector" class="org.archive.crawler.prefetch.Preselector">
|
||||
<!-- <property name="recheckScope" value="false" /> -->
|
||||
<!-- <property name="blockAll" value="false" /> -->
|
||||
<!-- <property name="blockByRegex" value="" /> -->
|
||||
<!-- <property name="allowByRegex" value="" /> -->
|
||||
</bean>
|
||||
<bean id="preconditions" class="org.archive.crawler.prefetch.PreconditionEnforcer">
|
||||
<!-- <property name="ipValidityDurationSeconds" value="21600" /> -->
|
||||
<!-- <property name="robotsValidityDurationSeconds" value="86400" /> -->
|
||||
<!-- <property name="calculateRobotsOnly" value="false" /> -->
|
||||
</bean>
|
||||
<bean id="fetchDns" class="org.archive.modules.fetcher.FetchDNS">
|
||||
<!-- <property name="acceptNonDnsResolves" value="false" /> -->
|
||||
<!-- <property name="digestContent" value="true" /> -->
|
||||
<!-- <property name="digestAlgorithm" value="sha1" /> -->
|
||||
</bean>
|
||||
<bean id="fetchHttp" class="org.archive.modules.fetcher.FetchHTTP">
|
||||
<!-- <property name="maxLengthBytes" value="0" /> -->
|
||||
<!-- <property name="timeoutSeconds" value="1200" /> -->
|
||||
<!-- <property name="maxFetchKBSec" value="0" /> -->
|
||||
<!-- <property name="defaultEncoding" value="ISO-8859-1" /> -->
|
||||
<!-- <property name="shouldFetchBodyRule">
|
||||
<bean class="org.archive.modules.deciderules.AcceptDecideRule"/>
|
||||
</property> -->
|
||||
<!-- <property name="soTimeoutMs" value="20000" /> -->
|
||||
<!-- <property name="sendIfModifiedSince" value="true" /> -->
|
||||
<!-- <property name="sendIfNoneMatch" value="true" /> -->
|
||||
<!-- <property name="sendConnectionClose" value="true" /> -->
|
||||
<!-- <property name="sendReferer" value="true" /> -->
|
||||
<!-- <property name="sendRange" value="false" /> -->
|
||||
<!-- <property name="ignoreCookies" value="false" /> -->
|
||||
<!-- <property name="sslTrustLevel" value="OPEN" /> -->
|
||||
<!-- <property name="acceptHeaders">
|
||||
<list>
|
||||
</list>
|
||||
</property>
|
||||
-->
|
||||
<!-- <property name="httpBindAddress" value="" /> -->
|
||||
<!-- <property name="httpProxyHost" value="" /> -->
|
||||
<!-- <property name="httpProxyPort" value="0" /> -->
|
||||
<!-- <property name="digestContent" value="true" /> -->
|
||||
<!-- <property name="digestAlgorithm" value="sha1" /> -->
|
||||
</bean>
|
||||
<bean id="extractorHttp" class="org.archive.modules.extractor.ExtractorHTTP">
|
||||
</bean>
|
||||
<bean id="extractorHtml" class="org.archive.modules.extractor.ExtractorHTML">
|
||||
<!-- <property name="extractJavascript" value="true" /> -->
|
||||
<!-- <property name="extractValueAttributes" value="true" /> -->
|
||||
<!-- <property name="ignoreFormActionUrls" value="false" /> -->
|
||||
<!-- <property name="extractOnlyFormGets" value="true" /> -->
|
||||
<!-- <property name="treatFramesAsEmbedLinks" value="true" /> -->
|
||||
<!-- <property name="ignoreUnexpectedHtml" value="true" /> -->
|
||||
<!-- <property name="maxElementLength" value="1024" /> -->
|
||||
<!-- <property name="maxAttributeNameLength" value="1024" /> -->
|
||||
<!-- <property name="maxAttributeValueLength" value="16384" /> -->
|
||||
</bean>
|
||||
<bean id="extractorCss" class="org.archive.modules.extractor.ExtractorCSS">
|
||||
</bean>
|
||||
<bean id="extractorJs" class="org.archive.modules.extractor.ExtractorJS">
|
||||
</bean>
|
||||
<bean id="extractorSwf" class="org.archive.modules.extractor.ExtractorSWF">
|
||||
</bean>
|
||||
<!-- assembled into ordered FetchChain bean -->
|
||||
<bean id="fetchProcessors" class="org.archive.modules.FetchChain">
|
||||
<property name="processors">
|
||||
<list>
|
||||
<!-- recheck scope, if so enabled... -->
|
||||
<ref bean="preselector"/>
|
||||
<!-- ...then verify or trigger prerequisite URIs fetched, allow crawling... -->
|
||||
<ref bean="preconditions"/>
|
||||
<!-- ...fetch if DNS URI... -->
|
||||
<ref bean="fetchDns"/>
|
||||
<!-- ...fetch if HTTP URI... -->
|
||||
<ref bean="fetchHttp"/>
|
||||
<!-- ...extract oulinks from HTTP headers... -->
|
||||
<ref bean="extractorHttp"/>
|
||||
<!-- ...extract oulinks from HTML content... -->
|
||||
<ref bean="extractorHtml"/>
|
||||
<!-- ...extract oulinks from CSS content... -->
|
||||
<ref bean="extractorCss"/>
|
||||
<!-- ...extract oulinks from Javascript content... -->
|
||||
<ref bean="extractorJs"/>
|
||||
<!-- ...extract oulinks from Flash content... -->
|
||||
<ref bean="extractorSwf"/>
|
||||
</list>
|
||||
</property>
|
||||
</bean>
|
||||
|
||||
<!-- DISPOSITION CHAIN -->
|
||||
<!-- processors declared as named beans -->
|
||||
<bean id="warcWriter" class="org.archive.modules.writer.WARCWriterProcessor">
|
||||
<!-- <property name="compress" value="true" /> -->
|
||||
<!-- <property name="prefix" value="IAH" /> -->
|
||||
<!-- <property name="suffix" value="${HOSTNAME}" /> -->
|
||||
<!-- <property name="maxFileSizeBytes" value="1000000000" /> -->
|
||||
<!-- <property name="poolMaxActive" value="1" /> -->
|
||||
<!-- <property name="poolMaxWaitMs" value="300000" /> -->
|
||||
<!-- <property name="skipIdenticalDigests" value="false" /> -->
|
||||
<!-- <property name="maxTotalBytesToWrite" value="0" /> -->
|
||||
<!-- <property name="directory" value="." /> -->
|
||||
<!-- <property name="storePaths">
|
||||
<list>
|
||||
<value>warcs</value>
|
||||
</list>
|
||||
</property> -->
|
||||
<!-- <property name="writeRequests" value="true" /> -->
|
||||
<!-- <property name="writeMetadata" value="true" /> -->
|
||||
<!-- <property name="writeRevisitForIdenticalDigests" value="true" /> -->
|
||||
<!-- <property name="writeRevisitForNotModified" value="true" /> -->
|
||||
</bean>
|
||||
<bean id="candidates" class="org.archive.crawler.postprocessor.CandidatesProcessor">
|
||||
<!-- <property name="seedsRedirectNewSeeds" value="true" /> -->
|
||||
</bean>
|
||||
<bean id="disposition" class="org.archive.crawler.postprocessor.DispositionProcessor">
|
||||
<!-- <property name="delayFactor" value="5.0" /> -->
|
||||
<!-- <property name="minDelayMs" value="3000" /> -->
|
||||
<!-- <property name="respectCrawlDelayUpToSeconds" value="300" /> -->
|
||||
<!-- <property name="maxDelayMs" value="30000" /> -->
|
||||
<!-- <property name="maxPerHostBandwidthUsageKbSec" value="0" /> -->
|
||||
|
||||
</bean>
|
||||
<!-- assembled into ordered DispositionChain bean -->
|
||||
<bean id="dispositionProcessors" class="org.archive.modules.DispositionChain">
|
||||
<property name="processors">
|
||||
<list>
|
||||
<!-- write to aggregate archival files... -->
|
||||
<ref bean="warcWriter"/>
|
||||
<!-- ...send each outlink candidate URI to CandidatesChain,
|
||||
and enqueue those ACCEPTed to the frontier... -->
|
||||
<ref bean="candidates"/>
|
||||
<!-- ...then update stats, shared-structures, frontier decisions -->
|
||||
<ref bean="disposition"/>
|
||||
</list>
|
||||
</property>
|
||||
</bean>
|
||||
|
||||
<!-- CRAWLCONTROLLER: Control interface, unifying context -->
|
||||
<bean id="crawlController"
|
||||
class="org.archive.crawler.framework.CrawlController">
|
||||
<!-- <property name="maxToeThreads" value="25" /> -->
|
||||
<!-- <property name="pauseAtStart" value="true" /> -->
|
||||
<!-- <property name="pauseAtFinish" value="false" /> -->
|
||||
<!-- <property name="recorderInBufferBytes" value="524288" /> -->
|
||||
<!-- <property name="recorderOutBufferBytes" value="16384" /> -->
|
||||
<!-- <property name="checkpointerPeriod" value="-1" /> -->
|
||||
<!-- <property name="scratchDir" value="scratch" /> -->
|
||||
<!-- <property name="checkpointsDir" value="checkpoints" /> -->
|
||||
</bean>
|
||||
|
||||
<!-- FRONTIER: Record of all URIs discovered and queued-for-collection -->
|
||||
<bean id="frontier"
|
||||
class="org.archive.crawler.frontier.BdbFrontier">
|
||||
<!-- <property name="holdQueues" value="true" /> -->
|
||||
<!-- <property name="queueTotalBudget" value="-1" /> -->
|
||||
<!-- <property name="balanceReplenishAmount" value="3000" /> -->
|
||||
<!-- <property name="errorPenaltyAmount" value="100" /> -->
|
||||
<!-- <property name="precedenceFloor" value="255" /> -->
|
||||
<!-- <property name="queuePrecedencePolicy">
|
||||
<bean class="org.archive.crawler.frontier.precedence.BaseQueuePrecedencePolicy" />
|
||||
</property> -->
|
||||
<!-- <property name="snoozeLongMs" value="300000" /> -->
|
||||
<!-- <property name="retryDelaySeconds" value="900" /> -->
|
||||
<!-- <property name="maxRetries" value="30" /> -->
|
||||
<!-- <property name="recoveryDir" value="logs" /> -->
|
||||
<!-- <property name="recoveryLogEnabled" value="true" /> -->
|
||||
<!-- <property name="maxOutlinks" value="6000" /> -->
|
||||
<!-- <property name="outboundQueueCapacity" value="50" /> -->
|
||||
<!-- <property name="inboundQueueMultiple" value="3" /> -->
|
||||
<!-- <property name="dumpPendingAtClose" value="false" /> -->
|
||||
</bean>
|
||||
|
||||
<!-- URI UNIQ FILTER: Used by frontier to remember already-included URIs -->
|
||||
<bean id="uriUniqFilter"
|
||||
class="org.archive.crawler.util.BdbUriUniqFilter">
|
||||
</bean>
|
||||
|
||||
<!--
|
||||
OPTIONAL BUT RECOMMENDED BEANS
|
||||
-->
|
||||
|
||||
<!-- ACTIONDIRECTORY: disk directory for mid-crawl operations
|
||||
Running job will watch directory for new files with URIs,
|
||||
scripts, and other data to be processed during a crawl. -->
|
||||
<bean id="actionDirectory" class="org.archive.crawler.framework.ActionDirectory">
|
||||
<!-- <property name="actionDir" value="action" /> -->
|
||||
<!-- <property name="initialDelaySeconds" value="10" /> -->
|
||||
<!-- <property name="delaySeconds" value="30" /> -->
|
||||
</bean>
|
||||
|
||||
<!-- CRAWLLIMITENFORCER: stops crawl when it reaches configured limits -->
|
||||
<bean id="crawlLimiter" class="org.archive.crawler.framework.CrawlLimitEnforcer">
|
||||
<!-- <property name="maxBytesDownload" value="0" /> -->
|
||||
<!-- <property name="maxDocumentsDownload" value="0" /> -->
|
||||
<!-- <property name="maxTimeSeconds" value="0" /> -->
|
||||
</bean>
|
||||
|
||||
<!--
|
||||
OPTIONAL BEANS
|
||||
Uncomment and expand as needed, or if non-default alternate
|
||||
implementations are preferred.
|
||||
-->
|
||||
|
||||
<!-- CANONICALIZATION POLICY -->
|
||||
<!--
|
||||
<bean id="canonicalizationPolicy"
|
||||
class="org.archive.modules.canonicalize.RulesCanonicalizationPolicy">
|
||||
<list>
|
||||
<bean class="org.archive.modules.canonicalize.LowercaseRule" />
|
||||
<bean class="org.archive.modules.canonicalize.StripUserinfoRule" />
|
||||
<bean class="org.archive.modules.canonicalize.StripWWWNRule" />
|
||||
<bean class="org.archive.modules.canonicalize.StripSessionIDs" />
|
||||
<bean class="org.archive.modules.canonicalize.StripSessionCFIDs" />
|
||||
<bean class="org.archive.modules.canonicalize.FixupQueryString" />
|
||||
</list>
|
||||
</bean>
|
||||
-->
|
||||
|
||||
<!-- QUEUE ASSIGNMENT POLICY -->
|
||||
<!--
|
||||
<bean id="queueAssignmentPolicy"
|
||||
class="org.archive.crawler.frontier.SurtAuthorityQueueAssignmentPolicy">
|
||||
<property name="forceQueueAssignment" value="" />
|
||||
<property name="deferToPrevious" value="true" />
|
||||
<property name="parallelQueues" value="1" />
|
||||
</bean>
|
||||
-->
|
||||
|
||||
<!-- URI PRECEDENCE POLICY -->
|
||||
<!--
|
||||
<bean id="uriPrecedencePolicy"
|
||||
class="org.archive.crawler.frontier.precedence.CostUriPrecedencePolicy">
|
||||
</bean>
|
||||
-->
|
||||
|
||||
<!-- COST ASSIGNMENT POLICY -->
|
||||
<!--
|
||||
<bean id="costAssignmentPolicy"
|
||||
class="org.archive.crawler.frontier.UnitCostAssignmentPolicy">
|
||||
</bean>
|
||||
-->
|
||||
|
||||
<!-- CREDENTIAL STORE: HTTP authentication or FORM POST credentials -->
|
||||
<!--
|
||||
<bean id="credentialStore"
|
||||
class="org.archive.modules.credential.CredentialStore">
|
||||
</bean>
|
||||
-->
|
||||
|
||||
<!--
|
||||
REQUIRED STANDARD BEANS
|
||||
It will be very rare to replace or reconfigure the following beans.
|
||||
-->
|
||||
|
||||
<!-- STATISTICSTRACKER: standard stats/reporting collector -->
|
||||
<bean id="statisticsTracker"
|
||||
class="org.archive.crawler.reporting.StatisticsTracker" autowire="byName">
|
||||
<!-- <property name="reportsDir" value="reports" /> -->
|
||||
<!-- <property name="liveHostReportSize" value="20" /> -->
|
||||
<!-- <property name="intervalSeconds" value="20" /> -->
|
||||
<!-- <property name="keepSnapshotsCount" value="5" /> -->
|
||||
<!-- <property name="liveHostReportSize" value="20" /> -->
|
||||
</bean>
|
||||
|
||||
<!-- CRAWLERLOGGERMODULE: shared logging facility -->
|
||||
<bean id="loggerModule"
|
||||
class="org.archive.crawler.reporting.CrawlerLoggerModule">
|
||||
<!-- <property name="path" value="logs" /> -->
|
||||
<!-- <property name="crawlLogPath" value="crawl.log" /> -->
|
||||
<!-- <property name="alertsLogPath" value="alerts.log" /> -->
|
||||
<!-- <property name="progressLogPath" value="progress-statistics.log" /> -->
|
||||
<!-- <property name="uriErrorsLogPath" value="uri-errors.log" /> -->
|
||||
<!-- <property name="runtimeErrorsLogPath" value="runtime-errors.log" /> -->
|
||||
<!-- <property name="nonfatalErrorsLogPath" value="nonfatal-errors.log" /> -->
|
||||
</bean>
|
||||
|
||||
<!-- SHEETOVERLAYMANAGER: manager of sheets of contextual overlays
|
||||
Autowired to include any SheetForSurtPrefix or
|
||||
SheetForDecideRuled beans -->
|
||||
<bean id="sheetOverlaysManager" autowire="byType"
|
||||
class="org.archive.crawler.spring.SheetOverlaysManager">
|
||||
</bean>
|
||||
|
||||
<!-- BDBMODULE: shared BDB-JE disk persistence manager -->
|
||||
<bean id="bdb"
|
||||
class="org.archive.bdb.BdbModule">
|
||||
<!-- <property name="dir" value="state" /> -->
|
||||
<!-- <property name="cachePercentage" value="60" /> -->
|
||||
<!-- <property name="useSharedCache" value="true" /> -->
|
||||
<!-- <property name="checkpointCopyLogs" value="true" /> -->
|
||||
<!-- <property name="expectedConcurrency" value="25" /> -->
|
||||
</bean>
|
||||
|
||||
<!-- BDBCOOKIESTORAGE: disk-based cookie storage for FetchHTTP -->
|
||||
<bean id="cookieStorage"
|
||||
class="org.archive.modules.fetcher.BdbCookieStorage">
|
||||
<!-- <property name="cookiesLoadFile"><null/></property> -->
|
||||
<!-- <property name="cookiesSaveFile"><null/></property> -->
|
||||
<!-- <property name="bdb">
|
||||
<ref bean="bdb"/>
|
||||
</property> -->
|
||||
</bean>
|
||||
|
||||
<!-- SERVERCACHE: shared cache of server/host info -->
|
||||
<bean id="serverCache"
|
||||
class="org.archive.modules.net.BdbServerCache">
|
||||
<!-- <property name="bdb">
|
||||
<ref bean="bdb"/>
|
||||
</property> -->
|
||||
</bean>
|
||||
|
||||
<!-- CONFIG PATH CONFIGURER: required helper making crawl paths relative
|
||||
to crawler-beans.cxml file, and tracking crawl files for web UI -->
|
||||
<bean id="configPathConfigurer"
|
||||
class="org.archive.spring.ConfigPathConfigurer">
|
||||
</bean>
|
||||
|
||||
</beans>
|
||||
Reference in new issue
Block a user