lee/heritrix3
1
0
Fork 0
mirror of https://github.com/internetarchive/heritrix3.git synced 2026-09-24 06:36:12 +00:00
Code Issues Packages Projects Releases Wiki Activity

[HER-1676] improve bundled profiles to match best-practice, be self-documenting

Browse Source 
* profile-crawler-beans.cxml
    add comments for all unstated default values an operator might want to change
* (many) 
    reorder, refactor, rename to better minimize/match simple configuration
This commit is contained in:
gojomo
2009-10-06 19:35:05 +00:00
parent 6b1d1963d8
commit 258da53d57
13 changed files with 465 additions and 198 deletions
Download Patch File Download Diff File Expand all files Collapse all files
commons/src/main/java/org/archive/spring/ConfigPathConfigurer.java
+11 -13
View File
@@ -57,6 +57,16 @@ implements
Ordered {
Map<String,Object> allBeans = new HashMap<String,Object>();
//// BEAN PROPERTIES
/** 'home' directory for all other paths to be resolved
* relative to; defaults to directory of primary XML config file */
ConfigPath path;
public ConfigPath getPath() {
return path;
}
//// BEANPOSTPROCESSOR IMPLEMENTATION
/**
* Remember all beans for later fixup.
@@ -117,19 +127,7 @@ implements
}
return bean;
}
//// BEAN PROPERTIES
/** 'home' directory for all other paths to be resolved
* relative to; defaults to directory of primary XML config file */
ConfigPath path;
public ConfigPath getPath() {
return path;
}
public void setPath(ConfigPath p) {
path = p;
}
//// APPLICATIONCONTEXTAWARE IMPLEMENTATION
AbstractApplicationContext appCtx;
dist/src/main/conf/jobs/profile-defaults/profile-crawler-beans.cxml
Vendored
+373 -88
View File
@@ -1,4 +1,7 @@
<?xml version="1.0" encoding="UTF-8"?>
<!--
HERITRIX 3 CRAWL JOB CONFIGURATION FILE
-->
<beans xmlns="http://www.springframework.org/schema/beans"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xmlns:context="http://www.springframework.org/schema/context"
@@ -11,18 +14,33 @@
<context:annotation-config/>
<!--
OVERRIDES
Values elsewhere in the configuration may be replaced ('overridden')
by a Properties map declared in a PropertiesOverrideConfigurer,
using a dotted-bean-path to address individual bean properties.
This allows us to collect a few of the most-often changed values
in an easy-to-edit format here at the beginning of the model
configuration.
-->
<!-- overrides from a text property list -->
<bean id="simpleOverrides" class="org.springframework.beans.factory.config.PropertyOverrideConfigurer">
<property name="properties">
<value>
# This Properties map is specified in the Java 'property list' text format
# http://java.sun.com/javase/6/docs/api/java/util/Properties.html#load%28java.io.Reader%29
metadata.operatorContactUrl=ENTER_AN_URL_WITH_YOUR_CONTACT_INFO_HERE_FOR_WEBMASTERS_AFFECTED_BY_YOUR_CRAWL
metadata.description=basic crawl
metadata.jobName=basic
metadata.description=Basic crawl starting with useful defaults
##..more?..##
</value>
</property>
</bean>
<!-- overrides from declared <prop> elements, more easily allowing
multiline values or even declared beans -->
<bean id="longerOverrides" class="org.springframework.beans.factory.config.PropertyOverrideConfigurer">
<property name="properties">
<props>
@@ -36,24 +54,52 @@ http://example.example/example
</property>
</bean>
<!-- Crawl metadata, including identification of crawler/operator -->
<!-- CRAWL METADATA: including identification of crawler/operator -->
<bean id="metadata" class="org.archive.modules.CrawlMetadata" autowire="byName">
<property name="description" value="[see override above]"/>
<property name="jobName" value="[see override above]"/>
<property name="operatorContactUrl" value="[see override above]"/>
<property name="operatorContactUrl" value="[see override above]"/>
<property name="jobName" value="[see override above]"/>
<property name="description" value="[see override above]"/>
<!-- <property name="operator" value=""/> -->
<!-- <property name="operatorFrom" value=""/> -->
<!-- <property name="organization" value=""/> -->
<!-- <property name="audience" value=""/> -->
<!-- <property name="userAgentTemplate"
value="Mozilla/5.0 (compatible; heritrix/@VERSION@ +@OPERATOR_CONTACT_URL@)"/> -->
</bean>
<!-- Seeds: crawl starting points -->
<!-- SEEDS: crawl starting points
ConfigString allows simple, inline specification of a moderate
number of seeds; see below comment for example of using an
arbitrarily-large external file. -->
<bean id="seeds" class="org.archive.modules.seeds.TextSeedModule">
<property name="textSource">
<bean class="org.archive.spring.ConfigString">
<property name="value">
<value>
# [see override above]
</value>
</property>
</bean>
</property>
<!-- <property name='sourceTagSeeds' value='false'/> -->
</bean>
<!-- SEEDS ALTERNATE APPROACH: specifying external seeds.txt file
Use either the above, or this, but not both. -->
<!--
<bean id="seeds" class="org.archive.modules.seeds.TextSeedModule">
<property name="textSource">
<bean class="org.archive.spring.ConfigString">
<property name="value" value="# [see override above]"/>
<bean class="org.archive.spring.ConfigFile">
<property name="path" value="seeds.txt" />
</bean>
</property>
<property name='sourceTagSeeds' value='false'/>
</bean>
-->
<!-- Scope: what discovered URIs to crawl; order is very important
because last rule to 'decide' rather than PASS wins. -->
<!-- SCOPE: rules for which discovered URIs to crawl; order is very
important because last decision returned other than 'NONE' wins. -->
<bean id="scope" class="org.archive.modules.deciderules.DecideRuleSequence">
<property name="rules">
<list>
@@ -61,27 +107,42 @@ http://example.example/example
<bean class="org.archive.modules.deciderules.RejectDecideRule">
</bean>
<!-- ...then ACCEPT those within configured/seed-implied SURT prefixes... -->
<bean class="org.archive.modules.deciderules.surt.SurtPrefixedDecideRule" autowire="byName">
<bean class="org.archive.modules.deciderules.surt.SurtPrefixedDecideRule">
<!-- <property name="seedsAsSurtPrefixes" value="true" /> -->
<!-- <property name="surtsSourceFile" value="" /> -->
<!-- <property name="surtsDumpFile" value="surts.dump" /> -->
</bean>
<!-- ...but REJECT those more than a configured link-hop-count from start... -->
<bean class="org.archive.modules.deciderules.TooManyHopsDecideRule">
<!-- <property name="maxHops" value="20" /> -->
</bean>
<!-- ...but ACCEPT those more than a configured link-hop-count from start... -->
<bean class="org.archive.modules.deciderules.TransclusionDecideRule">
<!-- <property name="maxTransHops" value="2" /> -->
<!-- <property name="maxSpeculativeHops" value="1" /> -->
</bean>
<!-- ...but REJECT those from a configurable (initially empty) set of REJECT SURTs... -->
<bean class="org.archive.modules.deciderules.surt.SurtPrefixedDecideRule" autowire="byName">
<property name="decision" value="REJECT"/>
<property name="seedsAsSurtPrefixes" value="false"/>
<bean class="org.archive.modules.deciderules.surt.SurtPrefixedDecideRule">
<property name="decision" value="REJECT"/>
<property name="seedsAsSurtPrefixes" value="false"/>
<property name="surtsDumpFile" value="negative-surts.dump" />
<!-- <property name="surtsSourceFile" value="" /> -->
</bean>
<!-- ...and REJECT those from a configurable (initially empty) set of URI regexes... -->
<bean class="org.archive.modules.deciderules.MatchesListRegexDecideRule">
<!-- <property name="listLogicalOr" value="true" /> -->
<!-- <property name="regexList">
<list>
</list>
</property> -->
</bean>
<!-- ...and REJECT those with suspicious repeating path-segments... -->
<bean class="org.archive.modules.deciderules.PathologicalPathDecideRule">
<!-- <property name="maxRepetitions" value="2" /> -->
</bean>
<!-- ...and REJECT those with more than threshold number of path-segments... -->
<bean class="org.archive.modules.deciderules.TooManyPathSegmentsDecideRule">
<!-- <property name="maxPathDepth" value="20" /> -->
</bean>
<!-- ...but always ACCEPT those marked as prerequisitee for another URI... -->
<bean class="org.archive.modules.deciderules.PrerequisiteAcceptDecideRule">
@@ -90,18 +151,106 @@ http://example.example/example
</property>
</bean>
<!-- Processors as named beans; assembled into FetchChain later -->
<bean id="preselector" class="org.archive.crawler.prefetch.Preselector">
<!--
PROCESSING CHAINS
Much of the crawler's work is specified by the sequential
application of swappable Processor modules. These Processors
are collected into three 'chains. The CandidateChain is applied
to URIs being considered for inclusion, before a URI is enqueued
for collection. The FetchChain is applied to URIs when their
turn for collection comes up. The DispositionChain is applied
after a URI is fetched and analyzed/link-extracted.
-->
<!-- CANDIDATE CHAIN -->
<!-- processors declared as named beans -->
<bean id="candidateScoper" class="org.archive.crawler.prefetch.CandidateScoper">
</bean>
<bean id="preconditionEnforcer" class="org.archive.crawler.prefetch.PreconditionEnforcer">
<bean id="preparer" class="org.archive.crawler.prefetch.FrontierPreparer">
<!-- <property name="preferenceDepthHops" value="-1" /> -->
<!-- <property name="preferenceEmbedHops" value="1" /> -->
<!-- <property name="canonicalizationPolicy">
<ref bean="canonicalizationPolicy" />
</property> -->
<!-- <property name="queueAssignmentPolicy">
<ref bean="queueAssignmentPolicy" />
</property> -->
<!-- <property name="uriPrecedencePolicy">
<ref bean="uriPrecedencePolicy" />
</property> -->
<!-- <property name="costAssignmentPolicy">
<ref bean="costAssignmentPolicy" />
</property> -->
</bean>
<!-- assembled into ordered CandidateChain bean -->
<bean id="candidateProcessors" class="org.archive.modules.CandidateChain">
<property name="processors">
<list>
<!-- apply scoping rules to each individual candidate URI... -->
<ref bean="candidateScoper"/>
<!-- ...then prepare those ACCEPTed for enqueuing to frontier. -->
<ref bean="preparer"/>
</list>
</property>
</bean>
<!-- FETCH CHAIN -->
<!-- processors declared as named beans -->
<bean id="preselector" class="org.archive.crawler.prefetch.Preselector">
<!-- <property name="recheckScope" value="false" /> -->
<!-- <property name="blockAll" value="false" /> -->
<!-- <property name="blockByRegex" value="" /> -->
<!-- <property name="allowByRegex" value="" /> -->
</bean>
<bean id="preconditions" class="org.archive.crawler.prefetch.PreconditionEnforcer">
<!-- <property name="ipValidityDurationSeconds" value="21600" /> -->
<!-- <property name="robotsValidityDurationSeconds" value="86400" /> -->
<!-- <property name="calculateRobotsOnly" value="false" /> -->
</bean>
<bean id="fetchDns" class="org.archive.modules.fetcher.FetchDNS">
<!-- <property name="acceptNonDnsResolves" value="false" /> -->
<!-- <property name="digestContent" value="true" /> -->
<!-- <property name="digestAlgorithm" value="sha1" /> -->
</bean>
<bean id="fetchHttp" class="org.archive.modules.fetcher.FetchHTTP">
<!-- <property name="maxLengthBytes" value="0" /> -->
<!-- <property name="timeoutSeconds" value="1200" /> -->
<!-- <property name="maxFetchKBSec" value="0" /> -->
<!-- <property name="defaultEncoding" value="ISO-8859-1" /> -->
<!-- <property name="shouldFetchBodyRule">
<bean class="org.archive.modules.deciderules.AcceptDecideRule"/>
</property> -->
<!-- <property name="soTimeoutMs" value="20000" /> -->
<!-- <property name="sendIfModifiedSince" value="true" /> -->
<!-- <property name="sendIfNoneMatch" value="true" /> -->
<!-- <property name="sendConnectionClose" value="true" /> -->
<!-- <property name="sendReferer" value="true" /> -->
<!-- <property name="sendRange" value="false" /> -->
<!-- <property name="ignoreCookies" value="false" /> -->
<!-- <property name="sslTrustLevel" value="OPEN" /> -->
<!-- <property name="acceptHeaders">
<list>
</list>
</property>
-->
<!-- <property name="httpBindAddress" value="" /> -->
<!-- <property name="httpProxyHost" value="" /> -->
<!-- <property name="httpProxyPort" value="0" /> -->
<!-- <property name="digestContent" value="true" /> -->
<!-- <property name="digestAlgorithm" value="sha1" /> -->
</bean>
<bean id="extractorHttp" class="org.archive.modules.extractor.ExtractorHTTP">
</bean>
<bean id="extractorHtml" class="org.archive.modules.extractor.ExtractorHTML">
<!-- <property name="extractJavascript" value="true" /> -->
<!-- <property name="extractValueAttributes" value="true" /> -->
<!-- <property name="ignoreFormActionUrls" value="false" /> -->
<!-- <property name="extractOnlyFormGets" value="true" /> -->
<!-- <property name="treatFramesAsEmbedLinks" value="true" /> -->
<!-- <property name="ignoreUnexpectedHtml" value="true" /> -->
<!-- <property name="maxElementLength" value="1024" /> -->
<!-- <property name="maxAttributeNameLength" value="1024" /> -->
<!-- <property name="maxAttributeValueLength" value="16384" /> -->
</bean>
<bean id="extractorCss" class="org.archive.modules.extractor.ExtractorCSS">
</bean>
@@ -109,109 +258,245 @@ http://example.example/example
</bean>
<bean id="extractorSwf" class="org.archive.modules.extractor.ExtractorSWF">
</bean>
<bean id="arcWriterProcessor" class="org.archive.modules.writer.ARCWriterProcessor">
</bean>
<!-- Processors as named beans; assembled into DispositionChain later -->
<bean id="disposition" class="org.archive.crawler.postprocessor.DispositionProcessor">
</bean>
<bean id="candidates" class="org.archive.crawler.postprocessor.CandidatesProcessor">
</bean>
<!-- Processors as named beans; assembled into CandidateChain later -->
<bean id="candidateScoper" class="org.archive.crawler.prefetch.CandidateScoper">
</bean>
<bean id="preparer" class="org.archive.crawler.prefetch.FrontierPreparer">
</bean>
<bean id="candidateProcessors" class="org.archive.modules.CandidateChain">
<property name="processors">
<list>
<ref bean="candidateScoper"/>
<ref bean="preparer"/>
</list>
</property>
</bean>
<!-- assembled into ordered FetchChain bean -->
<bean id="fetchProcessors" class="org.archive.modules.FetchChain">
<property name="processors">
<list>
<!-- recheck scope, if so enabled... -->
<ref bean="preselector"/>
<ref bean="preconditionEnforcer"/>
<!-- ...then verify or trigger prerequisite URIs fetched, allow crawling... -->
<ref bean="preconditions"/>
<!-- ...fetch if DNS URI... -->
<ref bean="fetchDns"/>
<!-- ...fetch if HTTP URI... -->
<ref bean="fetchHttp"/>
<!-- ...extract oulinks from HTTP headers... -->
<ref bean="extractorHttp"/>
<!-- ...extract oulinks from HTML content... -->
<ref bean="extractorHtml"/>
<!-- ...extract oulinks from CSS content... -->
<ref bean="extractorCss"/>
<!-- ...extract oulinks from Javascript content... -->
<ref bean="extractorJs"/>
<!-- ...extract oulinks from Flash content... -->
<ref bean="extractorSwf"/>
<ref bean="arcWriterProcessor"/>
</list>
</property>
</bean>
<bean id="dispositionProcessors" class="org.archive.modules.DispositionChain">
<!-- DISPOSITION CHAIN -->
<!-- processors declared as named beans -->
<bean id="warcWriter" class="org.archive.modules.writer.WARCWriterProcessor">
<!-- <property name="compress" value="true" /> -->
<!-- <property name="prefix" value="IAH" /> -->
<!-- <property name="suffix" value="${HOSTNAME}" /> -->
<!-- <property name="maxFileSizeBytes" value="1000000000" /> -->
<!-- <property name="poolMaxActive" value="1" /> -->
<!-- <property name="poolMaxWaitMs" value="300000" /> -->
<!-- <property name="skipIdenticalDigests" value="false" /> -->
<!-- <property name="maxTotalBytesToWrite" value="0" /> -->
<!-- <property name="directory" value="." /> -->
<!-- <property name="storePaths">
<list>
<value>warcs</value>
</list>
</property> -->
<!-- <property name="writeRequests" value="true" /> -->
<!-- <property name="writeMetadata" value="true" /> -->
<!-- <property name="writeRevisitForIdenticalDigests" value="true" /> -->
<!-- <property name="writeRevisitForNotModified" value="true" /> -->
</bean>
<bean id="candidates" class="org.archive.crawler.postprocessor.CandidatesProcessor">
<!-- <property name="seedsRedirectNewSeeds" value="true" /> -->
</bean>
<bean id="disposition" class="org.archive.crawler.postprocessor.DispositionProcessor">
<!-- <property name="delayFactor" value="5.0" /> -->
<!-- <property name="minDelayMs" value="3000" /> -->
<!-- <property name="respectCrawlDelayUpToSeconds" value="300" /> -->
<!-- <property name="maxDelayMs" value="30000" /> -->
<!-- <property name="maxPerHostBandwidthUsageKbSec" value="0" /> -->
</bean>
<!-- assembled into ordered DispositionChain bean -->
<bean id="dispositionProcessors" class="org.archive.modules.DispositionChain">
<property name="processors">
<list>
<ref bean="disposition"/>
<!-- write to aggregate archival files... -->
<ref bean="warcWriter"/>
<!-- ...send each outlink candidate URI to CandidatesChain,
and enqueue those ACCEPTed to the frontier... -->
<ref bean="candidates"/>
<!-- ...then update stats, shared-structures, frontier decisions -->
<ref bean="disposition"/>
</list>
</property>
</bean>
<!-- http or form login credentials -->
<!-- CRAWLCONTROLLER: Control interface, unifying context -->
<bean id="crawlController"
class="org.archive.crawler.framework.CrawlController">
<!-- <property name="maxToeThreads" value="25" /> -->
<!-- <property name="pauseAtStart" value="true" /> -->
<!-- <property name="pauseAtFinish" value="false" /> -->
<!-- <property name="recorderInBufferBytes" value="524288" /> -->
<!-- <property name="recorderOutBufferBytes" value="16384" /> -->
<!-- <property name="checkpointerPeriod" value="-1" /> -->
<!-- <property name="scratchDir" value="scratch" /> -->
<!-- <property name="checkpointsDir" value="checkpoints" /> -->
</bean>
<!-- FRONTIER: Record of all URIs discovered and queued-for-collection -->
<bean id="frontier"
class="org.archive.crawler.frontier.BdbFrontier">
<!-- <property name="dumpPendingAtClose" value="false" /> -->
<!-- <property name="holdQueues" value="true" /> -->
<!-- <property name="queueTotalBudget" value="-1" /> -->
<!-- <property name="balanceReplenishAmount" value="3000" /> -->
<!-- <property name="errorPenaltyAmount" value="100" /> -->
<!-- <property name="precedenceFloor" value="255" /> -->
<!-- <property name="queuePrecedencePolicy">
<bean class="org.archive.crawler.frontier.precedence.BaseQueuePrecedencePolicy" />
</property> -->
<!-- <property name="snoozeLongMs" value="300000" /> -->
<!-- <property name="dumpPendingAtClose" value="false" /> -->
</bean>
<!-- URI UNIQ FILTER: Used by frontier to remember already-included URIs -->
<bean id="uriUniqFilter"
class="org.archive.crawler.util.BdbUriUniqFilter">
</bean>
<!--
OPTIONAL BUT RECOMMENDED BEANS
-->
<!-- ACTIONDIRECTORY: disk directory for mid-crawl operations
Running job will watch directory for new files with URIs,
scripts, and other data to be processed during a crawl. -->
<bean id="actionDirectory" class="org.archive.crawler.framework.ActionDirectory">
<!-- <property name="actionDir" value="action" /> -->
<!-- <property name="initialDelaySeconds" value="10" /> -->
<!-- <property name="delaySeconds" value="30" /> -->
</bean>
<!--
OPTIONAL BEANS
Uncomment and expand as needed, or if non-default alternate
implementations are preferred.
-->
<!-- CANONICALIZATION POLICY -->
<!--
<bean id="canonicalizationPolicy"
class="org.archive.modules.canonicalize.RulesCanonicalizationPolicy">
<list>
<bean class="org.archive.modules.canonicalize.LowercaseRule" />
<bean class="org.archive.modules.canonicalize.StripUserinfoRule" />
<bean class="org.archive.modules.canonicalize.StripWWWNRule" />
<bean class="org.archive.modules.canonicalize.StripSessionIDs" />
<bean class="org.archive.modules.canonicalize.StripSessionCFIDs" />
<bean class="org.archive.modules.canonicalize.FixupQueryString" />
</list>
</bean>
-->
<!-- QUEUE ASSIGNMENT POLICY -->
<!--
<bean id="queueAssignmentPolicy"
class="org.archive.crawler.frontier.SurtAuthorityQueueAssignmentPolicy">
<property name="forceQueueAssignment" value="" />
<property name="deferToPrevious" value="true" />
<property name="parallelQueues" value="1" />
</bean>
-->
<!-- URI PRECEDENCE POLICY -->
<!--
<bean id="uriPrecedencePolicy"
class="org.archive.crawler.frontier.precedence.CostUriPrecedencePolicy">
</bean>
-->
<!-- COST ASSIGNMENT POLICY -->
<!--
<bean id="costAssignmentPolicy"
class="org.archive.crawler.frontier.UnitCostAssignmentPolicy">
</bean>
-->
<!-- CREDENTIAL STORE: HTTP authentication or FORM POST credentials -->
<!--
<bean id="credentialStore"
class="org.archive.modules.credential.CredentialStore">
</bean>
<bean id="frontier"
class="org.archive.crawler.frontier.BdbFrontier" autowire="byName">
-->
<!--
REQUIRED STANDARD BEANS
It will be very rare to replace or reconfigure the following beans.
-->
<!-- STATISTICSTRACKER: standard stats/reporting collector -->
<bean id="statisticsTracker"
class="org.archive.crawler.reporting.StatisticsTracker" autowire="byName">
<!-- <property name="reportsDir" value="reports" /> -->
<!-- <property name="liveHostReportSize" value="20" /> -->
<!-- <property name="intervalSeconds" value="20" /> -->
<!-- <property name="keepSnapshotsCount" value="5" /> -->
<!-- <property name="liveHostReportSize" value="20" /> -->
</bean>
<bean id="uriUniqFilter"
class="org.archive.crawler.util.BdbUriUniqFilter" autowire="byName">
<!-- CRAWLERLOGGERMODULE: shared logging facility -->
<bean id="loggerModule"
class="org.archive.crawler.reporting.CrawlerLoggerModule">
<!-- <property name="path" value="logs" /> -->
<!-- <property name="crawlLogPath" value="crawl.log" /> -->
<!-- <property name="alertsLogPath" value="alerts.log" /> -->
<!-- <property name="progressLogPath" value="progress-statistics.log" /> -->
<!-- <property name="uriErrorsLogPath" value="uri-errors.log" /> -->
<!-- <property name="runtimeErrorsLogPath" value="runtime-errors.log" /> -->
<!-- <property name="nonfatalErrorsLogPath" value="nonfatal-errors.log" /> -->
</bean>
<bean id="crawlController"
class="org.archive.crawler.framework.CrawlController" autowire="byName">
<!-- SHEETOVERLAYMANAGER: manager of sheets of contextual overlays
Autowired to include any SheetForSurtPrefix or
SheetForDecideRuled beans -->
<bean id="sheetOverlaysManager" autowire="byType"
class="org.archive.crawler.spring.SheetOverlaysManager">
</bean>
<!-- BDBMODULE: shared BDB-JE disk persistence manager -->
<bean id="bdb"
class="org.archive.bdb.BdbModule">
<!-- <property name="dir" value="state" /> -->
<!-- <property name="cachePercentage" value="60" /> -->
<!-- <property name="useSharedCache" value="true" /> -->
<!-- <property name="checkpointCopyLogs" value="true" /> -->
<!-- <property name="expectedConcurrency" value="25" /> -->
</bean>
<!-- BDBCOOKIESTORAGE: disk-based cookie storage for FetchHTTP -->
<bean id="cookieStorage"
class="org.archive.modules.fetcher.BdbCookieStorage">
<!-- <property name="cookiesLoadFile"><null/></property> -->
<!-- <property name="cookiesSaveFile"><null/></property> -->
<!-- <property name="bdb">
<ref bean="bdb"/>
</property> -->
</bean>
<!-- SERVERCACHE: shared cache of server/host info -->
<bean id="serverCache"
class="org.archive.modules.net.BdbServerCache">
<!-- <property name="bdb">
<ref bean="bdb"/>
</property> -->
</bean>
<!-- CONFIG PATH CONFIGURER: required helper making crawl paths relative
to crawler-beans.cxml file, and tracking crawl files for web UI -->
<bean id="configPathConfigurer"
class="org.archive.spring.ConfigPathConfigurer">
</bean>
<!-- IT WILL BE VERY RARE TO REPLACE OR CONFIGURE THE BELOW BEANS -->
<!-- standard stats/reporting collector -->
<bean id="statisticsTracker"
class="org.archive.crawler.reporting.StatisticsTracker" autowire="byName">
</bean>
<!-- Shared logging facility -->
<bean id="loggerModule"
class="org.archive.crawler.reporting.CrawlerLoggerModule">
</bean>
<!-- manager of sheets of contextual overlays; autowired to include
any SheetForSurtPrefix or SheetForDecideRuled beans -->
<bean id="sheetOverlaysManager"
class="org.archive.crawler.spring.SheetOverlaysManager" autowire="byType">
</bean>
<!-- shared BDB manager -->
<bean id="bdb" class="org.archive.bdb.BdbModule">
</bean>
<!-- cookie storage for FetchHTTP -->
<bean id="cookieStorage" class="org.archive.modules.fetcher.BdbCookieStorage">
</bean>
<!-- shared cache of server/host info -->
<bean id="serverCache" class="org.archive.modules.net.BdbServerCache">
</bean>
<!-- utility for injecting data to crawl via filesystem 'action' directory -->
<bean id="actionDirectory" class="org.archive.crawler.framework.ActionDirectory">
</bean>
</beans>
engine/src/main/java/org/archive/crawler/framework/ActionDirectory.java
+11 -11
View File
@@ -84,20 +84,20 @@ public class ActionDirectory implements ApplicationContextAware, Lifecycle, Runn
ScheduledExecutorService executor;
/** how long after crawl start to first scan action directory */
protected int initialDelay = 10;
public int getInitialDelay() {
return initialDelay;
protected int initialDelaySeconds = 10;
public int getInitialDelaySeconds() {
return initialDelaySeconds;
}
public void setInitialDelay(int initialDelay) {
this.initialDelay = initialDelay;
public void setInitialDelaySeconds(int initialDelay) {
this.initialDelaySeconds = initialDelay;
}
/** delay between scans of actionDirectory for new files */
protected int delay = 30;
public int getDelay() {
return delay;
protected int delaySeconds = 30;
public int getDelaySeconds() {
return delaySeconds;
}
public void setDelay(int delay) {
this.delay = delay;
public void setDelaySeconds(int delay) {
this.delaySeconds = delay;
}
/**
@@ -161,7 +161,7 @@ public class ActionDirectory implements ApplicationContextAware, Lifecycle, Runn
getDoneDir().getFile().mkdirs();
// start background executor
executor = Executors.newSingleThreadScheduledExecutor();
executor.scheduleWithFixedDelay(this, getInitialDelay(), getDelay(), TimeUnit.SECONDS);
executor.scheduleWithFixedDelay(this, getInitialDelaySeconds(), getDelaySeconds(), TimeUnit.SECONDS);
}
public void stop() {
engine/src/main/java/org/archive/crawler/framework/Scoper.java
+9 -9
View File
@@ -43,6 +43,15 @@ public abstract class Scoper extends Processor implements Lifecycle {
private static Logger LOGGER =
Logger.getLogger(Scoper.class.getName());
protected DecideRule scope;
public DecideRule getScope() {
return this.scope;
}
@Autowired
public void setScope(DecideRule scope) {
this.scope = scope;
}
FileHandler fileLogger = null;
/**
@@ -72,15 +81,6 @@ public abstract class Scoper extends Processor implements Lifecycle {
this.loggerModule = loggerModule;
}
protected DecideRule scope;
public DecideRule getScope() {
return this.scope;
}
@Autowired
public void setScope(DecideRule scope) {
this.scope = scope;
}
// FIXME: Weirdo log overriding might not work on a per-subclass basis,
// we may need to cut and paste it to the three subclasses, or eliminate
// it in favor of java.util.logging best practice.
engine/src/main/java/org/archive/crawler/postprocessor/DispositionProcessor.java
+1 -12
View File
@@ -31,7 +31,6 @@ import org.archive.modules.CrawlMetadata;
import org.archive.modules.CrawlURI;
import org.archive.modules.ModuleAttributeConstants;
import org.archive.modules.Processor;
import org.archive.modules.fetcher.UserAgentProvider;
import org.archive.modules.net.CrawlHost;
import org.archive.modules.net.CrawlServer;
import org.archive.modules.net.RobotsExclusionPolicy;
@@ -142,15 +141,6 @@ public class DispositionProcessor extends Processor {
this.metadata = provider;
}
public UserAgentProvider getUserAgentProvider() {
return (UserAgentProvider) kp.get("userAgentProvider");
}
@Autowired
public void setUserAgentProvider(UserAgentProvider provider) {
kp.put("userAgentProvider",provider);
}
public DispositionProcessor() {
super();
}
@@ -234,10 +224,9 @@ public class DispositionProcessor extends Processor {
// may need to extend wait
CrawlServer s = ServerCacheUtil.getServerFor(
getServerCache(),curi.getUURI());
UserAgentProvider uap = getUserAgentProvider();
String ua = curi.getUserAgent();
if (ua == null) {
ua = uap.getUserAgent();
ua = metadata.getUserAgent();
}
RobotsExclusionPolicy rep = s.getRobots();
if (rep != null) {
engine/src/main/java/org/archive/crawler/prefetch/PreconditionEnforcer.java
+22 -21
View File
@@ -57,21 +57,10 @@ import org.springframework.beans.factory.annotation.Autowired;
* @author gojomo
*/
public class PreconditionEnforcer extends Processor {
private static final long serialVersionUID = 3L;
private static final Logger logger =
Logger.getLogger(PreconditionEnforcer.class.getName());
public UserAgentProvider getUserAgentProvider() {
return (UserAgentProvider) kp.get("userAgentProvider");
}
@Autowired
public void setUserAgentProvider(UserAgentProvider provider) {
kp.put("userAgentProvider",provider);
}
/**
* The minimum interval for which a dns-record will be considered
* valid (in seconds). If the record's DNS TTL is larger, that will
@@ -113,10 +102,30 @@ public class PreconditionEnforcer extends Processor {
public boolean getCalculateRobotsOnly() {
return (Boolean) kp.get("calculateRobotsOnly");
}
public void setCalculateRobotsOnly(boolean recheck) {
kp.put("calculateRobotsOnly",recheck);
public void setCalculateRobotsOnly(boolean calcOnly) {
kp.put("calculateRobotsOnly",calcOnly);
}
public UserAgentProvider getUserAgentProvider() {
return (UserAgentProvider) kp.get("userAgentProvider");
}
@Autowired
public void setUserAgentProvider(UserAgentProvider provider) {
kp.put("userAgentProvider",provider);
}
{
// initialize with empty store so declaration not required
setCredentialStore(new CredentialStore());
}
public CredentialStore getCredentialStore() {
return (CredentialStore) kp.get("credentialStore");
}
@Autowired(required=false)
public void setCredentialStore(CredentialStore credentials) {
kp.put("credentialStore",credentials);
}
protected ServerCache serverCache;
public ServerCache getServerCache() {
return this.serverCache;
@@ -126,14 +135,6 @@ public class PreconditionEnforcer extends Processor {
this.serverCache = serverCache;
}
public CredentialStore getCredentialStore() {
return (CredentialStore) kp.get("credentialStore");
}
@Autowired
public void setCredentialStore(CredentialStore credentials) {
kp.put("credentialStore",credentials);
}
protected CrawlerLoggerModule loggerModule;
public CrawlerLoggerModule getLoggerModule() {
return this.loggerModule;
modules/src/main/java/org/archive/modules/deciderules/surt/SurtPrefixedDecideRule.java
+1 -1
View File
@@ -95,7 +95,7 @@ implements
* Dump file to save SURT prefixes actually used: Useful debugging SURTs.
*/
protected ConfigFile surtsDumpFile =
new ConfigFile("surtsDumpFile","");
new ConfigFile("surtsDumpFile","surts.dump");
public ConfigFile getSurtsDumpFile() {
return surtsDumpFile;
}
modules/src/main/java/org/archive/modules/extractor/ExtractorHTML.java
+12 -12
View File
@@ -120,28 +120,28 @@ public class ExtractorHTML extends ContentExtractor implements InitializingBean
// "(?is)(\\w+)\\s+.*?(?:(robots))\\s*=(?:(?:\\s*\"(.+)\")|(?:\\s*'(.+)')|(\\S+))");
{
setMaxAttrNameLength(1024); // 1K
setMaxAttributeNameLength(1024); // 1K
}
public int getMaxAttrNameLength() {
return (Integer) kp.get("maxAttrNameLength");
public int getMaxAttributeNameLength() {
return (Integer) kp.get("maxAttributeNameLength");
}
public void setMaxAttrNameLength(int max) {
kp.put("maxAttrNameLength", max);
public void setMaxAttributeNameLength(int max) {
kp.put("maxAttributeNameLength", max);
}
{
setMaxAttrValLength(16384); // 16K
setMaxAttributeValLength(16384); // 16K
}
public int getMaxAttrValLength() {
return (Integer) kp.get("maxAttrValLength");
public int getMaxAttributeValLength() {
return (Integer) kp.get("maxAttributeValLength");
}
public void setMaxAttrValLength(int max) {
kp.put("maxAttrValLength", max);
public void setMaxAttributeValLength(int max) {
kp.put("maxAttributeValLength", max);
}
// TODO: perhaps cut to near MAX_URI_LENGTH
@@ -317,9 +317,9 @@ public class ExtractorHTML extends ContentExtractor implements InitializingBean
regex = EACH_ATTRIBUTE_EXTRACTOR;
regex = regex.replace(MAX_ATTR_NAME_REPLACE,
Integer.toString(getMaxAttrNameLength()));
Integer.toString(getMaxAttributeNameLength()));
regex = regex.replace(MAX_ATTR_VAL_REPLACE,
Integer.toString(getMaxAttrValLength()));
Integer.toString(getMaxAttributeValLength()));
this.eachAttributeExtractor = Pattern.compile(regex);
}
modules/src/main/java/org/archive/modules/fetcher/FetchHTTP.java
+17 -9
View File
@@ -113,9 +113,10 @@ import org.springframework.context.Lifecycle;
* href="http://jakarta.apache.org/commons/httpclient/">Apache Jakarta Commons
* HttpClient</a> library.
*
* @author Gordon Mohr
* @author Igor Ranitovic
* @author others
* @contributor gojomo
* @contributor Igor Ranitovic
* @contributor stack
* @contributor others
* @version $Id$
*/
public class FetchHTTP extends Processor implements Lifecycle {
@@ -171,7 +172,7 @@ public class FetchHTTP extends Processor implements Lifecycle {
* retry read.
*/
{
setSoTimeoutMs(20*1000); // 20 minutes
setSoTimeoutMs(20*1000); // 20 seconds
}
public int getSoTimeoutMs() {
return (Integer) kp.get("soTimeoutMs");
@@ -278,11 +279,14 @@ public class FetchHTTP extends Processor implements Lifecycle {
* 'normal' (all valid certificates not including selfsigned) to 'strict'
* (Cert is valid and DN must match servername).
*/
public TrustLevel getTrustLevel() {
{
setSslTrustLevel(TrustLevel.OPEN);
}
public TrustLevel getSslTrustLevel() {
return (TrustLevel) kp.get("trustLevel");
}
public void getTrustLevel(TrustLevel trustLevel) {
kp.put("trustLevel",trustLevel);
public void setSslTrustLevel(TrustLevel trustLevel) {
kp.put("sslTrustLevel",trustLevel);
}
private transient HttpClient http = null;
@@ -452,10 +456,14 @@ public class FetchHTTP extends Processor implements Lifecycle {
/**
* Used to store credentials.
*/
{
// initialize with empty store so declaration not required
setCredentialStore(new CredentialStore());
}
public CredentialStore getCredentialStore() {
return (CredentialStore) kp.get("credentialStore");
}
@Autowired
@Autowired(required=false)
public void setCredentialStore(CredentialStore credentials) {
kp.put("credentialStore",credentials);
}
@@ -1311,7 +1319,7 @@ public class FetchHTTP extends Processor implements Lifecycle {
SSLContext context = SSLContext.getInstance("SSL");
context.init(null,
new TrustManager[] { new ConfigurableX509TrustManager(
getTrustLevel()) }, null);
getSslTrustLevel()) }, null);
this.sslfactory = context.getSocketFactory();
} catch (Exception e) {
logger.log(Level.WARNING, "Failed configure of ssl context "
modules/src/main/java/org/archive/modules/seeds/SeedModule.java
+1 -15
View File
@@ -40,7 +40,7 @@ public abstract class SeedModule implements Serializable
* from that seed. When present, such source tags appear in the
* second-to-last crawl.log field.
*/
boolean sourceTagSeeds;
boolean sourceTagSeeds = false;
public boolean getSourceTagSeeds() {
return sourceTagSeeds;
}
@@ -48,20 +48,6 @@ public abstract class SeedModule implements Serializable
this.sourceTagSeeds = sourceTagSeeds;
}
/**
* Whether to reread the seeds specification, whether it has changed or not,
* every time any configuration change occurs. If true, seeds are reread
* even when (for example) new domain overrides are set. Rereading the seeds
* can take a long time with large seed lists.
*/
protected boolean rereadSeedsOnConfig = true;
public boolean getRereadSeedsOnConfig() {
return rereadSeedsOnConfig;
}
public void setRereadSeedsOnConfig(boolean rereadSeedsOnConfig) {
this.rereadSeedsOnConfig = rereadSeedsOnConfig;
}
protected Set<SeedListener> seedListeners =
new HashSet<SeedListener>();
public Set<SeedListener> getSeedListeners() {
modules/src/main/java/org/archive/modules/writer/ARCWriterProcessor.java
+1 -1
View File
@@ -78,7 +78,7 @@ public class ARCWriterProcessor extends WriterPoolProcessor {
@Override
protected void setupPool(AtomicInteger serialNo) {
WriterPoolSettings wps = getWriterPoolSettings();
setPool(new ARCWriterPool(serialNo, wps, getPoolMaxActive(), getPoolMaxWait()));
setPool(new ARCWriterPool(serialNo, wps, getPoolMaxActive(), getPoolMaxWaitMs()));
}
/**
modules/src/main/java/org/archive/modules/writer/WARCWriterProcessor.java
+1 -1
View File
@@ -164,7 +164,7 @@ public class WARCWriterProcessor extends WriterPoolProcessor {
@Override
protected void setupPool(final AtomicInteger serialNo) {
WriterPoolSettings wps = getWriterPoolSettings();
setPool(new WARCWriterPool(serialNo, wps, getPoolMaxActive(), getPoolMaxWait()));
setPool(new WARCWriterPool(serialNo, wps, getPoolMaxActive(), getPoolMaxWaitMs()));
}
/**
modules/src/main/java/org/archive/modules/writer/WriterPoolProcessor.java
+5 -5
View File
@@ -129,12 +129,12 @@ implements Lifecycle {
* Maximum time to wait on pool element (milliseconds). This setting cannot
* be varied over the life of a crawl.
*/
int poolMaxWait = WriterPool.DEFAULT_MAXIMUM_WAIT;
public int getPoolMaxWait() {
return poolMaxWait;
int poolMaxWaitMs = WriterPool.DEFAULT_MAXIMUM_WAIT;
public int getPoolMaxWaitMs() {
return poolMaxWaitMs;
}
public void setPoolMaxWait(int poolMaxWait) {
this.poolMaxWait = poolMaxWait;
public void setPoolMaxWaitMs(int poolMaxWait) {
this.poolMaxWaitMs = poolMaxWait;
}
/**