mirror of
https://github.com/internetarchive/heritrix3.git
synced 2026-09-27 08:05:46 +00:00
[HER-1641] migration tool to convert Heritrix 1.14.x order.xml to H3 crawler-beans.cxml
* MigrateH1to3Tool.java
comment, clean unused code
* H1toH3.map
text map of {H3-override-beanpath, H1-settings-XPath, H1-default-value}
* migrate-template-crawler-beans.cxml
version of bundled profile slightly changed for better mapping to H1
(target replace range, seeds.txt, arcWriter)
This commit is contained in:
gojomo
@@ -1,121 +1,121 @@
|
||||
metadata.jobName //meta/name default
|
||||
metadata.description //meta/description Default Profile
|
||||
metadata.operator //meta/operator Admin
|
||||
# //meta/date 20091006215050
|
||||
# //controller/*[@name="settings-directory"] settings
|
||||
loggerModule.path //controller/*[@name="logs-path"] logs
|
||||
crawlController.checkpointsDir //controller/*[@name="checkpoints-path"] checkpoints
|
||||
bdb.dir //controller/*[@name="state-path"] state
|
||||
crawlController.scratchDir //controller/*[@name="scratch-path"] scratch
|
||||
crawlLimiter.maxBytesDownload //controller/*[@name="max-bytes-download"] 0
|
||||
crawlLimiter.maxDocumentsDownload //controller/*[@name="max-document-download"] 0
|
||||
crawlLimiter.maxTimeSeconds //controller/*[@name="max-time-sec"] 0
|
||||
crawlController.maxToeThreads //controller/*[@name="max-toe-threads"] 50
|
||||
crawlController.recorderOutBufferBytes //controller/*[@name="recorder-out-buffer-bytes"] 4096
|
||||
crawlController.recorderInBufferBytes //controller/*[@name="recorder-in-buffer-bytes"] 65536
|
||||
bdb.cachePercent //controller/*[@name="bdb-cache-percent"] 0
|
||||
! //controller/*[@name="scope"]/*[@name="enabled"] true
|
||||
seeds.textSource.path //controller/*[@name="scope"]/*[@name="seedsfile"] seeds.txt
|
||||
# //controller/*[@name="scope"]/*[@name="reread-seeds-on-config"] true
|
||||
?scope.rules[1].decision //controller/*[@name="scope"]/*[@name="decide-rules"]/*[@name="rules"]/*[@name="acceptIfSurtPrefixed"]/*[@name="decision"] ACCEPT
|
||||
?scope.rules[1].seedsAsSurtPrefixes //controller/*[@name="scope"]/*[@name="decide-rules"]/*[@name="rules"]/*[@name="acceptIfSurtPrefixed"]/*[@name="seeds-as-surt-prefixes"] true
|
||||
?scope.rules[1].alsoCheckVia //controller/*[@name="scope"]/*[@name="decide-rules"]/*[@name="rules"]/*[@name="acceptIfSurtPrefixed"]/*[@name="also-check-via"] false
|
||||
# //controller/*[@name="scope"]/*[@name="decide-rules"]/*[@name="rules"]/*[@name="acceptIfSurtPrefixed"]/*[@name="rebuild-on-reconfig"] true
|
||||
?scope.rules[2].maxHops //controller/*[@name="scope"]/*[@name="decide-rules"]/*[@name="rules"]/*[@name="rejectIfTooManyHops"]/*[@name="max-hops"] 20
|
||||
?scope.rules[3].maxTransHops //controller/*[@name="scope"]/*[@name="decide-rules"]/*[@name="rules"]/*[@name="acceptIfTranscluded"]/*[@name="max-trans-hops"] 3
|
||||
?scope.rules[3].maxSpeculativeHops //controller/*[@name="scope"]/*[@name="decide-rules"]/*[@name="rules"]/*[@name="acceptIfTranscluded"]/*[@name="max-speculative-hops"] 1
|
||||
?scope.rules[6].maxRepetitions //controller/*[@name="scope"]/*[@name="decide-rules"]/*[@name="rules"]/*[@name="rejectIfPathological"]/*[@name="max-repetitions"] 2
|
||||
?scope.rules[7].maxPathDepth //controller/*[@name="scope"]/*[@name="decide-rules"]/*[@name="rules"]/*[@name="rejectIfTooManyPathSegs"]/*[@name="max-path-depth"] 20
|
||||
*metadata.userAgentTemplate //controller/*[@name="http-headers"]/*[@name="user-agent"] Mozilla/5.0 (compatible; heritrix/@VERSION@ +PROJECT_URL_HERE)
|
||||
metadata.operatorFrom //controller/*[@name="http-headers"]/*[@name="from"] CONTACT_EMAIL_ADDRESS_HERE
|
||||
//controller/*[@name="robots-honoring-policy"]/*[@name="type"] classic
|
||||
//controller/*[@name="robots-honoring-policy"]/*[@name="masquerade"] false
|
||||
//controller/*[@name="frontier"]/*[@name="delay-factor"] 4.0
|
||||
//controller/*[@name="frontier"]/*[@name="max-delay-ms"] 20000
|
||||
//controller/*[@name="frontier"]/*[@name="min-delay-ms"] 2000
|
||||
//controller/*[@name="frontier"]/*[@name="respect-crawl-delay-up-to-secs"] 300
|
||||
//controller/*[@name="frontier"]/*[@name="max-retries"] 30
|
||||
//controller/*[@name="frontier"]/*[@name="retry-delay-seconds"] 900
|
||||
//controller/*[@name="frontier"]/*[@name="preference-embed-hops"] 1
|
||||
//controller/*[@name="frontier"]/*[@name="total-bandwidth-usage-KB-sec"] 0
|
||||
//controller/*[@name="frontier"]/*[@name="max-per-host-bandwidth-usage-KB-sec"] 0
|
||||
//controller/*[@name="frontier"]/*[@name="queue-assignment-policy"] org.archive.crawler.frontier.HostnameQueueAssignmentPolicy
|
||||
//controller/*[@name="frontier"]/*[@name="pause-at-start"] false
|
||||
//controller/*[@name="frontier"]/*[@name="pause-at-finish"] false
|
||||
//controller/*[@name="frontier"]/*[@name="source-tag-seeds"] false
|
||||
//controller/*[@name="frontier"]/*[@name="recovery-log-enabled"] true
|
||||
//controller/*[@name="frontier"]/*[@name="hold-queues"] true
|
||||
//controller/*[@name="frontier"]/*[@name="balance-replenish-amount"] 3000
|
||||
//controller/*[@name="frontier"]/*[@name="error-penalty-amount"] 100
|
||||
//controller/*[@name="frontier"]/*[@name="queue-total-budget"] -1
|
||||
//controller/*[@name="frontier"]/*[@name="cost-policy"] org.archive.crawler.frontier.ZeroCostAssignmentPolicy
|
||||
//controller/*[@name="frontier"]/*[@name="snooze-deactivate-ms"] 300000
|
||||
//controller/*[@name="frontier"]/*[@name="target-ready-backlog"] 50
|
||||
//controller/*[@name="frontier"]/*[@name="uri-included-structure"] org.archive.crawler.util.BdbUriUniqFilter
|
||||
//controller/*[@name="frontier"]/*[@name="dump-pending-at-close"] false
|
||||
//controller/*[@name="uri-canonicalization-rules"]/*[@name="Lowercase"]/*[@name="enabled"] true
|
||||
//controller/*[@name="uri-canonicalization-rules"]/*[@name="Userinfo"]/*[@name="enabled"] true
|
||||
//controller/*[@name="uri-canonicalization-rules"]/*[@name="WWW[0-9]*"]/*[@name="enabled"] true
|
||||
//controller/*[@name="uri-canonicalization-rules"]/*[@name="SessionIDs"]/*[@name="enabled"] true
|
||||
//controller/*[@name="uri-canonicalization-rules"]/*[@name="SessionCFIDs"]/*[@name="enabled"] true
|
||||
//controller/*[@name="uri-canonicalization-rules"]/*[@name="QueryStrPrefix"]/*[@name="enabled"] true
|
||||
//controller/*[@name="pre-fetch-processors"]/*[@name="Preselector"]/*[@name="enabled"] true
|
||||
//controller/*[@name="pre-fetch-processors"]/*[@name="Preselector"]/*[@name="override-logger"] false
|
||||
//controller/*[@name="pre-fetch-processors"]/*[@name="Preselector"]/*[@name="recheck-scope"] true
|
||||
//controller/*[@name="pre-fetch-processors"]/*[@name="Preselector"]/*[@name="block-all"] false
|
||||
//controller/*[@name="pre-fetch-processors"]/*[@name="Preprocessor"]/*[@name="enabled"] true
|
||||
//controller/*[@name="pre-fetch-processors"]/*[@name="Preprocessor"]/*[@name="ip-validity-duration-seconds"] 21600
|
||||
//controller/*[@name="pre-fetch-processors"]/*[@name="Preprocessor"]/*[@name="robot-validity-duration-seconds"] 86400
|
||||
//controller/*[@name="pre-fetch-processors"]/*[@name="Preprocessor"]/*[@name="calculate-robots-only"] false
|
||||
//controller/*[@name="fetch-processors"]/*[@name="DNS"]/*[@name="enabled"] true
|
||||
//controller/*[@name="fetch-processors"]/*[@name="DNS"]/*[@name="accept-non-dns-resolves"] false
|
||||
//controller/*[@name="fetch-processors"]/*[@name="DNS"]/*[@name="digest-content"] true
|
||||
//controller/*[@name="fetch-processors"]/*[@name="DNS"]/*[@name="digest-algorithm"] sha1
|
||||
//controller/*[@name="fetch-processors"]/*[@name="HTTP"]/*[@name="enabled"] true
|
||||
//controller/*[@name="fetch-processors"]/*[@name="HTTP"]/*[@name="timeout-seconds"] 1200
|
||||
//controller/*[@name="fetch-processors"]/*[@name="HTTP"]/*[@name="sotimeout-ms"] 20000
|
||||
//controller/*[@name="fetch-processors"]/*[@name="HTTP"]/*[@name="fetch-bandwidth"] 0
|
||||
//controller/*[@name="fetch-processors"]/*[@name="HTTP"]/*[@name="max-length-bytes"] 0
|
||||
//controller/*[@name="fetch-processors"]/*[@name="HTTP"]/*[@name="ignore-cookies"] false
|
||||
//controller/*[@name="fetch-processors"]/*[@name="HTTP"]/*[@name="use-bdb-for-cookies"] true
|
||||
//controller/*[@name="fetch-processors"]/*[@name="HTTP"]/*[@name="trust-level"] open
|
||||
//controller/*[@name="fetch-processors"]/*[@name="HTTP"]/*[@name="default-encoding"] ISO-8859-1
|
||||
//controller/*[@name="fetch-processors"]/*[@name="HTTP"]/*[@name="digest-content"] true
|
||||
//controller/*[@name="fetch-processors"]/*[@name="HTTP"]/*[@name="digest-algorithm"] sha1
|
||||
//controller/*[@name="fetch-processors"]/*[@name="HTTP"]/*[@name="send-if-modified-since"] true
|
||||
//controller/*[@name="fetch-processors"]/*[@name="HTTP"]/*[@name="send-if-none-match"] true
|
||||
//controller/*[@name="fetch-processors"]/*[@name="HTTP"]/*[@name="send-connection-close"] true
|
||||
//controller/*[@name="fetch-processors"]/*[@name="HTTP"]/*[@name="send-referer"] true
|
||||
//controller/*[@name="fetch-processors"]/*[@name="HTTP"]/*[@name="send-range"] false
|
||||
//controller/*[@name="extract-processors"]/*[@name="ExtractorHTTP"]/*[@name="enabled"] true
|
||||
//controller/*[@name="extract-processors"]/*[@name="ExtractorHTML"]/*[@name="enabled"] true
|
||||
//controller/*[@name="extract-processors"]/*[@name="ExtractorHTML"]/*[@name="extract-javascript"] true
|
||||
//controller/*[@name="extract-processors"]/*[@name="ExtractorHTML"]/*[@name="treat-frames-as-embed-links"] true
|
||||
//controller/*[@name="extract-processors"]/*[@name="ExtractorHTML"]/*[@name="ignore-form-action-urls"] false
|
||||
//controller/*[@name="extract-processors"]/*[@name="ExtractorHTML"]/*[@name="extract-only-form-gets"] true
|
||||
//controller/*[@name="extract-processors"]/*[@name="ExtractorHTML"]/*[@name="extract-value-attributes"] true
|
||||
//controller/*[@name="extract-processors"]/*[@name="ExtractorHTML"]/*[@name="ignore-unexpected-html"] true
|
||||
? //controller/*[@name="extract-processors"]/*[@name="ExtractorCSS"]/*[@name="enabled"] true
|
||||
? //controller/*[@name="extract-processors"]/*[@name="ExtractorJS"]/*[@name="enabled"] true
|
||||
? //controller/*[@name="extract-processors"]/*[@name="ExtractorSWF"]/*[@name="enabled"] true
|
||||
? //controller/*[@name="write-processors"]/*[@name="Archiver"]/*[@name="enabled"] true
|
||||
warcWriter.compress //controller/*[@name="write-processors"]/*[@name="Archiver"]/*[@name="compress"] true
|
||||
warcWriter.prefix //controller/*[@name="write-processors"]/*[@name="Archiver"]/*[@name="prefix"] IAH
|
||||
warcWriter.suffix //controller/*[@name="write-processors"]/*[@name="Archiver"]/*[@name="suffix"] ${HOSTNAME}
|
||||
warcWriter.maxFileSizeBytes //controller/*[@name="write-processors"]/*[@name="Archiver"]/*[@name="max-size-bytes"] 100000000
|
||||
? warcWriter.storePaths.(list) //controller/*[@name="write-processors"]/*[@name="Archiver"]/*[@name="path"]/string arcs
|
||||
warcWriter.poolMaxActive //controller/*[@name="write-processors"]/*[@name="Archiver"]/*[@name="pool-max-active"] 5
|
||||
warcWriter.poolMaxWaitMs //controller/*[@name="write-processors"]/*[@name="Archiver"]/*[@name="pool-max-wait"] 300000
|
||||
warcWriter.maxTotalBytesToWrite //controller/*[@name="write-processors"]/*[@name="Archiver"]/*[@name="total-bytes-to-write"] 0
|
||||
warcWriter.skipIdenticalDigests //controller/*[@name="write-processors"]/*[@name="Archiver"]/*[@name="skip-identical-digests"] false
|
||||
? //controller/*[@name="post-processors"]/*[@name="Updater"]/*[@name="enabled"] true
|
||||
+ //controller/*[@name="post-processors"]/*[@name="LinksScoper"]/*[@name="enabled"] true
|
||||
+ linkScoper //controller/*[@name="post-processors"]/*[@name="LinksScoper"]/*[@name="override-logger"] false
|
||||
candidates.seedsRedirectNewSeeds //controller/*[@name="post-processors"]/*[@name="LinksScoper"]/*[@name="seed-redirects-new-seed"] true
|
||||
preparer.preferenceDepthHops //controller/*[@name="post-processors"]/*[@name="LinksScoper"]/*[@name="preference-depth-hops"] -1
|
||||
? //controller/*[@name="post-processors"]/*[@name="Scheduler"]/*[@name="enabled"] true
|
||||
statisticsTracker.intervalSeconds //controller/*[@name="loggers"]/*[@name="crawl-statistics"]/*[@name="interval-seconds"] 20
|
||||
? //controller/*[@name="checkpoint-copy-bdbje-logs"] true
|
||||
? //controller/*[@name="recover-retain-failures"] false
|
||||
? //controller/*[@name="recover-scope-includes"] true
|
||||
? //controller/*[@name="recover-scope-enqueues"] true
|
||||
metadata.jobName|//meta/name|default
|
||||
metadata.description|//meta/description|Default Profile
|
||||
metadata.operator|//meta/operator|Admin
|
||||
$na|//meta/date|20091006215050
|
||||
$na|//controller/*[@name="settings-directory"]|settings
|
||||
loggerModule.path|//controller/*[@name="logs-path"]|logs
|
||||
crawlController.checkpointsDir|//controller/*[@name="checkpoints-path"]|checkpoints
|
||||
bdb.dir|//controller/*[@name="state-path"]|state
|
||||
crawlController.scratchDir|//controller/*[@name="scratch-path"]|scratch
|
||||
crawlLimiter.maxBytesDownload|//controller/*[@name="max-bytes-download"]|0
|
||||
crawlLimiter.maxDocumentsDownload|//controller/*[@name="max-document-download"]|0
|
||||
crawlLimiter.maxTimeSeconds|//controller/*[@name="max-time-sec"]|0
|
||||
crawlController.maxToeThreads|//controller/*[@name="max-toe-threads"]|50
|
||||
crawlController.recorderOutBufferBytes|//controller/*[@name="recorder-out-buffer-bytes"]|4096
|
||||
crawlController.recorderInBufferBytes|//controller/*[@name="recorder-in-buffer-bytes"]|65536
|
||||
bdb.cachePercent|//controller/*[@name="bdb-cache-percent"]|0
|
||||
$na|//controller/*[@name="scope"]/*[@name="enabled"]|true
|
||||
seeds.textSource.path|//controller/*[@name="scope"]/*[@name="seedsfile"]|seeds.txt
|
||||
$na|//controller/*[@name="scope"]/*[@name="reread-seeds-on-config"]|true
|
||||
scope.rules[1].decision|//controller/*[@name="scope"]/*[@name="decide-rules"]/*[@name="rules"]/*[@name="acceptIfSurtPrefixed"]/*[@name="decision"]|ACCEPT
|
||||
scope.rules[1].seedsAsSurtPrefixes|//controller/*[@name="scope"]/*[@name="decide-rules"]/*[@name="rules"]/*[@name="acceptIfSurtPrefixed"]/*[@name="seeds-as-surt-prefixes"]|true
|
||||
scope.rules[1].alsoCheckVia|//controller/*[@name="scope"]/*[@name="decide-rules"]/*[@name="rules"]/*[@name="acceptIfSurtPrefixed"]/*[@name="also-check-via"]|false
|
||||
$na|//controller/*[@name="scope"]/*[@name="decide-rules"]/*[@name="rules"]/*[@name="acceptIfSurtPrefixed"]/*[@name="rebuild-on-reconfig"]|true
|
||||
scope.rules[2].maxHops|//controller/*[@name="scope"]/*[@name="decide-rules"]/*[@name="rules"]/*[@name="rejectIfTooManyHops"]/*[@name="max-hops"]|20
|
||||
scope.rules[3].maxTransHops|//controller/*[@name="scope"]/*[@name="decide-rules"]/*[@name="rules"]/*[@name="acceptIfTranscluded"]/*[@name="max-trans-hops"]|3
|
||||
scope.rules[3].maxSpeculativeHops|//controller/*[@name="scope"]/*[@name="decide-rules"]/*[@name="rules"]/*[@name="acceptIfTranscluded"]/*[@name="max-speculative-hops"]|1
|
||||
scope.rules[6].maxRepetitions|//controller/*[@name="scope"]/*[@name="decide-rules"]/*[@name="rules"]/*[@name="rejectIfPathological"]/*[@name="max-repetitions"]|2
|
||||
scope.rules[7].maxPathDepth|//controller/*[@name="scope"]/*[@name="decide-rules"]/*[@name="rules"]/*[@name="rejectIfTooManyPathSegs"]/*[@name="max-path-depth"]|20
|
||||
*metadata.userAgentTemplate|//controller/*[@name="http-headers"]/*[@name="user-agent"]|Mozilla/5.0|(compatible;|heritrix/@VERSION@|+PROJECT_URL_HERE)
|
||||
metadata.operatorFrom|//controller/*[@name="http-headers"]/*[@name="from"]|CONTACT_EMAIL_ADDRESS_HERE
|
||||
*metadata.robotsHonoringPolicy.type|//controller/*[@name="robots-honoring-policy"]/*[@name="type"]|classic
|
||||
metadata.robotsHonoringPolicy.masquerade|//controller/*[@name="robots-honoring-policy"]/*[@name="masquerade"]|false
|
||||
disposition.delayFactor|//controller/*[@name="frontier"]/*[@name="delay-factor"]|4.0
|
||||
disposition.maxDelayMs|//controller/*[@name="frontier"]/*[@name="max-delay-ms"]|20000
|
||||
disposition.minDelayMs|//controller/*[@name="frontier"]/*[@name="min-delay-ms"]|2000
|
||||
disposition.respectCrawlDelayUpToSeconds|//controller/*[@name="frontier"]/*[@name="respect-crawl-delay-up-to-secs"]|300
|
||||
frontier.maxRetries|//controller/*[@name="frontier"]/*[@name="max-retries"]|30
|
||||
frontier.retryDelaySeconds|//controller/*[@name="frontier"]/*[@name="retry-delay-seconds"]|900
|
||||
preparer.preferenceEmbedHops|//controller/*[@name="frontier"]/*[@name="preference-embed-hops"]|1
|
||||
$na|//controller/*[@name="frontier"]/*[@name="total-bandwidth-usage-KB-sec"]|0
|
||||
disposition.maxPerHostBandwidthUsageKbSec|//controller/*[@name="frontier"]/*[@name="max-per-host-bandwidth-usage-KB-sec"]|0
|
||||
*preparer.queueAssignmentPolicy|//controller/*[@name="frontier"]/*[@name="queue-assignment-policy"]|org.archive.crawler.frontier.HostnameQueueAssignmentPolicy
|
||||
crawlController.pauseAtStart|//controller/*[@name="frontier"]/*[@name="pause-at-start"]|false
|
||||
crawlController.pauseAtFinish|//controller/*[@name="frontier"]/*[@name="pause-at-finish"]|false
|
||||
seeds.sourceTagSeeds|//controller/*[@name="frontier"]/*[@name="source-tag-seeds"]|false
|
||||
frontier.recoveryLogEnabled|//controller/*[@name="frontier"]/*[@name="recovery-log-enabled"]|true
|
||||
frontier.holdQueues|//controller/*[@name="frontier"]/*[@name="hold-queues"]|true
|
||||
frontier.balanceReplenishAmount|//controller/*[@name="frontier"]/*[@name="balance-replenish-amount"]|3000
|
||||
frontier.errorPenaltyAmount|//controller/*[@name="frontier"]/*[@name="error-penalty-amount"]|100
|
||||
frontier.queueTotalBudget|//controller/*[@name="frontier"]/*[@name="queue-total-budget"]|-1
|
||||
*preparer.costAssignmentPolicy|//controller/*[@name="frontier"]/*[@name="cost-policy"]|org.archive.crawler.frontier.ZeroCostAssignmentPolicy
|
||||
frontier.snoozeLongMs|//controller/*[@name="frontier"]/*[@name="snooze-deactivate-ms"]|300000
|
||||
$na|//controller/*[@name="frontier"]/*[@name="target-ready-backlog"]|50
|
||||
*frontier.uriUniqFilter|//controller/*[@name="frontier"]/*[@name="uri-included-structure"]|org.archive.crawler.util.BdbUriUniqFilter
|
||||
frontier.dumpPendingAtClose|//controller/*[@name="frontier"]/*[@name="dump-pending-at-close"]|false
|
||||
$na|//controller/*[@name="uri-canonicalization-rules"]/*[@name="Lowercase"]/*[@name="enabled"]|true
|
||||
$na|//controller/*[@name="uri-canonicalization-rules"]/*[@name="Userinfo"]/*[@name="enabled"]|true
|
||||
$na|//controller/*[@name="uri-canonicalization-rules"]/*[@name="WWW[0-9]*"]/*[@name="enabled"]|true
|
||||
$na|//controller/*[@name="uri-canonicalization-rules"]/*[@name="SessionIDs"]/*[@name="enabled"]|true
|
||||
$na|//controller/*[@name="uri-canonicalization-rules"]/*[@name="SessionCFIDs"]/*[@name="enabled"]|true
|
||||
$na|//controller/*[@name="uri-canonicalization-rules"]/*[@name="QueryStrPrefix"]/*[@name="enabled"]|true
|
||||
preselector.enabled|//controller/*[@name="pre-fetch-processors"]/*[@name="Preselector"]/*[@name="enabled"]|true
|
||||
preselector.logToFile|//controller/*[@name="pre-fetch-processors"]/*[@name="Preselector"]/*[@name="override-logger"]|false
|
||||
preselector.recheckScope|//controller/*[@name="pre-fetch-processors"]/*[@name="Preselector"]/*[@name="recheck-scope"]|true
|
||||
preselector.blockAll|//controller/*[@name="pre-fetch-processors"]/*[@name="Preselector"]/*[@name="block-all"]|false
|
||||
preconditions.enabled|//controller/*[@name="pre-fetch-processors"]/*[@name="Preprocessor"]/*[@name="enabled"]|true
|
||||
preconditions.ipValidityDurationSeconds|//controller/*[@name="pre-fetch-processors"]/*[@name="Preprocessor"]/*[@name="ip-validity-duration-seconds"]|21600
|
||||
preconditions.robotsValidityDurationSeconds|//controller/*[@name="pre-fetch-processors"]/*[@name="Preprocessor"]/*[@name="robot-validity-duration-seconds"]|86400
|
||||
preconditions.calculateRobotsOnly|//controller/*[@name="pre-fetch-processors"]/*[@name="Preprocessor"]/*[@name="calculate-robots-only"]|false
|
||||
fetchDns.enabled|//controller/*[@name="fetch-processors"]/*[@name="DNS"]/*[@name="enabled"]|true
|
||||
fetchDns.acceptNonDnsResolves|//controller/*[@name="fetch-processors"]/*[@name="DNS"]/*[@name="accept-non-dns-resolves"]|false
|
||||
fetchDns.digestContent|//controller/*[@name="fetch-processors"]/*[@name="DNS"]/*[@name="digest-content"]|true
|
||||
fetchDns.digestAlgorithm|//controller/*[@name="fetch-processors"]/*[@name="DNS"]/*[@name="digest-algorithm"]|sha1
|
||||
fetchHttp.enabled|//controller/*[@name="fetch-processors"]/*[@name="HTTP"]/*[@name="enabled"]|true
|
||||
fetchHttp.timeoutSeconds|//controller/*[@name="fetch-processors"]/*[@name="HTTP"]/*[@name="timeout-seconds"]|1200
|
||||
fetchHttp.soTimeoutMs|//controller/*[@name="fetch-processors"]/*[@name="HTTP"]/*[@name="sotimeout-ms"]|20000
|
||||
fetchHttp.maxFetchKBSec|//controller/*[@name="fetch-processors"]/*[@name="HTTP"]/*[@name="fetch-bandwidth"]|0
|
||||
fetchHttp.maxLengthBytes|//controller/*[@name="fetch-processors"]/*[@name="HTTP"]/*[@name="max-length-bytes"]|0
|
||||
fetchHttp.ignoreCookies|//controller/*[@name="fetch-processors"]/*[@name="HTTP"]/*[@name="ignore-cookies"]|false
|
||||
$na|//controller/*[@name="fetch-processors"]/*[@name="HTTP"]/*[@name="use-bdb-for-cookies"]|true
|
||||
*fetchHttp.sslTrustLevel|//controller/*[@name="fetch-processors"]/*[@name="HTTP"]/*[@name="trust-level"]|open
|
||||
fetchHttp.defaultEncoding|//controller/*[@name="fetch-processors"]/*[@name="HTTP"]/*[@name="default-encoding"]|ISO-8859-1
|
||||
fetchHttp.digestContent|//controller/*[@name="fetch-processors"]/*[@name="HTTP"]/*[@name="digest-content"]|true
|
||||
fetchHttp.digestAlgorithm|//controller/*[@name="fetch-processors"]/*[@name="HTTP"]/*[@name="digest-algorithm"]|sha1
|
||||
fetchHttp.sendIfModifiedSince|//controller/*[@name="fetch-processors"]/*[@name="HTTP"]/*[@name="send-if-modified-since"]|true
|
||||
fetchHttp.sendIfNoneMatch|//controller/*[@name="fetch-processors"]/*[@name="HTTP"]/*[@name="send-if-none-match"]|true
|
||||
fetchHttp.sendConnectionClose|//controller/*[@name="fetch-processors"]/*[@name="HTTP"]/*[@name="send-connection-close"]|true
|
||||
fetchHttp.sendReferer|//controller/*[@name="fetch-processors"]/*[@name="HTTP"]/*[@name="send-referer"]|true
|
||||
fetchHttp.sendRange|//controller/*[@name="fetch-processors"]/*[@name="HTTP"]/*[@name="send-range"]|false
|
||||
extractorHttp.enabled|//controller/*[@name="extract-processors"]/*[@name="ExtractorHTTP"]/*[@name="enabled"]|true
|
||||
extractorHtml.enabled|//controller/*[@name="extract-processors"]/*[@name="ExtractorHTML"]/*[@name="enabled"]|true
|
||||
extractorHtml.extractJavascript|//controller/*[@name="extract-processors"]/*[@name="ExtractorHTML"]/*[@name="extract-javascript"]|true
|
||||
extractorHtml.treatFramesAsEmbedLinks|//controller/*[@name="extract-processors"]/*[@name="ExtractorHTML"]/*[@name="treat-frames-as-embed-links"]|true
|
||||
extractorHtml.ignoreFormActionUrls|//controller/*[@name="extract-processors"]/*[@name="ExtractorHTML"]/*[@name="ignore-form-action-urls"]|false
|
||||
extractorHtml.extractOnlyFormGets|//controller/*[@name="extract-processors"]/*[@name="ExtractorHTML"]/*[@name="extract-only-form-gets"]|true
|
||||
extractorHtml.extractValueAttributes|//controller/*[@name="extract-processors"]/*[@name="ExtractorHTML"]/*[@name="extract-value-attributes"]|true
|
||||
extractorHtml.ignoreUnexpectedHtml|//controller/*[@name="extract-processors"]/*[@name="ExtractorHTML"]/*[@name="ignore-unexpected-html"]|true
|
||||
extractorCss.enabled|//controller/*[@name="extract-processors"]/*[@name="ExtractorCSS"]/*[@name="enabled"]|true
|
||||
extractorJs.enabled|//controller/*[@name="extract-processors"]/*[@name="ExtractorJS"]/*[@name="enabled"]|true
|
||||
extractorSwf.enabled|//controller/*[@name="extract-processors"]/*[@name="ExtractorSWF"]/*[@name="enabled"]|true
|
||||
arcWriter.enabled|//controller/*[@name="write-processors"]/*[@name="Archiver"]/*[@name="enabled"]|true
|
||||
arcWriter.compress|//controller/*[@name="write-processors"]/*[@name="Archiver"]/*[@name="compress"]|true
|
||||
arcWriter.prefix|//controller/*[@name="write-processors"]/*[@name="Archiver"]/*[@name="prefix"]|IAH
|
||||
arcWriter.suffix|//controller/*[@name="write-processors"]/*[@name="Archiver"]/*[@name="suffix"]|${HOSTNAME}
|
||||
arcWriter.maxFileSizeBytes|//controller/*[@name="write-processors"]/*[@name="Archiver"]/*[@name="max-size-bytes"]|100000000
|
||||
*arcWriter.storePaths[0]|//controller/*[@name="write-processors"]/*[@name="Archiver"]/*[@name="path"]/string|arcs
|
||||
arcWriter.poolMaxActive|//controller/*[@name="write-processors"]/*[@name="Archiver"]/*[@name="pool-max-active"]|5
|
||||
arcWriter.poolMaxWaitMs|//controller/*[@name="write-processors"]/*[@name="Archiver"]/*[@name="pool-max-wait"]|300000
|
||||
arcWriter.maxTotalBytesToWrite|//controller/*[@name="write-processors"]/*[@name="Archiver"]/*[@name="total-bytes-to-write"]|0
|
||||
arcWriter.skipIdenticalDigests|//controller/*[@name="write-processors"]/*[@name="Archiver"]/*[@name="skip-identical-digests"]|false
|
||||
$na|//controller/*[@name="post-processors"]/*[@name="Updater"]/*[@name="enabled"]|true
|
||||
$na|//controller/*[@name="post-processors"]/*[@name="LinksScoper"]/*[@name="enabled"]|true
|
||||
$na|//controller/*[@name="post-processors"]/*[@name="LinksScoper"]/*[@name="override-logger"]|false
|
||||
candidates.seedsRedirectNewSeeds|//controller/*[@name="post-processors"]/*[@name="LinksScoper"]/*[@name="seed-redirects-new-seed"]|true
|
||||
preparer.preferenceDepthHops|//controller/*[@name="post-processors"]/*[@name="LinksScoper"]/*[@name="preference-depth-hops"]|-1
|
||||
$na|//controller/*[@name="post-processors"]/*[@name="Scheduler"]/*[@name="enabled"]|true
|
||||
statisticsTracker.intervalSeconds|//controller/*[@name="loggers"]/*[@name="crawl-statistics"]/*[@name="interval-seconds"]|20
|
||||
bdb.checkpointCopyLogs|//controller/*[@name="checkpoint-copy-bdbje-logs"]|true
|
||||
$na|//controller/*[@name="recover-retain-failures"]|false
|
||||
$na|//controller/*[@name="recover-scope-includes"]|true
|
||||
$na|//controller/*[@name="recover-scope-enqueues"]|true
|
||||
@@ -1,27 +1,59 @@
|
||||
/*
|
||||
* This file is part of the Heritrix web crawler (crawler.archive.org).
|
||||
*
|
||||
* Licensed to the Internet Archive (IA) by one or more individual
|
||||
* contributors.
|
||||
*
|
||||
* The IA licenses this file to You under the Apache License, Version 2.0
|
||||
* (the "License"); you may not use this file except in compliance with
|
||||
* the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package org.archive.crawler.migrate;
|
||||
|
||||
import java.io.File;
|
||||
import java.io.IOException;
|
||||
import java.io.InputStream;
|
||||
import java.util.HashMap;
|
||||
import java.util.LinkedHashMap;
|
||||
import java.util.Map;
|
||||
|
||||
import javax.xml.parsers.DocumentBuilder;
|
||||
import javax.xml.parsers.DocumentBuilderFactory;
|
||||
import javax.xml.parsers.ParserConfigurationException;
|
||||
import javax.xml.xpath.XPath;
|
||||
import javax.xml.xpath.XPathConstants;
|
||||
import javax.xml.xpath.XPathExpression;
|
||||
import javax.xml.xpath.XPathExpressionException;
|
||||
import javax.xml.xpath.XPathFactory;
|
||||
|
||||
import org.apache.commons.io.FileUtils;
|
||||
import org.apache.commons.io.IOUtils;
|
||||
import org.apache.commons.io.LineIterator;
|
||||
import org.apache.commons.lang.StringUtils;
|
||||
import org.w3c.dom.Document;
|
||||
import org.w3c.dom.Element;
|
||||
import org.w3c.dom.Node;
|
||||
import org.w3c.dom.NodeList;
|
||||
import org.xml.sax.SAXException;
|
||||
|
||||
/**
|
||||
* Utility class which takes a H1 order.xml and creates a similar
|
||||
* H3 job directory, with as many simple settings converted over
|
||||
* (as top-of-crawler-beans overrides) as possible at this time.
|
||||
*
|
||||
* (Future versions will handle more complicated H1 settings
|
||||
* customizations, such as per-host overrides or choices of
|
||||
* alternate implementing classes for Scope, Processors, etc.)
|
||||
*
|
||||
* @contributor siznax
|
||||
* @contributor gojomo
|
||||
*/
|
||||
public class MigrateH1to3Tool {
|
||||
static DocumentBuilder DOCUMENT_BUILDER;
|
||||
static {
|
||||
@@ -32,195 +64,87 @@ public class MigrateH1to3Tool {
|
||||
}
|
||||
}
|
||||
|
||||
// TODO: fixxx
|
||||
protected File templateH3job = new File("/Users/steve/Documents/workspace-3.4.2/h3/dist/src/main/conf/jobs/profile-defaults");
|
||||
protected File defaultBeansXmlFile = new File(templateH3job,"profile-crawler-beans.cxml");
|
||||
|
||||
protected File sourceOrderXmlFile;
|
||||
protected File destinationH3JobDir;
|
||||
protected File targetBeansXmlFile;
|
||||
|
||||
protected Document sourceOrderXmlDom;
|
||||
protected Document targetBeansXmlDom;
|
||||
|
||||
public static void main(String[] args) throws Exception {
|
||||
new MigrateH1to3Tool().instanceMain(args);
|
||||
}
|
||||
|
||||
public void instanceMain(String[] args) throws Exception {
|
||||
|
||||
sourceOrderXmlFile = new File(args[0]);
|
||||
destinationH3JobDir = new File(args[1]);
|
||||
if(args.length>2) {
|
||||
templateH3job = new File(args[2]);
|
||||
File sourceOrderXmlFile = new File(args[0]);
|
||||
File destinationH3JobDir = new File(args[1]);
|
||||
|
||||
destinationH3JobDir.mkdirs();
|
||||
|
||||
InputStream inStream = getClass().getResourceAsStream(
|
||||
"migrate-template-crawler-beans.cxml");
|
||||
String template = IOUtils.toString(inStream);
|
||||
inStream.close();
|
||||
|
||||
Map<String,String> migrateH1toH3Map = getMigrateMap();
|
||||
|
||||
Document sourceOrderXmlDom = DOCUMENT_BUILDER.parse(sourceOrderXmlFile);
|
||||
Map<String,String> h1simpleSettings = flattenH1Order(sourceOrderXmlDom);
|
||||
|
||||
StringBuilder sb = new StringBuilder();
|
||||
for(String key : h1simpleSettings.keySet()) {
|
||||
String beanPath = migrateH1toH3Map.get(key);
|
||||
if(beanPath==null) {
|
||||
System.err.println("no rule found for: "+key);
|
||||
} else if (beanPath.startsWith("$")) {
|
||||
System.err.println(beanPath.substring(1)+": "+key);
|
||||
} else if (beanPath.startsWith("*")) {
|
||||
// TODO: needs special handling
|
||||
if(beanPath.equals("*metadata.userAgentTemplate")) {
|
||||
splitH1userAgent(h1simpleSettings.get(key),sb);
|
||||
} else {
|
||||
System.err.println(beanPath+": "+key);
|
||||
}
|
||||
} else {
|
||||
sb
|
||||
.append(beanPath)
|
||||
.append("=")
|
||||
.append(h1simpleSettings.get(key))
|
||||
.append("\n");
|
||||
}
|
||||
}
|
||||
|
||||
targetBeansXmlFile = new File(destinationH3JobDir,"crawler-beans.cxml");
|
||||
|
||||
FileUtils.copyFile(defaultBeansXmlFile, targetBeansXmlFile);
|
||||
|
||||
sourceOrderXmlDom = DOCUMENT_BUILDER.parse(sourceOrderXmlFile);
|
||||
// patch all overrides derived from H1 into H3 template
|
||||
String beansCxml = template.replace("###MIGRATE_OVERRIDES###", sb.toString());
|
||||
|
||||
domFindOrderXmlControllerStrings(sourceOrderXmlDom);
|
||||
domFindOrderXmlScopeElements(sourceOrderXmlDom);
|
||||
xpathFindOrderXmlScopeNode(sourceOrderXmlDom);
|
||||
|
||||
targetBeansXmlDom = DOCUMENT_BUILDER.parse(targetBeansXmlFile);
|
||||
System.out.println("targetBeansXmlDom = " + targetBeansXmlDom.getLocalName());
|
||||
unRollNode(targetBeansXmlDom.getElementsByTagName("property"));
|
||||
|
||||
migrateMetadata();
|
||||
|
||||
File targetBeansXmlFile = new File(destinationH3JobDir,"crawler-beans.cxml");
|
||||
FileUtils.writeStringToFile(targetBeansXmlFile, beansCxml);
|
||||
|
||||
FileUtils.copyFile(
|
||||
new File(sourceOrderXmlFile.getParentFile(), "seeds.txt"),
|
||||
new File(destinationH3JobDir, "seeds.txt"));
|
||||
}
|
||||
|
||||
protected void unRollNode(NodeList node) {
|
||||
System.out.println("unRollNode " + node.toString()); // + node.getLength());
|
||||
for (int i=0; i<node.getLength(); i++) {
|
||||
System.out.println(node.item(i).getNodeName()
|
||||
+ " " + node.item(i).getNodeValue()
|
||||
+ " " + node.item(i).getTextContent());
|
||||
for ( int j=0; j<node.item(i).getChildNodes().getLength(); j++) {
|
||||
Node beanChild = node.item(i).getChildNodes().item(j);
|
||||
System.out.println(" > " + beanChild.getNodeName()
|
||||
+ " " + beanChild.getNodeValue()
|
||||
+ " " + beanChild.getTextContent());
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
protected void setBeanProperty(String id, String property, String value)
|
||||
throws Exception {
|
||||
System.out.println("setBeanProperty("+id+","+property+","+value+")");
|
||||
|
||||
// String xpathStr = "//bean[@id=\""+id+"\"]//property[@name=\""+property+"\"]";
|
||||
String xpathStr = "//bean[@id=\"" + id + "\"]";
|
||||
System.out.println(" xpathStr = " + xpathStr);
|
||||
NodeList node = getBeanNode(xpathStr);
|
||||
unRollNode(node);
|
||||
|
||||
}
|
||||
|
||||
protected NodeList getBeanNode(String xpath) throws Exception {
|
||||
return getDocNode(targetBeansXmlDom,xpath);
|
||||
}
|
||||
|
||||
protected NodeList getDocNode(Document doc, String str) throws Exception {
|
||||
System.out.println("getDocNode(" + doc.getLocalName() + "," + str + " )");
|
||||
XPathFactory factory = XPathFactory.newInstance();
|
||||
XPath xpath = factory.newXPath();
|
||||
try {
|
||||
XPathExpression expr = xpath.compile(str);
|
||||
NodeList list = (NodeList) expr.evaluate(doc,XPathConstants.NODE);
|
||||
System.out.println("xpath \"" + str + "\" found " + list.getLength() + " members");
|
||||
return list;
|
||||
} catch (XPathExpressionException e) {
|
||||
System.out.println("XPathExpressionException!");
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
protected String getDocString(Document doc, String str) {
|
||||
XPathFactory factory = XPathFactory.newInstance();
|
||||
XPath xpath = factory.newXPath();
|
||||
try {
|
||||
XPathExpression expr = xpath.compile(str);
|
||||
return expr.evaluate(doc);
|
||||
} catch (XPathExpressionException e) {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
// JOB_NAME => order_meta_name = bean_metadata_jobName
|
||||
// DESCRIPTION => order_meta_description = bean_metadata_description
|
||||
protected void migrateMetadata() throws Exception {
|
||||
|
||||
String order_meta_name = getDocString(sourceOrderXmlDom,"//meta/name");
|
||||
String order_meta_description = getDocString(sourceOrderXmlDom,"//meta/description");
|
||||
System.out.println("order_meta/name = " + order_meta_name);
|
||||
System.out.println("order_meta/description = " + order_meta_description);
|
||||
|
||||
// NodeList bean_simpleOverrides = getBeanNode("//bean[@=\"simpleOverrides\"]");
|
||||
// unRollNode(bean_simpleOverrides);
|
||||
|
||||
setBeanProperty("simpleOverrides","jobName",order_meta_name);
|
||||
// setBeanProperty("metadata","description",order_meta_description);
|
||||
}
|
||||
|
||||
// CONTACT_URL => map_http-headers_from = bean_metadata_operatorContactUrl
|
||||
// HVERSION => map_http-headers_user-agent=" heritrix/(.*) " =>
|
||||
protected void migrateHttpHeaders() throws Exception {
|
||||
}
|
||||
|
||||
// protected void migrateControllerStrings()
|
||||
// protected void migrateScope()
|
||||
|
||||
protected void xpathFindOrderXmlScopeNode(Document orderXmlDom) throws Exception {
|
||||
// use Xpath to extract scope node
|
||||
XPathFactory factory = XPathFactory.newInstance();
|
||||
XPath xpath = factory.newXPath();
|
||||
String xp = "//controller/newObject[@name=\"scope\"]";
|
||||
System.out.print("==== XPATH {" + xp + "} ====\n");
|
||||
XPathExpression expr = xpath.compile(xp);
|
||||
NodeList scope = (NodeList) expr.evaluate(orderXmlDom,XPathConstants.NODE);
|
||||
for (int j=0; j< scope.getLength(); j++) {
|
||||
if (scope.item(j).getNodeType() == org.w3c.dom.Node.ELEMENT_NODE) {
|
||||
System.out.println(" " + scope.item(j).getNodeName());
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
protected void domFindOrderXmlScopeElements(Document orderXmlDom) {
|
||||
Element crawlOrder = orderXmlDom.getDocumentElement();
|
||||
NodeList newObjects = crawlOrder.getElementsByTagName("newObject");
|
||||
for (int i=0; i<newObjects.getLength(); i++) {
|
||||
if (newObjects.item(i).hasAttributes()) {
|
||||
Node name = newObjects.item(i).getAttributes().getNamedItem("name");
|
||||
|
||||
// scope
|
||||
if (name.getNodeValue().equalsIgnoreCase("scope")) {
|
||||
String npath = newObjects.item(i).getNodeName()
|
||||
+ "[" + i + "][@" + name.getNodeName()
|
||||
+ "=\"" + name.getNodeValue() + "\"]";
|
||||
System.out.println("==== DOM search {" + npath + "} ====");
|
||||
NodeList scope = newObjects.item(i).getChildNodes();
|
||||
for (int j=0; j<scope.getLength(); j++) {
|
||||
if (scope.item(j).getNodeType() == org.w3c.dom.Node.ELEMENT_NODE) {
|
||||
System.out.println(" " + scope.item(j).getNodeName());
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
protected void domFindOrderXmlControllerStrings(Document orderXmlDom) {
|
||||
Element crawlOrder = orderXmlDom.getDocumentElement();
|
||||
NodeList strings = crawlOrder.getElementsByTagName("string");
|
||||
for (int i=0; i<strings.getLength(); i++) {
|
||||
if (strings.item(i).getParentNode().getNodeName() == "controller") {
|
||||
if (strings.item(i).hasAttributes()) {
|
||||
Node s = strings.item(i);
|
||||
System.out.println(s.getParentNode().getNodeName()
|
||||
+ "/"
|
||||
+ s.getNodeName()
|
||||
+ "[" + i + "]"
|
||||
+ "@"
|
||||
+ s.getAttributes().item(0).getNodeName()
|
||||
+ "["
|
||||
+ s.getAttributes().item(0).getNodeValue()
|
||||
+ "]"
|
||||
+ " = "
|
||||
+ s.getTextContent());
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
public static void main(String[] args) throws Exception {
|
||||
new MigrateH1to3Tool().instanceMain(args);
|
||||
}
|
||||
|
||||
public static Map<String,String> flattenH1Order(String path) throws Exception {
|
||||
return flattenH1Order(DOCUMENT_BUILDER.parse(new File(path)));
|
||||
protected void splitH1userAgent(String userAgent, StringBuilder sb) {
|
||||
String originalUrl = userAgent.replaceAll(
|
||||
"^.*?\\+(http://[^)]*).*$",
|
||||
"$1");
|
||||
String newTemplate = userAgent.replace(originalUrl,"@OPERATOR_CONTACT_URL@");
|
||||
sb
|
||||
.append("metadata.operatorContactUrl=")
|
||||
.append(originalUrl)
|
||||
.append("\n")
|
||||
.append("metadata.userAgentTemplate=")
|
||||
.append(newTemplate)
|
||||
.append("\n");
|
||||
}
|
||||
|
||||
protected Map<String, String> getMigrateMap() throws IOException {
|
||||
Map<String,String> map = new HashMap<String,String>();
|
||||
InputStream inStream = getClass().getResourceAsStream("H1toH3.map");
|
||||
LineIterator iter = IOUtils.lineIterator(inStream, "UTF-8");
|
||||
while(iter.hasNext()) {
|
||||
String[] fields = iter.nextLine().split("\\|");
|
||||
map.put(fields[1], fields[0]);
|
||||
}
|
||||
inStream.close();
|
||||
return map;
|
||||
}
|
||||
|
||||
/**
|
||||
* Given a Document, return a Map of all non-blank simple text
|
||||
* nodes, keyed by the pseudo-XPath to their parent element.
|
||||
@@ -230,7 +154,7 @@ public class MigrateH1to3Tool {
|
||||
* @throws XPathExpressionException
|
||||
*/
|
||||
public static Map<String,String> flattenH1Order(Document h1order) throws XPathExpressionException {
|
||||
Map<String,String> flattened = new HashMap<String,String>();
|
||||
Map<String,String> flattened = new LinkedHashMap<String,String>();
|
||||
XPathExpression xpath = XPathFactory.newInstance().newXPath().compile("//text()");
|
||||
NodeList nodes = (NodeList) xpath.evaluate(h1order,XPathConstants.NODESET);
|
||||
for(int i = 0; i< nodes.getLength(); i++) {
|
||||
@@ -239,15 +163,15 @@ public class MigrateH1to3Tool {
|
||||
String pseudoXPath = getPseudoXpath(node.getParentNode());
|
||||
pseudoXPath = pseudoXPath.replaceFirst("/crawl-order", "/");
|
||||
|
||||
System.out.println(
|
||||
pseudoXPath
|
||||
+" "+node.getTextContent());
|
||||
// System.out.println(
|
||||
// pseudoXPath
|
||||
// +" "+node.getTextContent());
|
||||
|
||||
flattened.put(getPseudoXpath(node.getParentNode()), node.getTextContent());
|
||||
flattened.put(pseudoXPath, node.getTextContent());
|
||||
}
|
||||
}
|
||||
System.err.println(flattened.size());
|
||||
System.err.println(flattened);
|
||||
// System.err.println(flattened.size());
|
||||
// System.err.println(flattened);
|
||||
|
||||
return flattened;
|
||||
}
|
||||
|
||||
@@ -0,0 +1,511 @@
|
||||
<?xml version="1.0" encoding="UTF-8"?>
|
||||
<!--
|
||||
HERITRIX 3 CRAWL JOB CONFIGURATION FILE - MIGRATE TEMPLATE
|
||||
|
||||
This is a relatively minimal configuration suitable for many crawls.
|
||||
|
||||
Commented-out beans and properties are provided as an example; values
|
||||
shown in comments reflect the actual defaults which are in effect
|
||||
without specification. (To change from the default behavior,
|
||||
uncomment AND alter the shown values.)
|
||||
-->
|
||||
<beans xmlns="http://www.springframework.org/schema/beans"
|
||||
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
|
||||
xmlns:context="http://www.springframework.org/schema/context"
|
||||
xmlns:aop="http://www.springframework.org/schema/aop"
|
||||
xmlns:tx="http://www.springframework.org/schema/tx"
|
||||
xsi:schemaLocation="http://www.springframework.org/schema/beans http://www.springframework.org/schema/beans/spring-beans-2.0.xsd
|
||||
http://www.springframework.org/schema/aop http://www.springframework.org/schema/aop/spring-aop-2.0.xsd
|
||||
http://www.springframework.org/schema/tx http://www.springframework.org/schema/tx/spring-tx-2.0.xsd
|
||||
http://www.springframework.org/schema/context http://www.springframework.org/schema/context/spring-context-2.5.xsd">
|
||||
|
||||
<context:annotation-config/>
|
||||
|
||||
<!--
|
||||
OVERRIDES
|
||||
Values elsewhere in the configuration may be replaced ('overridden')
|
||||
by a Properties map declared in a PropertiesOverrideConfigurer,
|
||||
using a dotted-bean-path to address individual bean properties.
|
||||
This allows us to collect a few of the most-often changed values
|
||||
in an easy-to-edit format here at the beginning of the model
|
||||
configuration.
|
||||
-->
|
||||
<!-- overrides from a text property list -->
|
||||
<bean id="simpleOverrides" class="org.springframework.beans.factory.config.PropertyOverrideConfigurer">
|
||||
<property name="properties">
|
||||
<value>
|
||||
# This Properties map is specified in the Java 'property list' text format
|
||||
# http://java.sun.com/javase/6/docs/api/java/util/Properties.html#load%28java.io.Reader%29
|
||||
|
||||
###MIGRATE_OVERRIDES###
|
||||
|
||||
</value>
|
||||
</property>
|
||||
</bean>
|
||||
|
||||
<!-- overrides from declared <prop> elements, more easily allowing
|
||||
multiline values or even declared beans -->
|
||||
<bean id="longerOverrides" class="org.springframework.beans.factory.config.PropertyOverrideConfigurer">
|
||||
<property name="properties">
|
||||
<props>
|
||||
</props>
|
||||
</property>
|
||||
</bean>
|
||||
|
||||
<!-- CRAWL METADATA: including identification of crawler/operator -->
|
||||
<bean id="metadata" class="org.archive.modules.CrawlMetadata" autowire="byName">
|
||||
<property name="operatorContactUrl" value="[see override above]"/>
|
||||
<property name="jobName" value="[see override above]"/>
|
||||
<property name="description" value="[see override above]"/>
|
||||
<!-- <property name="operator" value=""/> -->
|
||||
<!-- <property name="operatorFrom" value=""/> -->
|
||||
<!-- <property name="organization" value=""/> -->
|
||||
<!-- <property name="audience" value=""/> -->
|
||||
<!-- <property name="userAgentTemplate"
|
||||
value="Mozilla/5.0 (compatible; heritrix/@VERSION@ +@OPERATOR_CONTACT_URL@)"/> -->
|
||||
|
||||
</bean>
|
||||
|
||||
<!-- SEEDS: crawl starting points -->
|
||||
<!-- ConfigFile approach: specifying external seeds.txt file -->
|
||||
<bean id="seeds" class="org.archive.modules.seeds.TextSeedModule">
|
||||
<property name="textSource">
|
||||
<bean class="org.archive.spring.ConfigFile">
|
||||
<property name="path" value="seeds.txt" />
|
||||
</bean>
|
||||
</property>
|
||||
<!-- <property name='sourceTagSeeds' value='false'/> -->
|
||||
</bean>
|
||||
|
||||
<!-- SEEDS ALTERNATE APPROACH:
|
||||
ConfigString allows simple, inline specification of a moderate
|
||||
number of seeds; see below comment for example of using an
|
||||
arbitrarily-large external file.
|
||||
Use either the above, or this, but not both. -->
|
||||
<!--
|
||||
<bean id="seeds" class="org.archive.modules.seeds.TextSeedModule">
|
||||
<property name="textSource">
|
||||
<bean class="org.archive.spring.ConfigString">
|
||||
<property name="value">
|
||||
<value>
|
||||
# [see override above]
|
||||
</value>
|
||||
</property>
|
||||
</bean>
|
||||
</property>
|
||||
<property name='sourceTagSeeds' value='false'/>
|
||||
</bean>
|
||||
-->
|
||||
|
||||
<!-- SCOPE: rules for which discovered URIs to crawl; order is very
|
||||
important because last decision returned other than 'NONE' wins. -->
|
||||
<bean id="scope" class="org.archive.modules.deciderules.DecideRuleSequence">
|
||||
<property name="rules">
|
||||
<list>
|
||||
<!-- Begin by REJECTing all... -->
|
||||
<bean class="org.archive.modules.deciderules.RejectDecideRule">
|
||||
</bean>
|
||||
<!-- ...then ACCEPT those within configured/seed-implied SURT prefixes... -->
|
||||
<bean class="org.archive.modules.deciderules.surt.SurtPrefixedDecideRule">
|
||||
<!-- <property name="seedsAsSurtPrefixes" value="true" /> -->
|
||||
<!-- <property name="alsoCheckVia" value="true" /> -->
|
||||
<!-- <property name="surtsSourceFile" value="" /> -->
|
||||
<!-- <property name="surtsDumpFile" value="surts.dump" /> -->
|
||||
</bean>
|
||||
<!-- ...but REJECT those more than a configured link-hop-count from start... -->
|
||||
<bean class="org.archive.modules.deciderules.TooManyHopsDecideRule">
|
||||
<!-- <property name="maxHops" value="20" /> -->
|
||||
</bean>
|
||||
<!-- ...but ACCEPT those more than a configured link-hop-count from start... -->
|
||||
<bean class="org.archive.modules.deciderules.TransclusionDecideRule">
|
||||
<!-- <property name="maxTransHops" value="2" /> -->
|
||||
<!-- <property name="maxSpeculativeHops" value="1" /> -->
|
||||
</bean>
|
||||
<!-- ...but REJECT those from a configurable (initially empty) set of REJECT SURTs... -->
|
||||
<bean class="org.archive.modules.deciderules.surt.SurtPrefixedDecideRule">
|
||||
<property name="decision" value="REJECT"/>
|
||||
<property name="seedsAsSurtPrefixes" value="false"/>
|
||||
<property name="surtsDumpFile" value="negative-surts.dump" />
|
||||
<!-- <property name="surtsSourceFile" value="" /> -->
|
||||
</bean>
|
||||
<!-- ...and REJECT those from a configurable (initially empty) set of URI regexes... -->
|
||||
<bean class="org.archive.modules.deciderules.MatchesListRegexDecideRule">
|
||||
<!-- <property name="listLogicalOr" value="true" /> -->
|
||||
<!-- <property name="regexList">
|
||||
<list>
|
||||
</list>
|
||||
</property> -->
|
||||
</bean>
|
||||
<!-- ...and REJECT those with suspicious repeating path-segments... -->
|
||||
<bean class="org.archive.modules.deciderules.PathologicalPathDecideRule">
|
||||
<!-- <property name="maxRepetitions" value="2" /> -->
|
||||
</bean>
|
||||
<!-- ...and REJECT those with more than threshold number of path-segments... -->
|
||||
<bean class="org.archive.modules.deciderules.TooManyPathSegmentsDecideRule">
|
||||
<!-- <property name="maxPathDepth" value="20" /> -->
|
||||
</bean>
|
||||
<!-- ...but always ACCEPT those marked as prerequisitee for another URI... -->
|
||||
<bean class="org.archive.modules.deciderules.PrerequisiteAcceptDecideRule">
|
||||
</bean>
|
||||
</list>
|
||||
</property>
|
||||
</bean>
|
||||
|
||||
<!--
|
||||
PROCESSING CHAINS
|
||||
Much of the crawler's work is specified by the sequential
|
||||
application of swappable Processor modules. These Processors
|
||||
are collected into three 'chains. The CandidateChain is applied
|
||||
to URIs being considered for inclusion, before a URI is enqueued
|
||||
for collection. The FetchChain is applied to URIs when their
|
||||
turn for collection comes up. The DispositionChain is applied
|
||||
after a URI is fetched and analyzed/link-extracted.
|
||||
-->
|
||||
|
||||
<!-- CANDIDATE CHAIN -->
|
||||
<!-- processors declared as named beans -->
|
||||
<bean id="candidateScoper" class="org.archive.crawler.prefetch.CandidateScoper">
|
||||
</bean>
|
||||
<bean id="preparer" class="org.archive.crawler.prefetch.FrontierPreparer">
|
||||
<!-- <property name="preferenceDepthHops" value="-1" /> -->
|
||||
<!-- <property name="preferenceEmbedHops" value="1" /> -->
|
||||
<!-- <property name="canonicalizationPolicy">
|
||||
<ref bean="canonicalizationPolicy" />
|
||||
</property> -->
|
||||
<!-- <property name="queueAssignmentPolicy">
|
||||
<ref bean="queueAssignmentPolicy" />
|
||||
</property> -->
|
||||
<!-- <property name="uriPrecedencePolicy">
|
||||
<ref bean="uriPrecedencePolicy" />
|
||||
</property> -->
|
||||
<!-- <property name="costAssignmentPolicy">
|
||||
<ref bean="costAssignmentPolicy" />
|
||||
</property> -->
|
||||
</bean>
|
||||
<!-- assembled into ordered CandidateChain bean -->
|
||||
<bean id="candidateProcessors" class="org.archive.modules.CandidateChain">
|
||||
<property name="processors">
|
||||
<list>
|
||||
<!-- apply scoping rules to each individual candidate URI... -->
|
||||
<ref bean="candidateScoper"/>
|
||||
<!-- ...then prepare those ACCEPTed for enqueuing to frontier. -->
|
||||
<ref bean="preparer"/>
|
||||
</list>
|
||||
</property>
|
||||
</bean>
|
||||
|
||||
<!-- FETCH CHAIN -->
|
||||
<!-- processors declared as named beans -->
|
||||
<bean id="preselector" class="org.archive.crawler.prefetch.Preselector">
|
||||
<!-- <property name="recheckScope" value="false" /> -->
|
||||
<!-- <property name="blockAll" value="false" /> -->
|
||||
<!-- <property name="blockByRegex" value="" /> -->
|
||||
<!-- <property name="allowByRegex" value="" /> -->
|
||||
</bean>
|
||||
<bean id="preconditions" class="org.archive.crawler.prefetch.PreconditionEnforcer">
|
||||
<!-- <property name="ipValidityDurationSeconds" value="21600" /> -->
|
||||
<!-- <property name="robotsValidityDurationSeconds" value="86400" /> -->
|
||||
<!-- <property name="calculateRobotsOnly" value="false" /> -->
|
||||
</bean>
|
||||
<bean id="fetchDns" class="org.archive.modules.fetcher.FetchDNS">
|
||||
<!-- <property name="acceptNonDnsResolves" value="false" /> -->
|
||||
<!-- <property name="digestContent" value="true" /> -->
|
||||
<!-- <property name="digestAlgorithm" value="sha1" /> -->
|
||||
</bean>
|
||||
<bean id="fetchHttp" class="org.archive.modules.fetcher.FetchHTTP">
|
||||
<!-- <property name="maxLengthBytes" value="0" /> -->
|
||||
<!-- <property name="timeoutSeconds" value="1200" /> -->
|
||||
<!-- <property name="maxFetchKBSec" value="0" /> -->
|
||||
<!-- <property name="defaultEncoding" value="ISO-8859-1" /> -->
|
||||
<!-- <property name="shouldFetchBodyRule">
|
||||
<bean class="org.archive.modules.deciderules.AcceptDecideRule"/>
|
||||
</property> -->
|
||||
<!-- <property name="soTimeoutMs" value="20000" /> -->
|
||||
<!-- <property name="sendIfModifiedSince" value="true" /> -->
|
||||
<!-- <property name="sendIfNoneMatch" value="true" /> -->
|
||||
<!-- <property name="sendConnectionClose" value="true" /> -->
|
||||
<!-- <property name="sendReferer" value="true" /> -->
|
||||
<!-- <property name="sendRange" value="false" /> -->
|
||||
<!-- <property name="ignoreCookies" value="false" /> -->
|
||||
<!-- <property name="sslTrustLevel" value="OPEN" /> -->
|
||||
<!-- <property name="acceptHeaders">
|
||||
<list>
|
||||
</list>
|
||||
</property>
|
||||
-->
|
||||
<!-- <property name="httpBindAddress" value="" /> -->
|
||||
<!-- <property name="httpProxyHost" value="" /> -->
|
||||
<!-- <property name="httpProxyPort" value="0" /> -->
|
||||
<!-- <property name="digestContent" value="true" /> -->
|
||||
<!-- <property name="digestAlgorithm" value="sha1" /> -->
|
||||
</bean>
|
||||
<bean id="extractorHttp" class="org.archive.modules.extractor.ExtractorHTTP">
|
||||
</bean>
|
||||
<bean id="extractorHtml" class="org.archive.modules.extractor.ExtractorHTML">
|
||||
<!-- <property name="extractJavascript" value="true" /> -->
|
||||
<!-- <property name="extractValueAttributes" value="true" /> -->
|
||||
<!-- <property name="ignoreFormActionUrls" value="false" /> -->
|
||||
<!-- <property name="extractOnlyFormGets" value="true" /> -->
|
||||
<!-- <property name="treatFramesAsEmbedLinks" value="true" /> -->
|
||||
<!-- <property name="ignoreUnexpectedHtml" value="true" /> -->
|
||||
<!-- <property name="maxElementLength" value="1024" /> -->
|
||||
<!-- <property name="maxAttributeNameLength" value="1024" /> -->
|
||||
<!-- <property name="maxAttributeValueLength" value="16384" /> -->
|
||||
</bean>
|
||||
<bean id="extractorCss" class="org.archive.modules.extractor.ExtractorCSS">
|
||||
</bean>
|
||||
<bean id="extractorJs" class="org.archive.modules.extractor.ExtractorJS">
|
||||
</bean>
|
||||
<bean id="extractorSwf" class="org.archive.modules.extractor.ExtractorSWF">
|
||||
</bean>
|
||||
<!-- assembled into ordered FetchChain bean -->
|
||||
<bean id="fetchProcessors" class="org.archive.modules.FetchChain">
|
||||
<property name="processors">
|
||||
<list>
|
||||
<!-- recheck scope, if so enabled... -->
|
||||
<ref bean="preselector"/>
|
||||
<!-- ...then verify or trigger prerequisite URIs fetched, allow crawling... -->
|
||||
<ref bean="preconditions"/>
|
||||
<!-- ...fetch if DNS URI... -->
|
||||
<ref bean="fetchDns"/>
|
||||
<!-- ...fetch if HTTP URI... -->
|
||||
<ref bean="fetchHttp"/>
|
||||
<!-- ...extract oulinks from HTTP headers... -->
|
||||
<ref bean="extractorHttp"/>
|
||||
<!-- ...extract oulinks from HTML content... -->
|
||||
<ref bean="extractorHtml"/>
|
||||
<!-- ...extract oulinks from CSS content... -->
|
||||
<ref bean="extractorCss"/>
|
||||
<!-- ...extract oulinks from Javascript content... -->
|
||||
<ref bean="extractorJs"/>
|
||||
<!-- ...extract oulinks from Flash content... -->
|
||||
<ref bean="extractorSwf"/>
|
||||
</list>
|
||||
</property>
|
||||
</bean>
|
||||
|
||||
<!-- DISPOSITION CHAIN -->
|
||||
<!-- processors declared as named beans -->
|
||||
<bean id="arcWriter" class="org.archive.modules.writer.ARCWriterProcessor">
|
||||
<!-- <property name="compress" value="true" /> -->
|
||||
<!-- <property name="prefix" value="IAH" /> -->
|
||||
<!-- <property name="suffix" value="${HOSTNAME}" /> -->
|
||||
<!-- <property name="maxFileSizeBytes" value="100000000" /> -->
|
||||
<!-- <property name="poolMaxActive" value="1" /> -->
|
||||
<!-- <property name="poolMaxWaitMs" value="300000" /> -->
|
||||
<!-- <property name="skipIdenticalDigests" value="false" /> -->
|
||||
<!-- <property name="maxTotalBytesToWrite" value="0" /> -->
|
||||
<!-- <property name="directory" value="." /> -->
|
||||
<!-- <property name="storePaths">
|
||||
<list>
|
||||
<value>arcs</value>
|
||||
</list>
|
||||
</property> -->
|
||||
</bean>
|
||||
<bean id="candidates" class="org.archive.crawler.postprocessor.CandidatesProcessor">
|
||||
<!-- <property name="seedsRedirectNewSeeds" value="true" /> -->
|
||||
</bean>
|
||||
<bean id="disposition" class="org.archive.crawler.postprocessor.DispositionProcessor">
|
||||
<!-- <property name="delayFactor" value="5.0" /> -->
|
||||
<!-- <property name="minDelayMs" value="3000" /> -->
|
||||
<!-- <property name="respectCrawlDelayUpToSeconds" value="300" /> -->
|
||||
<!-- <property name="maxDelayMs" value="30000" /> -->
|
||||
<!-- <property name="maxPerHostBandwidthUsageKbSec" value="0" /> -->
|
||||
|
||||
</bean>
|
||||
<!-- assembled into ordered DispositionChain bean -->
|
||||
<bean id="dispositionProcessors" class="org.archive.modules.DispositionChain">
|
||||
<property name="processors">
|
||||
<list>
|
||||
<!-- write to aggregate archival files... -->
|
||||
<ref bean="arcWriter"/>
|
||||
<!-- ...send each outlink candidate URI to CandidatesChain,
|
||||
and enqueue those ACCEPTed to the frontier... -->
|
||||
<ref bean="candidates"/>
|
||||
<!-- ...then update stats, shared-structures, frontier decisions -->
|
||||
<ref bean="disposition"/>
|
||||
</list>
|
||||
</property>
|
||||
</bean>
|
||||
|
||||
<!-- CRAWLCONTROLLER: Control interface, unifying context -->
|
||||
<bean id="crawlController"
|
||||
class="org.archive.crawler.framework.CrawlController">
|
||||
<!-- <property name="maxToeThreads" value="25" /> -->
|
||||
<!-- <property name="pauseAtStart" value="true" /> -->
|
||||
<!-- <property name="pauseAtFinish" value="false" /> -->
|
||||
<!-- <property name="recorderInBufferBytes" value="524288" /> -->
|
||||
<!-- <property name="recorderOutBufferBytes" value="16384" /> -->
|
||||
<!-- <property name="checkpointerPeriod" value="-1" /> -->
|
||||
<!-- <property name="scratchDir" value="scratch" /> -->
|
||||
<!-- <property name="checkpointsDir" value="checkpoints" /> -->
|
||||
</bean>
|
||||
|
||||
<!-- FRONTIER: Record of all URIs discovered and queued-for-collection -->
|
||||
<bean id="frontier"
|
||||
class="org.archive.crawler.frontier.BdbFrontier">
|
||||
<!-- <property name="holdQueues" value="true" /> -->
|
||||
<!-- <property name="queueTotalBudget" value="-1" /> -->
|
||||
<!-- <property name="balanceReplenishAmount" value="3000" /> -->
|
||||
<!-- <property name="errorPenaltyAmount" value="100" /> -->
|
||||
<!-- <property name="precedenceFloor" value="255" /> -->
|
||||
<!-- <property name="queuePrecedencePolicy">
|
||||
<bean class="org.archive.crawler.frontier.precedence.BaseQueuePrecedencePolicy" />
|
||||
</property> -->
|
||||
<!-- <property name="snoozeLongMs" value="300000" /> -->
|
||||
<!-- <property name="retryDelaySeconds" value="900" /> -->
|
||||
<!-- <property name="maxRetries" value="30" /> -->
|
||||
<!-- <property name="recoveryDir" value="logs" /> -->
|
||||
<!-- <property name="recoveryLogEnabled" value="true" /> -->
|
||||
<!-- <property name="maxOutlinks" value="6000" /> -->
|
||||
<!-- <property name="outboundQueueCapacity" value="50" /> -->
|
||||
<!-- <property name="inboundQueueMultiple" value="3" /> -->
|
||||
<!-- <property name="dumpPendingAtClose" value="false" /> -->
|
||||
</bean>
|
||||
|
||||
<!-- URI UNIQ FILTER: Used by frontier to remember already-included URIs -->
|
||||
<bean id="uriUniqFilter"
|
||||
class="org.archive.crawler.util.BdbUriUniqFilter">
|
||||
</bean>
|
||||
|
||||
<!--
|
||||
OPTIONAL BUT RECOMMENDED BEANS
|
||||
-->
|
||||
|
||||
<!-- ACTIONDIRECTORY: disk directory for mid-crawl operations
|
||||
Running job will watch directory for new files with URIs,
|
||||
scripts, and other data to be processed during a crawl. -->
|
||||
<bean id="actionDirectory" class="org.archive.crawler.framework.ActionDirectory">
|
||||
<!-- <property name="actionDir" value="action" /> -->
|
||||
<!-- <property name="initialDelaySeconds" value="10" /> -->
|
||||
<!-- <property name="delaySeconds" value="30" /> -->
|
||||
</bean>
|
||||
|
||||
<!-- CRAWLLIMITENFORCER: stops crawl when it reaches configured limits -->
|
||||
<bean id="crawlLimiter" class="org.archive.crawler.framework.CrawlLimitEnforcer">
|
||||
<!-- <property name="maxBytesDownload" value="0" /> -->
|
||||
<!-- <property name="maxDocumentsDownload" value="0" /> -->
|
||||
<!-- <property name="maxTimeSeconds" value="0" /> -->
|
||||
</bean>
|
||||
|
||||
<!--
|
||||
OPTIONAL BEANS
|
||||
Uncomment and expand as needed, or if non-default alternate
|
||||
implementations are preferred.
|
||||
-->
|
||||
|
||||
<!-- CANONICALIZATION POLICY -->
|
||||
<!--
|
||||
<bean id="canonicalizationPolicy"
|
||||
class="org.archive.modules.canonicalize.RulesCanonicalizationPolicy">
|
||||
<list>
|
||||
<bean class="org.archive.modules.canonicalize.LowercaseRule" />
|
||||
<bean class="org.archive.modules.canonicalize.StripUserinfoRule" />
|
||||
<bean class="org.archive.modules.canonicalize.StripWWWNRule" />
|
||||
<bean class="org.archive.modules.canonicalize.StripSessionIDs" />
|
||||
<bean class="org.archive.modules.canonicalize.StripSessionCFIDs" />
|
||||
<bean class="org.archive.modules.canonicalize.FixupQueryString" />
|
||||
</list>
|
||||
</bean>
|
||||
-->
|
||||
|
||||
<!-- QUEUE ASSIGNMENT POLICY -->
|
||||
<!--
|
||||
<bean id="queueAssignmentPolicy"
|
||||
class="org.archive.crawler.frontier.SurtAuthorityQueueAssignmentPolicy">
|
||||
<property name="forceQueueAssignment" value="" />
|
||||
<property name="deferToPrevious" value="true" />
|
||||
<property name="parallelQueues" value="1" />
|
||||
</bean>
|
||||
-->
|
||||
|
||||
<!-- URI PRECEDENCE POLICY -->
|
||||
<!--
|
||||
<bean id="uriPrecedencePolicy"
|
||||
class="org.archive.crawler.frontier.precedence.CostUriPrecedencePolicy">
|
||||
</bean>
|
||||
-->
|
||||
|
||||
<!-- COST ASSIGNMENT POLICY -->
|
||||
<!--
|
||||
<bean id="costAssignmentPolicy"
|
||||
class="org.archive.crawler.frontier.UnitCostAssignmentPolicy">
|
||||
</bean>
|
||||
-->
|
||||
|
||||
<!-- CREDENTIAL STORE: HTTP authentication or FORM POST credentials -->
|
||||
<!--
|
||||
<bean id="credentialStore"
|
||||
class="org.archive.modules.credential.CredentialStore">
|
||||
</bean>
|
||||
-->
|
||||
|
||||
<!--
|
||||
REQUIRED STANDARD BEANS
|
||||
It will be very rare to replace or reconfigure the following beans.
|
||||
-->
|
||||
|
||||
<!-- STATISTICSTRACKER: standard stats/reporting collector -->
|
||||
<bean id="statisticsTracker"
|
||||
class="org.archive.crawler.reporting.StatisticsTracker" autowire="byName">
|
||||
<!-- <property name="reportsDir" value="reports" /> -->
|
||||
<!-- <property name="liveHostReportSize" value="20" /> -->
|
||||
<!-- <property name="intervalSeconds" value="20" /> -->
|
||||
<!-- <property name="keepSnapshotsCount" value="5" /> -->
|
||||
<!-- <property name="liveHostReportSize" value="20" /> -->
|
||||
</bean>
|
||||
|
||||
<!-- CRAWLERLOGGERMODULE: shared logging facility -->
|
||||
<bean id="loggerModule"
|
||||
class="org.archive.crawler.reporting.CrawlerLoggerModule">
|
||||
<!-- <property name="path" value="logs" /> -->
|
||||
<!-- <property name="crawlLogPath" value="crawl.log" /> -->
|
||||
<!-- <property name="alertsLogPath" value="alerts.log" /> -->
|
||||
<!-- <property name="progressLogPath" value="progress-statistics.log" /> -->
|
||||
<!-- <property name="uriErrorsLogPath" value="uri-errors.log" /> -->
|
||||
<!-- <property name="runtimeErrorsLogPath" value="runtime-errors.log" /> -->
|
||||
<!-- <property name="nonfatalErrorsLogPath" value="nonfatal-errors.log" /> -->
|
||||
</bean>
|
||||
|
||||
<!-- SHEETOVERLAYMANAGER: manager of sheets of contextual overlays
|
||||
Autowired to include any SheetForSurtPrefix or
|
||||
SheetForDecideRuled beans -->
|
||||
<bean id="sheetOverlaysManager" autowire="byType"
|
||||
class="org.archive.crawler.spring.SheetOverlaysManager">
|
||||
</bean>
|
||||
|
||||
<!-- BDBMODULE: shared BDB-JE disk persistence manager -->
|
||||
<bean id="bdb"
|
||||
class="org.archive.bdb.BdbModule">
|
||||
<!-- <property name="dir" value="state" /> -->
|
||||
<!-- <property name="cachePercentage" value="60" /> -->
|
||||
<!-- <property name="useSharedCache" value="true" /> -->
|
||||
<!-- <property name="checkpointCopyLogs" value="true" /> -->
|
||||
<!-- <property name="expectedConcurrency" value="25" /> -->
|
||||
</bean>
|
||||
|
||||
<!-- BDBCOOKIESTORAGE: disk-based cookie storage for FetchHTTP -->
|
||||
<bean id="cookieStorage"
|
||||
class="org.archive.modules.fetcher.BdbCookieStorage">
|
||||
<!-- <property name="cookiesLoadFile"><null/></property> -->
|
||||
<!-- <property name="cookiesSaveFile"><null/></property> -->
|
||||
<!-- <property name="bdb">
|
||||
<ref bean="bdb"/>
|
||||
</property> -->
|
||||
</bean>
|
||||
|
||||
<!-- SERVERCACHE: shared cache of server/host info -->
|
||||
<bean id="serverCache"
|
||||
class="org.archive.modules.net.BdbServerCache">
|
||||
<!-- <property name="bdb">
|
||||
<ref bean="bdb"/>
|
||||
</property> -->
|
||||
</bean>
|
||||
|
||||
<!-- CONFIG PATH CONFIGURER: required helper making crawl paths relative
|
||||
to crawler-beans.cxml file, and tracking crawl files for web UI -->
|
||||
<bean id="configPathConfigurer"
|
||||
class="org.archive.spring.ConfigPathConfigurer">
|
||||
</bean>
|
||||
|
||||
</beans>
|
||||
Reference in New Issue
Block a user