lee/heritrix3
1
0 Fork 0
mirror of https://github.com/internetarchive/heritrix3.git synced 2026-11-09 07:24:41 +00:00
Code Issues Packages Projects Releases Wiki Activity

[HER-1803] H3: make it easier to change robots settings

Browse Source 
* RobotsPolicy.java
    simplified unification of RobotsHonoring and RobotsExclusion policy classes; subclasses specialize
* ObeyRobotsPolicy.java
    classic obey-robots-as-declared policy
* IgnoreRobotsPolicy.java
    ignore declared robots policy
* CustomRobotsPolicy.java
    follow custom-crafted policy in robots.txt format
* MostFavoredRobotsPolicy.java
    follow the most-permissive policy available to a set of user-agents; optionally adopt ('masquerade') the user-agent whose rules are chosen
* RobotsExclusionPolicy.java, RobotsHonoringPolicy.java
    deleted
* CrawlMetadata.java
    use new string robotsPolicyName as main determinant of policy
    maintain map of available policies ('obey','ignore', and any the operator added)
* CrawlServer.java
    retain Robotstxt instance rather than customized RobotsExclusionPolicy
* PreconditionEnforcer.java
    use full CrawlMetadata rather than just UserAgentProvider subset
    consult RobotsPolicy looked up through CrawlMetadata for allow/disallow decision
* DispositionProcessor.java
    update CrawlServer with Robotstxt only
    consult Robotstxt for crawl-delay
* profile-crawler-beans.cxml (*2), migrate-template-crawler-beans.cxml, selftest-crawler-beans.cxml
    remove old declarations/examples; add new
* ExtractorHTML.java, JerichoExtractorHTML.java
    consult RobotsPolicy.obeyMetaRobotsNofollow for NOFOLLOW
* Robotstxt.java
    new constructors for new use patterns
* **/*Test.java
    update/remove to match new use patterns
This commit is contained in:
gojomo committed 2010-09-10 00:27:20 +00:00
1 parent 3e43cf7bd5
commit d6a4d158b0
23 files changed
+503 -625
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

engine/src/main/java/org/archive/crawler/postprocessor/DispositionProcessor.java
+16 -4
View File
@@ -23,6 +23,7 @@ package org.archive.crawler.postprocessor;
import static org.archive.modules.CoreAttributeConstants.A_FETCH_COMPLETED_TIME;
import static org.archive.modules.fetcher.FetchStatusCodes.S_CONNECT_FAILED;
import static org.archive.modules.fetcher.FetchStatusCodes.S_CONNECT_LOST;
import static org.archive.modules.fetcher.FetchStatusCodes.S_DEEMED_NOT_FOUND;
import java.util.Map;
import java.util.logging.Logger;
@@ -34,7 +35,8 @@ import org.archive.modules.ModuleAttributeConstants;
import org.archive.modules.Processor;
import org.archive.modules.net.CrawlHost;
import org.archive.modules.net.CrawlServer;
import org.archive.modules.net.RobotsExclusionPolicy;
import org.archive.modules.net.IgnoreRobotsPolicy;
import org.archive.modules.net.Robotstxt;
import org.archive.modules.net.ServerCache;
import org.springframework.beans.factory.annotation.Autowired;
@@ -172,7 +174,17 @@ public class DispositionProcessor extends Processor {
if ("/robots.txt".equals(curi.getUURI().getPath())) {
// Update server with robots info
// NOTE: in some cases the curi's status can be changed here
server.updateRobots(metadata.getRobotsHonoringPolicy(), curi);
// server.updateRobots(metadata.getRobotsHonoringPolicy(), curi);
server.updateRobots(curi);
// shortcut retries w/ DEEMED when ignore-all
if (metadata.getRobotsPolicy() instanceof IgnoreRobotsPolicy) {
if(curi.getFetchStatus() < 0) {
// prevent the rest of the usual retries
curi.setFetchStatus(S_DEEMED_NOT_FOUND);
}
}
}
}
catch (URIException e) {
@@ -226,9 +238,9 @@ public class DispositionProcessor extends Processor {
if (ua == null) {
ua = metadata.getUserAgent();
}
RobotsExclusionPolicy rep = s.getRobots();
Robotstxt rep = s.getRobotstxt();
if (rep != null) {
long crawlDelay = (long)(1000 * s.getRobots().getCrawlDelay(ua));
long crawlDelay = (long)(1000 * rep.getDirectivesFor(ua).getCrawlDelay());
crawlDelay =
(crawlDelay > respectThreshold)
? respectThreshold
engine/src/main/java/org/archive/crawler/prefetch/PreconditionEnforcer.java
+14 -7
View File
@@ -30,6 +30,7 @@ import java.util.logging.Logger;
import org.apache.commons.httpclient.URIException;
import org.archive.crawler.reporting.CrawlerLoggerModule;
import org.archive.modules.CrawlMetadata;
import org.archive.modules.CrawlURI;
import org.archive.modules.ProcessResult;
import org.archive.modules.Processor;
@@ -39,9 +40,9 @@ import org.archive.modules.credential.CredentialStore;
import org.archive.modules.extractor.Hop;
import org.archive.modules.extractor.Link;
import org.archive.modules.extractor.LinkContext;
import org.archive.modules.fetcher.UserAgentProvider;
import org.archive.modules.net.CrawlHost;
import org.archive.modules.net.CrawlServer;
import org.archive.modules.net.RobotsPolicy;
import org.archive.modules.net.ServerCache;
import org.archive.net.UURI;
import org.archive.net.UURIFactory;
@@ -105,12 +106,17 @@ public class PreconditionEnforcer extends Processor {
kp.put("calculateRobotsOnly",calcOnly);
}
public UserAgentProvider getUserAgentProvider() {
return (UserAgentProvider) kp.get("userAgentProvider");
/**
* Auto-discovered module providing configured (or overridden)
* User-Agent value and RobotsHonoringPolicy
*/
CrawlMetadata metadata;
public CrawlMetadata getMetadata() {
return metadata;
}
@Autowired
public void setUserAgentProvider(UserAgentProvider provider) {
kp.put("userAgentProvider",provider);
public void setMetadata(CrawlMetadata provider) {
this.metadata = provider;
}
{
@@ -237,8 +243,9 @@ public class PreconditionEnforcer extends Processor {
}
// test against robots.txt if available
if (cs.isValidRobots()) {
String ua = getUserAgentProvider().getUserAgent();
if(cs.getRobots().disallows(curi, ua)) {
String ua = metadata.getUserAgent();
RobotsPolicy robots = metadata.getRobotsPolicy();
if(!robots.allows(ua, curi, cs.getRobotstxt())) {
if(getCalculateRobotsOnly()) {
// annotate URI as excluded, but continue to process normally
curi.getAnnotations().add("robotExcluded");