From d6a4d158b03789c7f4d5da2505e236eb6712afbf Mon Sep 17 00:00:00 2001 From: gojomo Date: Fri, 10 Sep 2010 00:27:20 +0000 Subject: [PATCH] [HER-1803] H3: make it easier to change robots settings * RobotsPolicy.java simplified unification of RobotsHonoring and RobotsExclusion policy classes; subclasses specialize * ObeyRobotsPolicy.java classic obey-robots-as-declared policy * IgnoreRobotsPolicy.java ignore declared robots policy * CustomRobotsPolicy.java follow custom-crafted policy in robots.txt format * MostFavoredRobotsPolicy.java follow the most-permissive policy available to a set of user-agents; optionally adopt ('masquerade') the user-agent whose rules are chosen * RobotsExclusionPolicy.java, RobotsHonoringPolicy.java deleted * CrawlMetadata.java use new string robotsPolicyName as main determinant of policy maintain map of available policies ('obey','ignore', and any the operator added) * CrawlServer.java retain Robotstxt instance rather than customized RobotsExclusionPolicy * PreconditionEnforcer.java use full CrawlMetadata rather than just UserAgentProvider subset consult RobotsPolicy looked up through CrawlMetadata for allow/disallow decision * DispositionProcessor.java update CrawlServer with Robotstxt only consult Robotstxt for crawl-delay * profile-crawler-beans.cxml (*2), migrate-template-crawler-beans.cxml, selftest-crawler-beans.cxml remove old declarations/examples; add new * ExtractorHTML.java, JerichoExtractorHTML.java consult RobotsPolicy.obeyMetaRobotsNofollow for NOFOLLOW * Robotstxt.java new constructors for new use patterns * **/*Test.java update/remove to match new use patterns --- .../java/org/archive/spring/ConfigString.java | 32 +++ .../profile-crawler-beans.cxml | 2 +- .../postprocessor/DispositionProcessor.java | 20 +- .../prefetch/PreconditionEnforcer.java | 21 +- .../migrate-template-crawler-beans.cxml | 24 +- .../restlet/profile-crawler-beans.cxml | 1 + .../selftest/conf/selftest-crawler-beans.cxml | 4 - .../org/archive/modules/CrawlMetadata.java | 52 +++- .../modules/extractor/ExtractorHTML.java | 8 +- .../extractor/JerichoExtractorHTML.java | 12 +- .../org/archive/modules/net/CrawlServer.java | 184 +++++------- .../modules/net/CustomRobotsPolicy.java | 69 +++++ .../modules/net/IgnoreRobotsPolicy.java} | 25 +- .../modules/net/MostFavoredRobotsPolicy.java | 84 ++++++ .../archive/modules/net/ObeyRobotsPolicy.java | 41 +++ .../modules/net/RobotsExclusionPolicy.java | 264 ------------------ .../modules/net/RobotsHonoringPolicy.java | 186 ------------ .../org/archive/modules/net/RobotsPolicy.java | 54 ++++ .../org/archive/modules/net/Robotstxt.java | 32 ++- .../modules/extractor/ExtractorHTMLTest.java | 3 +- .../extractor/JerichoExtractorHTMLTest.java | 3 +- .../writer/ARCWriterProcessorTest.java | 4 +- .../writer/WARCWriterProcessorTest.java | 3 +- 23 files changed, 503 insertions(+), 625 deletions(-) create mode 100644 modules/src/main/java/org/archive/modules/net/CustomRobotsPolicy.java rename modules/src/{test/java/org/archive/modules/net/RobotsHonoringPolicyTest.java => main/java/org/archive/modules/net/IgnoreRobotsPolicy.java} (62%) create mode 100644 modules/src/main/java/org/archive/modules/net/MostFavoredRobotsPolicy.java create mode 100644 modules/src/main/java/org/archive/modules/net/ObeyRobotsPolicy.java delete mode 100644 modules/src/main/java/org/archive/modules/net/RobotsExclusionPolicy.java delete mode 100644 modules/src/main/java/org/archive/modules/net/RobotsHonoringPolicy.java create mode 100644 modules/src/main/java/org/archive/modules/net/RobotsPolicy.java diff --git a/commons/src/main/java/org/archive/spring/ConfigString.java b/commons/src/main/java/org/archive/spring/ConfigString.java index 1372a99e..30cd0488 100644 --- a/commons/src/main/java/org/archive/spring/ConfigString.java +++ b/commons/src/main/java/org/archive/spring/ConfigString.java @@ -1,3 +1,21 @@ +/* + * This file is part of the Heritrix web crawler (crawler.archive.org). + * + * Licensed to the Internet Archive (IA) by one or more individual + * contributors. + * + * The IA licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package org.archive.spring; import java.io.Reader; @@ -5,6 +23,14 @@ import java.io.StringReader; import org.archive.io.ReadSource; +/** + * A configuration string that provides its own reader via the ReadSource + * interface, for convenient use in spring configuration where any of an + * inline string, path to local file (ConfigPath), or any other + * readable-text-source would all be equally welcome. + * + * @contributor gojomo + */ public class ConfigString implements ReadSource { String value; public String getValue() { @@ -16,4 +42,10 @@ public class ConfigString implements ReadSource { public Reader obtainReader() { return new StringReader(value); } + + public ConfigString() { + } + public ConfigString(String value) { + setValue(value); + } } diff --git a/dist/src/main/conf/jobs/profile-defaults/profile-crawler-beans.cxml b/dist/src/main/conf/jobs/profile-defaults/profile-crawler-beans.cxml index 1739330e..bb2e7c83 100644 --- a/dist/src/main/conf/jobs/profile-defaults/profile-crawler-beans.cxml +++ b/dist/src/main/conf/jobs/profile-defaults/profile-crawler-beans.cxml @@ -66,6 +66,7 @@ http://example.example/example + @@ -201,7 +202,6 @@ http://example.example/example - diff --git a/engine/src/main/java/org/archive/crawler/postprocessor/DispositionProcessor.java b/engine/src/main/java/org/archive/crawler/postprocessor/DispositionProcessor.java index 17f4f054..3c86a89f 100644 --- a/engine/src/main/java/org/archive/crawler/postprocessor/DispositionProcessor.java +++ b/engine/src/main/java/org/archive/crawler/postprocessor/DispositionProcessor.java @@ -23,6 +23,7 @@ package org.archive.crawler.postprocessor; import static org.archive.modules.CoreAttributeConstants.A_FETCH_COMPLETED_TIME; import static org.archive.modules.fetcher.FetchStatusCodes.S_CONNECT_FAILED; import static org.archive.modules.fetcher.FetchStatusCodes.S_CONNECT_LOST; +import static org.archive.modules.fetcher.FetchStatusCodes.S_DEEMED_NOT_FOUND; import java.util.Map; import java.util.logging.Logger; @@ -34,7 +35,8 @@ import org.archive.modules.ModuleAttributeConstants; import org.archive.modules.Processor; import org.archive.modules.net.CrawlHost; import org.archive.modules.net.CrawlServer; -import org.archive.modules.net.RobotsExclusionPolicy; +import org.archive.modules.net.IgnoreRobotsPolicy; +import org.archive.modules.net.Robotstxt; import org.archive.modules.net.ServerCache; import org.springframework.beans.factory.annotation.Autowired; @@ -172,7 +174,17 @@ public class DispositionProcessor extends Processor { if ("/robots.txt".equals(curi.getUURI().getPath())) { // Update server with robots info // NOTE: in some cases the curi's status can be changed here - server.updateRobots(metadata.getRobotsHonoringPolicy(), curi); +// server.updateRobots(metadata.getRobotsHonoringPolicy(), curi); + + server.updateRobots(curi); + + // shortcut retries w/ DEEMED when ignore-all + if (metadata.getRobotsPolicy() instanceof IgnoreRobotsPolicy) { + if(curi.getFetchStatus() < 0) { + // prevent the rest of the usual retries + curi.setFetchStatus(S_DEEMED_NOT_FOUND); + } + } } } catch (URIException e) { @@ -226,9 +238,9 @@ public class DispositionProcessor extends Processor { if (ua == null) { ua = metadata.getUserAgent(); } - RobotsExclusionPolicy rep = s.getRobots(); + Robotstxt rep = s.getRobotstxt(); if (rep != null) { - long crawlDelay = (long)(1000 * s.getRobots().getCrawlDelay(ua)); + long crawlDelay = (long)(1000 * rep.getDirectivesFor(ua).getCrawlDelay()); crawlDelay = (crawlDelay > respectThreshold) ? respectThreshold diff --git a/engine/src/main/java/org/archive/crawler/prefetch/PreconditionEnforcer.java b/engine/src/main/java/org/archive/crawler/prefetch/PreconditionEnforcer.java index ff1a775d..18f3ed73 100644 --- a/engine/src/main/java/org/archive/crawler/prefetch/PreconditionEnforcer.java +++ b/engine/src/main/java/org/archive/crawler/prefetch/PreconditionEnforcer.java @@ -30,6 +30,7 @@ import java.util.logging.Logger; import org.apache.commons.httpclient.URIException; import org.archive.crawler.reporting.CrawlerLoggerModule; +import org.archive.modules.CrawlMetadata; import org.archive.modules.CrawlURI; import org.archive.modules.ProcessResult; import org.archive.modules.Processor; @@ -39,9 +40,9 @@ import org.archive.modules.credential.CredentialStore; import org.archive.modules.extractor.Hop; import org.archive.modules.extractor.Link; import org.archive.modules.extractor.LinkContext; -import org.archive.modules.fetcher.UserAgentProvider; import org.archive.modules.net.CrawlHost; import org.archive.modules.net.CrawlServer; +import org.archive.modules.net.RobotsPolicy; import org.archive.modules.net.ServerCache; import org.archive.net.UURI; import org.archive.net.UURIFactory; @@ -105,12 +106,17 @@ public class PreconditionEnforcer extends Processor { kp.put("calculateRobotsOnly",calcOnly); } - public UserAgentProvider getUserAgentProvider() { - return (UserAgentProvider) kp.get("userAgentProvider"); + /** + * Auto-discovered module providing configured (or overridden) + * User-Agent value and RobotsHonoringPolicy + */ + CrawlMetadata metadata; + public CrawlMetadata getMetadata() { + return metadata; } @Autowired - public void setUserAgentProvider(UserAgentProvider provider) { - kp.put("userAgentProvider",provider); + public void setMetadata(CrawlMetadata provider) { + this.metadata = provider; } { @@ -237,8 +243,9 @@ public class PreconditionEnforcer extends Processor { } // test against robots.txt if available if (cs.isValidRobots()) { - String ua = getUserAgentProvider().getUserAgent(); - if(cs.getRobots().disallows(curi, ua)) { + String ua = metadata.getUserAgent(); + RobotsPolicy robots = metadata.getRobotsPolicy(); + if(!robots.allows(ua, curi, cs.getRobotstxt())) { if(getCalculateRobotsOnly()) { // annotate URI as excluded, but continue to process normally curi.getAnnotations().add("robotExcluded"); diff --git a/engine/src/main/resources/org/archive/crawler/migrate/migrate-template-crawler-beans.cxml b/engine/src/main/resources/org/archive/crawler/migrate/migrate-template-crawler-beans.cxml index 252157af..5200aa01 100644 --- a/engine/src/main/resources/org/archive/crawler/migrate/migrate-template-crawler-beans.cxml +++ b/engine/src/main/resources/org/archive/crawler/migrate/migrate-template-crawler-beans.cxml @@ -57,6 +57,7 @@ + @@ -144,9 +145,12 @@ - + + + + @@ -336,9 +340,7 @@ - - @@ -355,11 +357,20 @@ - - - + + @@ -489,7 +500,6 @@ - diff --git a/engine/src/main/resources/org/archive/crawler/restlet/profile-crawler-beans.cxml b/engine/src/main/resources/org/archive/crawler/restlet/profile-crawler-beans.cxml index ec4f7189..7b397e53 100644 --- a/engine/src/main/resources/org/archive/crawler/restlet/profile-crawler-beans.cxml +++ b/engine/src/main/resources/org/archive/crawler/restlet/profile-crawler-beans.cxml @@ -66,6 +66,7 @@ http://example.example/example + diff --git a/engine/testdata/selftest/conf/selftest-crawler-beans.cxml b/engine/testdata/selftest/conf/selftest-crawler-beans.cxml index c9af3519..8bf81679 100644 --- a/engine/testdata/selftest/conf/selftest-crawler-beans.cxml +++ b/engine/testdata/selftest/conf/selftest-crawler-beans.cxml @@ -43,10 +43,6 @@ crawlController.pauseAtStart=false - - - - diff --git a/modules/src/main/java/org/archive/modules/CrawlMetadata.java b/modules/src/main/java/org/archive/modules/CrawlMetadata.java index a73ff28d..5b174a59 100644 --- a/modules/src/main/java/org/archive/modules/CrawlMetadata.java +++ b/modules/src/main/java/org/archive/modules/CrawlMetadata.java @@ -20,10 +20,12 @@ package org.archive.modules; import java.io.Serializable; +import java.util.HashMap; +import java.util.Map; import java.util.regex.Matcher; import org.archive.modules.fetcher.UserAgentProvider; -import org.archive.modules.net.RobotsHonoringPolicy; +import org.archive.modules.net.RobotsPolicy; import org.archive.spring.BeanFieldsPatternValidator; import org.archive.spring.HasKeyedProperties; import org.archive.spring.HasValidator; @@ -53,18 +55,45 @@ implements UserAgentProvider, } /** - * Robots honoring + * Robots policy name */ { - setRobotsHonoringPolicy(new RobotsHonoringPolicy()); + setRobotsPolicyName("obey"); } - public RobotsHonoringPolicy getRobotsHonoringPolicy() { - return (RobotsHonoringPolicy) kp.get("robotsHonoringPolicy"); + public String getRobotsPolicyName() { + return (String) kp.get("robotsPolicyName"); } @Autowired(required=false) - public void setRobotsHonoringPolicy(RobotsHonoringPolicy policy) { - kp.put("robotsHonoringPolicy",policy); + public void setRobotsPolicyName(String policy) { + kp.put("robotsPolicyName",policy); } + + /** Map of all available RobotsPolicies, by name, to choose from. + * assembled from declared instances in configuration plus the standard + * 'obey' (aka 'classic') and 'ignore' policies. */ + Map availableRobotsPolicies = new HashMap(); + public Map getAvailableRobotsPolicies() { + return availableRobotsPolicies; + } + @Autowired(required=false) + public void setAvailableRobotsPolicies(Map policies) { + availableRobotsPolicies = policies; + ensureStandardPoliciesAvailable(); + } + protected void ensureStandardPoliciesAvailable() { + availableRobotsPolicies.putAll(RobotsPolicy.STANDARD_POLICIES); + } + + /** + * Get the currently-effective RobotsPolicy, as specified by the + * string name and chosen from the full available map. (Setting + * a different policy for some sites/URL patterns is best acheived + * by establishing a setting overlay for the robotsPolicyName + * property.) + */ + public RobotsPolicy getRobotsPolicy() { + return availableRobotsPolicies.get(getRobotsPolicyName()); + } String operator = ""; public String getOperator() { @@ -159,17 +188,14 @@ implements UserAgentProvider, this.jobName = jobName; } - public String getRobotsPolicyName() { - return getRobotsHonoringPolicy().getType().toString(); - } - public String getFrom() { return getOperatorFrom(); } - public void afterPropertiesSet() throws Exception { + public void afterPropertiesSet() { // force revalidation, throwing exception if invalid setOperatorContactUrl(getOperatorContactUrl()); + ensureStandardPoliciesAvailable(); } static Validator VALIDATOR = new BeanFieldsPatternValidator( @@ -193,5 +219,5 @@ implements UserAgentProvider, public Validator getValidator() { return VALIDATOR; - } + } } diff --git a/modules/src/main/java/org/archive/modules/extractor/ExtractorHTML.java b/modules/src/main/java/org/archive/modules/extractor/ExtractorHTML.java index 94945a0c..b704a85f 100644 --- a/modules/src/main/java/org/archive/modules/extractor/ExtractorHTML.java +++ b/modules/src/main/java/org/archive/modules/extractor/ExtractorHTML.java @@ -33,7 +33,7 @@ import org.apache.commons.httpclient.URIException; import org.archive.io.ReplayCharSequence; import org.archive.modules.CrawlMetadata; import org.archive.modules.CrawlURI; -import org.archive.modules.net.RobotsHonoringPolicy; +import org.archive.modules.net.RobotsPolicy; import org.archive.net.UURI; import org.archive.net.UURIFactory; import org.archive.util.ArchiveUtils; @@ -825,11 +825,9 @@ public class ExtractorHTML extends ContentExtractor implements InitializingBean // Look for the 'robots' meta-tag if("robots".equalsIgnoreCase(name) && content != null ) { curi.getData().put(A_META_ROBOTS, content); - RobotsHonoringPolicy policy = metadata.getRobotsHonoringPolicy(); + RobotsPolicy policy = metadata.getRobotsPolicy(); String contentLower = content.toLowerCase(); - if ((policy == null - || (!policy.isType(RobotsHonoringPolicy.Type.IGNORE) - && !policy.isType(RobotsHonoringPolicy.Type.CUSTOM))) + if (policy.obeyMetaRobotsNofollow() && (contentLower.indexOf("nofollow") >= 0 || contentLower.indexOf("none") >= 0)) { // if 'nofollow' or 'none' is specified and the diff --git a/modules/src/main/java/org/archive/modules/extractor/JerichoExtractorHTML.java b/modules/src/main/java/org/archive/modules/extractor/JerichoExtractorHTML.java index d3dfb695..8e1069a4 100644 --- a/modules/src/main/java/org/archive/modules/extractor/JerichoExtractorHTML.java +++ b/modules/src/main/java/org/archive/modules/extractor/JerichoExtractorHTML.java @@ -37,7 +37,7 @@ import org.apache.commons.httpclient.URIException; import org.apache.commons.lang.StringEscapeUtils; import org.apache.commons.lang.StringUtils; import org.archive.modules.CrawlURI; -import org.archive.modules.net.RobotsHonoringPolicy; +import org.archive.modules.net.RobotsPolicy; import org.archive.net.UURI; import org.archive.net.UURIFactory; import org.archive.util.DevUtils; @@ -304,13 +304,11 @@ public class JerichoExtractorHTML extends ExtractorHTML { if ("robots".equals(name) && content != null) { curi.getData().put(A_META_ROBOTS, content); - RobotsHonoringPolicy policy = metadata.getRobotsHonoringPolicy(); + RobotsPolicy policy = metadata.getRobotsPolicy(); String contentLower = content.toLowerCase(); - if ((policy == null || (!policy.isType( - RobotsHonoringPolicy.Type.IGNORE) && !policy.isType( - RobotsHonoringPolicy.Type.CUSTOM))) - && (contentLower.indexOf("nofollow") >= 0 || contentLower - .indexOf("none") >= 0)) { + if (policy.obeyMetaRobotsNofollow() + && (contentLower.indexOf("nofollow") >= 0 + || contentLower.indexOf("none") >= 0)) { // if 'nofollow' or 'none' is specified and the // honoring policy is not IGNORE or CUSTOM, end html extraction logger.fine("HTML extraction skipped due to robots meta-tag " + diff --git a/modules/src/main/java/org/archive/modules/net/CrawlServer.java b/modules/src/main/java/org/archive/modules/net/CrawlServer.java index b78979fe..6ab10743 100644 --- a/modules/src/main/java/org/archive/modules/net/CrawlServer.java +++ b/modules/src/main/java/org/archive/modules/net/CrawlServer.java @@ -26,9 +26,10 @@ import java.io.BufferedReader; import java.io.IOException; import java.io.InputStreamReader; import java.io.Serializable; -import java.io.StringReader; import java.util.HashSet; import java.util.Set; +import java.util.logging.Level; +import java.util.logging.Logger; import java.util.zip.Checksum; import org.apache.commons.collections.CollectionUtils; @@ -52,7 +53,8 @@ import org.archive.net.UURIFactory; * @author gojomo */ public class CrawlServer implements Serializable, FetchStats.HasFetchStats { - + private static final Logger logger = + Logger.getLogger(CrawlServer.class.getName()); private static final long serialVersionUID = 3L; public static final long ROBOTS_NOT_FETCHED = -1; @@ -62,7 +64,7 @@ public class CrawlServer implements Serializable, FetchStats.HasFetchStats { private final String server; // actually, host+port in the https case private int port; - private RobotsExclusionPolicy robots; + protected Robotstxt robotstxt; long robotsFetched = ROBOTS_NOT_FETCHED; boolean validRobots = false; Checksum robotstxtChecksum; @@ -98,22 +100,6 @@ public class CrawlServer implements Serializable, FetchStats.HasFetchStats { } } - /** Get the robots exclusion policy for this server. - * - * @return the robots exclusion policy for this server. - */ - public RobotsExclusionPolicy getRobots() { - return robots; - } - - /** Set the robots exclusion policy for this server. - * - * @param policy the policy to set. - */ - public void setRobots(RobotsExclusionPolicy policy) { - robots = policy; - } - public String toString() { return "CrawlServer("+server+")"; } @@ -139,96 +125,78 @@ public class CrawlServer implements Serializable, FetchStats.HasFetchStats { return true; } - /** Update the robots exclusion policy. - * - * @param curi the crawl URI containing the fetched robots.txt - * @throws IOException - */ - public synchronized void updateRobots(RobotsHonoringPolicy honoringPolicy, - CrawlURI curi) { - - robotsFetched = System.currentTimeMillis(); - - boolean gotSomething = curi.getFetchType() == HTTP_GET - && (curi.getFetchStatus() > 0 || curi.getFetchStatus() == S_DEEMED_NOT_FOUND ); - - - if (!gotSomething && curi.getFetchAttempts() < MIN_ROBOTS_RETRIES) { - // robots.txt lookup failed, still trying, no reason to consider IGNORE yet - validRobots = false; - return; - } - - - RobotsHonoringPolicy.Type type = honoringPolicy.getType(); - if (type == RobotsHonoringPolicy.Type.IGNORE) { - // IGNORE = ALLOWALL - robots = RobotsExclusionPolicy.ALLOWALL; - validRobots = true; - if(curi.getFetchStatus() < 0) { - // prevent the rest of the usual retries - curi.setFetchStatus(S_DEEMED_NOT_FOUND); - } - return; - } - - // special deeming for a particular kind of connection-lost (empty server response) - if(curi.getFetchStatus() == S_CONNECT_LOST && CollectionUtils.exists(curi.getNonFatalFailures(),new Predicate() { - public boolean evaluate(Object obj) { - return obj instanceof NoHttpResponseException; - } - })) { - curi.setFetchStatus(S_DEEMED_NOT_FOUND); - gotSomething = true; - } - - if (!gotSomething) { - // robots.txt fetch failed and exceptions (ignore/deeming) don't apply; no valid robots info yet - validRobots = false; - return; - } - - int fetchStatus = curi.getFetchStatus(); - if (fetchStatus < 200 || fetchStatus >= 300) { - // Not found or anything but a status code in the 2xx range is - // treated as giving access to all of a sites' content. - // This is the prevailing practice of Google, since 4xx - // responses on robots.txt are usually indicative of a - // misconfiguration or blanket-block, not an intentional - // indicator of partial blocking. - // TODO: consider handling server errors, redirects differently - robots = RobotsExclusionPolicy.ALLOWALL; - validRobots = true; - return; - } - - ReplayInputStream contentBodyStream = null; - try { - try { - BufferedReader reader; - if (type == RobotsHonoringPolicy.Type.CUSTOM) { - reader = new BufferedReader(new StringReader(honoringPolicy - .getCustomRobots())); - } else { - contentBodyStream = curi.getRecorder() - .getRecordedInput().getContentReplayInputStream(); - - contentBodyStream.setToResponseBodyStart(); - reader = new BufferedReader(new InputStreamReader( - contentBodyStream)); - } - robots = RobotsExclusionPolicy.policyFor( - reader, honoringPolicy); - validRobots = true; - } finally { - IOUtils.closeQuietly(contentBodyStream); - } - } catch (IOException e) { - robots = RobotsExclusionPolicy.ALLOWALL; - validRobots = true; - curi.getNonFatalFailures().add(e); - } + public Robotstxt getRobotstxt() { + return robotstxt; } + + /** Update the robotstxt + * + * @param curi the crawl URI containing the fetched robots.txt + * @throws IOException + */ + public synchronized void updateRobots(CrawlURI curi) { + + robotsFetched = System.currentTimeMillis(); + + boolean gotSomething = curi.getFetchType() == HTTP_GET + && (curi.getFetchStatus() > 0 || curi.getFetchStatus() == S_DEEMED_NOT_FOUND ); + + + if (!gotSomething && curi.getFetchAttempts() < MIN_ROBOTS_RETRIES) { + // robots.txt lookup failed, still trying, no reason to consider IGNORE yet + validRobots = false; + return; + } + + // special deeming for a particular kind of connection-lost (empty server response) + if(curi.getFetchStatus() == S_CONNECT_LOST && CollectionUtils.exists(curi.getNonFatalFailures(),new Predicate() { + public boolean evaluate(Object obj) { + return obj instanceof NoHttpResponseException; + } + })) { + curi.setFetchStatus(S_DEEMED_NOT_FOUND); + gotSomething = true; + } + + if (!gotSomething) { + // robots.txt fetch failed and exceptions (ignore/deeming) don't apply; no valid robots info yet + validRobots = false; + return; + } + + int fetchStatus = curi.getFetchStatus(); + if (fetchStatus < 200 || fetchStatus >= 300) { + // Not found or anything but a status code in the 2xx range is + // treated as giving access to all of a sites' content. + // This is the prevailing practice of Google, since 4xx + // responses on robots.txt are usually indicative of a + // misconfiguration or blanket-block, not an intentional + // indicator of partial blocking. + // TODO: consider handling server errors, redirects differently + robotstxt = Robotstxt.NO_ROBOTS; + validRobots = true; + return; + } + + ReplayInputStream contentBodyStream = null; + try { + BufferedReader reader; + contentBodyStream = curi.getRecorder() + .getRecordedInput().getContentReplayInputStream(); + + contentBodyStream.setToResponseBodyStart(); + reader = new BufferedReader(new InputStreamReader(contentBodyStream)); + robotstxt = new Robotstxt(reader); + validRobots = true; + } catch (IOException e) { + robotstxt = Robotstxt.NO_ROBOTS; + logger.log(Level.WARNING,"problem reading robots.txt for "+curi,e); + validRobots = true; + curi.getNonFatalFailures().add(e); + } finally { + IOUtils.closeQuietly(contentBodyStream); + } + } /** * @return The server string which might include a port number. diff --git a/modules/src/main/java/org/archive/modules/net/CustomRobotsPolicy.java b/modules/src/main/java/org/archive/modules/net/CustomRobotsPolicy.java new file mode 100644 index 00000000..1796189e --- /dev/null +++ b/modules/src/main/java/org/archive/modules/net/CustomRobotsPolicy.java @@ -0,0 +1,69 @@ +/* + * This file is part of the Heritrix web crawler (crawler.archive.org). + * + * Licensed to the Internet Archive (IA) by one or more individual + * contributors. + * + * The IA licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.archive.modules.net; + +import org.archive.io.ReadSource; +import org.archive.modules.CrawlURI; +import org.archive.spring.ConfigString; + +/** + * Follow a custom-written robots policy, rather than the site's own declarations + * + * Does not support overlays of different custom-robots; instead it is + * recommended each custom policy be declared as a separate bean, with a + * distinct name. + * + * Then, overlay the CrawlMetadata robotsPolicyName for each group of URIs + * that needs custom treatment + * + * @contributor gojomo + */ +public class CustomRobotsPolicy extends RobotsPolicy { + + Robotstxt customRobotstxt; + + /** textual alternate robots.txt rules to follow */ + ReadSource customRobots = new ConfigString(""); + public ReadSource getCustomRobots() { + return customRobots; + } + public void setCustomRobots(ReadSource customRobots) { + this.customRobots = customRobots; + customRobotstxt = new Robotstxt(customRobots); + } + + /** whether to obey the 'nofollow' directive in an HTML META ROBOTS element */ + boolean obeyMetaRobotsNofollow = true; + public boolean isObeyMetaRobotsNofollow() { + return obeyMetaRobotsNofollow; + } + public void setObeyMetaRobotsNofollow(boolean obeyMetaRobotsNofollow) { + this.obeyMetaRobotsNofollow = obeyMetaRobotsNofollow; + } + + @Override + public boolean allows(String userAgent, CrawlURI curi, Robotstxt robotstxt) { + return customRobotstxt.getDirectivesFor(userAgent).allows(getPath(curi)); + } + + @Override + public boolean obeyMetaRobotsNofollow() { + return obeyMetaRobotsNofollow; + } +} diff --git a/modules/src/test/java/org/archive/modules/net/RobotsHonoringPolicyTest.java b/modules/src/main/java/org/archive/modules/net/IgnoreRobotsPolicy.java similarity index 62% rename from modules/src/test/java/org/archive/modules/net/RobotsHonoringPolicyTest.java rename to modules/src/main/java/org/archive/modules/net/IgnoreRobotsPolicy.java index 5f153d1a..a0293202 100644 --- a/modules/src/test/java/org/archive/modules/net/RobotsHonoringPolicyTest.java +++ b/modules/src/main/java/org/archive/modules/net/IgnoreRobotsPolicy.java @@ -15,17 +15,26 @@ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. - */ + */package org.archive.modules.net; -package org.archive.modules.net; - -import org.archive.state.ModuleTestBase; +import org.archive.modules.CrawlURI; /** - * @author pjack + * Policy to ignore robots. + * + * @contributor gojomo */ -public class RobotsHonoringPolicyTest extends ModuleTestBase { - - // TODO TESTME +public class IgnoreRobotsPolicy extends RobotsPolicy { + public static RobotsPolicy INSTANCE = new IgnoreRobotsPolicy(); + @Override + public boolean allows(String userAgent, CrawlURI curi, Robotstxt robotstxt) { + return true; + } + + @Override + public boolean obeyMetaRobotsNofollow() { + return false; + } + } diff --git a/modules/src/main/java/org/archive/modules/net/MostFavoredRobotsPolicy.java b/modules/src/main/java/org/archive/modules/net/MostFavoredRobotsPolicy.java new file mode 100644 index 00000000..01bb028d --- /dev/null +++ b/modules/src/main/java/org/archive/modules/net/MostFavoredRobotsPolicy.java @@ -0,0 +1,84 @@ +/* + * This file is part of the Heritrix web crawler (crawler.archive.org). + * + * Licensed to the Internet Archive (IA) by one or more individual + * contributors. + * + * The IA licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.archive.modules.net; + +import java.util.LinkedList; +import java.util.List; + +import org.archive.modules.CrawlURI; + +/** + * Follow a most-favored robots policy -- allowing an URL if any of a number + * of user-agents would be allowed. + * + * @contributor gojomo + */ +public class MostFavoredRobotsPolicy extends RobotsPolicy { + + /** list of user-agents to try; if any are allowed, a URI will be crawled */ + List candidateUserAgents = new LinkedList(); + public List getCandidateUserAgents() { + return candidateUserAgents; + } + public void setCandidateUserAgents(List candidateUserAgents) { + this.candidateUserAgents = candidateUserAgents; + } + + /** whether to adopt the user-agent that is allowed for the fetch */ + boolean shouldMasquerade = false; + public boolean getShouldMasquerade() { + return shouldMasquerade; + } + public void setShouldMasquerade(boolean shouldMasquerade) { + this.shouldMasquerade = shouldMasquerade; + } + + /** whether to obey the 'nofollow' directive in an HTML META ROBOTS element */ + boolean obeyMetaRobotsNofollow = false; + public boolean isObeyMetaRobotsNofollow() { + return obeyMetaRobotsNofollow; + } + public void setObeyMetaRobotsNofollow(boolean obeyMetaRobotsNofollow) { + this.obeyMetaRobotsNofollow = obeyMetaRobotsNofollow; + } + + @Override + public boolean allows(String userAgent, CrawlURI curi, Robotstxt robotstxt) { + if (robotstxt.getDirectivesFor(userAgent).allows(getPath(curi))) { + return true; + } + for(String candidate : candidateUserAgents) { + if (robotstxt.getDirectivesFor(candidate).allows(getPath(curi))) { + if(shouldMasquerade) { + curi.setUserAgent(candidate); + } + return true; + } + } + // TODO: expand to offer option of following other rules in site's + // robots.txt, even if they don't match any of candidate set. + // TBD: which user-agent to use in that case. + return false; + } + + @Override + public boolean obeyMetaRobotsNofollow() { + return obeyMetaRobotsNofollow; + } +} diff --git a/modules/src/main/java/org/archive/modules/net/ObeyRobotsPolicy.java b/modules/src/main/java/org/archive/modules/net/ObeyRobotsPolicy.java new file mode 100644 index 00000000..60fb4e7a --- /dev/null +++ b/modules/src/main/java/org/archive/modules/net/ObeyRobotsPolicy.java @@ -0,0 +1,41 @@ +/* + * This file is part of the Heritrix web crawler (crawler.archive.org). + * + * Licensed to the Internet Archive (IA) by one or more individual + * contributors. + * + * The IA licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.archive.modules.net; + +import org.archive.modules.CrawlURI; + +/** + * Classic obey-robots-as-declared policy. + * + * @contributor gojomo + */ +public class ObeyRobotsPolicy extends RobotsPolicy { + public static RobotsPolicy INSTANCE = new ObeyRobotsPolicy(); + + @Override + public boolean allows(String userAgent, CrawlURI curi, Robotstxt robotstxt) { + return robotstxt.getDirectivesFor(userAgent).allows(getPath(curi)); + } + + @Override + public boolean obeyMetaRobotsNofollow() { + return true; + } + +} diff --git a/modules/src/main/java/org/archive/modules/net/RobotsExclusionPolicy.java b/modules/src/main/java/org/archive/modules/net/RobotsExclusionPolicy.java deleted file mode 100644 index 7e49dfe3..00000000 --- a/modules/src/main/java/org/archive/modules/net/RobotsExclusionPolicy.java +++ /dev/null @@ -1,264 +0,0 @@ -/* - * This file is part of the Heritrix web crawler (crawler.archive.org). - * - * Licensed to the Internet Archive (IA) by one or more individual - * contributors. - * - * The IA licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.archive.modules.net; - -import java.io.BufferedReader; -import java.io.IOException; -import java.io.ObjectInputStream; -import java.io.ObjectOutputStream; -import java.io.Serializable; -import java.util.ArrayList; -import java.util.Iterator; -import java.util.List; -import java.util.logging.Level; -import java.util.logging.Logger; - -import org.apache.commons.httpclient.URIException; -import org.archive.modules.CrawlURI; - -/** - * RobotsExclusionPolicy represents the actual policy adopted with - * respect to a specific remote server, usually constructed from - * consulting the robots.txt, if any, the server provided. - * - * (The similarly named RobotsHonoringPolicy, on the other hand, - * describes the strategy used by the crawler to determine to what - * extent it respects exclusion rules.) - * - * The expiration of policies after a suitable amount of time has - * elapsed since last fetch is handled outside this class, in - * CrawlServer itself. - * - * TODO: refactor RobotsHonoringPolicy to be a class-per-policy, and - * then see if a CrawlServer with a HonoringPolicy and a RobotsTxt - * makes this mediating class unnecessary. - * - * @author gojomo - * - */ -public class RobotsExclusionPolicy implements Serializable { - private static final long serialVersionUID = 3L; - - private static final Logger logger = - Logger.getLogger(RobotsExclusionPolicy.class.getName()); - - public static enum Type { - NORMAL, - ALLOWALL, - DENYALL - } - - private transient Type type = Type.NORMAL; - - final public static RobotsExclusionPolicy ALLOWALL = - new RobotsExclusionPolicy(Type.ALLOWALL); - final public static RobotsExclusionPolicy DENYALL = - new RobotsExclusionPolicy(Type.DENYALL); - - private Robotstxt robotstxt = null; - // FIXME?: this 'transient' seems wrong -- likely to cause - // all non-normal policies to break when CrawlServer - // go through a serialization/deserialization cycle - RobotsHonoringPolicy honoringPolicy = null; // was transient - - private String lastUsedUserAgent = null; - private List userAgentsToTest = null; - - /** - * @param settings - * @param reader - * @param honoringPolicy - * @return Robot exclusion policy. - * @throws IOException - */ - public static RobotsExclusionPolicy policyFor( - BufferedReader reader, RobotsHonoringPolicy honoringPolicy) - throws IOException { - Robotstxt robots = new Robotstxt(reader); - return (robots.allowsAll())? - ALLOWALL: - new RobotsExclusionPolicy(robots, honoringPolicy); - } - - - /** - * @param settings - * @param u - * @param d - * @param honoringPolicy - */ - public RobotsExclusionPolicy( - Robotstxt robotstxt, - RobotsHonoringPolicy honoringPolicy) { - this.robotstxt = robotstxt; - this.honoringPolicy = honoringPolicy; - - if(honoringPolicy == null) return; - - // If honoring policy is most favored user agent, all rules should be checked - if(honoringPolicy.isType(RobotsHonoringPolicy.Type.MOST_FAVORED)) { - userAgentsToTest = robotstxt.getUserAgents(); - - // IF honoring policy is most favored of set, then make a list with only the set as members - } else if(honoringPolicy.isType(RobotsHonoringPolicy.Type.MOST_FAVORED_SET)) { - userAgentsToTest = new ArrayList(); - Iterator userAgentSet = honoringPolicy.getUserAgents().iterator(); - while(userAgentSet.hasNext()) { - String userAgent = (String) userAgentSet.next(); - - Iterator iter = robotstxt.getUserAgents().iterator(); - while ( iter.hasNext() ) { - String ua = (String)iter.next(); - if (userAgent.indexOf(ua)>-1) { - userAgentsToTest.add(ua); - break; - } - } - } - } - } - - public RobotsExclusionPolicy(Type type) { - this(null, null); - this.type = type; - } - - public boolean disallows(CrawlURI curi, String userAgent) { - if (this == ALLOWALL) - return false; - if (this == DENYALL) - return true; - - // In the common case with policy=Classic, the useragent is remembered from uri to uri on - // the same server - if((honoringPolicy.isType(RobotsHonoringPolicy.Type.CLASSIC) - || honoringPolicy.isType(RobotsHonoringPolicy.Type.CUSTOM)) - && (lastUsedUserAgent == null - || !lastUsedUserAgent.equals(userAgent))) { - - lastUsedUserAgent = userAgent; - userAgentsToTest = new ArrayList(); - Iterator iter = robotstxt.getUserAgents().iterator(); - String lowerCaseUserAgent = userAgent.toLowerCase(); - while ( iter.hasNext() ) { - String ua = (String)iter.next(); - // ua in below is already lowercase. See Robotstxt.java line 60. - if (lowerCaseUserAgent.indexOf(ua)>-1) { - userAgentsToTest.add(ua); - break; // consider no more sections - } - } - } - - boolean disallow = false; - String ua = null; - - // Go thru list of all user agents we might act as - Iterator uas = userAgentsToTest.iterator(); - while(uas.hasNext()) { - ua = (String) uas.next(); - String path = null; - try { - path = curi.getUURI().getPathQuery(); - } catch (URIException e) { - logger.log(Level.SEVERE,"Failed getPathQuery from " + curi, e); - disallow = false; - break; - } - if(robotstxt.getDirectivesFor(ua).allows(path)) { - // at least one applicable set of rules allows - disallow = false; - break; - } else { - // at least one applicable set of rules disallows - // so disallow unless later test allows - disallow = true; - } - } - - // Are we supposed to masquerade as the user agent to which restrictions - // we follow? - if(honoringPolicy.shouldMasquerade() && ua != null && !ua.equals("")) { - curi.setUserAgent(ua); - } - return disallow; - } - - // Methods for object serialization. - - /** If object is DENYALL or ALLOWALL, only the object identity and type - * is written in the serialization stream. - * - * @param stream the serialization stream. - * @throws IOException - */ - private void writeObject(ObjectOutputStream stream) throws IOException { - stream.writeInt(type.ordinal()); - if (type == Type.NORMAL) { - stream.defaultWriteObject(); - } - } - - /** If object is DENYALL or ALLOWALL, only the object identity and type - * is read from the serialization stream. - * - * @param stream the serialization stream. - * @throws IOException - * @throws ClassNotFoundException - */ - private void readObject(ObjectInputStream stream) - throws IOException, ClassNotFoundException { - int ordinal = stream.readInt(); - type = Type.values()[ordinal]; - if (type == Type.NORMAL) { - stream.defaultReadObject(); - } - } - - /** If object is DENYALL or ALLOWALL, the object is replaced by constants - * so that check for object equality works. - * @return Object. - */ - private Object readResolve() { - if (type == Type.NORMAL) { - return this; - } else if (type == Type.ALLOWALL) { - return ALLOWALL; - } else if (type == Type.DENYALL) { - return DENYALL; - } - return null; - } - - /** - * Get the crawl-delay that applies to the given user-agent, or - * -1 (indicating no crawl-delay known) if not internal RobotsTxt - * instance. - * - * @param userAgent - * @return int Crawl-Delay value, or -1 if non available - */ - public float getCrawlDelay(String userAgent) { - if (robotstxt==null) { - return -1; - } - return robotstxt.getDirectivesFor(userAgent).getCrawlDelay(); - } -} diff --git a/modules/src/main/java/org/archive/modules/net/RobotsHonoringPolicy.java b/modules/src/main/java/org/archive/modules/net/RobotsHonoringPolicy.java deleted file mode 100644 index 532dfce9..00000000 --- a/modules/src/main/java/org/archive/modules/net/RobotsHonoringPolicy.java +++ /dev/null @@ -1,186 +0,0 @@ -/* - * This file is part of the Heritrix web crawler (crawler.archive.org). - * - * Licensed to the Internet Archive (IA) by one or more individual - * contributors. - * - * The IA licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.archive.modules.net; - -import java.io.Serializable; -import java.util.ArrayList; -import java.util.List; - -import org.archive.spring.HasKeyedProperties; -import org.archive.spring.KeyedProperties; - -/** - * RobotsHonoringPolicy represent the strategy used by the crawler - * for determining how robots.txt files will be honored. - * - * Five kinds of policies exist: - *
- *
classic:
- *
obey the first set of robots.txt directives that apply to your - * current user-agent
- *
ignore:
- *
ignore robots.txt directives entirely
- *
custom:
- *
obey a specific operator-entered set of robots.txt directives - * for a given host
- *
most-favored:
- *
obey the most liberal restrictions offered (if *any* crawler is - * allowed to get a page, get it)
- *
most-favored-set:
- *
given some set of user-agent patterns, obey the most liberal - * restriction offered to any
- *
- * - * The two last ones has the opportunity of adopting a different user-agent - * to reflect the restrictions we've opted to use. - * - * @author John Erik Halse - * - */ -public class RobotsHonoringPolicy implements Serializable, HasKeyedProperties { - private static final long serialVersionUID = 3L; - - KeyedProperties kp = new KeyedProperties(); - public KeyedProperties getKeyedProperties() { - return kp; - } - - /** - * Policy type. - */ - public static enum Type { - - /** Obeys all robts.txt rules for the configured user-agent. */ - CLASSIC, - - /** Ignores all robots rules. */ - IGNORE, - - /** Defers to custom-robots setting. */ - CUSTOM, - - /** Crawls URIs if the robots.txt allows any user-agent to crawl it. */ - MOST_FAVORED, - - /** - * Requires you to supply an list of alternate user-agents, and for - * every page, if any agent of the set is allowed, the page will be - * crawled. - */ - MOST_FAVORED_SET - } - - /* set default values of overridable properties */ - { - /** - * Policy type. The 'classic' policy simply obeys all robots.txt rules for - * the configured user-agent. The 'ignore' policy ignores all robots rules. - * The 'custom' policy allows you to specify a policy, in robots.txt format, - * as a setting. The 'most-favored' policy will crawl an URL if the - * robots.txt allows any user-agent to crawl it. The 'most-favored-set' - * policy requires you to supply an list of alternate user-agents, and for - * every page, if any agent of the set is allowed, the page will be crawled. - */ -// public final static Key TYPE = Key.make(Type.CLASSIC); - setType(Type.CLASSIC); - - /** - * Should we masquerade as another user agent when obeying the rules - * declared for it. Only relevant if the policy type is 'most-favored' or - * 'most-favored-set'. - */ -// public final static Key MASQUERADE = Key.make(false); - setMasquerade(false); - - /** - * Custom robots to use if policy type is 'custom'. Compose as if an actual - * robots.txt file. - */ -// public final static Key CUSTOM_ROBOTS = Key.make(""); - setCustomRobots(""); - - /** - * Alternate user-agent values to consider using for the 'most-favored-set' - * policy. - */ -// public final static Key> USER_AGENTS = Key.makeList(String.class); - setUserAgents(new ArrayList(0)); - } - - /** - * Creates a new instance of RobotsHonoringPolicy. - */ - public RobotsHonoringPolicy() { - } - - /** - * Check if policy is of a certain type. - * - * @param context An object that can be resolved into a settings object. - * @param type the type to check against. - * @return true if the policy is of the submitted type - */ - public boolean isType(Type type) { - return type == getType(); - } - - /** - * Get the supplied custom robots.txt - * - * @return String with content of alternate robots.txt - */ - public String getCustomRobots() { - return (String) kp.get("customRobots"); - } - public void setCustomRobots(String customRobots) { - kp.put("customRobots",customRobots); - } - - /** - * This method returns true if the crawler should masquerade as the user agent - * which restrictions it opted to use. - * - * (Only relevant for policy-types: most-favored and most-favored-set). - * - * @return true if we should masquerade - */ - public boolean shouldMasquerade() { - return (Boolean) kp.get("masquerade"); - } - public void setMasquerade(boolean masquerade) { - kp.put("masquerade",masquerade); - } - - - public Type getType() { - return (Type) kp.get("type"); - } - public void setType(Type type) { - kp.put("type",type); - } - - - @SuppressWarnings("unchecked") - public List getUserAgents() { - return (List) kp.get("userAgents"); - } - public void setUserAgents(List userAgents) { - kp.put("userAgents",userAgents); - } -} diff --git a/modules/src/main/java/org/archive/modules/net/RobotsPolicy.java b/modules/src/main/java/org/archive/modules/net/RobotsPolicy.java new file mode 100644 index 00000000..1bace024 --- /dev/null +++ b/modules/src/main/java/org/archive/modules/net/RobotsPolicy.java @@ -0,0 +1,54 @@ +/* + * This file is part of the Heritrix web crawler (crawler.archive.org). + * + * Licensed to the Internet Archive (IA) by one or more individual + * contributors. + * + * The IA licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.archive.modules.net; + +import java.util.HashMap; +import java.util.Map; + +import org.apache.commons.httpclient.URIException; +import org.archive.modules.CrawlURI; + +/** + * RobotsPolicy represents the strategy used by the crawler + * for determining how robots.txt files will be honored. + * + * @contributor gojomo + */ +abstract public class RobotsPolicy { + public static Map STANDARD_POLICIES = new HashMap(); + static { + STANDARD_POLICIES.put("obey", ObeyRobotsPolicy.INSTANCE); + // the obey policy has also historically been called 'classic' + STANDARD_POLICIES.put("classic", ObeyRobotsPolicy.INSTANCE); + STANDARD_POLICIES.put("ignore", IgnoreRobotsPolicy.INSTANCE); + } + + public abstract boolean allows(String userAgent, CrawlURI curi, Robotstxt robotstxt); + + public abstract boolean obeyMetaRobotsNofollow(); + + public String getPath(CrawlURI curi) { + try { + return curi.getUURI().getPath(); + } catch (URIException e) { + // unlikely + return ""; + } + } +} diff --git a/modules/src/main/java/org/archive/modules/net/Robotstxt.java b/modules/src/main/java/org/archive/modules/net/Robotstxt.java index af076a9c..8c127678 100644 --- a/modules/src/main/java/org/archive/modules/net/Robotstxt.java +++ b/modules/src/main/java/org/archive/modules/net/Robotstxt.java @@ -25,6 +25,11 @@ import java.util.HashMap; import java.util.LinkedList; import java.util.List; import java.util.Map; +import java.util.logging.Level; +import java.util.logging.Logger; + +import org.apache.commons.io.IOUtils; +import org.archive.io.ReadSource; /** * Utility class for parsing and representing 'robots.txt' format @@ -33,7 +38,9 @@ import java.util.Map; */ public class Robotstxt implements Serializable { static final long serialVersionUID = 7025386509301303890L; - + private static final Logger logger = + Logger.getLogger(Robotstxt.class.getName()); + // all user agents contained in this robots.txt // may be thinned of irrelevant entries LinkedList userAgents = new LinkedList(); @@ -44,8 +51,30 @@ public class Robotstxt implements Serializable { boolean hasErrors = false; static RobotsDirectives NO_DIRECTIVES = new RobotsDirectives(); + /** empty, reusable instance for all sites providing no rules */ + public static Robotstxt NO_ROBOTS = new Robotstxt(); + public Robotstxt() { + } + public Robotstxt(BufferedReader reader) throws IOException { + initializeFromReader(reader); + } + + public Robotstxt(ReadSource customRobots) { + BufferedReader reader = new BufferedReader(customRobots.obtainReader()); + try { + initializeFromReader(reader); + } catch (IOException e) { + logger.log(Level.SEVERE, + "robots ReadSource problem: potential for inadvertent overcrawling", + e); + } finally { + IOUtils.closeQuietly(reader); + } + } + + protected void initializeFromReader(BufferedReader reader) throws IOException { String read; // current is the disallowed paths for the preceding User-Agent(s) RobotsDirectives current = null; @@ -151,7 +180,6 @@ public class Robotstxt implements Serializable { } } - /** * Does this policy effectively allow everything? (No * disallows or timing (crawl-delay) directives?) diff --git a/modules/src/test/java/org/archive/modules/extractor/ExtractorHTMLTest.java b/modules/src/test/java/org/archive/modules/extractor/ExtractorHTMLTest.java index 0954fda3..3780a807 100644 --- a/modules/src/test/java/org/archive/modules/extractor/ExtractorHTMLTest.java +++ b/modules/src/test/java/org/archive/modules/extractor/ExtractorHTMLTest.java @@ -29,7 +29,6 @@ import org.apache.commons.collections.Predicate; import org.apache.commons.httpclient.URIException; import org.archive.modules.CrawlMetadata; import org.archive.modules.CrawlURI; -import org.archive.modules.net.RobotsHonoringPolicy; import org.archive.net.UURI; import org.archive.net.UURIFactory; import org.archive.util.Recorder; @@ -81,7 +80,7 @@ public class ExtractorHTMLTest extends StringExtractorTestBase { UriErrorLoggerModule ulm = new UnitTestUriLoggerModule(); result.setLoggerModule(ulm); CrawlMetadata metadata = new CrawlMetadata(); - metadata.setRobotsHonoringPolicy(new RobotsHonoringPolicy()); + metadata.afterPropertiesSet(); result.setMetadata(metadata); result.afterPropertiesSet(); return result; diff --git a/modules/src/test/java/org/archive/modules/extractor/JerichoExtractorHTMLTest.java b/modules/src/test/java/org/archive/modules/extractor/JerichoExtractorHTMLTest.java index f2ef95d3..9668c5bd 100644 --- a/modules/src/test/java/org/archive/modules/extractor/JerichoExtractorHTMLTest.java +++ b/modules/src/test/java/org/archive/modules/extractor/JerichoExtractorHTMLTest.java @@ -25,7 +25,6 @@ import org.apache.commons.collections.Predicate; import org.apache.commons.httpclient.URIException; import org.archive.modules.CrawlMetadata; import org.archive.modules.CrawlURI; -import org.archive.modules.net.RobotsHonoringPolicy; import org.archive.net.UURI; import org.archive.net.UURIFactory; @@ -44,7 +43,7 @@ public class JerichoExtractorHTMLTest extends ExtractorHTMLTest { UriErrorLoggerModule ulm = new UnitTestUriLoggerModule(); result.setLoggerModule(ulm); CrawlMetadata metadata = new CrawlMetadata(); - metadata.setRobotsHonoringPolicy(new RobotsHonoringPolicy()); + metadata.afterPropertiesSet(); result.setMetadata(metadata); result.afterPropertiesSet(); return result; diff --git a/modules/src/test/java/org/archive/modules/writer/ARCWriterProcessorTest.java b/modules/src/test/java/org/archive/modules/writer/ARCWriterProcessorTest.java index 6d60d578..ddf400e8 100644 --- a/modules/src/test/java/org/archive/modules/writer/ARCWriterProcessorTest.java +++ b/modules/src/test/java/org/archive/modules/writer/ARCWriterProcessorTest.java @@ -24,8 +24,6 @@ import java.io.File; import org.archive.modules.CrawlMetadata; import org.archive.modules.ProcessorTestBase; import org.archive.modules.fetcher.DefaultServerCache; -import org.archive.modules.net.RobotsHonoringPolicy; -import org.archive.modules.writer.ARCWriterProcessor; import org.archive.spring.ConfigPath; import org.archive.util.TmpDirTestCase; @@ -47,7 +45,7 @@ public class ARCWriterProcessorTest extends ProcessorTestBase { result.setDirectory(new ConfigPath("test",tmp.getAbsolutePath())); result.setServerCache(new DefaultServerCache()); CrawlMetadata metadata = new CrawlMetadata(); - metadata.setRobotsHonoringPolicy(new RobotsHonoringPolicy()); + metadata.afterPropertiesSet(); result.setMetadataProvider(metadata); result.start(); return result; diff --git a/modules/src/test/java/org/archive/modules/writer/WARCWriterProcessorTest.java b/modules/src/test/java/org/archive/modules/writer/WARCWriterProcessorTest.java index 98affe11..067c75d9 100644 --- a/modules/src/test/java/org/archive/modules/writer/WARCWriterProcessorTest.java +++ b/modules/src/test/java/org/archive/modules/writer/WARCWriterProcessorTest.java @@ -24,7 +24,6 @@ import java.io.File; import org.archive.modules.CrawlMetadata; import org.archive.modules.ProcessorTestBase; import org.archive.modules.fetcher.DefaultServerCache; -import org.archive.modules.net.RobotsHonoringPolicy; import org.archive.spring.ConfigPath; import org.archive.util.TmpDirTestCase; @@ -45,7 +44,7 @@ public class WARCWriterProcessorTest extends ProcessorTestBase { result.setDirectory(new ConfigPath("test",tmp.getAbsolutePath())); result.setServerCache(new DefaultServerCache()); CrawlMetadata metadata = new CrawlMetadata(); - metadata.setRobotsHonoringPolicy(new RobotsHonoringPolicy()); + metadata.afterPropertiesSet(); result.setMetadataProvider(metadata); result.start(); return result;