From 99a49b2df7f31c37b71968c442e298a1b7322eff Mon Sep 17 00:00:00 2001 From: gojomo Date: Fri, 27 May 2011 01:24:49 +0000 Subject: [PATCH] Avoid treating temporarily-DEFERRED robots.txt handling as-if a failure-to-ignore in robots-ignored case * DispositionProcessor treat S_DEFERRED different from other negative error statuses --- .../archive/crawler/postprocessor/DispositionProcessor.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/engine/src/main/java/org/archive/crawler/postprocessor/DispositionProcessor.java b/engine/src/main/java/org/archive/crawler/postprocessor/DispositionProcessor.java index 17dedc67..49ae9e72 100644 --- a/engine/src/main/java/org/archive/crawler/postprocessor/DispositionProcessor.java +++ b/engine/src/main/java/org/archive/crawler/postprocessor/DispositionProcessor.java @@ -25,6 +25,7 @@ import static org.archive.modules.CoreAttributeConstants.A_FETCH_BEGAN_TIME; import static org.archive.modules.fetcher.FetchStatusCodes.S_CONNECT_FAILED; import static org.archive.modules.fetcher.FetchStatusCodes.S_CONNECT_LOST; import static org.archive.modules.fetcher.FetchStatusCodes.S_DEEMED_NOT_FOUND; +import static org.archive.modules.fetcher.FetchStatusCodes.S_DEFERRED; import java.util.Map; import java.util.logging.Logger; @@ -196,7 +197,7 @@ public class DispositionProcessor extends Processor { // shortcut retries w/ DEEMED when ignore-all if (metadata.getRobotsPolicy() instanceof IgnoreRobotsPolicy) { - if(curi.getFetchStatus() < 0) { + if(curi.getFetchStatus() < 0 && curi.getFetchStatus()!=S_DEFERRED) { // prevent the rest of the usual retries curi.setFetchStatus(S_DEEMED_NOT_FOUND); }