Avoid treating temporarily-DEFERRED robots.txt handling as-if a failure-to-ignore in robots-ignored case

* DispositionProcessor
    treat S_DEFERRED different from other negative error statuses
This commit is contained in:
gojomo committed 2011-05-27 01:24:49 +00:00
1 parent 728951d1ff
commit 99a49b2df7
1 file changed
+2 -1
@@ -25,6 +25,7 @@ import static org.archive.modules.CoreAttributeConstants.A_FETCH_BEGAN_TIME;
import static org.archive.modules.fetcher.FetchStatusCodes.S_CONNECT_FAILED;
import static org.archive.modules.fetcher.FetchStatusCodes.S_CONNECT_LOST;
import static org.archive.modules.fetcher.FetchStatusCodes.S_DEEMED_NOT_FOUND;
import static org.archive.modules.fetcher.FetchStatusCodes.S_DEFERRED;
import java.util.Map;
import java.util.logging.Logger;
@@ -196,7 +197,7 @@ public class DispositionProcessor extends Processor {
// shortcut retries w/ DEEMED when ignore-all
if (metadata.getRobotsPolicy() instanceof IgnoreRobotsPolicy) {
if(curi.getFetchStatus() < 0) {
if(curi.getFetchStatus() < 0 && curi.getFetchStatus()!=S_DEFERRED) {
// prevent the rest of the usual retries
curi.setFetchStatus(S_DEEMED_NOT_FOUND);
}