From ada8cbd198eb8439d6449b44f86ca114e9cca9ea Mon Sep 17 00:00:00 2001 From: nlevitt Date: Thu, 4 Aug 2011 01:32:09 +0000 Subject: [PATCH] Fix bug reported by Travis where in certain corner cases where urls are processed in a particular order, a seed can be incorrectly tagged with S_ROBOTS_PREREQUISITE_FAILURE * DispositionProcessor.java innerProcess() - do not update server's robots info if robots fetch has been deferred --- .../archive/crawler/postprocessor/DispositionProcessor.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/engine/src/main/java/org/archive/crawler/postprocessor/DispositionProcessor.java b/engine/src/main/java/org/archive/crawler/postprocessor/DispositionProcessor.java index 38f3284d..01879ac0 100644 --- a/engine/src/main/java/org/archive/crawler/postprocessor/DispositionProcessor.java +++ b/engine/src/main/java/org/archive/crawler/postprocessor/DispositionProcessor.java @@ -20,8 +20,8 @@ package org.archive.crawler.postprocessor; -import static org.archive.modules.CoreAttributeConstants.A_FETCH_COMPLETED_TIME; import static org.archive.modules.CoreAttributeConstants.A_FETCH_BEGAN_TIME; +import static org.archive.modules.CoreAttributeConstants.A_FETCH_COMPLETED_TIME; import static org.archive.modules.fetcher.FetchStatusCodes.S_CONNECT_FAILED; import static org.archive.modules.fetcher.FetchStatusCodes.S_CONNECT_LOST; import static org.archive.modules.fetcher.FetchStatusCodes.S_DEEMED_NOT_FOUND; @@ -188,7 +188,7 @@ public class DispositionProcessor extends Processor { // Update robots info try { - if ("/robots.txt".equals(curi.getUURI().getPath())) { + if ("/robots.txt".equals(curi.getUURI().getPath()) && curi.getFetchStatus() != S_DEFERRED) { // shortcut retries w/ DEEMED when ignore-all if (metadata.getRobotsPolicy() instanceof IgnoreRobotsPolicy) { if(curi.getFetchStatus() < 0 && curi.getFetchStatus()!=S_DEFERRED) {