diff --git a/engine/src/main/java/org/archive/crawler/postprocessor/DispositionProcessor.java b/engine/src/main/java/org/archive/crawler/postprocessor/DispositionProcessor.java index 17dedc67..49ae9e72 100644 --- a/engine/src/main/java/org/archive/crawler/postprocessor/DispositionProcessor.java +++ b/engine/src/main/java/org/archive/crawler/postprocessor/DispositionProcessor.java @@ -25,6 +25,7 @@ import static org.archive.modules.CoreAttributeConstants.A_FETCH_BEGAN_TIME; import static org.archive.modules.fetcher.FetchStatusCodes.S_CONNECT_FAILED; import static org.archive.modules.fetcher.FetchStatusCodes.S_CONNECT_LOST; import static org.archive.modules.fetcher.FetchStatusCodes.S_DEEMED_NOT_FOUND; +import static org.archive.modules.fetcher.FetchStatusCodes.S_DEFERRED; import java.util.Map; import java.util.logging.Logger; @@ -196,7 +197,7 @@ public class DispositionProcessor extends Processor { // shortcut retries w/ DEEMED when ignore-all if (metadata.getRobotsPolicy() instanceof IgnoreRobotsPolicy) { - if(curi.getFetchStatus() < 0) { + if(curi.getFetchStatus() < 0 && curi.getFetchStatus()!=S_DEFERRED) { // prevent the rest of the usual retries curi.setFetchStatus(S_DEEMED_NOT_FOUND); }