mirror of
https://github.com/internetarchive/heritrix3.git
synced 2026-11-16 10:47:42 +00:00
Avoid treating temporarily-DEFERRED robots.txt handling as-if a failure-to-ignore in robots-ignored case
* DispositionProcessor
treat S_DEFERRED different from other negative error statuses
This commit is contained in:
1 parent
728951d1ff
commit
99a49b2df7
1 file changed
+2
-1
@@ -25,6 +25,7 @@ import static org.archive.modules.CoreAttributeConstants.A_FETCH_BEGAN_TIME;
|
||||
import static org.archive.modules.fetcher.FetchStatusCodes.S_CONNECT_FAILED;
|
||||
import static org.archive.modules.fetcher.FetchStatusCodes.S_CONNECT_LOST;
|
||||
import static org.archive.modules.fetcher.FetchStatusCodes.S_DEEMED_NOT_FOUND;
|
||||
import static org.archive.modules.fetcher.FetchStatusCodes.S_DEFERRED;
|
||||
|
||||
import java.util.Map;
|
||||
import java.util.logging.Logger;
|
||||
@@ -196,7 +197,7 @@ public class DispositionProcessor extends Processor {
|
||||
|
||||
// shortcut retries w/ DEEMED when ignore-all
|
||||
if (metadata.getRobotsPolicy() instanceof IgnoreRobotsPolicy) {
|
||||
if(curi.getFetchStatus() < 0) {
|
||||
if(curi.getFetchStatus() < 0 && curi.getFetchStatus()!=S_DEFERRED) {
|
||||
// prevent the rest of the usual retries
|
||||
curi.setFetchStatus(S_DEEMED_NOT_FOUND);
|
||||
}
|
||||
|
||||
Reference in new issue
Block a user