lee/heritrix3
1
0
Fork 0
mirror of https://github.com/internetarchive/heritrix3.git synced 2026-09-25 07:05:49 +00:00
Code Issues Packages Projects Releases Wiki Activity

Fix bug reported by Travis where in certain corner cases where urls are processed in a particular order, a seed can be incorrectly tagged with S_ROBOTS_PREREQUISITE_FAILURE

Browse Source 
* DispositionProcessor.java
    innerProcess() - do not update server's robots info if robots fetch has been deferred
This commit is contained in:
nlevitt
2011-08-04 01:32:09 +00:00
parent 706869e586
commit ada8cbd198
1 changed files with 2 additions and 2 deletions
Download Patch File Download Diff File Expand all files Collapse all files
engine/src/main/java/org/archive/crawler/postprocessor/DispositionProcessor.java
+2 -2
View File
@@ -20,8 +20,8 @@
package org.archive.crawler.postprocessor;
import static org.archive.modules.CoreAttributeConstants.A_FETCH_COMPLETED_TIME;
import static org.archive.modules.CoreAttributeConstants.A_FETCH_BEGAN_TIME;
import static org.archive.modules.CoreAttributeConstants.A_FETCH_COMPLETED_TIME;
import static org.archive.modules.fetcher.FetchStatusCodes.S_CONNECT_FAILED;
import static org.archive.modules.fetcher.FetchStatusCodes.S_CONNECT_LOST;
import static org.archive.modules.fetcher.FetchStatusCodes.S_DEEMED_NOT_FOUND;
@@ -188,7 +188,7 @@ public class DispositionProcessor extends Processor {
// Update robots info
try {
if ("/robots.txt".equals(curi.getUURI().getPath())) {
if ("/robots.txt".equals(curi.getUURI().getPath()) && curi.getFetchStatus() != S_DEFERRED) {
// shortcut retries w/ DEEMED when ignore-all
if (metadata.getRobotsPolicy() instanceof IgnoreRobotsPolicy) {
if(curi.getFetchStatus() < 0 && curi.getFetchStatus()!=S_DEFERRED) {