lee/heritrix3
1
0
Fork 0
mirror of https://github.com/internetarchive/heritrix3.git synced 2026-09-25 23:25:44 +00:00
Code Issues Packages Projects Releases Wiki Activity

Fix for HER-1891 seed redirect url sometimes "0 NOTCRAWLED" in seeds-report.txt

Browse Source 
even when crawled, when original seed also has a regular link to the redirect
url
* CandidatesProcessor.java
    innerProcess() - present seed outlinks to the frontier ahead of non-seed outlinks, so that seed version of any duplicated outlink is always the one that's crawled
This commit is contained in:
nlevitt
2011-05-25 04:16:53 +00:00
parent c2c24d1939
commit 29fa5f6563
1 changed files with 29 additions and 11 deletions
Download Patch File Download Diff File Expand all files Collapse all files
engine/src/main/java/org/archive/crawler/postprocessor/CandidatesProcessor.java
+29 -11
View File
@@ -23,6 +23,8 @@ package org.archive.crawler.postprocessor;
import static org.archive.modules.fetcher.FetchStatusCodes.S_DEFERRED;
import static org.archive.modules.fetcher.FetchStatusCodes.S_PREREQUISITE_UNSCHEDULABLE_FAILURE;
import java.util.LinkedList;
import org.apache.commons.httpclient.URIException;
import org.archive.crawler.framework.Frontier;
import org.archive.crawler.reporting.CrawlerLoggerModule;
@@ -160,6 +162,15 @@ public class CandidatesProcessor extends Processor {
return;
}
/*
* The same outlink may have been discovered both in a context that
* makes it a seed, and in a context that doesn't. In that case we want
* to make sure the seed version of the outlink is the one that gets
* processed. In order to do that, we put all the candidates in a list,
* with seeds at the front of the list, then traverse the list in order.
*/
LinkedList<CrawlURI> candidates = new LinkedList<CrawlURI>();
for (Link wref: curi.getOutLinks()) {
CrawlURI candidate;
try {
@@ -179,23 +190,30 @@ public class CandidatesProcessor extends Processor {
if(getSeedsRedirectNewSeeds() && curi.isSeed()
&& wref.getHopType() == Hop.REFER) {
candidate.setSeed(true);
candidate.setSeed(true);
candidates.addFirst(candidate); // seeds at front of list
} else {
candidates.addLast(candidate); // non-seeds at end of list
}
getCandidateChain().process(candidate, null);
if(candidate.getFetchStatus()>=0) {
if(checkForSeedPromotion(candidate)) {
getSeeds().addSeed(candidate);
} else {
frontier.schedule(candidate);
}
curi.getOutCandidates().add(candidate);
}
} finally {
KeyedProperties.clearOverridesFrom(candidate);
KeyedProperties.loadOverridesFrom(curi);
}
}
// process candidates in order, seeds first
for (CrawlURI candidate: candidates) {
getCandidateChain().process(candidate, null);
if(candidate.getFetchStatus()>=0) {
if(checkForSeedPromotion(candidate)) {
getSeeds().addSeed(candidate);
} else {
frontier.schedule(candidate);
}
curi.getOutCandidates().add(candidate);
}
}
curi.getOutLinks().clear();
}