diff --git a/engine/src/main/java/org/archive/crawler/postprocessor/CandidatesProcessor.java b/engine/src/main/java/org/archive/crawler/postprocessor/CandidatesProcessor.java index a2746e93..3fff1d30 100644 --- a/engine/src/main/java/org/archive/crawler/postprocessor/CandidatesProcessor.java +++ b/engine/src/main/java/org/archive/crawler/postprocessor/CandidatesProcessor.java @@ -23,6 +23,8 @@ package org.archive.crawler.postprocessor; import static org.archive.modules.fetcher.FetchStatusCodes.S_DEFERRED; import static org.archive.modules.fetcher.FetchStatusCodes.S_PREREQUISITE_UNSCHEDULABLE_FAILURE; +import java.util.LinkedList; + import org.apache.commons.httpclient.URIException; import org.archive.crawler.framework.Frontier; import org.archive.crawler.reporting.CrawlerLoggerModule; @@ -160,6 +162,15 @@ public class CandidatesProcessor extends Processor { return; } + /* + * The same outlink may have been discovered both in a context that + * makes it a seed, and in a context that doesn't. In that case we want + * to make sure the seed version of the outlink is the one that gets + * processed. In order to do that, we put all the candidates in a list, + * with seeds at the front of the list, then traverse the list in order. + */ + LinkedList candidates = new LinkedList(); + for (Link wref: curi.getOutLinks()) { CrawlURI candidate; try { @@ -179,23 +190,30 @@ public class CandidatesProcessor extends Processor { if(getSeedsRedirectNewSeeds() && curi.isSeed() && wref.getHopType() == Hop.REFER) { - candidate.setSeed(true); + candidate.setSeed(true); + candidates.addFirst(candidate); // seeds at front of list + } else { + candidates.addLast(candidate); // non-seeds at end of list } - getCandidateChain().process(candidate, null); - if(candidate.getFetchStatus()>=0) { - if(checkForSeedPromotion(candidate)) { - getSeeds().addSeed(candidate); - } else { - frontier.schedule(candidate); - } - curi.getOutCandidates().add(candidate); - } - } finally { KeyedProperties.clearOverridesFrom(candidate); KeyedProperties.loadOverridesFrom(curi); } } + + // process candidates in order, seeds first + for (CrawlURI candidate: candidates) { + getCandidateChain().process(candidate, null); + if(candidate.getFetchStatus()>=0) { + if(checkForSeedPromotion(candidate)) { + getSeeds().addSeed(candidate); + } else { + frontier.schedule(candidate); + } + curi.getOutCandidates().add(candidate); + } + } + curi.getOutLinks().clear(); }