From 29fa5f6563793b3d27e8cd3ff58e5193cf3b8ac2 Mon Sep 17 00:00:00 2001
From: nlevitt <nlevitt@daa5b2f2-a927-0410-8b2d-f5f262fa301a>
Date: Wed, 25 May 2011 04:16:53 +0000
Subject: [PATCH] Fix for HER-1891 seed redirect url sometimes "0 NOTCRAWLED"
 in seeds-report.txt even when crawled, when original seed also has a regular
 link to the redirect url * CandidatesProcessor.java     innerProcess() -
 present seed outlinks to the frontier ahead of non-seed outlinks, so that
 seed version of any duplicated outlink is always the one that's crawled

---
 .../postprocessor/CandidatesProcessor.java    | 40 ++++++++++++++-----
 1 file changed, 29 insertions(+), 11 deletions(-)

diff --git a/engine/src/main/java/org/archive/crawler/postprocessor/CandidatesProcessor.java b/engine/src/main/java/org/archive/crawler/postprocessor/CandidatesProcessor.java
index a2746e93..3fff1d30 100644
--- a/engine/src/main/java/org/archive/crawler/postprocessor/CandidatesProcessor.java
+++ b/engine/src/main/java/org/archive/crawler/postprocessor/CandidatesProcessor.java
@@ -23,6 +23,8 @@ package org.archive.crawler.postprocessor;
 import static org.archive.modules.fetcher.FetchStatusCodes.S_DEFERRED;
 import static org.archive.modules.fetcher.FetchStatusCodes.S_PREREQUISITE_UNSCHEDULABLE_FAILURE;
 
+import java.util.LinkedList;
+
 import org.apache.commons.httpclient.URIException;
 import org.archive.crawler.framework.Frontier;
 import org.archive.crawler.reporting.CrawlerLoggerModule;
@@ -160,6 +162,15 @@ public class CandidatesProcessor extends Processor {
             return;
         }
 
+        /*
+         * The same outlink may have been discovered both in a context that
+         * makes it a seed, and in a context that doesn't. In that case we want
+         * to make sure the seed version of the outlink is the one that gets
+         * processed. In order to do that, we put all the candidates in a list,
+         * with seeds at the front of the list, then traverse the list in order.
+         */
+        LinkedList<CrawlURI> candidates = new LinkedList<CrawlURI>();
+        
         for (Link wref: curi.getOutLinks()) {
             CrawlURI candidate;
             try {
@@ -179,23 +190,30 @@ public class CandidatesProcessor extends Processor {
                 
                 if(getSeedsRedirectNewSeeds() && curi.isSeed() 
                         && wref.getHopType() == Hop.REFER) {
-                    candidate.setSeed(true); 
+                    candidate.setSeed(true);
+                    candidates.addFirst(candidate); // seeds at front of list
+                } else {
+                    candidates.addLast(candidate);  // non-seeds at end of list
                 }
-                getCandidateChain().process(candidate, null); 
-                if(candidate.getFetchStatus()>=0) {
-                    if(checkForSeedPromotion(candidate)) {
-                        getSeeds().addSeed(candidate);
-                    } else {
-                        frontier.schedule(candidate);
-                    }
-                    curi.getOutCandidates().add(candidate);
-                }
-                
             } finally {
                 KeyedProperties.clearOverridesFrom(candidate); 
                 KeyedProperties.loadOverridesFrom(curi);
             }
         }
+
+        // process candidates in order, seeds first
+        for (CrawlURI candidate: candidates) {
+            getCandidateChain().process(candidate, null); 
+            if(candidate.getFetchStatus()>=0) {
+                if(checkForSeedPromotion(candidate)) {
+                    getSeeds().addSeed(candidate);
+                } else {
+                    frontier.schedule(candidate);
+                }
+                curi.getOutCandidates().add(candidate);
+            }
+        }
+        
         curi.getOutLinks().clear();
     }