HER-2039: Initial work. Eliminated Link class and dealt with resultant

errors. Not tested any further than that.

engine/src/main/java/org/archive/crawler/postprocessor/CandidatesProcessor.java

@@ -23,7 +23,6 @@ package org.archive.crawler.postprocessor;
 import static org.archive.modules.fetcher.FetchStatusCodes.S_DEFERRED;
 import static org.archive.modules.fetcher.FetchStatusCodes.S_PREREQUISITE_UNSCHEDULABLE_FAILURE;
 
-import org.apache.commons.httpclient.URIException;
 import org.archive.crawler.framework.Frontier;
 import org.archive.crawler.reporting.CrawlerLoggerModule;
 import org.archive.crawler.spring.SheetOverlaysManager;
@@ -32,7 +31,6 @@ import org.archive.modules.CrawlURI;
 import org.archive.modules.Processor;
 import org.archive.modules.SchedulingConstants;
 import org.archive.modules.extractor.Hop;
-import org.archive.modules.extractor.Link;
 import org.archive.modules.seeds.SeedModule;
 import org.archive.spring.KeyedProperties;
 import org.springframework.beans.factory.annotation.Autowired;
@@ -219,15 +217,6 @@ public class CandidatesProcessor extends Processor {
             return;
         }
 
-        // (2) NEW: also (and before-outlinks) run outCandidates (usually empty;
-        // only current use is a form-submission CrawlURI; could 
-        // potentially take over prerequisite duties for consistency
-        for(CrawlURI candidate : curi.getOutCandidates()) {
-            
-            runCandidateChain(candidate, curi);
-            
-        }
-        
         // Only consider candidate links of error pages if configured to do so
         if (!getProcessErrorOutlinks() 
                 && (curi.getFetchStatus() < 200 || curi.getFetchStatus() >= 400)) {
@@ -236,21 +225,10 @@ public class CandidatesProcessor extends Processor {
         }
 
         // (3) Handle outlinks (usual bulk of discoveries) 
-        for (Link wref: curi.getOutLinks()) {
-            CrawlURI candidate;
-            try {
-                candidate = curi.createCrawlURI(curi.getBaseURI(),wref);
-            } catch (URIException e) {
-                loggerModule.logUriError(e, curi.getUURI(), 
-                        wref.getDestination().toString());
-                continue;
-            }
+        for (CrawlURI candidate: curi.getOutLinks()) {
             
             runCandidateChain(candidate, curi);
 
-            // TODO: evaluate if this necessary (anyone uses?); wise (bloat?) 
-            curi.getOutCandidates().add(candidate);
-
         }
         curi.getOutLinks().clear();
     }

engine/src/main/java/org/archive/crawler/postprocessor/LinksScoper.java

@@ -27,14 +27,12 @@ import static org.archive.modules.fetcher.FetchStatusCodes.S_PREREQUISITE_UNSCHE
 import java.util.logging.Level;
 import java.util.logging.Logger;
 
-import org.apache.commons.httpclient.URIException;
 import org.archive.crawler.framework.Scoper;
 import org.archive.modules.CrawlURI;
 import org.archive.modules.deciderules.DecideResult;
 import org.archive.modules.deciderules.DecideRule;
 import org.archive.modules.deciderules.RejectDecideRule;
 import org.archive.modules.extractor.Hop;
-import org.archive.modules.extractor.Link;
 
 /**
  * Determine which extracted links are within scope.
@@ -151,18 +149,11 @@ public class LinksScoper extends Scoper {
         final boolean redirectsNewSeeds = getSeedsRedirectNewSeeds(); 
         int preferenceDepthHops = getPreferenceDepthHops(); 
         
-        for (Link wref: curi.getOutLinks()) try {
-            int directive = getSchedulingFor(curi, wref, preferenceDepthHops);
-            CrawlURI caURI = curi.createCrawlURI(curi.getBaseURI(), 
-                    wref, directive, 
-                    considerAsSeed(curi, wref, redirectsNewSeeds));
-            if (isInScope(caURI)) {
-                curi.getOutCandidates().add(caURI);
-            }
-        } catch (URIException e) {
-            loggerModule.logUriError(e, curi.getUURI(), 
-                    wref.getDestination().toString());
-        }
+        for (CrawlURI caURI: curi.getOutLinks()) {
+            int directive = getSchedulingFor(curi, caURI, preferenceDepthHops);
+            caURI.setSchedulingDirective(directive );
+            caURI.setSeed(considerAsSeed(curi, caURI, redirectsNewSeeds));
+        } 
         curi.getOutLinks().clear();
     }
     
@@ -197,10 +188,10 @@ public class LinksScoper extends Scoper {
         }
     }
     
-    private boolean considerAsSeed(final CrawlURI curi, final Link wref,
+    private boolean considerAsSeed(final CrawlURI curi, final CrawlURI link,
             final boolean redirectsNewSeeds) {
         return redirectsNewSeeds && curi.isSeed()
-                && wref.getHopType() == Hop.REFER;
+                && link.getLastHopType() == Hop.REFER;
     }
     
     /**
@@ -210,14 +201,14 @@ public class LinksScoper extends Scoper {
      * Imports into the frontier similarly do not pass through here, 
      * but are given NORMAL priority.
      */
-    protected int getSchedulingFor(final CrawlURI curi, final Link wref,
+    protected int getSchedulingFor(final CrawlURI curi, final CrawlURI link,
             final int preferenceDepthHops) {
-        final Hop c = wref.getHopType();
+        final Hop c = link.getLastHopType();
         if (LOGGER.isLoggable(Level.FINEST)) {
             LOGGER.finest(curi + " with path=" + curi.getPathFromSeed() +
                 " isSeed=" + curi.isSeed() + " with fetchStatus=" +
-                curi.getFetchStatus() + " -> " + wref.getDestination() +
-                " type " + c + " with context=" + wref.getContext());
+                curi.getFetchStatus() + " -> " + link.getURI() +
+                " type " + c + " with context=" + link.getViaContext());
         }
 
         switch (c) {

engine/src/main/java/org/archive/crawler/postprocessor/SupplementaryLinksScoper.java

@@ -87,7 +87,7 @@ public class SupplementaryLinksScoper extends Scoper {
         }
         
 //        Collection<CrawlURI> inScopeLinks = new HashSet<CrawlURI>();
-        Iterator<CrawlURI> iter = curi.getOutCandidates().iterator();
+        Iterator<CrawlURI> iter = curi.getOutLinks().iterator();
         while (iter.hasNext()) {
             CrawlURI cauri = iter.next();
             if (!isInScope(cauri)) {

engine/src/main/java/org/archive/crawler/processor/CrawlMapper.java

@@ -215,7 +215,7 @@ public abstract class CrawlMapper extends Processor implements Lifecycle {
         
         if (getCheckOutlinks()) {
             // consider outlinks for mapping
-            Iterator<CrawlURI> iter = curi.getOutCandidates().iterator(); 
+            Iterator<CrawlURI> iter = curi.getOutLinks().iterator(); 
             while(iter.hasNext()) {
                 CrawlURI cauri = iter.next();
                 if (decideToMapOutlink(cauri)) {

engine/src/main/java/org/archive/crawler/reporting/SeedRecord.java

@@ -110,7 +110,7 @@ public class SeedRecord implements CoreAttributeConstants, Serializable, Identit
         this.statusCode = curi.getFetchStatus();
         this.disposition = disposition;
         if (statusCode==301 || statusCode == 302) {
-            for (CrawlURI cauri: curi.getOutCandidates()) {
+            for (CrawlURI cauri: curi.getOutLinks()) {
                 if("location:".equalsIgnoreCase(cauri.getViaContext().
                         toString())) {
                     redirectUri = cauri.toString();