From acedca17886ccdb965989adc1444319dd50e79bc Mon Sep 17 00:00:00 2001
From: nlevitt <nlevitt@daa5b2f2-a927-0410-8b2d-f5f262fa301a>
Date: Tue, 26 Oct 2010 23:49:42 +0000
Subject: [PATCH] Part of [HER-1836] let more than one extractor run on a
 single url - new setting "independentExtractors" - when enabled, extractors
 run regardless of whether other extractors have run *
 profile-crawler-beans.cxml, AbstractFrontier.java, ExtractorParameters.java, 
  Extractor.java     new setting independentExtractors * ContentExtractor.java
     shouldProcess() - respect independentExtractors

---
 .../profile-crawler-beans.cxml                |  1 +
 .../crawler/frontier/AbstractFrontier.java    |  9 +++++
 .../modules/extractor/ContentExtractor.java   | 35 ++++++++++++-------
 .../archive/modules/extractor/Extractor.java  |  3 ++
 .../extractor/ExtractorParameters.java        |  1 +
 5 files changed, 36 insertions(+), 13 deletions(-)

diff --git a/dist/src/main/conf/jobs/profile-defaults/profile-crawler-beans.cxml b/dist/src/main/conf/jobs/profile-defaults/profile-crawler-beans.cxml
index 18fec7b1..71c0872a 100644
--- a/dist/src/main/conf/jobs/profile-defaults/profile-crawler-beans.cxml
+++ b/dist/src/main/conf/jobs/profile-defaults/profile-crawler-beans.cxml
@@ -373,6 +373,7 @@ http://example.example/example
   <!-- <property name="maxRetries" value="30" /> -->
   <!-- <property name="recoveryLogEnabled" value="true" /> -->
   <!-- <property name="maxOutlinks" value="6000" /> -->
+  <!-- <property name="independentExtractors" value="false" /> -->
   <!-- <property name="outbound">
         <bean class="java.util.concurrent.ArrayBlockingQueue">
          <constructor-arg value="200"/>
diff --git a/engine/src/main/java/org/archive/crawler/frontier/AbstractFrontier.java b/engine/src/main/java/org/archive/crawler/frontier/AbstractFrontier.java
index 705eed9e..bf8d3a19 100644
--- a/engine/src/main/java/org/archive/crawler/frontier/AbstractFrontier.java
+++ b/engine/src/main/java/org/archive/crawler/frontier/AbstractFrontier.java
@@ -145,6 +145,15 @@ public abstract class AbstractFrontier
         kp.put("maxOutlinks", max);
     }
     
+    {
+        setIndependentExtractors(false);
+    }
+    public boolean isIndependentExtractors() {
+        return (Boolean) kp.get("independentExtractors");
+    }
+    public void setIndependentExtractors(boolean enabled) {
+        kp.put("independentExtractors", enabled);
+    }
     
     /** inbound updates: URIs to be scheduled, finished; requested state changes */
     protected BlockingQueue<InEvent> inbound;
diff --git a/modules/src/main/java/org/archive/modules/extractor/ContentExtractor.java b/modules/src/main/java/org/archive/modules/extractor/ContentExtractor.java
index f5d5effc..811a3bac 100644
--- a/modules/src/main/java/org/archive/modules/extractor/ContentExtractor.java
+++ b/modules/src/main/java/org/archive/modules/extractor/ContentExtractor.java
@@ -39,24 +39,33 @@ public abstract class ContentExtractor extends Extractor {
         }
     }
 
-
     /**
-     * Determines if links should be extracted from the given URI.  This 
-     * method performs two checks. 
+     * Determines if links should be extracted from the given URI. This method
+     * performs three checks. The first check runs only if
+     * {@link ExtractorParameters#isIndependentExtractors()} is false. It checks
+     * {@link ExtractorURI#hasBeenLinkExtracted()} result. If that result is
+     * true, then this method returns false, as some other extractor has claimed
+     * that links are already extracted.
      * 
-     * <p>First, this method checks that the content length of the URI is
-     * greater than zero (in other words, that there is actually content
-     * for links to be extracted from).  If the content length of the URI
-     * is zero or less, then this method returns false.
+     * <p>
+     * Next, this method checks that the content length of the URI is greater
+     * than zero (in other words, that there is actually content for links to be
+     * extracted from). If the content length of the URI is zero or less, then
+     * this method returns false.
      * 
-     * <p>Finally, this method delegates to {@link #shouldExtract(CrawlURI))}
-     * and returns that result.
-     *
-     * @param uri   the URI to check
-     * @return   true if links should be extracted from the URI, 
-     *   false otherwise
+     * <p>
+     * Finally, this method delegates to {@link #innerExtract(ExtractorURI)} and
+     * returns that result.
+     * 
+     * @param uri
+     *            the URI to check
+     * @return true if links should be extracted from the URI, false otherwise
      */
     final protected boolean shouldProcess(CrawlURI uri) {
+        if (!getExtractorParameters().isIndependentExtractors()
+                && uri.hasBeenLinkExtracted()) {
+            return false;
+        }
         if (uri.getContentLength() <= 0) {
             return false;
         }
diff --git a/modules/src/main/java/org/archive/modules/extractor/Extractor.java b/modules/src/main/java/org/archive/modules/extractor/Extractor.java
index 2a7051e0..045ae0da 100644
--- a/modules/src/main/java/org/archive/modules/extractor/Extractor.java
+++ b/modules/src/main/java/org/archive/modules/extractor/Extractor.java
@@ -55,6 +55,9 @@ public abstract class Extractor extends Processor {
             public int getMaxOutlinks() {
                 return 6000;
             }
+            public boolean isIndependentExtractors() {
+                return false;
+            }
         };
 
     transient protected UriErrorLoggerModule loggerModule;
diff --git a/modules/src/main/java/org/archive/modules/extractor/ExtractorParameters.java b/modules/src/main/java/org/archive/modules/extractor/ExtractorParameters.java
index 30ea1de7..7c87ce94 100644
--- a/modules/src/main/java/org/archive/modules/extractor/ExtractorParameters.java
+++ b/modules/src/main/java/org/archive/modules/extractor/ExtractorParameters.java
@@ -29,4 +29,5 @@ package org.archive.modules.extractor;
  */
 public interface ExtractorParameters {
     public int getMaxOutlinks();
+    public boolean isIndependentExtractors();
 }