Part of [HER-1836] let more than one extractor run on a single url - new

setting "independentExtractors" - when enabled, extractors run regardless of
whether other extractors have run
* profile-crawler-beans.cxml, AbstractFrontier.java, ExtractorParameters.java,
  Extractor.java
    new setting independentExtractors
* ContentExtractor.java
    shouldProcess() - respect independentExtractors

dist/src/main/conf/jobs/profile-defaults/profile-crawler-beans.cxml

@@ -373,6 +373,7 @@ http://example.example/example
   <!-- <property name="maxRetries" value="30" /> -->
   <!-- <property name="recoveryLogEnabled" value="true" /> -->
   <!-- <property name="maxOutlinks" value="6000" /> -->
+  <!-- <property name="independentExtractors" value="false" /> -->
   <!-- <property name="outbound">
         <bean class="java.util.concurrent.ArrayBlockingQueue">
          <constructor-arg value="200"/>

engine/src/main/java/org/archive/crawler/frontier/AbstractFrontier.java

@@ -145,6 +145,15 @@ public abstract class AbstractFrontier
         kp.put("maxOutlinks", max);
     }
     
+    {
+        setIndependentExtractors(false);
+    }
+    public boolean isIndependentExtractors() {
+        return (Boolean) kp.get("independentExtractors");
+    }
+    public void setIndependentExtractors(boolean enabled) {
+        kp.put("independentExtractors", enabled);
+    }
     
     /** inbound updates: URIs to be scheduled, finished; requested state changes */
     protected BlockingQueue<InEvent> inbound;

modules/src/main/java/org/archive/modules/extractor/ContentExtractor.java

@@ -39,24 +39,33 @@ public abstract class ContentExtractor extends Extractor {
         }
     }
 
-
     /**
-     * Determines if links should be extracted from the given URI.  This 
-     * method performs two checks. 
+     * Determines if links should be extracted from the given URI. This method
+     * performs three checks. The first check runs only if
+     * {@link ExtractorParameters#isIndependentExtractors()} is false. It checks
+     * {@link ExtractorURI#hasBeenLinkExtracted()} result. If that result is
+     * true, then this method returns false, as some other extractor has claimed
+     * that links are already extracted.
      * 
-     * <p>First, this method checks that the content length of the URI is
-     * greater than zero (in other words, that there is actually content
-     * for links to be extracted from).  If the content length of the URI
-     * is zero or less, then this method returns false.
+     * <p>
+     * Next, this method checks that the content length of the URI is greater
+     * than zero (in other words, that there is actually content for links to be
+     * extracted from). If the content length of the URI is zero or less, then
+     * this method returns false.
      * 
-     * <p>Finally, this method delegates to {@link #shouldExtract(CrawlURI))}
-     * and returns that result.
-     *
-     * @param uri   the URI to check
-     * @return   true if links should be extracted from the URI, 
-     *   false otherwise
+     * <p>
+     * Finally, this method delegates to {@link #innerExtract(ExtractorURI)} and
+     * returns that result.
+     * 
+     * @param uri
+     *            the URI to check
+     * @return true if links should be extracted from the URI, false otherwise
      */
     final protected boolean shouldProcess(CrawlURI uri) {
+        if (!getExtractorParameters().isIndependentExtractors()
+                && uri.hasBeenLinkExtracted()) {
+            return false;
+        }
         if (uri.getContentLength() <= 0) {
             return false;
         }

modules/src/main/java/org/archive/modules/extractor/Extractor.java

@@ -55,6 +55,9 @@ public abstract class Extractor extends Processor {
             public int getMaxOutlinks() {
                 return 6000;
             }
+            public boolean isIndependentExtractors() {
+                return false;
+            }
         };
 
     transient protected UriErrorLoggerModule loggerModule;

modules/src/main/java/org/archive/modules/extractor/ExtractorParameters.java

@@ -29,4 +29,5 @@ package org.archive.modules.extractor;
  */
 public interface ExtractorParameters {
     public int getMaxOutlinks();
+    public boolean isIndependentExtractors();
 }