More ProcessorURI -> CrawlURI updates

engine/src/main/java/org/archive/crawler/frontier/precedence/HighestUriQueuePrecedencePolicy.java

@@ -28,7 +28,6 @@ import java.io.PrintWriter;
 
 import org.archive.crawler.frontier.WorkQueue;
 import org.archive.modules.CrawlURI;
-import org.archive.modules.ProcessorURI;
 import org.archive.modules.fetcher.FetchStats.Stage;
 import org.archive.util.Histotable;
 
@@ -66,11 +65,10 @@ public class HighestUriQueuePrecedencePolicy extends BaseQueuePrecedencePolicy {
         }
         
         /* (non-Javadoc)
-         * @see org.archive.crawler.frontier.precedence.PrecedenceProvider#tally(org.archive.modules.ProcessorURI, org.archive.modules.fetcher.FetchStats.Stage)
+         * @see org.archive.crawler.frontier.precedence.PrecedenceProvider#tally(org.archive.modules.CrawlURI, org.archive.modules.fetcher.FetchStats.Stage)
          */
         @Override
-        public void tally(ProcessorURI puri, Stage stage) {
-            CrawlURI curi = (CrawlURI)puri; 
+        public void tally(CrawlURI curi, Stage stage) {
             switch(stage) {
             case SCHEDULED:
                 // enqueued

engine/src/main/java/org/archive/crawler/frontier/precedence/PrecedenceProvider.java

@@ -27,7 +27,7 @@ package org.archive.crawler.frontier.precedence;
 import java.io.PrintWriter;
 import java.io.Serializable;
 
-import org.archive.modules.ProcessorURI;
+import org.archive.modules.CrawlURI;
 import org.archive.modules.fetcher.FetchStats;
 import org.archive.modules.fetcher.FetchStats.Stage;
 import org.archive.util.ArchiveUtils;
@@ -43,9 +43,9 @@ FetchStats.CollectsFetchStats, Serializable {
     abstract public int getPrecedence();
 
     /* (non-Javadoc)
-     * @see org.archive.modules.fetcher.FetchStats.CollectsFetchStats#tally(org.archive.modules.ProcessorURI, org.archive.modules.fetcher.FetchStats.Stage)
+     * @see org.archive.modules.fetcher.FetchStats.CollectsFetchStats#tally(org.archive.modules.CrawlURI, org.archive.modules.fetcher.FetchStats.Stage)
      */
-    public void tally(ProcessorURI curi, Stage stage) {
+    public void tally(CrawlURI curi, Stage stage) {
         // by default do nothing; subclasses do more
     }
 

engine/src/main/java/org/archive/crawler/postprocessor/LowDiskPauseProcessor.java

@@ -34,10 +34,11 @@ import java.util.regex.Matcher;
 import java.util.regex.Pattern;
 
 import org.apache.commons.io.IOUtils;
-import org.archive.modules.PostProcessor;
+import org.archive.crawler.framework.CrawlController;
 import org.archive.modules.ProcessResult;
 import org.archive.modules.Processor;
-import org.archive.modules.ProcessorURI;
+import org.archive.modules.CrawlURI;
+import org.springframework.beans.factory.annotation.Autowired;
 
 /**
  * Processor module which uses 'df -k', where available and with
@@ -45,7 +46,7 @@ import org.archive.modules.ProcessorURI;
  * disk space and pause the crawl if free space on  monitored 
  * filesystems falls below certain thresholds.
  */
-public class LowDiskPauseProcessor extends Processor implements PostProcessor {
+public class LowDiskPauseProcessor extends Processor {
 
     private static final long serialVersionUID = 3L;
 
@@ -56,6 +57,15 @@ public class LowDiskPauseProcessor extends Processor implements PostProcessor {
         Logger.getLogger(LowDiskPauseProcessor.class.getName());
 
 
+    protected CrawlController controller;
+    public CrawlController getCrawlController() {
+        return this.controller;
+    }
+    @Autowired
+    public void setCrawlController(CrawlController controller) {
+        this.controller = controller;
+    }
+    
     /**
      * List of filessystem mounts whose 'available' space should be monitored
      * via 'df' (if available).
@@ -107,12 +117,12 @@ public class LowDiskPauseProcessor extends Processor implements PostProcessor {
     
     
     @Override
-    protected boolean shouldProcess(ProcessorURI curi) {
+    protected boolean shouldProcess(CrawlURI curi) {
         return true;
     }
 
     @Override
-    protected void innerProcess(ProcessorURI uri) {
+    protected void innerProcess(CrawlURI uri) {
         throw new AssertionError();
     }
     
@@ -125,7 +135,7 @@ public class LowDiskPauseProcessor extends Processor implements PostProcessor {
      * @param curi CrawlURI to process.
      */
     @Override
-    protected ProcessResult innerProcessResult(ProcessorURI curi) {
+    protected ProcessResult innerProcessResult(CrawlURI curi) {
         synchronized (this) {
             contentSinceCheck += curi.getContentSize();
             if (contentSinceCheck/1024 > getRecheckThresholdKb()) {
@@ -145,7 +155,7 @@ public class LowDiskPauseProcessor extends Processor implements PostProcessor {
      * crawl pause. 
      * @param curi Current context.
      */
-    private ProcessResult checkAvailableSpace(ProcessorURI curi) {
+    private ProcessResult checkAvailableSpace(CrawlURI curi) {
         try {
             String df = IOUtils.toString(Runtime.getRuntime().exec(
                     "df -k").getInputStream());
@@ -166,7 +176,8 @@ public class LowDiskPauseProcessor extends Processor implements PostProcessor {
                                 availKilobytes + "K available on " + mount
                                         + " (below threshold "
                                         + thresholdKilobytes + "K)");
-                        return ProcessResult.STUCK;
+                        controller.requestCrawlPause();
+                        return ProcessResult.PROCEED;
                     }
                 }
             }

engine/src/main/java/org/archive/crawler/postprocessor/SupplementaryLinksScoper.java

@@ -1,28 +1,22 @@
-/* SupplementaryLinksScoper
- * 
- * $Id$
+/*
+ *  This file is part of the Heritrix web crawler (crawler.archive.org).
  *
- * Created on Oct 2, 2003
- * 
- * Copyright (C) 2003 Internet Archive.
+ *  Licensed to the Internet Archive (IA) by one or more individual 
+ *  contributors. 
  *
- * This file is part of the Heritrix web crawler (crawler.archive.org).
+ *  The IA licenses this file to You under the Apache License, Version 2.0
+ *  (the "License"); you may not use this file except in compliance with
+ *  the License.  You may obtain a copy of the License at
  *
- * Heritrix is free software; you can redistribute it and/or modify
- * it under the terms of the GNU Lesser Public License as published by
- * the Free Software Foundation; either version 2.1 of the License, or
- * any later version.
- *
- * Heritrix is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU Lesser Public License for more details.
- *
- * You should have received a copy of the GNU Lesser Public License
- * along with Heritrix; if not, write to the Free Software
- * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+ *      http://www.apache.org/licenses/LICENSE-2.0
  *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
  */
+
 package org.archive.crawler.postprocessor;
 
 import java.util.Iterator;
@@ -31,8 +25,6 @@ import java.util.logging.Logger;
 
 import org.archive.crawler.framework.Scoper;
 import org.archive.modules.CrawlURI;
-import org.archive.modules.PostProcessor;
-import org.archive.modules.ProcessorURI;
 import org.archive.modules.deciderules.AcceptDecideRule;
 import org.archive.modules.deciderules.DecideResult;
 import org.archive.modules.deciderules.DecideRule;
@@ -49,7 +41,7 @@ import org.archive.modules.deciderules.DecideRule;
  * to handle rejected URLs otherwise.
  * @author stack
  */
-public class SupplementaryLinksScoper extends Scoper implements PostProcessor {
+public class SupplementaryLinksScoper extends Scoper {
 
     private static final long serialVersionUID = -3L;
 
@@ -80,12 +72,12 @@ public class SupplementaryLinksScoper extends Scoper implements PostProcessor {
     }
 
     
-    protected boolean shouldProcess(ProcessorURI puri) {
+    protected boolean shouldProcess(CrawlURI puri) {
         return puri instanceof CrawlURI;
     }
     
     
-    protected void innerProcess(final ProcessorURI puri) {
+    protected void innerProcess(final CrawlURI puri) {
         CrawlURI curi = (CrawlURI)puri;
         
         // If prerequisites or no links, nothing to be done in here.

engine/src/main/java/org/archive/crawler/prefetch/Preselector.java

@@ -1,26 +1,22 @@
-/* Copyright (C) 2003 Internet Archive.
+/*
+ *  This file is part of the Heritrix web crawler (crawler.archive.org).
  *
- * This file is part of the Heritrix web crawler (crawler.archive.org).
+ *  Licensed to the Internet Archive (IA) by one or more individual 
+ *  contributors. 
  *
- * Heritrix is free software; you can redistribute it and/or modify
- * it under the terms of the GNU Lesser Public License as published by
- * the Free Software Foundation; either version 2.1 of the License, or
- * any later version.
+ *  The IA licenses this file to You under the Apache License, Version 2.0
+ *  (the "License"); you may not use this file except in compliance with
+ *  the License.  You may obtain a copy of the License at
  *
- * Heritrix is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU Lesser Public License for more details.
+ *      http://www.apache.org/licenses/LICENSE-2.0
  *
- * You should have received a copy of the GNU Lesser Public License
- * along with Heritrix; if not, write to the Free Software
- * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
- *
- * SimplePreselector.java
- * Created on Sep 22, 2003
- *
- * $Header$
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
  */
+
 package org.archive.crawler.prefetch;
 
 
@@ -30,7 +26,6 @@ import static org.archive.modules.fetcher.FetchStatusCodes.*;
 import org.archive.crawler.framework.Scoper;
 import org.archive.modules.CrawlURI;
 import org.archive.modules.ProcessResult;
-import org.archive.modules.ProcessorURI;
 import org.archive.util.TextUtils;
 
 
@@ -112,19 +107,19 @@ public class Preselector extends Scoper {
     }
 
     @Override
-    protected boolean shouldProcess(ProcessorURI puri) {
+    protected boolean shouldProcess(CrawlURI puri) {
         return puri instanceof CrawlURI;
     }
 
     
     @Override
-    protected void innerProcess(ProcessorURI puri) {
+    protected void innerProcess(CrawlURI puri) {
         throw new AssertionError();
     }
     
 
     @Override
-    protected ProcessResult innerProcessResult(ProcessorURI puri) {
+    protected ProcessResult innerProcessResult(CrawlURI puri) {
         CrawlURI curi = (CrawlURI)puri;
         
         // Check if uris should be blocked