Merge branch 'master' into fetchhttp-hc

Conflicts:
	modules/src/main/java/org/archive/modules/fetcher/FetchHTTP.java

(conflicts are resolved)

modules/src/main/java/org/archive/modules/ScriptedProcessor.java

@@ -59,6 +59,7 @@ import org.springframework.context.ApplicationContextAware;
 public class ScriptedProcessor extends Processor 
 implements ApplicationContextAware, InitializingBean {
 
+    @SuppressWarnings("unused")
     private static final long serialVersionUID = 3L;
 
     private static final Logger logger =

modules/src/main/java/org/archive/modules/deciderules/TooManyPathSegmentsDecideRule.java

@@ -18,6 +18,8 @@
  */
 package org.archive.modules.deciderules;
 
+import java.util.logging.Logger;
+
 import org.archive.modules.CrawlURI;
 
 /**
@@ -30,6 +32,8 @@ import org.archive.modules.CrawlURI;
 public class TooManyPathSegmentsDecideRule extends PredicatedDecideRule {
 
     private static final long serialVersionUID = 3L;
+    @SuppressWarnings("unused")
+    private static final Logger logger = Logger.getLogger(TooManyPathSegmentsDecideRule.class.getName()); 
 
     /** default for this class is to REJECT */
     {
@@ -63,11 +67,15 @@ public class TooManyPathSegmentsDecideRule extends PredicatedDecideRule {
      */
     @Override
     protected boolean evaluate(CrawlURI curi) {
-        String uri = curi.toString();
+        String uriPath = curi.getUURI().getEscapedPath();
+        if (uriPath == null) {
+            // no path means no segments
+            return false;
+        }
         int count = 0;
         int threshold = getMaxPathDepth();
-        for (int i = 0; i < uri.length(); i++) {
-            if (uri.charAt(i) == '/') {
+        for (int i = 0; i < uriPath.length(); i++) {
+            if (uriPath.charAt(i) == '/') {
                 count++;
             }
             if (count > threshold) {

modules/src/main/java/org/archive/modules/extractor/AggressiveExtractorHTML.java

@@ -32,6 +32,7 @@ import org.archive.modules.CrawlURI;
  */
 public class AggressiveExtractorHTML
 extends ExtractorHTML {
+    @SuppressWarnings("unused")
     private static final long serialVersionUID = 3L;
 
     protected static Logger logger =

modules/src/main/java/org/archive/modules/extractor/ExtractorCSS.java

@@ -57,6 +57,7 @@ import org.archive.util.TextUtils;
 public class ExtractorCSS extends ContentExtractor {
 
 
+    @SuppressWarnings("unused")
     private static final long serialVersionUID = 2L;
 
     private static Logger logger =

modules/src/main/java/org/archive/modules/extractor/ExtractorDOC.java

@@ -42,6 +42,7 @@ import org.archive.util.ms.Doc;
  */
 public class ExtractorDOC extends ContentExtractor {
 
+    @SuppressWarnings("unused")
     private static final long serialVersionUID = 3L;
     
     private static Pattern PATTERN = Pattern.compile("HYPERLINK.*?\"(.*?)\"");

modules/src/main/java/org/archive/modules/extractor/ExtractorHTML.java

@@ -61,6 +61,7 @@ import org.springframework.beans.factory.annotation.Autowired;
  */
 public class ExtractorHTML extends ContentExtractor implements InitializingBean {
 
+    @SuppressWarnings("unused")
     private static final long serialVersionUID = 2L;
 
     private static Logger logger =

modules/src/main/java/org/archive/modules/extractor/ExtractorHTTP.java

@@ -32,6 +32,7 @@ import org.archive.net.UURIFactory;
  */
 public class ExtractorHTTP extends Extractor {
 
+    @SuppressWarnings("unused")
     private static final long serialVersionUID = 3L;
 
     public ExtractorHTTP() {

modules/src/main/java/org/archive/modules/extractor/ExtractorImpliedURI.java

@@ -51,6 +51,7 @@ import org.archive.net.UURIFactory;
 
 public class ExtractorImpliedURI extends Extractor {
 
+    @SuppressWarnings("unused")
     private static final long serialVersionUID = 3L;
 
     private static Logger LOGGER =

modules/src/main/java/org/archive/modules/extractor/ExtractorJS.java

@@ -56,6 +56,7 @@ import org.archive.util.UriUtils;
  */
 public class ExtractorJS extends ContentExtractor {
 
+    @SuppressWarnings("unused")
     private static final long serialVersionUID = 2L;
 
     private static Logger LOGGER =

modules/src/main/java/org/archive/modules/extractor/ExtractorPDF.java

@@ -38,6 +38,7 @@ import org.archive.util.FileUtils;
  */
 public class ExtractorPDF extends ContentExtractor {
 
+    @SuppressWarnings("unused")
     private static final long serialVersionUID = 3L;
 
     private static final Logger LOGGER =

modules/src/main/java/org/archive/modules/extractor/ExtractorSWF.java

@@ -46,6 +46,7 @@ import com.anotherbigidea.io.InStream;
  */
 public class ExtractorSWF extends ContentExtractor {
 
+    @SuppressWarnings("unused")
     private static final long serialVersionUID = 3L;
 
     private static Logger logger =

modules/src/main/java/org/archive/modules/extractor/ExtractorURI.java

@@ -48,6 +48,7 @@ import org.archive.util.TextUtils;
 
 public class ExtractorURI extends Extractor {
 
+    @SuppressWarnings("unused")
     private static final long serialVersionUID = 3L;
 
     private static Logger LOGGER =

modules/src/main/java/org/archive/modules/extractor/ExtractorUniversal.java

@@ -49,6 +49,7 @@ import org.archive.net.UURIFactory;
  */
 public class ExtractorUniversal extends ContentExtractor {
 
+    @SuppressWarnings("unused")
     private static final long serialVersionUID = 3L;
 
     /**

modules/src/main/java/org/archive/modules/extractor/ExtractorXML.java

@@ -49,6 +49,7 @@ import org.archive.util.UriUtils;
  */
 public class ExtractorXML extends ContentExtractor {
 
+    @SuppressWarnings("unused")
     private static final long serialVersionUID = 3L;
 
     private static Logger logger =

modules/src/main/java/org/archive/modules/extractor/HTTPContentDigest.java

@@ -65,6 +65,7 @@ import org.archive.util.TextUtils;
  */
 public class HTTPContentDigest extends Processor {
 
+    @SuppressWarnings("unused")
     private static final long serialVersionUID = 3L;
 
     private static Logger logger =

modules/src/main/java/org/archive/modules/extractor/JerichoExtractorHTML.java

@@ -76,6 +76,7 @@ import au.id.jericho.lib.html.StartTagType;
 @SuppressWarnings("unchecked")
 public class JerichoExtractorHTML extends ExtractorHTML {
 
+    @SuppressWarnings("unused")
     private static final long serialVersionUID = 1684681316546343615L;
 
     final private static Logger logger = 

modules/src/main/java/org/archive/modules/extractor/TrapSuppressExtractor.java

@@ -28,6 +28,7 @@ import org.archive.modules.CrawlURI;
  *
  */
 public class TrapSuppressExtractor extends ContentExtractor  {
+    @SuppressWarnings("unused")
     private static final long serialVersionUID = -1028783453022579530L;
 
     /** ALIst attribute key for carrying-forward content-digest from 'via'*/

modules/src/main/java/org/archive/modules/fetcher/BdbCookieStorage.java

@@ -43,6 +43,7 @@ import com.sleepycat.je.DatabaseException;
  * @author pjack
  */
 public class BdbCookieStorage extends AbstractCookieStorage implements Checkpointable {
+    @SuppressWarnings("unused")
     private static final long serialVersionUID = 1L;
     
     protected BdbModule bdb;

modules/src/main/java/org/archive/modules/fetcher/FetchDNS.java

@@ -61,6 +61,7 @@ import org.xbill.DNS.Type;
  */
 public class FetchDNS extends Processor {
 
+    @SuppressWarnings("unused")
     private static final long serialVersionUID = 3L;
 
     private static Logger logger = Logger.getLogger(FetchDNS.class.getName());

modules/src/main/java/org/archive/modules/fetcher/FetchFTP.java

@@ -62,6 +62,7 @@ import org.archive.util.Recorder;
  *
  */
 public class FetchFTP extends Processor  {
+    @SuppressWarnings("unused")
     private static final long serialVersionUID = 1L;
 
     /** Logger for this class. */

modules/src/main/java/org/archive/modules/fetcher/FetchHTTP.java

@@ -122,6 +122,7 @@ import org.springframework.context.Lifecycle;
  * @version $Id$
  */
 public class FetchHTTP extends AbstractFetchHTTP implements Lifecycle {
+    @SuppressWarnings("unused")
     private static final long serialVersionUID = 1L;
     private static Logger logger = Logger.getLogger(FetchHTTP.class.getName());
 

modules/src/main/java/org/archive/modules/fetcher/FetchWhois.java

@@ -122,6 +122,7 @@ import com.sleepycat.je.DatabaseException;
 public class FetchWhois extends Processor implements CoreAttributeConstants,
         FetchStatusCodes, Lifecycle {
 
+    @SuppressWarnings("unused")
     private static final long serialVersionUID = 1L;
 
     private static Logger logger = Logger.getLogger(FetchWhois.class.getName());

modules/src/main/java/org/archive/modules/fetcher/SimpleCookieStorage.java

@@ -26,6 +26,7 @@ import org.apache.commons.httpclient.Cookie;
 
 public class SimpleCookieStorage extends AbstractCookieStorage {
 
+    @SuppressWarnings("unused")
     private static final long serialVersionUID = 1L;
 
     final private SortedMap<String,Cookie> map = new TreeMap<String,Cookie>();

modules/src/main/java/org/archive/modules/recrawl/FetchHistoryProcessor.java

@@ -40,6 +40,7 @@ import org.archive.modules.Processor;
  * @version $Date: 2006-09-25 20:19:54 +0000 (Mon, 25 Sep 2006) $, $Revision: 4654 $
  */
 public class FetchHistoryProcessor extends Processor {
+    @SuppressWarnings("unused")
     private static final long serialVersionUID = 1L;
     
     /** Desired history array length. */

modules/src/main/java/org/archive/modules/recrawl/PersistLoadProcessor.java

@@ -39,6 +39,7 @@ import com.sleepycat.je.DatabaseException;
  * @version $Date: 2006-09-25 20:19:54 +0000 (Mon, 25 Sep 2006) $, $Revision: 4654 $
  */
 public class PersistLoadProcessor extends PersistOnlineProcessor {
+    @SuppressWarnings("unused")
     private static final long serialVersionUID = -1917169316015093131L;
     private static final Logger logger =
         Logger.getLogger(PersistLoadProcessor.class.getName());

modules/src/main/java/org/archive/modules/recrawl/PersistLogProcessor.java

@@ -43,6 +43,7 @@ import org.springframework.context.Lifecycle;
 public class PersistLogProcessor extends PersistProcessor 
 implements Checkpointable, Lifecycle {
 
+    @SuppressWarnings("unused")
     private static final long serialVersionUID = 1678691994065439346L;
     
     protected CrawlerJournal log;

modules/src/main/java/org/archive/modules/recrawl/PersistOnlineProcessor.java

@@ -40,6 +40,7 @@ import com.sleepycat.je.DatabaseException;
 public abstract class PersistOnlineProcessor extends PersistProcessor 
 implements Lifecycle {
     
+    @SuppressWarnings("unused")
     private static final long serialVersionUID = -666479480942267268L;
     
     protected BdbModule bdb;

modules/src/main/java/org/archive/modules/recrawl/PersistProcessor.java

@@ -64,6 +64,7 @@ import com.sleepycat.je.EnvironmentConfig;
  */
 public abstract class PersistProcessor extends AbstractPersistProcessor {
 
+    @SuppressWarnings("unused")
     private static final long serialVersionUID = 1L;
 
     private static final Logger logger =

modules/src/main/java/org/archive/modules/recrawl/PersistStoreProcessor.java

@@ -29,6 +29,7 @@ import org.archive.modules.CrawlURI;
  */ 
 public class PersistStoreProcessor extends PersistOnlineProcessor 
 {
+    @SuppressWarnings("unused")
     private static final long serialVersionUID = -8308356194337303758L;
 
 //    class description: "PersistStoreProcessor. Stores CrawlURI attributes " +

modules/src/main/java/org/archive/modules/writer/ARCWriterProcessor.java

@@ -61,6 +61,7 @@ public class ARCWriterProcessor extends WriterPoolProcessor {
 
     final static private String METADATA_TEMPLATE = readMetadataTemplate();
     
+    @SuppressWarnings("unused")
     private static final long serialVersionUID = 3L;
 
     private static final Logger logger = 

modules/src/main/java/org/archive/modules/writer/Kw3WriterProcessor.java

@@ -78,6 +78,7 @@ import org.springframework.beans.factory.annotation.Autowired;
 public class Kw3WriterProcessor extends Processor {
 
 
+  @SuppressWarnings("unused")
   private static final long serialVersionUID = 3L;
   
   private static String COLON = ":";

modules/src/main/java/org/archive/modules/writer/MirrorWriterProcessor.java

@@ -86,6 +86,7 @@ import org.archive.util.FileUtils;
    @author Howard Lee Gayle
 */
 public class MirrorWriterProcessor extends Processor {
+    @SuppressWarnings("unused")
     private static final long serialVersionUID = 3L;
     private static final Logger logger =
         Logger.getLogger(MirrorWriterProcessor.class.getName());

modules/src/main/java/org/archive/modules/writer/WARCWriterProcessor.java

@@ -99,6 +99,7 @@ import org.archive.util.anvl.ANVLRecord;
  * @contributor stack
  */
 public class WARCWriterProcessor extends WriterPoolProcessor implements WARCWriterPoolSettings {
+    @SuppressWarnings("unused")
     private static final long serialVersionUID = 6182850087635847443L;
     private static final Logger logger = 
         Logger.getLogger(WARCWriterProcessor.class.getName());

modules/src/main/java/org/archive/modules/writer/WriterPoolProcessor.java

@@ -61,6 +61,7 @@ import org.springframework.context.Lifecycle;
  */
 public abstract class WriterPoolProcessor extends Processor 
 implements Lifecycle, Checkpointable, WriterPoolSettings {
+    @SuppressWarnings("unused")
     private static final long serialVersionUID = 1L;
     private static final Logger logger = 
         Logger.getLogger(WriterPoolProcessor.class.getName());

modules/src/test/java/org/archive/modules/extractor/UnitTestUriLoggerModule.java

@@ -14,6 +14,7 @@ import org.archive.net.UURI;
  *
  */
 public class UnitTestUriLoggerModule implements UriErrorLoggerModule {
+    @SuppressWarnings("unused")
     private static final long serialVersionUID = 1L;
     
     final private static Logger LOGGER =