Work for [HER-1656] add multiple-queues-per-host ('parallelQueues') capabilities to QueueAssignmentPolicies

* LongToIntConsistentHash.java + Test
    consistent-hashing utility class
* URIAuthorityBasedQueueAssignmentPolicy.java
    shared superclass for the hostname and surtauthority QAPs
    setting for 'deferToPrevious' -- avoid changing assignments
    setting for 'parallelQueues' -- when > 1, consistent-hash URIs across that many separate queues (with numerical suffix)
* SurtAuthorityQueueAssignmentPolicy.java, HostnameQueueAssignmentPolicy.java
    refactor to derive from URIAuthorityBasedQueueAssignmentPolicy
* QueueAssignmentPolicy.java
    add Apache license
* Hop.java
    add String representation for convenience

engine/src/main/java/org/archive/crawler/frontier/HostnameQueueAssignmentPolicy.java

@@ -1,34 +1,26 @@
-/* HostnameQueueAssignmentPolicy
-*
-* $Id$
-*
-* Created on Oct 5, 2004
-*
-* Copyright (C) 2004 Internet Archive.
-*
-* This file is part of the Heritrix web crawler (crawler.archive.org).
-*
-* Heritrix is free software; you can redistribute it and/or modify
-* it under the terms of the GNU Lesser Public License as published by
-* the Free Software Foundation; either version 2.1 of the License, or
-* any later version.
-*
-* Heritrix is distributed in the hope that it will be useful,
-* but WITHOUT ANY WARRANTY; without even the implied warranty of
-* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-* GNU Lesser Public License for more details.
-*
-* You should have received a copy of the GNU Lesser Public License
-* along with Heritrix; if not, write to the Free Software
-* Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
-*/ 
+/*
+ *  This file is part of the Heritrix web crawler (crawler.archive.org).
+ *
+ *  Licensed to the Internet Archive (IA) by one or more individual 
+ *  contributors. 
+ *
+ *  The IA licenses this file to You under the Apache License, Version 2.0
+ *  (the "License"); you may not use this file except in compliance with
+ *  the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ */
+
 package org.archive.crawler.frontier;
 
-import java.util.logging.Level;
-import java.util.logging.Logger;
-
 import org.apache.commons.httpclient.URIException;
-import org.archive.crawler.datamodel.CrawlURI;
+import org.apache.commons.lang.StringUtils;
 import org.archive.net.UURI;
 import org.archive.net.UURIFactory;
 
@@ -38,51 +30,22 @@ import org.archive.net.UURIFactory;
  * 
  * @author gojomo
  */
-public class HostnameQueueAssignmentPolicy extends QueueAssignmentPolicy {
-
+public class HostnameQueueAssignmentPolicy 
+extends URIAuthorityBasedQueueAssignmentPolicy {
     private static final long serialVersionUID = 3L;
 
-    private static final Logger logger = Logger
-        .getLogger(HostnameQueueAssignmentPolicy.class.getName());
-    /**
-     * When neat host-based class-key fails us
-     */
-    private static String DEFAULT_CLASS_KEY = "default...";
-    
-    private static final String DNS = "dns";
-
-    public String getClassKey(CrawlURI cauri) {
-        String scheme = cauri.getUURI().getScheme();
+    @Override
+    String getCoreKey(UURI basis) {
+        String scheme = basis.getScheme();
         String candidate = null;
         try {
-            if (scheme.equals(DNS)){
-                if (cauri.getVia() != null) {
-                    // Special handling for DNS: treat as being
-                    // of the same class as the triggering URI.
-                    // When a URI includes a port, this ensures 
-                    // the DNS lookup goes atop the host:port
-                    // queue that triggered it, rather than 
-                    // some other host queue
-                	UURI viaUuri = UURIFactory.getInstance(cauri.flattenVia());
-                    candidate = viaUuri.getAuthorityMinusUserinfo();
-                    // adopt scheme of triggering URI
-                    scheme = viaUuri.getScheme();
-                } else {
-                    candidate= cauri.getUURI().getReferencedHost();
-                }
-            } else {
-                candidate =  cauri.getUURI().getAuthorityMinusUserinfo();
-            }
-            
-            if(candidate == null || candidate.length() == 0) {
-                candidate = DEFAULT_CLASS_KEY;
-            }
-        } catch (URIException e) {
-            logger.log(Level.INFO,
-                    "unable to extract class key; using default", e);
-            candidate = DEFAULT_CLASS_KEY;
+            candidate = basis.getAuthorityMinusUserinfo();
+        } catch (URIException ue) {}// let next line handle
+        
+        if(StringUtils.isEmpty(candidate)) {
+            return null; 
         }
-        if (scheme != null && scheme.equals(UURIFactory.HTTPS)) {
+        if (UURIFactory.HTTPS.equals(scheme)) {
             // If https and no port specified, add default https port to
             // distinguish https from http server without a port.
             if (!candidate.matches(".+:[0-9]+")) {
@@ -93,4 +56,5 @@ public class HostnameQueueAssignmentPolicy extends QueueAssignmentPolicy {
         return candidate.replace(':','#');
     }
 
+
 }

engine/src/main/java/org/archive/crawler/frontier/QueueAssignmentPolicy.java

@@ -1,27 +1,22 @@
-/* QueueAssignmentPolicy
-*
-* $Id$
-*
-* Created on Oct 5, 2004
-*
-* Copyright (C) 2004 Internet Archive.
-*
-* This file is part of the Heritrix web crawler (crawler.archive.org).
-*
-* Heritrix is free software; you can redistribute it and/or modify
-* it under the terms of the GNU Lesser Public License as published by
-* the Free Software Foundation; either version 2.1 of the License, or
-* any later version.
-*
-* Heritrix is distributed in the hope that it will be useful,
-* but WITHOUT ANY WARRANTY; without even the implied warranty of
-* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-* GNU Lesser Public License for more details.
-*
-* You should have received a copy of the GNU Lesser Public License
-* along with Heritrix; if not, write to the Free Software
-* Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
-*/ 
+/*
+ *  This file is part of the Heritrix web crawler (crawler.archive.org).
+ *
+ *  Licensed to the Internet Archive (IA) by one or more individual 
+ *  contributors. 
+ *
+ *  The IA licenses this file to You under the Apache License, Version 2.0
+ *  (the "License"); you may not use this file except in compliance with
+ *  the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ */
+
 package org.archive.crawler.frontier;
 
 import java.io.Serializable;

engine/src/main/java/org/archive/crawler/frontier/SurtAuthorityQueueAssignmentPolicy.java

@@ -1,92 +1,39 @@
-/* SurtAuthorityQueueAssignmentPolicy
-*
-* $Id$
-*
-* Created on Oct 5, 2004
-*
-* Copyright (C) 2004 Internet Archive.
-*
-* This file is part of the Heritrix web crawler (crawler.archive.org).
-*
-* Heritrix is free software; you can redistribute it and/or modify
-* it under the terms of the GNU Lesser Public License as published by
-* the Free Software Foundation; either version 2.1 of the License, or
-* any later version.
-*
-* Heritrix is distributed in the hope that it will be useful,
-* but WITHOUT ANY WARRANTY; without even the implied warranty of
-* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-* GNU Lesser Public License for more details.
-*
-* You should have received a copy of the GNU Lesser Public License
-* along with Heritrix; if not, write to the Free Software
-* Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
-*/ 
+/*
+ *  This file is part of the Heritrix web crawler (crawler.archive.org).
+ *
+ *  Licensed to the Internet Archive (IA) by one or more individual 
+ *  contributors. 
+ *
+ *  The IA licenses this file to You under the Apache License, Version 2.0
+ *  (the "License"); you may not use this file except in compliance with
+ *  the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ */
+
 package org.archive.crawler.frontier;
 
-import java.util.logging.Level;
-import java.util.logging.Logger;
-
-import org.apache.commons.httpclient.URIException;
-import org.archive.crawler.datamodel.CrawlURI;
 import org.archive.net.UURI;
-import org.archive.net.UURIFactory;
 
 /**
  * SurtAuthorityQueueAssignmentPolicy based on the surt form of hostname.
  */
 public class SurtAuthorityQueueAssignmentPolicy
-extends QueueAssignmentPolicy {
-
+extends URIAuthorityBasedQueueAssignmentPolicy {
     private static final long serialVersionUID = 3L;
-
-    private static final Logger logger = Logger
-            .getLogger(SurtAuthorityQueueAssignmentPolicy.class.getName());
     
-    /**
-     * When neat host-based class-key fails us
-     */
-    private static String DEFAULT_CLASS_KEY = "default...";
-    
-    private static final String DNS = "dns";
-
-    public String getClassKey(CrawlURI cauri) {
-        String scheme = cauri.getUURI().getScheme();
-        String candidate = null;
-        try {
-            if (scheme.equals(DNS)) {
-            	UURI effectiveuuri;
-                if (cauri.getVia() != null) {
-                    // Special handling for DNS: treat as being
-                    // of the same class as the triggering URI.
-                    // When a URI includes a port, this ensures 
-                    // the DNS lookup goes atop the host:port
-                    // queue that triggered it, rather than 
-                    // some other host queue
-                	effectiveuuri = UURIFactory.getInstance(cauri.flattenVia());
-                } else {
-                	// To get the dns surt form, create a fake http version
-                	// (Gordon suggestion).
-                    effectiveuuri = UURIFactory.getInstance("http://" +
-                        cauri.getUURI().getPath());
-                }
-                candidate = getSurtAuthority(effectiveuuri.getSurtForm());
-            } else {
-                candidate = getSurtAuthority(cauri.getUURI().getSurtForm());
-            }
-            
-            if(candidate == null || candidate.length() == 0) {
-                candidate = DEFAULT_CLASS_KEY;
-            }
-        } catch (URIException e) {
-            logger.log(Level.INFO,
-                    "unable to extract class key; using default", e);
-            candidate = DEFAULT_CLASS_KEY;
-        }
-        // Ensure classKeys are safe as filenames on NTFS
+    @Override
+    String getCoreKey(UURI basis) {
+        String candidate = getSurtAuthority(basis.getSurtForm());
         return candidate.replace(':','#');
     }
-
+    
     protected String getSurtAuthority(String surt) {
         int indexOfOpen = surt.indexOf("://(");
         int indexOfClose = surt.indexOf(")");

engine/src/main/java/org/archive/crawler/frontier/URIAuthorityBasedQueueAssignmentPolicy.java

@@ -0,0 +1,120 @@
+/*
+ *  This file is part of the Heritrix web crawler (crawler.archive.org).
+ *
+ *  Licensed to the Internet Archive (IA) by one or more individual 
+ *  contributors. 
+ *
+ *  The IA licenses this file to You under the Apache License, Version 2.0
+ *  (the "License"); you may not use this file except in compliance with
+ *  the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ */
+
+package org.archive.crawler.frontier;
+
+import org.apache.commons.lang.StringUtils;
+import org.archive.crawler.datamodel.CrawlURI;
+import org.archive.modules.extractor.Hop;
+import org.archive.net.UURI;
+import org.archive.spring.HasKeyedProperties;
+import org.archive.spring.KeyedProperties;
+import org.archive.util.LongToIntConsistentHash;
+
+/**
+ * SurtAuthorityQueueAssignmentPolicy based on the surt form of hostname.
+ */
+public abstract class URIAuthorityBasedQueueAssignmentPolicy
+extends 
+ QueueAssignmentPolicy 
+implements
+ HasKeyedProperties {
+    private static final long serialVersionUID = 3L;
+    
+    KeyedProperties kp = new KeyedProperties();
+    public KeyedProperties getKeyedProperties() {
+        return kp;
+    }
+    //for when neat class-key fails us
+    protected static String DEFAULT_CLASS_KEY = "default...";
+
+    LongToIntConsistentHash conhash = new LongToIntConsistentHash();
+    
+    /**
+     * Whether to always defer to a previously-assigned key inside 
+     * the CrawlURI. If true, any key already in the CrawlURI will
+     * be returned as the classKey. 
+     */
+    public boolean getDeferToPrevious() {
+        return (Boolean) kp.get("deferToPrevious");
+    }
+    {
+        setDeferToPrevious(true);
+    }
+    public void setDeferToPrevious(boolean defer) {
+        kp.put("deferToPrevious",defer);
+    }
+    
+    /**
+     * The number of parallel queues to split a core key into. By 
+     * default is 1. If larger than 1, the non-authority-based portion
+     * of the URI will be used to distribute over that many separate
+     * queues. 
+     * 
+     */
+    public int getParallelQueues() {
+        return (Integer) kp.get("parallelQueues");
+    }
+    {
+        setParallelQueues(1);
+    }
+    public void setParallelQueues(int count) {
+        kp.put("parallelQueues",count);
+    }
+
+    public String getClassKey(CrawlURI curi) {
+        if(getDeferToPrevious() && !StringUtils.isEmpty(curi.getClassKey())) {
+            return curi.getClassKey();
+        }
+        
+        UURI basis = getBasisURI(curi); 
+        String candidate = getCoreKey(basis); 
+        
+        if(StringUtils.isEmpty(candidate)) {
+            return DEFAULT_CLASS_KEY;
+        }
+        
+        if(getParallelQueues()>1) {
+            int subqueue = getSubqueue(basis,getParallelQueues());
+            if (subqueue>0) {
+                candidate += "+"+subqueue;
+            }
+        }
+        return candidate; 
+    }
+    
+    protected int getSubqueue(UURI basis, int parallelQueues) {
+        return conhash.bucketFor(basis.getRawPathQuery(), parallelQueues);
+    }
+
+    abstract String getCoreKey(UURI basis);
+
+    protected UURI getBasisURI(CrawlURI curi) {
+        UURI effectiveuuri = null;
+        // always use 'via' of prerequisite URIs, if available, so
+        // prerequisites go to same queue as trigger URI
+        if (curi.getPathFromSeed().endsWith(Hop.PREREQ.getHopString())) {
+            effectiveuuri = curi.getVia();
+        }
+        if(effectiveuuri==null) {
+            effectiveuuri = curi.getUURI();
+        }
+        return effectiveuuri;
+    }
+}