Crawl-distribution-related improvements

* URIAuthorityBasedQueueAssignmentPolicy.java
    choose numbered subqueue on first-path-segment only, so similar URIs land in same subqueue
* CrawlMapper.java
    add queueKey to diversion lines
* HashCrawlMapper.java
    update license

engine/src/main/java/org/archive/crawler/frontier/URIAuthorityBasedQueueAssignmentPolicy.java

@@ -102,11 +102,29 @@ implements
         return candidate; 
     }
     
-    protected int getSubqueue(UURI basis, int parallelQueues) {
-        if(null==basis.getRawPathQuery()) {
+    protected int getSubqueue(UURI basisUuri, int parallelQueues) {
+        String basis = bucketBasis(basisUuri);
+        if(StringUtils.isEmpty(basis)) {
             return 0; 
         }
-        return conhash.bucketFor(basis.getRawPathQuery(), parallelQueues);
+        return conhash.bucketFor(basis, parallelQueues);
+    }
+    
+    /**
+     * Base subqueue on first path-segment, if any. (Means unbalanced
+     * subqueues, but consistency for most-common case where fanout
+     * can be at first segment, and it's beneficial to keep similar
+     * URIs in same queue.)
+     * @param uuri
+     * @return
+     */
+    protected String bucketBasis(UURI uuri) {
+        String path = new String(uuri.getRawPath());
+        int i = path.indexOf('/',1);
+        if(i<0) {
+            return null; 
+        }
+        return path.substring(1,i);
     }
 
     abstract String getCoreKey(UURI basis);

engine/src/main/java/org/archive/crawler/processor/CrawlMapper.java

@@ -283,6 +283,8 @@ public abstract class CrawlMapper extends Processor implements Lifecycle {
             return;
         }
         PrintWriter diversionLog = getDiversionLog(target);
+        diversionLog.print(cauri.getClassKey());
+        diversionLog.print(" ");
         cauri.singleLineReportTo(diversionLog);
         diversionLog.println();
     }

engine/src/main/java/org/archive/crawler/processor/HashCrawlMapper.java

@@ -1,24 +1,20 @@
-/* HashCrawlMapper
- * 
- * Created on Sep 30, 2005
+/*
+ *  This file is part of the Heritrix web crawler (crawler.archive.org).
  *
- * Copyright (C) 2005 Internet Archive.
- * 
- * This file is part of the Heritrix web crawler (crawler.archive.org).
- * 
- * Heritrix is free software; you can redistribute it and/or modify
- * it under the terms of the GNU Lesser Public License as published by
- * the Free Software Foundation; either version 2.1 of the License, or
- * any later version.
- * 
- * Heritrix is distributed in the hope that it will be useful, 
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU Lesser Public License for more details.
- * 
- * You should have received a copy of the GNU Lesser Public License
- * along with Heritrix; if not, write to the Free Software
- * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+ *  Licensed to the Internet Archive (IA) by one or more individual 
+ *  contributors. 
+ *
+ *  The IA licenses this file to You under the Apache License, Version 2.0
+ *  (the "License"); you may not use this file except in compliance with
+ *  the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
  */
 package org.archive.crawler.processor;