diff --git a/engine/src/main/java/org/archive/crawler/frontier/URIAuthorityBasedQueueAssignmentPolicy.java b/engine/src/main/java/org/archive/crawler/frontier/URIAuthorityBasedQueueAssignmentPolicy.java index 03be7a0a..e6f19966 100644 --- a/engine/src/main/java/org/archive/crawler/frontier/URIAuthorityBasedQueueAssignmentPolicy.java +++ b/engine/src/main/java/org/archive/crawler/frontier/URIAuthorityBasedQueueAssignmentPolicy.java @@ -102,11 +102,29 @@ implements return candidate; } - protected int getSubqueue(UURI basis, int parallelQueues) { - if(null==basis.getRawPathQuery()) { + protected int getSubqueue(UURI basisUuri, int parallelQueues) { + String basis = bucketBasis(basisUuri); + if(StringUtils.isEmpty(basis)) { return 0; } - return conhash.bucketFor(basis.getRawPathQuery(), parallelQueues); + return conhash.bucketFor(basis, parallelQueues); + } + + /** + * Base subqueue on first path-segment, if any. (Means unbalanced + * subqueues, but consistency for most-common case where fanout + * can be at first segment, and it's beneficial to keep similar + * URIs in same queue.) + * @param uuri + * @return + */ + protected String bucketBasis(UURI uuri) { + String path = new String(uuri.getRawPath()); + int i = path.indexOf('/',1); + if(i<0) { + return null; + } + return path.substring(1,i); } abstract String getCoreKey(UURI basis); diff --git a/engine/src/main/java/org/archive/crawler/processor/CrawlMapper.java b/engine/src/main/java/org/archive/crawler/processor/CrawlMapper.java index 4065d7dd..f8119f4c 100644 --- a/engine/src/main/java/org/archive/crawler/processor/CrawlMapper.java +++ b/engine/src/main/java/org/archive/crawler/processor/CrawlMapper.java @@ -283,6 +283,8 @@ public abstract class CrawlMapper extends Processor implements Lifecycle { return; } PrintWriter diversionLog = getDiversionLog(target); + diversionLog.print(cauri.getClassKey()); + diversionLog.print(" "); cauri.singleLineReportTo(diversionLog); diversionLog.println(); } diff --git a/engine/src/main/java/org/archive/crawler/processor/HashCrawlMapper.java b/engine/src/main/java/org/archive/crawler/processor/HashCrawlMapper.java index 8c5164e2..21c717c4 100644 --- a/engine/src/main/java/org/archive/crawler/processor/HashCrawlMapper.java +++ b/engine/src/main/java/org/archive/crawler/processor/HashCrawlMapper.java @@ -1,24 +1,20 @@ -/* HashCrawlMapper - * - * Created on Sep 30, 2005 +/* + * This file is part of the Heritrix web crawler (crawler.archive.org). * - * Copyright (C) 2005 Internet Archive. - * - * This file is part of the Heritrix web crawler (crawler.archive.org). - * - * Heritrix is free software; you can redistribute it and/or modify - * it under the terms of the GNU Lesser Public License as published by - * the Free Software Foundation; either version 2.1 of the License, or - * any later version. - * - * Heritrix is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Lesser Public License for more details. - * - * You should have received a copy of the GNU Lesser Public License - * along with Heritrix; if not, write to the Free Software - * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + * Licensed to the Internet Archive (IA) by one or more individual + * contributors. + * + * The IA licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. */ package org.archive.crawler.processor;