From 0cee24922fea80acd0c7ef357b32168e35f2c195 Mon Sep 17 00:00:00 2001 From: Adam Miller Date: Fri, 11 Oct 2019 19:44:35 +0000 Subject: [PATCH 1/2] AssignmentLevelSurtQueueAssignmentPolicy.java - Add support for forced queue assignment and parallel queues URIAuthorityBasedQueueAssignmentPolicy.java - Add interoperability between forced queue assignment and parallel queues QuotaEnforcer.java - Fix javadoc to match default behavior --- ...ignmentLevelSurtQueueAssignmentPolicy.java | 33 +++++++++++++++++-- ...RIAuthorityBasedQueueAssignmentPolicy.java | 8 ++--- .../crawler/prefetch/QuotaEnforcer.java | 2 +- 3 files changed, 35 insertions(+), 8 deletions(-) diff --git a/engine/src/main/java/org/archive/crawler/frontier/AssignmentLevelSurtQueueAssignmentPolicy.java b/engine/src/main/java/org/archive/crawler/frontier/AssignmentLevelSurtQueueAssignmentPolicy.java index d187bfc6..88603e9c 100644 --- a/engine/src/main/java/org/archive/crawler/frontier/AssignmentLevelSurtQueueAssignmentPolicy.java +++ b/engine/src/main/java/org/archive/crawler/frontier/AssignmentLevelSurtQueueAssignmentPolicy.java @@ -18,7 +18,9 @@ */ package org.archive.crawler.frontier; +import org.apache.commons.lang.StringUtils; import org.archive.modules.CrawlURI; +import org.archive.net.UURI; import org.archive.net.PublicSuffixes; /** @@ -32,9 +34,34 @@ public class AssignmentLevelSurtQueueAssignmentPolicy extends private static final long serialVersionUID = -1533545293624791702L; @Override - public String getClassKey(CrawlURI cauri) { - String candidate = super.getClassKey(cauri); - candidate = PublicSuffixes.reduceSurtToAssignmentLevel(candidate); + public String getClassKey(CrawlURI curi) { + if(getDeferToPrevious() && !StringUtils.isEmpty(curi.getClassKey())) { + return curi.getClassKey(); + } + + UURI basis = curi.getPolicyBasisUURI(); + String candidate = super.getClassKey(curi); + candidate = PublicSuffixes.reduceSurtToAssignmentLevel(candidate); + + if(!StringUtils.isEmpty(getForceQueueAssignment())) { + candidate = getForceQueueAssignment(); + } + + // all whois urls in the same queue + if (curi.getUURI().getScheme().equals("whois")) { + return "whois..."; + } + + if(StringUtils.isEmpty(candidate)) { + return DEFAULT_CLASS_KEY; + } + if(getParallelQueues()>1) { + int subqueue = getSubqueue(basis,getParallelQueues()); + if (subqueue>0) { + candidate += "+"+subqueue; + } + } + return candidate; } diff --git a/engine/src/main/java/org/archive/crawler/frontier/URIAuthorityBasedQueueAssignmentPolicy.java b/engine/src/main/java/org/archive/crawler/frontier/URIAuthorityBasedQueueAssignmentPolicy.java index 8001540d..4fcec33e 100644 --- a/engine/src/main/java/org/archive/crawler/frontier/URIAuthorityBasedQueueAssignmentPolicy.java +++ b/engine/src/main/java/org/archive/crawler/frontier/URIAuthorityBasedQueueAssignmentPolicy.java @@ -90,9 +90,12 @@ implements if(getDeferToPrevious() && !StringUtils.isEmpty(curi.getClassKey())) { return curi.getClassKey(); } + + UURI basis = curi.getPolicyBasisUURI(); + String candidate = getCoreKey(basis); if(!StringUtils.isEmpty(getForceQueueAssignment())) { - return getForceQueueAssignment(); + candidate = getForceQueueAssignment(); } // all whois urls in the same queue @@ -100,9 +103,6 @@ implements return "whois..."; } - UURI basis = curi.getPolicyBasisUURI(); - String candidate = getCoreKey(basis); - if(StringUtils.isEmpty(candidate)) { return DEFAULT_CLASS_KEY; } diff --git a/engine/src/main/java/org/archive/crawler/prefetch/QuotaEnforcer.java b/engine/src/main/java/org/archive/crawler/prefetch/QuotaEnforcer.java index d7630227..171bb5d0 100644 --- a/engine/src/main/java/org/archive/crawler/prefetch/QuotaEnforcer.java +++ b/engine/src/main/java/org/archive/crawler/prefetch/QuotaEnforcer.java @@ -348,7 +348,7 @@ public class QuotaEnforcer extends Processor { * being force-retired (if the Frontier supports this). Note that if your * queues combine URIs that are different with regard to the quota category, * the retirement may hold back URIs not in the same quota category. Default - * is false. + * is true. */ { setForceRetire(true); From 74c0739bb00bf084a64a44aefea0bd6bd452098e Mon Sep 17 00:00:00 2001 From: Noah Levitt Date: Fri, 11 Oct 2019 13:49:49 -0700 Subject: [PATCH 2/2] fix line ending and indentation issues --- .../AssignmentLevelSurtQueueAssignmentPolicy.java | 2 +- .../frontier/URIAuthorityBasedQueueAssignmentPolicy.java | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/engine/src/main/java/org/archive/crawler/frontier/AssignmentLevelSurtQueueAssignmentPolicy.java b/engine/src/main/java/org/archive/crawler/frontier/AssignmentLevelSurtQueueAssignmentPolicy.java index 88603e9c..75875184 100644 --- a/engine/src/main/java/org/archive/crawler/frontier/AssignmentLevelSurtQueueAssignmentPolicy.java +++ b/engine/src/main/java/org/archive/crawler/frontier/AssignmentLevelSurtQueueAssignmentPolicy.java @@ -41,7 +41,7 @@ public class AssignmentLevelSurtQueueAssignmentPolicy extends UURI basis = curi.getPolicyBasisUURI(); String candidate = super.getClassKey(curi); - candidate = PublicSuffixes.reduceSurtToAssignmentLevel(candidate); + candidate = PublicSuffixes.reduceSurtToAssignmentLevel(candidate); if(!StringUtils.isEmpty(getForceQueueAssignment())) { candidate = getForceQueueAssignment(); diff --git a/engine/src/main/java/org/archive/crawler/frontier/URIAuthorityBasedQueueAssignmentPolicy.java b/engine/src/main/java/org/archive/crawler/frontier/URIAuthorityBasedQueueAssignmentPolicy.java index 4fcec33e..3212c4f0 100644 --- a/engine/src/main/java/org/archive/crawler/frontier/URIAuthorityBasedQueueAssignmentPolicy.java +++ b/engine/src/main/java/org/archive/crawler/frontier/URIAuthorityBasedQueueAssignmentPolicy.java @@ -90,12 +90,12 @@ implements if(getDeferToPrevious() && !StringUtils.isEmpty(curi.getClassKey())) { return curi.getClassKey(); } - - UURI basis = curi.getPolicyBasisUURI(); - String candidate = getCoreKey(basis); + + UURI basis = curi.getPolicyBasisUURI(); + String candidate = getCoreKey(basis); if(!StringUtils.isEmpty(getForceQueueAssignment())) { - candidate = getForceQueueAssignment(); + candidate = getForceQueueAssignment(); } // all whois urls in the same queue