From 8f7ad4eae413dd2d46c2b3e60d8106b571ab05a7 Mon Sep 17 00:00:00 2001 From: gojomo Date: Mon, 5 Oct 2009 23:04:37 +0000 Subject: [PATCH] work for [HER-1605] Refactor parameterizable policies out of frontier * QueueAssignmentPolicy.java, URIAuthorityBasedQueueAssignmentPolicy.java move 'force-queue-assignment' setting from frontier to policy --- .../frontier/QueueAssignmentPolicy.java | 22 ++++++++++++++++++- ...RIAuthorityBasedQueueAssignmentPolicy.java | 4 ++++ 2 files changed, 25 insertions(+), 1 deletion(-) diff --git a/engine/src/main/java/org/archive/crawler/frontier/QueueAssignmentPolicy.java b/engine/src/main/java/org/archive/crawler/frontier/QueueAssignmentPolicy.java index 6677471c..9181c710 100644 --- a/engine/src/main/java/org/archive/crawler/frontier/QueueAssignmentPolicy.java +++ b/engine/src/main/java/org/archive/crawler/frontier/QueueAssignmentPolicy.java @@ -22,13 +22,33 @@ package org.archive.crawler.frontier; import java.io.Serializable; import org.archive.modules.CrawlURI; +import org.archive.spring.HasKeyedProperties; +import org.archive.spring.KeyedProperties; /** * Establishes a mapping from CrawlURIs to String keys (queue names). * * @author gojomo */ -public abstract class QueueAssignmentPolicy implements Serializable { +public abstract class QueueAssignmentPolicy implements Serializable, HasKeyedProperties { + + KeyedProperties kp = new KeyedProperties(); + public KeyedProperties getKeyedProperties() { + return kp; + } + + /** queue assignment to force onto CrawlURIs; intended to be overridden */ + { + setForceQueueAssignment(""); + } + public String getForceQueueAssignment() { + return (String) kp.get("forceQueueAssignment"); + } + public void setForceQueueAssignment(String forceQueueAssignment) { + kp.put("forceQueueAssignment",forceQueueAssignment); + } + + /** * Get the String key (name) of the queue to which the * CrawlURI should be assigned. diff --git a/engine/src/main/java/org/archive/crawler/frontier/URIAuthorityBasedQueueAssignmentPolicy.java b/engine/src/main/java/org/archive/crawler/frontier/URIAuthorityBasedQueueAssignmentPolicy.java index 48ecb13b..03be7a0a 100644 --- a/engine/src/main/java/org/archive/crawler/frontier/URIAuthorityBasedQueueAssignmentPolicy.java +++ b/engine/src/main/java/org/archive/crawler/frontier/URIAuthorityBasedQueueAssignmentPolicy.java @@ -82,6 +82,10 @@ implements return curi.getClassKey(); } + if(!StringUtils.isEmpty(getForceQueueAssignment())) { + return getForceQueueAssignment(); + } + UURI basis = curi.getPolicyBasisUURI(); String candidate = getCoreKey(basis);