From 4d06b239ee4b79fa9fe039bef19e161724af89af Mon Sep 17 00:00:00 2001 From: gojomo Date: Thu, 6 Aug 2009 20:22:00 +0000 Subject: [PATCH] Push CrawlURI and related constants down to modules subproject/package (many) --- .../datamodel/SchedulingConstants.java | 59 ------------------- .../crawler/datamodel/UriUniqFilter.java | 2 + .../ClassKeyMatchesRegExpDecideRule.java | 2 +- .../event/CrawlURIDispositionEvent.java | 2 +- .../archive/crawler/framework/Frontier.java | 2 +- .../org/archive/crawler/framework/Scoper.java | 2 +- .../archive/crawler/framework/ToeThread.java | 4 +- .../crawler/frontier/AbstractFrontier.java | 12 ++-- .../AntiCalendarCostAssignmentPolicy.java | 2 +- ...ignmentLevelSurtQueueAssignmentPolicy.java | 2 +- .../archive/crawler/frontier/BdbFrontier.java | 2 +- .../frontier/BdbMultipleWorkQueues.java | 2 +- .../crawler/frontier/BdbWorkQueue.java | 2 +- .../frontier/BucketQueueAssignmentPolicy.java | 2 +- .../frontier/CostAssignmentPolicy.java | 2 +- .../crawler/frontier/FrontierJournal.java | 2 +- .../frontier/IPQueueAssignmentPolicy.java | 2 +- .../frontier/QueueAssignmentPolicy.java | 2 +- ...RIAuthorityBasedQueueAssignmentPolicy.java | 2 +- .../frontier/UnitCostAssignmentPolicy.java | 2 +- .../frontier/WagCostAssignmentPolicy.java | 4 +- .../archive/crawler/frontier/WorkQueue.java | 2 +- .../crawler/frontier/WorkQueueFrontier.java | 10 ++-- .../frontier/ZeroCostAssignmentPolicy.java | 2 +- .../precedence/BaseUriPrecedencePolicy.java | 2 +- .../precedence/CostUriPrecedencePolicy.java | 2 +- .../HighestUriQueuePrecedencePolicy.java | 2 +- .../precedence/HopsUriPrecedencePolicy.java | 2 +- .../frontier/precedence/PrecedenceLoader.java | 3 +- .../PreloadedUriPrecedencePolicy.java | 4 +- .../precedence/UriPrecedencePolicy.java | 2 +- .../crawler/io/NonFatalErrorFormatter.java | 2 +- .../crawler/io/RuntimeErrorFormatter.java | 4 +- .../archive/crawler/io/UriErrorFormatter.java | 2 +- .../crawler/io/UriProcessingFormatter.java | 4 +- .../postprocessor/CrawlStateUpdater.java | 2 +- .../postprocessor/FrontierScheduler.java | 2 +- .../crawler/postprocessor/LinksScoper.java | 8 +-- .../SupplementaryLinksScoper.java | 2 +- .../prefetch/PreconditionEnforcer.java | 2 +- .../archive/crawler/prefetch/Preselector.java | 2 +- .../crawler/prefetch/QuotaEnforcer.java | 2 +- .../prefetch/RuntimeLimitEnforcer.java | 2 +- .../crawler/processor/CrawlMapper.java | 2 +- .../crawler/processor/HashCrawlMapper.java | 2 +- .../crawler/processor/LexicalCrawlMapper.java | 2 +- .../archive/crawler/reporting/SeedRecord.java | 4 +- .../crawler/reporting/StatisticsTracker.java | 4 +- .../crawler/spring/SheetOverlaysManager.java | 2 +- .../crawler/util/BenchmarkUriUniqFilters.java | 2 +- .../crawler/util/BloomUriUniqFilter.java | 2 +- .../crawler/util/CrawledBytesHistotable.java | 4 +- .../crawler/util/FPMergeUriUniqFilter.java | 2 +- .../crawler/util/SetBasedUriUniqFilter.java | 2 +- .../crawler/datamodel/CandidateURITest.java | 1 + .../crawler/datamodel/CrawlURITest.java | 2 + .../frontier/BdbMultipleWorkQueuesTest.java | 4 +- .../crawler/selftest/KeyWordProcessor.java | 2 +- .../selftest/KeyWordUriPrecedencePolicy.java | 2 +- .../crawler/util/BdbUriUniqFilterTest.java | 2 +- .../crawler/util/BloomUriUniqFilterTest.java | 2 +- .../crawler/util/FPUriUniqFilterTest.java | 2 +- .../modules}/CoreAttributeConstants.java | 3 +- .../java/org/archive/modules}/CrawlURI.java | 26 ++++---- .../archive/modules/SchedulingConstants.java | 52 ++++++++++++++++ 65 files changed, 150 insertions(+), 154 deletions(-) delete mode 100644 engine/src/main/java/org/archive/crawler/datamodel/SchedulingConstants.java rename {engine/src/main/java/org/archive/crawler/datamodel => modules/src/main/java/org/archive/modules}/CoreAttributeConstants.java (98%) rename {engine/src/main/java/org/archive/crawler/datamodel => modules/src/main/java/org/archive/modules}/CrawlURI.java (98%) create mode 100644 modules/src/main/java/org/archive/modules/SchedulingConstants.java diff --git a/engine/src/main/java/org/archive/crawler/datamodel/SchedulingConstants.java b/engine/src/main/java/org/archive/crawler/datamodel/SchedulingConstants.java deleted file mode 100644 index 67909524..00000000 --- a/engine/src/main/java/org/archive/crawler/datamodel/SchedulingConstants.java +++ /dev/null @@ -1,59 +0,0 @@ -/* - * Copyright (C) 2007 Internet Archive. - * - * This file is part of the Heritrix web crawler (crawler.archive.org). - * - * Heritrix is free software; you can redistribute it and/or modify - * it under the terms of the GNU Lesser Public License as published by - * the Free Software Foundation; either version 2.1 of the License, or - * any later version. - * - * Heritrix is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Lesser Public License for more details. - * - * You should have received a copy of the GNU Lesser Public License - * along with Heritrix; if not, write to the Free Software - * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA - * - * SchedulingConstants.java - * - * Created on Mar 23, 2007 - * - * $Id:$ - */ - -package org.archive.crawler.datamodel; - -/** - * @author pjack - * - */ -public class SchedulingConstants { - - - /** Highest scheduling priority. - * Before any others of its class. - */ - public static final int HIGHEST = 0; - - /** High scheduling priority. - * After any {@link #HIGHEST}. - */ - public static final int HIGH = 1; - - /** Medium priority. - * After any {@link #HIGH}. - */ - public static final int MEDIUM = 2; - - /** Normal/low priority. - * Whenever/end of queue. - */ - public static final int NORMAL = 3; - - - private SchedulingConstants() { - } -} diff --git a/engine/src/main/java/org/archive/crawler/datamodel/UriUniqFilter.java b/engine/src/main/java/org/archive/crawler/datamodel/UriUniqFilter.java index a93ddf94..341c53a9 100644 --- a/engine/src/main/java/org/archive/crawler/datamodel/UriUniqFilter.java +++ b/engine/src/main/java/org/archive/crawler/datamodel/UriUniqFilter.java @@ -24,6 +24,8 @@ package org.archive.crawler.datamodel; import java.io.File; +import org.archive.modules.CrawlURI; + /** * A UriUniqFilter passes URI objects to a destination * (receiver) if the passed URI object has not been previously seen. diff --git a/engine/src/main/java/org/archive/crawler/deciderules/ClassKeyMatchesRegExpDecideRule.java b/engine/src/main/java/org/archive/crawler/deciderules/ClassKeyMatchesRegExpDecideRule.java index ee2a4880..2d731071 100644 --- a/engine/src/main/java/org/archive/crawler/deciderules/ClassKeyMatchesRegExpDecideRule.java +++ b/engine/src/main/java/org/archive/crawler/deciderules/ClassKeyMatchesRegExpDecideRule.java @@ -25,8 +25,8 @@ package org.archive.crawler.deciderules; -import org.archive.crawler.datamodel.CrawlURI; import org.archive.crawler.framework.CrawlController; +import org.archive.modules.CrawlURI; import org.archive.modules.ProcessorURI; import org.archive.modules.deciderules.MatchesRegExpDecideRule; diff --git a/engine/src/main/java/org/archive/crawler/event/CrawlURIDispositionEvent.java b/engine/src/main/java/org/archive/crawler/event/CrawlURIDispositionEvent.java index 599602a9..a82d093a 100644 --- a/engine/src/main/java/org/archive/crawler/event/CrawlURIDispositionEvent.java +++ b/engine/src/main/java/org/archive/crawler/event/CrawlURIDispositionEvent.java @@ -18,7 +18,7 @@ */ package org.archive.crawler.event; -import org.archive.crawler.datamodel.CrawlURI; +import org.archive.modules.CrawlURI; import org.springframework.context.ApplicationEvent; public class CrawlURIDispositionEvent extends ApplicationEvent { diff --git a/engine/src/main/java/org/archive/crawler/framework/Frontier.java b/engine/src/main/java/org/archive/crawler/framework/Frontier.java index de60bc4d..bd0a801c 100644 --- a/engine/src/main/java/org/archive/crawler/framework/Frontier.java +++ b/engine/src/main/java/org/archive/crawler/framework/Frontier.java @@ -28,8 +28,8 @@ import java.io.IOException; import javax.management.openmbean.CompositeData; -import org.archive.crawler.datamodel.CrawlURI; import org.archive.crawler.frontier.FrontierJournal; +import org.archive.modules.CrawlURI; import org.archive.modules.deciderules.DecideRule; import org.archive.modules.fetcher.FetchStats; import org.archive.net.UURI; diff --git a/engine/src/main/java/org/archive/crawler/framework/Scoper.java b/engine/src/main/java/org/archive/crawler/framework/Scoper.java index b10c3933..3c8faf86 100644 --- a/engine/src/main/java/org/archive/crawler/framework/Scoper.java +++ b/engine/src/main/java/org/archive/crawler/framework/Scoper.java @@ -26,9 +26,9 @@ import java.util.logging.FileHandler; import java.util.logging.Level; import java.util.logging.Logger; -import org.archive.crawler.datamodel.CrawlURI; import org.archive.crawler.reporting.CrawlerLoggerModule; import org.archive.crawler.util.LogUtils; +import org.archive.modules.CrawlURI; import org.archive.modules.Processor; import org.archive.modules.deciderules.DecideResult; import org.archive.modules.deciderules.DecideRule; diff --git a/engine/src/main/java/org/archive/crawler/framework/ToeThread.java b/engine/src/main/java/org/archive/crawler/framework/ToeThread.java index cd0d96ba..1bd4c2c3 100644 --- a/engine/src/main/java/org/archive/crawler/framework/ToeThread.java +++ b/engine/src/main/java/org/archive/crawler/framework/ToeThread.java @@ -19,7 +19,7 @@ package org.archive.crawler.framework; -import static org.archive.crawler.datamodel.CoreAttributeConstants.A_RUNTIME_EXCEPTION; +import static org.archive.modules.CoreAttributeConstants.A_RUNTIME_EXCEPTION; import static org.archive.modules.fetcher.FetchStatusCodes.S_PROCESSING_THREAD_KILLED; import static org.archive.modules.fetcher.FetchStatusCodes.S_RUNTIME_EXCEPTION; import static org.archive.modules.fetcher.FetchStatusCodes.S_SERIOUS_ERROR; @@ -33,8 +33,8 @@ import java.util.Iterator; import java.util.logging.Level; import java.util.logging.Logger; -import org.archive.crawler.datamodel.CrawlURI; import org.archive.io.SinkHandlerLogThread; +import org.archive.modules.CrawlURI; import org.archive.modules.PostProcessor; import org.archive.modules.ProcessResult; import org.archive.modules.Processor; diff --git a/engine/src/main/java/org/archive/crawler/frontier/AbstractFrontier.java b/engine/src/main/java/org/archive/crawler/frontier/AbstractFrontier.java index 8ac1acb9..a3bea317 100644 --- a/engine/src/main/java/org/archive/crawler/frontier/AbstractFrontier.java +++ b/engine/src/main/java/org/archive/crawler/frontier/AbstractFrontier.java @@ -19,8 +19,8 @@ package org.archive.crawler.frontier; -import static org.archive.crawler.datamodel.CoreAttributeConstants.A_FETCH_COMPLETED_TIME; -import static org.archive.crawler.datamodel.CoreAttributeConstants.A_NONFATAL_ERRORS; +import static org.archive.modules.CoreAttributeConstants.A_FETCH_COMPLETED_TIME; +import static org.archive.modules.CoreAttributeConstants.A_NONFATAL_ERRORS; import static org.archive.modules.fetcher.FetchStatusCodes.S_BLOCKED_BY_CUSTOM_PROCESSOR; import static org.archive.modules.fetcher.FetchStatusCodes.S_BLOCKED_BY_USER; import static org.archive.modules.fetcher.FetchStatusCodes.S_CONNECT_FAILED; @@ -60,15 +60,15 @@ import org.apache.commons.httpclient.HttpStatus; import org.apache.commons.httpclient.URIException; import org.apache.commons.lang.StringUtils; import org.archive.checkpointing.CheckpointRecovery; -import org.archive.crawler.datamodel.CrawlURI; -import org.archive.crawler.datamodel.SchedulingConstants; import org.archive.crawler.event.CrawlStateEvent; import org.archive.crawler.framework.CrawlController; import org.archive.crawler.framework.Frontier; import org.archive.crawler.reporting.CrawlerLoggerModule; import org.archive.crawler.spring.SheetOverlaysManager; +import org.archive.modules.CrawlURI; import org.archive.modules.ModuleAttributeConstants; import org.archive.modules.ProcessorURI; +import org.archive.modules.SchedulingConstants; import org.archive.modules.canonicalize.UriCanonicalizationPolicy; import org.archive.modules.deciderules.DecideRule; import org.archive.modules.extractor.ExtractorParameters; @@ -730,7 +730,7 @@ public abstract class AbstractFrontier * Arrange for the given CrawlURI to be visited, if it is not * already scheduled/completed. * - * @see org.archive.crawler.framework.Frontier#schedule(org.archive.crawler.datamodel.CrawlURI) + * @see org.archive.crawler.framework.Frontier#schedule(org.archive.modules.CrawlURI) */ public void schedule(CrawlURI curi) { applyOverridesTo(curi); @@ -761,7 +761,7 @@ public abstract class AbstractFrontier * via the next next() call, as a result of finished(). * * (non-Javadoc) - * @see org.archive.crawler.framework.Frontier#finished(org.archive.crawler.datamodel.CrawlURI) + * @see org.archive.crawler.framework.Frontier#finished(org.archive.modules.CrawlURI) */ public void finished(CrawlURI curi) { enqueueOrDo(new Finish(curi)); diff --git a/engine/src/main/java/org/archive/crawler/frontier/AntiCalendarCostAssignmentPolicy.java b/engine/src/main/java/org/archive/crawler/frontier/AntiCalendarCostAssignmentPolicy.java index 6a622f65..1dc94b72 100644 --- a/engine/src/main/java/org/archive/crawler/frontier/AntiCalendarCostAssignmentPolicy.java +++ b/engine/src/main/java/org/archive/crawler/frontier/AntiCalendarCostAssignmentPolicy.java @@ -26,7 +26,7 @@ package org.archive.crawler.frontier; import java.util.regex.Matcher; -import org.archive.crawler.datamodel.CrawlURI; +import org.archive.modules.CrawlURI; import org.archive.util.TextUtils; /** diff --git a/engine/src/main/java/org/archive/crawler/frontier/AssignmentLevelSurtQueueAssignmentPolicy.java b/engine/src/main/java/org/archive/crawler/frontier/AssignmentLevelSurtQueueAssignmentPolicy.java index 37ccb1e7..d187bfc6 100644 --- a/engine/src/main/java/org/archive/crawler/frontier/AssignmentLevelSurtQueueAssignmentPolicy.java +++ b/engine/src/main/java/org/archive/crawler/frontier/AssignmentLevelSurtQueueAssignmentPolicy.java @@ -18,7 +18,7 @@ */ package org.archive.crawler.frontier; -import org.archive.crawler.datamodel.CrawlURI; +import org.archive.modules.CrawlURI; import org.archive.net.PublicSuffixes; /** diff --git a/engine/src/main/java/org/archive/crawler/frontier/BdbFrontier.java b/engine/src/main/java/org/archive/crawler/frontier/BdbFrontier.java index 5f78a013..34638605 100644 --- a/engine/src/main/java/org/archive/crawler/frontier/BdbFrontier.java +++ b/engine/src/main/java/org/archive/crawler/frontier/BdbFrontier.java @@ -46,7 +46,7 @@ import org.apache.commons.collections.Closure; import org.archive.bdb.BdbModule; import org.archive.checkpointing.Checkpointable; import org.archive.checkpointing.RecoverAction; -import org.archive.crawler.datamodel.CrawlURI; +import org.archive.modules.CrawlURI; import org.archive.queue.StoredQueue; import org.springframework.beans.factory.annotation.Autowired; diff --git a/engine/src/main/java/org/archive/crawler/frontier/BdbMultipleWorkQueues.java b/engine/src/main/java/org/archive/crawler/frontier/BdbMultipleWorkQueues.java index cebfe549..4fceb954 100644 --- a/engine/src/main/java/org/archive/crawler/frontier/BdbMultipleWorkQueues.java +++ b/engine/src/main/java/org/archive/crawler/frontier/BdbMultipleWorkQueues.java @@ -35,7 +35,7 @@ import javax.management.openmbean.CompositeDataSupport; import javax.management.openmbean.OpenDataException; import org.apache.commons.collections.Closure; -import org.archive.crawler.datamodel.CrawlURI; +import org.archive.modules.CrawlURI; import org.archive.util.ArchiveUtils; import com.sleepycat.bind.serial.StoredClassCatalog; diff --git a/engine/src/main/java/org/archive/crawler/frontier/BdbWorkQueue.java b/engine/src/main/java/org/archive/crawler/frontier/BdbWorkQueue.java index bc2a28fa..a36b4b78 100644 --- a/engine/src/main/java/org/archive/crawler/frontier/BdbWorkQueue.java +++ b/engine/src/main/java/org/archive/crawler/frontier/BdbWorkQueue.java @@ -28,7 +28,7 @@ import java.io.UnsupportedEncodingException; import java.util.logging.Level; import java.util.logging.Logger; -import org.archive.crawler.datamodel.CrawlURI; +import org.archive.modules.CrawlURI; import org.archive.util.ArchiveUtils; import org.archive.util.IoUtils; diff --git a/engine/src/main/java/org/archive/crawler/frontier/BucketQueueAssignmentPolicy.java b/engine/src/main/java/org/archive/crawler/frontier/BucketQueueAssignmentPolicy.java index 08ba4a43..d0b9561d 100644 --- a/engine/src/main/java/org/archive/crawler/frontier/BucketQueueAssignmentPolicy.java +++ b/engine/src/main/java/org/archive/crawler/frontier/BucketQueueAssignmentPolicy.java @@ -26,7 +26,7 @@ package org.archive.crawler.frontier; import org.apache.commons.httpclient.URIException; -import org.archive.crawler.datamodel.CrawlURI; +import org.archive.modules.CrawlURI; import org.archive.modules.net.CrawlHost; import org.archive.modules.net.ServerCache; import org.springframework.beans.factory.annotation.Autowired; diff --git a/engine/src/main/java/org/archive/crawler/frontier/CostAssignmentPolicy.java b/engine/src/main/java/org/archive/crawler/frontier/CostAssignmentPolicy.java index 2a393c47..174cd1ef 100644 --- a/engine/src/main/java/org/archive/crawler/frontier/CostAssignmentPolicy.java +++ b/engine/src/main/java/org/archive/crawler/frontier/CostAssignmentPolicy.java @@ -26,7 +26,7 @@ package org.archive.crawler.frontier; import java.io.Serializable; -import org.archive.crawler.datamodel.CrawlURI; +import org.archive.modules.CrawlURI; /** * Calculate a integer 'cost' value for the given CrawlURI. diff --git a/engine/src/main/java/org/archive/crawler/frontier/FrontierJournal.java b/engine/src/main/java/org/archive/crawler/frontier/FrontierJournal.java index 7850d5ee..7d4300be 100644 --- a/engine/src/main/java/org/archive/crawler/frontier/FrontierJournal.java +++ b/engine/src/main/java/org/archive/crawler/frontier/FrontierJournal.java @@ -34,10 +34,10 @@ import java.util.logging.Logger; import org.apache.commons.httpclient.URIException; import org.archive.checkpointing.Checkpointable; -import org.archive.crawler.datamodel.CrawlURI; import org.archive.crawler.framework.CrawlController; import org.archive.crawler.framework.Frontier; import org.archive.io.CrawlerJournal; +import org.archive.modules.CrawlURI; import org.archive.modules.deciderules.DecideRule; import org.archive.net.UURI; import org.archive.net.UURIFactory; diff --git a/engine/src/main/java/org/archive/crawler/frontier/IPQueueAssignmentPolicy.java b/engine/src/main/java/org/archive/crawler/frontier/IPQueueAssignmentPolicy.java index 30c3f90f..6cf121d8 100644 --- a/engine/src/main/java/org/archive/crawler/frontier/IPQueueAssignmentPolicy.java +++ b/engine/src/main/java/org/archive/crawler/frontier/IPQueueAssignmentPolicy.java @@ -24,7 +24,7 @@ */ package org.archive.crawler.frontier; -import org.archive.crawler.datamodel.CrawlURI; +import org.archive.modules.CrawlURI; import org.archive.modules.net.CrawlHost; import org.archive.modules.net.ServerCache; import org.archive.modules.net.ServerCacheUtil; diff --git a/engine/src/main/java/org/archive/crawler/frontier/QueueAssignmentPolicy.java b/engine/src/main/java/org/archive/crawler/frontier/QueueAssignmentPolicy.java index 7c1a4bae..6677471c 100644 --- a/engine/src/main/java/org/archive/crawler/frontier/QueueAssignmentPolicy.java +++ b/engine/src/main/java/org/archive/crawler/frontier/QueueAssignmentPolicy.java @@ -21,7 +21,7 @@ package org.archive.crawler.frontier; import java.io.Serializable; -import org.archive.crawler.datamodel.CrawlURI; +import org.archive.modules.CrawlURI; /** * Establishes a mapping from CrawlURIs to String keys (queue names). diff --git a/engine/src/main/java/org/archive/crawler/frontier/URIAuthorityBasedQueueAssignmentPolicy.java b/engine/src/main/java/org/archive/crawler/frontier/URIAuthorityBasedQueueAssignmentPolicy.java index 875f8d16..553e1957 100644 --- a/engine/src/main/java/org/archive/crawler/frontier/URIAuthorityBasedQueueAssignmentPolicy.java +++ b/engine/src/main/java/org/archive/crawler/frontier/URIAuthorityBasedQueueAssignmentPolicy.java @@ -20,7 +20,7 @@ package org.archive.crawler.frontier; import org.apache.commons.lang.StringUtils; -import org.archive.crawler.datamodel.CrawlURI; +import org.archive.modules.CrawlURI; import org.archive.modules.extractor.Hop; import org.archive.net.UURI; import org.archive.spring.HasKeyedProperties; diff --git a/engine/src/main/java/org/archive/crawler/frontier/UnitCostAssignmentPolicy.java b/engine/src/main/java/org/archive/crawler/frontier/UnitCostAssignmentPolicy.java index ec341124..e0817063 100644 --- a/engine/src/main/java/org/archive/crawler/frontier/UnitCostAssignmentPolicy.java +++ b/engine/src/main/java/org/archive/crawler/frontier/UnitCostAssignmentPolicy.java @@ -24,7 +24,7 @@ */ package org.archive.crawler.frontier; -import org.archive.crawler.datamodel.CrawlURI; +import org.archive.modules.CrawlURI; /** * A CostAssignment policy that uses a constant value of 1 for all CrawlURIs. diff --git a/engine/src/main/java/org/archive/crawler/frontier/WagCostAssignmentPolicy.java b/engine/src/main/java/org/archive/crawler/frontier/WagCostAssignmentPolicy.java index d560f42d..dfd19c74 100644 --- a/engine/src/main/java/org/archive/crawler/frontier/WagCostAssignmentPolicy.java +++ b/engine/src/main/java/org/archive/crawler/frontier/WagCostAssignmentPolicy.java @@ -24,7 +24,7 @@ */ package org.archive.crawler.frontier; -import org.archive.crawler.datamodel.CrawlURI; +import org.archive.modules.CrawlURI; import org.archive.net.UURI; /** @@ -42,7 +42,7 @@ public class WagCostAssignmentPolicy extends CostAssignmentPolicy { * * @param curi CrawlURI to be assigned a cost * - * @see org.archive.crawler.frontier.CostAssignmentPolicy#costOf(org.archive.crawler.datamodel.CrawlURI) + * @see org.archive.crawler.frontier.CostAssignmentPolicy#costOf(org.archive.modules.CrawlURI) */ public int costOf(CrawlURI curi) { int cost = 1; diff --git a/engine/src/main/java/org/archive/crawler/frontier/WorkQueue.java b/engine/src/main/java/org/archive/crawler/frontier/WorkQueue.java index bd17ad43..ccba9ca7 100644 --- a/engine/src/main/java/org/archive/crawler/frontier/WorkQueue.java +++ b/engine/src/main/java/org/archive/crawler/frontier/WorkQueue.java @@ -10,10 +10,10 @@ import java.util.concurrent.TimeUnit; import java.util.logging.Level; import java.util.logging.Logger; -import org.archive.crawler.datamodel.CrawlURI; import org.archive.crawler.framework.Frontier; import org.archive.crawler.frontier.precedence.PrecedenceProvider; import org.archive.crawler.frontier.precedence.SimplePrecedenceProvider; +import org.archive.modules.CrawlURI; import org.archive.modules.ProcessorURI; import org.archive.modules.fetcher.FetchStats; import org.archive.modules.fetcher.FetchStats.Stage; diff --git a/engine/src/main/java/org/archive/crawler/frontier/WorkQueueFrontier.java b/engine/src/main/java/org/archive/crawler/frontier/WorkQueueFrontier.java index 826b0ad1..831b003d 100644 --- a/engine/src/main/java/org/archive/crawler/frontier/WorkQueueFrontier.java +++ b/engine/src/main/java/org/archive/crawler/frontier/WorkQueueFrontier.java @@ -19,11 +19,11 @@ package org.archive.crawler.frontier; -import static org.archive.crawler.datamodel.CoreAttributeConstants.A_FORCE_RETIRE; import static org.archive.crawler.event.CrawlURIDispositionEvent.Disposition.DEFERRED_FOR_RETRY; import static org.archive.crawler.event.CrawlURIDispositionEvent.Disposition.DISREGARDED; import static org.archive.crawler.event.CrawlURIDispositionEvent.Disposition.FAILED; import static org.archive.crawler.event.CrawlURIDispositionEvent.Disposition.SUCCEEDED; +import static org.archive.modules.CoreAttributeConstants.A_FORCE_RETIRE; import static org.archive.modules.fetcher.FetchStatusCodes.S_DEFERRED; import static org.archive.modules.fetcher.FetchStatusCodes.S_RUNTIME_EXCEPTION; @@ -52,7 +52,6 @@ import java.util.regex.Pattern; import org.apache.commons.collections.Bag; import org.apache.commons.collections.BagUtils; import org.apache.commons.collections.bag.HashBag; -import org.archive.crawler.datamodel.CrawlURI; import org.archive.crawler.datamodel.UriUniqFilter; import org.archive.crawler.datamodel.UriUniqFilter.CrawlUriReceiver; import org.archive.crawler.event.CrawlURIDispositionEvent; @@ -61,6 +60,7 @@ import org.archive.crawler.frontier.precedence.BaseQueuePrecedencePolicy; import org.archive.crawler.frontier.precedence.CostUriPrecedencePolicy; import org.archive.crawler.frontier.precedence.QueuePrecedencePolicy; import org.archive.crawler.frontier.precedence.UriPrecedencePolicy; +import org.archive.modules.CrawlURI; import org.archive.net.UURI; import org.archive.spring.KeyedProperties; import org.archive.util.ArchiveUtils; @@ -354,7 +354,7 @@ ApplicationContextAware { * Arrange for the given CrawlURI to be visited, if it is not * already scheduled/completed. * - * @see org.archive.crawler.framework.Frontier#schedule(org.archive.crawler.datamodel.CrawlURI) + * @see org.archive.crawler.framework.Frontier#schedule(org.archive.modules.CrawlURI) */ protected void processScheduleIfUnique(CrawlURI curi) { assert Thread.currentThread() == managerThread; @@ -929,7 +929,7 @@ ApplicationContextAware { * via the next next() call, as a result of finished(). * * (non-Javadoc) - * @see org.archive.crawler.framework.Frontier#finished(org.archive.crawler.datamodel.CrawlURI) + * @see org.archive.crawler.framework.Frontier#finished(org.archive.modules.CrawlURI) */ protected void processFinish(CrawlURI curi) { assert Thread.currentThread() == managerThread; @@ -1489,7 +1489,7 @@ ApplicationContextAware { /** * Force logging, etc. of operator- deleted CrawlURIs * - * @see org.archive.crawler.framework.Frontier#deleted(org.archive.crawler.datamodel.CrawlURI) + * @see org.archive.crawler.framework.Frontier#deleted(org.archive.modules.CrawlURI) */ public void deleted(CrawlURI curi) { //treat as disregarded diff --git a/engine/src/main/java/org/archive/crawler/frontier/ZeroCostAssignmentPolicy.java b/engine/src/main/java/org/archive/crawler/frontier/ZeroCostAssignmentPolicy.java index 9d309f7d..1021f19e 100644 --- a/engine/src/main/java/org/archive/crawler/frontier/ZeroCostAssignmentPolicy.java +++ b/engine/src/main/java/org/archive/crawler/frontier/ZeroCostAssignmentPolicy.java @@ -6,7 +6,7 @@ */ package org.archive.crawler.frontier; -import org.archive.crawler.datamodel.CrawlURI; +import org.archive.modules.CrawlURI; /** * CostAssignmentPolicy considering all URIs costless -- essentially diff --git a/engine/src/main/java/org/archive/crawler/frontier/precedence/BaseUriPrecedencePolicy.java b/engine/src/main/java/org/archive/crawler/frontier/precedence/BaseUriPrecedencePolicy.java index 2dd983d4..31776044 100644 --- a/engine/src/main/java/org/archive/crawler/frontier/precedence/BaseUriPrecedencePolicy.java +++ b/engine/src/main/java/org/archive/crawler/frontier/precedence/BaseUriPrecedencePolicy.java @@ -19,7 +19,7 @@ package org.archive.crawler.frontier.precedence; -import org.archive.crawler.datamodel.CrawlURI; +import org.archive.modules.CrawlURI; import org.archive.spring.HasKeyedProperties; import org.archive.spring.KeyedProperties; diff --git a/engine/src/main/java/org/archive/crawler/frontier/precedence/CostUriPrecedencePolicy.java b/engine/src/main/java/org/archive/crawler/frontier/precedence/CostUriPrecedencePolicy.java index 02a64888..bace401c 100644 --- a/engine/src/main/java/org/archive/crawler/frontier/precedence/CostUriPrecedencePolicy.java +++ b/engine/src/main/java/org/archive/crawler/frontier/precedence/CostUriPrecedencePolicy.java @@ -24,7 +24,7 @@ */ package org.archive.crawler.frontier.precedence; -import org.archive.crawler.datamodel.CrawlURI; +import org.archive.modules.CrawlURI; /** * UriPrecedencePolicy which sets a URI's precedence to its 'cost' -- which diff --git a/engine/src/main/java/org/archive/crawler/frontier/precedence/HighestUriQueuePrecedencePolicy.java b/engine/src/main/java/org/archive/crawler/frontier/precedence/HighestUriQueuePrecedencePolicy.java index d0b5a434..d173d9d7 100644 --- a/engine/src/main/java/org/archive/crawler/frontier/precedence/HighestUriQueuePrecedencePolicy.java +++ b/engine/src/main/java/org/archive/crawler/frontier/precedence/HighestUriQueuePrecedencePolicy.java @@ -26,8 +26,8 @@ package org.archive.crawler.frontier.precedence; import java.io.PrintWriter; -import org.archive.crawler.datamodel.CrawlURI; import org.archive.crawler.frontier.WorkQueue; +import org.archive.modules.CrawlURI; import org.archive.modules.ProcessorURI; import org.archive.modules.fetcher.FetchStats.Stage; import org.archive.util.Histotable; diff --git a/engine/src/main/java/org/archive/crawler/frontier/precedence/HopsUriPrecedencePolicy.java b/engine/src/main/java/org/archive/crawler/frontier/precedence/HopsUriPrecedencePolicy.java index 5b349b32..1415029d 100644 --- a/engine/src/main/java/org/archive/crawler/frontier/precedence/HopsUriPrecedencePolicy.java +++ b/engine/src/main/java/org/archive/crawler/frontier/precedence/HopsUriPrecedencePolicy.java @@ -24,7 +24,7 @@ */ package org.archive.crawler.frontier.precedence; -import org.archive.crawler.datamodel.CrawlURI; +import org.archive.modules.CrawlURI; /** * UriPrecedencePolicy which assigns URIs a precedence equal to the number diff --git a/engine/src/main/java/org/archive/crawler/frontier/precedence/PrecedenceLoader.java b/engine/src/main/java/org/archive/crawler/frontier/precedence/PrecedenceLoader.java index 82807369..9a683ae4 100644 --- a/engine/src/main/java/org/archive/crawler/frontier/precedence/PrecedenceLoader.java +++ b/engine/src/main/java/org/archive/crawler/frontier/precedence/PrecedenceLoader.java @@ -31,7 +31,8 @@ import java.util.HashMap; import java.util.Iterator; import java.util.Map; -import static org.archive.crawler.datamodel.CoreAttributeConstants.*; +import static org.archive.modules.CoreAttributeConstants.*; + import org.archive.io.CrawlerJournal; import org.archive.modules.recrawl.PersistProcessor; import org.archive.util.bdbje.EnhancedEnvironment; diff --git a/engine/src/main/java/org/archive/crawler/frontier/precedence/PreloadedUriPrecedencePolicy.java b/engine/src/main/java/org/archive/crawler/frontier/precedence/PreloadedUriPrecedencePolicy.java index 77b1c489..80ffb06f 100644 --- a/engine/src/main/java/org/archive/crawler/frontier/precedence/PreloadedUriPrecedencePolicy.java +++ b/engine/src/main/java/org/archive/crawler/frontier/precedence/PreloadedUriPrecedencePolicy.java @@ -24,12 +24,12 @@ */ package org.archive.crawler.frontier.precedence; -import static org.archive.crawler.datamodel.CoreAttributeConstants.A_PRECALC_PRECEDENCE; +import static org.archive.modules.CoreAttributeConstants.A_PRECALC_PRECEDENCE; import java.util.Map; import org.archive.bdb.BdbModule; -import org.archive.crawler.datamodel.CrawlURI; +import org.archive.modules.CrawlURI; import org.archive.modules.recrawl.PersistProcessor; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.context.Lifecycle; diff --git a/engine/src/main/java/org/archive/crawler/frontier/precedence/UriPrecedencePolicy.java b/engine/src/main/java/org/archive/crawler/frontier/precedence/UriPrecedencePolicy.java index 53160397..791d1b4e 100644 --- a/engine/src/main/java/org/archive/crawler/frontier/precedence/UriPrecedencePolicy.java +++ b/engine/src/main/java/org/archive/crawler/frontier/precedence/UriPrecedencePolicy.java @@ -26,7 +26,7 @@ package org.archive.crawler.frontier.precedence; import java.io.Serializable; -import org.archive.crawler.datamodel.CrawlURI; +import org.archive.modules.CrawlURI; /** * Superclass for URI precedence policies, which set a integer diff --git a/engine/src/main/java/org/archive/crawler/io/NonFatalErrorFormatter.java b/engine/src/main/java/org/archive/crawler/io/NonFatalErrorFormatter.java index bc7430e8..f3f0b356 100644 --- a/engine/src/main/java/org/archive/crawler/io/NonFatalErrorFormatter.java +++ b/engine/src/main/java/org/archive/crawler/io/NonFatalErrorFormatter.java @@ -27,7 +27,7 @@ import java.io.PrintWriter; import java.io.StringWriter; import java.util.logging.LogRecord; -import org.archive.crawler.datamodel.CoreAttributeConstants; +import org.archive.modules.CoreAttributeConstants; /** * @author gojomo diff --git a/engine/src/main/java/org/archive/crawler/io/RuntimeErrorFormatter.java b/engine/src/main/java/org/archive/crawler/io/RuntimeErrorFormatter.java index de27b5c2..bbecd5fa 100644 --- a/engine/src/main/java/org/archive/crawler/io/RuntimeErrorFormatter.java +++ b/engine/src/main/java/org/archive/crawler/io/RuntimeErrorFormatter.java @@ -28,8 +28,8 @@ import java.io.PrintWriter; import java.io.StringWriter; import java.util.logging.LogRecord; -import org.archive.crawler.datamodel.CoreAttributeConstants; -import org.archive.crawler.datamodel.CrawlURI; +import org.archive.modules.CoreAttributeConstants; +import org.archive.modules.CrawlURI; /** * Runtime exception log formatter. diff --git a/engine/src/main/java/org/archive/crawler/io/UriErrorFormatter.java b/engine/src/main/java/org/archive/crawler/io/UriErrorFormatter.java index 29446771..948ca3a0 100644 --- a/engine/src/main/java/org/archive/crawler/io/UriErrorFormatter.java +++ b/engine/src/main/java/org/archive/crawler/io/UriErrorFormatter.java @@ -26,7 +26,7 @@ package org.archive.crawler.io; import java.util.logging.Formatter; import java.util.logging.LogRecord; -import org.archive.crawler.datamodel.CoreAttributeConstants; +import org.archive.modules.CoreAttributeConstants; import org.archive.net.UURI; import org.archive.util.ArchiveUtils; diff --git a/engine/src/main/java/org/archive/crawler/io/UriProcessingFormatter.java b/engine/src/main/java/org/archive/crawler/io/UriProcessingFormatter.java index e4178da7..5361bdb1 100644 --- a/engine/src/main/java/org/archive/crawler/io/UriProcessingFormatter.java +++ b/engine/src/main/java/org/archive/crawler/io/UriProcessingFormatter.java @@ -31,8 +31,8 @@ import java.util.Iterator; import java.util.logging.Formatter; import java.util.logging.LogRecord; -import org.archive.crawler.datamodel.CoreAttributeConstants; -import org.archive.crawler.datamodel.CrawlURI; +import org.archive.modules.CoreAttributeConstants; +import org.archive.modules.CrawlURI; import org.archive.util.ArchiveUtils; import org.archive.util.MimetypeUtils; diff --git a/engine/src/main/java/org/archive/crawler/postprocessor/CrawlStateUpdater.java b/engine/src/main/java/org/archive/crawler/postprocessor/CrawlStateUpdater.java index ac9f7930..bcfb7081 100644 --- a/engine/src/main/java/org/archive/crawler/postprocessor/CrawlStateUpdater.java +++ b/engine/src/main/java/org/archive/crawler/postprocessor/CrawlStateUpdater.java @@ -28,7 +28,7 @@ import static org.archive.modules.fetcher.FetchStatusCodes.S_CONNECT_FAILED; import java.util.logging.Logger; import org.apache.commons.httpclient.URIException; -import org.archive.crawler.datamodel.CrawlURI; +import org.archive.modules.CrawlURI; import org.archive.modules.PostProcessor; import org.archive.modules.Processor; import org.archive.modules.ProcessorURI; diff --git a/engine/src/main/java/org/archive/crawler/postprocessor/FrontierScheduler.java b/engine/src/main/java/org/archive/crawler/postprocessor/FrontierScheduler.java index 5415031b..1eae1d95 100644 --- a/engine/src/main/java/org/archive/crawler/postprocessor/FrontierScheduler.java +++ b/engine/src/main/java/org/archive/crawler/postprocessor/FrontierScheduler.java @@ -30,8 +30,8 @@ import static org.archive.modules.fetcher.FetchStatusCodes.S_DEFERRED; import java.util.concurrent.locks.ReentrantLock; -import org.archive.crawler.datamodel.CrawlURI; import org.archive.crawler.framework.Frontier; +import org.archive.modules.CrawlURI; import org.archive.modules.PostProcessor; import org.archive.modules.Processor; import org.archive.modules.ProcessorURI; diff --git a/engine/src/main/java/org/archive/crawler/postprocessor/LinksScoper.java b/engine/src/main/java/org/archive/crawler/postprocessor/LinksScoper.java index ff2d55c3..b2348fa5 100644 --- a/engine/src/main/java/org/archive/crawler/postprocessor/LinksScoper.java +++ b/engine/src/main/java/org/archive/crawler/postprocessor/LinksScoper.java @@ -25,17 +25,17 @@ */ package org.archive.crawler.postprocessor; -import static org.archive.crawler.datamodel.SchedulingConstants.HIGH; -import static org.archive.crawler.datamodel.SchedulingConstants.MEDIUM; -import static org.archive.crawler.datamodel.SchedulingConstants.NORMAL; +import static org.archive.modules.SchedulingConstants.HIGH; +import static org.archive.modules.SchedulingConstants.MEDIUM; +import static org.archive.modules.SchedulingConstants.NORMAL; import static org.archive.modules.fetcher.FetchStatusCodes.S_PREREQUISITE_UNSCHEDULABLE_FAILURE; import java.util.logging.Level; import java.util.logging.Logger; import org.apache.commons.httpclient.URIException; -import org.archive.crawler.datamodel.CrawlURI; import org.archive.crawler.framework.Scoper; +import org.archive.modules.CrawlURI; import org.archive.modules.PostProcessor; import org.archive.modules.ProcessorURI; import org.archive.modules.deciderules.DecideResult; diff --git a/engine/src/main/java/org/archive/crawler/postprocessor/SupplementaryLinksScoper.java b/engine/src/main/java/org/archive/crawler/postprocessor/SupplementaryLinksScoper.java index f24c35db..b8a4ac10 100644 --- a/engine/src/main/java/org/archive/crawler/postprocessor/SupplementaryLinksScoper.java +++ b/engine/src/main/java/org/archive/crawler/postprocessor/SupplementaryLinksScoper.java @@ -29,8 +29,8 @@ import java.util.Iterator; import java.util.logging.Level; import java.util.logging.Logger; -import org.archive.crawler.datamodel.CrawlURI; import org.archive.crawler.framework.Scoper; +import org.archive.modules.CrawlURI; import org.archive.modules.PostProcessor; import org.archive.modules.ProcessorURI; import org.archive.modules.deciderules.AcceptDecideRule; diff --git a/engine/src/main/java/org/archive/crawler/prefetch/PreconditionEnforcer.java b/engine/src/main/java/org/archive/crawler/prefetch/PreconditionEnforcer.java index 68008c75..fe3e6731 100644 --- a/engine/src/main/java/org/archive/crawler/prefetch/PreconditionEnforcer.java +++ b/engine/src/main/java/org/archive/crawler/prefetch/PreconditionEnforcer.java @@ -34,8 +34,8 @@ import java.util.logging.Level; import java.util.logging.Logger; import org.apache.commons.httpclient.URIException; -import org.archive.crawler.datamodel.CrawlURI; import org.archive.crawler.reporting.CrawlerLoggerModule; +import org.archive.modules.CrawlURI; import org.archive.modules.ProcessResult; import org.archive.modules.Processor; import org.archive.modules.ProcessorURI; diff --git a/engine/src/main/java/org/archive/crawler/prefetch/Preselector.java b/engine/src/main/java/org/archive/crawler/prefetch/Preselector.java index f2b3b992..31e3940b 100644 --- a/engine/src/main/java/org/archive/crawler/prefetch/Preselector.java +++ b/engine/src/main/java/org/archive/crawler/prefetch/Preselector.java @@ -24,11 +24,11 @@ package org.archive.crawler.prefetch; -import org.archive.crawler.datamodel.CrawlURI; import static org.archive.modules.fetcher.FetchStatusCodes.*; import org.archive.crawler.framework.Scoper; +import org.archive.modules.CrawlURI; import org.archive.modules.ProcessResult; import org.archive.modules.ProcessorURI; import org.archive.util.TextUtils; diff --git a/engine/src/main/java/org/archive/crawler/prefetch/QuotaEnforcer.java b/engine/src/main/java/org/archive/crawler/prefetch/QuotaEnforcer.java index 0beaf41e..d4baacf8 100644 --- a/engine/src/main/java/org/archive/crawler/prefetch/QuotaEnforcer.java +++ b/engine/src/main/java/org/archive/crawler/prefetch/QuotaEnforcer.java @@ -27,8 +27,8 @@ import static org.archive.modules.fetcher.FetchStatusCodes.S_BLOCKED_BY_QUOTA; import java.util.logging.Level; import java.util.logging.Logger; -import org.archive.crawler.datamodel.CrawlURI; import org.archive.crawler.framework.Frontier; +import org.archive.modules.CrawlURI; import org.archive.modules.ProcessResult; import org.archive.modules.Processor; import org.archive.modules.ProcessorURI; diff --git a/engine/src/main/java/org/archive/crawler/prefetch/RuntimeLimitEnforcer.java b/engine/src/main/java/org/archive/crawler/prefetch/RuntimeLimitEnforcer.java index e2f7c8ee..a5b55058 100644 --- a/engine/src/main/java/org/archive/crawler/prefetch/RuntimeLimitEnforcer.java +++ b/engine/src/main/java/org/archive/crawler/prefetch/RuntimeLimitEnforcer.java @@ -26,13 +26,13 @@ package org.archive.crawler.prefetch; import java.util.logging.Level; import java.util.logging.Logger; -import org.archive.crawler.datamodel.CrawlURI; import static org.archive.modules.fetcher.FetchStatusCodes.*; import org.archive.crawler.framework.CrawlController; import org.archive.crawler.framework.CrawlStatus; import org.archive.crawler.reporting.StatisticsTracker; +import org.archive.modules.CrawlURI; import org.archive.modules.ProcessResult; import org.archive.modules.Processor; import org.archive.modules.ProcessorURI; diff --git a/engine/src/main/java/org/archive/crawler/processor/CrawlMapper.java b/engine/src/main/java/org/archive/crawler/processor/CrawlMapper.java index 46439f19..67e8c3c9 100644 --- a/engine/src/main/java/org/archive/crawler/processor/CrawlMapper.java +++ b/engine/src/main/java/org/archive/crawler/processor/CrawlMapper.java @@ -32,7 +32,7 @@ import java.io.PrintWriter; import java.util.HashMap; import java.util.Iterator; -import org.archive.crawler.datamodel.CrawlURI; +import org.archive.modules.CrawlURI; import org.archive.modules.ProcessResult; import org.archive.modules.Processor; import org.archive.modules.ProcessorURI; diff --git a/engine/src/main/java/org/archive/crawler/processor/HashCrawlMapper.java b/engine/src/main/java/org/archive/crawler/processor/HashCrawlMapper.java index e0050329..8c5164e2 100644 --- a/engine/src/main/java/org/archive/crawler/processor/HashCrawlMapper.java +++ b/engine/src/main/java/org/archive/crawler/processor/HashCrawlMapper.java @@ -24,8 +24,8 @@ package org.archive.crawler.processor; import java.util.regex.Matcher; -import org.archive.crawler.datamodel.CrawlURI; import org.archive.crawler.framework.Frontier; +import org.archive.modules.CrawlURI; import org.archive.net.PublicSuffixes; import org.archive.util.TextUtils; import org.springframework.beans.factory.annotation.Autowired; diff --git a/engine/src/main/java/org/archive/crawler/processor/LexicalCrawlMapper.java b/engine/src/main/java/org/archive/crawler/processor/LexicalCrawlMapper.java index 537d617d..344b60ef 100644 --- a/engine/src/main/java/org/archive/crawler/processor/LexicalCrawlMapper.java +++ b/engine/src/main/java/org/archive/crawler/processor/LexicalCrawlMapper.java @@ -34,8 +34,8 @@ import java.util.Iterator; import java.util.SortedMap; import java.util.TreeMap; -import org.archive.crawler.datamodel.CrawlURI; import org.archive.crawler.framework.Frontier; +import org.archive.modules.CrawlURI; import org.archive.spring.ConfigPath; import org.archive.util.iterator.LineReadingIterator; import org.archive.util.iterator.RegexpLineIterator; diff --git a/engine/src/main/java/org/archive/crawler/reporting/SeedRecord.java b/engine/src/main/java/org/archive/crawler/reporting/SeedRecord.java index 6e39e386..75a1f24b 100644 --- a/engine/src/main/java/org/archive/crawler/reporting/SeedRecord.java +++ b/engine/src/main/java/org/archive/crawler/reporting/SeedRecord.java @@ -22,8 +22,8 @@ package org.archive.crawler.reporting; import java.io.Serializable; import java.util.logging.Logger; -import org.archive.crawler.datamodel.CrawlURI; -import org.archive.crawler.datamodel.CoreAttributeConstants; +import org.archive.modules.CoreAttributeConstants; +import org.archive.modules.CrawlURI; /** * Record of all interesting info about the most-recent diff --git a/engine/src/main/java/org/archive/crawler/reporting/StatisticsTracker.java b/engine/src/main/java/org/archive/crawler/reporting/StatisticsTracker.java index 209005da..8db565a5 100644 --- a/engine/src/main/java/org/archive/crawler/reporting/StatisticsTracker.java +++ b/engine/src/main/java/org/archive/crawler/reporting/StatisticsTracker.java @@ -19,7 +19,7 @@ package org.archive.crawler.reporting; -import static org.archive.crawler.datamodel.CoreAttributeConstants.A_SOURCE_TAG; +import static org.archive.modules.CoreAttributeConstants.A_SOURCE_TAG; import java.io.File; import java.io.FileWriter; @@ -46,7 +46,6 @@ import java.util.logging.Level; import java.util.logging.Logger; import org.archive.bdb.BdbModule; -import org.archive.crawler.datamodel.CrawlURI; import org.archive.crawler.event.CrawlStateEvent; import org.archive.crawler.event.CrawlURIDispositionEvent; import org.archive.crawler.event.StatSnapshotEvent; @@ -54,6 +53,7 @@ import org.archive.crawler.framework.CrawlController; import org.archive.crawler.framework.Engine; import org.archive.crawler.util.CrawledBytesHistotable; import org.archive.crawler.util.TopNSet; +import org.archive.modules.CrawlURI; import org.archive.modules.net.ServerCache; import org.archive.modules.net.ServerCacheUtil; import org.archive.modules.seeds.SeedModule; diff --git a/engine/src/main/java/org/archive/crawler/spring/SheetOverlaysManager.java b/engine/src/main/java/org/archive/crawler/spring/SheetOverlaysManager.java index 848b282d..e5baff50 100644 --- a/engine/src/main/java/org/archive/crawler/spring/SheetOverlaysManager.java +++ b/engine/src/main/java/org/archive/crawler/spring/SheetOverlaysManager.java @@ -30,7 +30,7 @@ import java.util.TreeMap; import java.util.TreeSet; import java.util.logging.Logger; -import org.archive.crawler.datamodel.CrawlURI; +import org.archive.modules.CrawlURI; import org.archive.spring.OverlayMapsSource; import org.archive.spring.Sheet; import org.archive.util.PrefixFinder; diff --git a/engine/src/main/java/org/archive/crawler/util/BenchmarkUriUniqFilters.java b/engine/src/main/java/org/archive/crawler/util/BenchmarkUriUniqFilters.java index f3be6d74..86c8ec45 100644 --- a/engine/src/main/java/org/archive/crawler/util/BenchmarkUriUniqFilters.java +++ b/engine/src/main/java/org/archive/crawler/util/BenchmarkUriUniqFilters.java @@ -30,8 +30,8 @@ import java.io.FileReader; import java.io.FileWriter; import java.io.IOException; -import org.archive.crawler.datamodel.CrawlURI; import org.archive.crawler.datamodel.UriUniqFilter; +import org.archive.modules.CrawlURI; import org.archive.util.fingerprint.MemLongFPSet; diff --git a/engine/src/main/java/org/archive/crawler/util/BloomUriUniqFilter.java b/engine/src/main/java/org/archive/crawler/util/BloomUriUniqFilter.java index 268f950a..c4310966 100644 --- a/engine/src/main/java/org/archive/crawler/util/BloomUriUniqFilter.java +++ b/engine/src/main/java/org/archive/crawler/util/BloomUriUniqFilter.java @@ -27,7 +27,7 @@ package org.archive.crawler.util; import java.io.Serializable; import java.util.logging.Logger; -import org.archive.crawler.datamodel.CrawlURI; +import org.archive.modules.CrawlURI; import org.archive.util.BloomFilter; import org.archive.util.BloomFilter32bitSplit; diff --git a/engine/src/main/java/org/archive/crawler/util/CrawledBytesHistotable.java b/engine/src/main/java/org/archive/crawler/util/CrawledBytesHistotable.java index fe21be8d..96060420 100644 --- a/engine/src/main/java/org/archive/crawler/util/CrawledBytesHistotable.java +++ b/engine/src/main/java/org/archive/crawler/util/CrawledBytesHistotable.java @@ -20,8 +20,8 @@ package org.archive.crawler.util; import org.apache.commons.httpclient.HttpStatus; -import org.archive.crawler.datamodel.CoreAttributeConstants; -import org.archive.crawler.datamodel.CrawlURI; +import org.archive.modules.CoreAttributeConstants; +import org.archive.modules.CrawlURI; import org.archive.modules.deciderules.recrawl.IdenticalDigestDecideRule; import org.archive.util.ArchiveUtils; import org.archive.util.Histotable; diff --git a/engine/src/main/java/org/archive/crawler/util/FPMergeUriUniqFilter.java b/engine/src/main/java/org/archive/crawler/util/FPMergeUriUniqFilter.java index 5782469c..3aafb764 100644 --- a/engine/src/main/java/org/archive/crawler/util/FPMergeUriUniqFilter.java +++ b/engine/src/main/java/org/archive/crawler/util/FPMergeUriUniqFilter.java @@ -36,8 +36,8 @@ import java.util.TreeSet; import java.util.logging.Level; import java.util.logging.Logger; -import org.archive.crawler.datamodel.CrawlURI; import org.archive.crawler.datamodel.UriUniqFilter; +import org.archive.modules.CrawlURI; import org.archive.util.fingerprint.ArrayLongFPCache; import st.ata.util.FPGenerator; diff --git a/engine/src/main/java/org/archive/crawler/util/SetBasedUriUniqFilter.java b/engine/src/main/java/org/archive/crawler/util/SetBasedUriUniqFilter.java index 73771c23..6fbfaa3c 100644 --- a/engine/src/main/java/org/archive/crawler/util/SetBasedUriUniqFilter.java +++ b/engine/src/main/java/org/archive/crawler/util/SetBasedUriUniqFilter.java @@ -33,8 +33,8 @@ import java.io.Serializable; import java.util.logging.Level; import java.util.logging.Logger; -import org.archive.crawler.datamodel.CrawlURI; import org.archive.crawler.datamodel.UriUniqFilter; +import org.archive.modules.CrawlURI; /** * UriUniqFilter based on an underlying UriSet (essentially a Set). diff --git a/engine/src/test/java/org/archive/crawler/datamodel/CandidateURITest.java b/engine/src/test/java/org/archive/crawler/datamodel/CandidateURITest.java index e6cdb53b..d0f6ac42 100644 --- a/engine/src/test/java/org/archive/crawler/datamodel/CandidateURITest.java +++ b/engine/src/test/java/org/archive/crawler/datamodel/CandidateURITest.java @@ -32,6 +32,7 @@ import java.io.ObjectOutputStream; import junit.framework.TestCase; +import org.archive.modules.CrawlURI; import org.archive.net.UURIFactory; /** diff --git a/engine/src/test/java/org/archive/crawler/datamodel/CrawlURITest.java b/engine/src/test/java/org/archive/crawler/datamodel/CrawlURITest.java index d056ea2c..12af602b 100644 --- a/engine/src/test/java/org/archive/crawler/datamodel/CrawlURITest.java +++ b/engine/src/test/java/org/archive/crawler/datamodel/CrawlURITest.java @@ -30,6 +30,8 @@ import java.io.ObjectInputStream; import java.io.ObjectOutputStream; import org.apache.commons.httpclient.URIException; +import org.archive.modules.CrawlURI; +import org.archive.modules.SchedulingConstants; import org.archive.net.UURI; import org.archive.net.UURIFactory; import org.archive.util.TmpDirTestCase; diff --git a/engine/src/test/java/org/archive/crawler/frontier/BdbMultipleWorkQueuesTest.java b/engine/src/test/java/org/archive/crawler/frontier/BdbMultipleWorkQueuesTest.java index 82929af9..c0144756 100644 --- a/engine/src/test/java/org/archive/crawler/frontier/BdbMultipleWorkQueuesTest.java +++ b/engine/src/test/java/org/archive/crawler/frontier/BdbMultipleWorkQueuesTest.java @@ -27,8 +27,8 @@ package org.archive.crawler.frontier; import java.util.logging.Logger; import org.apache.commons.httpclient.URIException; -import org.archive.crawler.datamodel.CrawlURI; -import org.archive.crawler.datamodel.SchedulingConstants; +import org.archive.modules.CrawlURI; +import org.archive.modules.SchedulingConstants; import org.archive.net.UURIFactory; import com.sleepycat.je.tree.Key; diff --git a/engine/src/test/java/org/archive/crawler/selftest/KeyWordProcessor.java b/engine/src/test/java/org/archive/crawler/selftest/KeyWordProcessor.java index 9f534b52..9cacadc6 100644 --- a/engine/src/test/java/org/archive/crawler/selftest/KeyWordProcessor.java +++ b/engine/src/test/java/org/archive/crawler/selftest/KeyWordProcessor.java @@ -2,7 +2,7 @@ package org.archive.crawler.selftest; import java.util.regex.Pattern; -import org.archive.crawler.datamodel.CrawlURI; +import org.archive.modules.CrawlURI; import org.archive.modules.Processor; import org.archive.modules.ProcessorURI; diff --git a/engine/src/test/java/org/archive/crawler/selftest/KeyWordUriPrecedencePolicy.java b/engine/src/test/java/org/archive/crawler/selftest/KeyWordUriPrecedencePolicy.java index f067a347..0bcec82c 100644 --- a/engine/src/test/java/org/archive/crawler/selftest/KeyWordUriPrecedencePolicy.java +++ b/engine/src/test/java/org/archive/crawler/selftest/KeyWordUriPrecedencePolicy.java @@ -19,8 +19,8 @@ package org.archive.crawler.selftest; -import org.archive.crawler.datamodel.CrawlURI; import org.archive.crawler.frontier.precedence.BaseUriPrecedencePolicy; +import org.archive.modules.CrawlURI; /** * Testing policy which uses a precedence inside the CrawlURI (presumably diff --git a/engine/src/test/java/org/archive/crawler/util/BdbUriUniqFilterTest.java b/engine/src/test/java/org/archive/crawler/util/BdbUriUniqFilterTest.java index c4105926..18f84e0f 100644 --- a/engine/src/test/java/org/archive/crawler/util/BdbUriUniqFilterTest.java +++ b/engine/src/test/java/org/archive/crawler/util/BdbUriUniqFilterTest.java @@ -30,8 +30,8 @@ import junit.framework.TestSuite; import org.apache.commons.httpclient.URIException; import org.apache.commons.io.FileUtils; -import org.archive.crawler.datamodel.CrawlURI; import org.archive.crawler.datamodel.UriUniqFilter; +import org.archive.modules.CrawlURI; import org.archive.net.UURI; import org.archive.net.UURIFactory; import org.archive.util.TmpDirTestCase; diff --git a/engine/src/test/java/org/archive/crawler/util/BloomUriUniqFilterTest.java b/engine/src/test/java/org/archive/crawler/util/BloomUriUniqFilterTest.java index f57ae45a..d852f7b4 100644 --- a/engine/src/test/java/org/archive/crawler/util/BloomUriUniqFilterTest.java +++ b/engine/src/test/java/org/archive/crawler/util/BloomUriUniqFilterTest.java @@ -28,8 +28,8 @@ import java.util.logging.Logger; import junit.framework.TestCase; import org.apache.commons.httpclient.URIException; -import org.archive.crawler.datamodel.CrawlURI; import org.archive.crawler.datamodel.UriUniqFilter; +import org.archive.modules.CrawlURI; import org.archive.net.UURI; import org.archive.net.UURIFactory; diff --git a/engine/src/test/java/org/archive/crawler/util/FPUriUniqFilterTest.java b/engine/src/test/java/org/archive/crawler/util/FPUriUniqFilterTest.java index 2e241489..92e864eb 100644 --- a/engine/src/test/java/org/archive/crawler/util/FPUriUniqFilterTest.java +++ b/engine/src/test/java/org/archive/crawler/util/FPUriUniqFilterTest.java @@ -28,8 +28,8 @@ import java.util.logging.Logger; import junit.framework.TestCase; import org.apache.commons.httpclient.URIException; -import org.archive.crawler.datamodel.CrawlURI; import org.archive.crawler.datamodel.UriUniqFilter; +import org.archive.modules.CrawlURI; import org.archive.net.UURI; import org.archive.net.UURIFactory; import org.archive.util.fingerprint.MemLongFPSet; diff --git a/engine/src/main/java/org/archive/crawler/datamodel/CoreAttributeConstants.java b/modules/src/main/java/org/archive/modules/CoreAttributeConstants.java similarity index 98% rename from engine/src/main/java/org/archive/crawler/datamodel/CoreAttributeConstants.java rename to modules/src/main/java/org/archive/modules/CoreAttributeConstants.java index 0adf44cd..4e698376 100644 --- a/engine/src/main/java/org/archive/crawler/datamodel/CoreAttributeConstants.java +++ b/modules/src/main/java/org/archive/modules/CoreAttributeConstants.java @@ -23,9 +23,8 @@ * @author gojomo * */ -package org.archive.crawler.datamodel; +package org.archive.modules; -import org.archive.modules.ModuleAttributeConstants; /** * CrawlURI attribute keys used by the core crawler diff --git a/engine/src/main/java/org/archive/crawler/datamodel/CrawlURI.java b/modules/src/main/java/org/archive/modules/CrawlURI.java similarity index 98% rename from engine/src/main/java/org/archive/crawler/datamodel/CrawlURI.java rename to modules/src/main/java/org/archive/modules/CrawlURI.java index 488b3343..3d23a29f 100644 --- a/engine/src/main/java/org/archive/crawler/datamodel/CrawlURI.java +++ b/modules/src/main/java/org/archive/modules/CrawlURI.java @@ -17,19 +17,19 @@ * limitations under the License. */ -package org.archive.crawler.datamodel; +package org.archive.modules; -import static org.archive.crawler.datamodel.CoreAttributeConstants.A_ANNOTATIONS; -import static org.archive.crawler.datamodel.CoreAttributeConstants.A_CREDENTIAL_AVATARS_KEY; -import static org.archive.crawler.datamodel.CoreAttributeConstants.A_DNS_SERVER_IP_LABEL; -import static org.archive.crawler.datamodel.CoreAttributeConstants.A_FETCH_COMPLETED_TIME; -import static org.archive.crawler.datamodel.CoreAttributeConstants.A_FORCE_RETIRE; -import static org.archive.crawler.datamodel.CoreAttributeConstants.A_HERITABLE_KEYS; -import static org.archive.crawler.datamodel.CoreAttributeConstants.A_HTML_BASE; -import static org.archive.crawler.datamodel.CoreAttributeConstants.A_NONFATAL_ERRORS; -import static org.archive.crawler.datamodel.CoreAttributeConstants.A_PREREQUISITE_URI; -import static org.archive.crawler.datamodel.CoreAttributeConstants.A_SOURCE_TAG; -import static org.archive.crawler.datamodel.SchedulingConstants.NORMAL; +import static org.archive.modules.CoreAttributeConstants.A_ANNOTATIONS; +import static org.archive.modules.CoreAttributeConstants.A_CREDENTIAL_AVATARS_KEY; +import static org.archive.modules.CoreAttributeConstants.A_DNS_SERVER_IP_LABEL; +import static org.archive.modules.CoreAttributeConstants.A_FETCH_COMPLETED_TIME; +import static org.archive.modules.CoreAttributeConstants.A_FORCE_RETIRE; +import static org.archive.modules.CoreAttributeConstants.A_HERITABLE_KEYS; +import static org.archive.modules.CoreAttributeConstants.A_HTML_BASE; +import static org.archive.modules.CoreAttributeConstants.A_NONFATAL_ERRORS; +import static org.archive.modules.CoreAttributeConstants.A_PREREQUISITE_URI; +import static org.archive.modules.CoreAttributeConstants.A_SOURCE_TAG; +import static org.archive.modules.SchedulingConstants.NORMAL; import static org.archive.modules.fetcher.FetchStatusCodes.S_BLOCKED_BY_CUSTOM_PROCESSOR; import static org.archive.modules.fetcher.FetchStatusCodes.S_BLOCKED_BY_USER; import static org.archive.modules.fetcher.FetchStatusCodes.S_CONNECT_FAILED; @@ -76,8 +76,6 @@ import org.apache.commons.httpclient.HttpStatus; import org.apache.commons.httpclient.URIException; import org.apache.commons.httpclient.methods.GetMethod; import org.apache.commons.httpclient.methods.PostMethod; -import org.archive.modules.ModuleAttributeConstants; -import org.archive.modules.ProcessorURI; import org.archive.modules.credential.CredentialAvatar; import org.archive.modules.credential.HttpAuthenticationCredential; import org.archive.modules.extractor.HTMLLinkContext; diff --git a/modules/src/main/java/org/archive/modules/SchedulingConstants.java b/modules/src/main/java/org/archive/modules/SchedulingConstants.java new file mode 100644 index 00000000..77bd21e8 --- /dev/null +++ b/modules/src/main/java/org/archive/modules/SchedulingConstants.java @@ -0,0 +1,52 @@ +/* + * This file is part of the Heritrix web crawler (crawler.archive.org). + * + * Licensed to the Internet Archive (IA) by one or more individual + * contributors. + * + * The IA licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.archive.modules; + +/** + * @author pjack + * + */ +public class SchedulingConstants { + + + /** Highest scheduling priority. + * Before any others of its class. + */ + public static final int HIGHEST = 0; + + /** High scheduling priority. + * After any {@link #HIGHEST}. + */ + public static final int HIGH = 1; + + /** Medium priority. + * After any {@link #HIGH}. + */ + public static final int MEDIUM = 2; + + /** Normal/low priority. + * Whenever/end of queue. + */ + public static final int NORMAL = 3; + + + private SchedulingConstants() { + } +}