Push CrawlURI and related constants down to modules subproject/package

(many)

engine/src/main/java/org/archive/crawler/datamodel/SchedulingConstants.java

@@ -1,59 +0,0 @@
-/* 
- * Copyright (C) 2007 Internet Archive.
- *
- * This file is part of the Heritrix web crawler (crawler.archive.org).
- *
- * Heritrix is free software; you can redistribute it and/or modify
- * it under the terms of the GNU Lesser Public License as published by
- * the Free Software Foundation; either version 2.1 of the License, or
- * any later version.
- *
- * Heritrix is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU Lesser Public License for more details.
- *
- * You should have received a copy of the GNU Lesser Public License
- * along with Heritrix; if not, write to the Free Software
- * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
- *
- * SchedulingConstants.java
- *
- * Created on Mar 23, 2007
- *
- * $Id:$
- */
-
-package org.archive.crawler.datamodel;
-
-/**
- * @author pjack
- *
- */
-public class SchedulingConstants {
-
-    
-    /** Highest scheduling priority.
-     * Before any others of its class.
-     */
-    public static final int HIGHEST = 0;
-    
-    /** High scheduling priority.
-     * After any {@link #HIGHEST}.
-     */
-    public static final int HIGH = 1;
-    
-    /** Medium priority.
-     * After any {@link #HIGH}.
-     */
-    public static final int MEDIUM = 2;
-    
-    /** Normal/low priority.
-     * Whenever/end of queue.
-     */
-    public static final int NORMAL = 3;
-
-    
-    private SchedulingConstants() {
-    }
-}

engine/src/main/java/org/archive/crawler/datamodel/UriUniqFilter.java

@@ -24,6 +24,8 @@ package org.archive.crawler.datamodel;
 
 import java.io.File;
 
+import org.archive.modules.CrawlURI;
+
 /**
  * A UriUniqFilter passes URI objects to a destination
  * (receiver) if the passed URI object has not been previously seen.

engine/src/main/java/org/archive/crawler/deciderules/ClassKeyMatchesRegExpDecideRule.java

@@ -25,8 +25,8 @@
 package org.archive.crawler.deciderules;
 
 
-import org.archive.crawler.datamodel.CrawlURI;
 import org.archive.crawler.framework.CrawlController;
+import org.archive.modules.CrawlURI;
 import org.archive.modules.ProcessorURI;
 import org.archive.modules.deciderules.MatchesRegExpDecideRule;
 

engine/src/main/java/org/archive/crawler/event/CrawlURIDispositionEvent.java

@@ -18,7 +18,7 @@
  */
 package org.archive.crawler.event;
 
-import org.archive.crawler.datamodel.CrawlURI;
+import org.archive.modules.CrawlURI;
 import org.springframework.context.ApplicationEvent;
 
 public class CrawlURIDispositionEvent extends ApplicationEvent {

engine/src/main/java/org/archive/crawler/framework/Frontier.java

@@ -28,8 +28,8 @@ import java.io.IOException;
 
 import javax.management.openmbean.CompositeData;
 
-import org.archive.crawler.datamodel.CrawlURI;
 import org.archive.crawler.frontier.FrontierJournal;
+import org.archive.modules.CrawlURI;
 import org.archive.modules.deciderules.DecideRule;
 import org.archive.modules.fetcher.FetchStats;
 import org.archive.net.UURI;

engine/src/main/java/org/archive/crawler/framework/Scoper.java

@@ -26,9 +26,9 @@ import java.util.logging.FileHandler;
 import java.util.logging.Level;
 import java.util.logging.Logger;
 
-import org.archive.crawler.datamodel.CrawlURI;
 import org.archive.crawler.reporting.CrawlerLoggerModule;
 import org.archive.crawler.util.LogUtils;
+import org.archive.modules.CrawlURI;
 import org.archive.modules.Processor;
 import org.archive.modules.deciderules.DecideResult;
 import org.archive.modules.deciderules.DecideRule;

engine/src/main/java/org/archive/crawler/framework/ToeThread.java

@@ -19,7 +19,7 @@
 
 package org.archive.crawler.framework;
 
-import static org.archive.crawler.datamodel.CoreAttributeConstants.A_RUNTIME_EXCEPTION;
+import static org.archive.modules.CoreAttributeConstants.A_RUNTIME_EXCEPTION;
 import static org.archive.modules.fetcher.FetchStatusCodes.S_PROCESSING_THREAD_KILLED;
 import static org.archive.modules.fetcher.FetchStatusCodes.S_RUNTIME_EXCEPTION;
 import static org.archive.modules.fetcher.FetchStatusCodes.S_SERIOUS_ERROR;
@@ -33,8 +33,8 @@ import java.util.Iterator;
 import java.util.logging.Level;
 import java.util.logging.Logger;
 
-import org.archive.crawler.datamodel.CrawlURI;
 import org.archive.io.SinkHandlerLogThread;
+import org.archive.modules.CrawlURI;
 import org.archive.modules.PostProcessor;
 import org.archive.modules.ProcessResult;
 import org.archive.modules.Processor;

engine/src/main/java/org/archive/crawler/frontier/AbstractFrontier.java

@@ -19,8 +19,8 @@
 
 package org.archive.crawler.frontier;
 
-import static org.archive.crawler.datamodel.CoreAttributeConstants.A_FETCH_COMPLETED_TIME;
-import static org.archive.crawler.datamodel.CoreAttributeConstants.A_NONFATAL_ERRORS;
+import static org.archive.modules.CoreAttributeConstants.A_FETCH_COMPLETED_TIME;
+import static org.archive.modules.CoreAttributeConstants.A_NONFATAL_ERRORS;
 import static org.archive.modules.fetcher.FetchStatusCodes.S_BLOCKED_BY_CUSTOM_PROCESSOR;
 import static org.archive.modules.fetcher.FetchStatusCodes.S_BLOCKED_BY_USER;
 import static org.archive.modules.fetcher.FetchStatusCodes.S_CONNECT_FAILED;
@@ -60,15 +60,15 @@ import org.apache.commons.httpclient.HttpStatus;
 import org.apache.commons.httpclient.URIException;
 import org.apache.commons.lang.StringUtils;
 import org.archive.checkpointing.CheckpointRecovery;
-import org.archive.crawler.datamodel.CrawlURI;
-import org.archive.crawler.datamodel.SchedulingConstants;
 import org.archive.crawler.event.CrawlStateEvent;
 import org.archive.crawler.framework.CrawlController;
 import org.archive.crawler.framework.Frontier;
 import org.archive.crawler.reporting.CrawlerLoggerModule;
 import org.archive.crawler.spring.SheetOverlaysManager;
+import org.archive.modules.CrawlURI;
 import org.archive.modules.ModuleAttributeConstants;
 import org.archive.modules.ProcessorURI;
+import org.archive.modules.SchedulingConstants;
 import org.archive.modules.canonicalize.UriCanonicalizationPolicy;
 import org.archive.modules.deciderules.DecideRule;
 import org.archive.modules.extractor.ExtractorParameters;
@@ -730,7 +730,7 @@ public abstract class AbstractFrontier
      * Arrange for the given CrawlURI to be visited, if it is not
      * already scheduled/completed.
      *
-     * @see org.archive.crawler.framework.Frontier#schedule(org.archive.crawler.datamodel.CrawlURI)
+     * @see org.archive.crawler.framework.Frontier#schedule(org.archive.modules.CrawlURI)
      */
     public void schedule(CrawlURI curi) {
         applyOverridesTo(curi);
@@ -761,7 +761,7 @@ public abstract class AbstractFrontier
      * via the next next() call, as a result of finished().
      *
      *  (non-Javadoc)
-     * @see org.archive.crawler.framework.Frontier#finished(org.archive.crawler.datamodel.CrawlURI)
+     * @see org.archive.crawler.framework.Frontier#finished(org.archive.modules.CrawlURI)
      */
     public void finished(CrawlURI curi) {
         enqueueOrDo(new Finish(curi));

engine/src/main/java/org/archive/crawler/frontier/AntiCalendarCostAssignmentPolicy.java

@@ -26,7 +26,7 @@ package org.archive.crawler.frontier;
 
 import java.util.regex.Matcher;
 
-import org.archive.crawler.datamodel.CrawlURI;
+import org.archive.modules.CrawlURI;
 import org.archive.util.TextUtils;
 
 /**

engine/src/main/java/org/archive/crawler/frontier/AssignmentLevelSurtQueueAssignmentPolicy.java

@@ -18,7 +18,7 @@
  */
 package org.archive.crawler.frontier;
 
-import org.archive.crawler.datamodel.CrawlURI;
+import org.archive.modules.CrawlURI;
 import org.archive.net.PublicSuffixes;
 
 /**

engine/src/main/java/org/archive/crawler/frontier/BdbFrontier.java

@@ -46,7 +46,7 @@ import org.apache.commons.collections.Closure;
 import org.archive.bdb.BdbModule;
 import org.archive.checkpointing.Checkpointable;
 import org.archive.checkpointing.RecoverAction;
-import org.archive.crawler.datamodel.CrawlURI;
+import org.archive.modules.CrawlURI;
 import org.archive.queue.StoredQueue;
 import org.springframework.beans.factory.annotation.Autowired;
 

engine/src/main/java/org/archive/crawler/frontier/BdbMultipleWorkQueues.java

@@ -35,7 +35,7 @@ import javax.management.openmbean.CompositeDataSupport;
 import javax.management.openmbean.OpenDataException;
 
 import org.apache.commons.collections.Closure;
-import org.archive.crawler.datamodel.CrawlURI;
+import org.archive.modules.CrawlURI;
 import org.archive.util.ArchiveUtils;
 
 import com.sleepycat.bind.serial.StoredClassCatalog;

engine/src/main/java/org/archive/crawler/frontier/BdbWorkQueue.java

@@ -28,7 +28,7 @@ import java.io.UnsupportedEncodingException;
 import java.util.logging.Level;
 import java.util.logging.Logger;
 
-import org.archive.crawler.datamodel.CrawlURI;
+import org.archive.modules.CrawlURI;
 import org.archive.util.ArchiveUtils;
 import org.archive.util.IoUtils;
 

engine/src/main/java/org/archive/crawler/frontier/BucketQueueAssignmentPolicy.java

@@ -26,7 +26,7 @@
 package org.archive.crawler.frontier;
 
 import org.apache.commons.httpclient.URIException;
-import org.archive.crawler.datamodel.CrawlURI;
+import org.archive.modules.CrawlURI;
 import org.archive.modules.net.CrawlHost;
 import org.archive.modules.net.ServerCache;
 import org.springframework.beans.factory.annotation.Autowired;

engine/src/main/java/org/archive/crawler/frontier/CostAssignmentPolicy.java

@@ -26,7 +26,7 @@ package org.archive.crawler.frontier;
 
 import java.io.Serializable;
 
-import org.archive.crawler.datamodel.CrawlURI;
+import org.archive.modules.CrawlURI;
 
 /**
  * Calculate a integer 'cost' value for the given CrawlURI.

engine/src/main/java/org/archive/crawler/frontier/FrontierJournal.java

@@ -34,10 +34,10 @@ import java.util.logging.Logger;
 
 import org.apache.commons.httpclient.URIException;
 import org.archive.checkpointing.Checkpointable;
-import org.archive.crawler.datamodel.CrawlURI;
 import org.archive.crawler.framework.CrawlController;
 import org.archive.crawler.framework.Frontier;
 import org.archive.io.CrawlerJournal;
+import org.archive.modules.CrawlURI;
 import org.archive.modules.deciderules.DecideRule;
 import org.archive.net.UURI;
 import org.archive.net.UURIFactory;

engine/src/main/java/org/archive/crawler/frontier/IPQueueAssignmentPolicy.java

@@ -24,7 +24,7 @@
 */ 
 package org.archive.crawler.frontier;
 
-import org.archive.crawler.datamodel.CrawlURI;
+import org.archive.modules.CrawlURI;
 import org.archive.modules.net.CrawlHost;
 import org.archive.modules.net.ServerCache;
 import org.archive.modules.net.ServerCacheUtil;

engine/src/main/java/org/archive/crawler/frontier/QueueAssignmentPolicy.java

@@ -21,7 +21,7 @@ package org.archive.crawler.frontier;
 
 import java.io.Serializable;
 
-import org.archive.crawler.datamodel.CrawlURI;
+import org.archive.modules.CrawlURI;
 
 /**
  * Establishes a mapping from CrawlURIs to String keys (queue names).

engine/src/main/java/org/archive/crawler/frontier/URIAuthorityBasedQueueAssignmentPolicy.java

@@ -20,7 +20,7 @@
 package org.archive.crawler.frontier;
 
 import org.apache.commons.lang.StringUtils;
-import org.archive.crawler.datamodel.CrawlURI;
+import org.archive.modules.CrawlURI;
 import org.archive.modules.extractor.Hop;
 import org.archive.net.UURI;
 import org.archive.spring.HasKeyedProperties;

engine/src/main/java/org/archive/crawler/frontier/UnitCostAssignmentPolicy.java

@@ -24,7 +24,7 @@
 */ 
 package org.archive.crawler.frontier;
 
-import org.archive.crawler.datamodel.CrawlURI;
+import org.archive.modules.CrawlURI;
 
 /**
  * A CostAssignment policy that uses a constant value of 1 for all CrawlURIs.

engine/src/main/java/org/archive/crawler/frontier/WagCostAssignmentPolicy.java

@@ -24,7 +24,7 @@
 */ 
 package org.archive.crawler.frontier;
 
-import org.archive.crawler.datamodel.CrawlURI;
+import org.archive.modules.CrawlURI;
 import org.archive.net.UURI;
 
 /**
@@ -42,7 +42,7 @@ public class WagCostAssignmentPolicy extends CostAssignmentPolicy {
      * 
      * @param curi CrawlURI to be assigned a cost
      * 
-     * @see org.archive.crawler.frontier.CostAssignmentPolicy#costOf(org.archive.crawler.datamodel.CrawlURI)
+     * @see org.archive.crawler.frontier.CostAssignmentPolicy#costOf(org.archive.modules.CrawlURI)
      */
     public int costOf(CrawlURI curi) {
         int cost = 1;

engine/src/main/java/org/archive/crawler/frontier/WorkQueue.java

@@ -10,10 +10,10 @@ import java.util.concurrent.TimeUnit;
 import java.util.logging.Level;
 import java.util.logging.Logger;
 
-import org.archive.crawler.datamodel.CrawlURI;
 import org.archive.crawler.framework.Frontier;
 import org.archive.crawler.frontier.precedence.PrecedenceProvider;
 import org.archive.crawler.frontier.precedence.SimplePrecedenceProvider;
+import org.archive.modules.CrawlURI;
 import org.archive.modules.ProcessorURI;
 import org.archive.modules.fetcher.FetchStats;
 import org.archive.modules.fetcher.FetchStats.Stage;

engine/src/main/java/org/archive/crawler/frontier/WorkQueueFrontier.java

@@ -19,11 +19,11 @@
 
 package org.archive.crawler.frontier;
 
-import static org.archive.crawler.datamodel.CoreAttributeConstants.A_FORCE_RETIRE;
 import static org.archive.crawler.event.CrawlURIDispositionEvent.Disposition.DEFERRED_FOR_RETRY;
 import static org.archive.crawler.event.CrawlURIDispositionEvent.Disposition.DISREGARDED;
 import static org.archive.crawler.event.CrawlURIDispositionEvent.Disposition.FAILED;
 import static org.archive.crawler.event.CrawlURIDispositionEvent.Disposition.SUCCEEDED;
+import static org.archive.modules.CoreAttributeConstants.A_FORCE_RETIRE;
 import static org.archive.modules.fetcher.FetchStatusCodes.S_DEFERRED;
 import static org.archive.modules.fetcher.FetchStatusCodes.S_RUNTIME_EXCEPTION;
 
@@ -52,7 +52,6 @@ import java.util.regex.Pattern;
 import org.apache.commons.collections.Bag;
 import org.apache.commons.collections.BagUtils;
 import org.apache.commons.collections.bag.HashBag;
-import org.archive.crawler.datamodel.CrawlURI;
 import org.archive.crawler.datamodel.UriUniqFilter;
 import org.archive.crawler.datamodel.UriUniqFilter.CrawlUriReceiver;
 import org.archive.crawler.event.CrawlURIDispositionEvent;
@@ -61,6 +60,7 @@ import org.archive.crawler.frontier.precedence.BaseQueuePrecedencePolicy;
 import org.archive.crawler.frontier.precedence.CostUriPrecedencePolicy;
 import org.archive.crawler.frontier.precedence.QueuePrecedencePolicy;
 import org.archive.crawler.frontier.precedence.UriPrecedencePolicy;
+import org.archive.modules.CrawlURI;
 import org.archive.net.UURI;
 import org.archive.spring.KeyedProperties;
 import org.archive.util.ArchiveUtils;
@@ -354,7 +354,7 @@ ApplicationContextAware {
      * Arrange for the given CrawlURI to be visited, if it is not
      * already scheduled/completed.
      *
-     * @see org.archive.crawler.framework.Frontier#schedule(org.archive.crawler.datamodel.CrawlURI)
+     * @see org.archive.crawler.framework.Frontier#schedule(org.archive.modules.CrawlURI)
      */
     protected void processScheduleIfUnique(CrawlURI curi) {
         assert Thread.currentThread() == managerThread;
@@ -929,7 +929,7 @@ ApplicationContextAware {
      * via the next next() call, as a result of finished().
      *
      *  (non-Javadoc)
-     * @see org.archive.crawler.framework.Frontier#finished(org.archive.crawler.datamodel.CrawlURI)
+     * @see org.archive.crawler.framework.Frontier#finished(org.archive.modules.CrawlURI)
      */
     protected void processFinish(CrawlURI curi) {
         assert Thread.currentThread() == managerThread;
@@ -1489,7 +1489,7 @@ ApplicationContextAware {
     /**
      * Force logging, etc. of operator- deleted CrawlURIs
      * 
-     * @see org.archive.crawler.framework.Frontier#deleted(org.archive.crawler.datamodel.CrawlURI)
+     * @see org.archive.crawler.framework.Frontier#deleted(org.archive.modules.CrawlURI)
      */
     public void deleted(CrawlURI curi) {
         //treat as disregarded

engine/src/main/java/org/archive/crawler/frontier/ZeroCostAssignmentPolicy.java

@@ -6,7 +6,7 @@
  */
 package org.archive.crawler.frontier;
 
-import org.archive.crawler.datamodel.CrawlURI;
+import org.archive.modules.CrawlURI;
 
 /**
  * CostAssignmentPolicy considering all URIs costless -- essentially

engine/src/main/java/org/archive/crawler/frontier/precedence/BaseUriPrecedencePolicy.java

@@ -19,7 +19,7 @@
  
 package org.archive.crawler.frontier.precedence;
 
-import org.archive.crawler.datamodel.CrawlURI;
+import org.archive.modules.CrawlURI;
 import org.archive.spring.HasKeyedProperties;
 import org.archive.spring.KeyedProperties;
 

engine/src/main/java/org/archive/crawler/frontier/precedence/CostUriPrecedencePolicy.java

@@ -24,7 +24,7 @@
 */
 package org.archive.crawler.frontier.precedence;
 
-import org.archive.crawler.datamodel.CrawlURI;
+import org.archive.modules.CrawlURI;
 
 /**
  * UriPrecedencePolicy which sets a URI's precedence to its 'cost' -- which

engine/src/main/java/org/archive/crawler/frontier/precedence/HighestUriQueuePrecedencePolicy.java

@@ -26,8 +26,8 @@ package org.archive.crawler.frontier.precedence;
 
 import java.io.PrintWriter;
 
-import org.archive.crawler.datamodel.CrawlURI;
 import org.archive.crawler.frontier.WorkQueue;
+import org.archive.modules.CrawlURI;
 import org.archive.modules.ProcessorURI;
 import org.archive.modules.fetcher.FetchStats.Stage;
 import org.archive.util.Histotable;

engine/src/main/java/org/archive/crawler/frontier/precedence/HopsUriPrecedencePolicy.java

@@ -24,7 +24,7 @@
 */
 package org.archive.crawler.frontier.precedence;
 
-import org.archive.crawler.datamodel.CrawlURI;
+import org.archive.modules.CrawlURI;
 
 /**
  * UriPrecedencePolicy which assigns URIs a precedence equal to the number

engine/src/main/java/org/archive/crawler/frontier/precedence/PrecedenceLoader.java

@@ -31,7 +31,8 @@ import java.util.HashMap;
 import java.util.Iterator;
 import java.util.Map;
 
-import static org.archive.crawler.datamodel.CoreAttributeConstants.*;
+import static org.archive.modules.CoreAttributeConstants.*;
+
 import org.archive.io.CrawlerJournal;
 import org.archive.modules.recrawl.PersistProcessor;
 import org.archive.util.bdbje.EnhancedEnvironment;

engine/src/main/java/org/archive/crawler/frontier/precedence/PreloadedUriPrecedencePolicy.java

@@ -24,12 +24,12 @@
 */
 package org.archive.crawler.frontier.precedence;
 
-import static org.archive.crawler.datamodel.CoreAttributeConstants.A_PRECALC_PRECEDENCE;
+import static org.archive.modules.CoreAttributeConstants.A_PRECALC_PRECEDENCE;
 
 import java.util.Map;
 
 import org.archive.bdb.BdbModule;
-import org.archive.crawler.datamodel.CrawlURI;
+import org.archive.modules.CrawlURI;
 import org.archive.modules.recrawl.PersistProcessor;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.context.Lifecycle;

engine/src/main/java/org/archive/crawler/frontier/precedence/UriPrecedencePolicy.java

@@ -26,7 +26,7 @@ package org.archive.crawler.frontier.precedence;
 
 import java.io.Serializable;
 
-import org.archive.crawler.datamodel.CrawlURI;
+import org.archive.modules.CrawlURI;
 
 /**
  * Superclass for URI precedence policies, which set a integer 

engine/src/main/java/org/archive/crawler/io/NonFatalErrorFormatter.java

@@ -27,7 +27,7 @@ import java.io.PrintWriter;
 import java.io.StringWriter;
 import java.util.logging.LogRecord;
 
-import org.archive.crawler.datamodel.CoreAttributeConstants;
+import org.archive.modules.CoreAttributeConstants;
 
 /**
  * @author gojomo

engine/src/main/java/org/archive/crawler/io/RuntimeErrorFormatter.java

@@ -28,8 +28,8 @@ import java.io.PrintWriter;
 import java.io.StringWriter;
 import java.util.logging.LogRecord;
 
-import org.archive.crawler.datamodel.CoreAttributeConstants;
-import org.archive.crawler.datamodel.CrawlURI;
+import org.archive.modules.CoreAttributeConstants;
+import org.archive.modules.CrawlURI;
 
 /**
  * Runtime exception log formatter.

engine/src/main/java/org/archive/crawler/io/UriErrorFormatter.java

@@ -26,7 +26,7 @@ package org.archive.crawler.io;
 import java.util.logging.Formatter;
 import java.util.logging.LogRecord;
 
-import org.archive.crawler.datamodel.CoreAttributeConstants;
+import org.archive.modules.CoreAttributeConstants;
 import org.archive.net.UURI;
 import org.archive.util.ArchiveUtils;
 

engine/src/main/java/org/archive/crawler/io/UriProcessingFormatter.java

@@ -31,8 +31,8 @@ import java.util.Iterator;
 import java.util.logging.Formatter;
 import java.util.logging.LogRecord;
 
-import org.archive.crawler.datamodel.CoreAttributeConstants;
-import org.archive.crawler.datamodel.CrawlURI;
+import org.archive.modules.CoreAttributeConstants;
+import org.archive.modules.CrawlURI;
 import org.archive.util.ArchiveUtils;
 import org.archive.util.MimetypeUtils;
 

engine/src/main/java/org/archive/crawler/postprocessor/CrawlStateUpdater.java

@@ -28,7 +28,7 @@ import static org.archive.modules.fetcher.FetchStatusCodes.S_CONNECT_FAILED;
 import java.util.logging.Logger;
 
 import org.apache.commons.httpclient.URIException;
-import org.archive.crawler.datamodel.CrawlURI;
+import org.archive.modules.CrawlURI;
 import org.archive.modules.PostProcessor;
 import org.archive.modules.Processor;
 import org.archive.modules.ProcessorURI;

engine/src/main/java/org/archive/crawler/postprocessor/FrontierScheduler.java

@@ -30,8 +30,8 @@ import static org.archive.modules.fetcher.FetchStatusCodes.S_DEFERRED;
 
 import java.util.concurrent.locks.ReentrantLock;
 
-import org.archive.crawler.datamodel.CrawlURI;
 import org.archive.crawler.framework.Frontier;
+import org.archive.modules.CrawlURI;
 import org.archive.modules.PostProcessor;
 import org.archive.modules.Processor;
 import org.archive.modules.ProcessorURI;

engine/src/main/java/org/archive/crawler/postprocessor/LinksScoper.java

@@ -25,17 +25,17 @@
  */
 package org.archive.crawler.postprocessor;
 
-import static org.archive.crawler.datamodel.SchedulingConstants.HIGH;
-import static org.archive.crawler.datamodel.SchedulingConstants.MEDIUM;
-import static org.archive.crawler.datamodel.SchedulingConstants.NORMAL;
+import static org.archive.modules.SchedulingConstants.HIGH;
+import static org.archive.modules.SchedulingConstants.MEDIUM;
+import static org.archive.modules.SchedulingConstants.NORMAL;
 import static org.archive.modules.fetcher.FetchStatusCodes.S_PREREQUISITE_UNSCHEDULABLE_FAILURE;
 
 import java.util.logging.Level;
 import java.util.logging.Logger;
 
 import org.apache.commons.httpclient.URIException;
-import org.archive.crawler.datamodel.CrawlURI;
 import org.archive.crawler.framework.Scoper;
+import org.archive.modules.CrawlURI;
 import org.archive.modules.PostProcessor;
 import org.archive.modules.ProcessorURI;
 import org.archive.modules.deciderules.DecideResult;

engine/src/main/java/org/archive/crawler/postprocessor/SupplementaryLinksScoper.java

@@ -29,8 +29,8 @@ import java.util.Iterator;
 import java.util.logging.Level;
 import java.util.logging.Logger;
 
-import org.archive.crawler.datamodel.CrawlURI;
 import org.archive.crawler.framework.Scoper;
+import org.archive.modules.CrawlURI;
 import org.archive.modules.PostProcessor;
 import org.archive.modules.ProcessorURI;
 import org.archive.modules.deciderules.AcceptDecideRule;

engine/src/main/java/org/archive/crawler/prefetch/PreconditionEnforcer.java

@@ -34,8 +34,8 @@ import java.util.logging.Level;
 import java.util.logging.Logger;
 
 import org.apache.commons.httpclient.URIException;
-import org.archive.crawler.datamodel.CrawlURI;
 import org.archive.crawler.reporting.CrawlerLoggerModule;
+import org.archive.modules.CrawlURI;
 import org.archive.modules.ProcessResult;
 import org.archive.modules.Processor;
 import org.archive.modules.ProcessorURI;

engine/src/main/java/org/archive/crawler/prefetch/Preselector.java

@@ -24,11 +24,11 @@
 package org.archive.crawler.prefetch;
 
 
-import org.archive.crawler.datamodel.CrawlURI;
 
 import static org.archive.modules.fetcher.FetchStatusCodes.*;
 
 import org.archive.crawler.framework.Scoper;
+import org.archive.modules.CrawlURI;
 import org.archive.modules.ProcessResult;
 import org.archive.modules.ProcessorURI;
 import org.archive.util.TextUtils;

engine/src/main/java/org/archive/crawler/prefetch/QuotaEnforcer.java

@@ -27,8 +27,8 @@ import static org.archive.modules.fetcher.FetchStatusCodes.S_BLOCKED_BY_QUOTA;
 import java.util.logging.Level;
 import java.util.logging.Logger;
 
-import org.archive.crawler.datamodel.CrawlURI;
 import org.archive.crawler.framework.Frontier;
+import org.archive.modules.CrawlURI;
 import org.archive.modules.ProcessResult;
 import org.archive.modules.Processor;
 import org.archive.modules.ProcessorURI;

engine/src/main/java/org/archive/crawler/prefetch/RuntimeLimitEnforcer.java

@@ -26,13 +26,13 @@ package org.archive.crawler.prefetch;
 import java.util.logging.Level;
 import java.util.logging.Logger;
 
-import org.archive.crawler.datamodel.CrawlURI;
 
 import static org.archive.modules.fetcher.FetchStatusCodes.*;
 
 import org.archive.crawler.framework.CrawlController;
 import org.archive.crawler.framework.CrawlStatus;
 import org.archive.crawler.reporting.StatisticsTracker;
+import org.archive.modules.CrawlURI;
 import org.archive.modules.ProcessResult;
 import org.archive.modules.Processor;
 import org.archive.modules.ProcessorURI;

engine/src/main/java/org/archive/crawler/processor/CrawlMapper.java

@@ -32,7 +32,7 @@ import java.io.PrintWriter;
 import java.util.HashMap;
 import java.util.Iterator;
 
-import org.archive.crawler.datamodel.CrawlURI;
+import org.archive.modules.CrawlURI;
 import org.archive.modules.ProcessResult;
 import org.archive.modules.Processor;
 import org.archive.modules.ProcessorURI;

engine/src/main/java/org/archive/crawler/processor/HashCrawlMapper.java

@@ -24,8 +24,8 @@ package org.archive.crawler.processor;
 
 import java.util.regex.Matcher;
 
-import org.archive.crawler.datamodel.CrawlURI;
 import org.archive.crawler.framework.Frontier;
+import org.archive.modules.CrawlURI;
 import org.archive.net.PublicSuffixes;
 import org.archive.util.TextUtils;
 import org.springframework.beans.factory.annotation.Autowired;

engine/src/main/java/org/archive/crawler/processor/LexicalCrawlMapper.java

@@ -34,8 +34,8 @@ import java.util.Iterator;
 import java.util.SortedMap;
 import java.util.TreeMap;
 
-import org.archive.crawler.datamodel.CrawlURI;
 import org.archive.crawler.framework.Frontier;
+import org.archive.modules.CrawlURI;
 import org.archive.spring.ConfigPath;
 import org.archive.util.iterator.LineReadingIterator;
 import org.archive.util.iterator.RegexpLineIterator;

engine/src/main/java/org/archive/crawler/reporting/SeedRecord.java

@@ -22,8 +22,8 @@ package org.archive.crawler.reporting;
 import java.io.Serializable;
 import java.util.logging.Logger;
 
-import org.archive.crawler.datamodel.CrawlURI;
-import org.archive.crawler.datamodel.CoreAttributeConstants;
+import org.archive.modules.CoreAttributeConstants;
+import org.archive.modules.CrawlURI;
 
 /**
  * Record of all interesting info about the most-recent

engine/src/main/java/org/archive/crawler/reporting/StatisticsTracker.java

@@ -19,7 +19,7 @@
 
 package org.archive.crawler.reporting;
 
-import static org.archive.crawler.datamodel.CoreAttributeConstants.A_SOURCE_TAG;
+import static org.archive.modules.CoreAttributeConstants.A_SOURCE_TAG;
 
 import java.io.File;
 import java.io.FileWriter;
@@ -46,7 +46,6 @@ import java.util.logging.Level;
 import java.util.logging.Logger;
 
 import org.archive.bdb.BdbModule;
-import org.archive.crawler.datamodel.CrawlURI;
 import org.archive.crawler.event.CrawlStateEvent;
 import org.archive.crawler.event.CrawlURIDispositionEvent;
 import org.archive.crawler.event.StatSnapshotEvent;
@@ -54,6 +53,7 @@ import org.archive.crawler.framework.CrawlController;
 import org.archive.crawler.framework.Engine;
 import org.archive.crawler.util.CrawledBytesHistotable;
 import org.archive.crawler.util.TopNSet;
+import org.archive.modules.CrawlURI;
 import org.archive.modules.net.ServerCache;
 import org.archive.modules.net.ServerCacheUtil;
 import org.archive.modules.seeds.SeedModule;

engine/src/main/java/org/archive/crawler/spring/SheetOverlaysManager.java

@@ -30,7 +30,7 @@ import java.util.TreeMap;
 import java.util.TreeSet;
 import java.util.logging.Logger;
 
-import org.archive.crawler.datamodel.CrawlURI;
+import org.archive.modules.CrawlURI;
 import org.archive.spring.OverlayMapsSource;
 import org.archive.spring.Sheet;
 import org.archive.util.PrefixFinder;

engine/src/main/java/org/archive/crawler/util/BenchmarkUriUniqFilters.java

@@ -30,8 +30,8 @@ import java.io.FileReader;
 import java.io.FileWriter;
 import java.io.IOException;
 
-import org.archive.crawler.datamodel.CrawlURI;
 import org.archive.crawler.datamodel.UriUniqFilter;
+import org.archive.modules.CrawlURI;
 import org.archive.util.fingerprint.MemLongFPSet;
 
 

engine/src/main/java/org/archive/crawler/util/BloomUriUniqFilter.java

@@ -27,7 +27,7 @@ package org.archive.crawler.util;
 import java.io.Serializable;
 import java.util.logging.Logger;
 
-import org.archive.crawler.datamodel.CrawlURI;
+import org.archive.modules.CrawlURI;
 import org.archive.util.BloomFilter;
 import org.archive.util.BloomFilter32bitSplit;
 

engine/src/main/java/org/archive/crawler/util/CrawledBytesHistotable.java

@@ -20,8 +20,8 @@
 package org.archive.crawler.util;
 
 import org.apache.commons.httpclient.HttpStatus;
-import org.archive.crawler.datamodel.CoreAttributeConstants;
-import org.archive.crawler.datamodel.CrawlURI;
+import org.archive.modules.CoreAttributeConstants;
+import org.archive.modules.CrawlURI;
 import org.archive.modules.deciderules.recrawl.IdenticalDigestDecideRule;
 import org.archive.util.ArchiveUtils;
 import org.archive.util.Histotable;

engine/src/main/java/org/archive/crawler/util/FPMergeUriUniqFilter.java

@@ -36,8 +36,8 @@ import java.util.TreeSet;
 import java.util.logging.Level;
 import java.util.logging.Logger;
 
-import org.archive.crawler.datamodel.CrawlURI;
 import org.archive.crawler.datamodel.UriUniqFilter;
+import org.archive.modules.CrawlURI;
 import org.archive.util.fingerprint.ArrayLongFPCache;
 
 import st.ata.util.FPGenerator;

engine/src/main/java/org/archive/crawler/util/SetBasedUriUniqFilter.java

@@ -33,8 +33,8 @@ import java.io.Serializable;
 import java.util.logging.Level;
 import java.util.logging.Logger;
 
-import org.archive.crawler.datamodel.CrawlURI;
 import org.archive.crawler.datamodel.UriUniqFilter;
+import org.archive.modules.CrawlURI;
 
 /**
  * UriUniqFilter based on an underlying UriSet (essentially a Set).

engine/src/test/java/org/archive/crawler/datamodel/CandidateURITest.java

@@ -32,6 +32,7 @@ import java.io.ObjectOutputStream;
 
 import junit.framework.TestCase;
 
+import org.archive.modules.CrawlURI;
 import org.archive.net.UURIFactory;
 
 /**

engine/src/test/java/org/archive/crawler/datamodel/CrawlURITest.java

@@ -30,6 +30,8 @@ import java.io.ObjectInputStream;
 import java.io.ObjectOutputStream;
 
 import org.apache.commons.httpclient.URIException;
+import org.archive.modules.CrawlURI;
+import org.archive.modules.SchedulingConstants;
 import org.archive.net.UURI;
 import org.archive.net.UURIFactory;
 import org.archive.util.TmpDirTestCase;

engine/src/test/java/org/archive/crawler/frontier/BdbMultipleWorkQueuesTest.java

@@ -27,8 +27,8 @@ package org.archive.crawler.frontier;
 import java.util.logging.Logger;
 
 import org.apache.commons.httpclient.URIException;
-import org.archive.crawler.datamodel.CrawlURI;
-import org.archive.crawler.datamodel.SchedulingConstants;
+import org.archive.modules.CrawlURI;
+import org.archive.modules.SchedulingConstants;
 import org.archive.net.UURIFactory;
 
 import com.sleepycat.je.tree.Key;

engine/src/test/java/org/archive/crawler/selftest/KeyWordProcessor.java

@@ -2,7 +2,7 @@ package org.archive.crawler.selftest;
 
 import java.util.regex.Pattern;
 
-import org.archive.crawler.datamodel.CrawlURI;
+import org.archive.modules.CrawlURI;
 import org.archive.modules.Processor;
 import org.archive.modules.ProcessorURI;
 

engine/src/test/java/org/archive/crawler/selftest/KeyWordUriPrecedencePolicy.java

@@ -19,8 +19,8 @@
 
 package org.archive.crawler.selftest;
 
-import org.archive.crawler.datamodel.CrawlURI;
 import org.archive.crawler.frontier.precedence.BaseUriPrecedencePolicy;
+import org.archive.modules.CrawlURI;
 
 /**
  * Testing policy which uses a precedence inside the CrawlURI (presumably

engine/src/test/java/org/archive/crawler/util/BdbUriUniqFilterTest.java

@@ -30,8 +30,8 @@ import junit.framework.TestSuite;
 
 import org.apache.commons.httpclient.URIException;
 import org.apache.commons.io.FileUtils;
-import org.archive.crawler.datamodel.CrawlURI;
 import org.archive.crawler.datamodel.UriUniqFilter;
+import org.archive.modules.CrawlURI;
 import org.archive.net.UURI;
 import org.archive.net.UURIFactory;
 import org.archive.util.TmpDirTestCase;

engine/src/test/java/org/archive/crawler/util/BloomUriUniqFilterTest.java

@@ -28,8 +28,8 @@ import java.util.logging.Logger;
 import junit.framework.TestCase;
 
 import org.apache.commons.httpclient.URIException;
-import org.archive.crawler.datamodel.CrawlURI;
 import org.archive.crawler.datamodel.UriUniqFilter;
+import org.archive.modules.CrawlURI;
 import org.archive.net.UURI;
 import org.archive.net.UURIFactory;
 

engine/src/test/java/org/archive/crawler/util/FPUriUniqFilterTest.java

@@ -28,8 +28,8 @@ import java.util.logging.Logger;
 import junit.framework.TestCase;
 
 import org.apache.commons.httpclient.URIException;
-import org.archive.crawler.datamodel.CrawlURI;
 import org.archive.crawler.datamodel.UriUniqFilter;
+import org.archive.modules.CrawlURI;
 import org.archive.net.UURI;
 import org.archive.net.UURIFactory;
 import org.archive.util.fingerprint.MemLongFPSet;

modules/src/main/java/org/archive/modules/CoreAttributeConstants.java

@@ -23,9 +23,8 @@
  * @author gojomo
  *
  */
-package org.archive.crawler.datamodel;
+package org.archive.modules;
 
-import org.archive.modules.ModuleAttributeConstants;
 
 /**
  * CrawlURI attribute keys used by the core crawler

modules/src/main/java/org/archive/modules/CrawlURI.java

@@ -17,19 +17,19 @@
  *  limitations under the License.
  */
  
-package org.archive.crawler.datamodel;
+package org.archive.modules;
 
-import static org.archive.crawler.datamodel.CoreAttributeConstants.A_ANNOTATIONS;
-import static org.archive.crawler.datamodel.CoreAttributeConstants.A_CREDENTIAL_AVATARS_KEY;
-import static org.archive.crawler.datamodel.CoreAttributeConstants.A_DNS_SERVER_IP_LABEL;
-import static org.archive.crawler.datamodel.CoreAttributeConstants.A_FETCH_COMPLETED_TIME;
-import static org.archive.crawler.datamodel.CoreAttributeConstants.A_FORCE_RETIRE;
-import static org.archive.crawler.datamodel.CoreAttributeConstants.A_HERITABLE_KEYS;
-import static org.archive.crawler.datamodel.CoreAttributeConstants.A_HTML_BASE;
-import static org.archive.crawler.datamodel.CoreAttributeConstants.A_NONFATAL_ERRORS;
-import static org.archive.crawler.datamodel.CoreAttributeConstants.A_PREREQUISITE_URI;
-import static org.archive.crawler.datamodel.CoreAttributeConstants.A_SOURCE_TAG;
-import static org.archive.crawler.datamodel.SchedulingConstants.NORMAL;
+import static org.archive.modules.CoreAttributeConstants.A_ANNOTATIONS;
+import static org.archive.modules.CoreAttributeConstants.A_CREDENTIAL_AVATARS_KEY;
+import static org.archive.modules.CoreAttributeConstants.A_DNS_SERVER_IP_LABEL;
+import static org.archive.modules.CoreAttributeConstants.A_FETCH_COMPLETED_TIME;
+import static org.archive.modules.CoreAttributeConstants.A_FORCE_RETIRE;
+import static org.archive.modules.CoreAttributeConstants.A_HERITABLE_KEYS;
+import static org.archive.modules.CoreAttributeConstants.A_HTML_BASE;
+import static org.archive.modules.CoreAttributeConstants.A_NONFATAL_ERRORS;
+import static org.archive.modules.CoreAttributeConstants.A_PREREQUISITE_URI;
+import static org.archive.modules.CoreAttributeConstants.A_SOURCE_TAG;
+import static org.archive.modules.SchedulingConstants.NORMAL;
 import static org.archive.modules.fetcher.FetchStatusCodes.S_BLOCKED_BY_CUSTOM_PROCESSOR;
 import static org.archive.modules.fetcher.FetchStatusCodes.S_BLOCKED_BY_USER;
 import static org.archive.modules.fetcher.FetchStatusCodes.S_CONNECT_FAILED;
@@ -76,8 +76,6 @@ import org.apache.commons.httpclient.HttpStatus;
 import org.apache.commons.httpclient.URIException;
 import org.apache.commons.httpclient.methods.GetMethod;
 import org.apache.commons.httpclient.methods.PostMethod;
-import org.archive.modules.ModuleAttributeConstants;
-import org.archive.modules.ProcessorURI;
 import org.archive.modules.credential.CredentialAvatar;
 import org.archive.modules.credential.HttpAuthenticationCredential;
 import org.archive.modules.extractor.HTMLLinkContext;

modules/src/main/java/org/archive/modules/SchedulingConstants.java

@@ -0,0 +1,52 @@
+/*
+ *  This file is part of the Heritrix web crawler (crawler.archive.org).
+ *
+ *  Licensed to the Internet Archive (IA) by one or more individual 
+ *  contributors. 
+ *
+ *  The IA licenses this file to You under the Apache License, Version 2.0
+ *  (the "License"); you may not use this file except in compliance with
+ *  the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ */
+
+package org.archive.modules;
+
+/**
+ * @author pjack
+ *
+ */
+public class SchedulingConstants {
+
+    
+    /** Highest scheduling priority.
+     * Before any others of its class.
+     */
+    public static final int HIGHEST = 0;
+    
+    /** High scheduling priority.
+     * After any {@link #HIGHEST}.
+     */
+    public static final int HIGH = 1;
+    
+    /** Medium priority.
+     * After any {@link #HIGH}.
+     */
+    public static final int MEDIUM = 2;
+    
+    /** Normal/low priority.
+     * Whenever/end of queue.
+     */
+    public static final int NORMAL = 3;
+
+    
+    private SchedulingConstants() {
+    }
+}