Push CrawlURI and related constants down to modules subproject/package

(many)

engine/src/main/java/org/archive/crawler/datamodel/CoreAttributeConstants.java

@@ -1,123 +0,0 @@
-/* Copyright (C) 2003 Internet Archive.
- *
- * This file is part of the Heritrix web crawler (crawler.archive.org).
- *
- * Heritrix is free software; you can redistribute it and/or modify
- * it under the terms of the GNU Lesser Public License as published by
- * the Free Software Foundation; either version 2.1 of the License, or
- * any later version.
- *
- * Heritrix is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU Lesser Public License for more details.
- *
- * You should have received a copy of the GNU Lesser Public License
- * along with Heritrix; if not, write to the Free Software
- * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
- *
- * CoreAttributeConstants.java
- * Created on Jun 17, 2003
- *
- * $Header: /cvsroot/archive-crawler/ArchiveOpenCrawler/src//**
- * @author gojomo
- *
- */
-package org.archive.crawler.datamodel;
-
-import org.archive.modules.ModuleAttributeConstants;
-
-/**
- * CrawlURI attribute keys used by the core crawler
- * classes.
- *
- * @author gojomo
- *
- */
-public interface CoreAttributeConstants {
-
-    /**
-     * Extracted MIME type of fetched content; should be
-     * set immediately by fetching module if possible
-     * (rather than waiting for a later analyzer)
-     */
-    public static String A_CONTENT_TYPE = "content-type";
-
-    /**
-     * Multiplier of last fetch duration to wait before
-     * fetching another item of the same class (eg host)
-     */
-    public static String A_DELAY_FACTOR = "delay-factor";
-    /**
-     * Minimum delay before fetching another item of th
-     * same class (eg host). Even if lastFetchTime*delayFactor
-     * is less than this, this period will be waited.
-     */
-    public static String A_MINIMUM_DELAY = "minimum-delay";
-
-    public static String A_RRECORD_SET_LABEL = "dns-records";
-    public static String A_DNS_FETCH_TIME    = "dns-fetch-time";
-    public static String A_DNS_SERVER_IP_LABEL = ModuleAttributeConstants.A_DNS_SERVER_IP_LABEL;
-    public static String A_FETCH_COMPLETED_TIME = "fetch-completed-time";
-    public static String A_HTTP_TRANSACTION = ModuleAttributeConstants.A_HTTP_TRANSACTION;
-
-    public static String A_RUNTIME_EXCEPTION = "runtime-exception";
-    public static String A_NONFATAL_ERRORS = "nonfatal-errors";
-
-    /** shorthand string tokens indicating notable occurences,
-     * separated by commas */
-    public static String A_ANNOTATIONS = "annotations";
-
-    public static String A_PREREQUISITE_URI = "prerequisite-uri";
-    public static String A_DISTANCE_FROM_SEED = "distance-from-seed";
-    public static String A_HTML_BASE = "html-base-href";
-    public static String A_RETRY_DELAY = "retry-delay";
-
-    /** 
-     * Define for org.archive.crawler.writer.MirrorWriterProcessor.
-     */
-    public static String A_MIRROR_PATH = "mirror-path";
-
-    /**
-     * Key to get credential avatars from A_LIST.
-     */
-    public static final String A_CREDENTIAL_AVATARS_KEY =
-        "credential-avatars";
-    
-    /** a 'source' (usu. URI) that's inherited by discovered URIs */
-    public static String A_SOURCE_TAG = ModuleAttributeConstants.A_SOURCE_TAG;
-    
-    /**
-     * Key to (optional) attribute specifying a list of keys that
-     * are passed to CandidateURIs that 'descend' (are discovered 
-     * via) this URI. 
-     */
-    public static final String A_HERITABLE_KEYS = "heritable";
-    
-    /** flag indicating the containing queue should be retired */ 
-    public static final String A_FORCE_RETIRE = "force-retire";
-    
-    /** key to atribute containing precalculated precedence */
-    public static final String A_PRECALC_PRECEDENCE = "precalc-precedence";
-    
-    /** local override of proxy host */ 
-    public static final String A_HTTP_PROXY_HOST = "http-proxy-host";
-    /** local override of proxy port */ 
-    public static final String A_HTTP_PROXY_PORT = "http-proxy-port";
-
-    /**
-     * Fetch truncation codes present in {@link CrawlURI} annotations.
-     * All truncation annotations have a <code>TRUNC_SUFFIX</code> suffix (TODO:
-     * Make for-sure unique or redo truncation so definitive flag marked
-     * against {@link CrawlURI}).
-     */
-    public static final String TRUNC_SUFFIX = ModuleAttributeConstants.TRUNC_SUFFIX;
-    // headerTrunc
-    public static final String HEADER_TRUNC = ModuleAttributeConstants.HEADER_TRUNC; 
-    // timeTrunc
-    public static final String TIMER_TRUNC = ModuleAttributeConstants.TIMER_TRUNC;
-    // lenTrunc
-    public static final String LENGTH_TRUNC = ModuleAttributeConstants.LENGTH_TRUNC;
-
-
-}

engine/src/main/java/org/archive/crawler/datamodel/SchedulingConstants.java

@@ -1,59 +0,0 @@
-/* 
- * Copyright (C) 2007 Internet Archive.
- *
- * This file is part of the Heritrix web crawler (crawler.archive.org).
- *
- * Heritrix is free software; you can redistribute it and/or modify
- * it under the terms of the GNU Lesser Public License as published by
- * the Free Software Foundation; either version 2.1 of the License, or
- * any later version.
- *
- * Heritrix is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU Lesser Public License for more details.
- *
- * You should have received a copy of the GNU Lesser Public License
- * along with Heritrix; if not, write to the Free Software
- * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
- *
- * SchedulingConstants.java
- *
- * Created on Mar 23, 2007
- *
- * $Id:$
- */
-
-package org.archive.crawler.datamodel;
-
-/**
- * @author pjack
- *
- */
-public class SchedulingConstants {
-
-    
-    /** Highest scheduling priority.
-     * Before any others of its class.
-     */
-    public static final int HIGHEST = 0;
-    
-    /** High scheduling priority.
-     * After any {@link #HIGHEST}.
-     */
-    public static final int HIGH = 1;
-    
-    /** Medium priority.
-     * After any {@link #HIGH}.
-     */
-    public static final int MEDIUM = 2;
-    
-    /** Normal/low priority.
-     * Whenever/end of queue.
-     */
-    public static final int NORMAL = 3;
-
-    
-    private SchedulingConstants() {
-    }
-}

engine/src/main/java/org/archive/crawler/datamodel/UriUniqFilter.java

@@ -24,6 +24,8 @@ package org.archive.crawler.datamodel;
 
 import java.io.File;
 
+import org.archive.modules.CrawlURI;
+
 /**
  * A UriUniqFilter passes URI objects to a destination
  * (receiver) if the passed URI object has not been previously seen.

engine/src/main/java/org/archive/crawler/deciderules/ClassKeyMatchesRegExpDecideRule.java

@@ -25,8 +25,8 @@
 package org.archive.crawler.deciderules;
 
 
-import org.archive.crawler.datamodel.CrawlURI;
 import org.archive.crawler.framework.CrawlController;
+import org.archive.modules.CrawlURI;
 import org.archive.modules.ProcessorURI;
 import org.archive.modules.deciderules.MatchesRegExpDecideRule;
 

engine/src/main/java/org/archive/crawler/event/CrawlURIDispositionEvent.java

@@ -18,7 +18,7 @@
  */
 package org.archive.crawler.event;
 
-import org.archive.crawler.datamodel.CrawlURI;
+import org.archive.modules.CrawlURI;
 import org.springframework.context.ApplicationEvent;
 
 public class CrawlURIDispositionEvent extends ApplicationEvent {

engine/src/main/java/org/archive/crawler/framework/Frontier.java

@@ -28,8 +28,8 @@ import java.io.IOException;
 
 import javax.management.openmbean.CompositeData;
 
-import org.archive.crawler.datamodel.CrawlURI;
 import org.archive.crawler.frontier.FrontierJournal;
+import org.archive.modules.CrawlURI;
 import org.archive.modules.deciderules.DecideRule;
 import org.archive.modules.fetcher.FetchStats;
 import org.archive.net.UURI;

engine/src/main/java/org/archive/crawler/framework/Scoper.java

@@ -26,9 +26,9 @@ import java.util.logging.FileHandler;
 import java.util.logging.Level;
 import java.util.logging.Logger;
 
-import org.archive.crawler.datamodel.CrawlURI;
 import org.archive.crawler.reporting.CrawlerLoggerModule;
 import org.archive.crawler.util.LogUtils;
+import org.archive.modules.CrawlURI;
 import org.archive.modules.Processor;
 import org.archive.modules.deciderules.DecideResult;
 import org.archive.modules.deciderules.DecideRule;

engine/src/main/java/org/archive/crawler/framework/ToeThread.java

@@ -19,7 +19,7 @@
 
 package org.archive.crawler.framework;
 
-import static org.archive.crawler.datamodel.CoreAttributeConstants.A_RUNTIME_EXCEPTION;
+import static org.archive.modules.CoreAttributeConstants.A_RUNTIME_EXCEPTION;
 import static org.archive.modules.fetcher.FetchStatusCodes.S_PROCESSING_THREAD_KILLED;
 import static org.archive.modules.fetcher.FetchStatusCodes.S_RUNTIME_EXCEPTION;
 import static org.archive.modules.fetcher.FetchStatusCodes.S_SERIOUS_ERROR;
@@ -33,8 +33,8 @@ import java.util.Iterator;
 import java.util.logging.Level;
 import java.util.logging.Logger;
 
-import org.archive.crawler.datamodel.CrawlURI;
 import org.archive.io.SinkHandlerLogThread;
+import org.archive.modules.CrawlURI;
 import org.archive.modules.PostProcessor;
 import org.archive.modules.ProcessResult;
 import org.archive.modules.Processor;

engine/src/main/java/org/archive/crawler/frontier/AbstractFrontier.java

@@ -19,8 +19,8 @@
 
 package org.archive.crawler.frontier;
 
-import static org.archive.crawler.datamodel.CoreAttributeConstants.A_FETCH_COMPLETED_TIME;
-import static org.archive.crawler.datamodel.CoreAttributeConstants.A_NONFATAL_ERRORS;
+import static org.archive.modules.CoreAttributeConstants.A_FETCH_COMPLETED_TIME;
+import static org.archive.modules.CoreAttributeConstants.A_NONFATAL_ERRORS;
 import static org.archive.modules.fetcher.FetchStatusCodes.S_BLOCKED_BY_CUSTOM_PROCESSOR;
 import static org.archive.modules.fetcher.FetchStatusCodes.S_BLOCKED_BY_USER;
 import static org.archive.modules.fetcher.FetchStatusCodes.S_CONNECT_FAILED;
@@ -60,15 +60,15 @@ import org.apache.commons.httpclient.HttpStatus;
 import org.apache.commons.httpclient.URIException;
 import org.apache.commons.lang.StringUtils;
 import org.archive.checkpointing.CheckpointRecovery;
-import org.archive.crawler.datamodel.CrawlURI;
-import org.archive.crawler.datamodel.SchedulingConstants;
 import org.archive.crawler.event.CrawlStateEvent;
 import org.archive.crawler.framework.CrawlController;
 import org.archive.crawler.framework.Frontier;
 import org.archive.crawler.reporting.CrawlerLoggerModule;
 import org.archive.crawler.spring.SheetOverlaysManager;
+import org.archive.modules.CrawlURI;
 import org.archive.modules.ModuleAttributeConstants;
 import org.archive.modules.ProcessorURI;
+import org.archive.modules.SchedulingConstants;
 import org.archive.modules.canonicalize.UriCanonicalizationPolicy;
 import org.archive.modules.deciderules.DecideRule;
 import org.archive.modules.extractor.ExtractorParameters;
@@ -730,7 +730,7 @@ public abstract class AbstractFrontier
      * Arrange for the given CrawlURI to be visited, if it is not
      * already scheduled/completed.
      *
-     * @see org.archive.crawler.framework.Frontier#schedule(org.archive.crawler.datamodel.CrawlURI)
+     * @see org.archive.crawler.framework.Frontier#schedule(org.archive.modules.CrawlURI)
      */
     public void schedule(CrawlURI curi) {
         applyOverridesTo(curi);
@@ -761,7 +761,7 @@ public abstract class AbstractFrontier
      * via the next next() call, as a result of finished().
      *
      *  (non-Javadoc)
-     * @see org.archive.crawler.framework.Frontier#finished(org.archive.crawler.datamodel.CrawlURI)
+     * @see org.archive.crawler.framework.Frontier#finished(org.archive.modules.CrawlURI)
      */
     public void finished(CrawlURI curi) {
         enqueueOrDo(new Finish(curi));

engine/src/main/java/org/archive/crawler/frontier/AntiCalendarCostAssignmentPolicy.java

@@ -26,7 +26,7 @@ package org.archive.crawler.frontier;
 
 import java.util.regex.Matcher;
 
-import org.archive.crawler.datamodel.CrawlURI;
+import org.archive.modules.CrawlURI;
 import org.archive.util.TextUtils;
 
 /**

engine/src/main/java/org/archive/crawler/frontier/AssignmentLevelSurtQueueAssignmentPolicy.java

@@ -18,7 +18,7 @@
  */
 package org.archive.crawler.frontier;
 
-import org.archive.crawler.datamodel.CrawlURI;
+import org.archive.modules.CrawlURI;
 import org.archive.net.PublicSuffixes;
 
 /**

engine/src/main/java/org/archive/crawler/frontier/BdbFrontier.java

@@ -46,7 +46,7 @@ import org.apache.commons.collections.Closure;
 import org.archive.bdb.BdbModule;
 import org.archive.checkpointing.Checkpointable;
 import org.archive.checkpointing.RecoverAction;
-import org.archive.crawler.datamodel.CrawlURI;
+import org.archive.modules.CrawlURI;
 import org.archive.queue.StoredQueue;
 import org.springframework.beans.factory.annotation.Autowired;
 

engine/src/main/java/org/archive/crawler/frontier/BdbMultipleWorkQueues.java

@@ -35,7 +35,7 @@ import javax.management.openmbean.CompositeDataSupport;
 import javax.management.openmbean.OpenDataException;
 
 import org.apache.commons.collections.Closure;
-import org.archive.crawler.datamodel.CrawlURI;
+import org.archive.modules.CrawlURI;
 import org.archive.util.ArchiveUtils;
 
 import com.sleepycat.bind.serial.StoredClassCatalog;

engine/src/main/java/org/archive/crawler/frontier/BdbWorkQueue.java

@@ -28,7 +28,7 @@ import java.io.UnsupportedEncodingException;
 import java.util.logging.Level;
 import java.util.logging.Logger;
 
-import org.archive.crawler.datamodel.CrawlURI;
+import org.archive.modules.CrawlURI;
 import org.archive.util.ArchiveUtils;
 import org.archive.util.IoUtils;
 

engine/src/main/java/org/archive/crawler/frontier/BucketQueueAssignmentPolicy.java

@@ -26,7 +26,7 @@
 package org.archive.crawler.frontier;
 
 import org.apache.commons.httpclient.URIException;
-import org.archive.crawler.datamodel.CrawlURI;
+import org.archive.modules.CrawlURI;
 import org.archive.modules.net.CrawlHost;
 import org.archive.modules.net.ServerCache;
 import org.springframework.beans.factory.annotation.Autowired;

engine/src/main/java/org/archive/crawler/frontier/CostAssignmentPolicy.java

@@ -26,7 +26,7 @@ package org.archive.crawler.frontier;
 
 import java.io.Serializable;
 
-import org.archive.crawler.datamodel.CrawlURI;
+import org.archive.modules.CrawlURI;
 
 /**
  * Calculate a integer 'cost' value for the given CrawlURI.

engine/src/main/java/org/archive/crawler/frontier/FrontierJournal.java

@@ -34,10 +34,10 @@ import java.util.logging.Logger;
 
 import org.apache.commons.httpclient.URIException;
 import org.archive.checkpointing.Checkpointable;
-import org.archive.crawler.datamodel.CrawlURI;
 import org.archive.crawler.framework.CrawlController;
 import org.archive.crawler.framework.Frontier;
 import org.archive.io.CrawlerJournal;
+import org.archive.modules.CrawlURI;
 import org.archive.modules.deciderules.DecideRule;
 import org.archive.net.UURI;
 import org.archive.net.UURIFactory;

engine/src/main/java/org/archive/crawler/frontier/IPQueueAssignmentPolicy.java

@@ -24,7 +24,7 @@
 */ 
 package org.archive.crawler.frontier;
 
-import org.archive.crawler.datamodel.CrawlURI;
+import org.archive.modules.CrawlURI;
 import org.archive.modules.net.CrawlHost;
 import org.archive.modules.net.ServerCache;
 import org.archive.modules.net.ServerCacheUtil;

engine/src/main/java/org/archive/crawler/frontier/QueueAssignmentPolicy.java

@@ -21,7 +21,7 @@ package org.archive.crawler.frontier;
 
 import java.io.Serializable;
 
-import org.archive.crawler.datamodel.CrawlURI;
+import org.archive.modules.CrawlURI;
 
 /**
  * Establishes a mapping from CrawlURIs to String keys (queue names).

engine/src/main/java/org/archive/crawler/frontier/URIAuthorityBasedQueueAssignmentPolicy.java

@@ -20,7 +20,7 @@
 package org.archive.crawler.frontier;
 
 import org.apache.commons.lang.StringUtils;
-import org.archive.crawler.datamodel.CrawlURI;
+import org.archive.modules.CrawlURI;
 import org.archive.modules.extractor.Hop;
 import org.archive.net.UURI;
 import org.archive.spring.HasKeyedProperties;

engine/src/main/java/org/archive/crawler/frontier/UnitCostAssignmentPolicy.java

@@ -24,7 +24,7 @@
 */ 
 package org.archive.crawler.frontier;
 
-import org.archive.crawler.datamodel.CrawlURI;
+import org.archive.modules.CrawlURI;
 
 /**
  * A CostAssignment policy that uses a constant value of 1 for all CrawlURIs.

engine/src/main/java/org/archive/crawler/frontier/WagCostAssignmentPolicy.java

@@ -24,7 +24,7 @@
 */ 
 package org.archive.crawler.frontier;
 
-import org.archive.crawler.datamodel.CrawlURI;
+import org.archive.modules.CrawlURI;
 import org.archive.net.UURI;
 
 /**
@@ -42,7 +42,7 @@ public class WagCostAssignmentPolicy extends CostAssignmentPolicy {
      * 
      * @param curi CrawlURI to be assigned a cost
      * 
-     * @see org.archive.crawler.frontier.CostAssignmentPolicy#costOf(org.archive.crawler.datamodel.CrawlURI)
+     * @see org.archive.crawler.frontier.CostAssignmentPolicy#costOf(org.archive.modules.CrawlURI)
      */
     public int costOf(CrawlURI curi) {
         int cost = 1;

engine/src/main/java/org/archive/crawler/frontier/WorkQueue.java

@@ -10,10 +10,10 @@ import java.util.concurrent.TimeUnit;
 import java.util.logging.Level;
 import java.util.logging.Logger;
 
-import org.archive.crawler.datamodel.CrawlURI;
 import org.archive.crawler.framework.Frontier;
 import org.archive.crawler.frontier.precedence.PrecedenceProvider;
 import org.archive.crawler.frontier.precedence.SimplePrecedenceProvider;
+import org.archive.modules.CrawlURI;
 import org.archive.modules.ProcessorURI;
 import org.archive.modules.fetcher.FetchStats;
 import org.archive.modules.fetcher.FetchStats.Stage;

engine/src/main/java/org/archive/crawler/frontier/WorkQueueFrontier.java

@@ -19,11 +19,11 @@
 
 package org.archive.crawler.frontier;
 
-import static org.archive.crawler.datamodel.CoreAttributeConstants.A_FORCE_RETIRE;
 import static org.archive.crawler.event.CrawlURIDispositionEvent.Disposition.DEFERRED_FOR_RETRY;
 import static org.archive.crawler.event.CrawlURIDispositionEvent.Disposition.DISREGARDED;
 import static org.archive.crawler.event.CrawlURIDispositionEvent.Disposition.FAILED;
 import static org.archive.crawler.event.CrawlURIDispositionEvent.Disposition.SUCCEEDED;
+import static org.archive.modules.CoreAttributeConstants.A_FORCE_RETIRE;
 import static org.archive.modules.fetcher.FetchStatusCodes.S_DEFERRED;
 import static org.archive.modules.fetcher.FetchStatusCodes.S_RUNTIME_EXCEPTION;
 
@@ -52,7 +52,6 @@ import java.util.regex.Pattern;
 import org.apache.commons.collections.Bag;
 import org.apache.commons.collections.BagUtils;
 import org.apache.commons.collections.bag.HashBag;
-import org.archive.crawler.datamodel.CrawlURI;
 import org.archive.crawler.datamodel.UriUniqFilter;
 import org.archive.crawler.datamodel.UriUniqFilter.CrawlUriReceiver;
 import org.archive.crawler.event.CrawlURIDispositionEvent;
@@ -61,6 +60,7 @@ import org.archive.crawler.frontier.precedence.BaseQueuePrecedencePolicy;
 import org.archive.crawler.frontier.precedence.CostUriPrecedencePolicy;
 import org.archive.crawler.frontier.precedence.QueuePrecedencePolicy;
 import org.archive.crawler.frontier.precedence.UriPrecedencePolicy;
+import org.archive.modules.CrawlURI;
 import org.archive.net.UURI;
 import org.archive.spring.KeyedProperties;
 import org.archive.util.ArchiveUtils;
@@ -354,7 +354,7 @@ ApplicationContextAware {
      * Arrange for the given CrawlURI to be visited, if it is not
      * already scheduled/completed.
      *
-     * @see org.archive.crawler.framework.Frontier#schedule(org.archive.crawler.datamodel.CrawlURI)
+     * @see org.archive.crawler.framework.Frontier#schedule(org.archive.modules.CrawlURI)
      */
     protected void processScheduleIfUnique(CrawlURI curi) {
         assert Thread.currentThread() == managerThread;
@@ -929,7 +929,7 @@ ApplicationContextAware {
      * via the next next() call, as a result of finished().
      *
      *  (non-Javadoc)
-     * @see org.archive.crawler.framework.Frontier#finished(org.archive.crawler.datamodel.CrawlURI)
+     * @see org.archive.crawler.framework.Frontier#finished(org.archive.modules.CrawlURI)
      */
     protected void processFinish(CrawlURI curi) {
         assert Thread.currentThread() == managerThread;
@@ -1489,7 +1489,7 @@ ApplicationContextAware {
     /**
      * Force logging, etc. of operator- deleted CrawlURIs
      * 
-     * @see org.archive.crawler.framework.Frontier#deleted(org.archive.crawler.datamodel.CrawlURI)
+     * @see org.archive.crawler.framework.Frontier#deleted(org.archive.modules.CrawlURI)
      */
     public void deleted(CrawlURI curi) {
         //treat as disregarded

engine/src/main/java/org/archive/crawler/frontier/ZeroCostAssignmentPolicy.java

@@ -6,7 +6,7 @@
  */
 package org.archive.crawler.frontier;
 
-import org.archive.crawler.datamodel.CrawlURI;
+import org.archive.modules.CrawlURI;
 
 /**
  * CostAssignmentPolicy considering all URIs costless -- essentially

engine/src/main/java/org/archive/crawler/frontier/precedence/BaseUriPrecedencePolicy.java

@@ -19,7 +19,7 @@
  
 package org.archive.crawler.frontier.precedence;
 
-import org.archive.crawler.datamodel.CrawlURI;
+import org.archive.modules.CrawlURI;
 import org.archive.spring.HasKeyedProperties;
 import org.archive.spring.KeyedProperties;
 

engine/src/main/java/org/archive/crawler/frontier/precedence/CostUriPrecedencePolicy.java

@@ -24,7 +24,7 @@
 */
 package org.archive.crawler.frontier.precedence;
 
-import org.archive.crawler.datamodel.CrawlURI;
+import org.archive.modules.CrawlURI;
 
 /**
  * UriPrecedencePolicy which sets a URI's precedence to its 'cost' -- which

engine/src/main/java/org/archive/crawler/frontier/precedence/HighestUriQueuePrecedencePolicy.java

@@ -26,8 +26,8 @@ package org.archive.crawler.frontier.precedence;
 
 import java.io.PrintWriter;
 
-import org.archive.crawler.datamodel.CrawlURI;
 import org.archive.crawler.frontier.WorkQueue;
+import org.archive.modules.CrawlURI;
 import org.archive.modules.ProcessorURI;
 import org.archive.modules.fetcher.FetchStats.Stage;
 import org.archive.util.Histotable;

engine/src/main/java/org/archive/crawler/frontier/precedence/HopsUriPrecedencePolicy.java

@@ -24,7 +24,7 @@
 */
 package org.archive.crawler.frontier.precedence;
 
-import org.archive.crawler.datamodel.CrawlURI;
+import org.archive.modules.CrawlURI;
 
 /**
  * UriPrecedencePolicy which assigns URIs a precedence equal to the number

engine/src/main/java/org/archive/crawler/frontier/precedence/PrecedenceLoader.java

@@ -31,7 +31,8 @@ import java.util.HashMap;
 import java.util.Iterator;
 import java.util.Map;
 
-import static org.archive.crawler.datamodel.CoreAttributeConstants.*;
+import static org.archive.modules.CoreAttributeConstants.*;
+
 import org.archive.io.CrawlerJournal;
 import org.archive.modules.recrawl.PersistProcessor;
 import org.archive.util.bdbje.EnhancedEnvironment;

engine/src/main/java/org/archive/crawler/frontier/precedence/PreloadedUriPrecedencePolicy.java

@@ -24,12 +24,12 @@
 */
 package org.archive.crawler.frontier.precedence;
 
-import static org.archive.crawler.datamodel.CoreAttributeConstants.A_PRECALC_PRECEDENCE;
+import static org.archive.modules.CoreAttributeConstants.A_PRECALC_PRECEDENCE;
 
 import java.util.Map;
 
 import org.archive.bdb.BdbModule;
-import org.archive.crawler.datamodel.CrawlURI;
+import org.archive.modules.CrawlURI;
 import org.archive.modules.recrawl.PersistProcessor;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.context.Lifecycle;

engine/src/main/java/org/archive/crawler/frontier/precedence/UriPrecedencePolicy.java

@@ -26,7 +26,7 @@ package org.archive.crawler.frontier.precedence;
 
 import java.io.Serializable;
 
-import org.archive.crawler.datamodel.CrawlURI;
+import org.archive.modules.CrawlURI;
 
 /**
  * Superclass for URI precedence policies, which set a integer 

engine/src/main/java/org/archive/crawler/io/NonFatalErrorFormatter.java

@@ -27,7 +27,7 @@ import java.io.PrintWriter;
 import java.io.StringWriter;
 import java.util.logging.LogRecord;
 
-import org.archive.crawler.datamodel.CoreAttributeConstants;
+import org.archive.modules.CoreAttributeConstants;
 
 /**
  * @author gojomo

engine/src/main/java/org/archive/crawler/io/RuntimeErrorFormatter.java

@@ -28,8 +28,8 @@ import java.io.PrintWriter;
 import java.io.StringWriter;
 import java.util.logging.LogRecord;
 
-import org.archive.crawler.datamodel.CoreAttributeConstants;
-import org.archive.crawler.datamodel.CrawlURI;
+import org.archive.modules.CoreAttributeConstants;
+import org.archive.modules.CrawlURI;
 
 /**
  * Runtime exception log formatter.

engine/src/main/java/org/archive/crawler/io/UriErrorFormatter.java

@@ -26,7 +26,7 @@ package org.archive.crawler.io;
 import java.util.logging.Formatter;
 import java.util.logging.LogRecord;
 
-import org.archive.crawler.datamodel.CoreAttributeConstants;
+import org.archive.modules.CoreAttributeConstants;
 import org.archive.net.UURI;
 import org.archive.util.ArchiveUtils;
 

engine/src/main/java/org/archive/crawler/io/UriProcessingFormatter.java

@@ -31,8 +31,8 @@ import java.util.Iterator;
 import java.util.logging.Formatter;
 import java.util.logging.LogRecord;
 
-import org.archive.crawler.datamodel.CoreAttributeConstants;
-import org.archive.crawler.datamodel.CrawlURI;
+import org.archive.modules.CoreAttributeConstants;
+import org.archive.modules.CrawlURI;
 import org.archive.util.ArchiveUtils;
 import org.archive.util.MimetypeUtils;
 

engine/src/main/java/org/archive/crawler/postprocessor/CrawlStateUpdater.java

@@ -28,7 +28,7 @@ import static org.archive.modules.fetcher.FetchStatusCodes.S_CONNECT_FAILED;
 import java.util.logging.Logger;
 
 import org.apache.commons.httpclient.URIException;
-import org.archive.crawler.datamodel.CrawlURI;
+import org.archive.modules.CrawlURI;
 import org.archive.modules.PostProcessor;
 import org.archive.modules.Processor;
 import org.archive.modules.ProcessorURI;

engine/src/main/java/org/archive/crawler/postprocessor/FrontierScheduler.java

@@ -30,8 +30,8 @@ import static org.archive.modules.fetcher.FetchStatusCodes.S_DEFERRED;
 
 import java.util.concurrent.locks.ReentrantLock;
 
-import org.archive.crawler.datamodel.CrawlURI;
 import org.archive.crawler.framework.Frontier;
+import org.archive.modules.CrawlURI;
 import org.archive.modules.PostProcessor;
 import org.archive.modules.Processor;
 import org.archive.modules.ProcessorURI;

engine/src/main/java/org/archive/crawler/postprocessor/LinksScoper.java

@@ -25,17 +25,17 @@
  */
 package org.archive.crawler.postprocessor;
 
-import static org.archive.crawler.datamodel.SchedulingConstants.HIGH;
-import static org.archive.crawler.datamodel.SchedulingConstants.MEDIUM;
-import static org.archive.crawler.datamodel.SchedulingConstants.NORMAL;
+import static org.archive.modules.SchedulingConstants.HIGH;
+import static org.archive.modules.SchedulingConstants.MEDIUM;
+import static org.archive.modules.SchedulingConstants.NORMAL;
 import static org.archive.modules.fetcher.FetchStatusCodes.S_PREREQUISITE_UNSCHEDULABLE_FAILURE;
 
 import java.util.logging.Level;
 import java.util.logging.Logger;
 
 import org.apache.commons.httpclient.URIException;
-import org.archive.crawler.datamodel.CrawlURI;
 import org.archive.crawler.framework.Scoper;
+import org.archive.modules.CrawlURI;
 import org.archive.modules.PostProcessor;
 import org.archive.modules.ProcessorURI;
 import org.archive.modules.deciderules.DecideResult;

engine/src/main/java/org/archive/crawler/postprocessor/SupplementaryLinksScoper.java

@@ -29,8 +29,8 @@ import java.util.Iterator;
 import java.util.logging.Level;
 import java.util.logging.Logger;
 
-import org.archive.crawler.datamodel.CrawlURI;
 import org.archive.crawler.framework.Scoper;
+import org.archive.modules.CrawlURI;
 import org.archive.modules.PostProcessor;
 import org.archive.modules.ProcessorURI;
 import org.archive.modules.deciderules.AcceptDecideRule;

engine/src/main/java/org/archive/crawler/prefetch/PreconditionEnforcer.java

@@ -34,8 +34,8 @@ import java.util.logging.Level;
 import java.util.logging.Logger;
 
 import org.apache.commons.httpclient.URIException;
-import org.archive.crawler.datamodel.CrawlURI;
 import org.archive.crawler.reporting.CrawlerLoggerModule;
+import org.archive.modules.CrawlURI;
 import org.archive.modules.ProcessResult;
 import org.archive.modules.Processor;
 import org.archive.modules.ProcessorURI;

engine/src/main/java/org/archive/crawler/prefetch/Preselector.java

@@ -24,11 +24,11 @@
 package org.archive.crawler.prefetch;
 
 
-import org.archive.crawler.datamodel.CrawlURI;
 
 import static org.archive.modules.fetcher.FetchStatusCodes.*;
 
 import org.archive.crawler.framework.Scoper;
+import org.archive.modules.CrawlURI;
 import org.archive.modules.ProcessResult;
 import org.archive.modules.ProcessorURI;
 import org.archive.util.TextUtils;

engine/src/main/java/org/archive/crawler/prefetch/QuotaEnforcer.java

@@ -27,8 +27,8 @@ import static org.archive.modules.fetcher.FetchStatusCodes.S_BLOCKED_BY_QUOTA;
 import java.util.logging.Level;
 import java.util.logging.Logger;
 
-import org.archive.crawler.datamodel.CrawlURI;
 import org.archive.crawler.framework.Frontier;
+import org.archive.modules.CrawlURI;
 import org.archive.modules.ProcessResult;
 import org.archive.modules.Processor;
 import org.archive.modules.ProcessorURI;

engine/src/main/java/org/archive/crawler/prefetch/RuntimeLimitEnforcer.java

@@ -26,13 +26,13 @@ package org.archive.crawler.prefetch;
 import java.util.logging.Level;
 import java.util.logging.Logger;
 
-import org.archive.crawler.datamodel.CrawlURI;
 
 import static org.archive.modules.fetcher.FetchStatusCodes.*;
 
 import org.archive.crawler.framework.CrawlController;
 import org.archive.crawler.framework.CrawlStatus;
 import org.archive.crawler.reporting.StatisticsTracker;
+import org.archive.modules.CrawlURI;
 import org.archive.modules.ProcessResult;
 import org.archive.modules.Processor;
 import org.archive.modules.ProcessorURI;

engine/src/main/java/org/archive/crawler/processor/CrawlMapper.java

@@ -32,7 +32,7 @@ import java.io.PrintWriter;
 import java.util.HashMap;
 import java.util.Iterator;
 
-import org.archive.crawler.datamodel.CrawlURI;
+import org.archive.modules.CrawlURI;
 import org.archive.modules.ProcessResult;
 import org.archive.modules.Processor;
 import org.archive.modules.ProcessorURI;

engine/src/main/java/org/archive/crawler/processor/HashCrawlMapper.java

@@ -24,8 +24,8 @@ package org.archive.crawler.processor;
 
 import java.util.regex.Matcher;
 
-import org.archive.crawler.datamodel.CrawlURI;
 import org.archive.crawler.framework.Frontier;
+import org.archive.modules.CrawlURI;
 import org.archive.net.PublicSuffixes;
 import org.archive.util.TextUtils;
 import org.springframework.beans.factory.annotation.Autowired;

engine/src/main/java/org/archive/crawler/processor/LexicalCrawlMapper.java

@@ -34,8 +34,8 @@ import java.util.Iterator;
 import java.util.SortedMap;
 import java.util.TreeMap;
 
-import org.archive.crawler.datamodel.CrawlURI;
 import org.archive.crawler.framework.Frontier;
+import org.archive.modules.CrawlURI;
 import org.archive.spring.ConfigPath;
 import org.archive.util.iterator.LineReadingIterator;
 import org.archive.util.iterator.RegexpLineIterator;

engine/src/main/java/org/archive/crawler/reporting/SeedRecord.java

@@ -22,8 +22,8 @@ package org.archive.crawler.reporting;
 import java.io.Serializable;
 import java.util.logging.Logger;
 
-import org.archive.crawler.datamodel.CrawlURI;
-import org.archive.crawler.datamodel.CoreAttributeConstants;
+import org.archive.modules.CoreAttributeConstants;
+import org.archive.modules.CrawlURI;
 
 /**
  * Record of all interesting info about the most-recent

engine/src/main/java/org/archive/crawler/reporting/StatisticsTracker.java

@@ -19,7 +19,7 @@
 
 package org.archive.crawler.reporting;
 
-import static org.archive.crawler.datamodel.CoreAttributeConstants.A_SOURCE_TAG;
+import static org.archive.modules.CoreAttributeConstants.A_SOURCE_TAG;
 
 import java.io.File;
 import java.io.FileWriter;
@@ -46,7 +46,6 @@ import java.util.logging.Level;
 import java.util.logging.Logger;
 
 import org.archive.bdb.BdbModule;
-import org.archive.crawler.datamodel.CrawlURI;
 import org.archive.crawler.event.CrawlStateEvent;
 import org.archive.crawler.event.CrawlURIDispositionEvent;
 import org.archive.crawler.event.StatSnapshotEvent;
@@ -54,6 +53,7 @@ import org.archive.crawler.framework.CrawlController;
 import org.archive.crawler.framework.Engine;
 import org.archive.crawler.util.CrawledBytesHistotable;
 import org.archive.crawler.util.TopNSet;
+import org.archive.modules.CrawlURI;
 import org.archive.modules.net.ServerCache;
 import org.archive.modules.net.ServerCacheUtil;
 import org.archive.modules.seeds.SeedModule;

engine/src/main/java/org/archive/crawler/spring/SheetOverlaysManager.java

@@ -30,7 +30,7 @@ import java.util.TreeMap;
 import java.util.TreeSet;
 import java.util.logging.Logger;
 
-import org.archive.crawler.datamodel.CrawlURI;
+import org.archive.modules.CrawlURI;
 import org.archive.spring.OverlayMapsSource;
 import org.archive.spring.Sheet;
 import org.archive.util.PrefixFinder;

engine/src/main/java/org/archive/crawler/util/BenchmarkUriUniqFilters.java

@@ -30,8 +30,8 @@ import java.io.FileReader;
 import java.io.FileWriter;
 import java.io.IOException;
 
-import org.archive.crawler.datamodel.CrawlURI;
 import org.archive.crawler.datamodel.UriUniqFilter;
+import org.archive.modules.CrawlURI;
 import org.archive.util.fingerprint.MemLongFPSet;
 
 

engine/src/main/java/org/archive/crawler/util/BloomUriUniqFilter.java

@@ -27,7 +27,7 @@ package org.archive.crawler.util;
 import java.io.Serializable;
 import java.util.logging.Logger;
 
-import org.archive.crawler.datamodel.CrawlURI;
+import org.archive.modules.CrawlURI;
 import org.archive.util.BloomFilter;
 import org.archive.util.BloomFilter32bitSplit;
 

engine/src/main/java/org/archive/crawler/util/CrawledBytesHistotable.java

@@ -20,8 +20,8 @@
 package org.archive.crawler.util;
 
 import org.apache.commons.httpclient.HttpStatus;
-import org.archive.crawler.datamodel.CoreAttributeConstants;
-import org.archive.crawler.datamodel.CrawlURI;
+import org.archive.modules.CoreAttributeConstants;
+import org.archive.modules.CrawlURI;
 import org.archive.modules.deciderules.recrawl.IdenticalDigestDecideRule;
 import org.archive.util.ArchiveUtils;
 import org.archive.util.Histotable;

engine/src/main/java/org/archive/crawler/util/FPMergeUriUniqFilter.java

@@ -36,8 +36,8 @@ import java.util.TreeSet;
 import java.util.logging.Level;
 import java.util.logging.Logger;
 
-import org.archive.crawler.datamodel.CrawlURI;
 import org.archive.crawler.datamodel.UriUniqFilter;
+import org.archive.modules.CrawlURI;
 import org.archive.util.fingerprint.ArrayLongFPCache;
 
 import st.ata.util.FPGenerator;

engine/src/main/java/org/archive/crawler/util/SetBasedUriUniqFilter.java

@@ -33,8 +33,8 @@ import java.io.Serializable;
 import java.util.logging.Level;
 import java.util.logging.Logger;
 
-import org.archive.crawler.datamodel.CrawlURI;
 import org.archive.crawler.datamodel.UriUniqFilter;
+import org.archive.modules.CrawlURI;
 
 /**
  * UriUniqFilter based on an underlying UriSet (essentially a Set).