From daab317c9476d000611e02f2cd478957853944c5 Mon Sep 17 00:00:00 2001 From: gojomo Date: Fri, 20 Nov 2009 01:10:32 +0000 Subject: [PATCH] [HER-1546] Springify(5): Update checkpointing to work smoothly with spring-configured crawls processor-related bean checkpoint support: * Checkpoint.java move to commons for proper dependency graph * (others) recognize above --- commons/src/main/java/org/archive/bdb/BdbModule.java | 2 +- .../main/java/org/archive/checkpointing}/Checkpoint.java | 6 ++++-- .../main/java/org/archive/checkpointing/Checkpointable.java | 1 - commons/src/main/java/org/archive/io/CrawlerJournal.java | 2 +- .../test/java/org/archive/settings/file/BdbModuleTest.java | 2 +- .../org/archive/crawler/framework/CheckpointService.java | 1 + .../java/org/archive/crawler/framework/CrawlController.java | 1 + .../main/java/org/archive/crawler/frontier/BdbFrontier.java | 2 +- .../org/archive/crawler/reporting/CrawlerLoggerModule.java | 2 +- .../org/archive/crawler/reporting/StatisticsTracker.java | 2 +- .../main/java/org/archive/crawler/restlet/JobResource.java | 2 +- .../java/org/archive/crawler/util/BdbUriUniqFilter.java | 2 +- .../src/main/java/org/archive/modules/DispositionChain.java | 2 +- modules/src/main/java/org/archive/modules/Processor.java | 2 +- .../org/archive/modules/recrawl/PersistLogProcessor.java | 2 +- .../org/archive/modules/writer/WriterPoolProcessor.java | 2 +- 16 files changed, 18 insertions(+), 15 deletions(-) rename {engine/src/main/java/org/archive/crawler/framework => commons/src/main/java/org/archive/checkpointing}/Checkpoint.java (94%) diff --git a/commons/src/main/java/org/archive/bdb/BdbModule.java b/commons/src/main/java/org/archive/bdb/BdbModule.java index d9d6ec68..1aa2df9e 100644 --- a/commons/src/main/java/org/archive/bdb/BdbModule.java +++ b/commons/src/main/java/org/archive/bdb/BdbModule.java @@ -39,8 +39,8 @@ import java.util.logging.Logger; import org.apache.commons.io.FileUtils; import org.apache.commons.io.filefilter.FileFilterUtils; import org.apache.commons.io.filefilter.IOFileFilter; +import org.archive.checkpointing.Checkpoint; import org.archive.checkpointing.Checkpointable; -import org.archive.crawler.framework.Checkpoint; import org.archive.spring.ConfigPath; import org.archive.util.CachedBdbMap; import org.archive.util.ObjectIdentityBdbCache; diff --git a/engine/src/main/java/org/archive/crawler/framework/Checkpoint.java b/commons/src/main/java/org/archive/checkpointing/Checkpoint.java similarity index 94% rename from engine/src/main/java/org/archive/crawler/framework/Checkpoint.java rename to commons/src/main/java/org/archive/checkpointing/Checkpoint.java index 6539cd89..525ab4cc 100644 --- a/engine/src/main/java/org/archive/crawler/framework/Checkpoint.java +++ b/commons/src/main/java/org/archive/checkpointing/Checkpoint.java @@ -16,7 +16,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.archive.crawler.framework; +package org.archive.checkpointing; import java.io.File; import java.io.IOException; @@ -24,6 +24,7 @@ import java.util.logging.Level; import java.util.logging.Logger; import org.apache.commons.io.FileUtils; +import org.archive.crawler.framework.CheckpointService; import org.archive.spring.ConfigPath; import org.archive.util.ArchiveUtils; import org.json.JSONException; @@ -122,7 +123,8 @@ public class Checkpoint implements InitializingBean { public String getShortName() { return shortName; } - protected void writeValidity(String stamp) { + + public void writeValidity(String stamp) { if(!success) { return; } diff --git a/commons/src/main/java/org/archive/checkpointing/Checkpointable.java b/commons/src/main/java/org/archive/checkpointing/Checkpointable.java index 77908b26..6d356f74 100644 --- a/commons/src/main/java/org/archive/checkpointing/Checkpointable.java +++ b/commons/src/main/java/org/archive/checkpointing/Checkpointable.java @@ -21,7 +21,6 @@ package org.archive.checkpointing; import java.io.IOException; -import org.archive.crawler.framework.Checkpoint; /** * Interface for objects that can checkpoint their state, possibly diff --git a/commons/src/main/java/org/archive/io/CrawlerJournal.java b/commons/src/main/java/org/archive/io/CrawlerJournal.java index a7755406..c2b9fa43 100644 --- a/commons/src/main/java/org/archive/io/CrawlerJournal.java +++ b/commons/src/main/java/org/archive/io/CrawlerJournal.java @@ -30,7 +30,7 @@ import java.io.OutputStreamWriter; import java.io.Writer; import java.util.zip.GZIPOutputStream; -import org.archive.crawler.framework.Checkpoint; +import org.archive.checkpointing.Checkpoint; import org.archive.util.ArchiveUtils; import org.archive.util.FileUtils; diff --git a/commons/src/test/java/org/archive/settings/file/BdbModuleTest.java b/commons/src/test/java/org/archive/settings/file/BdbModuleTest.java index 1f3659c5..3aa905fa 100644 --- a/commons/src/test/java/org/archive/settings/file/BdbModuleTest.java +++ b/commons/src/test/java/org/archive/settings/file/BdbModuleTest.java @@ -22,7 +22,7 @@ package org.archive.settings.file; import org.apache.commons.io.FileUtils; import org.apache.commons.lang.math.RandomUtils; import org.archive.bdb.BdbModule; -import org.archive.crawler.framework.Checkpoint; +import org.archive.checkpointing.Checkpoint; import org.archive.spring.ConfigPath; import org.archive.util.ObjectIdentityBdbCache; import org.archive.util.Supplier; diff --git a/engine/src/main/java/org/archive/crawler/framework/CheckpointService.java b/engine/src/main/java/org/archive/crawler/framework/CheckpointService.java index 4629f526..1204d03d 100644 --- a/engine/src/main/java/org/archive/crawler/framework/CheckpointService.java +++ b/engine/src/main/java/org/archive/crawler/framework/CheckpointService.java @@ -29,6 +29,7 @@ import java.util.logging.Logger; import org.apache.commons.io.filefilter.FileFilterUtils; import org.apache.commons.lang.ArrayUtils; +import org.archive.checkpointing.Checkpoint; import org.archive.checkpointing.Checkpointable; import org.archive.crawler.reporting.CrawlStatSnapshot; import org.archive.spring.ConfigPath; diff --git a/engine/src/main/java/org/archive/crawler/framework/CrawlController.java b/engine/src/main/java/org/archive/crawler/framework/CrawlController.java index 0c987820..e2709437 100644 --- a/engine/src/main/java/org/archive/crawler/framework/CrawlController.java +++ b/engine/src/main/java/org/archive/crawler/framework/CrawlController.java @@ -28,6 +28,7 @@ import java.util.LinkedList; import java.util.logging.Level; import java.util.logging.Logger; +import org.archive.checkpointing.Checkpoint; import org.archive.checkpointing.Checkpointable; import org.archive.crawler.event.CrawlStateEvent; import org.archive.crawler.reporting.AlertThreadGroup; diff --git a/engine/src/main/java/org/archive/crawler/frontier/BdbFrontier.java b/engine/src/main/java/org/archive/crawler/frontier/BdbFrontier.java index e163f7fe..d5bc1919 100644 --- a/engine/src/main/java/org/archive/crawler/frontier/BdbFrontier.java +++ b/engine/src/main/java/org/archive/crawler/frontier/BdbFrontier.java @@ -34,8 +34,8 @@ import javax.management.openmbean.CompositeData; import org.apache.commons.collections.Closure; import org.archive.bdb.BdbModule; +import org.archive.checkpointing.Checkpoint; import org.archive.checkpointing.Checkpointable; -import org.archive.crawler.framework.Checkpoint; import org.archive.modules.CrawlURI; import org.archive.queue.StoredQueue; import org.archive.util.ArchiveUtils; diff --git a/engine/src/main/java/org/archive/crawler/reporting/CrawlerLoggerModule.java b/engine/src/main/java/org/archive/crawler/reporting/CrawlerLoggerModule.java index 8e75f356..df8104ca 100644 --- a/engine/src/main/java/org/archive/crawler/reporting/CrawlerLoggerModule.java +++ b/engine/src/main/java/org/archive/crawler/reporting/CrawlerLoggerModule.java @@ -31,8 +31,8 @@ import java.util.logging.Logger; import java.util.logging.SimpleFormatter; import org.apache.commons.httpclient.URIException; +import org.archive.checkpointing.Checkpoint; import org.archive.checkpointing.Checkpointable; -import org.archive.crawler.framework.Checkpoint; import org.archive.crawler.framework.Engine; import org.archive.crawler.io.NonFatalErrorFormatter; import org.archive.crawler.io.RuntimeErrorFormatter; diff --git a/engine/src/main/java/org/archive/crawler/reporting/StatisticsTracker.java b/engine/src/main/java/org/archive/crawler/reporting/StatisticsTracker.java index 2665a95a..8308f502 100644 --- a/engine/src/main/java/org/archive/crawler/reporting/StatisticsTracker.java +++ b/engine/src/main/java/org/archive/crawler/reporting/StatisticsTracker.java @@ -40,11 +40,11 @@ import java.util.logging.Logger; import org.archive.bdb.BdbModule; import org.archive.bdb.TempStoredSortedMap; +import org.archive.checkpointing.Checkpoint; import org.archive.checkpointing.Checkpointable; import org.archive.crawler.event.CrawlStateEvent; import org.archive.crawler.event.CrawlURIDispositionEvent; import org.archive.crawler.event.StatSnapshotEvent; -import org.archive.crawler.framework.Checkpoint; import org.archive.crawler.framework.CrawlController; import org.archive.crawler.framework.Engine; import org.archive.crawler.util.CrawledBytesHistotable; diff --git a/engine/src/main/java/org/archive/crawler/restlet/JobResource.java b/engine/src/main/java/org/archive/crawler/restlet/JobResource.java index e2e634ff..d5707daf 100644 --- a/engine/src/main/java/org/archive/crawler/restlet/JobResource.java +++ b/engine/src/main/java/org/archive/crawler/restlet/JobResource.java @@ -30,7 +30,7 @@ import java.util.List; import org.apache.commons.io.filefilter.IOFileFilter; import org.apache.commons.lang.StringEscapeUtils; import org.apache.commons.lang.StringUtils; -import org.archive.crawler.framework.Checkpoint; +import org.archive.checkpointing.Checkpoint; import org.archive.crawler.framework.CrawlJob; import org.archive.crawler.framework.Engine; import org.archive.crawler.reporting.AlertHandler; diff --git a/engine/src/main/java/org/archive/crawler/util/BdbUriUniqFilter.java b/engine/src/main/java/org/archive/crawler/util/BdbUriUniqFilter.java index e657e59c..cead8031 100644 --- a/engine/src/main/java/org/archive/crawler/util/BdbUriUniqFilter.java +++ b/engine/src/main/java/org/archive/crawler/util/BdbUriUniqFilter.java @@ -25,8 +25,8 @@ import java.util.logging.Level; import java.util.logging.Logger; import org.archive.bdb.BdbModule; +import org.archive.checkpointing.Checkpoint; import org.archive.checkpointing.Checkpointable; -import org.archive.crawler.framework.Checkpoint; import org.json.JSONException; import org.json.JSONObject; import org.springframework.beans.factory.BeanNameAware; diff --git a/modules/src/main/java/org/archive/modules/DispositionChain.java b/modules/src/main/java/org/archive/modules/DispositionChain.java index 67a71431..8c33b170 100644 --- a/modules/src/main/java/org/archive/modules/DispositionChain.java +++ b/modules/src/main/java/org/archive/modules/DispositionChain.java @@ -22,8 +22,8 @@ import java.io.IOException; import java.util.concurrent.locks.ReentrantReadWriteLock; +import org.archive.checkpointing.Checkpoint; import org.archive.checkpointing.Checkpointable; -import org.archive.crawler.framework.Checkpoint; import org.springframework.beans.factory.annotation.Autowired; public class DispositionChain extends ProcessorChain implements Checkpointable { diff --git a/modules/src/main/java/org/archive/modules/Processor.java b/modules/src/main/java/org/archive/modules/Processor.java index acd5bf41..38bd0149 100644 --- a/modules/src/main/java/org/archive/modules/Processor.java +++ b/modules/src/main/java/org/archive/modules/Processor.java @@ -25,8 +25,8 @@ import java.util.Set; import java.util.concurrent.atomic.AtomicLong; import org.apache.commons.httpclient.HttpStatus; +import org.archive.checkpointing.Checkpoint; import org.archive.checkpointing.Checkpointable; -import org.archive.crawler.framework.Checkpoint; import org.archive.modules.credential.CredentialAvatar; import org.archive.modules.credential.HttpAuthenticationCredential; import org.archive.modules.deciderules.AcceptDecideRule; diff --git a/modules/src/main/java/org/archive/modules/recrawl/PersistLogProcessor.java b/modules/src/main/java/org/archive/modules/recrawl/PersistLogProcessor.java index a514ef03..7420ebc9 100644 --- a/modules/src/main/java/org/archive/modules/recrawl/PersistLogProcessor.java +++ b/modules/src/main/java/org/archive/modules/recrawl/PersistLogProcessor.java @@ -24,8 +24,8 @@ import java.io.Serializable; import org.apache.commons.codec.binary.Base64; import org.apache.commons.lang.SerializationUtils; +import org.archive.checkpointing.Checkpoint; import org.archive.checkpointing.Checkpointable; -import org.archive.crawler.framework.Checkpoint; import org.archive.io.CrawlerJournal; import org.archive.modules.CrawlURI; import org.archive.spring.ConfigPath; diff --git a/modules/src/main/java/org/archive/modules/writer/WriterPoolProcessor.java b/modules/src/main/java/org/archive/modules/writer/WriterPoolProcessor.java index e6200321..70cc5547 100644 --- a/modules/src/main/java/org/archive/modules/writer/WriterPoolProcessor.java +++ b/modules/src/main/java/org/archive/modules/writer/WriterPoolProcessor.java @@ -31,8 +31,8 @@ import java.util.List; import java.util.concurrent.atomic.AtomicInteger; import java.util.logging.Logger; +import org.archive.checkpointing.Checkpoint; import org.archive.checkpointing.Checkpointable; -import org.archive.crawler.framework.Checkpoint; import org.archive.io.DefaultWriterPoolSettings; import org.archive.io.WriterPool; import org.archive.io.WriterPoolMember;