[HER-1546] Springify(5): Update checkpointing to work smoothly with spring-configured crawls

stats & logging checkpoint support
* StatisticsTracker.java
    implement Checkpointable; save small state in JSON, recycle BDB data when recovering
* CrawlerLoggerModule.java
    implement Checkpointable; rotate logs on checkpoint
* GenerationFileHandler.java
    avoid clobbering leftover logs from earlier futures via moveAside
* CrawlStatSnapshot.java
    (sameProgressAs) test for lack-of-progress
* ObjectIdentityMemCache.java, TopNSet.java
    offer deeper access to assist checkpoint/resume
This commit is contained in:
gojomo committed 2009-11-19 23:18:15 +00:00
1 parent b63c2bdc77
commit f01ad9ada8
6 files changed
+152 -18

No files matched your search

@@ -100,6 +100,7 @@ public class GenerationFileHandler extends FileHandler {
storeSuffix;
File activeFile = new File(filename);
File storeFile = new File(storeFilename);
FileUtils.moveAsideIfExists(storeFile);
if (!activeFile.renameTo(storeFile)) {
throw new IOException("Unable to move " + filename + " to " +
storeFilename);
@@ -19,6 +19,7 @@
package org.archive.util;
import java.util.Map;
import java.util.Set;
import java.util.concurrent.ConcurrentHashMap;
@@ -72,4 +73,12 @@ implements ObjectIdentityCache<String,V> {
public void sync() {
// do nothing
}
/**
* Offer raw map access for convenience of checkpoint/recovery.
* @return Map<String, V>
*/
public Map<String, V> getMap() {
return map;
}
}