mirror of
https://github.com/internetarchive/heritrix3.git
synced 2026-11-15 10:20:47 +00:00
[HER-1546] Springify(5): Update checkpointing to work smoothly with spring-configured crawls
stats & logging checkpoint support
* StatisticsTracker.java
implement Checkpointable; save small state in JSON, recycle BDB data when recovering
* CrawlerLoggerModule.java
implement Checkpointable; rotate logs on checkpoint
* GenerationFileHandler.java
avoid clobbering leftover logs from earlier futures via moveAside
* CrawlStatSnapshot.java
(sameProgressAs) test for lack-of-progress
* ObjectIdentityMemCache.java, TopNSet.java
offer deeper access to assist checkpoint/resume
This commit is contained in:
1 parent
b63c2bdc77
commit
f01ad9ada8
6 files changed
+152
-18
No files matched your search
@@ -100,6 +100,7 @@ public class GenerationFileHandler extends FileHandler {
|
||||
storeSuffix;
|
||||
File activeFile = new File(filename);
|
||||
File storeFile = new File(storeFilename);
|
||||
FileUtils.moveAsideIfExists(storeFile);
|
||||
if (!activeFile.renameTo(storeFile)) {
|
||||
throw new IOException("Unable to move " + filename + " to " +
|
||||
storeFilename);
|
||||
|
||||
@@ -19,6 +19,7 @@
|
||||
|
||||
package org.archive.util;
|
||||
|
||||
import java.util.Map;
|
||||
import java.util.Set;
|
||||
import java.util.concurrent.ConcurrentHashMap;
|
||||
|
||||
@@ -72,4 +73,12 @@ implements ObjectIdentityCache<String,V> {
|
||||
public void sync() {
|
||||
// do nothing
|
||||
}
|
||||
|
||||
/**
|
||||
* Offer raw map access for convenience of checkpoint/recovery.
|
||||
* @return Map<String, V>
|
||||
*/
|
||||
public Map<String, V> getMap() {
|
||||
return map;
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user