mirror of
https://github.com/internetarchive/heritrix3.git
synced 2026-08-02 21:01:06 +00:00
part of HER-2048 - WARCWriterProcessor stats resume from checkpoint
This commit is contained in:
@@ -100,6 +100,8 @@ import org.archive.uid.RecordIDGenerator;
|
||||
import org.archive.uid.UUIDGenerator;
|
||||
import org.archive.util.ArchiveUtils;
|
||||
import org.archive.util.anvl.ANVLRecord;
|
||||
import org.json.JSONException;
|
||||
import org.json.JSONObject;
|
||||
|
||||
/**
|
||||
* WARCWriterProcessor.
|
||||
@@ -959,9 +961,38 @@ public class WARCWriterProcessor extends WriterPoolProcessor implements WARCWrit
|
||||
record.addLabelValue(label, value);
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
protected JSONObject toCheckpointJson() throws JSONException {
|
||||
JSONObject json = super.toCheckpointJson();
|
||||
json.put("stats", stats);
|
||||
return json;
|
||||
}
|
||||
|
||||
@Override
|
||||
protected void fromCheckpointJson(JSONObject json) throws JSONException {
|
||||
super.fromCheckpointJson(json);
|
||||
|
||||
HashMap<String, Map<String, Long>> cpStats = new HashMap<String, Map<String, Long>>();
|
||||
JSONObject jsonStats = json.getJSONObject("stats");
|
||||
for (String key1: JSONObject.getNames(jsonStats)) {
|
||||
JSONObject jsonSubstats = jsonStats.getJSONObject(key1);
|
||||
if (!cpStats.containsKey(key1)) {
|
||||
cpStats.put(key1, new HashMap<String, Long>());
|
||||
}
|
||||
Map<String, Long> substats = cpStats.get(key1);
|
||||
|
||||
for (String key2: JSONObject.getNames(jsonSubstats)) {
|
||||
long value = jsonSubstats.getLong(key2);
|
||||
substats.put(key2, value);
|
||||
}
|
||||
}
|
||||
addStats(cpStats);
|
||||
}
|
||||
|
||||
@Override
|
||||
public String report() {
|
||||
// XXX note in report that stats include recovered checkpoint?
|
||||
logger.info("final stats: " + stats);
|
||||
|
||||
StringBuilder buf = new StringBuilder();
|
||||
|
||||
Reference in New Issue
Block a user