part of HER-2048 - WARCWriterProcessor stats resume from checkpoint

This commit is contained in:
Noah Levitt
2013-09-07 11:35:03 -07:00
parent e0f78976d7
commit 52d72e661d
@@ -100,6 +100,8 @@ import org.archive.uid.RecordIDGenerator;
import org.archive.uid.UUIDGenerator;
import org.archive.util.ArchiveUtils;
import org.archive.util.anvl.ANVLRecord;
import org.json.JSONException;
import org.json.JSONObject;
/**
* WARCWriterProcessor.
@@ -959,9 +961,38 @@ public class WARCWriterProcessor extends WriterPoolProcessor implements WARCWrit
record.addLabelValue(label, value);
}
}
@Override
protected JSONObject toCheckpointJson() throws JSONException {
JSONObject json = super.toCheckpointJson();
json.put("stats", stats);
return json;
}
@Override
protected void fromCheckpointJson(JSONObject json) throws JSONException {
super.fromCheckpointJson(json);
HashMap<String, Map<String, Long>> cpStats = new HashMap<String, Map<String, Long>>();
JSONObject jsonStats = json.getJSONObject("stats");
for (String key1: JSONObject.getNames(jsonStats)) {
JSONObject jsonSubstats = jsonStats.getJSONObject(key1);
if (!cpStats.containsKey(key1)) {
cpStats.put(key1, new HashMap<String, Long>());
}
Map<String, Long> substats = cpStats.get(key1);
for (String key2: JSONObject.getNames(jsonSubstats)) {
long value = jsonSubstats.getLong(key2);
substats.put(key2, value);
}
}
addStats(cpStats);
}
@Override
public String report() {
// XXX note in report that stats include recovered checkpoint?
logger.info("final stats: " + stats);
StringBuilder buf = new StringBuilder();