lee/heritrix3
1
0 Fork 0
mirror of https://github.com/internetarchive/heritrix3.git synced 2026-11-15 10:20:47 +00:00
Code Issues Packages Projects Releases Wiki Activity

Fix for HER-1985 H3: SurtPrefixDecideRule forgets learned/seed-derived/seed-directive SURT prefixes in checkpoint-resume

Browse Source 
* SurtPrefixedDecideRule.java
    make class implement Checkpointable, save surt prefixes to json on checkpoint and load them on recover
* profile-crawler-beans.cxml
    make main SurtPrefixedDecideRule a top-level bean so that it can be checkpointed
* Checkpoint.java, CheckpointService.java
    add some FINE level logging during checkpointing and recovery
This commit is contained in:
Noah Levitt committed 2012-02-09 10:05:07 -08:00
1 parent a652a9afd0
commit cfac7ead91
4 files changed
+97 -27
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

commons/src/main/java/org/archive/checkpointing/Checkpoint.java
+9
View File
@@ -125,6 +125,9 @@ public class Checkpoint implements InitializingBean {
public void saveJson(String beanName, JSONObject json) {
try {
File targetFile = new File(getCheckpointDir().getFile(),beanName);
if (LOGGER.isLoggable(Level.FINE)) {
LOGGER.fine("saving json to " + targetFile);
}
FileUtils.writeStringToFile(
targetFile,
json.toString());
@@ -137,6 +140,9 @@ public class Checkpoint implements InitializingBean {
public JSONObject loadJson(String beanName) {
File sourceFile = new File(getCheckpointDir().getFile(),beanName);
try {
if (LOGGER.isLoggable(Level.FINE)) {
LOGGER.fine("reading json from " + sourceFile);
}
return new JSONObject(FileUtils.readFileToString(sourceFile));
} catch (JSONException e) {
throw new RuntimeException(e);
@@ -148,6 +154,9 @@ public class Checkpoint implements InitializingBean {
public BufferedWriter saveWriter(String beanName, String extraName) throws IOException {
try {
File targetFile = new File(getCheckpointDir().getFile(),beanName+"-"+extraName);
if (LOGGER.isLoggable(Level.FINE)) {
LOGGER.fine("opening for writing: " + targetFile);
}
return new BufferedWriter(new FileWriter(targetFile));
} catch (IOException e) {
LOGGER.log(Level.SEVERE,"unable to save checkpoint writer state "+extraName+" of "+beanName,e);