diff --git a/commons/src/main/java/org/archive/checkpointing/Checkpoint.java b/commons/src/main/java/org/archive/checkpointing/Checkpoint.java index 2e5f2a72..47cda1ca 100644 --- a/commons/src/main/java/org/archive/checkpointing/Checkpoint.java +++ b/commons/src/main/java/org/archive/checkpointing/Checkpoint.java @@ -125,6 +125,9 @@ public class Checkpoint implements InitializingBean { public void saveJson(String beanName, JSONObject json) { try { File targetFile = new File(getCheckpointDir().getFile(),beanName); + if (LOGGER.isLoggable(Level.FINE)) { + LOGGER.fine("saving json to " + targetFile); + } FileUtils.writeStringToFile( targetFile, json.toString()); @@ -137,6 +140,9 @@ public class Checkpoint implements InitializingBean { public JSONObject loadJson(String beanName) { File sourceFile = new File(getCheckpointDir().getFile(),beanName); try { + if (LOGGER.isLoggable(Level.FINE)) { + LOGGER.fine("reading json from " + sourceFile); + } return new JSONObject(FileUtils.readFileToString(sourceFile)); } catch (JSONException e) { throw new RuntimeException(e); @@ -148,6 +154,9 @@ public class Checkpoint implements InitializingBean { public BufferedWriter saveWriter(String beanName, String extraName) throws IOException { try { File targetFile = new File(getCheckpointDir().getFile(),beanName+"-"+extraName); + if (LOGGER.isLoggable(Level.FINE)) { + LOGGER.fine("opening for writing: " + targetFile); + } return new BufferedWriter(new FileWriter(targetFile)); } catch (IOException e) { LOGGER.log(Level.SEVERE,"unable to save checkpoint writer state "+extraName+" of "+beanName,e); diff --git a/engine/src/main/java/org/archive/crawler/framework/CheckpointService.java b/engine/src/main/java/org/archive/crawler/framework/CheckpointService.java index ef62f364..3b5d8b2e 100644 --- a/engine/src/main/java/org/archive/crawler/framework/CheckpointService.java +++ b/engine/src/main/java/org/archive/crawler/framework/CheckpointService.java @@ -40,6 +40,7 @@ import org.archive.spring.ConfigPath; import org.archive.spring.ConfigPathConfigurer; import org.archive.spring.HasValidator; import org.springframework.beans.BeansException; +import org.springframework.beans.factory.BeanFactoryUtils; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.context.ApplicationContext; import org.springframework.context.ApplicationContextAware; @@ -222,6 +223,9 @@ public class CheckpointService implements Lifecycle, ApplicationContextAware, Ha } Map toCheckpoint = appCtx.getBeansOfType(Checkpointable.class); + if (LOGGER.isLoggable(Level.FINE)) { + LOGGER.fine("checkpointing beans " + toCheckpoint); + } checkpointInProgress = new Checkpoint(); try { diff --git a/engine/src/main/resources/org/archive/crawler/restlet/profile-crawler-beans.cxml b/engine/src/main/resources/org/archive/crawler/restlet/profile-crawler-beans.cxml index f7842df0..a53b476b 100644 --- a/engine/src/main/resources/org/archive/crawler/restlet/profile-crawler-beans.cxml +++ b/engine/src/main/resources/org/archive/crawler/restlet/profile-crawler-beans.cxml @@ -10,11 +10,11 @@ behavior, uncomment AND alter the shown values.) --> @@ -109,6 +109,25 @@ http://example.example/example --> + + + + + + + + + @@ -116,26 +135,9 @@ http://example.example/example - - + - - - - - - - + diff --git a/modules/src/main/java/org/archive/modules/deciderules/surt/SurtPrefixedDecideRule.java b/modules/src/main/java/org/archive/modules/deciderules/surt/SurtPrefixedDecideRule.java index ecdae30f..0f851f3f 100644 --- a/modules/src/main/java/org/archive/modules/deciderules/surt/SurtPrefixedDecideRule.java +++ b/modules/src/main/java/org/archive/modules/deciderules/surt/SurtPrefixedDecideRule.java @@ -23,10 +23,13 @@ import java.io.File; import java.io.FileWriter; import java.io.IOException; import java.io.Reader; +import java.util.logging.Level; import java.util.logging.Logger; import org.apache.commons.io.IOUtils; import org.apache.commons.lang.StringUtils; +import org.archive.checkpointing.Checkpoint; +import org.archive.checkpointing.Checkpointable; import org.archive.io.ReadSource; import org.archive.modules.CrawlURI; import org.archive.modules.deciderules.DecideResult; @@ -36,6 +39,10 @@ import org.archive.modules.seeds.SeedModule; import org.archive.net.UURI; import org.archive.spring.ConfigFile; import org.archive.util.SurtPrefixSet; +import org.json.JSONArray; +import org.json.JSONException; +import org.json.JSONObject; +import org.springframework.beans.factory.BeanNameAware; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.context.ApplicationEvent; import org.springframework.context.ApplicationListener; @@ -56,7 +63,8 @@ import org.springframework.context.event.ContextStartedEvent; * @author gojomo */ public class SurtPrefixedDecideRule extends PredicatedDecideRule implements - SeedListener, ApplicationListener { + SeedListener, ApplicationListener, Checkpointable, + BeanNameAware { private static final long serialVersionUID = 3L; private static final Logger logger = @@ -219,6 +227,9 @@ public class SurtPrefixedDecideRule extends PredicatedDecideRule implements */ protected void buildSurtPrefixSet() { if (getSurtsSource() != null) { + if (logger.isLoggable(Level.FINE)) { + logger.fine("reading surt prefixes from " + getSurtsSource()); + } Reader reader = getSurtsSource().obtainReader(); try { surtPrefixes.importFromMixed(reader, true); @@ -277,12 +288,56 @@ public class SurtPrefixedDecideRule extends PredicatedDecideRule implements } throw new IllegalArgumentException("decision must be ACCEPT or REJECT"); } - + @Override public void onApplicationEvent(ApplicationEvent event) { if (event instanceof ContextStartedEvent) { - readPrefixes(); + if (recoveryCheckpoint != null) { + JSONObject json = recoveryCheckpoint.loadJson(beanName); + try { + JSONArray jsonArray = json.getJSONArray("surtPrefixes"); + for (int i = 0; i < jsonArray.length(); i++) { + surtPrefixes.add(jsonArray.getString(i)); + } + } catch (JSONException e) { + throw new IllegalStateException(e); + } + } else { + readPrefixes(); + } } } + + // BeanNameAware + protected String beanName; + public void setBeanName(String name) { + this.beanName = name; + } + + @Override + public void startCheckpoint(Checkpoint checkpointInProgress) { + } + + @Override + public void doCheckpoint(Checkpoint checkpointInProgress) + throws IOException { + try { + JSONObject json = new JSONObject(); + json.put("surtPrefixes", surtPrefixes); + checkpointInProgress.saveJson(beanName, json); + } catch (JSONException e) { + throw new RuntimeException(e); + } + } + + @Override + public void finishCheckpoint(Checkpoint checkpointInProgress) { + } + + protected Checkpoint recoveryCheckpoint; + @Override + public void setRecoveryCheckpoint(Checkpoint recoveryCheckpoint) { + this.recoveryCheckpoint = recoveryCheckpoint; + } }//EOC