Fix for HER-1985 H3: SurtPrefixDecideRule forgets learned/seed-derived/seed-directive SURT prefixes in checkpoint-resume

* SurtPrefixedDecideRule.java
    make class implement Checkpointable, save surt prefixes to json on checkpoint and load them on recover
* profile-crawler-beans.cxml
    make main SurtPrefixedDecideRule a top-level bean so that it can be checkpointed
* Checkpoint.java, CheckpointService.java
    add some FINE level logging during checkpointing and recovery

commons/src/main/java/org/archive/checkpointing/Checkpoint.java

@@ -125,6 +125,9 @@ public class Checkpoint implements InitializingBean {
     public void saveJson(String beanName, JSONObject json) {
         try {
             File targetFile = new File(getCheckpointDir().getFile(),beanName);
+            if (LOGGER.isLoggable(Level.FINE)) {
+                LOGGER.fine("saving json to " + targetFile);
+            }
             FileUtils.writeStringToFile(
                     targetFile,
                     json.toString());
@@ -137,6 +140,9 @@ public class Checkpoint implements InitializingBean {
     public JSONObject loadJson(String beanName) {
         File sourceFile = new File(getCheckpointDir().getFile(),beanName);
         try {
+            if (LOGGER.isLoggable(Level.FINE)) {
+                LOGGER.fine("reading json from " + sourceFile);
+            }
             return new JSONObject(FileUtils.readFileToString(sourceFile));
         } catch (JSONException e) {
             throw new RuntimeException(e);
@@ -148,6 +154,9 @@ public class Checkpoint implements InitializingBean {
     public BufferedWriter saveWriter(String beanName, String extraName) throws IOException {
         try {
             File targetFile = new File(getCheckpointDir().getFile(),beanName+"-"+extraName);
+            if (LOGGER.isLoggable(Level.FINE)) {
+                LOGGER.fine("opening for writing: " + targetFile);
+            }
             return new BufferedWriter(new FileWriter(targetFile)); 
         } catch (IOException e) {
             LOGGER.log(Level.SEVERE,"unable to save checkpoint writer state "+extraName+" of "+beanName,e);

engine/src/main/java/org/archive/crawler/framework/CheckpointService.java

@@ -40,6 +40,7 @@ import org.archive.spring.ConfigPath;
 import org.archive.spring.ConfigPathConfigurer;
 import org.archive.spring.HasValidator;
 import org.springframework.beans.BeansException;
+import org.springframework.beans.factory.BeanFactoryUtils;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.context.ApplicationContext;
 import org.springframework.context.ApplicationContextAware;
@@ -222,6 +223,9 @@ public class CheckpointService implements Lifecycle, ApplicationContextAware, Ha
         }
         
         Map<String,Checkpointable> toCheckpoint = appCtx.getBeansOfType(Checkpointable.class);
+        if (LOGGER.isLoggable(Level.FINE)) {
+            LOGGER.fine("checkpointing beans " + toCheckpoint);
+        }
         
         checkpointInProgress = new Checkpoint();
         try {

engine/src/main/resources/org/archive/crawler/restlet/profile-crawler-beans.cxml

@@ -10,11 +10,11 @@
    behavior, uncomment AND alter the shown values.)   
  -->
 <beans xmlns="http://www.springframework.org/schema/beans"
-	     xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
-         xmlns:context="http://www.springframework.org/schema/context"
-	     xmlns:aop="http://www.springframework.org/schema/aop"
-	     xmlns:tx="http://www.springframework.org/schema/tx"
-	     xsi:schemaLocation="http://www.springframework.org/schema/beans http://www.springframework.org/schema/beans/spring-beans-3.0.xsd
+        xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+        xmlns:context="http://www.springframework.org/schema/context"
+        xmlns:aop="http://www.springframework.org/schema/aop"
+        xmlns:tx="http://www.springframework.org/schema/tx"
+        xsi:schemaLocation="http://www.springframework.org/schema/beans http://www.springframework.org/schema/beans/spring-beans-3.0.xsd
            http://www.springframework.org/schema/aop http://www.springframework.org/schema/aop/spring-aop-3.0.xsd
            http://www.springframework.org/schema/tx http://www.springframework.org/schema/tx/spring-tx-3.0.xsd
            http://www.springframework.org/schema/context http://www.springframework.org/schema/context/spring-context-3.0.xsd">
@@ -109,6 +109,25 @@ http://example.example/example
  </bean>
   -->
  
+ <bean id="acceptSurts" class="org.archive.modules.deciderules.surt.SurtPrefixedDecideRule">
+  <!-- <property name="decision" value="ACCEPT"/> -->
+  <!-- <property name="seedsAsSurtPrefixes" value="true" /> -->
+  <!-- <property name="alsoCheckVia" value="false" /> -->
+  <!-- <property name="surtsSourceFile" value="" /> -->
+  <!-- <property name="surtsDumpFile" value="${launchId}/surts.dump" /> -->
+  <!-- <property name="surtsSource">
+        <bean class="org.archive.spring.ConfigString">
+         <property name="value">
+          <value>
+           # example.com
+           # http://www.example.edu/path1/
+           # +http://(org,example,
+          </value>
+         </property> 
+        </bean>
+       </property> -->
+ </bean>
+
  <!-- SCOPE: rules for which discovered URIs to crawl; order is very 
       important because last decision returned other than 'NONE' wins. -->
  <bean id="scope" class="org.archive.modules.deciderules.DecideRuleSequence">
@@ -116,26 +135,9 @@ http://example.example/example
   <property name="rules">
    <list>
     <!-- Begin by REJECTing all... -->
-    <bean class="org.archive.modules.deciderules.RejectDecideRule">
-    </bean>
+    <bean class="org.archive.modules.deciderules.RejectDecideRule" />
     <!-- ...then ACCEPT those within configured/seed-implied SURT prefixes... -->
-    <bean class="org.archive.modules.deciderules.surt.SurtPrefixedDecideRule">
-     <!-- <property name="seedsAsSurtPrefixes" value="true" /> -->
-     <!-- <property name="alsoCheckVia" value="false" /> -->
-     <!-- <property name="surtsSourceFile" value="" /> -->
-     <!-- <property name="surtsDumpFile" value="${launchId}/surts.dump" /> -->
-     <!-- <property name="surtsSource">
-           <bean class="org.archive.spring.ConfigString">
-            <property name="value">
-             <value>
-              # example.com
-              # http://www.example.edu/path1/
-              # +http://(org,example,
-             </value>
-            </property> 
-           </bean>
-          </property> -->
-    </bean>
+    <ref bean="acceptSurts" />
     <!-- ...but REJECT those more than a configured link-hop-count from start... -->
     <bean class="org.archive.modules.deciderules.TooManyHopsDecideRule">
      <!-- <property name="maxHops" value="20" /> -->

modules/src/main/java/org/archive/modules/deciderules/surt/SurtPrefixedDecideRule.java

@@ -23,10 +23,13 @@ import java.io.File;
 import java.io.FileWriter;
 import java.io.IOException;
 import java.io.Reader;
+import java.util.logging.Level;
 import java.util.logging.Logger;
 
 import org.apache.commons.io.IOUtils;
 import org.apache.commons.lang.StringUtils;
+import org.archive.checkpointing.Checkpoint;
+import org.archive.checkpointing.Checkpointable;
 import org.archive.io.ReadSource;
 import org.archive.modules.CrawlURI;
 import org.archive.modules.deciderules.DecideResult;
@@ -36,6 +39,10 @@ import org.archive.modules.seeds.SeedModule;
 import org.archive.net.UURI;
 import org.archive.spring.ConfigFile;
 import org.archive.util.SurtPrefixSet;
+import org.json.JSONArray;
+import org.json.JSONException;
+import org.json.JSONObject;
+import org.springframework.beans.factory.BeanNameAware;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.context.ApplicationEvent;
 import org.springframework.context.ApplicationListener;
@@ -56,7 +63,8 @@ import org.springframework.context.event.ContextStartedEvent;
  * @author gojomo
  */
 public class SurtPrefixedDecideRule extends PredicatedDecideRule implements
-        SeedListener, ApplicationListener<ApplicationEvent> {
+        SeedListener, ApplicationListener<ApplicationEvent>, Checkpointable,
+        BeanNameAware {
     
     private static final long serialVersionUID = 3L;
     private static final Logger logger =
@@ -219,6 +227,9 @@ public class SurtPrefixedDecideRule extends PredicatedDecideRule implements
      */
     protected void buildSurtPrefixSet() {
         if (getSurtsSource() != null) {
+            if (logger.isLoggable(Level.FINE)) {
+                logger.fine("reading surt prefixes from " + getSurtsSource());
+            }
             Reader reader = getSurtsSource().obtainReader();
             try {
                 surtPrefixes.importFromMixed(reader, true);
@@ -277,12 +288,56 @@ public class SurtPrefixedDecideRule extends PredicatedDecideRule implements
         }
         throw new IllegalArgumentException("decision must be ACCEPT or REJECT");
     }
-    
+
     @Override
     public void onApplicationEvent(ApplicationEvent event) {
         if (event instanceof ContextStartedEvent) {
-            readPrefixes();
+            if (recoveryCheckpoint != null) {
+                JSONObject json = recoveryCheckpoint.loadJson(beanName);
+                try {
+                    JSONArray jsonArray = json.getJSONArray("surtPrefixes");
+                    for (int i = 0; i < jsonArray.length(); i++) {
+                        surtPrefixes.add(jsonArray.getString(i));
+                    }
+                } catch (JSONException e) {
+                    throw new IllegalStateException(e);
+                }
+            } else {
+                readPrefixes();
+            }
         }
     }
+    
+    // BeanNameAware
+    protected String beanName; 
+    public void setBeanName(String name) {
+        this.beanName = name;
+    }
+
+    @Override
+    public void startCheckpoint(Checkpoint checkpointInProgress) {
+    }
+    
+    @Override
+    public void doCheckpoint(Checkpoint checkpointInProgress)
+            throws IOException {
+        try {
+            JSONObject json = new JSONObject();
+            json.put("surtPrefixes", surtPrefixes);
+            checkpointInProgress.saveJson(beanName, json);
+        } catch (JSONException e) {
+            throw new RuntimeException(e);
+        }
+    }
+    
+    @Override
+    public void finishCheckpoint(Checkpoint checkpointInProgress) {
+    }
+    
+    protected Checkpoint recoveryCheckpoint;
+    @Override
+    public void setRecoveryCheckpoint(Checkpoint recoveryCheckpoint) {
+        this.recoveryCheckpoint = recoveryCheckpoint;
+    }
 
 }//EOC