More on HER-1901 - support ${launch-id} interpolation on W/ARCWriterProcessor storePaths

* WriterPoolProcessor.java, ARCWriterProcessor.java, WARCWriterProcessor.java
    change type of storePaths to List<ConfigPath> and handle appropriately
* ConfigPathConfigurer.java
    fixupPaths() - old code did not touch WriterPoolProcessor storePaths, since they're deeply nested inside the bean, but they need to be remembered for later interpolation of ${launch-id}, so add special handling

commons/src/main/java/org/archive/spring/ConfigPathConfigurer.java

@@ -22,9 +22,11 @@ package org.archive.spring;
 import java.beans.PropertyDescriptor;
 import java.io.File;
 import java.util.HashMap;
+import java.util.List;
 import java.util.Map;
 
 import org.apache.commons.lang.StringUtils;
+import org.archive.modules.writer.WriterPoolProcessor;
 import org.springframework.beans.BeanWrapperImpl;
 import org.springframework.beans.BeansException;
 import org.springframework.beans.factory.config.BeanPostProcessor;
@@ -125,6 +127,16 @@ implements
                 }
             }
         }
+        
+        // we want to remember writer pool store paths for later interpolation
+        // (but their base dirs are taken care of elsewhere)
+        if (bean instanceof WriterPoolProcessor) {
+            List<ConfigPath> storePaths = ((WriterPoolProcessor) bean).getStorePaths();
+            for (int i = 0; i < storePaths.size(); i++) {
+                remember(beanName + "." + "storePaths[" + i + "]", storePaths.get(i));
+            }
+        }
+        
         if(bean instanceof PathFixupListener) {
             ((PathFixupListener)bean).pathsFixedUp();
         }

modules/src/main/java/org/archive/modules/writer/ARCWriterProcessor.java

@@ -38,8 +38,9 @@ import org.archive.io.ReplayInputStream;
 import org.archive.io.WriterPoolMember;
 import org.archive.io.arc.ARCWriter;
 import org.archive.io.arc.ARCWriterPool;
-import org.archive.modules.ProcessResult;
 import org.archive.modules.CrawlURI;
+import org.archive.modules.ProcessResult;
+import org.archive.spring.ConfigPath;
 import org.archive.util.ArchiveUtils;
 
 /**
@@ -64,9 +65,9 @@ public class ARCWriterProcessor extends WriterPoolProcessor {
     public long getDefaultMaxFileSize() {
         return 100000000L; // 100 SI mega-bytes (10^8 bytes)
     }
-    public List<String> getDefaultStorePaths() {
-        List<String> paths = new ArrayList<String>();
-        paths.add("arcs");
+    public List<ConfigPath> getDefaultStorePaths() {
+        List<ConfigPath> paths = new ArrayList<ConfigPath>();
+        paths.add(new ConfigPath("arcs default store path", "arcs"));
         return paths;
     }
 

modules/src/main/java/org/archive/modules/writer/WARCWriterProcessor.java

@@ -81,6 +81,7 @@ import org.archive.modules.CrawlURI;
 import org.archive.modules.ProcessResult;
 import org.archive.modules.deciderules.recrawl.IdenticalDigestDecideRule;
 import org.archive.modules.extractor.Link;
+import org.archive.spring.ConfigPath;
 import org.archive.uid.RecordIDGenerator;
 import org.archive.uid.UUIDGenerator;
 import org.archive.util.ArchiveUtils;
@@ -107,9 +108,9 @@ public class WARCWriterProcessor extends WriterPoolProcessor implements WARCWrit
     public long getDefaultMaxFileSize() {
         return 1000000000L; // 1 SI giga-byte (10^9 bytes), per WARC appendix A
     }
-    public List<String> getDefaultStorePaths() {
-        List<String> paths = new ArrayList<String>();
-        paths.add("warcs");
+    public List<ConfigPath> getDefaultStorePaths() {
+        List<ConfigPath> paths = new ArrayList<ConfigPath>();
+        paths.add(new ConfigPath("warcs default store path", "warcs"));
         return paths;
     }
     

modules/src/main/java/org/archive/modules/writer/WriterPoolProcessor.java

@@ -241,12 +241,12 @@ implements Lifecycle, Checkpointable, WriterPoolSettings {
      * setting is safe to change midcrawl (You can remove and add new 
      * dirs as the crawler progresses).
      */
-    List<String> storePaths = getDefaultStorePaths();
-    abstract List<String> getDefaultStorePaths();
-    public List<String> getStorePaths() {
+    List<ConfigPath> storePaths = getDefaultStorePaths();
+    abstract List<ConfigPath> getDefaultStorePaths();
+    public List<ConfigPath> getStorePaths() {
         return storePaths;
     }
-    public void setStorePaths(List<String> paths) {
+    public void setStorePaths(List<ConfigPath> paths) {
         this.storePaths = paths; 
     }
     
@@ -430,12 +430,11 @@ implements Lifecycle, Checkpointable, WriterPoolSettings {
     public abstract List<String> getMetadata();
     
     public List<File> getOutputDirs() {
-        List<String> list = getStorePaths();
+        List<ConfigPath> list = getStorePaths();
         ArrayList<File> results = new ArrayList<File>();
-        for (String path: list) {
-            File f = new File(
-                    path.startsWith("/") ? null : getDirectory().getFile(), 
-                    path);
+        for (ConfigPath path: list) {
+            path.setBase(getDirectory());
+            File f = path.getFile();
             if (!f.exists()) {
                 try {
                     f.mkdirs();