More on HER-1901 - fix build by refactoring launch dir initialization into PathSharingContext; refactor config path interpolation mostly into ConfigPathConfigurer; various tweaks

* ActionDirectory.java, CrawlerLoggerModule.java, StatisticsTracker.java, SurtPrefixedDecideRule.java, WriterPoolProcessor.java, profile-crawler-beans.cxml
    use camelcase ${launchId} 
* CheckpointService.java, JobResource.java
    rename getAvailableCheckpointDirectories() to findAvailableCheckpointDirectories() so that it's not treated as a bean property
* WriterPoolProcessor.java, WriterPoolSettingsData.java, WriterPoolMember.java
    rename getOutputDirs() to calcOutputDirs() so that it's not treated as a bean property
* ConfigPath.java
    let ConfigPathConfigurer to do interpolation of ${launchId}
* ConfigFile.java
    let ConfigPathConfigurer do the snapshotting of config file
* CrawlJob.java, PathSharingContext.java
    remove launchDir initialization out of CrawlJob into PathSharingContext
* ConfigPathConfigurer.java
    - remove special handling of WriterPoolProcessor store paths; instead, look in beans for ConfigPaths within Iterables
    - have each ConfigPath hold reference to this ConfigPathConfigurer to use for interpolating ${launchId} and snapshotting config files

engine/src/main/java/org/archive/crawler/framework/ActionDirectory.java

@@ -111,7 +111,7 @@ public class ActionDirectory implements ApplicationContextAware, Lifecycle, Runn
     }
     
     protected ConfigPath doneDir = 
-        new ConfigPath("ActionDirectory done directory","${launch-id}/actions-done");
+        new ConfigPath("ActionDirectory done directory","${launchId}/actions-done");
     public ConfigPath getDoneDir() {
         return doneDir;
     }

engine/src/main/java/org/archive/crawler/framework/CheckpointService.java

@@ -37,6 +37,7 @@ import org.archive.checkpointing.Checkpoint;
 import org.archive.checkpointing.Checkpointable;
 import org.archive.crawler.reporting.CrawlStatSnapshot;
 import org.archive.spring.ConfigPath;
+import org.archive.spring.ConfigPathConfigurer;
 import org.archive.spring.HasValidator;
 import org.springframework.beans.BeansException;
 import org.springframework.beans.factory.annotation.Autowired;
@@ -223,7 +224,6 @@ public class CheckpointService implements Lifecycle, ApplicationContextAware, Ha
         checkpointInProgress = new Checkpoint();
         checkpointInProgress.generateFrom(getCheckpointsDir(),getNextCheckpointNumber());
         
-        @SuppressWarnings("unchecked")
         Map<String,Checkpointable> toCheckpoint = appCtx.getBeansOfType(Checkpointable.class);
         
         try {
@@ -288,8 +288,8 @@ public class CheckpointService implements Lifecycle, ApplicationContextAware, Ha
         if(getRecoveryCheckpoint()!=null || isRunning()) {
             return false;
         }
-        return (getAvailableCheckpointDirectories() != null 
-                && getAvailableCheckpointDirectories().size() > 0);
+        return (findAvailableCheckpointDirectories() != null 
+                && findAvailableCheckpointDirectories().size() > 0);
     }
 
     /**
@@ -300,7 +300,7 @@ public class CheckpointService implements Lifecycle, ApplicationContextAware, Ha
      * @return List of valid checkpoint directory File instances
      */
     @SuppressWarnings("unchecked")
-    public List<File> getAvailableCheckpointDirectories() {
+    public List<File> findAvailableCheckpointDirectories() {
         File[] dirs = getCheckpointsDir().getFile().listFiles((FileFilter)FileFilterUtils.directoryFileFilter());
         if (dirs == null) {
             return Collections.EMPTY_LIST;
@@ -325,7 +325,6 @@ public class CheckpointService implements Lifecycle, ApplicationContextAware, Ha
      * 
      * @param selectedCheckpoint
      */
-    @SuppressWarnings("unchecked")
     public void setRecoveryCheckpointByName(String selectedCheckpoint) {
         if(isRunning) {
             throw new RuntimeException("may not set recovery Checkpoint after launch");
@@ -333,6 +332,7 @@ public class CheckpointService implements Lifecycle, ApplicationContextAware, Ha
         Checkpoint recoveryCheckpoint = new Checkpoint();
         recoveryCheckpoint.getCheckpointDir().setBase(getCheckpointsDir());
         recoveryCheckpoint.getCheckpointDir().setPath(selectedCheckpoint);
+        recoveryCheckpoint.getCheckpointDir().setConfigurer(appCtx.getBean(ConfigPathConfigurer.class));
         recoveryCheckpoint.afterPropertiesSet();
         setRecoveryCheckpoint(recoveryCheckpoint);
         Map<String,Checkpointable> toSetRecovery = appCtx.getBeansOfType(Checkpointable.class);

engine/src/main/java/org/archive/crawler/framework/CrawlJob.java

@@ -57,7 +57,6 @@ import org.archive.spring.ConfigPath;
 import org.archive.spring.ConfigPathConfigurer;
 import org.archive.spring.PathSharingContext;
 import org.archive.util.ArchiveUtils;
-import org.archive.util.FilesystemLinkMaker;
 import org.archive.util.TextUtils;
 import org.joda.time.DateTime;
 import org.springframework.beans.BeanWrapperImpl;
@@ -430,7 +429,6 @@ public class CrawlJob implements Comparable<CrawlJob>, ApplicationListener<Appli
         Thread launcher = new Thread(alertThreadGroup, getShortName()+" launchthread") {
             public void run() {
                 CrawlController cc = getCrawlController();
-                initLaunchDir();
                 startContext();
                 if(cc!=null) {
                     cc.requestCrawlStart();
@@ -448,57 +446,7 @@ public class CrawlJob implements Comparable<CrawlJob>, ApplicationListener<Appli
         }
     }
     
-    protected transient String currentLaunchId;
-    protected void initLaunchId() {
-        currentLaunchId = ArchiveUtils.get17DigitDate();
-        LOGGER.info("launch id " + currentLaunchId);
-    }
-    public String getCurrentLaunchId() {
-        return currentLaunchId;
-    }
-
-    protected transient File currentLaunchDir;
-    public File getCurrentLaunchDir() {
-        return currentLaunchDir;
-    }
-    
     protected transient Handler currentLaunchJobLogHandler;
-    protected void initLaunchDir() {
-        initLaunchId();
-        try {
-            currentLaunchDir = new File(getJobDir(), getCurrentLaunchId());
-            if (!currentLaunchDir.mkdir()) {
-                throw new IOException("failed to create directory " + currentLaunchDir);
-            }
-            
-            // job log file covering just this launch
-            getJobLogger().removeHandler(currentLaunchJobLogHandler);
-            File f = new File(currentLaunchDir, "job.log");
-            currentLaunchJobLogHandler = new FileHandler(f.getAbsolutePath(), true);
-            currentLaunchJobLogHandler.setFormatter(new JobLogFormatter());
-            getJobLogger().addHandler(currentLaunchJobLogHandler);
-            
-            // copy cxml to launch dir
-            FileUtils.copyFileToDirectory(getPrimaryConfig(), currentLaunchDir);
-            
-            // attempt to symlink "current" to launch dir
-            File currentSymlink = new File(getJobDir(), "current");
-            currentSymlink.delete();
-            boolean success = FilesystemLinkMaker.makeSymbolicLink(currentLaunchDir.getName(), currentSymlink.getPath());
-            if (!success) {
-                LOGGER.warning("failed to create symlink from " + currentSymlink + " to " + currentLaunchDir);
-            }
-        } catch (IOException e) {
-            LOGGER.log(Level.SEVERE, "failed to initialize launch directory: " + e);
-            currentLaunchDir = null;
-        }
-        
-        // fill in ${launch-id} in all config paths, and let config files know
-        // where to snapshot themselves
-        for (ConfigPath configPath: getConfigPaths().values()) {
-            configPath.informOfLaunch(getCurrentLaunchId(), getCurrentLaunchDir());
-        }
-    }
 
     /**
      * Start the context, catching and reporting any BeansExceptions.
@@ -506,6 +454,15 @@ public class CrawlJob implements Comparable<CrawlJob>, ApplicationListener<Appli
     protected synchronized void startContext() {
         try {
             ac.start(); 
+            
+            // job log file covering just this launch
+            getJobLogger().removeHandler(currentLaunchJobLogHandler);
+            File f = new File(ac.getCurrentLaunchDir(), "job.log");
+            currentLaunchJobLogHandler = new FileHandler(f.getAbsolutePath(), true);
+            currentLaunchJobLogHandler.setFormatter(new JobLogFormatter());
+            getJobLogger().addHandler(currentLaunchJobLogHandler);
+            
+            
         } catch (BeansException be) {
             ac.close();
             ac = null; 
@@ -606,6 +563,7 @@ public class CrawlJob implements Comparable<CrawlJob>, ApplicationListener<Appli
         }
         xmlOkAt = new DateTime(0);
         getJobLogger().removeHandler(currentLaunchJobLogHandler);
+        currentLaunchJobLogHandler.close();
         getJobLogger().log(Level.INFO,"Job instance discarded");
         return true; 
     }
@@ -701,7 +659,8 @@ public class CrawlJob implements Comparable<CrawlJob>, ApplicationListener<Appli
      */
     public void onApplicationEvent(ApplicationEvent event) {
         if(event instanceof CrawlStateEvent) {
-            getJobLogger().log(Level.INFO, ((CrawlStateEvent)event).getState().toString());
+            getJobLogger().log(Level.INFO, ((CrawlStateEvent)event).getState() + 
+                    ac.getCurrentLaunchId() != null ? " " + ac.getCurrentLaunchId() : "");
         }
         if(event instanceof CheckpointSuccessEvent) {
             getJobLogger().log(Level.INFO, "CHECKPOINTED "+((CheckpointSuccessEvent)event).getCheckpoint().getName());

engine/src/main/java/org/archive/crawler/reporting/CrawlerLoggerModule.java

@@ -62,7 +62,7 @@ public class CrawlerLoggerModule
         Checkpointable, SimpleFileLoggerProvider {
     private static final long serialVersionUID = 1L;
 
-    protected ConfigPath path = new ConfigPath(Engine.LOGS_DIR_NAME,"${launch-id}/logs"); 
+    protected ConfigPath path = new ConfigPath(Engine.LOGS_DIR_NAME,"${launchId}/logs"); 
     public ConfigPath getPath() {
         return path;
     }

engine/src/main/java/org/archive/crawler/reporting/StatisticsTracker.java

@@ -153,7 +153,7 @@ public class StatisticsTracker
         this.bdb = bdb;
     }
 
-    protected ConfigPath reportsDir = new ConfigPath(Engine.REPORTS_DIR_NAME,"${launch-id}/reports");
+    protected ConfigPath reportsDir = new ConfigPath(Engine.REPORTS_DIR_NAME,"${launchId}/reports");
     public ConfigPath getReportsDir() {
         return reportsDir;
     }

engine/src/main/java/org/archive/crawler/restlet/JobResource.java

@@ -304,7 +304,7 @@ public class JobResource extends BaseResource {
             } else if (cj.getCheckpointService().hasAvailableCheckpoints() && cj.isLaunchable()) {
                 pw.println("select an available checkpoint before launch to recover:");
                 pw.println("<select name='checkpoint'><option> </option>");
-                for(File f : cj.getCheckpointService().getAvailableCheckpointDirectories()) {
+                for(File f : cj.getCheckpointService().findAvailableCheckpointDirectories()) {
                     pw.println("<option>"+f.getName()+"</option>");
                 }
                 pw.println("</select>");