HER-1901 timestamped subdirectory for each launch

* HardLinker.java
    renamed FilesystemLinkMaker.java
* FilesystemLinkMaker.java
    add support for symbolic links
* CLibrary.java
    new method symlink()
* BdbModule.java
    use new class name FilesystemLinkMaker
* CrawlJob.java
    at crawl launch, create launch directory launch-{timestamp17}, copy cxml there, symlink "current" to launch dir, inform ConfigPaths
* ConfigPath.java
    interpolate ${launch-id} in configured paths
* ConfigFile.java
    obtainReader() - snapshot config files to launch dir when they are read
* ActionDirectory.java
    default doneDir now ${launch-id}/actions-done
    actOn() - symlink from old style done dir action/done to done files
* SurtPrefixedDecideRule.java
    default surtsDumpFile now ${launch-id}/surts.dump
    pathsFixedUp() - this gets called at build time, but we don't want anything written to disk until launch time, so remove call to dumpSurtPrefixSet() here
* CrawlerLoggerModule.java
    default logs dir now ${launch-id}/logs
* StatisticsTracker.java 
    default reports dir now ${launch-id}/reports
* WriterPoolProcessor.java
    default writer base path now ${launch-id} 
* profile-crawler-beans.cxml
    update to reflect new default paths under launch dirs
* PropertyUtils.java
    fix javadoc typo

engine/src/main/java/org/archive/crawler/framework/ActionDirectory.java

@@ -41,6 +41,7 @@ import org.apache.commons.lang.StringUtils;
 import org.archive.modules.seeds.SeedModule;
 import org.archive.spring.ConfigPath;
 import org.archive.util.ArchiveUtils;
+import org.archive.util.FilesystemLinkMaker;
 import org.springframework.beans.BeansException;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.context.ApplicationContext;
@@ -100,9 +101,6 @@ public class ActionDirectory implements ApplicationContextAware, Lifecycle, Runn
         this.delaySeconds = delay;
     }
     
-    /**
-     * Scratch directory for temporary overflow-to-disk
-     */
     protected ConfigPath actionDir = 
         new ConfigPath("ActionDirectory source directory","action");
     public ConfigPath getActionDir() {
@@ -112,16 +110,13 @@ public class ActionDirectory implements ApplicationContextAware, Lifecycle, Runn
         this.actionDir = actionDir;
     }
     
-    /**
-     * Scratch directory for temporary overflow-to-disk
-     */
     protected ConfigPath doneDir = 
-        new ConfigPath("ActionDirectory done directory","action/done");
+        new ConfigPath("ActionDirectory done directory","${launch-id}/actions-done");
     public ConfigPath getDoneDir() {
         return doneDir;
     }
-    public void setDoneDir(ConfigPath scratchDir) {
-        this.doneDir = scratchDir;
+    public void setDoneDir(ConfigPath doneDir) {
+        this.doneDir = doneDir;
     }
     
     ApplicationContext appCtx;
@@ -258,7 +253,19 @@ public class ActionDirectory implements ApplicationContextAware, Lifecycle, Runn
         // move file to 'done' area with timestamp prefix
         while(actionFile.exists()) {
             try {
-                FileUtils.moveFile(actionFile, new File(doneDir.getFile(),timestamp+"."+actionFile.getName()));
+                File doneFile = new File(doneDir.getFile(),timestamp+"."+actionFile.getName());
+                FileUtils.moveFile(actionFile, doneFile);
+                
+                // attempt to symlink from action/done/ to done file
+                File actionDoneDirFile = new File(actionDir.getFile(), "done");
+                if (!actionDoneDirFile.equals(doneDir.getFile())) {
+                    actionDoneDirFile.mkdirs();
+                    File doneSymlinkFile = new File(actionDoneDirFile, doneFile.getName());
+                    boolean success = FilesystemLinkMaker.makeSymbolicLink(doneFile.getPath(), doneSymlinkFile.getPath());
+                    if (!success) {
+                        LOGGER.warning("failed to create symlink from " + doneSymlinkFile + " to " + doneFile);
+                    }
+                }
             } catch (IOException e) {
                 LOGGER.log(Level.SEVERE,"unable to move "+actionFile,e);
             }

engine/src/main/java/org/archive/crawler/framework/CrawlJob.java

@@ -57,6 +57,7 @@ import org.archive.spring.ConfigPath;
 import org.archive.spring.ConfigPathConfigurer;
 import org.archive.spring.PathSharingContext;
 import org.archive.util.ArchiveUtils;
+import org.archive.util.FilesystemLinkMaker;
 import org.archive.util.TextUtils;
 import org.joda.time.DateTime;
 import org.springframework.beans.BeanWrapperImpl;
@@ -80,7 +81,7 @@ import org.xml.sax.SAXException;
  * 
  * @contributor gojomo
  */
-public class CrawlJob implements Comparable<CrawlJob>, ApplicationListener {
+public class CrawlJob implements Comparable<CrawlJob>, ApplicationListener<ApplicationEvent> {
     private final static Logger LOGGER =
         Logger.getLogger(CrawlJob.class.getName());
 
@@ -97,7 +98,7 @@ public class CrawlJob implements Comparable<CrawlJob>, ApplicationListener {
     public CrawlJob(File cxml) {
         primaryConfig = cxml; 
         isLaunchInfoPartial = false;
-        scanJobLog(); 
+        scanJobLog(); // XXX look at launch directories instead/first? 
         alertThreadGroup = new AlertThreadGroup(getShortName());
     }
     
@@ -428,8 +429,9 @@ public class CrawlJob implements Comparable<CrawlJob>, ApplicationListener {
         alertThreadGroup.addLogger(getJobLogger());
         Thread launcher = new Thread(alertThreadGroup, getShortName()+" launchthread") {
             public void run() {
-                startContext();
                 CrawlController cc = getCrawlController();
+                initLaunchDir();
+                startContext();
                 if(cc!=null) {
                     cc.requestCrawlStart();
                 }
@@ -446,6 +448,50 @@ public class CrawlJob implements Comparable<CrawlJob>, ApplicationListener {
         }
     }
     
+    protected transient String currentLaunchId;
+    public void initLaunchId() {
+        currentLaunchId = "launch-" + ArchiveUtils.get17DigitDate();
+        LOGGER.info("launch id " + currentLaunchId);
+    }
+    public String getCurrentLaunchId() {
+        return currentLaunchId;
+    }
+
+    protected transient File currentLaunchDir;
+    public File getCurrentLaunchDir() {
+        return currentLaunchDir;
+    }
+    
+    protected void initLaunchDir() {
+        initLaunchId();
+        try {
+            currentLaunchDir = new File(getJobDir(), getCurrentLaunchId());
+            if (!currentLaunchDir.mkdir()) {
+                throw new IOException("failed to create directory " + currentLaunchDir);
+            }
+            
+            // copy cxml to launch dir
+            FileUtils.copyFileToDirectory(getPrimaryConfig(), currentLaunchDir);
+            
+            // attempt to symlink "current" to launch dir
+            File currentSymlink = new File(getJobDir(), "current");
+            currentSymlink.delete();
+            boolean success = FilesystemLinkMaker.makeSymbolicLink(currentLaunchDir.getName(), currentSymlink.getPath());
+            if (!success) {
+                LOGGER.warning("failed to create symlink from " + currentSymlink + " to " + currentLaunchDir);
+            }
+        } catch (IOException e) {
+            LOGGER.log(Level.SEVERE, "failed to initialize launch directory: " + e);
+            currentLaunchDir = null;
+        }
+        
+        // fill in ${launch-id} in all config paths, and let config files know
+        // where to snapshot themselves
+        for (ConfigPath configPath: getConfigPaths().values()) {
+            configPath.informOfLaunch(getCurrentLaunchId(), getCurrentLaunchDir());
+        }
+    }
+
     /**
      * Start the context, catching and reporting any BeansExceptions.
      */
@@ -515,7 +561,6 @@ public class CrawlJob implements Comparable<CrawlJob>, ApplicationListener {
      * 
      * @return Checkpointer
      */
-    @SuppressWarnings("unchecked")
     public synchronized CheckpointService getCheckpointService() {
         if(ac==null) {
             return null;

engine/src/main/java/org/archive/crawler/reporting/CrawlerLoggerModule.java

@@ -62,7 +62,7 @@ public class CrawlerLoggerModule
         Checkpointable, SimpleFileLoggerProvider {
     private static final long serialVersionUID = 1L;
 
-    protected ConfigPath path = new ConfigPath(Engine.LOGS_DIR_NAME,"logs"); 
+    protected ConfigPath path = new ConfigPath(Engine.LOGS_DIR_NAME,"${launch-id}/logs"); 
     public ConfigPath getPath() {
         return path;
     }

engine/src/main/java/org/archive/crawler/reporting/StatisticsTracker.java

@@ -130,9 +130,9 @@ import com.sleepycat.je.DatabaseException;
 public class StatisticsTracker 
     implements 
         ApplicationContextAware, 
-        ApplicationListener,
+        ApplicationListener<ApplicationEvent>,
         SeedListener,
-        Lifecycle, 
+        Lifecycle,
         Runnable, 
         Checkpointable,
         BeanNameAware {
@@ -153,7 +153,7 @@ public class StatisticsTracker
         this.bdb = bdb;
     }
 
-    protected ConfigPath reportsDir = new ConfigPath(Engine.REPORTS_DIR_NAME,"reports");
+    protected ConfigPath reportsDir = new ConfigPath(Engine.REPORTS_DIR_NAME,"${launch-id}/reports");
     public ConfigPath getReportsDir() {
         return reportsDir;
     }