Fix HER-1935 Many calls to File.mkdirs() and other file/dir methods don't check the return value. Also fix bug where pointless empty directories were created in scratch dir.

* BasicProfileTest.java, SelfTestBase.java, CrawlControllerTest.java, PrecedenceLoader.java, MigrateH1to3Tool.java, CrawlerLoggerModule.java, StatisticsTracker.java, CheckpointUtils.java, BdbUriUniqFilter.java, ARCWriterProcessorTest.java, WARCWriterProcessorTest.java, PersistProcessor.java, WriterPoolProcessor.java, PrefixFinderTest.java, StoredQueueTest.java, FileUtilsTest.java, ObjectIdentityBdbManualCacheTest.java, ObjectIdentityBdbCacheTest.java, ObjectPlusFilesOutputStream.java, TestUtils.java, TmpDirTestCase.java, Engine.java
    Replace calls to File.mkdirs() with FileUtils.ensureWriteableDirectory(dir). In these cases the calls were either already in a spot where the possible IOException would be handled appropriately, or the line was trivially moved into such a block.
* ActionDirectory.java, Engine.java
    replace calls to File.mkdirs() with FileUtils.ensureWriteableDirectory(dir), and throw IllegalStateException on failure
* BdbModule.java
    setup() - replace call to File.mkdirs() with FileUtils.ensureWriteableDirectory(dir) and add "throws IOException" - conveniently the place where this method is called was already in a try block that catches IOException
* Checkpoint.java
    generateFrom() - replace call to File.mkdirs() with FileUtils.ensureWriteableDirectory(dir) and add "throws IOException"
* CheckpointService.java
    move call to Checkpoint.generateFrom() inside existing try block since it now can throw IOException
* Recorder.java
    ensure(File) - replace call to File.mkdirs() with FileUtils.ensureWriteableDirectory(dir), and throw IllegalStateException on failure
    new Recorder(File,String,int,int) - call ensure() on the correct object, the containing directory; and remove redundant call to ensure()

engine/src/main/java/org/archive/crawler/framework/ActionDirectory.java

@@ -151,9 +151,13 @@ public class ActionDirectory implements ApplicationContextAware, Lifecycle, Runn
         if (isRunning()) {
             return;
         }
-        // create directories
-        getActionDir().getFile().mkdirs();
-        getDoneDir().getFile().mkdirs();
+        try {
+            // create directories
+            org.archive.util.FileUtils.ensureWriteableDirectory(getActionDir().getFile());
+            org.archive.util.FileUtils.ensureWriteableDirectory(getDoneDir().getFile());
+        } catch (IOException e) {
+            throw new IllegalStateException(e);
+        }
         // start background executor
         executor = Executors.newSingleThreadScheduledExecutor();
         executor.scheduleWithFixedDelay(this, getInitialDelaySeconds(), getDelaySeconds(), TimeUnit.SECONDS);

engine/src/main/java/org/archive/crawler/framework/CheckpointService.java

@@ -221,12 +221,12 @@ public class CheckpointService implements Lifecycle, ApplicationContextAware, Ha
             }
         }
         
-        checkpointInProgress = new Checkpoint();
-        checkpointInProgress.generateFrom(getCheckpointsDir(),getNextCheckpointNumber());
-        
         Map<String,Checkpointable> toCheckpoint = appCtx.getBeansOfType(Checkpointable.class);
         
+        checkpointInProgress = new Checkpoint();
         try {
+            checkpointInProgress.generateFrom(getCheckpointsDir(),getNextCheckpointNumber());
+            
             // pre (incl. acquire necessary locks)
 //            long startMs = System.currentTimeMillis();
             for(Checkpointable c : toCheckpoint.values()) {

engine/src/main/java/org/archive/crawler/framework/Engine.java

@@ -61,7 +61,12 @@ public class Engine {
     
     public Engine(File jobsDir) {
         this.jobsDir = jobsDir;
-        this.jobsDir.mkdirs();
+        
+        try {
+            org.archive.util.FileUtils.ensureWriteableDirectory(jobsDir);
+        } catch (IOException e) {
+            throw new IllegalStateException(e);
+        }
         
         findJobConfigs();
         // TODO: cleanup any cruft from improperly ended jobs 
@@ -178,7 +183,7 @@ public class Engine {
      */
     public synchronized void copy(CrawlJob orig, File destDir, boolean asProfile) 
     throws IOException {
-        destDir.mkdirs();
+        org.archive.util.FileUtils.ensureWriteableDirectory(destDir);
         if(destDir.list().length>0) {
             throw new IOException("destination dir not empty");
         }
@@ -352,7 +357,7 @@ public class Engine {
             inStream.close();
 
             // write default crawler-beans string to new job dir
-            newJobDir.mkdirs();
+            org.archive.util.FileUtils.ensureWriteableDirectory(newJobDir);
             File newJobCxml = new File(newJobDir,"crawler-beans.cxml");
             FileUtils.writeStringToFile(newJobCxml, defaultCxmlStr);
 

engine/src/main/java/org/archive/crawler/frontier/precedence/PrecedenceLoader.java

@@ -31,6 +31,7 @@ import java.util.Map;
 
 import org.archive.modules.recrawl.PersistProcessor;
 import org.archive.util.ArchiveUtils;
+import org.archive.util.FileUtils;
 import org.archive.util.bdbje.EnhancedEnvironment;
 import org.archive.util.iterator.LineReadingIterator;
 
@@ -100,9 +101,7 @@ public class PrecedenceLoader {
             FileNotFoundException, UnsupportedEncodingException, IOException {
         File source = new File(args[0]);
         File env = new File(args[1]);
-        if(!env.exists()) {
-            env.mkdirs();
-        }
+        FileUtils.ensureWriteableDirectory(env);
         
         // setup target environment
         EnhancedEnvironment targetEnv = PersistProcessor.setupCopyEnvironment(env);

engine/src/main/java/org/archive/crawler/migrate/MigrateH1to3Tool.java

@@ -92,7 +92,7 @@ public class MigrateH1to3Tool {
         }
         File destinationH3JobDir = new File(destinationH3JobDirArg);
 
-        destinationH3JobDir.mkdirs();
+        org.archive.util.FileUtils.ensureWriteableDirectory(destinationH3JobDir);
 
         System.out.println("H1 source: "+sourceOrderXmlFile.getAbsolutePath()); 
         System.out.println("H3 destination: "+destinationH3JobDir.getAbsolutePath());

engine/src/main/java/org/archive/crawler/reporting/CrawlerLoggerModule.java

@@ -46,6 +46,7 @@ import org.archive.modules.extractor.UriErrorLoggerModule;
 import org.archive.net.UURI;
 import org.archive.spring.ConfigPath;
 import org.archive.util.ArchiveUtils;
+import org.archive.util.FileUtils;
 import org.springframework.beans.factory.InitializingBean;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.context.Lifecycle;
@@ -209,9 +210,9 @@ public class CrawlerLoggerModule
         if(isRunning) {
             return; 
         }
-        getPath().getFile().mkdirs();
         this.atg = AlertThreadGroup.current();
         try {
+            FileUtils.ensureWriteableDirectory(getPath().getFile());
             setupLogs();
         } catch (IOException e) {
             throw new IllegalStateException(e);

engine/src/main/java/org/archive/crawler/reporting/StatisticsTracker.java

@@ -58,6 +58,7 @@ import org.archive.modules.seeds.SeedListener;
 import org.archive.modules.seeds.SeedModule;
 import org.archive.spring.ConfigPath;
 import org.archive.util.ArchiveUtils;
+import org.archive.util.FileUtils;
 import org.archive.util.JSONUtils;
 import org.archive.util.MimetypeUtils;
 import org.archive.util.ObjectIdentityCache;
@@ -902,8 +903,8 @@ public class StatisticsTracker
             return f;
         }
         
-        f.getParentFile().mkdirs();
         try {
+            FileUtils.ensureWriteableDirectory(f.getParentFile());
             PrintWriter bw = new PrintWriter(new FileWriter(f));
             report.write(bw, this);
             bw.close();

engine/src/main/java/org/archive/crawler/util/BdbUriUniqFilter.java

@@ -27,6 +27,7 @@ import java.util.logging.Logger;
 import org.archive.bdb.BdbModule;
 import org.archive.checkpointing.Checkpoint;
 import org.archive.checkpointing.Checkpointable;
+import org.archive.util.FileUtils;
 import org.json.JSONException;
 import org.json.JSONObject;
 import org.springframework.beans.factory.BeanNameAware;
@@ -165,9 +166,7 @@ implements Lifecycle, Checkpointable, BeanNameAware {
     public BdbUriUniqFilter(File bdbEnv, final int cacheSizePercentage)
     throws IOException {
         super();
-        if (!bdbEnv.exists()) {
-            bdbEnv.mkdirs();
-        }
+        FileUtils.ensureWriteableDirectory(bdbEnv);
         EnvironmentConfig envConfig = new EnvironmentConfig();
         envConfig.setAllowCreate(true);
         if (cacheSizePercentage > 0 && cacheSizePercentage < 100) {

engine/src/main/java/org/archive/crawler/util/CheckpointUtils.java

@@ -27,6 +27,8 @@ import java.io.IOException;
 import java.io.ObjectInputStream;
 import java.io.ObjectOutputStream;
 
+import org.archive.util.FileUtils;
+
 /**
  * Utilities useful checkpointing.
  * @author stack
@@ -76,7 +78,7 @@ public class CheckpointUtils {
     public static void writeObjectToFile(final Object o, final String suffix,
             final File dir)
     throws IOException {
-        dir.mkdirs();
+        FileUtils.ensureWriteableDirectory(dir);
         ObjectOutputStream out = new ObjectOutputStream(
             new FileOutputStream(getClassCheckpointFile(dir, suffix,
                 o.getClass())));