Fix HER-1935 Many calls to File.mkdirs() and other file/dir methods don't check the return value. Also fix bug where pointless empty directories were created in scratch dir.

* BasicProfileTest.java, SelfTestBase.java, CrawlControllerTest.java, PrecedenceLoader.java, MigrateH1to3Tool.java, CrawlerLoggerModule.java, StatisticsTracker.java, CheckpointUtils.java, BdbUriUniqFilter.java, ARCWriterProcessorTest.java, WARCWriterProcessorTest.java, PersistProcessor.java, WriterPoolProcessor.java, PrefixFinderTest.java, StoredQueueTest.java, FileUtilsTest.java, ObjectIdentityBdbManualCacheTest.java, ObjectIdentityBdbCacheTest.java, ObjectPlusFilesOutputStream.java, TestUtils.java, TmpDirTestCase.java, Engine.java
    Replace calls to File.mkdirs() with FileUtils.ensureWriteableDirectory(dir). In these cases the calls were either already in a spot where the possible IOException would be handled appropriately, or the line was trivially moved into such a block.
* ActionDirectory.java, Engine.java
    replace calls to File.mkdirs() with FileUtils.ensureWriteableDirectory(dir), and throw IllegalStateException on failure
* BdbModule.java
    setup() - replace call to File.mkdirs() with FileUtils.ensureWriteableDirectory(dir) and add "throws IOException" - conveniently the place where this method is called was already in a try block that catches IOException
* Checkpoint.java
    generateFrom() - replace call to File.mkdirs() with FileUtils.ensureWriteableDirectory(dir) and add "throws IOException"
* CheckpointService.java
    move call to Checkpoint.generateFrom() inside existing try block since it now can throw IOException
* Recorder.java
    ensure(File) - replace call to File.mkdirs() with FileUtils.ensureWriteableDirectory(dir), and throw IllegalStateException on failure
    new Recorder(File,String,int,int) - call ensure() on the correct object, the containing directory; and remove redundant call to ensure()

commons/src/main/java/org/archive/bdb/BdbModule.java

@@ -256,7 +256,7 @@ public class BdbModule implements Lifecycle, Checkpointable, Closeable {
     }
     
     protected void setup(File f, boolean create) 
-    throws DatabaseException {
+    throws DatabaseException, IOException {
         EnvironmentConfig config = new EnvironmentConfig();
         config.setAllowCreate(create);
         config.setLockTimeout(75, TimeUnit.MINUTES); // set to max
@@ -282,7 +282,7 @@ public class BdbModule implements Lifecycle, Checkpointable, Closeable {
             config.setConfigParam("je.cleaner.expunge", "false");
         } // else leave whatever other setting was already in place
 
-        f.mkdirs();
+        org.archive.util.FileUtils.ensureWriteableDirectory(f);
         this.bdbEnvironment = new EnhancedEnvironment(f, config);
         this.classCatalog = this.bdbEnvironment.getClassCatalog();
         if(!create) {
@@ -479,7 +479,7 @@ public class BdbModule implements Lifecycle, Checkpointable, Closeable {
                 dbBackup.startBackup();
                 
                 File envCpDir = new File(dir.getFile(),checkpointInProgress.getName());
-                envCpDir.mkdirs();
+                org.archive.util.FileUtils.ensureWriteableDirectory(envCpDir);
                 File logfilesList = new File(envCpDir,"jdbfiles.manifest");
                 String[] filedata = dbBackup.getLogFilesInBackupSet();
                 for (int i=0; i<filedata.length;i++) {

commons/src/main/java/org/archive/checkpointing/Checkpoint.java

@@ -78,15 +78,16 @@ public class Checkpoint implements InitializingBean {
      * 
      * @param checkpointsDir
      * @param nextCheckpointNumber
+     * @throws IOException 
      */
-    public void generateFrom(ConfigPath checkpointsDir, int nextCheckpointNumber) {
+    public void generateFrom(ConfigPath checkpointsDir, int nextCheckpointNumber) throws IOException {
         getCheckpointDir().setBase(checkpointsDir);
         getCheckpointDir().setPath(
                 "cp" 
                 + INDEX_FORMAT.format(nextCheckpointNumber) 
                 + "-" 
                 + ArchiveUtils.get14DigitDate());
-        getCheckpointDir().getFile().mkdirs();
+        org.archive.util.FileUtils.ensureWriteableDirectory(getCheckpointDir().getFile());
         afterPropertiesSet();
     }
     

commons/src/main/java/org/archive/io/ObjectPlusFilesOutputStream.java

@@ -99,7 +99,7 @@ public class ObjectPlusFilesOutputStream extends ObjectOutputStream {
         writeLong(file.length());
         File auxDir = getAuxiliaryDirectory();
         if(!auxDir.exists()) {
-        	auxDir.mkdirs();
+            FileUtils.ensureWriteableDirectory(auxDir);
         }
         File destination = new File(auxDir,name);
         hardlinkOrCopy(file, destination);

commons/src/main/java/org/archive/util/FileUtils.java

@@ -672,7 +672,10 @@ public class FileUtils {
     public static File ensureWriteableDirectory(File dir)
     throws IOException {
         if (!dir.exists()) {
-            dir.mkdirs();
+            boolean success = dir.mkdirs();
+            if (!success) {
+                throw new IOException("Failed to create directory: " + dir);
+            }
         } else {
             if (!dir.canWrite()) {
                 throw new IOException("Dir " + dir.getAbsolutePath() +

commons/src/main/java/org/archive/util/Recorder.java

@@ -120,14 +120,18 @@ public class Recorder {
      */
     public Recorder(File tempDir, String backingFilenameBase, 
             int outBufferSize, int inBufferSize) {
-        this(ensure(new File(tempDir, backingFilenameBase)), 
+        this(new File(ensure(tempDir), backingFilenameBase),
                 outBufferSize, inBufferSize);
-        tempDir.mkdirs();
     }
     
     
     private static File ensure(File tempDir) {
-        tempDir.mkdirs();
+        try {
+            org.archive.util.FileUtils.ensureWriteableDirectory(tempDir);
+        } catch (IOException e) {
+            throw new IllegalStateException(e);
+        }
+        
         return tempDir;
     }
     

commons/src/main/java/org/archive/util/TestUtils.java

@@ -140,10 +140,7 @@ public class TestUtils {
         String tmpDirStr = System.getProperty(TmpDirTestCase.TEST_TMP_SYSTEM_PROPERTY_NAME);
         tmpDirStr = (tmpDirStr == null)? TmpDirTestCase.DEFAULT_TEST_TMP_DIR: tmpDirStr;
         File tmpDir = new File(tmpDirStr);
-        if (!tmpDir.exists())
-        {
-            tmpDir.mkdirs();
-        }
+        FileUtils.ensureWriteableDirectory(tmpDir);
 
         if (!tmpDir.canWrite())
         {

commons/src/main/java/org/archive/util/TmpDirTestCase.java

@@ -106,10 +106,7 @@ public abstract class TmpDirTestCase extends TestCase
         String tmpDirStr = System.getProperty(TEST_TMP_SYSTEM_PROPERTY_NAME);
         tmpDirStr = (tmpDirStr == null)? DEFAULT_TEST_TMP_DIR: tmpDirStr;
         File tmpDir = new File(tmpDirStr);
-        if (!tmpDir.exists())
-        {
-            tmpDir.mkdirs();
-        }
+        FileUtils.ensureWriteableDirectory(tmpDir);
 
         if (!tmpDir.canWrite())
         {