diff --git a/commons/src/main/java/org/archive/bdb/BdbModule.java b/commons/src/main/java/org/archive/bdb/BdbModule.java index 5d854f15..9912dcc4 100644 --- a/commons/src/main/java/org/archive/bdb/BdbModule.java +++ b/commons/src/main/java/org/archive/bdb/BdbModule.java @@ -256,7 +256,7 @@ public class BdbModule implements Lifecycle, Checkpointable, Closeable { } protected void setup(File f, boolean create) - throws DatabaseException { + throws DatabaseException, IOException { EnvironmentConfig config = new EnvironmentConfig(); config.setAllowCreate(create); config.setLockTimeout(75, TimeUnit.MINUTES); // set to max @@ -282,7 +282,7 @@ public class BdbModule implements Lifecycle, Checkpointable, Closeable { config.setConfigParam("je.cleaner.expunge", "false"); } // else leave whatever other setting was already in place - f.mkdirs(); + org.archive.util.FileUtils.ensureWriteableDirectory(f); this.bdbEnvironment = new EnhancedEnvironment(f, config); this.classCatalog = this.bdbEnvironment.getClassCatalog(); if(!create) { @@ -479,7 +479,7 @@ public class BdbModule implements Lifecycle, Checkpointable, Closeable { dbBackup.startBackup(); File envCpDir = new File(dir.getFile(),checkpointInProgress.getName()); - envCpDir.mkdirs(); + org.archive.util.FileUtils.ensureWriteableDirectory(envCpDir); File logfilesList = new File(envCpDir,"jdbfiles.manifest"); String[] filedata = dbBackup.getLogFilesInBackupSet(); for (int i=0; i>>(); this.cache.initialize(env,"setUpCache",IdentityCacheableWrapper.class, env.getClassCatalog()); diff --git a/commons/src/test/java/org/archive/util/ObjectIdentityBdbManualCacheTest.java b/commons/src/test/java/org/archive/util/ObjectIdentityBdbManualCacheTest.java index 14f1153b..77a5eb2f 100644 --- a/commons/src/test/java/org/archive/util/ObjectIdentityBdbManualCacheTest.java +++ b/commons/src/test/java/org/archive/util/ObjectIdentityBdbManualCacheTest.java @@ -37,10 +37,10 @@ public class ObjectIdentityBdbManualCacheTest extends TmpDirTestCase { protected void setUp() throws Exception { super.setUp(); - File envDir = new File(getTmpDir(),"ObjectIdentityBdbCacheTest"); - envDir.mkdirs(); + File envDir = new File(getTmpDir(),"ObjectIdentityBdbCacheTest"); + org.archive.util.FileUtils.ensureWriteableDirectory(envDir); FileUtils.deleteDirectory(envDir); - envDir.mkdirs(); + org.archive.util.FileUtils.ensureWriteableDirectory(envDir); env = EnhancedEnvironment.getTestEnvironment(envDir); this.cache = new ObjectIdentityBdbManualCache>>(); this.cache.initialize(env,"setUpCache",IdentityCacheableWrapper.class, env.getClassCatalog()); diff --git a/dist/src/test/java/org/archive/crawler/BasicProfileTest.java b/dist/src/test/java/org/archive/crawler/BasicProfileTest.java index 0c1b6ee5..b01f26b5 100644 --- a/dist/src/test/java/org/archive/crawler/BasicProfileTest.java +++ b/dist/src/test/java/org/archive/crawler/BasicProfileTest.java @@ -57,7 +57,7 @@ public class BasicProfileTest extends TmpDirTestCase { System.out.println("\nNow testing " + srcDir.getName()); File tmpDir = new File(getTmpDir(), "validatorTest"); File configDir = new File(tmpDir, srcDir.getName()); - configDir.mkdirs(); + org.archive.util.FileUtils.ensureWriteableDirectory(configDir); FileUtils.copyDirectory(srcDir, configDir); PathSharingContext ac = null; diff --git a/engine/src/main/java/org/archive/crawler/framework/ActionDirectory.java b/engine/src/main/java/org/archive/crawler/framework/ActionDirectory.java index 9ef67a2e..447e8b0d 100644 --- a/engine/src/main/java/org/archive/crawler/framework/ActionDirectory.java +++ b/engine/src/main/java/org/archive/crawler/framework/ActionDirectory.java @@ -151,9 +151,13 @@ public class ActionDirectory implements ApplicationContextAware, Lifecycle, Runn if (isRunning()) { return; } - // create directories - getActionDir().getFile().mkdirs(); - getDoneDir().getFile().mkdirs(); + try { + // create directories + org.archive.util.FileUtils.ensureWriteableDirectory(getActionDir().getFile()); + org.archive.util.FileUtils.ensureWriteableDirectory(getDoneDir().getFile()); + } catch (IOException e) { + throw new IllegalStateException(e); + } // start background executor executor = Executors.newSingleThreadScheduledExecutor(); executor.scheduleWithFixedDelay(this, getInitialDelaySeconds(), getDelaySeconds(), TimeUnit.SECONDS); diff --git a/engine/src/main/java/org/archive/crawler/framework/CheckpointService.java b/engine/src/main/java/org/archive/crawler/framework/CheckpointService.java index f6bd4542..c22d45f3 100644 --- a/engine/src/main/java/org/archive/crawler/framework/CheckpointService.java +++ b/engine/src/main/java/org/archive/crawler/framework/CheckpointService.java @@ -221,12 +221,12 @@ public class CheckpointService implements Lifecycle, ApplicationContextAware, Ha } } - checkpointInProgress = new Checkpoint(); - checkpointInProgress.generateFrom(getCheckpointsDir(),getNextCheckpointNumber()); - Map toCheckpoint = appCtx.getBeansOfType(Checkpointable.class); + checkpointInProgress = new Checkpoint(); try { + checkpointInProgress.generateFrom(getCheckpointsDir(),getNextCheckpointNumber()); + // pre (incl. acquire necessary locks) // long startMs = System.currentTimeMillis(); for(Checkpointable c : toCheckpoint.values()) { diff --git a/engine/src/main/java/org/archive/crawler/framework/Engine.java b/engine/src/main/java/org/archive/crawler/framework/Engine.java index d861e7fa..7d94f5aa 100644 --- a/engine/src/main/java/org/archive/crawler/framework/Engine.java +++ b/engine/src/main/java/org/archive/crawler/framework/Engine.java @@ -61,7 +61,12 @@ public class Engine { public Engine(File jobsDir) { this.jobsDir = jobsDir; - this.jobsDir.mkdirs(); + + try { + org.archive.util.FileUtils.ensureWriteableDirectory(jobsDir); + } catch (IOException e) { + throw new IllegalStateException(e); + } findJobConfigs(); // TODO: cleanup any cruft from improperly ended jobs @@ -178,7 +183,7 @@ public class Engine { */ public synchronized void copy(CrawlJob orig, File destDir, boolean asProfile) throws IOException { - destDir.mkdirs(); + org.archive.util.FileUtils.ensureWriteableDirectory(destDir); if(destDir.list().length>0) { throw new IOException("destination dir not empty"); } @@ -352,7 +357,7 @@ public class Engine { inStream.close(); // write default crawler-beans string to new job dir - newJobDir.mkdirs(); + org.archive.util.FileUtils.ensureWriteableDirectory(newJobDir); File newJobCxml = new File(newJobDir,"crawler-beans.cxml"); FileUtils.writeStringToFile(newJobCxml, defaultCxmlStr); diff --git a/engine/src/main/java/org/archive/crawler/frontier/precedence/PrecedenceLoader.java b/engine/src/main/java/org/archive/crawler/frontier/precedence/PrecedenceLoader.java index 1b5c9d03..7031b99e 100644 --- a/engine/src/main/java/org/archive/crawler/frontier/precedence/PrecedenceLoader.java +++ b/engine/src/main/java/org/archive/crawler/frontier/precedence/PrecedenceLoader.java @@ -31,6 +31,7 @@ import java.util.Map; import org.archive.modules.recrawl.PersistProcessor; import org.archive.util.ArchiveUtils; +import org.archive.util.FileUtils; import org.archive.util.bdbje.EnhancedEnvironment; import org.archive.util.iterator.LineReadingIterator; @@ -100,9 +101,7 @@ public class PrecedenceLoader { FileNotFoundException, UnsupportedEncodingException, IOException { File source = new File(args[0]); File env = new File(args[1]); - if(!env.exists()) { - env.mkdirs(); - } + FileUtils.ensureWriteableDirectory(env); // setup target environment EnhancedEnvironment targetEnv = PersistProcessor.setupCopyEnvironment(env); diff --git a/engine/src/main/java/org/archive/crawler/migrate/MigrateH1to3Tool.java b/engine/src/main/java/org/archive/crawler/migrate/MigrateH1to3Tool.java index 115389a7..5f511b48 100644 --- a/engine/src/main/java/org/archive/crawler/migrate/MigrateH1to3Tool.java +++ b/engine/src/main/java/org/archive/crawler/migrate/MigrateH1to3Tool.java @@ -92,7 +92,7 @@ public class MigrateH1to3Tool { } File destinationH3JobDir = new File(destinationH3JobDirArg); - destinationH3JobDir.mkdirs(); + org.archive.util.FileUtils.ensureWriteableDirectory(destinationH3JobDir); System.out.println("H1 source: "+sourceOrderXmlFile.getAbsolutePath()); System.out.println("H3 destination: "+destinationH3JobDir.getAbsolutePath()); diff --git a/engine/src/main/java/org/archive/crawler/reporting/CrawlerLoggerModule.java b/engine/src/main/java/org/archive/crawler/reporting/CrawlerLoggerModule.java index 39c648f3..f52df015 100644 --- a/engine/src/main/java/org/archive/crawler/reporting/CrawlerLoggerModule.java +++ b/engine/src/main/java/org/archive/crawler/reporting/CrawlerLoggerModule.java @@ -46,6 +46,7 @@ import org.archive.modules.extractor.UriErrorLoggerModule; import org.archive.net.UURI; import org.archive.spring.ConfigPath; import org.archive.util.ArchiveUtils; +import org.archive.util.FileUtils; import org.springframework.beans.factory.InitializingBean; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.context.Lifecycle; @@ -209,9 +210,9 @@ public class CrawlerLoggerModule if(isRunning) { return; } - getPath().getFile().mkdirs(); this.atg = AlertThreadGroup.current(); try { + FileUtils.ensureWriteableDirectory(getPath().getFile()); setupLogs(); } catch (IOException e) { throw new IllegalStateException(e); diff --git a/engine/src/main/java/org/archive/crawler/reporting/StatisticsTracker.java b/engine/src/main/java/org/archive/crawler/reporting/StatisticsTracker.java index 6d56a825..842726f2 100644 --- a/engine/src/main/java/org/archive/crawler/reporting/StatisticsTracker.java +++ b/engine/src/main/java/org/archive/crawler/reporting/StatisticsTracker.java @@ -58,6 +58,7 @@ import org.archive.modules.seeds.SeedListener; import org.archive.modules.seeds.SeedModule; import org.archive.spring.ConfigPath; import org.archive.util.ArchiveUtils; +import org.archive.util.FileUtils; import org.archive.util.JSONUtils; import org.archive.util.MimetypeUtils; import org.archive.util.ObjectIdentityCache; @@ -902,8 +903,8 @@ public class StatisticsTracker return f; } - f.getParentFile().mkdirs(); try { + FileUtils.ensureWriteableDirectory(f.getParentFile()); PrintWriter bw = new PrintWriter(new FileWriter(f)); report.write(bw, this); bw.close(); diff --git a/engine/src/main/java/org/archive/crawler/util/BdbUriUniqFilter.java b/engine/src/main/java/org/archive/crawler/util/BdbUriUniqFilter.java index 7a48bccd..a33836a6 100644 --- a/engine/src/main/java/org/archive/crawler/util/BdbUriUniqFilter.java +++ b/engine/src/main/java/org/archive/crawler/util/BdbUriUniqFilter.java @@ -27,6 +27,7 @@ import java.util.logging.Logger; import org.archive.bdb.BdbModule; import org.archive.checkpointing.Checkpoint; import org.archive.checkpointing.Checkpointable; +import org.archive.util.FileUtils; import org.json.JSONException; import org.json.JSONObject; import org.springframework.beans.factory.BeanNameAware; @@ -165,9 +166,7 @@ implements Lifecycle, Checkpointable, BeanNameAware { public BdbUriUniqFilter(File bdbEnv, final int cacheSizePercentage) throws IOException { super(); - if (!bdbEnv.exists()) { - bdbEnv.mkdirs(); - } + FileUtils.ensureWriteableDirectory(bdbEnv); EnvironmentConfig envConfig = new EnvironmentConfig(); envConfig.setAllowCreate(true); if (cacheSizePercentage > 0 && cacheSizePercentage < 100) { diff --git a/engine/src/main/java/org/archive/crawler/util/CheckpointUtils.java b/engine/src/main/java/org/archive/crawler/util/CheckpointUtils.java index 6bff9937..7c4a9bc4 100644 --- a/engine/src/main/java/org/archive/crawler/util/CheckpointUtils.java +++ b/engine/src/main/java/org/archive/crawler/util/CheckpointUtils.java @@ -27,6 +27,8 @@ import java.io.IOException; import java.io.ObjectInputStream; import java.io.ObjectOutputStream; +import org.archive.util.FileUtils; + /** * Utilities useful checkpointing. * @author stack @@ -76,7 +78,7 @@ public class CheckpointUtils { public static void writeObjectToFile(final Object o, final String suffix, final File dir) throws IOException { - dir.mkdirs(); + FileUtils.ensureWriteableDirectory(dir); ObjectOutputStream out = new ObjectOutputStream( new FileOutputStream(getClassCheckpointFile(dir, suffix, o.getClass()))); diff --git a/engine/src/test/java/org/archive/crawler/framework/CrawlControllerTest.java b/engine/src/test/java/org/archive/crawler/framework/CrawlControllerTest.java index 0c2ad523..af39c2d3 100644 --- a/engine/src/test/java/org/archive/crawler/framework/CrawlControllerTest.java +++ b/engine/src/test/java/org/archive/crawler/framework/CrawlControllerTest.java @@ -30,6 +30,7 @@ import org.archive.modules.net.BdbServerCache; import org.archive.spring.ConfigPath; import org.archive.state.ModuleTestBase; import org.archive.util.ArchiveUtils; +import org.archive.util.FileUtils; /** * @@ -45,9 +46,7 @@ public class CrawlControllerTest extends ModuleTestBase { tmpPath = DEFAULT_TEST_TMP_DIR; } File tmp = new File(tmpPath); - if (!tmp.exists()) { - tmp.mkdirs(); - } + FileUtils.ensureWriteableDirectory(tmp); FileWriter fileWriter = null; try { @@ -59,10 +58,10 @@ public class CrawlControllerTest extends ModuleTestBase { } File state = new File(tmp, "state"); - state.mkdirs(); + FileUtils.ensureWriteableDirectory(state); File checkpoints = new File(tmp, "checkpoints"); - checkpoints.mkdirs(); + FileUtils.ensureWriteableDirectory(checkpoints); BdbModule bdb = new BdbModule(); bdb.setDir(new ConfigPath("test",state.getAbsolutePath())); diff --git a/engine/src/test/java/org/archive/crawler/selftest/SelfTestBase.java b/engine/src/test/java/org/archive/crawler/selftest/SelfTestBase.java index 479bc9bf..37260190 100644 --- a/engine/src/test/java/org/archive/crawler/selftest/SelfTestBase.java +++ b/engine/src/test/java/org/archive/crawler/selftest/SelfTestBase.java @@ -90,7 +90,7 @@ public abstract class SelfTestBase extends TmpDirTestCase { if(profileTemplate.exists()) { org.apache.commons.io.FileUtils.copyDirectory(profileTemplate, tmpDefProfile); } else { - tmpDefProfile.mkdirs(); + org.archive.util.FileUtils.ensureWriteableDirectory(tmpDefProfile); } // Start up a Jetty that serves the selftest's content directory. @@ -98,7 +98,7 @@ public abstract class SelfTestBase extends TmpDirTestCase { // Copy configuration for eg Logging over File tmpConfDir = new File(tmpTestDir, "conf"); - tmpConfDir.mkdirs(); + org.archive.util.FileUtils.ensureWriteableDirectory(tmpConfDir); File srcConf = new File(src.getParentFile(), "conf"); FileUtils.copyDirectory(srcConf, tmpConfDir); diff --git a/modules/src/main/java/org/archive/modules/recrawl/PersistProcessor.java b/modules/src/main/java/org/archive/modules/recrawl/PersistProcessor.java index 8c1f75f6..e05fe16f 100644 --- a/modules/src/main/java/org/archive/modules/recrawl/PersistProcessor.java +++ b/modules/src/main/java/org/archive/modules/recrawl/PersistProcessor.java @@ -42,6 +42,7 @@ import org.archive.bdb.BdbModule; import org.archive.modules.CrawlURI; import org.archive.modules.Processor; import org.archive.util.ArchiveUtils; +import org.archive.util.FileUtils; import org.archive.util.OneLineSimpleLogger; import org.archive.util.SURT; import org.archive.util.bdbje.EnhancedEnvironment; @@ -261,9 +262,7 @@ public abstract class PersistProcessor extends Processor { if (envFile != null) { // set up target environment - if (!envFile.exists()) { - envFile.mkdirs(); - } + FileUtils.ensureWriteableDirectory(envFile); targetEnv = setupCopyEnvironment(envFile); classCatalog = targetEnv.getClassCatalog(); historyDB = targetEnv.openDatabase(null, URI_HISTORY_DBNAME, diff --git a/modules/src/main/java/org/archive/modules/writer/Kw3WriterProcessor.java b/modules/src/main/java/org/archive/modules/writer/Kw3WriterProcessor.java index ce97f6bd..eee32184 100644 --- a/modules/src/main/java/org/archive/modules/writer/Kw3WriterProcessor.java +++ b/modules/src/main/java/org/archive/modules/writer/Kw3WriterProcessor.java @@ -48,6 +48,7 @@ import org.archive.modules.Processor; import org.archive.modules.net.CrawlHost; import org.archive.modules.net.ServerCache; import org.archive.spring.ConfigPath; +import org.archive.util.FileUtils; import org.springframework.beans.factory.annotation.Autowired; /** @@ -273,7 +274,7 @@ public class Kw3WriterProcessor extends Processor { File dir = new File(getPath().getFile(), md5.substring(0, 2) + "/" + host + "/current"); if (!dir.exists()) { - dir.mkdirs(); + FileUtils.ensureWriteableDirectory(dir); if (this.chmod) chmods(dir, getPath().getFile()); } diff --git a/modules/src/main/java/org/archive/modules/writer/WriterPoolProcessor.java b/modules/src/main/java/org/archive/modules/writer/WriterPoolProcessor.java index 4251fcb8..98630702 100644 --- a/modules/src/main/java/org/archive/modules/writer/WriterPoolProcessor.java +++ b/modules/src/main/java/org/archive/modules/writer/WriterPoolProcessor.java @@ -47,6 +47,7 @@ import org.archive.modules.deciderules.recrawl.IdenticalDigestDecideRule; import org.archive.modules.net.CrawlHost; import org.archive.modules.net.ServerCache; import org.archive.spring.ConfigPath; +import org.archive.util.FileUtils; import org.json.JSONException; import org.json.JSONObject; import org.springframework.beans.factory.annotation.Autowired; @@ -440,7 +441,7 @@ implements Lifecycle, Checkpointable, WriterPoolSettings { File f = path.getFile(); if (!f.exists()) { try { - f.mkdirs(); + FileUtils.ensureWriteableDirectory(f); } catch (Exception e) { e.printStackTrace(); continue; diff --git a/modules/src/test/java/org/archive/modules/writer/ARCWriterProcessorTest.java b/modules/src/test/java/org/archive/modules/writer/ARCWriterProcessorTest.java index ddf400e8..9d83cd85 100644 --- a/modules/src/test/java/org/archive/modules/writer/ARCWriterProcessorTest.java +++ b/modules/src/test/java/org/archive/modules/writer/ARCWriterProcessorTest.java @@ -25,6 +25,7 @@ import org.archive.modules.CrawlMetadata; import org.archive.modules.ProcessorTestBase; import org.archive.modules.fetcher.DefaultServerCache; import org.archive.spring.ConfigPath; +import org.archive.util.FileUtils; import org.archive.util.TmpDirTestCase; @@ -39,7 +40,7 @@ public class ARCWriterProcessorTest extends ProcessorTestBase { protected Object makeModule() throws Exception { File tmp = TmpDirTestCase.tmpDir(); tmp = new File(tmp, "ARCWriterProcessTest"); - tmp.mkdirs(); + FileUtils.ensureWriteableDirectory(tmp); ARCWriterProcessor result = new ARCWriterProcessor(); result.setDirectory(new ConfigPath("test",tmp.getAbsolutePath())); diff --git a/modules/src/test/java/org/archive/modules/writer/WARCWriterProcessorTest.java b/modules/src/test/java/org/archive/modules/writer/WARCWriterProcessorTest.java index 3de19b60..8ea8b660 100644 --- a/modules/src/test/java/org/archive/modules/writer/WARCWriterProcessorTest.java +++ b/modules/src/test/java/org/archive/modules/writer/WARCWriterProcessorTest.java @@ -42,6 +42,7 @@ import org.archive.net.UURIFactory; import org.archive.spring.ConfigPath; import org.archive.uid.RecordIDGenerator; import org.archive.uid.UUIDGenerator; +import org.archive.util.FileUtils; import org.archive.util.TmpDirTestCase; import org.archive.util.anvl.ANVLRecord; @@ -57,8 +58,8 @@ public class WARCWriterProcessorTest extends ProcessorTestBase { @Override protected Object makeModule() throws Exception { File tmp = TmpDirTestCase.tmpDir(); - tmp = new File(tmp, "ARCWriterProcessTest"); - tmp.mkdirs(); + tmp = new File(tmp, "WARCWriterProcessTest"); + FileUtils.ensureWriteableDirectory(tmp); WARCWriterProcessor result = new WARCWriterProcessor(); result.setDirectory(new ConfigPath("test",tmp.getAbsolutePath()));