diff --git a/commons/src/main/java/org/archive/bdb/BdbModule.java b/commons/src/main/java/org/archive/bdb/BdbModule.java index e986995e..7fdd54fb 100644 --- a/commons/src/main/java/org/archive/bdb/BdbModule.java +++ b/commons/src/main/java/org/archive/bdb/BdbModule.java @@ -280,13 +280,44 @@ Serializable, Closeable { db.sync(); db.close(); } catch (DatabaseException e) { - LOGGER.log(Level.SEVERE, "Error closing db " + name, e); + LOGGER.log(Level.WARNING, "Error closing db " + name, e); } } - public Database openDatabase(String name, BdbConfig config, - boolean recycle) + + /** + * Open a Database inside this BdbModule's environment. + * + * @param name + * @param config + * @param usePriorData use existing data in database, if any + * @return + * @throws DatabaseException + */ + public Database openDatabase(String name, BdbConfig config, boolean usePriorData) + throws DatabaseException { + if (!usePriorData) { + try { + bdbEnvironment.truncateDatabase(null, name, false); + } catch (DatabaseNotFoundException e) { + // Ignored + } + } + return bdbEnvironment.openDatabase(null, name, config.toDatabaseConfig()); + } + + /** + * Open a Database inside this BdbModule's environment, and + * remember it for automatic close-at-module-stop. + * + * @param name + * @param config + * @param usePriorData + * @return + * @throws DatabaseException + */ + public Database openManagedDatabase(String name, BdbConfig config, boolean usePriorData) throws DatabaseException { if(!isRunning()) { // proper initialization hasn't occurred @@ -296,15 +327,9 @@ Serializable, Closeable { if (databases.containsKey(name)) { throw new IllegalStateException("Database already exists: " +name); } - if (!recycle) { - try { - bdbEnvironment.truncateDatabase(null, name, false); - } catch (DatabaseNotFoundException e) { - // Ignored - } - } + DatabasePlusConfig dpc = new DatabasePlusConfig(); - dpc.database = bdbEnvironment.openDatabase(null, name, config.toDatabaseConfig()); + dpc.database = openDatabase(name, config, usePriorData); dpc.name = name; dpc.config = config; databases.put(name, dpc); diff --git a/commons/src/main/java/org/archive/util/ObjectIdentityBdbCache.java b/commons/src/main/java/org/archive/util/ObjectIdentityBdbCache.java index 71d51979..692e7c44 100644 --- a/commons/src/main/java/org/archive/util/ObjectIdentityBdbCache.java +++ b/commons/src/main/java/org/archive/util/ObjectIdentityBdbCache.java @@ -549,7 +549,7 @@ implements ObjectIdentityCache, Closeable, Serializable { final void clearPhantom() { if (this.phantom != null) { this.phantom.clear(); - this.phantom = null; +// this.phantom = null; } super.clear(); } diff --git a/commons/src/test/java/org/archive/settings/file/BdbModuleTest.java b/commons/src/test/java/org/archive/settings/file/BdbModuleTest.java index 05596a6f..afd32868 100644 --- a/commons/src/test/java/org/archive/settings/file/BdbModuleTest.java +++ b/commons/src/test/java/org/archive/settings/file/BdbModuleTest.java @@ -76,7 +76,7 @@ public class BdbModuleTest extends TmpDirTestCase { BdbModule.BdbConfig config = new BdbModule.BdbConfig(); config.setAllowCreate(true); - bdb.openDatabase("testOpen", config, false); + bdb.openManagedDatabase("testOpen", config, false); ConcurrentMap testData = bdb.getCBMMap("testData", false, String.class, String.class); diff --git a/engine/src/main/java/org/archive/crawler/frontier/BdbFrontier.java b/engine/src/main/java/org/archive/crawler/frontier/BdbFrontier.java index 0ecb24d6..00d3ebb8 100644 --- a/engine/src/main/java/org/archive/crawler/frontier/BdbFrontier.java +++ b/engine/src/main/java/org/archive/crawler/frontier/BdbFrontier.java @@ -123,7 +123,7 @@ implements Serializable, Checkpointable { dbConfig.setAllowCreate(!recycle); // Make database deferred write: URLs that are added then removed // before a page-out is required need never cause disk IO. - db = bdb.openDatabase("pending", dbConfig, recycle); + db = bdb.openManagedDatabase("pending", dbConfig, recycle); } return new BdbMultipleWorkQueues(db, bdb.getClassCatalog()); @@ -248,7 +248,7 @@ implements Serializable, Checkpointable { inactiveQueuesByPrecedence = new TreeMap>(); Database retiredQueuesDb; - retiredQueuesDb = bdb.openDatabase("retiredQueues", + retiredQueuesDb = bdb.openManagedDatabase("retiredQueues", StoredQueue.databaseConfig(), false); retiredQueues = new StoredQueue(retiredQueuesDb, String.class, null); @@ -270,7 +270,7 @@ implements Serializable, Checkpointable { Queue createInactiveQueueForPrecedence(int precedence) { Database inactiveQueuesDb; try { - inactiveQueuesDb = bdb.openDatabase("inactiveQueues-"+precedence, + inactiveQueuesDb = bdb.openManagedDatabase("inactiveQueues-"+precedence, StoredQueue.databaseConfig(), false); } catch (DatabaseException e) { throw new RuntimeException(e); diff --git a/engine/src/main/java/org/archive/crawler/frontier/precedence/PreloadedUriPrecedencePolicy.java b/engine/src/main/java/org/archive/crawler/frontier/precedence/PreloadedUriPrecedencePolicy.java index 80ffb06f..6d403b17 100644 --- a/engine/src/main/java/org/archive/crawler/frontier/precedence/PreloadedUriPrecedencePolicy.java +++ b/engine/src/main/java/org/archive/crawler/frontier/precedence/PreloadedUriPrecedencePolicy.java @@ -87,7 +87,7 @@ implements Lifecycle { StoredClassCatalog classCatalog = bdb.getClassCatalog(); BdbModule.BdbConfig dbConfig = PersistProcessor.historyDatabaseConfig(); - historyDb = bdb.openDatabase(dbName, dbConfig, true); + historyDb = bdb.openManagedDatabase(dbName, dbConfig, true); historyMap = new StoredSortedMap(historyDb, new StringBinding(), new SerialBinding(classCatalog, Map.class), true); diff --git a/engine/src/main/java/org/archive/crawler/util/BdbUriUniqFilter.java b/engine/src/main/java/org/archive/crawler/util/BdbUriUniqFilter.java index d37fbff2..65ec3486 100644 --- a/engine/src/main/java/org/archive/crawler/util/BdbUriUniqFilter.java +++ b/engine/src/main/java/org/archive/crawler/util/BdbUriUniqFilter.java @@ -103,7 +103,7 @@ implements Lifecycle, Serializable { try { BdbModule.BdbConfig config = getDatabaseConfig(); config.setAllowCreate(true); - initialize(bdb.openDatabase(DB_NAME, config, false)); + initialize(bdb.openManagedDatabase(DB_NAME, config, false)); } catch (DatabaseException e) { throw new IllegalStateException(e); } diff --git a/modules/src/main/java/org/archive/modules/fetcher/BdbCookieStorage.java b/modules/src/main/java/org/archive/modules/fetcher/BdbCookieStorage.java index 06081f72..5ca1abb6 100644 --- a/modules/src/main/java/org/archive/modules/fetcher/BdbCookieStorage.java +++ b/modules/src/main/java/org/archive/modules/fetcher/BdbCookieStorage.java @@ -65,7 +65,7 @@ public class BdbCookieStorage extends AbstractCookieStorage { BdbModule.BdbConfig dbConfig = new BdbModule.BdbConfig(); dbConfig.setTransactional(false); dbConfig.setAllowCreate(true); - cookieDb = bdb.openDatabase(COOKIEDB_NAME, dbConfig, true); + cookieDb = bdb.openManagedDatabase(COOKIEDB_NAME, dbConfig, true); cookies = new StoredSortedMap( cookieDb, diff --git a/modules/src/main/java/org/archive/modules/recrawl/PersistOnlineProcessor.java b/modules/src/main/java/org/archive/modules/recrawl/PersistOnlineProcessor.java index 590c6261..84dfcc2b 100644 --- a/modules/src/main/java/org/archive/modules/recrawl/PersistOnlineProcessor.java +++ b/modules/src/main/java/org/archive/modules/recrawl/PersistOnlineProcessor.java @@ -76,7 +76,7 @@ implements Lifecycle { StoredClassCatalog classCatalog = bdb.getClassCatalog(); BdbModule.BdbConfig dbConfig = historyDatabaseConfig(); - historyDb = bdb.openDatabase(getHistoryDbName(), dbConfig, true); + historyDb = bdb.openManagedDatabase(getHistoryDbName(), dbConfig, true); historyMap = new StoredSortedMap( historyDb,