diff --git a/commons/src/main/java/org/archive/bdb/BdbModule.java b/commons/src/main/java/org/archive/bdb/BdbModule.java index 74aebe3f..d9b69bd5 100644 --- a/commons/src/main/java/org/archive/bdb/BdbModule.java +++ b/commons/src/main/java/org/archive/bdb/BdbModule.java @@ -321,7 +321,7 @@ public class BdbModule implements Lifecycle, Checkpointable, Closeable { * @return * @throws DatabaseException */ - public Database openManagedDatabase(String name, BdbConfig config, boolean usePriorData) + public Database openDatabase(String name, BdbConfig config, boolean usePriorData) throws DatabaseException { if(!isRunning()) { // proper initialization hasn't occurred @@ -360,7 +360,7 @@ public class BdbModule implements Lifecycle, Checkpointable, Closeable { public StoredQueue getStoredQueue(String dbname, Class clazz, boolean b) { try { Database queueDb; - queueDb = openManagedDatabase(dbname, + queueDb = openDatabase(dbname, StoredQueue.databaseConfig(), false); return new StoredQueue(queueDb, clazz, getClassCatalog()); } catch (DatabaseException e) { @@ -637,10 +637,10 @@ public class BdbModule implements Lifecycle, Checkpointable, Closeable { * @param allowDuplicates whether duplicate keys allowed * @return */ - public DisposableStoredSortedMap getStoredMap(String dbName, Class keyClass, Class valueClass, boolean allowDuplicates) { + public DisposableStoredSortedMap getStoredMap(String dbName, Class keyClass, Class valueClass, boolean allowDuplicates, boolean usePriorData) { BdbConfig config = new BdbConfig(); - config.setSortedDuplicates(true); - config.setAllowCreate(true); + config.setSortedDuplicates(allowDuplicates); + config.setAllowCreate(!usePriorData); Database mapDb; if(dbName==null) { dbName = "tempMap-"+System.identityHashCode(this)+"-"+sn; @@ -648,7 +648,7 @@ public class BdbModule implements Lifecycle, Checkpointable, Closeable { } final String openName = dbName; try { - mapDb = openManagedDatabase(openName,config,false); + mapDb = openDatabase(openName,config,usePriorData); } catch (DatabaseException e) { throw new RuntimeException(e); } diff --git a/engine/src/main/java/org/archive/crawler/frontier/precedence/PreloadedUriPrecedencePolicy.java b/engine/src/main/java/org/archive/crawler/frontier/precedence/PreloadedUriPrecedencePolicy.java index f8147fdc..77be1e37 100644 --- a/engine/src/main/java/org/archive/crawler/frontier/precedence/PreloadedUriPrecedencePolicy.java +++ b/engine/src/main/java/org/archive/crawler/frontier/precedence/PreloadedUriPrecedencePolicy.java @@ -81,7 +81,7 @@ implements Lifecycle { StoredClassCatalog classCatalog = bdb.getClassCatalog(); BdbModule.BdbConfig dbConfig = PersistProcessor.HISTORY_DB_CONFIG; - historyDb = bdb.openManagedDatabase(dbName, dbConfig, true); + historyDb = bdb.openDatabase(dbName, dbConfig, true); historyMap = new StoredSortedMap(historyDb, new StringBinding(), new SerialBinding(classCatalog, Map.class), true); diff --git a/engine/src/main/java/org/archive/crawler/reporting/StatisticsTracker.java b/engine/src/main/java/org/archive/crawler/reporting/StatisticsTracker.java index a5369a20..285ac372 100644 --- a/engine/src/main/java/org/archive/crawler/reporting/StatisticsTracker.java +++ b/engine/src/main/java/org/archive/crawler/reporting/StatisticsTracker.java @@ -694,7 +694,8 @@ public class StatisticsTracker null, Long.class, String.class, - true); + true, + false); for(String k : mapOfAtomicLongValues.keySet()) { sortedMap.put(-mapOfAtomicLongValues.get(k).longValue(), k); } @@ -723,7 +724,8 @@ public class StatisticsTracker null, Long.class, String.class, - true); + true, + false); for(String k : cacheOfAtomicLongValues.keySet()) { sortedMap.put(-cacheOfAtomicLongValues.get(k).longValue(), k); } @@ -895,7 +897,8 @@ public class StatisticsTracker null, Integer.class, SeedRecord.class, - true); + true, + false); while (i.hasNext()) { String seed = i.next(); diff --git a/engine/src/main/java/org/archive/crawler/util/BdbUriUniqFilter.java b/engine/src/main/java/org/archive/crawler/util/BdbUriUniqFilter.java index cead8031..ef27d6ce 100644 --- a/engine/src/main/java/org/archive/crawler/util/BdbUriUniqFilter.java +++ b/engine/src/main/java/org/archive/crawler/util/BdbUriUniqFilter.java @@ -106,7 +106,7 @@ implements Lifecycle, Checkpointable, BeanNameAware { try { BdbModule.BdbConfig config = getDatabaseConfig(); config.setAllowCreate(!isRecovery); - initialize(bdb.openManagedDatabase(DB_NAME, config, isRecovery)); + initialize(bdb.openDatabase(DB_NAME, config, isRecovery)); } catch (DatabaseException e) { throw new IllegalStateException(e); } diff --git a/modules/src/main/java/org/archive/modules/fetcher/BdbCookieStorage.java b/modules/src/main/java/org/archive/modules/fetcher/BdbCookieStorage.java index 5ca1abb6..c310257c 100644 --- a/modules/src/main/java/org/archive/modules/fetcher/BdbCookieStorage.java +++ b/modules/src/main/java/org/archive/modules/fetcher/BdbCookieStorage.java @@ -19,9 +19,6 @@ package org.archive.modules.fetcher; -import java.io.IOException; -import java.io.ObjectInputStream; -import java.io.ObjectOutputStream; import java.util.Map; import java.util.SortedMap; @@ -65,7 +62,7 @@ public class BdbCookieStorage extends AbstractCookieStorage { BdbModule.BdbConfig dbConfig = new BdbModule.BdbConfig(); dbConfig.setTransactional(false); dbConfig.setAllowCreate(true); - cookieDb = bdb.openManagedDatabase(COOKIEDB_NAME, dbConfig, true); + cookieDb = bdb.openDatabase(COOKIEDB_NAME, dbConfig, true); cookies = new StoredSortedMap( cookieDb, @@ -80,37 +77,12 @@ public class BdbCookieStorage extends AbstractCookieStorage { } } - @SuppressWarnings("unchecked") public SortedMap getCookiesMap() { // assert cookies != null : "cookie map not set up"; return cookies; } - protected void innerSaveCookiesMap(Map map) { } - - - private void writeObject(ObjectOutputStream out) throws IOException { - out.defaultWriteObject(); - try { - out.writeUTF(cookieDb.getDatabaseName()); - } catch (DatabaseException e) { - IOException io = new IOException(); - io.initCause(e); - throw io; - } - } - - @SuppressWarnings("unchecked") - private void readObject(ObjectInputStream in) - throws IOException, ClassNotFoundException { - in.defaultReadObject(); - cookieDb = bdb.getDatabase(in.readUTF()); - cookies = new StoredSortedMap(cookieDb, - new StringBinding(), new SerialBinding(bdb.getClassCatalog(), - Cookie.class), true); - } - } diff --git a/modules/src/main/java/org/archive/modules/recrawl/PersistOnlineProcessor.java b/modules/src/main/java/org/archive/modules/recrawl/PersistOnlineProcessor.java index ece88d04..ec40ead7 100644 --- a/modules/src/main/java/org/archive/modules/recrawl/PersistOnlineProcessor.java +++ b/modules/src/main/java/org/archive/modules/recrawl/PersistOnlineProcessor.java @@ -76,7 +76,7 @@ implements Lifecycle { StoredClassCatalog classCatalog = bdb.getClassCatalog(); BdbModule.BdbConfig dbConfig = HISTORY_DB_CONFIG; - historyDb = bdb.openManagedDatabase(getHistoryDbName(), dbConfig, true); + historyDb = bdb.openDatabase(getHistoryDbName(), dbConfig, true); historyMap = new StoredSortedMap( historyDb,