From eb9bf5a8e2050d21a3595f48c307837d018d1b2e Mon Sep 17 00:00:00 2001 From: gojomo Date: Mon, 23 Nov 2009 04:41:32 +0000 Subject: [PATCH] BDBJE updates, refactoring * commons/pom.xml, .classpath update to BDB-JE 4.0.71 * BdbModule.java (getStoredQueue) added (getStoredMap) made more general (openDatabase) all databases now 'managed' * TempStoredSortedMap.java -> DisposableStoredSortedMap.java renamed * StatisticsTracker.java, **Report.java use new name * StoredQueue.java, StoredQueueTest.java move to org.archive.bdb package * BdbMultipleWorkQueues.java, PrefixFinderTest.java update for BDB-JE 4 changes * BdbFrontier.java use getStoredQueue drop unneeded custom serialization support * WorkQueueFrontier.java drop unneeded custom serialization support --- .classpath | 126 +++++++++--------- commons/pom.xml | 2 +- .../main/java/org/archive/bdb/BdbModule.java | 78 ++++++----- ...ap.java => DisposableStoredSortedMap.java} | 14 +- .../archive/{queue => bdb}/StoredQueue.java | 3 +- .../org/archive/bdb}/StoredQueueTest.java | 3 +- .../settings/file/PrefixFinderTest.java | 3 +- .../archive/crawler/frontier/BdbFrontier.java | 50 +------ .../frontier/BdbMultipleWorkQueues.java | 6 +- .../crawler/frontier/WorkQueueFrontier.java | 49 ++----- .../crawler/reporting/HostsReport.java | 6 +- .../crawler/reporting/MimetypesReport.java | 6 +- .../crawler/reporting/ResponseCodeReport.java | 6 +- .../crawler/reporting/SeedsReport.java | 6 +- .../crawler/reporting/SourceTagsReport.java | 6 +- .../crawler/reporting/StatisticsTracker.java | 18 +-- 16 files changed, 163 insertions(+), 219 deletions(-) rename commons/src/main/java/org/archive/bdb/{TempStoredSortedMap.java => DisposableStoredSortedMap.java} (75%) rename commons/src/main/java/org/archive/{queue => bdb}/StoredQueue.java (98%) rename commons/src/{main/java/org/archive/queue => test/java/org/archive/bdb}/StoredQueueTest.java (98%) diff --git a/.classpath b/.classpath index 1e78ebf7..82bebe6b 100644 --- a/.classpath +++ b/.classpath @@ -1,63 +1,63 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/commons/pom.xml b/commons/pom.xml index 5b645580..844f3e42 100644 --- a/commons/pom.xml +++ b/commons/pom.xml @@ -104,7 +104,7 @@ com.sleepycat je - 3.3.87 + 4.0.71 commons-lang diff --git a/commons/src/main/java/org/archive/bdb/BdbModule.java b/commons/src/main/java/org/archive/bdb/BdbModule.java index 1aa2df9e..74aebe3f 100644 --- a/commons/src/main/java/org/archive/bdb/BdbModule.java +++ b/commons/src/main/java/org/archive/bdb/BdbModule.java @@ -33,6 +33,7 @@ import java.util.List; import java.util.Map; import java.util.Set; import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.TimeUnit; import java.util.logging.Level; import java.util.logging.Logger; @@ -67,8 +68,7 @@ import com.sleepycat.je.util.DbBackup; * @contributor pjack * @contributor gojomo */ -public class BdbModule implements Lifecycle, Checkpointable, -Serializable, Closeable { +public class BdbModule implements Lifecycle, Checkpointable, Closeable { private static final long serialVersionUID = 1L; final private static Logger LOGGER = Logger.getLogger(BdbModule.class.getName()); @@ -248,7 +248,7 @@ Serializable, Closeable { throws DatabaseException { EnvironmentConfig config = new EnvironmentConfig(); config.setAllowCreate(create); - config.setLockTimeout(5000000); + config.setLockTimeout(75, TimeUnit.MINUTES); // set to max if(cachePercent>0) { config.setCachePercent(cachePercent); } @@ -297,29 +297,20 @@ Serializable, Closeable { } } - - - /** - * Open a Database inside this BdbModule's environment. - * - * @param name - * @param config - * @param usePriorData use existing data in database, if any - * @return - * @throws DatabaseException - */ - public Database openDatabase(String name, BdbConfig config, boolean usePriorData) - throws DatabaseException { - if (!usePriorData) { - try { - bdbEnvironment.truncateDatabase(null, name, false); - } catch (DatabaseNotFoundException e) { - // Ignored - } + public void disposeDatabase(String name) { + DatabasePlusConfig dpc = databases.remove(name); + if (dpc == null) { + throw new IllegalStateException("No such database: " + name); + } + Database db = dpc.database; + try { + db.close(); + bdbEnvironment.removeDatabase(null, name); + } catch (DatabaseException e) { + LOGGER.log(Level.WARNING, "Error closing db " + name, e); } - return bdbEnvironment.openDatabase(null, name, config.toDatabaseConfig()); } - + /** * Open a Database inside this BdbModule's environment, and * remember it for automatic close-at-module-stop. @@ -348,7 +339,14 @@ Serializable, Closeable { } DatabasePlusConfig dpc = new DatabasePlusConfig(); - dpc.database = openDatabase(name, config, usePriorData); + if (!usePriorData) { + try { + bdbEnvironment.truncateDatabase(null, name, false); + } catch (DatabaseNotFoundException e) { + // Ignored + } + } + dpc.database = bdbEnvironment.openDatabase(null, name, config.toDatabaseConfig()); dpc.name = name; dpc.config = config; databases.put(name, dpc); @@ -359,7 +357,19 @@ Serializable, Closeable { return classCatalog; } - + public StoredQueue getStoredQueue(String dbname, Class clazz, boolean b) { + try { + Database queueDb; + queueDb = openManagedDatabase(dbname, + StoredQueue.databaseConfig(), false); + return new StoredQueue(queueDb, clazz, getClassCatalog()); + } catch (DatabaseException e) { + throw new RuntimeException(e); + } + + } + + /** * Get a CachedBdbMap, backed by a BDB Database of the given name, * with the given key and value class types. If 'recycle' is true, @@ -627,7 +637,7 @@ Serializable, Closeable { * @param allowDuplicates whether duplicate keys allowed * @return */ - public TempStoredSortedMap getStoredMap(String dbName, Class keyClass, Class valueClass, boolean allowDuplicates) { + public DisposableStoredSortedMap getStoredMap(String dbName, Class keyClass, Class valueClass, boolean allowDuplicates) { BdbConfig config = new BdbConfig(); config.setSortedDuplicates(true); config.setAllowCreate(true); @@ -636,8 +646,9 @@ Serializable, Closeable { dbName = "tempMap-"+System.identityHashCode(this)+"-"+sn; sn++; } + final String openName = dbName; try { - mapDb = openDatabase(dbName,config,false); + mapDb = openManagedDatabase(openName,config,false); } catch (DatabaseException e) { throw new RuntimeException(e); } @@ -645,11 +656,18 @@ Serializable, Closeable { if(valueBinding == null) { valueBinding = new SerialBinding(classCatalog, valueClass); } - TempStoredSortedMap storedMap = new TempStoredSortedMap( + DisposableStoredSortedMap storedMap = new DisposableStoredSortedMap( mapDb, TupleBinding.getPrimitiveBinding(keyClass), valueBinding, - true); + true) { + @Override + public void dispose() { + BdbModule.this.disposeDatabase(openName); + } + ; + }; return storedMap; } + } diff --git a/commons/src/main/java/org/archive/bdb/TempStoredSortedMap.java b/commons/src/main/java/org/archive/bdb/DisposableStoredSortedMap.java similarity index 75% rename from commons/src/main/java/org/archive/bdb/TempStoredSortedMap.java rename to commons/src/main/java/org/archive/bdb/DisposableStoredSortedMap.java index 477653f8..5614335e 100644 --- a/commons/src/main/java/org/archive/bdb/TempStoredSortedMap.java +++ b/commons/src/main/java/org/archive/bdb/DisposableStoredSortedMap.java @@ -34,28 +34,28 @@ import com.sleepycat.je.DatabaseException; * @param * @param */ -public class TempStoredSortedMap extends StoredSortedMap { +public class DisposableStoredSortedMap extends StoredSortedMap { Database db; String dbName; - public TempStoredSortedMap(Database db, EntryBinding arg1, EntityBinding arg2, boolean arg3) { + public DisposableStoredSortedMap(Database db, EntryBinding arg1, EntityBinding arg2, boolean arg3) { super(db, arg1, arg2, arg3); this.db = db; } - public TempStoredSortedMap(Database db, EntryBinding arg1, EntityBinding arg2, PrimaryKeyAssigner arg3) { + public DisposableStoredSortedMap(Database db, EntryBinding arg1, EntityBinding arg2, PrimaryKeyAssigner arg3) { super(db, arg1, arg2, arg3); this.db = db; } - public TempStoredSortedMap(Database db, EntryBinding arg1, EntryBinding arg2, boolean arg3) { + public DisposableStoredSortedMap(Database db, EntryBinding arg1, EntryBinding arg2, boolean arg3) { super(db, arg1, arg2, arg3); this.db = db; } - public TempStoredSortedMap(Database db, EntryBinding arg1, EntryBinding arg2, PrimaryKeyAssigner arg3) { + public DisposableStoredSortedMap(Database db, EntryBinding arg1, EntryBinding arg2, PrimaryKeyAssigner arg3) { super(db, arg1, arg2, arg3); this.db = db; } - public void destroy() { + public void dispose() { try { if(this.db!=null) { String name = this.db.getDatabaseName(); @@ -70,7 +70,7 @@ public class TempStoredSortedMap extends StoredSortedMap { @Override protected void finalize() throws Throwable { super.finalize(); - destroy(); + dispose(); } } diff --git a/commons/src/main/java/org/archive/queue/StoredQueue.java b/commons/src/main/java/org/archive/bdb/StoredQueue.java similarity index 98% rename from commons/src/main/java/org/archive/queue/StoredQueue.java rename to commons/src/main/java/org/archive/bdb/StoredQueue.java index 5a160ddc..92d275a6 100644 --- a/commons/src/main/java/org/archive/queue/StoredQueue.java +++ b/commons/src/main/java/org/archive/bdb/StoredQueue.java @@ -17,7 +17,7 @@ * limitations under the License. */ -package org.archive.queue; +package org.archive.bdb; import java.io.Serializable; import java.util.AbstractQueue; @@ -25,7 +25,6 @@ import java.util.Iterator; import java.util.concurrent.atomic.AtomicLong; import java.util.logging.Logger; -import org.archive.bdb.BdbModule; import com.sleepycat.bind.EntryBinding; import com.sleepycat.bind.serial.SerialBinding; diff --git a/commons/src/main/java/org/archive/queue/StoredQueueTest.java b/commons/src/test/java/org/archive/bdb/StoredQueueTest.java similarity index 98% rename from commons/src/main/java/org/archive/queue/StoredQueueTest.java rename to commons/src/test/java/org/archive/bdb/StoredQueueTest.java index b5c9a1d5..615382d0 100644 --- a/commons/src/main/java/org/archive/queue/StoredQueueTest.java +++ b/commons/src/test/java/org/archive/bdb/StoredQueueTest.java @@ -17,7 +17,7 @@ * limitations under the License. */ -package org.archive.queue; +package org.archive.bdb; import java.io.File; import java.util.NoSuchElementException; @@ -25,6 +25,7 @@ import java.util.concurrent.LinkedBlockingQueue; import org.apache.commons.io.FileUtils; import org.archive.bdb.BdbModule; +import org.archive.bdb.StoredQueue; import org.archive.util.TmpDirTestCase; import org.archive.util.bdbje.EnhancedEnvironment; diff --git a/commons/src/test/java/org/archive/settings/file/PrefixFinderTest.java b/commons/src/test/java/org/archive/settings/file/PrefixFinderTest.java index fa95ee12..834e1a6e 100644 --- a/commons/src/test/java/org/archive/settings/file/PrefixFinderTest.java +++ b/commons/src/test/java/org/archive/settings/file/PrefixFinderTest.java @@ -77,8 +77,7 @@ public class PrefixFinderTest extends TmpDirTestCase { @SuppressWarnings("unchecked") public void testStoredSortedMap() throws Exception { EnvironmentConfig config = new EnvironmentConfig(); - config.setAllowCreate(true); - config.setLockTimeout(5000000); + config.setAllowCreate(true); config.setCachePercent(5); File f = new File(getTmpDir(), "PrefixFinderText"); diff --git a/engine/src/main/java/org/archive/crawler/frontier/BdbFrontier.java b/engine/src/main/java/org/archive/crawler/frontier/BdbFrontier.java index d5bc1919..111febeb 100644 --- a/engine/src/main/java/org/archive/crawler/frontier/BdbFrontier.java +++ b/engine/src/main/java/org/archive/crawler/frontier/BdbFrontier.java @@ -18,9 +18,6 @@ */ package org.archive.crawler.frontier; -import java.io.IOException; -import java.io.ObjectInputStream; -import java.io.Serializable; import java.util.Queue; import java.util.SortedMap; import java.util.TreeMap; @@ -34,10 +31,10 @@ import javax.management.openmbean.CompositeData; import org.apache.commons.collections.Closure; import org.archive.bdb.BdbModule; +import org.archive.bdb.StoredQueue; import org.archive.checkpointing.Checkpoint; import org.archive.checkpointing.Checkpointable; import org.archive.modules.CrawlURI; -import org.archive.queue.StoredQueue; import org.archive.util.ArchiveUtils; import org.archive.util.Supplier; import org.json.JSONException; @@ -55,7 +52,7 @@ import com.sleepycat.je.DatabaseException; * @author Gordon Mohr */ public class BdbFrontier extends WorkQueueFrontier -implements Serializable, Checkpointable, BeanNameAware { +implements Checkpointable, BeanNameAware { private static final long serialVersionUID = 1L; private static final Logger logger = @@ -289,11 +286,7 @@ implements Serializable, Checkpointable, BeanNameAware { inactiveQueuesByPrecedence = new TreeMap>(); - Database retiredQueuesDb; - retiredQueuesDb = bdb.openManagedDatabase("retiredQueues", - StoredQueue.databaseConfig(), false); - retiredQueues = new StoredQueue(retiredQueuesDb, - String.class, null); + retiredQueues = bdb.getStoredQueue("retiredQueues", String.class, false); // small risk of OutOfMemoryError: in large crawls with many // unresponsive queues, an unbounded number of snoozed queues @@ -310,42 +303,7 @@ implements Serializable, Checkpointable, BeanNameAware { */ @Override Queue createInactiveQueueForPrecedence(int precedence) { - Database inactiveQueuesDb; - try { - inactiveQueuesDb = bdb.openManagedDatabase("inactiveQueues-"+precedence, - StoredQueue.databaseConfig(), false); - } catch (DatabaseException e) { - throw new RuntimeException(e); - } - return new StoredQueue(inactiveQueuesDb, - String.class, null); - } - - private void readObject(ObjectInputStream in) - throws IOException, ClassNotFoundException { - in.defaultReadObject(); - - // rehook StoredQueues to their databases - for(int precedenceKey : inactiveQueuesByPrecedence.keySet()) { - Database inactiveQueuesDb = - bdb.getDatabase("inactiveQueues-"+precedenceKey); - ((StoredQueue)inactiveQueuesByPrecedence.get(precedenceKey)) - .hookupDatabase(inactiveQueuesDb, String.class, null); - } - - // rehook retiredQueues to its database - Database retiredQueuesDb = bdb.getDatabase("retiredQueues"); - retiredQueues.hookupDatabase(retiredQueuesDb, String.class, null); - - try { - this.pendingUris = new BdbMultipleWorkQueues(bdb.getDatabase("pending"), - bdb.getClassCatalog()); - } catch (DatabaseException e) { - IOException io = new IOException(); - io.initCause(e); - throw io; - } - startManagerThread(); + return bdb.getStoredQueue("inactiveQueues-"+precedence, String.class, false); } /** diff --git a/engine/src/main/java/org/archive/crawler/frontier/BdbMultipleWorkQueues.java b/engine/src/main/java/org/archive/crawler/frontier/BdbMultipleWorkQueues.java index 56d328b7..0561b6f2 100644 --- a/engine/src/main/java/org/archive/crawler/frontier/BdbMultipleWorkQueues.java +++ b/engine/src/main/java/org/archive/crawler/frontier/BdbMultipleWorkQueues.java @@ -290,12 +290,14 @@ public class BdbMultipleWorkQueues { // a queue-beginning cap entry (zero-length value) status = cursor.getSearchKey(headKey, result, null); if (status != OperationStatus.SUCCESS) { - throw new DatabaseException("bdb queue cap missing: " + LOGGER.severe("bdb queue cap missing: " + status.toString() + " " + new String(headKey.getData())); + return status; } if (result.getData().length > 0) { - throw new DatabaseException("bdb queue has nonzero size: " + LOGGER.severe("bdb queue has nonzero size: " + result.getData().length); + return OperationStatus.KEYEXIST; } // get next item (real first item of queue) status = cursor.getNext(headKey,result,null); diff --git a/engine/src/main/java/org/archive/crawler/frontier/WorkQueueFrontier.java b/engine/src/main/java/org/archive/crawler/frontier/WorkQueueFrontier.java index a5c41545..4309d0bf 100644 --- a/engine/src/main/java/org/archive/crawler/frontier/WorkQueueFrontier.java +++ b/engine/src/main/java/org/archive/crawler/frontier/WorkQueueFrontier.java @@ -33,7 +33,6 @@ import java.io.PrintWriter; import java.io.Serializable; import java.lang.ref.SoftReference; import java.util.ArrayList; -import java.util.Arrays; import java.util.Collection; import java.util.Iterator; import java.util.Map; @@ -81,8 +80,9 @@ import com.sleepycat.je.DatabaseException; * @author Christian Kohlschuetter */ public abstract class WorkQueueFrontier extends AbstractFrontier -implements Closeable, CrawlUriReceiver, Serializable, -ApplicationContextAware { +implements Closeable, + CrawlUriReceiver, + ApplicationContextAware { private static final long serialVersionUID = 570384305871965843L; /** truncate reporting of queues at some large but not unbounded number */ @@ -975,13 +975,11 @@ ApplicationContextAware { } long delay_ms = curi.getPolitenessDelay(); - - if (delay_ms > 0) { - snoozeQueue(wq,now,delay_ms); - } else { - reenqueueQueue(wq); - } - + if (delay_ms > 0) { + snoozeQueue(wq,now,delay_ms); + } else { + reenqueueQueue(wq); + } curi.stripToMinimal(); curi.processingCleanup(); @@ -1531,37 +1529,6 @@ ApplicationContextAware { protected int getInProcessCount() { return inProcessQueues.size(); } - - /** - * Custom deserialization: bring in snoozed queues as array of - * their names (aka 'classKeys'). - * @param stream - * @throws IOException - * @throws ClassNotFoundException - */ - private void readObject(java.io.ObjectInputStream stream) - throws IOException, ClassNotFoundException { - stream.defaultReadObject(); - DelayedWorkQueue[] snoozed = (DelayedWorkQueue[])stream.readObject(); - snoozedClassQueues = new DelayQueue( - Arrays.asList(snoozed)); - } - - /** - * Custom serialization: write snoozed queues as array of their - * names (aka 'classKeys'). - * - * @param stream - * @throws IOException - */ - private void writeObject(java.io.ObjectOutputStream stream) - throws IOException { - stream.defaultWriteObject(); - DelayedWorkQueue[] snoozed = snoozedClassQueues.toArray( - new DelayedWorkQueue[0]); - stream.writeObject(snoozed); - } - class DelayedWorkQueue implements Delayed, Serializable { diff --git a/engine/src/main/java/org/archive/crawler/reporting/HostsReport.java b/engine/src/main/java/org/archive/crawler/reporting/HostsReport.java index 7bf72b9d..00dcd2ec 100644 --- a/engine/src/main/java/org/archive/crawler/reporting/HostsReport.java +++ b/engine/src/main/java/org/archive/crawler/reporting/HostsReport.java @@ -23,7 +23,7 @@ import java.io.PrintWriter; import java.util.Map; import org.apache.commons.collections.Closure; -import org.archive.bdb.TempStoredSortedMap; +import org.archive.bdb.DisposableStoredSortedMap; import org.archive.modules.net.CrawlHost; /** @@ -37,7 +37,7 @@ public class HostsReport extends Report { public void write(final PrintWriter writer) { // TODO: use CrawlHosts for all stats; only perform sorting on // manageable number of hosts - TempStoredSortedMap hd = stats.calcReverseSortedHostsDistribution(); + DisposableStoredSortedMap hd = stats.calcReverseSortedHostsDistribution(); // header writer.print("[#urls] [#bytes] [host] [#robots] [#remaining] [#novel-urls] [#novel-bytes] [#dup-by-hash-urls] [#dup-by-hash-bytes] [#not-modified-urls] [#not-modified-bytes]\n"); for (Map.Entry entry : hd.entrySet()) { @@ -57,7 +57,7 @@ public class HostsReport extends Report { host.getSubstats().getNotModifiedUrls(), host.getSubstats().getNotModifiedBytes()); } - hd.destroy(); + hd.dispose(); // StatisticsTracker doesn't know of zero-completion hosts; // so supplement report with those entries from host cache Closure logZeros = new Closure() { diff --git a/engine/src/main/java/org/archive/crawler/reporting/MimetypesReport.java b/engine/src/main/java/org/archive/crawler/reporting/MimetypesReport.java index 3678996e..4b6c09a4 100644 --- a/engine/src/main/java/org/archive/crawler/reporting/MimetypesReport.java +++ b/engine/src/main/java/org/archive/crawler/reporting/MimetypesReport.java @@ -22,7 +22,7 @@ package org.archive.crawler.reporting; import java.io.PrintWriter; import java.util.Map; -import org.archive.bdb.TempStoredSortedMap; +import org.archive.bdb.DisposableStoredSortedMap; /** * The "Mimetypes Report", tallies by MIME type. @@ -35,7 +35,7 @@ public class MimetypesReport extends Report { public void write(PrintWriter writer) { // header writer.print("[#urls] [#bytes] [mime-types]\n"); - TempStoredSortedMap fd = stats.getReverseSortedCopy(stats.getFileDistribution()); + DisposableStoredSortedMap fd = stats.getReverseSortedCopy(stats.getFileDistribution()); for (Map.Entry entry : fd.entrySet()) { // key is -count, value is type writer.print(Math.abs(entry.getKey())); @@ -45,7 +45,7 @@ public class MimetypesReport extends Report { writer.print(entry.getValue()); writer.print("\n"); } - fd.destroy(); + fd.dispose(); } @Override diff --git a/engine/src/main/java/org/archive/crawler/reporting/ResponseCodeReport.java b/engine/src/main/java/org/archive/crawler/reporting/ResponseCodeReport.java index eb815fed..85dfde91 100644 --- a/engine/src/main/java/org/archive/crawler/reporting/ResponseCodeReport.java +++ b/engine/src/main/java/org/archive/crawler/reporting/ResponseCodeReport.java @@ -21,7 +21,7 @@ package org.archive.crawler.reporting; import java.io.PrintWriter; import java.util.Map; -import org.archive.bdb.TempStoredSortedMap; +import org.archive.bdb.DisposableStoredSortedMap; /** * The "Response Codes Report", tallies by response/disposition code. @@ -35,7 +35,7 @@ public class ResponseCodeReport extends Report { // header writer.print("[#urls] [rescode]\n"); - TempStoredSortedMap scd = + DisposableStoredSortedMap scd = stats.getReverseSortedCopy(stats.getStatusCodeDistribution()); for (Map.Entry entry : scd.entrySet()) { writer.print(Math.abs(entry.getKey())); @@ -43,7 +43,7 @@ public class ResponseCodeReport extends Report { writer.print(entry.getValue()); writer.print("\n"); } - scd.destroy(); + scd.dispose(); } @Override diff --git a/engine/src/main/java/org/archive/crawler/reporting/SeedsReport.java b/engine/src/main/java/org/archive/crawler/reporting/SeedsReport.java index 937b1853..94ccbc49 100644 --- a/engine/src/main/java/org/archive/crawler/reporting/SeedsReport.java +++ b/engine/src/main/java/org/archive/crawler/reporting/SeedsReport.java @@ -22,7 +22,7 @@ import java.io.PrintWriter; import java.util.Iterator; import java.util.Map; -import org.archive.bdb.TempStoredSortedMap; +import org.archive.bdb.DisposableStoredSortedMap; import com.sleepycat.collections.StoredIterator; @@ -41,7 +41,7 @@ public class SeedsReport extends Report { long seedsCrawled = 0; long seedsTotal = 0; - TempStoredSortedMap seedsByCode = stats.calcSeedRecordsSortedByStatusCode(); + DisposableStoredSortedMap seedsByCode = stats.calcSeedRecordsSortedByStatusCode(); // for (Map.Entry entry : seedsByCode.entrySet()) { Iterator> iter = seedsByCode.entrySet().iterator(); while(iter.hasNext()) { @@ -65,7 +65,7 @@ public class SeedsReport extends Report { writer.print("\n"); } StoredIterator.close(iter); - seedsByCode.destroy(); + seedsByCode.dispose(); stats.seedsTotal = seedsTotal; stats.seedsCrawled = seedsCrawled; } diff --git a/engine/src/main/java/org/archive/crawler/reporting/SourceTagsReport.java b/engine/src/main/java/org/archive/crawler/reporting/SourceTagsReport.java index d13206c3..4678215e 100644 --- a/engine/src/main/java/org/archive/crawler/reporting/SourceTagsReport.java +++ b/engine/src/main/java/org/archive/crawler/reporting/SourceTagsReport.java @@ -22,7 +22,7 @@ import java.io.PrintWriter; import java.util.Map; import java.util.concurrent.atomic.AtomicLong; -import org.archive.bdb.TempStoredSortedMap; +import org.archive.bdb.DisposableStoredSortedMap; /** * The "Source Report", tallies of source tags (usually seeds) by host. @@ -40,7 +40,7 @@ public class SourceTagsReport extends Report { Map hostCounts = (Map)stats.sourceHostDistribution.get(sourceKey); // sort hosts by #urls - TempStoredSortedMap sortedHostCounts = + DisposableStoredSortedMap sortedHostCounts = stats.getReverseSortedHostCounts(hostCounts); // for each host for (Map.Entry entry : sortedHostCounts.entrySet()) { @@ -51,7 +51,7 @@ public class SourceTagsReport extends Report { writer.print(Math.abs(entry.getKey())); writer.print("\n"); } - sortedHostCounts.destroy(); + sortedHostCounts.dispose(); } } diff --git a/engine/src/main/java/org/archive/crawler/reporting/StatisticsTracker.java b/engine/src/main/java/org/archive/crawler/reporting/StatisticsTracker.java index 8308f502..a5369a20 100644 --- a/engine/src/main/java/org/archive/crawler/reporting/StatisticsTracker.java +++ b/engine/src/main/java/org/archive/crawler/reporting/StatisticsTracker.java @@ -39,7 +39,7 @@ import java.util.logging.Level; import java.util.logging.Logger; import org.archive.bdb.BdbModule; -import org.archive.bdb.TempStoredSortedMap; +import org.archive.bdb.DisposableStoredSortedMap; import org.archive.checkpointing.Checkpoint; import org.archive.checkpointing.Checkpointable; import org.archive.crawler.event.CrawlStateEvent; @@ -687,9 +687,9 @@ public class StatisticsTracker * Assumes values are wrapped with AtomicLong. * @return a sorted set containing the same elements as the map. */ - public TempStoredSortedMap getReverseSortedCopy( + public DisposableStoredSortedMap getReverseSortedCopy( final Map mapOfAtomicLongValues) { - TempStoredSortedMap sortedMap = + DisposableStoredSortedMap sortedMap = bdb.getStoredMap( null, Long.class, @@ -716,9 +716,9 @@ public class StatisticsTracker * Assumes values are wrapped with AtomicLong. * @return a sorted set containing the same elements as the map. */ - public TempStoredSortedMap getReverseSortedCopy( + public DisposableStoredSortedMap getReverseSortedCopy( final ObjectIdentityCache cacheOfAtomicLongValues) { - TempStoredSortedMap sortedMap = + DisposableStoredSortedMap sortedMap = bdb.getStoredMap( null, Long.class, @@ -888,9 +888,9 @@ public class StatisticsTracker return processedSeedsRecords.keySet().iterator(); } - public TempStoredSortedMap calcSeedRecordsSortedByStatusCode() { + public DisposableStoredSortedMap calcSeedRecordsSortedByStatusCode() { Iterator i = getSeedsIterator(); - TempStoredSortedMap sortedMap = + DisposableStoredSortedMap sortedMap = bdb.getStoredMap( null, Integer.class, @@ -915,7 +915,7 @@ public class StatisticsTracker * * @return SortedMap of hosts distribution */ - public TempStoredSortedMap getReverseSortedHostCounts( + public DisposableStoredSortedMap getReverseSortedHostCounts( Map hostCounts) { synchronized(hostCounts){ return getReverseSortedCopy(hostCounts); @@ -927,7 +927,7 @@ public class StatisticsTracker * (largest first) order. * @return SortedMap of hosts distribution */ - public TempStoredSortedMap calcReverseSortedHostsDistribution() { + public DisposableStoredSortedMap calcReverseSortedHostsDistribution() { synchronized(hostsDistribution){ return getReverseSortedCopy(hostsDistribution); }