diff --git a/commons/src/main/java/org/archive/bdb/BdbModule.java b/commons/src/main/java/org/archive/bdb/BdbModule.java index cb47bc64..e986995e 100644 --- a/commons/src/main/java/org/archive/bdb/BdbModule.java +++ b/commons/src/main/java/org/archive/bdb/BdbModule.java @@ -52,7 +52,10 @@ import org.archive.util.ObjectIdentityCache; import org.archive.util.bdbje.EnhancedEnvironment; import org.springframework.context.Lifecycle; +import com.sleepycat.bind.EntryBinding; +import com.sleepycat.bind.serial.SerialBinding; import com.sleepycat.bind.serial.StoredClassCatalog; +import com.sleepycat.bind.tuple.TupleBinding; import com.sleepycat.je.CheckpointConfig; import com.sleepycat.je.Database; import com.sleepycat.je.DatabaseConfig; @@ -60,9 +63,6 @@ import com.sleepycat.je.DatabaseException; import com.sleepycat.je.DatabaseNotFoundException; import com.sleepycat.je.DbInternal; import com.sleepycat.je.EnvironmentConfig; -import com.sleepycat.je.SecondaryConfig; -import com.sleepycat.je.SecondaryDatabase; -import com.sleepycat.je.SecondaryKeyCreator; import com.sleepycat.je.dbi.EnvironmentImpl; import com.sleepycat.je.utilint.DbLsn; @@ -154,35 +154,6 @@ Serializable, Closeable { } } - - public static class SecondaryBdbConfig extends BdbConfig { - private static final long serialVersionUID = 1L; - - private SecondaryKeyCreator keyCreator; - - public SecondaryBdbConfig() { - } - - public SecondaryKeyCreator getKeyCreator() { - return keyCreator; - } - - public void setKeyCreator(SecondaryKeyCreator keyCreator) { - this.keyCreator = keyCreator; - } - - public SecondaryConfig toSecondaryConfig() { - SecondaryConfig result = new SecondaryConfig(); - result.setDeferredWrite(true); - result.setTransactional(transactional); - result.setAllowCreate(allowCreate); - result.setSortedDuplicates(sortedDuplicates); - result.setKeyCreator(keyCreator); - return result; - } - - } - protected ConfigPath dir = new ConfigPath("bdbmodule subdirectory","state"); public ConfigPath getDir() { return dir; @@ -340,23 +311,6 @@ Serializable, Closeable { return dpc.database; } - - public SecondaryDatabase openSecondaryDatabase(String name, Database db, - SecondaryBdbConfig config) throws DatabaseException { - if (databases.containsKey(name)) { - throw new IllegalStateException("Database already exists: " +name); - } - SecondaryDatabase result = bdbEnvironment.openSecondaryDatabase(null, - name, db, config.toSecondaryConfig()); - DatabasePlusConfig dpc = new DatabasePlusConfig(); - dpc.database = result; - dpc.name = name; - dpc.primaryName = db.getDatabaseName(); - dpc.config = config; - databases.put(name, dpc); - return result; - } - public StoredClassCatalog getClassCatalog() { return classCatalog; } @@ -482,18 +436,8 @@ Serializable, Closeable { // this.classCatalog); // } for (DatabasePlusConfig dpc: databases.values()) { - if (!(dpc.config instanceof SecondaryBdbConfig)) { - dpc.database = bdbEnvironment.openDatabase(null, - dpc.name, dpc.config.toDatabaseConfig()); - } - } - for (DatabasePlusConfig dpc: databases.values()) { - if (dpc.config instanceof SecondaryBdbConfig) { - SecondaryBdbConfig conf = (SecondaryBdbConfig)dpc.config; - Database primary = databases.get(dpc.primaryName).database; - dpc.database = bdbEnvironment.openSecondaryDatabase(null, - dpc.name, primary, conf.toSecondaryConfig()); - } + dpc.database = bdbEnvironment.openDatabase(null, + dpc.name, dpc.config.toDatabaseConfig()); } } catch (DatabaseException e) { IOException io = new IOException(); @@ -724,15 +668,12 @@ Serializable, Closeable { path = recovery.translatePath(path); FileUtils.copyDirectory(bdbDir, new File(path)); } - } private static class BdbShutdownHook extends Thread { - final private BdbModule bdb; - public BdbShutdownHook(BdbModule bdb) { this.bdb = bdb; } @@ -740,6 +681,47 @@ Serializable, Closeable { public void run() { this.bdb.close2(); } + } + + /** uniqueness serial number for temp map databases */ + long sn = 0; + /** + * Creates a database-backed TempStoredSortedMap for transient + * reporting requirements. Calling the returned map's destroy() + * method when done discards the associated Database. + * + * @param + * @param + * @param dbName Database name to use; if null a name will be synthesized + * @param keyClass Class of keys; should be a Java primitive type + * @param valueClass Class of values; may be any serializable type + * @param allowDuplicates whether duplicate keys allowed + * @return + */ + public TempStoredSortedMap getStoredMap(String dbName, Class keyClass, Class valueClass, boolean allowDuplicates) { + BdbConfig config = new BdbConfig(); + config.setSortedDuplicates(true); + config.setAllowCreate(true); + Database mapDb; + if(dbName==null) { + dbName = "tempMap-"+System.identityHashCode(this)+"-"+sn; + sn++; + } + try { + mapDb = openDatabase(dbName,config,false); + } catch (DatabaseException e) { + throw new RuntimeException(e); + } + EntryBinding valueBinding = TupleBinding.getPrimitiveBinding(valueClass); + if(valueBinding == null) { + valueBinding = new SerialBinding(classCatalog, valueClass); + } + TempStoredSortedMap storedMap = new TempStoredSortedMap( + mapDb, + TupleBinding.getPrimitiveBinding(keyClass), + valueBinding, + true); + return storedMap; } } diff --git a/commons/src/main/java/org/archive/bdb/TempStoredSortedMap.java b/commons/src/main/java/org/archive/bdb/TempStoredSortedMap.java new file mode 100644 index 00000000..477653f8 --- /dev/null +++ b/commons/src/main/java/org/archive/bdb/TempStoredSortedMap.java @@ -0,0 +1,76 @@ +/* + * This file is part of the Heritrix web crawler (crawler.archive.org). + * + * Licensed to the Internet Archive (IA) by one or more individual + * contributors. + * + * The IA licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.archive.bdb; + +import com.sleepycat.bind.EntityBinding; +import com.sleepycat.bind.EntryBinding; +import com.sleepycat.collections.PrimaryKeyAssigner; +import com.sleepycat.collections.StoredSortedMap; +import com.sleepycat.je.Database; +import com.sleepycat.je.DatabaseException; + +/** + * TempStoredSortedMap remembers its backing Database, and offers + * a destroy() method for closing/discarding the underlying Database. + * + * @contributor gojomo + * @param + * @param + */ +public class TempStoredSortedMap extends StoredSortedMap { + Database db; + String dbName; + + public TempStoredSortedMap(Database db, EntryBinding arg1, EntityBinding arg2, boolean arg3) { + super(db, arg1, arg2, arg3); + this.db = db; + } + public TempStoredSortedMap(Database db, EntryBinding arg1, EntityBinding arg2, PrimaryKeyAssigner arg3) { + super(db, arg1, arg2, arg3); + this.db = db; + } + public TempStoredSortedMap(Database db, EntryBinding arg1, EntryBinding arg2, boolean arg3) { + super(db, arg1, arg2, arg3); + this.db = db; + } + public TempStoredSortedMap(Database db, EntryBinding arg1, EntryBinding arg2, PrimaryKeyAssigner arg3) { + super(db, arg1, arg2, arg3); + this.db = db; + } + + public void destroy() { + try { + if(this.db!=null) { + String name = this.db.getDatabaseName(); + this.db.close(); + this.db.getEnvironment().removeDatabase(null, name); + this.db = null; + } + } catch (DatabaseException e) { + throw new RuntimeException(e); + } + } + @Override + protected void finalize() throws Throwable { + super.finalize(); + destroy(); + } + +} diff --git a/engine/src/main/java/org/archive/crawler/reporting/HostsReport.java b/engine/src/main/java/org/archive/crawler/reporting/HostsReport.java index 3b5faf99..a1404baf 100644 --- a/engine/src/main/java/org/archive/crawler/reporting/HostsReport.java +++ b/engine/src/main/java/org/archive/crawler/reporting/HostsReport.java @@ -20,11 +20,10 @@ package org.archive.crawler.reporting; import java.io.PrintWriter; -import java.util.Iterator; -import java.util.SortedMap; -import java.util.concurrent.atomic.AtomicLong; +import java.util.Map; import org.apache.commons.collections.Closure; +import org.archive.bdb.TempStoredSortedMap; import org.archive.modules.net.CrawlHost; /** @@ -38,21 +37,21 @@ public class HostsReport extends Report { public void write(final PrintWriter writer) { // TODO: use CrawlHosts for all stats; only perform sorting on // manageable number of hosts - SortedMap hd = stats.calcReverseSortedHostsDistribution(); + TempStoredSortedMap hd = stats.calcReverseSortedHostsDistribution(); // header writer.print("[#urls] [#bytes] [host] [#robots] [#remaining]\n"); - for (Iterator i = hd.keySet().iterator(); i.hasNext();) { - // Key is 'host'. - String key = (String) i.next(); - CrawlHost host = stats.serverCache.getHostFor(key); - AtomicLong val = (AtomicLong)hd.get(key); + for (Map.Entry entry : hd.entrySet()) { + // key is -count, value is hostname + CrawlHost host = stats.serverCache.getHostFor(entry.getValue()); + long count = Math.abs(entry.getKey()); writeReportLine(writer, - ((val==null)?"-":val.get()), - stats.getBytesPerHost(key), - key, + count, + stats.getBytesPerHost(entry.getValue()), + entry.getValue(), host.getSubstats().getRobotsDenials(), host.getSubstats().getRemaining()); } + hd.destroy(); // StatisticsTracker doesn't know of zero-completion hosts; // so supplement report with those entries from host cache Closure logZeros = new Closure() { diff --git a/engine/src/main/java/org/archive/crawler/reporting/MimetypesReport.java b/engine/src/main/java/org/archive/crawler/reporting/MimetypesReport.java index 45d6ffc3..3678996e 100644 --- a/engine/src/main/java/org/archive/crawler/reporting/MimetypesReport.java +++ b/engine/src/main/java/org/archive/crawler/reporting/MimetypesReport.java @@ -20,9 +20,9 @@ package org.archive.crawler.reporting; import java.io.PrintWriter; -import java.util.Iterator; -import java.util.TreeMap; -import java.util.concurrent.atomic.AtomicLong; +import java.util.Map; + +import org.archive.bdb.TempStoredSortedMap; /** * The "Mimetypes Report", tallies by MIME type. @@ -35,17 +35,18 @@ public class MimetypesReport extends Report { public void write(PrintWriter writer) { // header writer.print("[#urls] [#bytes] [mime-types]\n"); - TreeMap fd = stats.getReverseSortedCopy(stats.getFileDistribution()); - for (Iterator i = fd.keySet().iterator(); i.hasNext();) { - Object key = i.next(); - // Key is mime type. - writer.print(Long.toString(((AtomicLong)fd.get(key)).get())); + TempStoredSortedMap fd = stats.getReverseSortedCopy(stats.getFileDistribution()); + for (Map.Entry entry : fd.entrySet()) { + // key is -count, value is type + writer.print(Math.abs(entry.getKey())); writer.print(" "); - writer.print(Long.toString(stats.getBytesPerFileType((String)key))); + writer.print(stats.getBytesPerFileType(entry.getValue())); writer.print(" "); - writer.print((String)key); + writer.print(entry.getValue()); writer.print("\n"); - } } + } + fd.destroy(); + } @Override public String getFilename() { diff --git a/engine/src/main/java/org/archive/crawler/reporting/ResponseCodeReport.java b/engine/src/main/java/org/archive/crawler/reporting/ResponseCodeReport.java index 78c4907e..eb815fed 100644 --- a/engine/src/main/java/org/archive/crawler/reporting/ResponseCodeReport.java +++ b/engine/src/main/java/org/archive/crawler/reporting/ResponseCodeReport.java @@ -19,9 +19,9 @@ package org.archive.crawler.reporting; import java.io.PrintWriter; -import java.util.Iterator; -import java.util.TreeMap; -import java.util.concurrent.atomic.AtomicLong; +import java.util.Map; + +import org.archive.bdb.TempStoredSortedMap; /** * The "Response Codes Report", tallies by response/disposition code. @@ -33,17 +33,17 @@ public class ResponseCodeReport extends Report { @Override public void write(PrintWriter writer) { // header - writer.print("[rescode] [#urls]\n"); + writer.print("[#urls] [rescode]\n"); - TreeMap scd = + TempStoredSortedMap scd = stats.getReverseSortedCopy(stats.getStatusCodeDistribution()); - for (Iterator i = scd.keySet().iterator(); i.hasNext();) { - Object key = i.next(); - writer.print((String)key); + for (Map.Entry entry : scd.entrySet()) { + writer.print(Math.abs(entry.getKey())); writer.print(" "); - writer.print(Long.toString(((AtomicLong)scd.get(key)).get())); + writer.print(entry.getValue()); writer.print("\n"); } + scd.destroy(); } @Override diff --git a/engine/src/main/java/org/archive/crawler/reporting/SeedRecord.java b/engine/src/main/java/org/archive/crawler/reporting/SeedRecord.java index 75a1f24b..b65da729 100644 --- a/engine/src/main/java/org/archive/crawler/reporting/SeedRecord.java +++ b/engine/src/main/java/org/archive/crawler/reporting/SeedRecord.java @@ -130,4 +130,8 @@ public class SeedRecord implements CoreAttributeConstants, Serializable { public String getUri() { return uri; } + + public int sortShiftStatusCode() { + return -statusCode - Integer.MAX_VALUE; + } } \ No newline at end of file diff --git a/engine/src/main/java/org/archive/crawler/reporting/SeedsReport.java b/engine/src/main/java/org/archive/crawler/reporting/SeedsReport.java index 5f4510b2..ecbabbe7 100644 --- a/engine/src/main/java/org/archive/crawler/reporting/SeedsReport.java +++ b/engine/src/main/java/org/archive/crawler/reporting/SeedsReport.java @@ -20,6 +20,11 @@ package org.archive.crawler.reporting; import java.io.PrintWriter; import java.util.Iterator; +import java.util.Map; + +import org.archive.bdb.TempStoredSortedMap; + +import com.sleepycat.collections.StoredIterator; /** @@ -36,9 +41,12 @@ public class SeedsReport extends Report { long seedsCrawled = 0; long seedsTotal = 0; - for (Iterator i = stats.getSeedRecordsSortedByStatusCode(); - i.hasNext();) { - SeedRecord sr = (SeedRecord)i.next(); + TempStoredSortedMap seedsByCode = stats.getSeedRecordsSortedByStatusCode(); +// for (Map.Entry entry : seedsByCode.entrySet()) { + Iterator> iter = seedsByCode.entrySet().iterator(); + while(iter.hasNext()) { + Map.Entry entry = iter.next(); + SeedRecord sr = entry.getValue(); writer.print(sr.getStatusCode()); writer.print(" "); seedsTotal++; @@ -56,6 +64,8 @@ public class SeedsReport extends Report { } writer.print("\n"); } + StoredIterator.close(iter); + seedsByCode.destroy(); stats.seedsTotal = seedsTotal; stats.seedsCrawled = seedsCrawled; } diff --git a/engine/src/main/java/org/archive/crawler/reporting/SourceTagsReport.java b/engine/src/main/java/org/archive/crawler/reporting/SourceTagsReport.java index ee3f81bf..d13206c3 100644 --- a/engine/src/main/java/org/archive/crawler/reporting/SourceTagsReport.java +++ b/engine/src/main/java/org/archive/crawler/reporting/SourceTagsReport.java @@ -19,11 +19,11 @@ package org.archive.crawler.reporting; import java.io.PrintWriter; -import java.util.Iterator; import java.util.Map; -import java.util.SortedMap; import java.util.concurrent.atomic.AtomicLong; +import org.archive.bdb.TempStoredSortedMap; + /** * The "Source Report", tallies of source tags (usually seeds) by host. * @@ -36,24 +36,22 @@ public class SourceTagsReport extends Report { writer.print("[source] [host] [#urls]\n"); // for each source - for (Iterator i = stats.sourceHostDistribution.keySet().iterator(); i.hasNext();) { - String sourceKey = i.next(); + for (String sourceKey : stats.sourceHostDistribution.keySet()) { Map hostCounts = (Map)stats.sourceHostDistribution.get(sourceKey); // sort hosts by #urls - SortedMap sortedHostCounts = + TempStoredSortedMap sortedHostCounts = stats.getReverseSortedHostCounts(hostCounts); // for each host - for (Iterator j = sortedHostCounts.keySet().iterator(); j.hasNext();) { - Object hostKey = j.next(); - AtomicLong hostCount = (AtomicLong) hostCounts.get(hostKey); + for (Map.Entry entry : sortedHostCounts.entrySet()) { writer.print(sourceKey.toString()); writer.print(" "); - writer.print(hostKey.toString()); + writer.print(entry.getValue()); writer.print(" "); - writer.print(hostCount.get()); + writer.print(Math.abs(entry.getKey())); writer.print("\n"); } + sortedHostCounts.destroy(); } } diff --git a/engine/src/main/java/org/archive/crawler/reporting/StatisticsTracker.java b/engine/src/main/java/org/archive/crawler/reporting/StatisticsTracker.java index 5f043b0b..1f48a1fe 100644 --- a/engine/src/main/java/org/archive/crawler/reporting/StatisticsTracker.java +++ b/engine/src/main/java/org/archive/crawler/reporting/StatisticsTracker.java @@ -26,14 +26,11 @@ import java.io.FileWriter; import java.io.IOException; import java.io.PrintWriter; import java.io.Serializable; -import java.util.Comparator; import java.util.Date; import java.util.Iterator; import java.util.LinkedList; import java.util.Map; import java.util.SortedMap; -import java.util.TreeMap; -import java.util.TreeSet; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentMap; import java.util.concurrent.Executors; @@ -44,6 +41,7 @@ import java.util.logging.Level; import java.util.logging.Logger; import org.archive.bdb.BdbModule; +import org.archive.bdb.TempStoredSortedMap; import org.archive.crawler.event.CrawlStateEvent; import org.archive.crawler.event.CrawlURIDispositionEvent; import org.archive.crawler.event.StatSnapshotEvent; @@ -615,12 +613,12 @@ public class StatisticsTracker } /** - * Sort the entries of the given HashMap in descending order by their + * Sort the entries of the given Map in descending order by their * values, which must be longs wrapped with AtomicLong. *

* Elements are sorted by value from largest to smallest. Equal values are - * sorted in an arbitrary, but consistent manner by their keys. Only items - * with identical value and key are considered equal. + * sorted by their keys. The returned map is a StoredSortedMap, and + * thus may include duplicate keys. * * If the passed-in map requires access to be synchronized, the caller * should ensure this synchronization. @@ -629,32 +627,16 @@ public class StatisticsTracker * Assumes values are wrapped with AtomicLong. * @return a sorted set containing the same elements as the map. */ - public TreeMap getReverseSortedCopy( + public TempStoredSortedMap getReverseSortedCopy( final Map mapOfAtomicLongValues) { - TreeMap sortedMap = - new TreeMap(new Comparator() { - public int compare(String e1, String e2) { - long firstVal = mapOfAtomicLongValues.get(e1).get(); - long secondVal = mapOfAtomicLongValues.get(e2).get(); - if (firstVal < secondVal) { - return 1; - } - if (secondVal < firstVal) { - return -1; - } - // If the values are the same, sort by keys. - return e1.compareTo(e2); - } - }); - try { - sortedMap.putAll(mapOfAtomicLongValues); - } catch (UnsupportedOperationException e) { - Iterator i = mapOfAtomicLongValues.keySet().iterator(); - for (;i.hasNext();) { - // Ok. Try doing it the slow way then. - String key = i.next(); - sortedMap.put(key, mapOfAtomicLongValues.get(key)); - } + TempStoredSortedMap sortedMap = + bdb.getStoredMap( + null, + Long.class, + String.class, + true); + for(String k : mapOfAtomicLongValues.keySet()) { + sortedMap.put(-mapOfAtomicLongValues.get(k).longValue(), k); } return sortedMap; } @@ -674,25 +656,16 @@ public class StatisticsTracker * Assumes values are wrapped with AtomicLong. * @return a sorted set containing the same elements as the map. */ - public TreeMap getReverseSortedCopy( + public TempStoredSortedMap getReverseSortedCopy( final ObjectIdentityCache cacheOfAtomicLongValues) { - TreeMap sortedMap = - new TreeMap(new Comparator() { - public int compare(String e1, String e2) { - long firstVal = cacheOfAtomicLongValues.get(e1).get(); - long secondVal = cacheOfAtomicLongValues.get(e2).get(); - if (firstVal < secondVal) { - return 1; - } - if (secondVal < firstVal) { - return -1; - } - // If the values are the same, sort by keys. - return e1.compareTo(e2); - } - }); - for(String key : cacheOfAtomicLongValues.keySet()) { - sortedMap.put(key, cacheOfAtomicLongValues.get(key)); + TempStoredSortedMap sortedMap = + bdb.getStoredMap( + null, + Long.class, + String.class, + true); + for(String k : cacheOfAtomicLongValues.keySet()) { + sortedMap.put(-cacheOfAtomicLongValues.get(k).longValue(), k); } return sortedMap; } @@ -854,27 +827,15 @@ public class StatisticsTracker return processedSeedsRecords.keySet().iterator(); } - public Iterator getSeedRecordsSortedByStatusCode() { + public TempStoredSortedMap getSeedRecordsSortedByStatusCode() { Iterator i = getSeedsIterator(); - TreeSet sortedSet = - new TreeSet(new Comparator() { - public int compare(SeedRecord sr1, SeedRecord sr2) { - int code1 = sr1.getStatusCode(); - int code2 = sr2.getStatusCode(); - if (code1 == code2) { - // If the values are equal, sort by URIs. - return sr1.getUri().compareTo(sr2.getUri()); - } - // mirror and shift the nubmer line so as to - // place zero at the beginning, then all negatives - // in order of ascending absolute value, then all - // positives descending - code1 = -code1 - Integer.MAX_VALUE; - code2 = -code2 - Integer.MAX_VALUE; - - return new Integer(code1).compareTo(new Integer(code2)); - } - }); + TempStoredSortedMap sortedMap = + bdb.getStoredMap( + null, + Integer.class, + SeedRecord.class, + true); + while (i.hasNext()) { String seed = i.next(); SeedRecord sr = (SeedRecord) processedSeedsRecords.get(seed); @@ -882,9 +843,9 @@ public class StatisticsTracker sr = new SeedRecord(seed,"Seed has not been processed"); // no need to retain synthesized record } - sortedSet.add(sr); + sortedMap.put(sr.sortShiftStatusCode(), sr); } - return sortedSet.iterator(); + return sortedMap; } /** @@ -893,7 +854,7 @@ public class StatisticsTracker * * @return SortedMap of hosts distribution */ - public SortedMap getReverseSortedHostCounts( + public TempStoredSortedMap getReverseSortedHostCounts( Map hostCounts) { synchronized(hostCounts){ return getReverseSortedCopy(hostCounts); @@ -906,7 +867,7 @@ public class StatisticsTracker * @return SortedMap of hosts distribution * @deprecated Use {@link #calcReverseSortedHostsDistribution()} instead */ - public SortedMap getReverseSortedHostsDistribution() { + public SortedMap getReverseSortedHostsDistribution() { return calcReverseSortedHostsDistribution(); } /** @@ -914,7 +875,7 @@ public class StatisticsTracker * (largest first) order. * @return SortedMap of hosts distribution */ - public SortedMap calcReverseSortedHostsDistribution() { + public TempStoredSortedMap calcReverseSortedHostsDistribution() { synchronized(hostsDistribution){ return getReverseSortedCopy(hostsDistribution); }