[HER-769] Support case where millions of seeds.

* BdbModule.java
    add temp-stored-map service method
    remove unused secondary-db support
* TempStoredSortedMap.java
    StoredSortedMap that can destroy its underlying database after temporary use
* StatisticsTracker.java
    update various sorted-by-decreasing-frequency methods to use temp StoredMaps, with duplicate keys that are negative counts
* (Multiple)Report.java
    update to use new duplicate-keyed frequency maps

engine/src/main/java/org/archive/crawler/reporting/HostsReport.java

@@ -20,11 +20,10 @@
 package org.archive.crawler.reporting;
 
 import java.io.PrintWriter;
-import java.util.Iterator;
-import java.util.SortedMap;
-import java.util.concurrent.atomic.AtomicLong;
+import java.util.Map;
 
 import org.apache.commons.collections.Closure;
+import org.archive.bdb.TempStoredSortedMap;
 import org.archive.modules.net.CrawlHost;
 
 /**
@@ -38,21 +37,21 @@ public class HostsReport extends Report {
     public void write(final PrintWriter writer) {
         // TODO: use CrawlHosts for all stats; only perform sorting on 
         // manageable number of hosts
-        SortedMap<String,AtomicLong> hd = stats.calcReverseSortedHostsDistribution();
+        TempStoredSortedMap<Long,String> hd = stats.calcReverseSortedHostsDistribution();
         // header
         writer.print("[#urls] [#bytes] [host] [#robots] [#remaining]\n");
-        for (Iterator<String> i = hd.keySet().iterator(); i.hasNext();) {
-            // Key is 'host'.
-            String key = (String) i.next();
-            CrawlHost host = stats.serverCache.getHostFor(key);
-            AtomicLong val = (AtomicLong)hd.get(key);
+        for (Map.Entry<Long,String> entry : hd.entrySet()) {
+            // key is -count, value is hostname
+            CrawlHost host = stats.serverCache.getHostFor(entry.getValue());
+            long count = Math.abs(entry.getKey()); 
             writeReportLine(writer,
-                    ((val==null)?"-":val.get()),
-                    stats.getBytesPerHost(key),
-                    key,
+                    count,
+                    stats.getBytesPerHost(entry.getValue()),
+                    entry.getValue(),
                     host.getSubstats().getRobotsDenials(),
                     host.getSubstats().getRemaining());
         }
+        hd.destroy();
         // StatisticsTracker doesn't know of zero-completion hosts; 
         // so supplement report with those entries from host cache
         Closure logZeros = new Closure() {

engine/src/main/java/org/archive/crawler/reporting/MimetypesReport.java

@@ -20,9 +20,9 @@
 package org.archive.crawler.reporting;
 
 import java.io.PrintWriter;
-import java.util.Iterator;
-import java.util.TreeMap;
-import java.util.concurrent.atomic.AtomicLong;
+import java.util.Map;
+
+import org.archive.bdb.TempStoredSortedMap;
 
 /**
  * The "Mimetypes Report", tallies by MIME type.
@@ -35,17 +35,18 @@ public class MimetypesReport extends Report {
     public void write(PrintWriter writer) {
         // header
         writer.print("[#urls] [#bytes] [mime-types]\n");
-        TreeMap<String,AtomicLong> fd = stats.getReverseSortedCopy(stats.getFileDistribution());
-        for (Iterator<String> i = fd.keySet().iterator(); i.hasNext();) {
-            Object key = i.next();
-            // Key is mime type.
-            writer.print(Long.toString(((AtomicLong)fd.get(key)).get()));
+        TempStoredSortedMap<Long,String> fd = stats.getReverseSortedCopy(stats.getFileDistribution());
+        for (Map.Entry<Long,String> entry : fd.entrySet()) {
+            // key is -count, value is type
+            writer.print(Math.abs(entry.getKey()));
             writer.print(" ");
-            writer.print(Long.toString(stats.getBytesPerFileType((String)key)));
+            writer.print(stats.getBytesPerFileType(entry.getValue()));
             writer.print(" ");
-            writer.print((String)key);
+            writer.print(entry.getValue());
             writer.print("\n");
-        }    }
+        }
+        fd.destroy();
+    }
 
     @Override
     public String getFilename() {

engine/src/main/java/org/archive/crawler/reporting/ResponseCodeReport.java

@@ -19,9 +19,9 @@
 package org.archive.crawler.reporting;
 
 import java.io.PrintWriter;
-import java.util.Iterator;
-import java.util.TreeMap;
-import java.util.concurrent.atomic.AtomicLong;
+import java.util.Map;
+
+import org.archive.bdb.TempStoredSortedMap;
 
 /**
  * The "Response Codes Report", tallies by response/disposition code.
@@ -33,17 +33,17 @@ public class ResponseCodeReport extends Report {
     @Override
     public void write(PrintWriter writer) {
         // header
-        writer.print("[rescode] [#urls]\n");
+        writer.print("[#urls] [rescode]\n");
         
-        TreeMap<String,AtomicLong> scd = 
+        TempStoredSortedMap<Long,String> scd = 
             stats.getReverseSortedCopy(stats.getStatusCodeDistribution());
-        for (Iterator<String> i = scd.keySet().iterator(); i.hasNext();) {
-            Object key = i.next();
-            writer.print((String)key);
+        for (Map.Entry<Long,String> entry : scd.entrySet()) {
+            writer.print(Math.abs(entry.getKey()));
             writer.print(" ");
-            writer.print(Long.toString(((AtomicLong)scd.get(key)).get()));
+            writer.print(entry.getValue());
             writer.print("\n");
         }
+        scd.destroy();
     }
 
     @Override

engine/src/main/java/org/archive/crawler/reporting/SeedRecord.java

@@ -130,4 +130,8 @@ public class SeedRecord implements CoreAttributeConstants, Serializable {
     public String getUri() {
         return uri;
     }
+
+    public int sortShiftStatusCode() {
+        return -statusCode - Integer.MAX_VALUE;
+    }
 }

engine/src/main/java/org/archive/crawler/reporting/SeedsReport.java

@@ -20,6 +20,11 @@ package org.archive.crawler.reporting;
 
 import java.io.PrintWriter;
 import java.util.Iterator;
+import java.util.Map;
+
+import org.archive.bdb.TempStoredSortedMap;
+
+import com.sleepycat.collections.StoredIterator;
 
 
 /**
@@ -36,9 +41,12 @@ public class SeedsReport extends Report {
 
         long seedsCrawled = 0;
         long seedsTotal = 0;
-        for (Iterator<SeedRecord> i = stats.getSeedRecordsSortedByStatusCode();
-                i.hasNext();) {
-            SeedRecord sr = (SeedRecord)i.next();
+        TempStoredSortedMap<Integer, SeedRecord> seedsByCode = stats.getSeedRecordsSortedByStatusCode();
+//        for (Map.Entry<Integer,SeedRecord> entry : seedsByCode.entrySet()) {
+        Iterator<Map.Entry<Integer,SeedRecord>> iter = seedsByCode.entrySet().iterator();
+        while(iter.hasNext()) {
+            Map.Entry<Integer,SeedRecord> entry = iter.next();
+            SeedRecord sr = entry.getValue();
             writer.print(sr.getStatusCode());
             writer.print(" ");
             seedsTotal++;
@@ -56,6 +64,8 @@ public class SeedsReport extends Report {
             }
             writer.print("\n");
         }
+        StoredIterator.close(iter);
+        seedsByCode.destroy();
         stats.seedsTotal = seedsTotal;
         stats.seedsCrawled = seedsCrawled; 
     }

engine/src/main/java/org/archive/crawler/reporting/SourceTagsReport.java

@@ -19,11 +19,11 @@
 package org.archive.crawler.reporting;
 
 import java.io.PrintWriter;
-import java.util.Iterator;
 import java.util.Map;
-import java.util.SortedMap;
 import java.util.concurrent.atomic.AtomicLong;
 
+import org.archive.bdb.TempStoredSortedMap;
+
 /**
  * The "Source Report", tallies of source tags (usually seeds) by host.
  * 
@@ -36,24 +36,22 @@ public class SourceTagsReport extends Report {
 
         writer.print("[source] [host] [#urls]\n");
         // for each source
-        for (Iterator<String> i = stats.sourceHostDistribution.keySet().iterator(); i.hasNext();) {
-            String sourceKey = i.next();
+        for (String sourceKey : stats.sourceHostDistribution.keySet()) {
             Map<String,AtomicLong> hostCounts = 
                 (Map<String,AtomicLong>)stats.sourceHostDistribution.get(sourceKey);
             // sort hosts by #urls
-            SortedMap<String,AtomicLong> sortedHostCounts = 
+            TempStoredSortedMap<Long,String> sortedHostCounts = 
                 stats.getReverseSortedHostCounts(hostCounts);
             // for each host
-            for (Iterator<String> j = sortedHostCounts.keySet().iterator(); j.hasNext();) {
-                Object hostKey = j.next();
-                AtomicLong hostCount = (AtomicLong) hostCounts.get(hostKey);
+            for (Map.Entry<Long, String> entry : sortedHostCounts.entrySet()) {
                 writer.print(sourceKey.toString());
                 writer.print(" ");
-                writer.print(hostKey.toString());
+                writer.print(entry.getValue());
                 writer.print(" ");
-                writer.print(hostCount.get());
+                writer.print(Math.abs(entry.getKey()));
                 writer.print("\n");
             }
+            sortedHostCounts.destroy();
         }
     }
 

engine/src/main/java/org/archive/crawler/reporting/StatisticsTracker.java

@@ -26,14 +26,11 @@ import java.io.FileWriter;
 import java.io.IOException;
 import java.io.PrintWriter;
 import java.io.Serializable;
-import java.util.Comparator;
 import java.util.Date;
 import java.util.Iterator;
 import java.util.LinkedList;
 import java.util.Map;
 import java.util.SortedMap;
-import java.util.TreeMap;
-import java.util.TreeSet;
 import java.util.concurrent.ConcurrentHashMap;
 import java.util.concurrent.ConcurrentMap;
 import java.util.concurrent.Executors;
@@ -44,6 +41,7 @@ import java.util.logging.Level;
 import java.util.logging.Logger;
 
 import org.archive.bdb.BdbModule;
+import org.archive.bdb.TempStoredSortedMap;
 import org.archive.crawler.event.CrawlStateEvent;
 import org.archive.crawler.event.CrawlURIDispositionEvent;
 import org.archive.crawler.event.StatSnapshotEvent;
@@ -615,12 +613,12 @@ public class StatisticsTracker
     }
 
     /**
-     * Sort the entries of the given HashMap in descending order by their
+     * Sort the entries of the given Map in descending order by their
      * values, which must be longs wrapped with <code>AtomicLong</code>.
      * <p>
      * Elements are sorted by value from largest to smallest. Equal values are
-     * sorted in an arbitrary, but consistent manner by their keys. Only items
-     * with identical value and key are considered equal.
+     * sorted by their keys. The returned map is a StoredSortedMap, and
+     * thus may include duplicate keys. 
      *
      * If the passed-in map requires access to be synchronized, the caller
      * should ensure this synchronization. 
@@ -629,32 +627,16 @@ public class StatisticsTracker
      *            Assumes values are wrapped with AtomicLong.
      * @return a sorted set containing the same elements as the map.
      */
-    public TreeMap<String,AtomicLong> getReverseSortedCopy(
+    public TempStoredSortedMap<Long,String> getReverseSortedCopy(
             final Map<String,AtomicLong> mapOfAtomicLongValues) {
-        TreeMap<String,AtomicLong> sortedMap = 
-          new TreeMap<String,AtomicLong>(new Comparator<String>() {
-            public int compare(String e1, String e2) {
-                long firstVal = mapOfAtomicLongValues.get(e1).get();
-                long secondVal = mapOfAtomicLongValues.get(e2).get();
-                if (firstVal < secondVal) {
-                    return 1;
-                }
-                if (secondVal < firstVal) {
-                    return -1;
-                }
-                // If the values are the same, sort by keys.
-                return e1.compareTo(e2);
-            }
-        });
-        try {
-            sortedMap.putAll(mapOfAtomicLongValues);
-        } catch (UnsupportedOperationException e) {
-            Iterator<String> i = mapOfAtomicLongValues.keySet().iterator();
-            for (;i.hasNext();) {
-                // Ok. Try doing it the slow way then.
-                String key = i.next();
-                sortedMap.put(key, mapOfAtomicLongValues.get(key));
-            }
+        TempStoredSortedMap<Long,String> sortedMap = 
+            bdb.getStoredMap(
+                    null,
+                    Long.class,
+                    String.class,
+                    true);
+        for(String k : mapOfAtomicLongValues.keySet()) {
+            sortedMap.put(-mapOfAtomicLongValues.get(k).longValue(), k);
         }
         return sortedMap;
     }
@@ -674,25 +656,16 @@ public class StatisticsTracker
      *            Assumes values are wrapped with AtomicLong.
      * @return a sorted set containing the same elements as the map.
      */
-    public TreeMap<String,AtomicLong> getReverseSortedCopy(
+    public TempStoredSortedMap<Long,String> getReverseSortedCopy(
             final ObjectIdentityCache<String,AtomicLong> cacheOfAtomicLongValues) {
-        TreeMap<String,AtomicLong> sortedMap = 
-          new TreeMap<String,AtomicLong>(new Comparator<String>() {
-            public int compare(String e1, String e2) {
-                long firstVal = cacheOfAtomicLongValues.get(e1).get();
-                long secondVal = cacheOfAtomicLongValues.get(e2).get();
-                if (firstVal < secondVal) {
-                    return 1;
-                }
-                if (secondVal < firstVal) {
-                    return -1;
-                }
-                // If the values are the same, sort by keys.
-                return e1.compareTo(e2);
-            }
-        });
-        for(String key : cacheOfAtomicLongValues.keySet()) {
-            sortedMap.put(key, cacheOfAtomicLongValues.get(key));
+        TempStoredSortedMap<Long,String> sortedMap = 
+            bdb.getStoredMap(
+                    null,
+                    Long.class,
+                    String.class,
+                    true);
+        for(String k : cacheOfAtomicLongValues.keySet()) {
+            sortedMap.put(-cacheOfAtomicLongValues.get(k).longValue(), k);
         }
         return sortedMap;
     }
@@ -854,27 +827,15 @@ public class StatisticsTracker
         return processedSeedsRecords.keySet().iterator();
     }
 
-    public Iterator<SeedRecord> getSeedRecordsSortedByStatusCode() {
+    public TempStoredSortedMap<Integer,SeedRecord> getSeedRecordsSortedByStatusCode() {
         Iterator<String> i = getSeedsIterator();
-        TreeSet<SeedRecord> sortedSet = 
-          new TreeSet<SeedRecord>(new Comparator<SeedRecord>() {
-            public int compare(SeedRecord sr1, SeedRecord sr2) {
-                int code1 = sr1.getStatusCode();
-                int code2 = sr2.getStatusCode();
-                if (code1 == code2) {
-                    // If the values are equal, sort by URIs.
-                    return sr1.getUri().compareTo(sr2.getUri());
-                }
-                // mirror and shift the nubmer line so as to
-                // place zero at the beginning, then all negatives 
-                // in order of ascending absolute value, then all 
-                // positives descending
-                code1 = -code1 - Integer.MAX_VALUE;
-                code2 = -code2 - Integer.MAX_VALUE;
-                
-                return new Integer(code1).compareTo(new Integer(code2));
-            }
-        });
+        TempStoredSortedMap<Integer,SeedRecord> sortedMap = 
+            bdb.getStoredMap(
+                    null,
+                    Integer.class,
+                    SeedRecord.class,
+                    true);
+        
         while (i.hasNext()) {
             String seed = i.next();
             SeedRecord sr = (SeedRecord) processedSeedsRecords.get(seed);
@@ -882,9 +843,9 @@ public class StatisticsTracker
                 sr = new SeedRecord(seed,"Seed has not been processed");
                 // no need to retain synthesized record
             }
-            sortedSet.add(sr);
+            sortedMap.put(sr.sortShiftStatusCode(), sr); 
         }
-        return sortedSet.iterator();
+        return sortedMap;
     }
     
     /**
@@ -893,7 +854,7 @@ public class StatisticsTracker
      * 
      * @return SortedMap of hosts distribution
      */
-    public SortedMap<String,AtomicLong> getReverseSortedHostCounts(
+    public TempStoredSortedMap<Long,String> getReverseSortedHostCounts(
             Map<String,AtomicLong> hostCounts) {
         synchronized(hostCounts){
             return getReverseSortedCopy(hostCounts);
@@ -906,7 +867,7 @@ public class StatisticsTracker
      * @return SortedMap of hosts distribution
      * @deprecated Use {@link #calcReverseSortedHostsDistribution()} instead
      */
-    public SortedMap<String,AtomicLong> getReverseSortedHostsDistribution() {
+    public SortedMap<Long,String> getReverseSortedHostsDistribution() {
         return calcReverseSortedHostsDistribution();
     }
     /**
@@ -914,7 +875,7 @@ public class StatisticsTracker
      * (largest first) order. 
      * @return SortedMap of hosts distribution
      */
-    public SortedMap<String,AtomicLong> calcReverseSortedHostsDistribution() {
+    public TempStoredSortedMap<Long,String> calcReverseSortedHostsDistribution() {
         synchronized(hostsDistribution){
             return getReverseSortedCopy(hostsDistribution);
         }