diff --git a/commons/src/main/java/org/archive/util/Histotable.java b/commons/src/main/java/org/archive/util/Histotable.java index bbcae681..47ad437b 100644 --- a/commons/src/main/java/org/archive/util/Histotable.java +++ b/commons/src/main/java/org/archive/util/Histotable.java @@ -32,7 +32,12 @@ import java.util.TreeSet; * * Assumes external synchronization. * - * @author gojomo + * TODO: Histotable and TopNSet that they could possibly + * have a closer relationship and share some code (even though Histotable + * tracks 'all' keys, and handles increments, while TopNSet only remembers + * a small subset of keys, and requires a fresh full value on each update.) + * + * @contributor gojomo */ public class Histotable extends TreeMap { private static final long serialVersionUID = 310306238032568623L; @@ -62,32 +67,46 @@ public class Histotable extends TreeMap { } /** - * @return Return an up-to-date sorted version of the totalled info. + * Get a SortedSet that, when filled with (String key)->(long count) + * Entry instances, sorts them by (count, key) descending, as is useful + * for most-frequent displays. + * + * Static to allow reuse elsewhere (TopNSet) until a better home for + * this utility method is found. + * + * @return TreeSet with suitable Comparator */ - public TreeSet> getSortedByCounts() { + public static TreeSet> getEntryByFrequencySortedSet() { // sorted by count - TreeSet> sorted = - new TreeSet>( - new Comparator>() { - public int compare(Map.Entry e1, - Map.Entry e2) { + TreeSet> sorted = + new TreeSet>( + new Comparator>() { + public int compare(Map.Entry e1, + Map.Entry e2) { long firstVal = e1.getValue(); long secondVal = e2.getValue(); if (firstVal < secondVal) { return 1; } if (secondVal < firstVal) { return -1; } // If the values are the same, sort by keys. - String firstKey = ((Map.Entry) e1).getKey().toString(); - String secondKey = ((Map.Entry) e2).getKey().toString(); + String firstKey = ((Map.Entry) e1).getKey().toString(); + String secondKey = ((Map.Entry) e2).getKey().toString(); return firstKey.compareTo(secondKey); } }); - - sorted.addAll(entrySet()); return sorted; } /** - * @return Return an up-to-date sorted version of the totalled info. + * @return Return an up-to-date count-descending sorted version of the totaled info. + */ + public TreeSet> getSortedByCounts() { + TreeSet> sorted = getEntryByFrequencySortedSet(); + sorted.addAll(entrySet()); + return sorted; + } + + /** + * @return Return an up-to-date sorted version of the totaled info. */ public Set> getSortedByKeys() { return entrySet(); diff --git a/engine/src/main/java/org/archive/crawler/frontier/WorkQueueFrontier.java b/engine/src/main/java/org/archive/crawler/frontier/WorkQueueFrontier.java index 3b260175..4ec62bbc 100644 --- a/engine/src/main/java/org/archive/crawler/frontier/WorkQueueFrontier.java +++ b/engine/src/main/java/org/archive/crawler/frontier/WorkQueueFrontier.java @@ -56,6 +56,7 @@ import org.archive.crawler.event.CrawlURIDispositionEvent; import org.archive.crawler.framework.ToeThread; import org.archive.crawler.frontier.precedence.BaseQueuePrecedencePolicy; import org.archive.crawler.frontier.precedence.QueuePrecedencePolicy; +import org.archive.crawler.util.TopNSet; import org.archive.modules.CrawlURI; import org.archive.spring.KeyedProperties; import org.archive.util.ArchiveUtils; @@ -85,9 +86,6 @@ implements Closeable, ApplicationContextAware { private static final long serialVersionUID = 570384305871965843L; - /** truncate reporting of queues at some large but not unbounded number */ - private static final int REPORT_MAX_QUEUES = 2000; - /** * If we know that only a small amount of queues is held in memory, * we can avoid using a disk-based BigMap. @@ -189,7 +187,14 @@ implements Closeable, this.precedenceFloor = floor; } - + /** truncate reporting of queues at this large but not unbounded number */ + protected int maxQueuesPerReportCategory = 2000; + public int getMaxQueuesPerReportCategory() { + return this.maxQueuesPerReportCategory; + } + public void setMaxQueuesPerReportCategory(int max) { + this.maxQueuesPerReportCategory = max; + } /** All known queues. */ @@ -217,8 +222,19 @@ implements Closeable, /** URIs scheduled to be re-enqueued at future date */ protected StoredSortedMap futureUris; - transient protected WorkQueue longestActiveQueue = null; - + /** remember keys of small number of largest queues for reporting */ + transient protected TopNSet largestQueues = new TopNSet(20); + /** remember this many largest queues for reporting's sake; actual tracking + * can be somewhat approximate when some queues shrink before others' + * sizes are again noted, or if the size is adjusted mid-crawl. */ + public int getLargestQueuesCount() { + return largestQueues.getMaxSize(); + } + public void setLargestQueuesCount(int count) { + largestQueues.setMaxSize(count); + } + + protected int highestPrecedenceWaiting = Integer.MAX_VALUE; /** The UriUniqFilter to use, tracking those UURIs which are @@ -436,11 +452,7 @@ implements Closeable, readyQueue(wq); } } - WorkQueue laq = longestActiveQueue; - if(((laq==null) || wq.getCount() > laq.getCount())) { - longestActiveQueue = wq; - } - + largestQueues.update(wq.getClassKey(), wq.getCount()); } /** @@ -1441,39 +1453,36 @@ implements Closeable, w.print("\n -----===== MANAGER THREAD =====-----\n"); ToeThread.reportThread(managerThread, w); - WorkQueue longest = longestActiveQueue; - if (longest != null) { - w.print("\n -----===== LONGEST QUEUE =====-----\n"); - longest.reportTo(w); - } + w.print("\n -----===== "+largestQueues.size()+" LONGEST QUEUES =====-----\n"); + appendQueueReports(w, "LONGEST", largestQueues.getEntriesDescending().iterator(), largestQueues.size(), largestQueues.size()); w.print("\n -----===== IN-PROCESS QUEUES =====-----\n"); @SuppressWarnings("unchecked") Collection inProcess = inProcessQueues; - ArrayList copy = extractSome(inProcess, REPORT_MAX_QUEUES); - appendQueueReports(w, "IN-PROCESS", copy.iterator(), copy.size(), REPORT_MAX_QUEUES); + ArrayList copy = extractSome(inProcess, maxQueuesPerReportCategory); + appendQueueReports(w, "IN-PROCESS", copy.iterator(), copy.size(), maxQueuesPerReportCategory); w.print("\n -----===== READY QUEUES =====-----\n"); appendQueueReports(w, "READY", this.readyClassQueues.iterator(), - this.readyClassQueues.size(), REPORT_MAX_QUEUES); + this.readyClassQueues.size(), maxQueuesPerReportCategory); w.print("\n -----===== SNOOZED QUEUES =====-----\n"); Object[] objs = snoozedClassQueues.toArray(); DelayedWorkQueue[] qs = Arrays.copyOf(objs,objs.length,DelayedWorkQueue[].class); Arrays.sort(qs); - appendQueueReports(w, "SNOOZED", new ObjectArrayIterator(qs), getSnoozedCount(), REPORT_MAX_QUEUES); + appendQueueReports(w, "SNOOZED", new ObjectArrayIterator(qs), getSnoozedCount(), maxQueuesPerReportCategory); w.print("\n -----===== INACTIVE QUEUES =====-----\n"); SortedMap> sortedInactives = getInactiveQueuesByPrecedence(); for(Integer prec : sortedInactives.keySet()) { Queue inactiveQueues = sortedInactives.get(prec); appendQueueReports(w, "INACTIVE-p"+prec, inactiveQueues.iterator(), - inactiveQueues.size(), REPORT_MAX_QUEUES); + inactiveQueues.size(), maxQueuesPerReportCategory); } w.print("\n -----===== RETIRED QUEUES =====-----\n"); appendQueueReports(w, "RETIRED", getRetiredQueues().iterator(), - getRetiredQueues().size(), REPORT_MAX_QUEUES); + getRetiredQueues().size(), maxQueuesPerReportCategory); w.flush(); } @@ -1513,6 +1522,7 @@ implements Closeable, * @param total * @param max */ + @SuppressWarnings("unchecked") protected void appendQueueReports(PrintWriter w, String label, Iterator iterator, int total, int max) { Object obj; @@ -1527,6 +1537,8 @@ implements Closeable, q = (WorkQueue)obj; } else if (obj instanceof DelayedWorkQueue) { q = (WorkQueue)((DelayedWorkQueue)obj).getWorkQueue(this); + } else if (obj instanceof Map.Entry) { + q = this.allQueues.get((String)((Map.Entry)obj).getKey()); } else { q = this.allQueues.get((String)obj); } @@ -1616,7 +1628,7 @@ implements Closeable, return (float)(activeCount + eligibleInactiveCount) / (inProcessCount + snoozedCount); } public long deepestUri() { - return longestActiveQueue==null ? -1 : longestActiveQueue.getCount(); + return largestQueues.getTopSet().size()==0 ? -1 : largestQueues.getTopSet().get(largestQueues.getLargest()); } /** @@ -1640,5 +1652,4 @@ implements Closeable, return inProcessQueues.size(); } -} // TODO: slim class! Suspect it should be < 800 lines, shedding budgeting/reporting - +} // TODO: slim class! Suspect it should be < 800 lines, shedding budgeting/reporting \ No newline at end of file diff --git a/engine/src/main/java/org/archive/crawler/util/TopNSet.java b/engine/src/main/java/org/archive/crawler/util/TopNSet.java index f9e1c4bd..d1e85792 100644 --- a/engine/src/main/java/org/archive/crawler/util/TopNSet.java +++ b/engine/src/main/java/org/archive/crawler/util/TopNSet.java @@ -21,12 +21,24 @@ package org.archive.crawler.util; import java.io.Serializable; import java.util.HashMap; +import java.util.SortedSet; +import java.util.TreeSet; +import java.util.Map.Entry; + +import org.archive.util.Histotable; /** * Counting Set which only remembers the 'top N' of all String values - * reported (with counts) to it. Assumes counts reported for a - * certain String key only ever increase. - * + * reported (with counts) to it. Precise if counts reported for a + * certain String key only ever increase. Otherwise, the current top N + * might decrement to be smaller than other tallies, which won't be noticed + * until those other, previously ignored tallies, are re-reported. + * + * TODO: Histotable and TopNSet that they could possibly + * have a closer relationship and share some code (even though Histotable + * tracks 'all' keys, and handles increments, while TopNSet only remembers + * a small subset of keys, and requires a fresh full value on each update.) + * * @contributor gojomo */ public class TopNSet implements Serializable { @@ -37,53 +49,62 @@ public class TopNSet implements Serializable { HashMap set; long smallestKnownValue; String smallestKnownKey; + long largestKnownValue; + String largestKnownKey; public TopNSet(int size){ maxsize = size; set = new HashMap(size); } + /** + * Update the given String key with a new total value, perhaps displacing + * an existing top-valued entry, and updating the fields recording max/min + * keys in any case. + * + * @param key String key to update + * @param value long new total value (*not* increment/decrement) + */ public synchronized void update(String key, long value){ - if(set.containsKey(key)) { - // Update the value of an existing key + if(set.containsKey(key) || set.size() < maxsize || value > smallestKnownValue) { set.put(key,value); - // This may promote the key if it was the smallest - if(smallestKnownKey == null || smallestKnownKey.equals(key)){ - updateSmallest(); - } - } else if(set.size()smallestKnownValue){ - // Replace current smallest + if(set.size() > maxsize) { set.remove(smallestKnownKey); - updateSmallest(); - set.put(key, value); - } // Else do nothing. + updateBounds(); + } else if(value < smallestKnownValue + || value > largestKnownValue + || key.equals(smallestKnownKey) + || key.equals(largestKnownKey)) { + updateBounds(); + } } } - private void updateSmallest(){ - // Need to scan through for new smallest value. - long oldSmallest = smallestKnownValue; + public String getLargest() { + return largestKnownKey; + } + + public String getSmallest() { + return smallestKnownKey; + } + + /** + * After an operation invalidating the previous largest/smallest entry, + * find the new largest/smallest. + */ + protected void updateBounds(){ + // freshly determine smallestKnownValue = Long.MAX_VALUE; + largestKnownValue = Long.MIN_VALUE; for(String k : set.keySet()){ long v = set.get(k); if(vlargestKnownValue){ + largestKnownValue = v; + largestKnownKey = k; } } } @@ -95,4 +116,26 @@ public class TopNSet implements Serializable { public HashMap getTopSet() { return set; } + + public int size() { + return set.size(); + } + + /** + * Get descending ordered list of key,count Entries. + * + * @return SortedSet of Entry descending-frequency + */ + public SortedSet> getEntriesDescending() { + TreeSet> sorted = Histotable.getEntryByFrequencySortedSet(); + sorted.addAll(getTopSet().entrySet()); + return sorted; + } + + public int getMaxSize() { + return maxsize; + } + public void setMaxSize(int max) { + maxsize = max; + } } diff --git a/engine/src/test/java/org/archive/crawler/util/TopNSetTest.java b/engine/src/test/java/org/archive/crawler/util/TopNSetTest.java new file mode 100644 index 00000000..4e2d697b --- /dev/null +++ b/engine/src/test/java/org/archive/crawler/util/TopNSetTest.java @@ -0,0 +1,70 @@ +/* + * This file is part of the Heritrix web crawler (crawler.archive.org). + * + * Licensed to the Internet Archive (IA) by one or more individual + * contributors. + * + * The IA licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.archive.crawler.util; + +import junit.framework.TestCase; + +import org.apache.commons.httpclient.URIException; + +/** + * Test TopNSetTest. + + * @contributor gojomo + */ +public class TopNSetTest extends TestCase{ + + public void testOne() throws URIException { + TopNSet tops = new TopNSet(20); + tops.update("foo",999); + assertEquals("wrong-sized set",1, tops.getTopSet().size()); + assertEquals("bad largest","foo",tops.getLargest()); + assertEquals("bad smallest","foo",tops.getSmallest()); + } + + + public void testTwo() throws URIException { + TopNSet tops = new TopNSet(20); + tops.update("foo",999); + tops.update("bar",101); + assertEquals("wrong-sized set",2,tops.getTopSet().size()); + assertEquals("bad largest","foo",tops.getLargest()); + assertEquals("bad smallest","bar",tops.getSmallest()); + } + + public void testInversion() throws URIException { + TopNSet tops = new TopNSet(20); + tops.update("foo",999); + tops.update("bar",101); + tops.update("foo",9); + assertEquals("wrong-sized set",2,tops.getTopSet().size()); + assertEquals("bad largest","bar",tops.getLargest()); + assertEquals("bad smallest","foo",tops.getSmallest()); + } + + public void testOverflowUp() throws URIException { + TopNSet tops = new TopNSet(20); + for(int i = 0; i <= 100; i++) { + tops.update(Integer.toString(i),i); + } + assertEquals("wrong-sized set",20,tops.getTopSet().size()); + assertEquals("bad largest","100",tops.getLargest()); + assertEquals("bad smallest","81",tops.getSmallest()); + } +}