lee/heritrix3
1
0
Fork 0
mirror of https://github.com/internetarchive/heritrix3.git synced 2026-09-20 20:55:49 +00:00
Code Issues Packages Projects Releases Wiki Activity

[HER-1827] Have Frontier report include top X longest queues

Browse Source 
* Histotable.java
    expose static method useful for frequency-reporting
* TopNSet.java, TopNSet.java
    improve handling of case where updates may decrease frequency counts
    add convenience methods for common needs
* WorkQueueFrontier.java
    replace single longestActiveQueue with largestQueues TopNSet, of configurable size
    make max-queues-to-report-per-category configurable
This commit is contained in:
gojomo
2010-09-28 22:12:28 +00:00
parent 74d3fb54ce
commit 8f28d493a2
4 changed files with 213 additions and 70 deletions
Download Patch File Download Diff File Expand all files Collapse all files
commons/src/main/java/org/archive/util/Histotable.java
+32 -13
View File
@@ -32,7 +32,12 @@ import java.util.TreeSet;
*
* Assumes external synchronization.
*
* @author gojomo
* TODO: Histotable and TopNSet that they could possibly
* have a closer relationship and share some code (even though Histotable
* tracks 'all' keys, and handles increments, while TopNSet only remembers
* a small subset of keys, and requires a fresh full value on each update.)
*
* @contributor gojomo
*/
public class Histotable<K> extends TreeMap<K,Long> {
private static final long serialVersionUID = 310306238032568623L;
@@ -62,32 +67,46 @@ public class Histotable<K> extends TreeMap<K,Long> {
}
/**
* @return Return an up-to-date sorted version of the totalled info.
* Get a SortedSet that, when filled with (String key)->(long count)
* Entry instances, sorts them by (count, key) descending, as is useful
* for most-frequent displays.
*
* Static to allow reuse elsewhere (TopNSet) until a better home for
* this utility method is found.
*
* @return TreeSet with suitable Comparator
*/
public TreeSet<Map.Entry<K,Long>> getSortedByCounts() {
public static TreeSet<Map.Entry<?,Long>> getEntryByFrequencySortedSet() {
// sorted by count
TreeSet<Map.Entry<K,Long>> sorted =
new TreeSet<Map.Entry<K,Long>>(
new Comparator<Map.Entry<K,Long>>() {
public int compare(Map.Entry<K,Long> e1,
Map.Entry<K,Long> e2) {
TreeSet<Map.Entry<?,Long>> sorted =
new TreeSet<Map.Entry<?,Long>>(
new Comparator<Map.Entry<?,Long>>() {
public int compare(Map.Entry<?,Long> e1,
Map.Entry<?,Long> e2) {
long firstVal = e1.getValue();
long secondVal = e2.getValue();
if (firstVal < secondVal) { return 1; }
if (secondVal < firstVal) { return -1; }
// If the values are the same, sort by keys.
String firstKey = ((Map.Entry<K,Long>) e1).getKey().toString();
String secondKey = ((Map.Entry<K,Long>) e2).getKey().toString();
String firstKey = ((Map.Entry<?,Long>) e1).getKey().toString();
String secondKey = ((Map.Entry<?,Long>) e2).getKey().toString();
return firstKey.compareTo(secondKey);
}
});
sorted.addAll(entrySet());
return sorted;
}
/**
* @return Return an up-to-date sorted version of the totalled info.
* @return Return an up-to-date count-descending sorted version of the totaled info.
*/
public TreeSet<Map.Entry<?,Long>> getSortedByCounts() {
TreeSet<java.util.Map.Entry<?, Long>> sorted = getEntryByFrequencySortedSet();
sorted.addAll(entrySet());
return sorted;
}
/**
* @return Return an up-to-date sorted version of the totaled info.
*/
public Set<Map.Entry<K,Long>> getSortedByKeys() {
return entrySet();
engine/src/main/java/org/archive/crawler/frontier/WorkQueueFrontier.java
+36 -25
View File
@@ -56,6 +56,7 @@ import org.archive.crawler.event.CrawlURIDispositionEvent;
import org.archive.crawler.framework.ToeThread;
import org.archive.crawler.frontier.precedence.BaseQueuePrecedencePolicy;
import org.archive.crawler.frontier.precedence.QueuePrecedencePolicy;
import org.archive.crawler.util.TopNSet;
import org.archive.modules.CrawlURI;
import org.archive.spring.KeyedProperties;
import org.archive.util.ArchiveUtils;
@@ -85,9 +86,6 @@ implements Closeable,
ApplicationContextAware {
private static final long serialVersionUID = 570384305871965843L;
/** truncate reporting of queues at some large but not unbounded number */
private static final int REPORT_MAX_QUEUES = 2000;
/**
* If we know that only a small amount of queues is held in memory,
* we can avoid using a disk-based BigMap.
@@ -189,7 +187,14 @@ implements Closeable,
this.precedenceFloor = floor;
}
/** truncate reporting of queues at this large but not unbounded number */
protected int maxQueuesPerReportCategory = 2000;
public int getMaxQueuesPerReportCategory() {
return this.maxQueuesPerReportCategory;
}
public void setMaxQueuesPerReportCategory(int max) {
this.maxQueuesPerReportCategory = max;
}
/** All known queues.
*/
@@ -217,8 +222,19 @@ implements Closeable,
/** URIs scheduled to be re-enqueued at future date */
protected StoredSortedMap<Long, CrawlURI> futureUris;
transient protected WorkQueue longestActiveQueue = null;
/** remember keys of small number of largest queues for reporting */
transient protected TopNSet largestQueues = new TopNSet(20);
/** remember this many largest queues for reporting's sake; actual tracking
* can be somewhat approximate when some queues shrink before others'
* sizes are again noted, or if the size is adjusted mid-crawl. */
public int getLargestQueuesCount() {
return largestQueues.getMaxSize();
}
public void setLargestQueuesCount(int count) {
largestQueues.setMaxSize(count);
}
protected int highestPrecedenceWaiting = Integer.MAX_VALUE;
/** The UriUniqFilter to use, tracking those UURIs which are
@@ -436,11 +452,7 @@ implements Closeable,
readyQueue(wq);
}
}
WorkQueue laq = longestActiveQueue;
if(((laq==null) || wq.getCount() > laq.getCount())) {
longestActiveQueue = wq;
}
largestQueues.update(wq.getClassKey(), wq.getCount());
}
/**
@@ -1441,39 +1453,36 @@ implements Closeable,
w.print("\n -----===== MANAGER THREAD =====-----\n");
ToeThread.reportThread(managerThread, w);
WorkQueue longest = longestActiveQueue;
if (longest != null) {
w.print("\n -----===== LONGEST QUEUE =====-----\n");
longest.reportTo(w);
}
w.print("\n -----===== "+largestQueues.size()+" LONGEST QUEUES =====-----\n");
appendQueueReports(w, "LONGEST", largestQueues.getEntriesDescending().iterator(), largestQueues.size(), largestQueues.size());
w.print("\n -----===== IN-PROCESS QUEUES =====-----\n");
@SuppressWarnings("unchecked")
Collection<WorkQueue> inProcess = inProcessQueues;
ArrayList<WorkQueue> copy = extractSome(inProcess, REPORT_MAX_QUEUES);
appendQueueReports(w, "IN-PROCESS", copy.iterator(), copy.size(), REPORT_MAX_QUEUES);
ArrayList<WorkQueue> copy = extractSome(inProcess, maxQueuesPerReportCategory);
appendQueueReports(w, "IN-PROCESS", copy.iterator(), copy.size(), maxQueuesPerReportCategory);
w.print("\n -----===== READY QUEUES =====-----\n");
appendQueueReports(w, "READY", this.readyClassQueues.iterator(),
this.readyClassQueues.size(), REPORT_MAX_QUEUES);
this.readyClassQueues.size(), maxQueuesPerReportCategory);
w.print("\n -----===== SNOOZED QUEUES =====-----\n");
Object[] objs = snoozedClassQueues.toArray();
DelayedWorkQueue[] qs = Arrays.copyOf(objs,objs.length,DelayedWorkQueue[].class);
Arrays.sort(qs);
appendQueueReports(w, "SNOOZED", new ObjectArrayIterator(qs), getSnoozedCount(), REPORT_MAX_QUEUES);
appendQueueReports(w, "SNOOZED", new ObjectArrayIterator(qs), getSnoozedCount(), maxQueuesPerReportCategory);
w.print("\n -----===== INACTIVE QUEUES =====-----\n");
SortedMap<Integer,Queue<String>> sortedInactives = getInactiveQueuesByPrecedence();
for(Integer prec : sortedInactives.keySet()) {
Queue<String> inactiveQueues = sortedInactives.get(prec);
appendQueueReports(w, "INACTIVE-p"+prec, inactiveQueues.iterator(),
inactiveQueues.size(), REPORT_MAX_QUEUES);
inactiveQueues.size(), maxQueuesPerReportCategory);
}
w.print("\n -----===== RETIRED QUEUES =====-----\n");
appendQueueReports(w, "RETIRED", getRetiredQueues().iterator(),
getRetiredQueues().size(), REPORT_MAX_QUEUES);
getRetiredQueues().size(), maxQueuesPerReportCategory);
w.flush();
}
@@ -1513,6 +1522,7 @@ implements Closeable,
* @param total
* @param max
*/
@SuppressWarnings("unchecked")
protected void appendQueueReports(PrintWriter w, String label, Iterator<?> iterator,
int total, int max) {
Object obj;
@@ -1527,6 +1537,8 @@ implements Closeable,
q = (WorkQueue)obj;
} else if (obj instanceof DelayedWorkQueue) {
q = (WorkQueue)((DelayedWorkQueue)obj).getWorkQueue(this);
} else if (obj instanceof Map.Entry) {
q = this.allQueues.get((String)((Map.Entry)obj).getKey());
} else {
q = this.allQueues.get((String)obj);
}
@@ -1616,7 +1628,7 @@ implements Closeable,
return (float)(activeCount + eligibleInactiveCount) / (inProcessCount + snoozedCount);
}
public long deepestUri() {
return longestActiveQueue==null ? -1 : longestActiveQueue.getCount();
return largestQueues.getTopSet().size()==0 ? -1 : largestQueues.getTopSet().get(largestQueues.getLargest());
}
/**
@@ -1640,5 +1652,4 @@ implements Closeable,
return inProcessQueues.size();
}
} // TODO: slim class! Suspect it should be < 800 lines, shedding budgeting/reporting
} // TODO: slim class! Suspect it should be < 800 lines, shedding budgeting/reporting
engine/src/main/java/org/archive/crawler/util/TopNSet.java
+75 -32
View File
@@ -21,12 +21,24 @@ package org.archive.crawler.util;
import java.io.Serializable;
import java.util.HashMap;
import java.util.SortedSet;
import java.util.TreeSet;
import java.util.Map.Entry;
import org.archive.util.Histotable;
/**
* Counting Set which only remembers the 'top N' of all String values
* reported (with counts) to it. Assumes counts reported for a
* certain String key only ever increase.
*
* reported (with counts) to it. Precise if counts reported for a
* certain String key only ever increase. Otherwise, the current top N
* might decrement to be smaller than other tallies, which won't be noticed
* until those other, previously ignored tallies, are re-reported.
*
* TODO: Histotable and TopNSet that they could possibly
* have a closer relationship and share some code (even though Histotable
* tracks 'all' keys, and handles increments, while TopNSet only remembers
* a small subset of keys, and requires a fresh full value on each update.)
*
* @contributor gojomo
*/
public class TopNSet implements Serializable {
@@ -37,53 +49,62 @@ public class TopNSet implements Serializable {
HashMap<String, Long> set;
long smallestKnownValue;
String smallestKnownKey;
long largestKnownValue;
String largestKnownKey;
public TopNSet(int size){
maxsize = size;
set = new HashMap<String, Long>(size);
}
/**
* Update the given String key with a new total value, perhaps displacing
* an existing top-valued entry, and updating the fields recording max/min
* keys in any case.
*
* @param key String key to update
* @param value long new total value (*not* increment/decrement)
*/
public synchronized void update(String key, long value){
if(set.containsKey(key)) {
// Update the value of an existing key
if(set.containsKey(key) || set.size() < maxsize || value > smallestKnownValue) {
set.put(key,value);
// This may promote the key if it was the smallest
if(smallestKnownKey == null || smallestKnownKey.equals(key)){
updateSmallest();
}
} else if(set.size()<maxsize) {
// Can add a new key/value pair as we still have space
set.put(key, value);
// Check if this is new smallest known value
if(value<smallestKnownValue){
smallestKnownValue = value;
smallestKnownKey = key;
}
} else {
// Determine if value is large enough for inclusion
if(value>smallestKnownValue){
// Replace current smallest
if(set.size() > maxsize) {
set.remove(smallestKnownKey);
updateSmallest();
set.put(key, value);
} // Else do nothing.
updateBounds();
} else if(value < smallestKnownValue
|| value > largestKnownValue
|| key.equals(smallestKnownKey)
|| key.equals(largestKnownKey)) {
updateBounds();
}
}
}
private void updateSmallest(){
// Need to scan through for new smallest value.
long oldSmallest = smallestKnownValue;
public String getLargest() {
return largestKnownKey;
}
public String getSmallest() {
return smallestKnownKey;
}
/**
* After an operation invalidating the previous largest/smallest entry,
* find the new largest/smallest.
*/
protected void updateBounds(){
// freshly determine
smallestKnownValue = Long.MAX_VALUE;
largestKnownValue = Long.MIN_VALUE;
for(String k : set.keySet()){
long v = set.get(k);
if(v<smallestKnownValue){
smallestKnownValue = v;
smallestKnownKey = k;
if(v==oldSmallest){
// Found another key matching old smallest known value
// Can not be anything smaller.
return;
}
}
if(v>largestKnownValue){
largestKnownValue = v;
largestKnownKey = k;
}
}
}
@@ -95,4 +116,26 @@ public class TopNSet implements Serializable {
public HashMap<String,Long> getTopSet() {
return set;
}
public int size() {
return set.size();
}
/**
* Get descending ordered list of key,count Entries.
*
* @return SortedSet of Entry<key, count> descending-frequency
*/
public SortedSet<Entry<?, Long>> getEntriesDescending() {
TreeSet<Entry<?, Long>> sorted = Histotable.getEntryByFrequencySortedSet();
sorted.addAll(getTopSet().entrySet());
return sorted;
}
public int getMaxSize() {
return maxsize;
}
public void setMaxSize(int max) {
maxsize = max;
}
}
engine/src/test/java/org/archive/crawler/util/TopNSetTest.java
+70
View File
@@ -0,0 +1,70 @@
/*
* This file is part of the Heritrix web crawler (crawler.archive.org).
*
* Licensed to the Internet Archive (IA) by one or more individual
* contributors.
*
* The IA licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.archive.crawler.util;
import junit.framework.TestCase;
import org.apache.commons.httpclient.URIException;
/**
* Test TopNSetTest.
* @contributor gojomo
*/
public class TopNSetTest extends TestCase{
public void testOne() throws URIException {
TopNSet tops = new TopNSet(20);
tops.update("foo",999);
assertEquals("wrong-sized set",1, tops.getTopSet().size());
assertEquals("bad largest","foo",tops.getLargest());
assertEquals("bad smallest","foo",tops.getSmallest());
}
public void testTwo() throws URIException {
TopNSet tops = new TopNSet(20);
tops.update("foo",999);
tops.update("bar",101);
assertEquals("wrong-sized set",2,tops.getTopSet().size());
assertEquals("bad largest","foo",tops.getLargest());
assertEquals("bad smallest","bar",tops.getSmallest());
}
public void testInversion() throws URIException {
TopNSet tops = new TopNSet(20);
tops.update("foo",999);
tops.update("bar",101);
tops.update("foo",9);
assertEquals("wrong-sized set",2,tops.getTopSet().size());
assertEquals("bad largest","bar",tops.getLargest());
assertEquals("bad smallest","foo",tops.getSmallest());
}
public void testOverflowUp() throws URIException {
TopNSet tops = new TopNSet(20);
for(int i = 0; i <= 100; i++) {
tops.update(Integer.toString(i),i);
}
assertEquals("wrong-sized set",20,tops.getTopSet().size());
assertEquals("bad largest","100",tops.getLargest());
assertEquals("bad smallest","81",tops.getSmallest());
}
}