lee/heritrix3
1
0
Fork 0
mirror of https://github.com/internetarchive/heritrix3.git synced 2026-09-25 23:25:44 +00:00
Code Issues Packages Projects Releases Wiki Activity

[HER-769] Support case where millions of seeds.

Browse Source 
* BdbModule.java
    add temp-stored-map service method
    remove unused secondary-db support
* TempStoredSortedMap.java
    StoredSortedMap that can destroy its underlying database after temporary use
* StatisticsTracker.java
    update various sorted-by-decreasing-frequency methods to use temp StoredMaps, with duplicate keys that are negative counts
* (Multiple)Report.java
    update to use new duplicate-keyed frequency maps
This commit is contained in:
gojomo
2009-10-03 23:28:11 +00:00
parent 97778493e2
commit 3cb6b59b8c
9 changed files with 214 additions and 183 deletions
Download Patch File Download Diff File Expand all files Collapse all files
commons/src/main/java/org/archive/bdb/BdbModule.java
+46 -64
View File
@@ -52,7 +52,10 @@ import org.archive.util.ObjectIdentityCache;
import org.archive.util.bdbje.EnhancedEnvironment;
import org.springframework.context.Lifecycle;
import com.sleepycat.bind.EntryBinding;
import com.sleepycat.bind.serial.SerialBinding;
import com.sleepycat.bind.serial.StoredClassCatalog;
import com.sleepycat.bind.tuple.TupleBinding;
import com.sleepycat.je.CheckpointConfig;
import com.sleepycat.je.Database;
import com.sleepycat.je.DatabaseConfig;
@@ -60,9 +63,6 @@ import com.sleepycat.je.DatabaseException;
import com.sleepycat.je.DatabaseNotFoundException;
import com.sleepycat.je.DbInternal;
import com.sleepycat.je.EnvironmentConfig;
import com.sleepycat.je.SecondaryConfig;
import com.sleepycat.je.SecondaryDatabase;
import com.sleepycat.je.SecondaryKeyCreator;
import com.sleepycat.je.dbi.EnvironmentImpl;
import com.sleepycat.je.utilint.DbLsn;
@@ -154,35 +154,6 @@ Serializable, Closeable {
}
}
public static class SecondaryBdbConfig extends BdbConfig {
private static final long serialVersionUID = 1L;
private SecondaryKeyCreator keyCreator;
public SecondaryBdbConfig() {
}
public SecondaryKeyCreator getKeyCreator() {
return keyCreator;
}
public void setKeyCreator(SecondaryKeyCreator keyCreator) {
this.keyCreator = keyCreator;
}
public SecondaryConfig toSecondaryConfig() {
SecondaryConfig result = new SecondaryConfig();
result.setDeferredWrite(true);
result.setTransactional(transactional);
result.setAllowCreate(allowCreate);
result.setSortedDuplicates(sortedDuplicates);
result.setKeyCreator(keyCreator);
return result;
}
}
protected ConfigPath dir = new ConfigPath("bdbmodule subdirectory","state");
public ConfigPath getDir() {
return dir;
@@ -340,23 +311,6 @@ Serializable, Closeable {
return dpc.database;
}
public SecondaryDatabase openSecondaryDatabase(String name, Database db,
SecondaryBdbConfig config) throws DatabaseException {
if (databases.containsKey(name)) {
throw new IllegalStateException("Database already exists: " +name);
}
SecondaryDatabase result = bdbEnvironment.openSecondaryDatabase(null,
name, db, config.toSecondaryConfig());
DatabasePlusConfig dpc = new DatabasePlusConfig();
dpc.database = result;
dpc.name = name;
dpc.primaryName = db.getDatabaseName();
dpc.config = config;
databases.put(name, dpc);
return result;
}
public StoredClassCatalog getClassCatalog() {
return classCatalog;
}
@@ -482,18 +436,8 @@ Serializable, Closeable {
// this.classCatalog);
// }
for (DatabasePlusConfig dpc: databases.values()) {
if (!(dpc.config instanceof SecondaryBdbConfig)) {
dpc.database = bdbEnvironment.openDatabase(null,
dpc.name, dpc.config.toDatabaseConfig());
}
}
for (DatabasePlusConfig dpc: databases.values()) {
if (dpc.config instanceof SecondaryBdbConfig) {
SecondaryBdbConfig conf = (SecondaryBdbConfig)dpc.config;
Database primary = databases.get(dpc.primaryName).database;
dpc.database = bdbEnvironment.openSecondaryDatabase(null,
dpc.name, primary, conf.toSecondaryConfig());
}
dpc.database = bdbEnvironment.openDatabase(null,
dpc.name, dpc.config.toDatabaseConfig());
}
} catch (DatabaseException e) {
IOException io = new IOException();
@@ -724,15 +668,12 @@ Serializable, Closeable {
path = recovery.translatePath(path);
FileUtils.copyDirectory(bdbDir, new File(path));
}
}
private static class BdbShutdownHook extends Thread {
final private BdbModule bdb;
public BdbShutdownHook(BdbModule bdb) {
this.bdb = bdb;
}
@@ -740,6 +681,47 @@ Serializable, Closeable {
public void run() {
this.bdb.close2();
}
}
/** uniqueness serial number for temp map databases */
long sn = 0;
/**
* Creates a database-backed TempStoredSortedMap for transient
* reporting requirements. Calling the returned map's destroy()
* method when done discards the associated Database.
*
* @param <K>
* @param <V>
* @param dbName Database name to use; if null a name will be synthesized
* @param keyClass Class of keys; should be a Java primitive type
* @param valueClass Class of values; may be any serializable type
* @param allowDuplicates whether duplicate keys allowed
* @return
*/
public <K,V> TempStoredSortedMap<K, V> getStoredMap(String dbName, Class<K> keyClass, Class<V> valueClass, boolean allowDuplicates) {
BdbConfig config = new BdbConfig();
config.setSortedDuplicates(true);
config.setAllowCreate(true);
Database mapDb;
if(dbName==null) {
dbName = "tempMap-"+System.identityHashCode(this)+"-"+sn;
sn++;
}
try {
mapDb = openDatabase(dbName,config,false);
} catch (DatabaseException e) {
throw new RuntimeException(e);
}
EntryBinding<V> valueBinding = TupleBinding.getPrimitiveBinding(valueClass);
if(valueBinding == null) {
valueBinding = new SerialBinding<V>(classCatalog, valueClass);
}
TempStoredSortedMap<K,V> storedMap = new TempStoredSortedMap<K, V>(
mapDb,
TupleBinding.getPrimitiveBinding(keyClass),
valueBinding,
true);
return storedMap;
}
}
commons/src/main/java/org/archive/bdb/TempStoredSortedMap.java
+76
View File
@@ -0,0 +1,76 @@
/*
* This file is part of the Heritrix web crawler (crawler.archive.org).
*
* Licensed to the Internet Archive (IA) by one or more individual
* contributors.
*
* The IA licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.archive.bdb;
import com.sleepycat.bind.EntityBinding;
import com.sleepycat.bind.EntryBinding;
import com.sleepycat.collections.PrimaryKeyAssigner;
import com.sleepycat.collections.StoredSortedMap;
import com.sleepycat.je.Database;
import com.sleepycat.je.DatabaseException;
/**
* TempStoredSortedMap remembers its backing Database, and offers
* a destroy() method for closing/discarding the underlying Database.
*
* @contributor gojomo
* @param <K>
* @param <V>
*/
public class TempStoredSortedMap<K,V> extends StoredSortedMap<K,V> {
Database db;
String dbName;
public TempStoredSortedMap(Database db, EntryBinding<K> arg1, EntityBinding<V> arg2, boolean arg3) {
super(db, arg1, arg2, arg3);
this.db = db;
}
public TempStoredSortedMap(Database db, EntryBinding<K> arg1, EntityBinding<V> arg2, PrimaryKeyAssigner arg3) {
super(db, arg1, arg2, arg3);
this.db = db;
}
public TempStoredSortedMap(Database db, EntryBinding<K> arg1, EntryBinding<V> arg2, boolean arg3) {
super(db, arg1, arg2, arg3);
this.db = db;
}
public TempStoredSortedMap(Database db, EntryBinding<K> arg1, EntryBinding<V> arg2, PrimaryKeyAssigner arg3) {
super(db, arg1, arg2, arg3);
this.db = db;
}
public void destroy() {
try {
if(this.db!=null) {
String name = this.db.getDatabaseName();
this.db.close();
this.db.getEnvironment().removeDatabase(null, name);
this.db = null;
}
} catch (DatabaseException e) {
throw new RuntimeException(e);
}
}
@Override
protected void finalize() throws Throwable {
super.finalize();
destroy();
}
}
engine/src/main/java/org/archive/crawler/reporting/HostsReport.java
+11 -12
View File
@@ -20,11 +20,10 @@
package org.archive.crawler.reporting;
import java.io.PrintWriter;
import java.util.Iterator;
import java.util.SortedMap;
import java.util.concurrent.atomic.AtomicLong;
import java.util.Map;
import org.apache.commons.collections.Closure;
import org.archive.bdb.TempStoredSortedMap;
import org.archive.modules.net.CrawlHost;
/**
@@ -38,21 +37,21 @@ public class HostsReport extends Report {
public void write(final PrintWriter writer) {
// TODO: use CrawlHosts for all stats; only perform sorting on
// manageable number of hosts
SortedMap<String,AtomicLong> hd = stats.calcReverseSortedHostsDistribution();
TempStoredSortedMap<Long,String> hd = stats.calcReverseSortedHostsDistribution();
// header
writer.print("[#urls] [#bytes] [host] [#robots] [#remaining]\n");
for (Iterator<String> i = hd.keySet().iterator(); i.hasNext();) {
// Key is 'host'.
String key = (String) i.next();
CrawlHost host = stats.serverCache.getHostFor(key);
AtomicLong val = (AtomicLong)hd.get(key);
for (Map.Entry<Long,String> entry : hd.entrySet()) {
// key is -count, value is hostname
CrawlHost host = stats.serverCache.getHostFor(entry.getValue());
long count = Math.abs(entry.getKey());
writeReportLine(writer,
((val==null)?"-":val.get()),
stats.getBytesPerHost(key),
key,
count,
stats.getBytesPerHost(entry.getValue()),
entry.getValue(),
host.getSubstats().getRobotsDenials(),
host.getSubstats().getRemaining());
}
hd.destroy();
// StatisticsTracker doesn't know of zero-completion hosts;
// so supplement report with those entries from host cache
Closure logZeros = new Closure() {
engine/src/main/java/org/archive/crawler/reporting/MimetypesReport.java
+12 -11
View File
@@ -20,9 +20,9 @@
package org.archive.crawler.reporting;
import java.io.PrintWriter;
import java.util.Iterator;
import java.util.TreeMap;
import java.util.concurrent.atomic.AtomicLong;
import java.util.Map;
import org.archive.bdb.TempStoredSortedMap;
/**
* The "Mimetypes Report", tallies by MIME type.
@@ -35,17 +35,18 @@ public class MimetypesReport extends Report {
public void write(PrintWriter writer) {
// header
writer.print("[#urls] [#bytes] [mime-types]\n");
TreeMap<String,AtomicLong> fd = stats.getReverseSortedCopy(stats.getFileDistribution());
for (Iterator<String> i = fd.keySet().iterator(); i.hasNext();) {
Object key = i.next();
// Key is mime type.
writer.print(Long.toString(((AtomicLong)fd.get(key)).get()));
TempStoredSortedMap<Long,String> fd = stats.getReverseSortedCopy(stats.getFileDistribution());
for (Map.Entry<Long,String> entry : fd.entrySet()) {
// key is -count, value is type
writer.print(Math.abs(entry.getKey()));
writer.print(" ");
writer.print(Long.toString(stats.getBytesPerFileType((String)key)));
writer.print(stats.getBytesPerFileType(entry.getValue()));
writer.print(" ");
writer.print((String)key);
writer.print(entry.getValue());
writer.print("\n");
} }
}
fd.destroy();
}
@Override
public String getFilename() {
engine/src/main/java/org/archive/crawler/reporting/ResponseCodeReport.java
+9 -9
View File
@@ -19,9 +19,9 @@
package org.archive.crawler.reporting;
import java.io.PrintWriter;
import java.util.Iterator;
import java.util.TreeMap;
import java.util.concurrent.atomic.AtomicLong;
import java.util.Map;
import org.archive.bdb.TempStoredSortedMap;
/**
* The "Response Codes Report", tallies by response/disposition code.
@@ -33,17 +33,17 @@ public class ResponseCodeReport extends Report {
@Override
public void write(PrintWriter writer) {
// header
writer.print("[rescode] [#urls]\n");
writer.print("[#urls] [rescode]\n");
TreeMap<String,AtomicLong> scd =
TempStoredSortedMap<Long,String> scd =
stats.getReverseSortedCopy(stats.getStatusCodeDistribution());
for (Iterator<String> i = scd.keySet().iterator(); i.hasNext();) {
Object key = i.next();
writer.print((String)key);
for (Map.Entry<Long,String> entry : scd.entrySet()) {
writer.print(Math.abs(entry.getKey()));
writer.print(" ");
writer.print(Long.toString(((AtomicLong)scd.get(key)).get()));
writer.print(entry.getValue());
writer.print("\n");
}
scd.destroy();
}
@Override
engine/src/main/java/org/archive/crawler/reporting/SeedRecord.java
+4
View File
@@ -130,4 +130,8 @@ public class SeedRecord implements CoreAttributeConstants, Serializable {
public String getUri() {
return uri;
}
public int sortShiftStatusCode() {
return -statusCode - Integer.MAX_VALUE;
}
}
engine/src/main/java/org/archive/crawler/reporting/SeedsReport.java
+13 -3
View File
@@ -20,6 +20,11 @@ package org.archive.crawler.reporting;
import java.io.PrintWriter;
import java.util.Iterator;
import java.util.Map;
import org.archive.bdb.TempStoredSortedMap;
import com.sleepycat.collections.StoredIterator;
/**
@@ -36,9 +41,12 @@ public class SeedsReport extends Report {
long seedsCrawled = 0;
long seedsTotal = 0;
for (Iterator<SeedRecord> i = stats.getSeedRecordsSortedByStatusCode();
i.hasNext();) {
SeedRecord sr = (SeedRecord)i.next();
TempStoredSortedMap<Integer, SeedRecord> seedsByCode = stats.getSeedRecordsSortedByStatusCode();
// for (Map.Entry<Integer,SeedRecord> entry : seedsByCode.entrySet()) {
Iterator<Map.Entry<Integer,SeedRecord>> iter = seedsByCode.entrySet().iterator();
while(iter.hasNext()) {
Map.Entry<Integer,SeedRecord> entry = iter.next();
SeedRecord sr = entry.getValue();
writer.print(sr.getStatusCode());
writer.print(" ");
seedsTotal++;
@@ -56,6 +64,8 @@ public class SeedsReport extends Report {
}
writer.print("\n");
}
StoredIterator.close(iter);
seedsByCode.destroy();
stats.seedsTotal = seedsTotal;
stats.seedsCrawled = seedsCrawled;
}
engine/src/main/java/org/archive/crawler/reporting/SourceTagsReport.java
+8 -10
View File
@@ -19,11 +19,11 @@
package org.archive.crawler.reporting;
import java.io.PrintWriter;
import java.util.Iterator;
import java.util.Map;
import java.util.SortedMap;
import java.util.concurrent.atomic.AtomicLong;
import org.archive.bdb.TempStoredSortedMap;
/**
* The "Source Report", tallies of source tags (usually seeds) by host.
*
@@ -36,24 +36,22 @@ public class SourceTagsReport extends Report {
writer.print("[source] [host] [#urls]\n");
// for each source
for (Iterator<String> i = stats.sourceHostDistribution.keySet().iterator(); i.hasNext();) {
String sourceKey = i.next();
for (String sourceKey : stats.sourceHostDistribution.keySet()) {
Map<String,AtomicLong> hostCounts =
(Map<String,AtomicLong>)stats.sourceHostDistribution.get(sourceKey);
// sort hosts by #urls
SortedMap<String,AtomicLong> sortedHostCounts =
TempStoredSortedMap<Long,String> sortedHostCounts =
stats.getReverseSortedHostCounts(hostCounts);
// for each host
for (Iterator<String> j = sortedHostCounts.keySet().iterator(); j.hasNext();) {
Object hostKey = j.next();
AtomicLong hostCount = (AtomicLong) hostCounts.get(hostKey);
for (Map.Entry<Long, String> entry : sortedHostCounts.entrySet()) {
writer.print(sourceKey.toString());
writer.print(" ");
writer.print(hostKey.toString());
writer.print(entry.getValue());
writer.print(" ");
writer.print(hostCount.get());
writer.print(Math.abs(entry.getKey()));
writer.print("\n");
}
sortedHostCounts.destroy();
}
}
engine/src/main/java/org/archive/crawler/reporting/StatisticsTracker.java
+35 -74
View File
@@ -26,14 +26,11 @@ import java.io.FileWriter;
import java.io.IOException;
import java.io.PrintWriter;
import java.io.Serializable;
import java.util.Comparator;
import java.util.Date;
import java.util.Iterator;
import java.util.LinkedList;
import java.util.Map;
import java.util.SortedMap;
import java.util.TreeMap;
import java.util.TreeSet;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ConcurrentMap;
import java.util.concurrent.Executors;
@@ -44,6 +41,7 @@ import java.util.logging.Level;
import java.util.logging.Logger;
import org.archive.bdb.BdbModule;
import org.archive.bdb.TempStoredSortedMap;
import org.archive.crawler.event.CrawlStateEvent;
import org.archive.crawler.event.CrawlURIDispositionEvent;
import org.archive.crawler.event.StatSnapshotEvent;
@@ -615,12 +613,12 @@ public class StatisticsTracker
}
/**
* Sort the entries of the given HashMap in descending order by their
* Sort the entries of the given Map in descending order by their
* values, which must be longs wrapped with <code>AtomicLong</code>.
* <p>
* Elements are sorted by value from largest to smallest. Equal values are
* sorted in an arbitrary, but consistent manner by their keys. Only items
* with identical value and key are considered equal.
* sorted by their keys. The returned map is a StoredSortedMap, and
* thus may include duplicate keys.
*
* If the passed-in map requires access to be synchronized, the caller
* should ensure this synchronization.
@@ -629,32 +627,16 @@ public class StatisticsTracker
* Assumes values are wrapped with AtomicLong.
* @return a sorted set containing the same elements as the map.
*/
public TreeMap<String,AtomicLong> getReverseSortedCopy(
public TempStoredSortedMap<Long,String> getReverseSortedCopy(
final Map<String,AtomicLong> mapOfAtomicLongValues) {
TreeMap<String,AtomicLong> sortedMap =
new TreeMap<String,AtomicLong>(new Comparator<String>() {
public int compare(String e1, String e2) {
long firstVal = mapOfAtomicLongValues.get(e1).get();
long secondVal = mapOfAtomicLongValues.get(e2).get();
if (firstVal < secondVal) {
return 1;
}
if (secondVal < firstVal) {
return -1;
}
// If the values are the same, sort by keys.
return e1.compareTo(e2);
}
});
try {
sortedMap.putAll(mapOfAtomicLongValues);
} catch (UnsupportedOperationException e) {
Iterator<String> i = mapOfAtomicLongValues.keySet().iterator();
for (;i.hasNext();) {
// Ok. Try doing it the slow way then.
String key = i.next();
sortedMap.put(key, mapOfAtomicLongValues.get(key));
}
TempStoredSortedMap<Long,String> sortedMap =
bdb.getStoredMap(
null,
Long.class,
String.class,
true);
for(String k : mapOfAtomicLongValues.keySet()) {
sortedMap.put(-mapOfAtomicLongValues.get(k).longValue(), k);
}
return sortedMap;
}
@@ -674,25 +656,16 @@ public class StatisticsTracker
* Assumes values are wrapped with AtomicLong.
* @return a sorted set containing the same elements as the map.
*/
public TreeMap<String,AtomicLong> getReverseSortedCopy(
public TempStoredSortedMap<Long,String> getReverseSortedCopy(
final ObjectIdentityCache<String,AtomicLong> cacheOfAtomicLongValues) {
TreeMap<String,AtomicLong> sortedMap =
new TreeMap<String,AtomicLong>(new Comparator<String>() {
public int compare(String e1, String e2) {
long firstVal = cacheOfAtomicLongValues.get(e1).get();
long secondVal = cacheOfAtomicLongValues.get(e2).get();
if (firstVal < secondVal) {
return 1;
}
if (secondVal < firstVal) {
return -1;
}
// If the values are the same, sort by keys.
return e1.compareTo(e2);
}
});
for(String key : cacheOfAtomicLongValues.keySet()) {
sortedMap.put(key, cacheOfAtomicLongValues.get(key));
TempStoredSortedMap<Long,String> sortedMap =
bdb.getStoredMap(
null,
Long.class,
String.class,
true);
for(String k : cacheOfAtomicLongValues.keySet()) {
sortedMap.put(-cacheOfAtomicLongValues.get(k).longValue(), k);
}
return sortedMap;
}
@@ -854,27 +827,15 @@ public class StatisticsTracker
return processedSeedsRecords.keySet().iterator();
}
public Iterator<SeedRecord> getSeedRecordsSortedByStatusCode() {
public TempStoredSortedMap<Integer,SeedRecord> getSeedRecordsSortedByStatusCode() {
Iterator<String> i = getSeedsIterator();
TreeSet<SeedRecord> sortedSet =
new TreeSet<SeedRecord>(new Comparator<SeedRecord>() {
public int compare(SeedRecord sr1, SeedRecord sr2) {
int code1 = sr1.getStatusCode();
int code2 = sr2.getStatusCode();
if (code1 == code2) {
// If the values are equal, sort by URIs.
return sr1.getUri().compareTo(sr2.getUri());
}
// mirror and shift the nubmer line so as to
// place zero at the beginning, then all negatives
// in order of ascending absolute value, then all
// positives descending
code1 = -code1 - Integer.MAX_VALUE;
code2 = -code2 - Integer.MAX_VALUE;
return new Integer(code1).compareTo(new Integer(code2));
}
});
TempStoredSortedMap<Integer,SeedRecord> sortedMap =
bdb.getStoredMap(
null,
Integer.class,
SeedRecord.class,
true);
while (i.hasNext()) {
String seed = i.next();
SeedRecord sr = (SeedRecord) processedSeedsRecords.get(seed);
@@ -882,9 +843,9 @@ public class StatisticsTracker
sr = new SeedRecord(seed,"Seed has not been processed");
// no need to retain synthesized record
}
sortedSet.add(sr);
sortedMap.put(sr.sortShiftStatusCode(), sr);
}
return sortedSet.iterator();
return sortedMap;
}
/**
@@ -893,7 +854,7 @@ public class StatisticsTracker
*
* @return SortedMap of hosts distribution
*/
public SortedMap<String,AtomicLong> getReverseSortedHostCounts(
public TempStoredSortedMap<Long,String> getReverseSortedHostCounts(
Map<String,AtomicLong> hostCounts) {
synchronized(hostCounts){
return getReverseSortedCopy(hostCounts);
@@ -906,7 +867,7 @@ public class StatisticsTracker
* @return SortedMap of hosts distribution
* @deprecated Use {@link #calcReverseSortedHostsDistribution()} instead
*/
public SortedMap<String,AtomicLong> getReverseSortedHostsDistribution() {
public SortedMap<Long,String> getReverseSortedHostsDistribution() {
return calcReverseSortedHostsDistribution();
}
/**
@@ -914,7 +875,7 @@ public class StatisticsTracker
* (largest first) order.
* @return SortedMap of hosts distribution
*/
public SortedMap<String,AtomicLong> calcReverseSortedHostsDistribution() {
public TempStoredSortedMap<Long,String> calcReverseSortedHostsDistribution() {
synchronized(hostsDistribution){
return getReverseSortedCopy(hostsDistribution);
}