lee/heritrix3
1
0 Fork 0
mirror of https://github.com/internetarchive/heritrix3.git synced 2026-11-15 10:20:47 +00:00
Code Issues Packages Projects Releases Wiki Activity

[HER-769] Support case where millions of seeds.

Browse Source 
* BdbModule.java
    add temp-stored-map service method
    remove unused secondary-db support
* TempStoredSortedMap.java
    StoredSortedMap that can destroy its underlying database after temporary use
* StatisticsTracker.java
    update various sorted-by-decreasing-frequency methods to use temp StoredMaps, with duplicate keys that are negative counts
* (Multiple)Report.java
    update to use new duplicate-keyed frequency maps
This commit is contained in:
gojomo committed 2009-10-03 23:28:11 +00:00
1 parent 97778493e2
commit 3cb6b59b8c
9 files changed
+214 -183
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

commons/src/main/java/org/archive/bdb/BdbModule.java
+46 -64
View File
@@ -52,7 +52,10 @@ import org.archive.util.ObjectIdentityCache;
import org.archive.util.bdbje.EnhancedEnvironment;
import org.springframework.context.Lifecycle;
import com.sleepycat.bind.EntryBinding;
import com.sleepycat.bind.serial.SerialBinding;
import com.sleepycat.bind.serial.StoredClassCatalog;
import com.sleepycat.bind.tuple.TupleBinding;
import com.sleepycat.je.CheckpointConfig;
import com.sleepycat.je.Database;
import com.sleepycat.je.DatabaseConfig;
@@ -60,9 +63,6 @@ import com.sleepycat.je.DatabaseException;
import com.sleepycat.je.DatabaseNotFoundException;
import com.sleepycat.je.DbInternal;
import com.sleepycat.je.EnvironmentConfig;
import com.sleepycat.je.SecondaryConfig;
import com.sleepycat.je.SecondaryDatabase;
import com.sleepycat.je.SecondaryKeyCreator;
import com.sleepycat.je.dbi.EnvironmentImpl;
import com.sleepycat.je.utilint.DbLsn;
@@ -154,35 +154,6 @@ Serializable, Closeable {
}
}
public static class SecondaryBdbConfig extends BdbConfig {
private static final long serialVersionUID = 1L;
private SecondaryKeyCreator keyCreator;
public SecondaryBdbConfig() {
}
public SecondaryKeyCreator getKeyCreator() {
return keyCreator;
}
public void setKeyCreator(SecondaryKeyCreator keyCreator) {
this.keyCreator = keyCreator;
}
public SecondaryConfig toSecondaryConfig() {
SecondaryConfig result = new SecondaryConfig();
result.setDeferredWrite(true);
result.setTransactional(transactional);
result.setAllowCreate(allowCreate);
result.setSortedDuplicates(sortedDuplicates);
result.setKeyCreator(keyCreator);
return result;
}
}
protected ConfigPath dir = new ConfigPath("bdbmodule subdirectory","state");
public ConfigPath getDir() {
return dir;
@@ -340,23 +311,6 @@ Serializable, Closeable {
return dpc.database;
}
public SecondaryDatabase openSecondaryDatabase(String name, Database db,
SecondaryBdbConfig config) throws DatabaseException {
if (databases.containsKey(name)) {
throw new IllegalStateException("Database already exists: " +name);
}
SecondaryDatabase result = bdbEnvironment.openSecondaryDatabase(null,
name, db, config.toSecondaryConfig());
DatabasePlusConfig dpc = new DatabasePlusConfig();
dpc.database = result;
dpc.name = name;
dpc.primaryName = db.getDatabaseName();
dpc.config = config;
databases.put(name, dpc);
return result;
}
public StoredClassCatalog getClassCatalog() {
return classCatalog;
}
@@ -482,18 +436,8 @@ Serializable, Closeable {
// this.classCatalog);
// }
for (DatabasePlusConfig dpc: databases.values()) {
if (!(dpc.config instanceof SecondaryBdbConfig)) {
dpc.database = bdbEnvironment.openDatabase(null,
dpc.name, dpc.config.toDatabaseConfig());
}
}
for (DatabasePlusConfig dpc: databases.values()) {
if (dpc.config instanceof SecondaryBdbConfig) {
SecondaryBdbConfig conf = (SecondaryBdbConfig)dpc.config;
Database primary = databases.get(dpc.primaryName).database;
dpc.database = bdbEnvironment.openSecondaryDatabase(null,
dpc.name, primary, conf.toSecondaryConfig());
}
dpc.database = bdbEnvironment.openDatabase(null,
dpc.name, dpc.config.toDatabaseConfig());
}
} catch (DatabaseException e) {
IOException io = new IOException();
@@ -724,15 +668,12 @@ Serializable, Closeable {
path = recovery.translatePath(path);
FileUtils.copyDirectory(bdbDir, new File(path));
}
}
private static class BdbShutdownHook extends Thread {
final private BdbModule bdb;
public BdbShutdownHook(BdbModule bdb) {
this.bdb = bdb;
}
@@ -740,6 +681,47 @@ Serializable, Closeable {
public void run() {
this.bdb.close2();
}
}
/** uniqueness serial number for temp map databases */
long sn = 0;
/**
* Creates a database-backed TempStoredSortedMap for transient
* reporting requirements. Calling the returned map's destroy()
* method when done discards the associated Database.
*
* @param <K>
* @param <V>
* @param dbName Database name to use; if null a name will be synthesized
* @param keyClass Class of keys; should be a Java primitive type
* @param valueClass Class of values; may be any serializable type
* @param allowDuplicates whether duplicate keys allowed
* @return
*/
public <K,V> TempStoredSortedMap<K, V> getStoredMap(String dbName, Class<K> keyClass, Class<V> valueClass, boolean allowDuplicates) {
BdbConfig config = new BdbConfig();
config.setSortedDuplicates(true);
config.setAllowCreate(true);
Database mapDb;
if(dbName==null) {
dbName = "tempMap-"+System.identityHashCode(this)+"-"+sn;
sn++;
}
try {
mapDb = openDatabase(dbName,config,false);
} catch (DatabaseException e) {
throw new RuntimeException(e);
}
EntryBinding<V> valueBinding = TupleBinding.getPrimitiveBinding(valueClass);
if(valueBinding == null) {
valueBinding = new SerialBinding<V>(classCatalog, valueClass);
}
TempStoredSortedMap<K,V> storedMap = new TempStoredSortedMap<K, V>(
mapDb,
TupleBinding.getPrimitiveBinding(keyClass),
valueBinding,
true);
return storedMap;
}
}
commons/src/main/java/org/archive/bdb/TempStoredSortedMap.java
+76
View File
@@ -0,0 +1,76 @@
/*
* This file is part of the Heritrix web crawler (crawler.archive.org).
*
* Licensed to the Internet Archive (IA) by one or more individual
* contributors.
*
* The IA licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.archive.bdb;
import com.sleepycat.bind.EntityBinding;
import com.sleepycat.bind.EntryBinding;
import com.sleepycat.collections.PrimaryKeyAssigner;
import com.sleepycat.collections.StoredSortedMap;
import com.sleepycat.je.Database;
import com.sleepycat.je.DatabaseException;
/**
* TempStoredSortedMap remembers its backing Database, and offers
* a destroy() method for closing/discarding the underlying Database.
*
* @contributor gojomo
* @param <K>
* @param <V>
*/
public class TempStoredSortedMap<K,V> extends StoredSortedMap<K,V> {
Database db;
String dbName;
public TempStoredSortedMap(Database db, EntryBinding<K> arg1, EntityBinding<V> arg2, boolean arg3) {
super(db, arg1, arg2, arg3);
this.db = db;
}
public TempStoredSortedMap(Database db, EntryBinding<K> arg1, EntityBinding<V> arg2, PrimaryKeyAssigner arg3) {
super(db, arg1, arg2, arg3);
this.db = db;
}
public TempStoredSortedMap(Database db, EntryBinding<K> arg1, EntryBinding<V> arg2, boolean arg3) {
super(db, arg1, arg2, arg3);
this.db = db;
}
public TempStoredSortedMap(Database db, EntryBinding<K> arg1, EntryBinding<V> arg2, PrimaryKeyAssigner arg3) {
super(db, arg1, arg2, arg3);
this.db = db;
}
public void destroy() {
try {
if(this.db!=null) {
String name = this.db.getDatabaseName();
this.db.close();
this.db.getEnvironment().removeDatabase(null, name);
this.db = null;
}
} catch (DatabaseException e) {
throw new RuntimeException(e);
}
}
@Override
protected void finalize() throws Throwable {
super.finalize();
destroy();
}
}