lee/heritrix3
1
0
Fork 0
mirror of https://github.com/internetarchive/heritrix3.git synced 2026-09-26 07:35:39 +00:00
Code Issues Packages Projects Releases Wiki Activity

Minimize transient garbage volume by optimized serialization based on the 'Kryo' library

Browse Source 
* AutoKryo
    extension of Kryo to allow classes to control their own registration, trigger registration of associated classes, and deserialize classes without no-arg constructors
* KryoBinding
    binding for use with BDB that uses AutoKryo serialization for a 2X-4X reduction in byte[] size
* UURI
    improved serialization via Externalizable and Kryo's CustomSerialization methods
* BdbModule
    discard deprecated CachedBDBMap option
    (getObjectCache) extend with both declaredClass and valueClass (for when map values are specializations of the declared type, as with frontier.allQueues)
    adjust type declarations
* ObjectIdentityBdbCache
    use KryoBinding rather than SerialBinding
* CachedBdbMapTest
    discarded
* BdbFrontier, BdbServerCache, StatisticsTracker
    adjust type declarations, objectCache creation
* BdbMultipleWorkQueues
    use KryoBinding rather than (Recycling)SerialBinding
* BdbWorkQueue, CrawlServer, CrawlHost, CrawlURI
    add autoregister support
* LinkContext
    public for kryo registration
This commit is contained in:
gojomo
2010-11-05 23:23:58 +00:00
parent f4a01f1155
commit 0627a67dce
16 changed files with 360 additions and 275 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.classpath
+4
View File
@@ -51,5 +51,9 @@
<classpathentry kind="lib" path="commons/src/main/resources"/>
<classpathentry kind="lib" path="commons/src/test/resources"/>
<classpathentry kind="var" path="M2_REPO/org/json/json/20090211/json-20090211.jar" sourcepath="/M2SRC/json.zip"/>
<classpathentry kind="var" path="M2_REPO/com/esotericsoftware/kryo/1.01/kryo-1.01.jar" sourcepath="/M2SRC/kryo/src"/>
<classpathentry kind="var" path="M2_REPO/com/esotericsoftware/minlog/1.2/minlog-1.2.jar"/>
<classpathentry kind="var" path="M2_REPO/com/esotericsoftware/reflectasm/0.8/reflectasm-0.8.jar"/>
<classpathentry kind="var" path="M2_REPO/asm/asm/3.2/asm-3.2.jar"/>
<classpathentry kind="output" path="bin"/>
</classpath>
commons/src/main/java/org/archive/bdb/AutoKryo.java
+83
View File
@@ -0,0 +1,83 @@
package org.archive.bdb;
import java.lang.reflect.Constructor;
import java.lang.reflect.InvocationTargetException;
import java.util.ArrayList;
import sun.reflect.ReflectionFactory;
import com.esotericsoftware.kryo.Kryo;
import com.esotericsoftware.kryo.SerializationException;
/**
* Extensions to Kryo to let classes control their own registration, suggest
* other classes to register together, and use the same (Sun-JVM-only) trick
* for deserializing classes without no-arg constructors.
*
* TODO: more comments!
*
* @contributor gojomo
*/
@SuppressWarnings("unchecked")
public class AutoKryo extends Kryo {
ArrayList<Class> registeredClasses = new ArrayList<Class>();
@Override
protected void handleUnregisteredClass(Class type) {
System.err.println("UNREGISTERED FOR KRYO "+type+" in "+registeredClasses.get(0));
super.handleUnregisteredClass(type);
}
public void autoregister(Class type) {
if (registeredClasses.contains(type)) {
return;
}
registeredClasses.add(type);
try {
invokeStatic(
"autoregisterTo",
type,
new Class[]{ ((Class)AutoKryo.class), },
new Object[] { this, });
} catch (Exception e) {
register(type);
}
}
@Override
public <T> T newInstance(Class<T> type) {
SerializationException ex = null;
try {
return super.newInstance(type);
} catch (SerializationException se) {
ex = se;
}
try {
final Constructor<?> constructor =
ReflectionFactory.getReflectionFactory().newConstructorForSerialization(
type,
Object.class.getDeclaredConstructor( new Class[0] ) );
constructor.setAccessible( true );
Object inst = constructor.newInstance( new Object[0] );
return (T) inst;
} catch (SecurityException e) {
e.printStackTrace();
} catch (NoSuchMethodException e) {
e.printStackTrace();
} catch (IllegalArgumentException e) {
e.printStackTrace();
} catch (InstantiationException e) {
e.printStackTrace();
} catch (IllegalAccessException e) {
e.printStackTrace();
} catch (InvocationTargetException e) {
e.printStackTrace();
}
throw ex;
}
protected Object invokeStatic(String method, Class clazz, Class[] types, Object[] args) throws Exception {
return clazz.getMethod(method, types).invoke(null, args);
}
}
commons/src/main/java/org/archive/bdb/BdbModule.java
+11 -48
View File
@@ -43,7 +43,6 @@ import org.apache.commons.io.filefilter.IOFileFilter;
import org.archive.checkpointing.Checkpoint;
import org.archive.checkpointing.Checkpointable;
import org.archive.spring.ConfigPath;
import org.archive.util.CachedBdbMap;
import org.archive.util.ObjectIdentityBdbCache;
import org.archive.util.ObjectIdentityCache;
import org.archive.util.bdbje.EnhancedEnvironment;
@@ -77,7 +76,6 @@ public class BdbModule implements Lifecycle, Checkpointable, Closeable {
private static class DatabasePlusConfig implements Serializable {
private static final long serialVersionUID = 1L;
public transient Database database;
public String name;
public BdbConfig config;
}
@@ -189,6 +187,7 @@ public class BdbModule implements Lifecycle, Checkpointable, Closeable {
private transient StoredClassCatalog classCatalog;
@SuppressWarnings("unchecked")
private Map<String,ObjectIdentityCache> oiCaches =
new ConcurrentHashMap<String,ObjectIdentityCache>();
@@ -337,7 +336,6 @@ public class BdbModule implements Lifecycle, Checkpointable, Closeable {
}
}
dpc.database = bdbEnvironment.openDatabase(null, name, config.toDatabaseConfig());
dpc.name = name;
dpc.config = config;
databases.put(name, dpc);
return dpc.database;
@@ -358,39 +356,7 @@ public class BdbModule implements Lifecycle, Checkpointable, Closeable {
}
}
/**
* Get a CachedBdbMap, backed by a BDB Database of the given name,
* with the given key and value class types. If 'recycle' is true,
* reuse values already in the database; otherwise start with an
* empty map.
*
* @param <K>
* @param <V>
* @param dbName
* @param recycle
* @param key
* @param value
* @return
* @throws DatabaseException
* @deprecated use ObjectIdentityBdbCache instead
*/
public <K,V> CachedBdbMap<K,V> getCBMMap(String dbName, boolean recycle,
Class<? super K> key, Class<? super V> value)
throws DatabaseException {
if (!recycle) {
try {
bdbEnvironment.truncateDatabase(null, dbName, false);
} catch (DatabaseNotFoundException e) {
// ignored
}
}
CachedBdbMap<K, V> r = new CachedBdbMap<K,V>(dbName);
r.initialize(bdbEnvironment, key, value, classCatalog);
oiCaches.put(dbName, r);
return r;
}
/**
* Get an ObjectIdentityBdbCache, backed by a BDB Database of the
@@ -406,7 +372,7 @@ public class BdbModule implements Lifecycle, Checkpointable, Closeable {
* @throws DatabaseException
*/
public <V> ObjectIdentityBdbCache<V> getOIBCCache(String dbName, boolean recycle,
Class<? super V> valueClass)
Class<? extends V> valueClass)
throws DatabaseException {
if (!recycle) {
try {
@@ -420,11 +386,12 @@ public class BdbModule implements Lifecycle, Checkpointable, Closeable {
oiCaches.put(dbName, oic);
return oic;
}
/** controls which alternate ObjectIdentityCache implementation to use */
private static boolean USE_OIBC = true;
public <V> ObjectIdentityCache<String, V> getObjectCache(String dbName, boolean recycle,
Class<V> valueClass)
throws DatabaseException {
return getObjectCache(dbName, recycle, valueClass, valueClass);
}
/**
* Get an ObjectIdentityCache, backed by a BDB Database of the given
@@ -440,18 +407,14 @@ public class BdbModule implements Lifecycle, Checkpointable, Closeable {
* @throws DatabaseException
*/
public <V> ObjectIdentityCache<String, V> getObjectCache(String dbName, boolean recycle,
Class<? super V> valueClass)
Class<V> declaredClass, Class<? extends V> valueClass)
throws DatabaseException {
@SuppressWarnings("unchecked")
ObjectIdentityCache<String,V> oic = oiCaches.get(dbName);
if(oic!=null) {
return oic;
}
if(USE_OIBC) {
oic = getOIBCCache(dbName, recycle, valueClass);
} else {
oic = getCBMMap(dbName, recycle, String.class, valueClass);
}
oic = getOIBCCache(dbName, recycle, valueClass);
return oic;
}
commons/src/main/java/org/archive/bdb/KryoBinding.java
+75
View File
@@ -0,0 +1,75 @@
/*
* This file is part of the Heritrix web crawler (crawler.archive.org).
*
* Licensed to the Internet Archive (IA) by one or more individual
* contributors.
*
* The IA licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.archive.bdb;
import com.esotericsoftware.kryo.Kryo;
import com.esotericsoftware.kryo.ObjectBuffer;
import com.sleepycat.bind.EntryBinding;
import com.sleepycat.je.DatabaseEntry;
/**
* Binding for use with BerkeleyDB-JE that uses Kryo serialization rather
* than BDB's (custom version of) Java serialization.
*
* @contributor gojomo
*/
public class KryoBinding<K> implements EntryBinding<K> {
protected Class<K> baseClass;
protected AutoKryo kryo = new AutoKryo();
protected ThreadLocal<ObjectBuffer> threadBuffer = new ThreadLocal<ObjectBuffer>() {
@Override
protected ObjectBuffer initialValue() {
return new ObjectBuffer(kryo,16*1024,Integer.MAX_VALUE);
}
};
/**
* Constructor. Save parameters locally, as superclass
* fields are private.
*
* @param classCatalog is the catalog to hold shared class information
*
* @param baseClass is the base class for serialized objects stored using
* this binding
*/
@SuppressWarnings("unchecked")
public KryoBinding(Class baseClass) {
this.baseClass = baseClass;
kryo.autoregister(baseClass);
}
public Kryo getKryo() {
return kryo;
}
/**
* Copies superclass simply to allow different source for FastOoutputStream.
*
* @see com.sleepycat.bind.serial.SerialBinding#entryToObject
*/
public void objectToEntry(K object, DatabaseEntry entry) {
entry.setData(threadBuffer.get().writeObjectData(object));
}
@Override
public K entryToObject(DatabaseEntry entry) {
return threadBuffer.get().readObjectData(entry.getData(), baseClass);
}
}
commons/src/main/java/org/archive/net/UURI.java
+41 -2
View File
@@ -18,15 +18,23 @@
*/
package org.archive.net;
import java.io.Externalizable;
import java.io.File;
import java.io.Serializable;
import java.io.IOException;
import java.io.ObjectInput;
import java.io.ObjectOutput;
import java.net.URI;
import java.net.URISyntaxException;
import java.nio.ByteBuffer;
import org.apache.commons.httpclient.URIException;
import org.archive.util.SURT;
import org.archive.util.TextUtils;
import com.esotericsoftware.kryo.CustomSerialization;
import com.esotericsoftware.kryo.Kryo;
import com.esotericsoftware.kryo.serialize.StringSerializer;
/**
* Usable URI.
@@ -51,7 +59,7 @@ import org.archive.util.TextUtils;
* @see org.apache.commons.httpclient.URI
*/
public class UURI extends LaxURI
implements CharSequence, Serializable {
implements CharSequence, Externalizable, CustomSerialization {
private static final long serialVersionUID = -1277570889914647093L;
@@ -438,4 +446,35 @@ implements CharSequence, Serializable {
}
return (new File(path)).getName();
}
public void writeObjectData(Kryo kryo, ByteBuffer buffer) {
StringSerializer.put(buffer, toCustomString());
}
public void readObjectData (Kryo kryo, ByteBuffer buffer) {
try {
parseUriReference(StringSerializer.get(buffer),true);
} catch (URIException e) {
// TODO Auto-generated catch block
e.printStackTrace();
}
}
@Override
public void readExternal(ObjectInput in) throws IOException,
ClassNotFoundException {
try {
parseUriReference(in.readUTF(),true);
} catch (URIException e) {
// TODO Auto-generated catch block
e.printStackTrace();
}
}
@Override
public void writeExternal(ObjectOutput out) throws IOException {
out.writeUTF(toCustomString());
}
}
commons/src/main/java/org/archive/util/ObjectIdentityBdbCache.java
+11 -1
View File
@@ -33,6 +33,10 @@ import java.util.concurrent.atomic.AtomicLong;
import java.util.logging.Level;
import java.util.logging.Logger;
import org.archive.bdb.BenchmarkingBinding;
import org.archive.bdb.KryoBinding;
import org.archive.crawler.frontier.RecyclingSerialBinding;
import com.sleepycat.bind.EntryBinding;
import com.sleepycat.bind.serial.SerialBinding;
import com.sleepycat.bind.serial.StoredClassCatalog;
@@ -173,7 +177,13 @@ implements ObjectIdentityCache<String, V>, Closeable, Serializable {
EntryBinding keyBinding = TupleBinding.getPrimitiveBinding(String.class);
EntryBinding valueBinding = TupleBinding.getPrimitiveBinding(valueClass);
if(valueBinding == null) {
valueBinding = new SerialBinding(classCatalog, valueClass);
valueBinding =
new KryoBinding<V>(valueClass);
// new SerialBinding(classCatalog, valueClass);
// new BenchmarkingBinding<V>(new EntryBinding[] {
// new KryoBinding<V>(valueClass),
// new RecyclingSerialBinding<V>(classCatalog, valueClass),
// }, valueClass);
}
return new StoredSortedMap<String,V>(database, keyBinding, valueBinding, true);
}
commons/src/test/java/org/archive/util/CachedBdbMapTest.java
-133
View File
@@ -1,133 +0,0 @@
/*
* This file is part of the Heritrix web crawler (crawler.archive.org).
*
* Licensed to the Internet Archive (IA) by one or more individual
* contributors.
*
* The IA licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.archive.util;
import java.io.File;
import java.util.HashMap;
import java.util.logging.Handler;
import java.util.logging.Level;
import java.util.logging.Logger;
import org.apache.commons.io.FileUtils;
import org.archive.bdb.BdbModule;
/**
* Tests of CachedBdbMap
*
* @contributor stack
* @contributor gojomo
* @version $Date$, $Revision$
*/
public class CachedBdbMapTest extends TmpDirTestCase {
File envDir;
BdbModule bdb;
private CachedBdbMap<String,HashMap<String,String>> cache;
@SuppressWarnings("unchecked")
protected void setUp() throws Exception {
super.setUp();
this.envDir = new File(getTmpDir(),"CachedBdbMapTest");
this.envDir.mkdirs();
bdb = new BdbModule();
bdb.getDir().setBase(null);
bdb.getDir().setPath(envDir.getAbsolutePath());
bdb.start();
this.cache = bdb.getCBMMap(
this.getClass().getName(), false, String.class, HashMap.class);
}
protected void tearDown() throws Exception {
ArchiveUtils.closeQuietly(this.cache);
bdb.stop();
FileUtils.deleteDirectory(this.envDir);
super.tearDown();
}
public void testBackingDbGetsUpdated() {
// Enable all logging. Up the level on the handlers and then
// on the big map itself.
Handler [] handlers = Logger.getLogger("").getHandlers();
for (int index = 0; index < handlers.length; index++) {
handlers[index].setLevel(Level.FINEST);
}
Logger.getLogger(CachedBdbMap.class.getName()).
setLevel(Level.FINEST);
// Set up values.
final String value = "value";
final String key = "key";
final int upperbound = 3;
// First put in empty hashmap.
for (int i = 0; i < upperbound; i++) {
assertNull("unexpected prior entry",
this.cache.putIfAbsent(key + Integer.toString(i), new HashMap<String,String>()));
}
// Now add value to hash map.
for (int i = 0; i < upperbound; i++) {
HashMap<String,String> m = this.cache.get(key + Integer.toString(i));
m.put(key, value);
}
this.cache.sync();
for (int i = 0; i < upperbound; i++) {
HashMap<String,String> m = this.cache.get(key + Integer.toString(i));
String v = m.get(key);
if (v == null || !v.equals(value)) {
Logger.getLogger(CachedBdbMap.class.getName()).
warning("Wrong value " + i);
}
}
}
/**
* Test that in scarce memory conditions, the memory map is
* expunged of otherwise unreferenced entries as expected.
*
* NOTE: this test may be especially fragile with regard to
* GC/timing issues; relies on timely finalization, which is
* never guaranteed by JVM/GC. For example, it is so sensitive
* to CPU speed that a Thread.sleep(1000) succeeds when my
* laptop is plugged in, but fails when it is on battery!
*
* @throws InterruptedException
*/
public void testMemMapCleared() throws InterruptedException {
System.gc();
System.runFinalization();
System.gc();
assertEquals(cache.memMap.size(), 0);
for(int i=0; i < 10000; i++) {
cache.putIfAbsent(""+i, new HashMap<String,String>());
}
assertEquals(10000, cache.memMap.size());
assertEquals(10000, cache.size());
TestUtils.forceScarceMemory();
Thread.sleep(2000);
TestUtils.forceScarceMemory();
Thread.sleep(2000);
// The 'canary' trick makes this explicit expunge, or
// an expunge triggered by a get() or put...(), unnecessary
//cache.expungeStaleEntries();
System.out.println(cache.size()+","+cache.memMap.size());
assertEquals(0, cache.memMap.size());
}
public static void main(String [] args) {
junit.textui.TestRunner.run(CachedBdbMapTest.class);
}
}
engine/src/main/java/org/archive/crawler/frontier/BdbFrontier.java
+3 -3
View File
@@ -153,9 +153,9 @@ implements Checkpointable, BeanNameAware {
WorkQueue wq = allQueues.getOrUse(
classKey,
new Supplier<WorkQueue>() {
public WorkQueue get() {
public BdbWorkQueue get() {
String qKey = new String(classKey); // ensure private minimal key
WorkQueue q = new BdbWorkQueue(qKey, BdbFrontier.this);
BdbWorkQueue q = new BdbWorkQueue(qKey, BdbFrontier.this);
q.setTotalBudget(getQueueTotalBudget());
getQueuePrecedencePolicy().queueCreated(q);
return q;
@@ -257,7 +257,7 @@ implements Checkpointable, BeanNameAware {
@Override
protected void initAllQueues() throws DatabaseException {
boolean isRecovery = (recoveryCheckpoint != null);
this.allQueues = bdb.getObjectCache("allqueues", isRecovery, WorkQueue.class);
this.allQueues = bdb.getObjectCache("allqueues", isRecovery, WorkQueue.class, BdbWorkQueue.class);
if(isRecovery) {
JSONObject json = recoveryCheckpoint.loadJson(beanName);
try {
engine/src/main/java/org/archive/crawler/frontier/BdbMultipleWorkQueues.java
+11 -2
View File
@@ -31,9 +31,11 @@ import javax.management.openmbean.CompositeDataSupport;
import javax.management.openmbean.OpenDataException;
import org.apache.commons.collections.Closure;
import org.archive.bdb.KryoBinding;
import org.archive.modules.CrawlURI;
import org.archive.util.ArchiveUtils;
import com.sleepycat.bind.EntryBinding;
import com.sleepycat.bind.serial.StoredClassCatalog;
import com.sleepycat.je.Cursor;
import com.sleepycat.je.Database;
@@ -65,7 +67,7 @@ public class BdbMultipleWorkQueues {
private Database pendingUrisDB = null;
/** Supporting bdb serialization of CrawlURIs */
private RecyclingSerialBinding<CrawlURI> crawlUriBinding;
private EntryBinding<CrawlURI> crawlUriBinding;
/**
* Create the multi queue in the given environment.
@@ -80,7 +82,14 @@ public class BdbMultipleWorkQueues {
throws DatabaseException {
this.pendingUrisDB = db;
crawlUriBinding =
new RecyclingSerialBinding<CrawlURI>(classCatalog, CrawlURI.class);
new KryoBinding<CrawlURI>(CrawlURI.class);
// new RecyclingSerialBinding<CrawlURI>(classCatalog, CrawlURI.class);
// new BenchmarkingBinding<CrawlURI>(new EntryBinding[] {
// new KryoBinding<CrawlURI>(CrawlURI.class,true),
// new KryoBinding<CrawlURI>(CrawlURI.class,false),
// new RecyclingSerialBinding<CrawlURI>(classCatalog, CrawlURI.class),
// });
}
/**
engine/src/main/java/org/archive/crawler/frontier/BdbWorkQueue.java
+14
View File
@@ -21,10 +21,14 @@ package org.archive.crawler.frontier;
import java.io.IOException;
import java.io.Serializable;
import java.io.UnsupportedEncodingException;
import java.util.HashSet;
import java.util.logging.Level;
import java.util.logging.Logger;
import org.archive.bdb.AutoKryo;
import org.archive.crawler.frontier.precedence.SimplePrecedenceProvider;
import org.archive.modules.CrawlURI;
import org.archive.modules.fetcher.FetchStats;
import org.archive.util.ArchiveUtils;
import com.sleepycat.je.DatabaseEntry;
@@ -166,4 +170,14 @@ implements Serializable {
return e.getMessage();
}
}
// Kryo support
public static void autoregisterTo(AutoKryo kryo) {
kryo.register(BdbWorkQueue.class);
kryo.autoregister(FetchStats.class);
kryo.autoregister(HashSet.class);
kryo.autoregister(SimplePrecedenceProvider.class);
kryo.autoregister(byte[].class);
kryo.setRegistrationOptional(true);
}
}
engine/src/main/java/org/archive/crawler/reporting/StatisticsTracker.java
+10 -3
View File
@@ -331,8 +331,9 @@ public class StatisticsTracker
new ObjectIdentityMemCache<AtomicLong>(); // temp dummy
/** Keep track of URL counts per host per seed */
protected ObjectIdentityCache<String,ConcurrentMap<String,AtomicLong>> sourceHostDistribution =
new ObjectIdentityMemCache<ConcurrentMap<String,AtomicLong>>(); // temp dummy;
@SuppressWarnings("unchecked")
protected ObjectIdentityCache<String,ConcurrentMap> sourceHostDistribution =
new ObjectIdentityMemCache<ConcurrentMap>(); // temp dummy;
/* Keep track of 'top' hosts for live reports */
protected TopNSet hostsDistributionTop;
@@ -503,6 +504,10 @@ public class StatisticsTracker
return snapshot;
}
public LinkedList<CrawlStatSnapshot> getSnapshots() {
return snapshots;
}
public CrawlStatSnapshot getLastSnapshot() {
CrawlStatSnapshot snap = snapshots.peek();
return snap == null ? getSnapshot() : snap;
@@ -840,12 +845,13 @@ public class StatisticsTracker
}
}
@SuppressWarnings("unchecked")
protected void saveSourceStats(String source, String hostname) {
synchronized(sourceHostDistribution) {
ConcurrentMap<String,AtomicLong> hostUriCount =
sourceHostDistribution.getOrUse(
source,
new Supplier<ConcurrentMap<String,AtomicLong>>() {
new Supplier<ConcurrentMap>() {
public ConcurrentMap<String, AtomicLong> get() {
return new ConcurrentHashMap<String,AtomicLong>();
}});
@@ -1121,6 +1127,7 @@ public class StatisticsTracker
// CrawlController's only interest is in knowing that a Checkpoint is
// being recovered
public void startCheckpoint(Checkpoint checkpointInProgress) {}
@SuppressWarnings("unchecked")
public void doCheckpoint(Checkpoint checkpointInProgress) throws IOException {
JSONObject json = new JSONObject();
try {
modules/src/main/java/org/archive/modules/CrawlURI.java
+68 -79
View File
@@ -77,6 +77,7 @@ import org.apache.commons.httpclient.URIException;
import org.apache.commons.httpclient.methods.GetMethod;
import org.apache.commons.httpclient.methods.PostMethod;
import org.apache.commons.lang.StringUtils;
import org.archive.bdb.AutoKryo;
import org.archive.modules.credential.CredentialAvatar;
import org.archive.modules.credential.HttpAuthenticationCredential;
import org.archive.modules.extractor.HTMLLinkContext;
@@ -116,7 +117,7 @@ implements MultiReporter, Serializable, OverlayContext {
/**
* The URI being crawled. It's transient to save space when storing to BDB.
*/
private transient UURI uuri;
private UURI uuri;
/** Seed status */
@@ -138,7 +139,7 @@ implements MultiReporter, Serializable, OverlayContext {
* Where this URI was (presently) discovered. . Transient to allow
* more efficient custom serialization
*/
private transient UURI via;
private UURI via;
/**
* Context of URI's discovery, as per the 'context' in Link
@@ -199,7 +200,7 @@ implements MultiReporter, Serializable, OverlayContext {
*
* Package-protected so CrawlURI can access it directly.
*/
transient Map<String,Object> data;
Map<String,Object> data;
// private transient SheetManager manager;
@@ -646,7 +647,7 @@ implements MultiReporter, Serializable, OverlayContext {
* This methods removes the attribute list.
*/
public void stripToMinimal() {
data = null;
data = null;
}
/**
@@ -670,13 +671,13 @@ implements MultiReporter, Serializable, OverlayContext {
* @return the annotations set for this uri.
*/
public Collection<String> getAnnotations() {
@SuppressWarnings("unchecked")
List<String> list = (List<String>)getData().get(A_ANNOTATIONS);
@SuppressWarnings("unchecked")
List<String> list = (List<String>)getData().get(A_ANNOTATIONS);
if (list == null) {
list = new ArrayList<String>();
getData().put(A_ANNOTATIONS, list);
}
return list;
return list;
}
/**
@@ -867,20 +868,20 @@ implements MultiReporter, Serializable, OverlayContext {
}
public Map<String,Object> getPersistentDataMap() {
if (data == null) {
return null;
}
Map<String,Object> result = new HashMap<String,Object>(getData());
Set<String> retain = new HashSet<String>(persistentKeys);
if (containsDataKey(A_HERITABLE_KEYS)) {
@SuppressWarnings("unchecked")
if (data == null) {
return null;
}
Map<String,Object> result = new HashMap<String,Object>(getData());
Set<String> retain = new HashSet<String>(persistentKeys);
if (containsDataKey(A_HERITABLE_KEYS)) {
@SuppressWarnings("unchecked")
HashSet<String> heritable = (HashSet<String>)getData().get(A_HERITABLE_KEYS);
retain.addAll(heritable);
}
result.keySet().retainAll(retain);
return result;
}
result.keySet().retainAll(retain);
return result;
}
/**
@@ -954,26 +955,26 @@ implements MultiReporter, Serializable, OverlayContext {
* @see #isSuccess()
*/
public boolean is2XXSuccess() {
return this.fetchStatus >= 200 && this.fetchStatus < 300;
return this.fetchStatus >= 200 && this.fetchStatus < 300;
}
/**
* @return True if we have an rfc2617 payload.
*/
public boolean hasRfc2617CredentialAvatar() {
boolean result = false;
Set<CredentialAvatar> avatars = getCredentialAvatars();
if (avatars != null && avatars.size() > 0) {
for (Iterator<CredentialAvatar> i = avatars.iterator(); i.hasNext();) {
if (((CredentialAvatar)i.next()).
match(HttpAuthenticationCredential.class)) {
result = true;
break;
}
}
}
* @return True if we have an rfc2617 payload.
*/
public boolean hasRfc2617CredentialAvatar() {
boolean result = false;
Set<CredentialAvatar> avatars = getCredentialAvatars();
if (avatars != null && avatars.size() > 0) {
for (Iterator<CredentialAvatar> i = avatars.iterator(); i.hasNext();) {
if (((CredentialAvatar)i.next()).
match(HttpAuthenticationCredential.class)) {
result = true;
break;
}
}
}
return result;
}
}
/**
@@ -1153,7 +1154,7 @@ implements MultiReporter, Serializable, OverlayContext {
* @param key Key to add.
*/
public static Collection<String> getPersistentDataKeys() {
return persistentKeys;
return persistentKeys;
}
public void addPersistentDataMapKey(String s) {
@@ -1180,49 +1181,21 @@ implements MultiReporter, Serializable, OverlayContext {
return persistentKeys.remove(key);
}
/**
* Custom serialization writing an empty 'outLinks' as null. Estimated
* to save ~20 bytes in serialized form.
*
* @param stream
* @throws IOException
*/
private void writeObject(ObjectOutputStream stream) throws IOException {
stream.defaultWriteObject();
stream.writeUTF(uuri.toCustomString()); //
stream.writeObject((via == null) ? null : via.getURI());
stream.writeObject((data==null || data.isEmpty()) ? null : data);
stream.writeObject((outLinks.isEmpty()) ? null : outLinks);
stream.writeObject((outCandidates.isEmpty()) ? null : outCandidates);
}
/**
* Custom deserialization recreating empty HashSet from null in 'outLinks'
* slot.
*
* @param stream
* @throws IOException
* @throws ClassNotFoundException
*/
}
private void readObject(ObjectInputStream stream) throws IOException,
ClassNotFoundException {
ClassNotFoundException {
stream.defaultReadObject();
uuri = readUuri(stream.readUTF());
via = readUuri((String)stream.readObject());
@SuppressWarnings("unchecked")
Map<String,Object> temp = (Map<String,Object>)stream.readObject();
this.data = temp;
@SuppressWarnings("unchecked")
HashSet<Link> ol = (HashSet<Link>) stream.readObject();
outLinks = (ol == null) ? new HashSet<Link>() : ol;
@SuppressWarnings("unchecked")
HashSet<CrawlURI> oc = (HashSet<CrawlURI>)stream.readObject();
outCandidates = (oc == null) ? new HashSet<CrawlURI>() : oc;
outLinks = new HashSet<Link>();
outCandidates = new HashSet<CrawlURI>();
}
/**
* Read a UURI from a String, handling a null or URIException
*
@@ -1295,15 +1268,15 @@ implements MultiReporter, Serializable, OverlayContext {
}
public Collection<Throwable> getNonFatalFailures() {
@SuppressWarnings("unchecked")
List<Throwable> list = (List)getData().get(A_NONFATAL_ERRORS);
if (list == null) {
list = new ArrayList<Throwable>();
getData().put(A_NONFATAL_ERRORS, list);
}
// FIXME: Previous code automatically added annotation when "localized error"
// was added, override collection to implement that?
@SuppressWarnings("unchecked")
List<Throwable> list = (List)getData().get(A_NONFATAL_ERRORS);
if (list == null) {
list = new ArrayList<Throwable>();
getData().put(A_NONFATAL_ERRORS, list);
}
// FIXME: Previous code automatically added annotation when "localized error"
// was added, override collection to implement that?
return list;
}
@@ -1318,7 +1291,7 @@ implements MultiReporter, Serializable, OverlayContext {
}
public void setFetchBeginTime(long time) {
getData().put(ModuleAttributeConstants.A_FETCH_BEGAN_TIME, time);
getData().put(ModuleAttributeConstants.A_FETCH_BEGAN_TIME, time);
}
public void setFetchCompletedTime(long time) {
@@ -1837,4 +1810,20 @@ implements MultiReporter, Serializable, OverlayContext {
return containsDataKey(A_FORCE_RETIRE)
&& (Boolean)getData().get(A_FORCE_RETIRE);
}
// Kryo support
@SuppressWarnings("unused")
private CrawlURI() {}
public static void autoregisterTo(AutoKryo kryo) {
// kryo.register(CrawlURI.class,new DeflateCompressor(kryo.newSerializer(CrawlURI.class)));
kryo.register(CrawlURI.class);
kryo.autoregister(byte[].class);
kryo.autoregister(java.util.HashSet.class);
kryo.autoregister(java.util.HashMap.class);
kryo.autoregister(org.archive.net.UURI.class);
kryo.autoregister(org.archive.modules.extractor.HTMLLinkContext.class);
kryo.autoregister(org.archive.modules.extractor.LinkContext.SimpleLinkContext.class);
kryo.autoregister(java.util.HashMap[].class);
kryo.setRegistrationOptional(true);
}
}
modules/src/main/java/org/archive/modules/extractor/LinkContext.java
+1 -1
View File
@@ -32,7 +32,7 @@ public abstract class LinkContext implements Serializable {
/** Class for representing handy default LinkContext values. */
private static class SimpleLinkContext extends LinkContext {
public static class SimpleLinkContext extends LinkContext {
private static final long serialVersionUID = 1L;
modules/src/main/java/org/archive/modules/net/BdbServerCache.java
+2 -2
View File
@@ -50,8 +50,8 @@ implements Lifecycle {
return;
}
try {
this.servers = bdb.getObjectCache("servers", false, CrawlServer.class);
this.hosts = bdb.getObjectCache("hosts", false, CrawlHost.class);
this.servers = bdb.getObjectCache("servers", false, CrawlServer.class, CrawlServer.class);
this.hosts = bdb.getObjectCache("hosts", false, CrawlHost.class, CrawlHost.class);
} catch (DatabaseException e) {
throw new IllegalStateException(e);
}
modules/src/main/java/org/archive/modules/net/CrawlHost.java
+11
View File
@@ -20,10 +20,12 @@
package org.archive.modules.net;
import java.io.Serializable;
import java.net.Inet4Address;
import java.net.InetAddress;
import java.util.logging.Level;
import java.util.logging.Logger;
import org.archive.bdb.AutoKryo;
import org.archive.modules.fetcher.FetchStats;
import org.archive.util.InetAddressUtil;
@@ -223,4 +225,13 @@ public class CrawlHost implements Serializable, FetchStats.HasFetchStats {
public FetchStats getSubstats() {
return substats;
}
// Kryo support
// public CrawlHost() {}
public static void autoregisterTo(AutoKryo kryo) {
kryo.register(CrawlHost.class);
kryo.autoregister(FetchStats.class);
kryo.autoregister(Inet4Address.class);
kryo.setRegistrationOptional(true);
}
}
modules/src/main/java/org/archive/modules/net/CrawlServer.java
+15 -1
View File
@@ -37,6 +37,7 @@ import org.apache.commons.collections.Predicate;
import org.apache.commons.httpclient.NoHttpResponseException;
import org.apache.commons.httpclient.URIException;
import org.apache.commons.io.IOUtils;
import org.archive.bdb.AutoKryo;
import org.archive.io.ReplayInputStream;
import org.archive.modules.CrawlURI;
import org.archive.modules.credential.CredentialAvatar;
@@ -62,7 +63,7 @@ public class CrawlServer implements Serializable, FetchStats.HasFetchStats {
* after this many tries */
public static final long MIN_ROBOTS_RETRIES = 3;
private final String server; // actually, host+port in the https case
private String server; // actually, host+port in the https case
private int port;
protected Robotstxt robotstxt;
long robotsFetched = ROBOTS_NOT_FETCHED;
@@ -324,4 +325,17 @@ public class CrawlServer implements Serializable, FetchStats.HasFetchStats {
}
return false;
}
// Kryo support
// public CrawlServer() {}
public static void autoregisterTo(AutoKryo kryo) {
kryo.register(CrawlServer.class);
kryo.autoregister(FetchStats.class);
kryo.autoregister(org.archive.modules.net.Robotstxt.class);
kryo.autoregister(java.util.HashMap.class);
kryo.autoregister(org.archive.modules.net.RobotsDirectives.class);
kryo.autoregister(org.archive.util.PrefixSet.class);
kryo.autoregister(java.util.LinkedList.class);
kryo.setRegistrationOptional(true);
}
}