diff --git a/.classpath b/.classpath index 1307f9e7..c754bc3c 100644 --- a/.classpath +++ b/.classpath @@ -51,5 +51,9 @@ + + + + diff --git a/commons/src/main/java/org/archive/bdb/AutoKryo.java b/commons/src/main/java/org/archive/bdb/AutoKryo.java new file mode 100644 index 00000000..ccb47d37 --- /dev/null +++ b/commons/src/main/java/org/archive/bdb/AutoKryo.java @@ -0,0 +1,83 @@ +package org.archive.bdb; + +import java.lang.reflect.Constructor; +import java.lang.reflect.InvocationTargetException; +import java.util.ArrayList; + +import sun.reflect.ReflectionFactory; + +import com.esotericsoftware.kryo.Kryo; +import com.esotericsoftware.kryo.SerializationException; + +/** + * Extensions to Kryo to let classes control their own registration, suggest + * other classes to register together, and use the same (Sun-JVM-only) trick + * for deserializing classes without no-arg constructors. + * + * TODO: more comments! + * + * @contributor gojomo + */ +@SuppressWarnings("unchecked") +public class AutoKryo extends Kryo { + ArrayList registeredClasses = new ArrayList(); + + @Override + protected void handleUnregisteredClass(Class type) { + System.err.println("UNREGISTERED FOR KRYO "+type+" in "+registeredClasses.get(0)); + super.handleUnregisteredClass(type); + } + + public void autoregister(Class type) { + if (registeredClasses.contains(type)) { + return; + } + registeredClasses.add(type); + try { + invokeStatic( + "autoregisterTo", + type, + new Class[]{ ((Class)AutoKryo.class), }, + new Object[] { this, }); + } catch (Exception e) { + register(type); + } + } + + + @Override + public T newInstance(Class type) { + SerializationException ex = null; + try { + return super.newInstance(type); + } catch (SerializationException se) { + ex = se; + } + try { + final Constructor constructor = + ReflectionFactory.getReflectionFactory().newConstructorForSerialization( + type, + Object.class.getDeclaredConstructor( new Class[0] ) ); + constructor.setAccessible( true ); + Object inst = constructor.newInstance( new Object[0] ); + return (T) inst; + } catch (SecurityException e) { + e.printStackTrace(); + } catch (NoSuchMethodException e) { + e.printStackTrace(); + } catch (IllegalArgumentException e) { + e.printStackTrace(); + } catch (InstantiationException e) { + e.printStackTrace(); + } catch (IllegalAccessException e) { + e.printStackTrace(); + } catch (InvocationTargetException e) { + e.printStackTrace(); + } + throw ex; + } + + protected Object invokeStatic(String method, Class clazz, Class[] types, Object[] args) throws Exception { + return clazz.getMethod(method, types).invoke(null, args); + } +} diff --git a/commons/src/main/java/org/archive/bdb/BdbModule.java b/commons/src/main/java/org/archive/bdb/BdbModule.java index 2fa91a81..2cf259d0 100644 --- a/commons/src/main/java/org/archive/bdb/BdbModule.java +++ b/commons/src/main/java/org/archive/bdb/BdbModule.java @@ -43,7 +43,6 @@ import org.apache.commons.io.filefilter.IOFileFilter; import org.archive.checkpointing.Checkpoint; import org.archive.checkpointing.Checkpointable; import org.archive.spring.ConfigPath; -import org.archive.util.CachedBdbMap; import org.archive.util.ObjectIdentityBdbCache; import org.archive.util.ObjectIdentityCache; import org.archive.util.bdbje.EnhancedEnvironment; @@ -77,7 +76,6 @@ public class BdbModule implements Lifecycle, Checkpointable, Closeable { private static class DatabasePlusConfig implements Serializable { private static final long serialVersionUID = 1L; public transient Database database; - public String name; public BdbConfig config; } @@ -189,6 +187,7 @@ public class BdbModule implements Lifecycle, Checkpointable, Closeable { private transient StoredClassCatalog classCatalog; + @SuppressWarnings("unchecked") private Map oiCaches = new ConcurrentHashMap(); @@ -337,7 +336,6 @@ public class BdbModule implements Lifecycle, Checkpointable, Closeable { } } dpc.database = bdbEnvironment.openDatabase(null, name, config.toDatabaseConfig()); - dpc.name = name; dpc.config = config; databases.put(name, dpc); return dpc.database; @@ -358,39 +356,7 @@ public class BdbModule implements Lifecycle, Checkpointable, Closeable { } } - - - /** - * Get a CachedBdbMap, backed by a BDB Database of the given name, - * with the given key and value class types. If 'recycle' is true, - * reuse values already in the database; otherwise start with an - * empty map. - * - * @param - * @param - * @param dbName - * @param recycle - * @param key - * @param value - * @return - * @throws DatabaseException - * @deprecated use ObjectIdentityBdbCache instead - */ - public CachedBdbMap getCBMMap(String dbName, boolean recycle, - Class key, Class value) - throws DatabaseException { - if (!recycle) { - try { - bdbEnvironment.truncateDatabase(null, dbName, false); - } catch (DatabaseNotFoundException e) { - // ignored - } - } - CachedBdbMap r = new CachedBdbMap(dbName); - r.initialize(bdbEnvironment, key, value, classCatalog); - oiCaches.put(dbName, r); - return r; - } + /** * Get an ObjectIdentityBdbCache, backed by a BDB Database of the @@ -406,7 +372,7 @@ public class BdbModule implements Lifecycle, Checkpointable, Closeable { * @throws DatabaseException */ public ObjectIdentityBdbCache getOIBCCache(String dbName, boolean recycle, - Class valueClass) + Class valueClass) throws DatabaseException { if (!recycle) { try { @@ -420,11 +386,12 @@ public class BdbModule implements Lifecycle, Checkpointable, Closeable { oiCaches.put(dbName, oic); return oic; } - - /** controls which alternate ObjectIdentityCache implementation to use */ - private static boolean USE_OIBC = true; - - + + public ObjectIdentityCache getObjectCache(String dbName, boolean recycle, + Class valueClass) + throws DatabaseException { + return getObjectCache(dbName, recycle, valueClass, valueClass); + } /** * Get an ObjectIdentityCache, backed by a BDB Database of the given @@ -440,18 +407,14 @@ public class BdbModule implements Lifecycle, Checkpointable, Closeable { * @throws DatabaseException */ public ObjectIdentityCache getObjectCache(String dbName, boolean recycle, - Class valueClass) + Class declaredClass, Class valueClass) throws DatabaseException { @SuppressWarnings("unchecked") ObjectIdentityCache oic = oiCaches.get(dbName); if(oic!=null) { return oic; } - if(USE_OIBC) { - oic = getOIBCCache(dbName, recycle, valueClass); - } else { - oic = getCBMMap(dbName, recycle, String.class, valueClass); - } + oic = getOIBCCache(dbName, recycle, valueClass); return oic; } diff --git a/commons/src/main/java/org/archive/bdb/KryoBinding.java b/commons/src/main/java/org/archive/bdb/KryoBinding.java new file mode 100644 index 00000000..afe2bdda --- /dev/null +++ b/commons/src/main/java/org/archive/bdb/KryoBinding.java @@ -0,0 +1,75 @@ +/* + * This file is part of the Heritrix web crawler (crawler.archive.org). + * + * Licensed to the Internet Archive (IA) by one or more individual + * contributors. + * + * The IA licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.archive.bdb; + +import com.esotericsoftware.kryo.Kryo; +import com.esotericsoftware.kryo.ObjectBuffer; +import com.sleepycat.bind.EntryBinding; +import com.sleepycat.je.DatabaseEntry; + +/** + * Binding for use with BerkeleyDB-JE that uses Kryo serialization rather + * than BDB's (custom version of) Java serialization. + * + * @contributor gojomo + */ +public class KryoBinding implements EntryBinding { + + protected Class baseClass; + protected AutoKryo kryo = new AutoKryo(); + protected ThreadLocal threadBuffer = new ThreadLocal() { + @Override + protected ObjectBuffer initialValue() { + return new ObjectBuffer(kryo,16*1024,Integer.MAX_VALUE); + } + }; + + /** + * Constructor. Save parameters locally, as superclass + * fields are private. + * + * @param classCatalog is the catalog to hold shared class information + * + * @param baseClass is the base class for serialized objects stored using + * this binding + */ + @SuppressWarnings("unchecked") + public KryoBinding(Class baseClass) { + this.baseClass = baseClass; + kryo.autoregister(baseClass); + } + + public Kryo getKryo() { + return kryo; + } + + /** + * Copies superclass simply to allow different source for FastOoutputStream. + * + * @see com.sleepycat.bind.serial.SerialBinding#entryToObject + */ + public void objectToEntry(K object, DatabaseEntry entry) { + entry.setData(threadBuffer.get().writeObjectData(object)); + } + + @Override + public K entryToObject(DatabaseEntry entry) { + return threadBuffer.get().readObjectData(entry.getData(), baseClass); + } +} diff --git a/commons/src/main/java/org/archive/net/UURI.java b/commons/src/main/java/org/archive/net/UURI.java index 3df1ae4c..d24a09ae 100644 --- a/commons/src/main/java/org/archive/net/UURI.java +++ b/commons/src/main/java/org/archive/net/UURI.java @@ -18,15 +18,23 @@ */ package org.archive.net; +import java.io.Externalizable; import java.io.File; -import java.io.Serializable; +import java.io.IOException; +import java.io.ObjectInput; +import java.io.ObjectOutput; import java.net.URI; import java.net.URISyntaxException; +import java.nio.ByteBuffer; import org.apache.commons.httpclient.URIException; import org.archive.util.SURT; import org.archive.util.TextUtils; +import com.esotericsoftware.kryo.CustomSerialization; +import com.esotericsoftware.kryo.Kryo; +import com.esotericsoftware.kryo.serialize.StringSerializer; + /** * Usable URI. @@ -51,7 +59,7 @@ import org.archive.util.TextUtils; * @see org.apache.commons.httpclient.URI */ public class UURI extends LaxURI -implements CharSequence, Serializable { +implements CharSequence, Externalizable, CustomSerialization { private static final long serialVersionUID = -1277570889914647093L; @@ -438,4 +446,35 @@ implements CharSequence, Serializable { } return (new File(path)).getName(); } + + public void writeObjectData(Kryo kryo, ByteBuffer buffer) { + StringSerializer.put(buffer, toCustomString()); + } + + public void readObjectData (Kryo kryo, ByteBuffer buffer) { + try { + parseUriReference(StringSerializer.get(buffer),true); + } catch (URIException e) { + // TODO Auto-generated catch block + e.printStackTrace(); + } + } + + @Override + public void readExternal(ObjectInput in) throws IOException, + ClassNotFoundException { + try { + parseUriReference(in.readUTF(),true); + } catch (URIException e) { + // TODO Auto-generated catch block + e.printStackTrace(); + } + } + + @Override + public void writeExternal(ObjectOutput out) throws IOException { + out.writeUTF(toCustomString()); + } + + } diff --git a/commons/src/main/java/org/archive/util/ObjectIdentityBdbCache.java b/commons/src/main/java/org/archive/util/ObjectIdentityBdbCache.java index e944ed05..e7d2c32e 100644 --- a/commons/src/main/java/org/archive/util/ObjectIdentityBdbCache.java +++ b/commons/src/main/java/org/archive/util/ObjectIdentityBdbCache.java @@ -33,6 +33,10 @@ import java.util.concurrent.atomic.AtomicLong; import java.util.logging.Level; import java.util.logging.Logger; +import org.archive.bdb.BenchmarkingBinding; +import org.archive.bdb.KryoBinding; +import org.archive.crawler.frontier.RecyclingSerialBinding; + import com.sleepycat.bind.EntryBinding; import com.sleepycat.bind.serial.SerialBinding; import com.sleepycat.bind.serial.StoredClassCatalog; @@ -173,7 +177,13 @@ implements ObjectIdentityCache, Closeable, Serializable { EntryBinding keyBinding = TupleBinding.getPrimitiveBinding(String.class); EntryBinding valueBinding = TupleBinding.getPrimitiveBinding(valueClass); if(valueBinding == null) { - valueBinding = new SerialBinding(classCatalog, valueClass); + valueBinding = + new KryoBinding(valueClass); +// new SerialBinding(classCatalog, valueClass); +// new BenchmarkingBinding(new EntryBinding[] { +// new KryoBinding(valueClass), +// new RecyclingSerialBinding(classCatalog, valueClass), +// }, valueClass); } return new StoredSortedMap(database, keyBinding, valueBinding, true); } diff --git a/commons/src/test/java/org/archive/util/CachedBdbMapTest.java b/commons/src/test/java/org/archive/util/CachedBdbMapTest.java deleted file mode 100644 index db2e5b9c..00000000 --- a/commons/src/test/java/org/archive/util/CachedBdbMapTest.java +++ /dev/null @@ -1,133 +0,0 @@ -/* - * This file is part of the Heritrix web crawler (crawler.archive.org). - * - * Licensed to the Internet Archive (IA) by one or more individual - * contributors. - * - * The IA licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.archive.util; - -import java.io.File; -import java.util.HashMap; -import java.util.logging.Handler; -import java.util.logging.Level; -import java.util.logging.Logger; - -import org.apache.commons.io.FileUtils; -import org.archive.bdb.BdbModule; - -/** - * Tests of CachedBdbMap - * - * @contributor stack - * @contributor gojomo - * @version $Date$, $Revision$ - */ -public class CachedBdbMapTest extends TmpDirTestCase { - File envDir; - BdbModule bdb; - private CachedBdbMap> cache; - - @SuppressWarnings("unchecked") - protected void setUp() throws Exception { - super.setUp(); - this.envDir = new File(getTmpDir(),"CachedBdbMapTest"); - this.envDir.mkdirs(); - bdb = new BdbModule(); - bdb.getDir().setBase(null); - bdb.getDir().setPath(envDir.getAbsolutePath()); - bdb.start(); - this.cache = bdb.getCBMMap( - this.getClass().getName(), false, String.class, HashMap.class); - } - - protected void tearDown() throws Exception { - ArchiveUtils.closeQuietly(this.cache); - bdb.stop(); - FileUtils.deleteDirectory(this.envDir); - super.tearDown(); - } - - public void testBackingDbGetsUpdated() { - // Enable all logging. Up the level on the handlers and then - // on the big map itself. - Handler [] handlers = Logger.getLogger("").getHandlers(); - for (int index = 0; index < handlers.length; index++) { - handlers[index].setLevel(Level.FINEST); - } - Logger.getLogger(CachedBdbMap.class.getName()). - setLevel(Level.FINEST); - // Set up values. - final String value = "value"; - final String key = "key"; - final int upperbound = 3; - // First put in empty hashmap. - for (int i = 0; i < upperbound; i++) { - assertNull("unexpected prior entry", - this.cache.putIfAbsent(key + Integer.toString(i), new HashMap())); - } - // Now add value to hash map. - for (int i = 0; i < upperbound; i++) { - HashMap m = this.cache.get(key + Integer.toString(i)); - m.put(key, value); - } - this.cache.sync(); - for (int i = 0; i < upperbound; i++) { - HashMap m = this.cache.get(key + Integer.toString(i)); - String v = m.get(key); - if (v == null || !v.equals(value)) { - Logger.getLogger(CachedBdbMap.class.getName()). - warning("Wrong value " + i); - } - } - } - - /** - * Test that in scarce memory conditions, the memory map is - * expunged of otherwise unreferenced entries as expected. - * - * NOTE: this test may be especially fragile with regard to - * GC/timing issues; relies on timely finalization, which is - * never guaranteed by JVM/GC. For example, it is so sensitive - * to CPU speed that a Thread.sleep(1000) succeeds when my - * laptop is plugged in, but fails when it is on battery! - * - * @throws InterruptedException - */ - public void testMemMapCleared() throws InterruptedException { - System.gc(); - System.runFinalization(); - System.gc(); - assertEquals(cache.memMap.size(), 0); - for(int i=0; i < 10000; i++) { - cache.putIfAbsent(""+i, new HashMap()); - } - assertEquals(10000, cache.memMap.size()); - assertEquals(10000, cache.size()); - TestUtils.forceScarceMemory(); - Thread.sleep(2000); - TestUtils.forceScarceMemory(); - Thread.sleep(2000); - // The 'canary' trick makes this explicit expunge, or - // an expunge triggered by a get() or put...(), unnecessary - //cache.expungeStaleEntries(); - System.out.println(cache.size()+","+cache.memMap.size()); - assertEquals(0, cache.memMap.size()); - } - - public static void main(String [] args) { - junit.textui.TestRunner.run(CachedBdbMapTest.class); - } -} diff --git a/engine/src/main/java/org/archive/crawler/frontier/BdbFrontier.java b/engine/src/main/java/org/archive/crawler/frontier/BdbFrontier.java index f6e13775..dee0c3a3 100644 --- a/engine/src/main/java/org/archive/crawler/frontier/BdbFrontier.java +++ b/engine/src/main/java/org/archive/crawler/frontier/BdbFrontier.java @@ -153,9 +153,9 @@ implements Checkpointable, BeanNameAware { WorkQueue wq = allQueues.getOrUse( classKey, new Supplier() { - public WorkQueue get() { + public BdbWorkQueue get() { String qKey = new String(classKey); // ensure private minimal key - WorkQueue q = new BdbWorkQueue(qKey, BdbFrontier.this); + BdbWorkQueue q = new BdbWorkQueue(qKey, BdbFrontier.this); q.setTotalBudget(getQueueTotalBudget()); getQueuePrecedencePolicy().queueCreated(q); return q; @@ -257,7 +257,7 @@ implements Checkpointable, BeanNameAware { @Override protected void initAllQueues() throws DatabaseException { boolean isRecovery = (recoveryCheckpoint != null); - this.allQueues = bdb.getObjectCache("allqueues", isRecovery, WorkQueue.class); + this.allQueues = bdb.getObjectCache("allqueues", isRecovery, WorkQueue.class, BdbWorkQueue.class); if(isRecovery) { JSONObject json = recoveryCheckpoint.loadJson(beanName); try { diff --git a/engine/src/main/java/org/archive/crawler/frontier/BdbMultipleWorkQueues.java b/engine/src/main/java/org/archive/crawler/frontier/BdbMultipleWorkQueues.java index a13cc06e..ac5c8a03 100644 --- a/engine/src/main/java/org/archive/crawler/frontier/BdbMultipleWorkQueues.java +++ b/engine/src/main/java/org/archive/crawler/frontier/BdbMultipleWorkQueues.java @@ -31,9 +31,11 @@ import javax.management.openmbean.CompositeDataSupport; import javax.management.openmbean.OpenDataException; import org.apache.commons.collections.Closure; +import org.archive.bdb.KryoBinding; import org.archive.modules.CrawlURI; import org.archive.util.ArchiveUtils; +import com.sleepycat.bind.EntryBinding; import com.sleepycat.bind.serial.StoredClassCatalog; import com.sleepycat.je.Cursor; import com.sleepycat.je.Database; @@ -65,7 +67,7 @@ public class BdbMultipleWorkQueues { private Database pendingUrisDB = null; /** Supporting bdb serialization of CrawlURIs */ - private RecyclingSerialBinding crawlUriBinding; + private EntryBinding crawlUriBinding; /** * Create the multi queue in the given environment. @@ -80,7 +82,14 @@ public class BdbMultipleWorkQueues { throws DatabaseException { this.pendingUrisDB = db; crawlUriBinding = - new RecyclingSerialBinding(classCatalog, CrawlURI.class); + new KryoBinding(CrawlURI.class); +// new RecyclingSerialBinding(classCatalog, CrawlURI.class); +// new BenchmarkingBinding(new EntryBinding[] { +// new KryoBinding(CrawlURI.class,true), +// new KryoBinding(CrawlURI.class,false), +// new RecyclingSerialBinding(classCatalog, CrawlURI.class), +// }); + } /** diff --git a/engine/src/main/java/org/archive/crawler/frontier/BdbWorkQueue.java b/engine/src/main/java/org/archive/crawler/frontier/BdbWorkQueue.java index 6e5a2d5d..0d1ab07a 100644 --- a/engine/src/main/java/org/archive/crawler/frontier/BdbWorkQueue.java +++ b/engine/src/main/java/org/archive/crawler/frontier/BdbWorkQueue.java @@ -21,10 +21,14 @@ package org.archive.crawler.frontier; import java.io.IOException; import java.io.Serializable; import java.io.UnsupportedEncodingException; +import java.util.HashSet; import java.util.logging.Level; import java.util.logging.Logger; +import org.archive.bdb.AutoKryo; +import org.archive.crawler.frontier.precedence.SimplePrecedenceProvider; import org.archive.modules.CrawlURI; +import org.archive.modules.fetcher.FetchStats; import org.archive.util.ArchiveUtils; import com.sleepycat.je.DatabaseEntry; @@ -166,4 +170,14 @@ implements Serializable { return e.getMessage(); } } + + // Kryo support + public static void autoregisterTo(AutoKryo kryo) { + kryo.register(BdbWorkQueue.class); + kryo.autoregister(FetchStats.class); + kryo.autoregister(HashSet.class); + kryo.autoregister(SimplePrecedenceProvider.class); + kryo.autoregister(byte[].class); + kryo.setRegistrationOptional(true); + } } \ No newline at end of file diff --git a/engine/src/main/java/org/archive/crawler/reporting/StatisticsTracker.java b/engine/src/main/java/org/archive/crawler/reporting/StatisticsTracker.java index 3597d394..f84519b2 100644 --- a/engine/src/main/java/org/archive/crawler/reporting/StatisticsTracker.java +++ b/engine/src/main/java/org/archive/crawler/reporting/StatisticsTracker.java @@ -331,8 +331,9 @@ public class StatisticsTracker new ObjectIdentityMemCache(); // temp dummy /** Keep track of URL counts per host per seed */ - protected ObjectIdentityCache> sourceHostDistribution = - new ObjectIdentityMemCache>(); // temp dummy; + @SuppressWarnings("unchecked") + protected ObjectIdentityCache sourceHostDistribution = + new ObjectIdentityMemCache(); // temp dummy; /* Keep track of 'top' hosts for live reports */ protected TopNSet hostsDistributionTop; @@ -503,6 +504,10 @@ public class StatisticsTracker return snapshot; } + public LinkedList getSnapshots() { + return snapshots; + } + public CrawlStatSnapshot getLastSnapshot() { CrawlStatSnapshot snap = snapshots.peek(); return snap == null ? getSnapshot() : snap; @@ -840,12 +845,13 @@ public class StatisticsTracker } } + @SuppressWarnings("unchecked") protected void saveSourceStats(String source, String hostname) { synchronized(sourceHostDistribution) { ConcurrentMap hostUriCount = sourceHostDistribution.getOrUse( source, - new Supplier>() { + new Supplier() { public ConcurrentMap get() { return new ConcurrentHashMap(); }}); @@ -1121,6 +1127,7 @@ public class StatisticsTracker // CrawlController's only interest is in knowing that a Checkpoint is // being recovered public void startCheckpoint(Checkpoint checkpointInProgress) {} + @SuppressWarnings("unchecked") public void doCheckpoint(Checkpoint checkpointInProgress) throws IOException { JSONObject json = new JSONObject(); try { diff --git a/modules/src/main/java/org/archive/modules/CrawlURI.java b/modules/src/main/java/org/archive/modules/CrawlURI.java index 8c299b7c..1f9b32d3 100644 --- a/modules/src/main/java/org/archive/modules/CrawlURI.java +++ b/modules/src/main/java/org/archive/modules/CrawlURI.java @@ -77,6 +77,7 @@ import org.apache.commons.httpclient.URIException; import org.apache.commons.httpclient.methods.GetMethod; import org.apache.commons.httpclient.methods.PostMethod; import org.apache.commons.lang.StringUtils; +import org.archive.bdb.AutoKryo; import org.archive.modules.credential.CredentialAvatar; import org.archive.modules.credential.HttpAuthenticationCredential; import org.archive.modules.extractor.HTMLLinkContext; @@ -116,7 +117,7 @@ implements MultiReporter, Serializable, OverlayContext { /** * The URI being crawled. It's transient to save space when storing to BDB. */ - private transient UURI uuri; + private UURI uuri; /** Seed status */ @@ -138,7 +139,7 @@ implements MultiReporter, Serializable, OverlayContext { * Where this URI was (presently) discovered. . Transient to allow * more efficient custom serialization */ - private transient UURI via; + private UURI via; /** * Context of URI's discovery, as per the 'context' in Link @@ -199,7 +200,7 @@ implements MultiReporter, Serializable, OverlayContext { * * Package-protected so CrawlURI can access it directly. */ - transient Map data; + Map data; // private transient SheetManager manager; @@ -646,7 +647,7 @@ implements MultiReporter, Serializable, OverlayContext { * This methods removes the attribute list. */ public void stripToMinimal() { - data = null; + data = null; } /** @@ -670,13 +671,13 @@ implements MultiReporter, Serializable, OverlayContext { * @return the annotations set for this uri. */ public Collection getAnnotations() { - @SuppressWarnings("unchecked") - List list = (List)getData().get(A_ANNOTATIONS); + @SuppressWarnings("unchecked") + List list = (List)getData().get(A_ANNOTATIONS); if (list == null) { list = new ArrayList(); getData().put(A_ANNOTATIONS, list); } - return list; + return list; } /** @@ -867,20 +868,20 @@ implements MultiReporter, Serializable, OverlayContext { } public Map getPersistentDataMap() { - if (data == null) { - return null; - } - Map result = new HashMap(getData()); - Set retain = new HashSet(persistentKeys); - - if (containsDataKey(A_HERITABLE_KEYS)) { - @SuppressWarnings("unchecked") + if (data == null) { + return null; + } + Map result = new HashMap(getData()); + Set retain = new HashSet(persistentKeys); + + if (containsDataKey(A_HERITABLE_KEYS)) { + @SuppressWarnings("unchecked") HashSet heritable = (HashSet)getData().get(A_HERITABLE_KEYS); retain.addAll(heritable); - } - - result.keySet().retainAll(retain); - return result; + } + + result.keySet().retainAll(retain); + return result; } /** @@ -954,26 +955,26 @@ implements MultiReporter, Serializable, OverlayContext { * @see #isSuccess() */ public boolean is2XXSuccess() { - return this.fetchStatus >= 200 && this.fetchStatus < 300; + return this.fetchStatus >= 200 && this.fetchStatus < 300; } /** - * @return True if we have an rfc2617 payload. - */ - public boolean hasRfc2617CredentialAvatar() { - boolean result = false; - Set avatars = getCredentialAvatars(); - if (avatars != null && avatars.size() > 0) { - for (Iterator i = avatars.iterator(); i.hasNext();) { - if (((CredentialAvatar)i.next()). - match(HttpAuthenticationCredential.class)) { - result = true; - break; - } - } - } + * @return True if we have an rfc2617 payload. + */ + public boolean hasRfc2617CredentialAvatar() { + boolean result = false; + Set avatars = getCredentialAvatars(); + if (avatars != null && avatars.size() > 0) { + for (Iterator i = avatars.iterator(); i.hasNext();) { + if (((CredentialAvatar)i.next()). + match(HttpAuthenticationCredential.class)) { + result = true; + break; + } + } + } return result; - } + } /** @@ -1153,7 +1154,7 @@ implements MultiReporter, Serializable, OverlayContext { * @param key Key to add. */ public static Collection getPersistentDataKeys() { - return persistentKeys; + return persistentKeys; } public void addPersistentDataMapKey(String s) { @@ -1180,49 +1181,21 @@ implements MultiReporter, Serializable, OverlayContext { return persistentKeys.remove(key); } - /** - * Custom serialization writing an empty 'outLinks' as null. Estimated - * to save ~20 bytes in serialized form. - * - * @param stream - * @throws IOException - */ private void writeObject(ObjectOutputStream stream) throws IOException { stream.defaultWriteObject(); - stream.writeUTF(uuri.toCustomString()); // - stream.writeObject((via == null) ? null : via.getURI()); stream.writeObject((data==null || data.isEmpty()) ? null : data); - stream.writeObject((outLinks.isEmpty()) ? null : outLinks); - stream.writeObject((outCandidates.isEmpty()) ? null : outCandidates); - } - - /** - * Custom deserialization recreating empty HashSet from null in 'outLinks' - * slot. - * - * @param stream - * @throws IOException - * @throws ClassNotFoundException - */ + } + private void readObject(ObjectInputStream stream) throws IOException, - ClassNotFoundException { + ClassNotFoundException { stream.defaultReadObject(); - uuri = readUuri(stream.readUTF()); - via = readUuri((String)stream.readObject()); @SuppressWarnings("unchecked") Map temp = (Map)stream.readObject(); this.data = temp; - - @SuppressWarnings("unchecked") - HashSet ol = (HashSet) stream.readObject(); - outLinks = (ol == null) ? new HashSet() : ol; - @SuppressWarnings("unchecked") - HashSet oc = (HashSet)stream.readObject(); - outCandidates = (oc == null) ? new HashSet() : oc; + outLinks = new HashSet(); + outCandidates = new HashSet(); } - - /** * Read a UURI from a String, handling a null or URIException * @@ -1295,15 +1268,15 @@ implements MultiReporter, Serializable, OverlayContext { } public Collection getNonFatalFailures() { - @SuppressWarnings("unchecked") - List list = (List)getData().get(A_NONFATAL_ERRORS); - if (list == null) { - list = new ArrayList(); - getData().put(A_NONFATAL_ERRORS, list); - } - - // FIXME: Previous code automatically added annotation when "localized error" - // was added, override collection to implement that? + @SuppressWarnings("unchecked") + List list = (List)getData().get(A_NONFATAL_ERRORS); + if (list == null) { + list = new ArrayList(); + getData().put(A_NONFATAL_ERRORS, list); + } + + // FIXME: Previous code automatically added annotation when "localized error" + // was added, override collection to implement that? return list; } @@ -1318,7 +1291,7 @@ implements MultiReporter, Serializable, OverlayContext { } public void setFetchBeginTime(long time) { - getData().put(ModuleAttributeConstants.A_FETCH_BEGAN_TIME, time); + getData().put(ModuleAttributeConstants.A_FETCH_BEGAN_TIME, time); } public void setFetchCompletedTime(long time) { @@ -1837,4 +1810,20 @@ implements MultiReporter, Serializable, OverlayContext { return containsDataKey(A_FORCE_RETIRE) && (Boolean)getData().get(A_FORCE_RETIRE); } + + // Kryo support + @SuppressWarnings("unused") + private CrawlURI() {} + public static void autoregisterTo(AutoKryo kryo) { +// kryo.register(CrawlURI.class,new DeflateCompressor(kryo.newSerializer(CrawlURI.class))); + kryo.register(CrawlURI.class); + kryo.autoregister(byte[].class); + kryo.autoregister(java.util.HashSet.class); + kryo.autoregister(java.util.HashMap.class); + kryo.autoregister(org.archive.net.UURI.class); + kryo.autoregister(org.archive.modules.extractor.HTMLLinkContext.class); + kryo.autoregister(org.archive.modules.extractor.LinkContext.SimpleLinkContext.class); + kryo.autoregister(java.util.HashMap[].class); + kryo.setRegistrationOptional(true); + } } diff --git a/modules/src/main/java/org/archive/modules/extractor/LinkContext.java b/modules/src/main/java/org/archive/modules/extractor/LinkContext.java index 61fc5a14..b41e7962 100644 --- a/modules/src/main/java/org/archive/modules/extractor/LinkContext.java +++ b/modules/src/main/java/org/archive/modules/extractor/LinkContext.java @@ -32,7 +32,7 @@ public abstract class LinkContext implements Serializable { /** Class for representing handy default LinkContext values. */ - private static class SimpleLinkContext extends LinkContext { + public static class SimpleLinkContext extends LinkContext { private static final long serialVersionUID = 1L; diff --git a/modules/src/main/java/org/archive/modules/net/BdbServerCache.java b/modules/src/main/java/org/archive/modules/net/BdbServerCache.java index a810b508..2c5d8320 100644 --- a/modules/src/main/java/org/archive/modules/net/BdbServerCache.java +++ b/modules/src/main/java/org/archive/modules/net/BdbServerCache.java @@ -50,8 +50,8 @@ implements Lifecycle { return; } try { - this.servers = bdb.getObjectCache("servers", false, CrawlServer.class); - this.hosts = bdb.getObjectCache("hosts", false, CrawlHost.class); + this.servers = bdb.getObjectCache("servers", false, CrawlServer.class, CrawlServer.class); + this.hosts = bdb.getObjectCache("hosts", false, CrawlHost.class, CrawlHost.class); } catch (DatabaseException e) { throw new IllegalStateException(e); } diff --git a/modules/src/main/java/org/archive/modules/net/CrawlHost.java b/modules/src/main/java/org/archive/modules/net/CrawlHost.java index 4c96299e..180fed9a 100644 --- a/modules/src/main/java/org/archive/modules/net/CrawlHost.java +++ b/modules/src/main/java/org/archive/modules/net/CrawlHost.java @@ -20,10 +20,12 @@ package org.archive.modules.net; import java.io.Serializable; +import java.net.Inet4Address; import java.net.InetAddress; import java.util.logging.Level; import java.util.logging.Logger; +import org.archive.bdb.AutoKryo; import org.archive.modules.fetcher.FetchStats; import org.archive.util.InetAddressUtil; @@ -223,4 +225,13 @@ public class CrawlHost implements Serializable, FetchStats.HasFetchStats { public FetchStats getSubstats() { return substats; } + + // Kryo support +// public CrawlHost() {} + public static void autoregisterTo(AutoKryo kryo) { + kryo.register(CrawlHost.class); + kryo.autoregister(FetchStats.class); + kryo.autoregister(Inet4Address.class); + kryo.setRegistrationOptional(true); + } } diff --git a/modules/src/main/java/org/archive/modules/net/CrawlServer.java b/modules/src/main/java/org/archive/modules/net/CrawlServer.java index 6ab10743..5d71ebde 100644 --- a/modules/src/main/java/org/archive/modules/net/CrawlServer.java +++ b/modules/src/main/java/org/archive/modules/net/CrawlServer.java @@ -37,6 +37,7 @@ import org.apache.commons.collections.Predicate; import org.apache.commons.httpclient.NoHttpResponseException; import org.apache.commons.httpclient.URIException; import org.apache.commons.io.IOUtils; +import org.archive.bdb.AutoKryo; import org.archive.io.ReplayInputStream; import org.archive.modules.CrawlURI; import org.archive.modules.credential.CredentialAvatar; @@ -62,7 +63,7 @@ public class CrawlServer implements Serializable, FetchStats.HasFetchStats { * after this many tries */ public static final long MIN_ROBOTS_RETRIES = 3; - private final String server; // actually, host+port in the https case + private String server; // actually, host+port in the https case private int port; protected Robotstxt robotstxt; long robotsFetched = ROBOTS_NOT_FETCHED; @@ -324,4 +325,17 @@ public class CrawlServer implements Serializable, FetchStats.HasFetchStats { } return false; } + + // Kryo support +// public CrawlServer() {} + public static void autoregisterTo(AutoKryo kryo) { + kryo.register(CrawlServer.class); + kryo.autoregister(FetchStats.class); + kryo.autoregister(org.archive.modules.net.Robotstxt.class); + kryo.autoregister(java.util.HashMap.class); + kryo.autoregister(org.archive.modules.net.RobotsDirectives.class); + kryo.autoregister(org.archive.util.PrefixSet.class); + kryo.autoregister(java.util.LinkedList.class); + kryo.setRegistrationOptional(true); + } }