Minimize transient garbage volume by optimized serialization based on the 'Kryo' library

* AutoKryo
    extension of Kryo to allow classes to control their own registration, trigger registration of associated classes, and deserialize classes without no-arg constructors
* KryoBinding
    binding for use with BDB that uses AutoKryo serialization for a 2X-4X reduction in byte[] size
* UURI
    improved serialization via Externalizable and Kryo's CustomSerialization methods
* BdbModule
    discard deprecated CachedBDBMap option
    (getObjectCache) extend with both declaredClass and valueClass (for when map values are specializations of the declared type, as with frontier.allQueues)
    adjust type declarations
* ObjectIdentityBdbCache
    use KryoBinding rather than SerialBinding
* CachedBdbMapTest
    discarded
* BdbFrontier, BdbServerCache, StatisticsTracker
    adjust type declarations, objectCache creation
* BdbMultipleWorkQueues
    use KryoBinding rather than (Recycling)SerialBinding
* BdbWorkQueue, CrawlServer, CrawlHost, CrawlURI
    add autoregister support
* LinkContext
    public for kryo registration

commons/src/main/java/org/archive/bdb/AutoKryo.java

@@ -0,0 +1,83 @@
+package org.archive.bdb;
+
+import java.lang.reflect.Constructor;
+import java.lang.reflect.InvocationTargetException;
+import java.util.ArrayList;
+
+import sun.reflect.ReflectionFactory;
+
+import com.esotericsoftware.kryo.Kryo;
+import com.esotericsoftware.kryo.SerializationException;
+
+/**
+ * Extensions to Kryo to let classes control their own registration, suggest
+ * other classes to register together, and use the same (Sun-JVM-only) trick
+ * for deserializing classes without no-arg constructors.
+ * 
+ * TODO: more comments!
+ * 
+ * @contributor gojomo
+ */
+@SuppressWarnings("unchecked")
+public class AutoKryo extends Kryo {
+    ArrayList<Class> registeredClasses = new ArrayList<Class>(); 
+    
+    @Override
+    protected void handleUnregisteredClass(Class type) {
+        System.err.println("UNREGISTERED FOR KRYO "+type+" in "+registeredClasses.get(0));
+        super.handleUnregisteredClass(type);
+    }
+
+    public void autoregister(Class type) {
+        if (registeredClasses.contains(type)) {
+            return;
+        }
+        registeredClasses.add(type); 
+        try {
+            invokeStatic(
+                "autoregisterTo", 
+                type,
+                new Class[]{ ((Class)AutoKryo.class), }, 
+                new Object[] { this, });
+        } catch (Exception e) {
+            register(type); 
+        }
+    }
+
+    
+    @Override
+    public <T> T newInstance(Class<T> type) {
+        SerializationException ex = null; 
+        try {
+            return super.newInstance(type);
+        } catch (SerializationException se) {
+            ex = se;
+        }
+        try {
+            final Constructor<?> constructor = 
+            ReflectionFactory.getReflectionFactory().newConstructorForSerialization( 
+                    type, 
+                    Object.class.getDeclaredConstructor( new Class[0] ) );
+            constructor.setAccessible( true );
+            Object inst = constructor.newInstance( new Object[0] );
+            return (T) inst;
+        } catch (SecurityException e) {
+            e.printStackTrace();
+        } catch (NoSuchMethodException e) {
+            e.printStackTrace();
+        } catch (IllegalArgumentException e) {
+             e.printStackTrace();
+        } catch (InstantiationException e) {
+            e.printStackTrace();
+        } catch (IllegalAccessException e) {
+            e.printStackTrace();
+        } catch (InvocationTargetException e) {
+            e.printStackTrace();
+        }
+        throw ex;
+    }
+
+    protected Object invokeStatic(String method, Class clazz, Class[] types, Object[] args) throws Exception {
+        return clazz.getMethod(method, types).invoke(null, args);
+    }
+}

commons/src/main/java/org/archive/bdb/BdbModule.java

@@ -43,7 +43,6 @@ import org.apache.commons.io.filefilter.IOFileFilter;
 import org.archive.checkpointing.Checkpoint;
 import org.archive.checkpointing.Checkpointable;
 import org.archive.spring.ConfigPath;
-import org.archive.util.CachedBdbMap;
 import org.archive.util.ObjectIdentityBdbCache;
 import org.archive.util.ObjectIdentityCache;
 import org.archive.util.bdbje.EnhancedEnvironment;
@@ -77,7 +76,6 @@ public class BdbModule implements Lifecycle, Checkpointable, Closeable {
     private static class DatabasePlusConfig implements Serializable {
         private static final long serialVersionUID = 1L;
         public transient Database database;
-        public String name;
         public BdbConfig config;
     }
     
@@ -189,6 +187,7 @@ public class BdbModule implements Lifecycle, Checkpointable, Closeable {
         
     private transient StoredClassCatalog classCatalog;
     
+    @SuppressWarnings("unchecked")
     private Map<String,ObjectIdentityCache> oiCaches = 
         new ConcurrentHashMap<String,ObjectIdentityCache>();
 
@@ -337,7 +336,6 @@ public class BdbModule implements Lifecycle, Checkpointable, Closeable {
             }
         }
         dpc.database = bdbEnvironment.openDatabase(null, name, config.toDatabaseConfig());
-        dpc.name = name;
         dpc.config = config;
         databases.put(name, dpc);
         return dpc.database;
@@ -358,39 +356,7 @@ public class BdbModule implements Lifecycle, Checkpointable, Closeable {
         }
         
     }
-    
-    
-    /**
-     * Get a CachedBdbMap, backed by a BDB Database of the given name, 
-     * with the given key and value class types. If 'recycle' is true,
-     * reuse values already in the database; otherwise start with an 
-     * empty map.  
-     * 
-     * @param <K>
-     * @param <V>
-     * @param dbName
-     * @param recycle
-     * @param key
-     * @param value
-     * @return
-     * @throws DatabaseException
-     * @deprecated use ObjectIdentityBdbCache instead
-     */
-    public <K,V> CachedBdbMap<K,V> getCBMMap(String dbName, boolean recycle,
-            Class<? super K> key, Class<? super V> value) 
-    throws DatabaseException {
-        if (!recycle) {
-            try {
-                bdbEnvironment.truncateDatabase(null, dbName, false);
-            } catch (DatabaseNotFoundException e) {
-                // ignored
-            }
-        }
-        CachedBdbMap<K, V> r = new CachedBdbMap<K,V>(dbName);     
-        r.initialize(bdbEnvironment, key, value, classCatalog);
-        oiCaches.put(dbName, r);
-        return r;
-    }
+
 
     /**
      * Get an ObjectIdentityBdbCache, backed by a BDB Database of the 
@@ -406,7 +372,7 @@ public class BdbModule implements Lifecycle, Checkpointable, Closeable {
      * @throws DatabaseException
      */
     public <V> ObjectIdentityBdbCache<V> getOIBCCache(String dbName, boolean recycle,
-            Class<? super V> valueClass) 
+            Class<? extends V> valueClass) 
     throws DatabaseException {
         if (!recycle) {
             try {
@@ -420,11 +386,12 @@ public class BdbModule implements Lifecycle, Checkpointable, Closeable {
         oiCaches.put(dbName, oic);
         return oic;
     }
-
-    /** controls which alternate ObjectIdentityCache implementation to use */
-    private static boolean USE_OIBC = true;
-    
-
+  
+    public <V> ObjectIdentityCache<String, V> getObjectCache(String dbName, boolean recycle,
+            Class<V> valueClass) 
+    throws DatabaseException {
+        return getObjectCache(dbName, recycle, valueClass, valueClass);
+    }
     
     /**
      * Get an ObjectIdentityCache, backed by a BDB Database of the given 
@@ -440,18 +407,14 @@ public class BdbModule implements Lifecycle, Checkpointable, Closeable {
      * @throws DatabaseException
      */
     public <V> ObjectIdentityCache<String, V> getObjectCache(String dbName, boolean recycle,
-            Class<? super V> valueClass) 
+            Class<V> declaredClass, Class<? extends V> valueClass) 
     throws DatabaseException {
         @SuppressWarnings("unchecked")
         ObjectIdentityCache<String,V> oic = oiCaches.get(dbName);
         if(oic!=null) {
             return oic; 
         }
-        if(USE_OIBC) {
-            oic =  getOIBCCache(dbName, recycle, valueClass);
-        } else {
-            oic =  getCBMMap(dbName, recycle, String.class, valueClass);
-        }
+        oic =  getOIBCCache(dbName, recycle, valueClass);
         return oic; 
     }
     

commons/src/main/java/org/archive/bdb/KryoBinding.java

@@ -0,0 +1,75 @@
+/*
+ *  This file is part of the Heritrix web crawler (crawler.archive.org).
+ *
+ *  Licensed to the Internet Archive (IA) by one or more individual 
+ *  contributors. 
+ *
+ *  The IA licenses this file to You under the Apache License, Version 2.0
+ *  (the "License"); you may not use this file except in compliance with
+ *  the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ */
+package org.archive.bdb;
+
+import com.esotericsoftware.kryo.Kryo;
+import com.esotericsoftware.kryo.ObjectBuffer;
+import com.sleepycat.bind.EntryBinding;
+import com.sleepycat.je.DatabaseEntry;
+
+/**
+ * Binding for use with BerkeleyDB-JE that uses Kryo serialization rather
+ * than BDB's (custom version of) Java serialization.
+ * 
+ * @contributor gojomo
+ */
+public class KryoBinding<K> implements EntryBinding<K> {
+
+    protected Class<K> baseClass;
+    protected AutoKryo kryo = new AutoKryo(); 
+    protected ThreadLocal<ObjectBuffer> threadBuffer = new ThreadLocal<ObjectBuffer>() {
+        @Override
+        protected ObjectBuffer initialValue() {
+            return new ObjectBuffer(kryo,16*1024,Integer.MAX_VALUE);
+        }
+    };
+    
+    /**
+     * Constructor. Save parameters locally, as superclass 
+     * fields are private. 
+     * 
+     * @param classCatalog is the catalog to hold shared class information
+     *
+     * @param baseClass is the base class for serialized objects stored using
+     * this binding
+     */
+    @SuppressWarnings("unchecked")
+    public KryoBinding(Class baseClass) {
+        this.baseClass = baseClass;
+        kryo.autoregister(baseClass);
+    }
+
+    public Kryo getKryo() {
+        return kryo;
+    }
+    
+    /**
+     * Copies superclass simply to allow different source for FastOoutputStream.
+     * 
+     * @see com.sleepycat.bind.serial.SerialBinding#entryToObject
+     */
+    public void objectToEntry(K object, DatabaseEntry entry) {
+        entry.setData(threadBuffer.get().writeObjectData(object));
+    }
+
+    @Override
+    public K entryToObject(DatabaseEntry entry) {
+        return threadBuffer.get().readObjectData(entry.getData(), baseClass);
+    }
+}

commons/src/main/java/org/archive/net/UURI.java

@@ -18,15 +18,23 @@
  */
 package org.archive.net;
 
+import java.io.Externalizable;
 import java.io.File;
-import java.io.Serializable;
+import java.io.IOException;
+import java.io.ObjectInput;
+import java.io.ObjectOutput;
 import java.net.URI;
 import java.net.URISyntaxException;
+import java.nio.ByteBuffer;
 
 import org.apache.commons.httpclient.URIException;
 import org.archive.util.SURT;
 import org.archive.util.TextUtils;
 
+import com.esotericsoftware.kryo.CustomSerialization;
+import com.esotericsoftware.kryo.Kryo;
+import com.esotericsoftware.kryo.serialize.StringSerializer;
+
 
 /**
  * Usable URI.
@@ -51,7 +59,7 @@ import org.archive.util.TextUtils;
  * @see org.apache.commons.httpclient.URI
  */
 public class UURI extends LaxURI
-implements CharSequence, Serializable {
+implements CharSequence, Externalizable, CustomSerialization {
 
     private static final long serialVersionUID = -1277570889914647093L;
 
@@ -438,4 +446,35 @@ implements CharSequence, Serializable {
         }
         return (new File(path)).getName();
     }
+    
+    public void writeObjectData(Kryo kryo, ByteBuffer buffer) {
+        StringSerializer.put(buffer, toCustomString());
+    }
+
+    public void readObjectData (Kryo kryo, ByteBuffer buffer) {
+        try {
+            parseUriReference(StringSerializer.get(buffer),true);
+        } catch (URIException e) {
+            // TODO Auto-generated catch block
+            e.printStackTrace();
+        }
+    }
+
+    @Override
+    public void readExternal(ObjectInput in) throws IOException,
+            ClassNotFoundException {
+        try {
+            parseUriReference(in.readUTF(),true);
+        } catch (URIException e) {
+            // TODO Auto-generated catch block
+            e.printStackTrace();
+        }
+    }
+
+    @Override
+    public void writeExternal(ObjectOutput out) throws IOException {
+        out.writeUTF(toCustomString());
+    }
+    
+    
 }

commons/src/main/java/org/archive/util/ObjectIdentityBdbCache.java

@@ -33,6 +33,10 @@ import java.util.concurrent.atomic.AtomicLong;
 import java.util.logging.Level;
 import java.util.logging.Logger;
 
+import org.archive.bdb.BenchmarkingBinding;
+import org.archive.bdb.KryoBinding;
+import org.archive.crawler.frontier.RecyclingSerialBinding;
+
 import com.sleepycat.bind.EntryBinding;
 import com.sleepycat.bind.serial.SerialBinding;
 import com.sleepycat.bind.serial.StoredClassCatalog;
@@ -173,7 +177,13 @@ implements ObjectIdentityCache<String, V>, Closeable, Serializable {
         EntryBinding keyBinding = TupleBinding.getPrimitiveBinding(String.class);
         EntryBinding valueBinding = TupleBinding.getPrimitiveBinding(valueClass);
         if(valueBinding == null) {
-            valueBinding = new SerialBinding(classCatalog, valueClass);
+            valueBinding = 
+                new KryoBinding<V>(valueClass);
+//                new SerialBinding(classCatalog, valueClass);
+//                new BenchmarkingBinding<V>(new EntryBinding[] {
+//                      new KryoBinding<V>(valueClass),                   
+//                      new RecyclingSerialBinding<V>(classCatalog, valueClass),
+//                  }, valueClass);
         }
         return new StoredSortedMap<String,V>(database, keyBinding, valueBinding, true);
     }