diff --git a/commons/pom.xml b/commons/pom.xml index 2a679195..66b60e88 100644 --- a/commons/pom.xml +++ b/commons/pom.xml @@ -182,12 +182,6 @@ runtime - - com.google.guava - guava - r08 - - net.java.dev.jna jna diff --git a/commons/src/main/java/org/archive/util/ObjectIdentityBdbManualCache.java b/commons/src/main/java/org/archive/util/ObjectIdentityBdbManualCache.java index 84504e79..81ec0f7c 100644 --- a/commons/src/main/java/org/archive/util/ObjectIdentityBdbManualCache.java +++ b/commons/src/main/java/org/archive/util/ObjectIdentityBdbManualCache.java @@ -33,8 +33,9 @@ import java.util.logging.Logger; import org.archive.bdb.KryoBinding; -import com.google.common.collect.MapEvictionListener; -import com.google.common.collect.MapMaker; +import com.google.common.cache.CacheBuilder; +import com.google.common.cache.RemovalListener; +import com.google.common.cache.RemovalNotification; import com.sleepycat.bind.EntryBinding; import com.sleepycat.bind.serial.StoredClassCatalog; import com.sleepycat.bind.tuple.TupleBinding; @@ -64,8 +65,9 @@ import com.sleepycat.je.Environment; * @author paul baclace (conversion to ConcurrentMap) * */ -public class ObjectIdentityBdbManualCache -implements ObjectIdentityCache, Closeable, Serializable, MapEvictionListener { +@SuppressWarnings("ALL") +public class ObjectIdentityBdbManualCache +implements ObjectIdentityCache, Closeable, Serializable { private static final long serialVersionUID = 1L; private static final Logger logger = Logger.getLogger(ObjectIdentityBdbManualCache.class.getName()); @@ -110,6 +112,18 @@ implements ObjectIdentityCache, Closeable, Serializable, MapEvictionListener< */ public ObjectIdentityBdbManualCache() { super(); + dirtyItems = CacheBuilder.newBuilder() + .maximumSize(10000) + .expireAfterWrite(5, TimeUnit.MINUTES) + .removalListener(new RemovalListener() { + @Override + public void onRemoval(RemovalNotification stringVRemovalNotification) { + evictions.incrementAndGet(); + diskMap.put(stringVRemovalNotification.getKey(), stringVRemovalNotification.getValue()); + } + }) + .build() + .asMap(); } /** @@ -127,16 +141,18 @@ implements ObjectIdentityCache, Closeable, Serializable, MapEvictionListener< public void initialize(final Environment env, String dbName, final Class valueClass, final StoredClassCatalog classCatalog) throws DatabaseException { - // TODO: tune capacity for actual threads, expected size of key caches? - this.memMap = new MapMaker().concurrencyLevel(64).initialCapacity(8192).softValues().makeMap(); + // TODO: tune capacity for actual threads, expected size of key caches? + this.memMap = CacheBuilder.newBuilder() + .concurrencyLevel(64) + .initialCapacity(8192) + .softValues() + .build() + .asMap(); this.db = openDatabase(env, dbName); this.diskMap = createDiskMap(this.db, classCatalog, valueClass); // keep a record of items that must be persisted; auto-persist if // unchanged after 5 minutes, or more than 10K would collect - this.dirtyItems = new MapMaker().concurrencyLevel(64) - .maximumSize(10000).expireAfterWrite(5,TimeUnit.MINUTES) - .evictionListener(this).makeMap(); - + this.count = new AtomicLong(diskMap.size()); } @@ -363,9 +379,9 @@ implements ObjectIdentityCache, Closeable, Serializable, MapEvictionListener< dirtyItems.put(key,val); } - @Override - public void onEviction(String key, V val) { + /*@Override + public void onRemoval(RemovalNotification stringVRemovalNotification) { evictions.incrementAndGet(); - diskMap.put(key, val); - } + diskMap.put(stringVRemovalNotification.getKey(), stringVRemovalNotification.getValue()); + }*/ } diff --git a/commons/src/main/java/org/archive/util/TestUtils.java b/commons/src/main/java/org/archive/util/TestUtils.java index 889492a1..a1237483 100644 --- a/commons/src/main/java/org/archive/util/TestUtils.java +++ b/commons/src/main/java/org/archive/util/TestUtils.java @@ -127,7 +127,7 @@ public class TestUtils { } cname = cname.replace(File.separatorChar, '.'); cname = cname.substring(0, cname.length() - 5); - suite.addTestSuite(Class.forName(cname)); + suite.addTestSuite((Class) Class.forName(cname)); } } } diff --git a/engine/src/main/java/org/archive/crawler/util/TopNSet.java b/engine/src/main/java/org/archive/crawler/util/TopNSet.java index 4a8b2599..0c1d90ed 100644 --- a/engine/src/main/java/org/archive/crawler/util/TopNSet.java +++ b/engine/src/main/java/org/archive/crawler/util/TopNSet.java @@ -25,10 +25,9 @@ import java.util.SortedSet; import java.util.TreeSet; import java.util.concurrent.ConcurrentMap; +import com.google.common.cache.CacheBuilder; import org.archive.util.Histotable; -import com.google.common.collect.MapMaker; - /** * Counting Set which only remembers the 'top N' of all String values * reported (with counts) to it. Precise if counts reported for a @@ -56,7 +55,7 @@ public class TopNSet implements Serializable { public TopNSet(int size){ maxsize = size; - set = new MapMaker().concurrencyLevel(64).makeMap(); + set = CacheBuilder.newBuilder().concurrencyLevel(64).build().asMap(); } /** diff --git a/modules/src/main/java/org/archive/modules/forms/FormLoginProcessor.java b/modules/src/main/java/org/archive/modules/forms/FormLoginProcessor.java index 94716960..f2292f77 100644 --- a/modules/src/main/java/org/archive/modules/forms/FormLoginProcessor.java +++ b/modules/src/main/java/org/archive/modules/forms/FormLoginProcessor.java @@ -26,6 +26,9 @@ import java.util.concurrent.atomic.AtomicLong; import java.util.logging.Level; import java.util.logging.Logger; +import com.google.common.cache.CacheBuilder; +import com.google.common.cache.CacheLoader; +import com.google.common.cache.LoadingCache; import org.apache.commons.httpclient.URIException; import org.apache.commons.lang.StringUtils; import org.archive.checkpointing.Checkpointable; @@ -42,9 +45,6 @@ import org.json.JSONException; import org.json.JSONObject; import org.springframework.beans.factory.annotation.Autowired; -import com.google.common.base.Function; -import com.google.common.collect.MapMaker; - /** * A step, post-ExtractorHTMLForms, where a followup CrawlURI to * attempt a form submission may be synthesized. @@ -119,22 +119,24 @@ public class FormLoginProcessor extends Processor implements Checkpointable { Logger.getLogger(FormLoginProcessor.class.getName()); // formProvince (String) -> count - ConcurrentMap eligibleFormsSeenCount = - new MapMaker().makeComputingMap(new Function() { - @Override - public AtomicLong apply(String arg0) { - return new AtomicLong(0L); - } - }); + ConcurrentMap eligibleFormsSeenCount = + CacheBuilder.newBuilder() + .build( + new CacheLoader() { + public AtomicLong load(String arg0) { + return new AtomicLong(0L); + } + }).asMap(); // formProvince (String) -> count - ConcurrentMap eligibleFormsAttemptsCount = - new MapMaker().makeComputingMap(new Function() { - @Override - public AtomicLong apply(String arg0) { - return new AtomicLong(0L); - } - }); + ConcurrentMap eligibleFormsAttemptsCount = + CacheBuilder.newBuilder() + .build( + new CacheLoader() { + public AtomicLong load(String arg0) { + return new AtomicLong(0L); + } + }).asMap(); /** * SURT prefix against which configured username/password is