From 2cac0ebf801a2d398e00aa6dbdcf8e77519d2e5c Mon Sep 17 00:00:00 2001 From: Shriphani Palakodety Date: Thu, 13 Mar 2014 15:26:57 -0400 Subject: [PATCH 01/16] bumped guava version to 11 --- .../util/ObjectIdentityBdbManualCache.java | 25 +++++++++++-------- 1 file changed, 14 insertions(+), 11 deletions(-) diff --git a/commons/src/main/java/org/archive/util/ObjectIdentityBdbManualCache.java b/commons/src/main/java/org/archive/util/ObjectIdentityBdbManualCache.java index 84504e79..2d1410ea 100644 --- a/commons/src/main/java/org/archive/util/ObjectIdentityBdbManualCache.java +++ b/commons/src/main/java/org/archive/util/ObjectIdentityBdbManualCache.java @@ -26,15 +26,16 @@ import java.util.Iterator; import java.util.Set; import java.util.Map.Entry; import java.util.concurrent.ConcurrentMap; -import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicLong; import java.util.logging.Level; import java.util.logging.Logger; import org.archive.bdb.KryoBinding; -import com.google.common.collect.MapEvictionListener; import com.google.common.collect.MapMaker; +import com.google.common.cache.CacheBuilder; +import com.google.common.cache.RemovalListener; +import com.google.common.cache.RemovalNotification; import com.sleepycat.bind.EntryBinding; import com.sleepycat.bind.serial.StoredClassCatalog; import com.sleepycat.bind.tuple.TupleBinding; @@ -64,8 +65,9 @@ import com.sleepycat.je.Environment; * @author paul baclace (conversion to ConcurrentMap) * */ -public class ObjectIdentityBdbManualCache -implements ObjectIdentityCache, Closeable, Serializable, MapEvictionListener { +@SuppressWarnings("ALL") +public class ObjectIdentityBdbManualCache +implements ObjectIdentityCache, Closeable, Serializable, RemovalListener { private static final long serialVersionUID = 1L; private static final Logger logger = Logger.getLogger(ObjectIdentityBdbManualCache.class.getName()); @@ -110,6 +112,10 @@ implements ObjectIdentityCache, Closeable, Serializable, MapEvictionListener< */ public ObjectIdentityBdbManualCache() { super(); + dirtyItems = CacheBuilder.newBuilder() + .maximumSize(10000) + .build() + .asMap(); } /** @@ -128,15 +134,12 @@ implements ObjectIdentityCache, Closeable, Serializable, MapEvictionListener< final Class valueClass, final StoredClassCatalog classCatalog) throws DatabaseException { // TODO: tune capacity for actual threads, expected size of key caches? - this.memMap = new MapMaker().concurrencyLevel(64).initialCapacity(8192).softValues().makeMap(); + this.memMap = new MapMaker().concurrencyLevel(64).initialCapacity(8192).softValues().makeMap(); this.db = openDatabase(env, dbName); this.diskMap = createDiskMap(this.db, classCatalog, valueClass); // keep a record of items that must be persisted; auto-persist if // unchanged after 5 minutes, or more than 10K would collect - this.dirtyItems = new MapMaker().concurrencyLevel(64) - .maximumSize(10000).expireAfterWrite(5,TimeUnit.MINUTES) - .evictionListener(this).makeMap(); - + this.count = new AtomicLong(diskMap.size()); } @@ -364,8 +367,8 @@ implements ObjectIdentityCache, Closeable, Serializable, MapEvictionListener< } @Override - public void onEviction(String key, V val) { + public void onRemoval(RemovalNotification stringVRemovalNotification) { evictions.incrementAndGet(); - diskMap.put(key, val); + diskMap.put(stringVRemovalNotification.getKey(), stringVRemovalNotification.getValue()); } } From ee6a3bddb30e22134eaa7b171e44e9aa8d803bc7 Mon Sep 17 00:00:00 2001 From: Shriphani Palakodety Date: Thu, 13 Mar 2014 15:29:37 -0400 Subject: [PATCH 02/16] forgot to add expireAfterWrite --- .../java/org/archive/util/ObjectIdentityBdbManualCache.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/commons/src/main/java/org/archive/util/ObjectIdentityBdbManualCache.java b/commons/src/main/java/org/archive/util/ObjectIdentityBdbManualCache.java index 2d1410ea..7ebefe77 100644 --- a/commons/src/main/java/org/archive/util/ObjectIdentityBdbManualCache.java +++ b/commons/src/main/java/org/archive/util/ObjectIdentityBdbManualCache.java @@ -26,6 +26,7 @@ import java.util.Iterator; import java.util.Set; import java.util.Map.Entry; import java.util.concurrent.ConcurrentMap; +import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicLong; import java.util.logging.Level; import java.util.logging.Logger; @@ -114,6 +115,7 @@ implements ObjectIdentityCache, Closeable, Serializable, RemovalListenerbuild() .asMap(); } From 25ce16e8930b1ec98513f7bc5cd311c6a5b33b13 Mon Sep 17 00:00:00 2001 From: Shriphani Palakodety Date: Thu, 13 Mar 2014 20:02:58 -0400 Subject: [PATCH 03/16] bump version in pom.xml --- commons/pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/commons/pom.xml b/commons/pom.xml index c894bb4f..9bdbb34f 100644 --- a/commons/pom.xml +++ b/commons/pom.xml @@ -185,7 +185,7 @@ com.google.guava guava - r08 + 16.0.1 From ad8cfd7075bfe9266b0d909dd30b18bcc2a9ca4e Mon Sep 17 00:00:00 2001 From: Shriphani Palakodety Date: Thu, 13 Mar 2014 21:50:06 -0400 Subject: [PATCH 04/16] prevent intellij from complaining --- commons/src/main/java/org/archive/util/TestUtils.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/commons/src/main/java/org/archive/util/TestUtils.java b/commons/src/main/java/org/archive/util/TestUtils.java index 889492a1..a1237483 100644 --- a/commons/src/main/java/org/archive/util/TestUtils.java +++ b/commons/src/main/java/org/archive/util/TestUtils.java @@ -127,7 +127,7 @@ public class TestUtils { } cname = cname.replace(File.separatorChar, '.'); cname = cname.substring(0, cname.length() - 5); - suite.addTestSuite(Class.forName(cname)); + suite.addTestSuite((Class) Class.forName(cname)); } } } From abd61abbf2a2cf9c724e2ebbf0112d5153ff1ed6 Mon Sep 17 00:00:00 2001 From: Shriphani Palakodety Date: Fri, 14 Mar 2014 15:13:34 -0400 Subject: [PATCH 05/16] removed guava dependency from heritrix. use the webarchive-commons version --- commons/pom.xml | 6 ------ 1 file changed, 6 deletions(-) diff --git a/commons/pom.xml b/commons/pom.xml index 9bdbb34f..d0730efb 100644 --- a/commons/pom.xml +++ b/commons/pom.xml @@ -182,12 +182,6 @@ runtime - - com.google.guava - guava - 16.0.1 - - net.java.dev.jna jna From 439f78b038a78c1940f1ce22781816514a0e7189 Mon Sep 17 00:00:00 2001 From: Shriphani Palakodety Date: Fri, 14 Mar 2014 15:36:22 -0400 Subject: [PATCH 06/16] migration to latest guava complete --- .../util/ObjectIdentityBdbManualCache.java | 10 ++++-- .../org/archive/crawler/util/TopNSet.java | 5 ++- .../modules/forms/FormLoginProcessor.java | 36 ++++++++++--------- 3 files changed, 28 insertions(+), 23 deletions(-) diff --git a/commons/src/main/java/org/archive/util/ObjectIdentityBdbManualCache.java b/commons/src/main/java/org/archive/util/ObjectIdentityBdbManualCache.java index 7ebefe77..f0d43dc8 100644 --- a/commons/src/main/java/org/archive/util/ObjectIdentityBdbManualCache.java +++ b/commons/src/main/java/org/archive/util/ObjectIdentityBdbManualCache.java @@ -33,7 +33,6 @@ import java.util.logging.Logger; import org.archive.bdb.KryoBinding; -import com.google.common.collect.MapMaker; import com.google.common.cache.CacheBuilder; import com.google.common.cache.RemovalListener; import com.google.common.cache.RemovalNotification; @@ -135,8 +134,13 @@ implements ObjectIdentityCache, Closeable, Serializable, RemovalListenerbuild() + .asMap(); this.db = openDatabase(env, dbName); this.diskMap = createDiskMap(this.db, classCatalog, valueClass); // keep a record of items that must be persisted; auto-persist if diff --git a/engine/src/main/java/org/archive/crawler/util/TopNSet.java b/engine/src/main/java/org/archive/crawler/util/TopNSet.java index 4a8b2599..0c1d90ed 100644 --- a/engine/src/main/java/org/archive/crawler/util/TopNSet.java +++ b/engine/src/main/java/org/archive/crawler/util/TopNSet.java @@ -25,10 +25,9 @@ import java.util.SortedSet; import java.util.TreeSet; import java.util.concurrent.ConcurrentMap; +import com.google.common.cache.CacheBuilder; import org.archive.util.Histotable; -import com.google.common.collect.MapMaker; - /** * Counting Set which only remembers the 'top N' of all String values * reported (with counts) to it. Precise if counts reported for a @@ -56,7 +55,7 @@ public class TopNSet implements Serializable { public TopNSet(int size){ maxsize = size; - set = new MapMaker().concurrencyLevel(64).makeMap(); + set = CacheBuilder.newBuilder().concurrencyLevel(64).build().asMap(); } /** diff --git a/modules/src/main/java/org/archive/modules/forms/FormLoginProcessor.java b/modules/src/main/java/org/archive/modules/forms/FormLoginProcessor.java index 94716960..47d06cec 100644 --- a/modules/src/main/java/org/archive/modules/forms/FormLoginProcessor.java +++ b/modules/src/main/java/org/archive/modules/forms/FormLoginProcessor.java @@ -26,6 +26,9 @@ import java.util.concurrent.atomic.AtomicLong; import java.util.logging.Level; import java.util.logging.Logger; +import com.google.common.cache.CacheBuilder; +import com.google.common.cache.CacheLoader; +import com.google.common.cache.LoadingCache; import org.apache.commons.httpclient.URIException; import org.apache.commons.lang.StringUtils; import org.archive.checkpointing.Checkpointable; @@ -42,9 +45,6 @@ import org.json.JSONException; import org.json.JSONObject; import org.springframework.beans.factory.annotation.Autowired; -import com.google.common.base.Function; -import com.google.common.collect.MapMaker; - /** * A step, post-ExtractorHTMLForms, where a followup CrawlURI to * attempt a form submission may be synthesized. @@ -119,22 +119,24 @@ public class FormLoginProcessor extends Processor implements Checkpointable { Logger.getLogger(FormLoginProcessor.class.getName()); // formProvince (String) -> count - ConcurrentMap eligibleFormsSeenCount = - new MapMaker().makeComputingMap(new Function() { - @Override - public AtomicLong apply(String arg0) { - return new AtomicLong(0L); - } - }); + ConcurrentMap eligibleFormsSeenCount = + CacheBuilder.newBuilder() + .build( + new CacheLoader() { + public AtomicLong load(String arg0) { + return new AtomicLong(0L); + } + }).asMap(); // formProvince (String) -> count - ConcurrentMap eligibleFormsAttemptsCount = - new MapMaker().makeComputingMap(new Function() { - @Override - public AtomicLong apply(String arg0) { - return new AtomicLong(0L); - } - }); + ConcurrentMap eligibleFormsAttemptsCount = + CacheBuilder.newBuilder() + .build( + new CacheLoader() { + public AtomicLong load(String arg0) { + return new AtomicLong(0L); + } + }).asMap(); /** * SURT prefix against which configured username/password is From d4d23639b615c4ecbf4e3529083bd727e9df51fb Mon Sep 17 00:00:00 2001 From: Shriphani Palakodety Date: Fri, 14 Mar 2014 16:20:50 -0400 Subject: [PATCH 07/16] added removal listener --- .../archive/util/ObjectIdentityBdbManualCache.java | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/commons/src/main/java/org/archive/util/ObjectIdentityBdbManualCache.java b/commons/src/main/java/org/archive/util/ObjectIdentityBdbManualCache.java index f0d43dc8..81ec0f7c 100644 --- a/commons/src/main/java/org/archive/util/ObjectIdentityBdbManualCache.java +++ b/commons/src/main/java/org/archive/util/ObjectIdentityBdbManualCache.java @@ -67,7 +67,7 @@ import com.sleepycat.je.Environment; */ @SuppressWarnings("ALL") public class ObjectIdentityBdbManualCache -implements ObjectIdentityCache, Closeable, Serializable, RemovalListener { +implements ObjectIdentityCache, Closeable, Serializable { private static final long serialVersionUID = 1L; private static final Logger logger = Logger.getLogger(ObjectIdentityBdbManualCache.class.getName()); @@ -115,6 +115,13 @@ implements ObjectIdentityCache, Closeable, Serializable, RemovalListener() { + @Override + public void onRemoval(RemovalNotification stringVRemovalNotification) { + evictions.incrementAndGet(); + diskMap.put(stringVRemovalNotification.getKey(), stringVRemovalNotification.getValue()); + } + }) .build() .asMap(); } @@ -372,9 +379,9 @@ implements ObjectIdentityCache, Closeable, Serializable, RemovalListener stringVRemovalNotification) { evictions.incrementAndGet(); diskMap.put(stringVRemovalNotification.getKey(), stringVRemovalNotification.getValue()); - } + }*/ } From 4c302f84be79ed41766577b307f8971cac5107ea Mon Sep 17 00:00:00 2001 From: Shriphani Palakodety Date: Sun, 16 Mar 2014 21:00:10 -0400 Subject: [PATCH 08/16] build with the right key and value types --- .../java/org/archive/modules/forms/FormLoginProcessor.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/modules/src/main/java/org/archive/modules/forms/FormLoginProcessor.java b/modules/src/main/java/org/archive/modules/forms/FormLoginProcessor.java index 47d06cec..f2292f77 100644 --- a/modules/src/main/java/org/archive/modules/forms/FormLoginProcessor.java +++ b/modules/src/main/java/org/archive/modules/forms/FormLoginProcessor.java @@ -121,7 +121,7 @@ public class FormLoginProcessor extends Processor implements Checkpointable { // formProvince (String) -> count ConcurrentMap eligibleFormsSeenCount = CacheBuilder.newBuilder() - .build( + .build( new CacheLoader() { public AtomicLong load(String arg0) { return new AtomicLong(0L); @@ -131,7 +131,7 @@ public class FormLoginProcessor extends Processor implements Checkpointable { // formProvince (String) -> count ConcurrentMap eligibleFormsAttemptsCount = CacheBuilder.newBuilder() - .build( + .build( new CacheLoader() { public AtomicLong load(String arg0) { return new AtomicLong(0L); From bd1047d2d21fbc5ecb945bbd9235704c9720e686 Mon Sep 17 00:00:00 2001 From: Shriphani Palakodety Date: Thu, 13 Mar 2014 15:26:57 -0400 Subject: [PATCH 09/16] bumped guava version to 11 --- .../util/ObjectIdentityBdbManualCache.java | 25 +++++++++++-------- 1 file changed, 14 insertions(+), 11 deletions(-) diff --git a/commons/src/main/java/org/archive/util/ObjectIdentityBdbManualCache.java b/commons/src/main/java/org/archive/util/ObjectIdentityBdbManualCache.java index 84504e79..2d1410ea 100644 --- a/commons/src/main/java/org/archive/util/ObjectIdentityBdbManualCache.java +++ b/commons/src/main/java/org/archive/util/ObjectIdentityBdbManualCache.java @@ -26,15 +26,16 @@ import java.util.Iterator; import java.util.Set; import java.util.Map.Entry; import java.util.concurrent.ConcurrentMap; -import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicLong; import java.util.logging.Level; import java.util.logging.Logger; import org.archive.bdb.KryoBinding; -import com.google.common.collect.MapEvictionListener; import com.google.common.collect.MapMaker; +import com.google.common.cache.CacheBuilder; +import com.google.common.cache.RemovalListener; +import com.google.common.cache.RemovalNotification; import com.sleepycat.bind.EntryBinding; import com.sleepycat.bind.serial.StoredClassCatalog; import com.sleepycat.bind.tuple.TupleBinding; @@ -64,8 +65,9 @@ import com.sleepycat.je.Environment; * @author paul baclace (conversion to ConcurrentMap) * */ -public class ObjectIdentityBdbManualCache -implements ObjectIdentityCache, Closeable, Serializable, MapEvictionListener { +@SuppressWarnings("ALL") +public class ObjectIdentityBdbManualCache +implements ObjectIdentityCache, Closeable, Serializable, RemovalListener { private static final long serialVersionUID = 1L; private static final Logger logger = Logger.getLogger(ObjectIdentityBdbManualCache.class.getName()); @@ -110,6 +112,10 @@ implements ObjectIdentityCache, Closeable, Serializable, MapEvictionListener< */ public ObjectIdentityBdbManualCache() { super(); + dirtyItems = CacheBuilder.newBuilder() + .maximumSize(10000) + .build() + .asMap(); } /** @@ -128,15 +134,12 @@ implements ObjectIdentityCache, Closeable, Serializable, MapEvictionListener< final Class valueClass, final StoredClassCatalog classCatalog) throws DatabaseException { // TODO: tune capacity for actual threads, expected size of key caches? - this.memMap = new MapMaker().concurrencyLevel(64).initialCapacity(8192).softValues().makeMap(); + this.memMap = new MapMaker().concurrencyLevel(64).initialCapacity(8192).softValues().makeMap(); this.db = openDatabase(env, dbName); this.diskMap = createDiskMap(this.db, classCatalog, valueClass); // keep a record of items that must be persisted; auto-persist if // unchanged after 5 minutes, or more than 10K would collect - this.dirtyItems = new MapMaker().concurrencyLevel(64) - .maximumSize(10000).expireAfterWrite(5,TimeUnit.MINUTES) - .evictionListener(this).makeMap(); - + this.count = new AtomicLong(diskMap.size()); } @@ -364,8 +367,8 @@ implements ObjectIdentityCache, Closeable, Serializable, MapEvictionListener< } @Override - public void onEviction(String key, V val) { + public void onRemoval(RemovalNotification stringVRemovalNotification) { evictions.incrementAndGet(); - diskMap.put(key, val); + diskMap.put(stringVRemovalNotification.getKey(), stringVRemovalNotification.getValue()); } } From d0e0ee6bc68311a5cf76661c268d9370bd6e4fc4 Mon Sep 17 00:00:00 2001 From: Shriphani Palakodety Date: Thu, 13 Mar 2014 15:29:37 -0400 Subject: [PATCH 10/16] forgot to add expireAfterWrite --- .../java/org/archive/util/ObjectIdentityBdbManualCache.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/commons/src/main/java/org/archive/util/ObjectIdentityBdbManualCache.java b/commons/src/main/java/org/archive/util/ObjectIdentityBdbManualCache.java index 2d1410ea..7ebefe77 100644 --- a/commons/src/main/java/org/archive/util/ObjectIdentityBdbManualCache.java +++ b/commons/src/main/java/org/archive/util/ObjectIdentityBdbManualCache.java @@ -26,6 +26,7 @@ import java.util.Iterator; import java.util.Set; import java.util.Map.Entry; import java.util.concurrent.ConcurrentMap; +import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicLong; import java.util.logging.Level; import java.util.logging.Logger; @@ -114,6 +115,7 @@ implements ObjectIdentityCache, Closeable, Serializable, RemovalListenerbuild() .asMap(); } From b0ad9358e88782c18425ba892cd2e49f706e4c0e Mon Sep 17 00:00:00 2001 From: Shriphani Palakodety Date: Thu, 13 Mar 2014 20:02:58 -0400 Subject: [PATCH 11/16] bump version in pom.xml --- commons/pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/commons/pom.xml b/commons/pom.xml index 2a679195..e267340b 100644 --- a/commons/pom.xml +++ b/commons/pom.xml @@ -185,7 +185,7 @@ com.google.guava guava - r08 + 16.0.1 From 1e8e2e2016b5ab3d5fd8a32669a4010ccc2cd006 Mon Sep 17 00:00:00 2001 From: Shriphani Palakodety Date: Thu, 13 Mar 2014 21:50:06 -0400 Subject: [PATCH 12/16] prevent intellij from complaining --- commons/src/main/java/org/archive/util/TestUtils.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/commons/src/main/java/org/archive/util/TestUtils.java b/commons/src/main/java/org/archive/util/TestUtils.java index 889492a1..a1237483 100644 --- a/commons/src/main/java/org/archive/util/TestUtils.java +++ b/commons/src/main/java/org/archive/util/TestUtils.java @@ -127,7 +127,7 @@ public class TestUtils { } cname = cname.replace(File.separatorChar, '.'); cname = cname.substring(0, cname.length() - 5); - suite.addTestSuite(Class.forName(cname)); + suite.addTestSuite((Class) Class.forName(cname)); } } } From 8c9892586a80a378adac733e89f2890e3d69d551 Mon Sep 17 00:00:00 2001 From: Shriphani Palakodety Date: Fri, 14 Mar 2014 15:13:34 -0400 Subject: [PATCH 13/16] removed guava dependency from heritrix. use the webarchive-commons version --- commons/pom.xml | 6 ------ 1 file changed, 6 deletions(-) diff --git a/commons/pom.xml b/commons/pom.xml index e267340b..66b60e88 100644 --- a/commons/pom.xml +++ b/commons/pom.xml @@ -182,12 +182,6 @@ runtime - - com.google.guava - guava - 16.0.1 - - net.java.dev.jna jna From 2f60b5f935d0dd52ca8e52e52a83ead28aaba864 Mon Sep 17 00:00:00 2001 From: Shriphani Palakodety Date: Fri, 14 Mar 2014 15:36:22 -0400 Subject: [PATCH 14/16] migration to latest guava complete --- .../util/ObjectIdentityBdbManualCache.java | 10 ++++-- .../org/archive/crawler/util/TopNSet.java | 5 ++- .../modules/forms/FormLoginProcessor.java | 36 ++++++++++--------- 3 files changed, 28 insertions(+), 23 deletions(-) diff --git a/commons/src/main/java/org/archive/util/ObjectIdentityBdbManualCache.java b/commons/src/main/java/org/archive/util/ObjectIdentityBdbManualCache.java index 7ebefe77..f0d43dc8 100644 --- a/commons/src/main/java/org/archive/util/ObjectIdentityBdbManualCache.java +++ b/commons/src/main/java/org/archive/util/ObjectIdentityBdbManualCache.java @@ -33,7 +33,6 @@ import java.util.logging.Logger; import org.archive.bdb.KryoBinding; -import com.google.common.collect.MapMaker; import com.google.common.cache.CacheBuilder; import com.google.common.cache.RemovalListener; import com.google.common.cache.RemovalNotification; @@ -135,8 +134,13 @@ implements ObjectIdentityCache, Closeable, Serializable, RemovalListenerbuild() + .asMap(); this.db = openDatabase(env, dbName); this.diskMap = createDiskMap(this.db, classCatalog, valueClass); // keep a record of items that must be persisted; auto-persist if diff --git a/engine/src/main/java/org/archive/crawler/util/TopNSet.java b/engine/src/main/java/org/archive/crawler/util/TopNSet.java index 4a8b2599..0c1d90ed 100644 --- a/engine/src/main/java/org/archive/crawler/util/TopNSet.java +++ b/engine/src/main/java/org/archive/crawler/util/TopNSet.java @@ -25,10 +25,9 @@ import java.util.SortedSet; import java.util.TreeSet; import java.util.concurrent.ConcurrentMap; +import com.google.common.cache.CacheBuilder; import org.archive.util.Histotable; -import com.google.common.collect.MapMaker; - /** * Counting Set which only remembers the 'top N' of all String values * reported (with counts) to it. Precise if counts reported for a @@ -56,7 +55,7 @@ public class TopNSet implements Serializable { public TopNSet(int size){ maxsize = size; - set = new MapMaker().concurrencyLevel(64).makeMap(); + set = CacheBuilder.newBuilder().concurrencyLevel(64).build().asMap(); } /** diff --git a/modules/src/main/java/org/archive/modules/forms/FormLoginProcessor.java b/modules/src/main/java/org/archive/modules/forms/FormLoginProcessor.java index 94716960..47d06cec 100644 --- a/modules/src/main/java/org/archive/modules/forms/FormLoginProcessor.java +++ b/modules/src/main/java/org/archive/modules/forms/FormLoginProcessor.java @@ -26,6 +26,9 @@ import java.util.concurrent.atomic.AtomicLong; import java.util.logging.Level; import java.util.logging.Logger; +import com.google.common.cache.CacheBuilder; +import com.google.common.cache.CacheLoader; +import com.google.common.cache.LoadingCache; import org.apache.commons.httpclient.URIException; import org.apache.commons.lang.StringUtils; import org.archive.checkpointing.Checkpointable; @@ -42,9 +45,6 @@ import org.json.JSONException; import org.json.JSONObject; import org.springframework.beans.factory.annotation.Autowired; -import com.google.common.base.Function; -import com.google.common.collect.MapMaker; - /** * A step, post-ExtractorHTMLForms, where a followup CrawlURI to * attempt a form submission may be synthesized. @@ -119,22 +119,24 @@ public class FormLoginProcessor extends Processor implements Checkpointable { Logger.getLogger(FormLoginProcessor.class.getName()); // formProvince (String) -> count - ConcurrentMap eligibleFormsSeenCount = - new MapMaker().makeComputingMap(new Function() { - @Override - public AtomicLong apply(String arg0) { - return new AtomicLong(0L); - } - }); + ConcurrentMap eligibleFormsSeenCount = + CacheBuilder.newBuilder() + .build( + new CacheLoader() { + public AtomicLong load(String arg0) { + return new AtomicLong(0L); + } + }).asMap(); // formProvince (String) -> count - ConcurrentMap eligibleFormsAttemptsCount = - new MapMaker().makeComputingMap(new Function() { - @Override - public AtomicLong apply(String arg0) { - return new AtomicLong(0L); - } - }); + ConcurrentMap eligibleFormsAttemptsCount = + CacheBuilder.newBuilder() + .build( + new CacheLoader() { + public AtomicLong load(String arg0) { + return new AtomicLong(0L); + } + }).asMap(); /** * SURT prefix against which configured username/password is From c81eeced7ae775b99901927ee16fa2bb6f096b7f Mon Sep 17 00:00:00 2001 From: Shriphani Palakodety Date: Fri, 14 Mar 2014 16:20:50 -0400 Subject: [PATCH 15/16] added removal listener --- .../archive/util/ObjectIdentityBdbManualCache.java | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/commons/src/main/java/org/archive/util/ObjectIdentityBdbManualCache.java b/commons/src/main/java/org/archive/util/ObjectIdentityBdbManualCache.java index f0d43dc8..81ec0f7c 100644 --- a/commons/src/main/java/org/archive/util/ObjectIdentityBdbManualCache.java +++ b/commons/src/main/java/org/archive/util/ObjectIdentityBdbManualCache.java @@ -67,7 +67,7 @@ import com.sleepycat.je.Environment; */ @SuppressWarnings("ALL") public class ObjectIdentityBdbManualCache -implements ObjectIdentityCache, Closeable, Serializable, RemovalListener { +implements ObjectIdentityCache, Closeable, Serializable { private static final long serialVersionUID = 1L; private static final Logger logger = Logger.getLogger(ObjectIdentityBdbManualCache.class.getName()); @@ -115,6 +115,13 @@ implements ObjectIdentityCache, Closeable, Serializable, RemovalListener() { + @Override + public void onRemoval(RemovalNotification stringVRemovalNotification) { + evictions.incrementAndGet(); + diskMap.put(stringVRemovalNotification.getKey(), stringVRemovalNotification.getValue()); + } + }) .build() .asMap(); } @@ -372,9 +379,9 @@ implements ObjectIdentityCache, Closeable, Serializable, RemovalListener stringVRemovalNotification) { evictions.incrementAndGet(); diskMap.put(stringVRemovalNotification.getKey(), stringVRemovalNotification.getValue()); - } + }*/ } From 84f29e4e10529331ce0f48f7165b8c6aa2dac04a Mon Sep 17 00:00:00 2001 From: Shriphani Palakodety Date: Sun, 16 Mar 2014 21:00:10 -0400 Subject: [PATCH 16/16] build with the right key and value types --- .../java/org/archive/modules/forms/FormLoginProcessor.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/modules/src/main/java/org/archive/modules/forms/FormLoginProcessor.java b/modules/src/main/java/org/archive/modules/forms/FormLoginProcessor.java index 47d06cec..f2292f77 100644 --- a/modules/src/main/java/org/archive/modules/forms/FormLoginProcessor.java +++ b/modules/src/main/java/org/archive/modules/forms/FormLoginProcessor.java @@ -121,7 +121,7 @@ public class FormLoginProcessor extends Processor implements Checkpointable { // formProvince (String) -> count ConcurrentMap eligibleFormsSeenCount = CacheBuilder.newBuilder() - .build( + .build( new CacheLoader() { public AtomicLong load(String arg0) { return new AtomicLong(0L); @@ -131,7 +131,7 @@ public class FormLoginProcessor extends Processor implements Checkpointable { // formProvince (String) -> count ConcurrentMap eligibleFormsAttemptsCount = CacheBuilder.newBuilder() - .build( + .build( new CacheLoader() { public AtomicLong load(String arg0) { return new AtomicLong(0L);