lee/heritrix3
1
0
Fork 0
mirror of https://github.com/internetarchive/heritrix3.git synced 2026-09-25 15:16:24 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #50 from shriphani/master

Browse Source 
Replace deprecated routines in guava
This commit is contained in:
Noah Levitt
2014-03-18 11:27:49 -07:00
parent 2018da4cd6 74f017c53e
commit ef2889cf17
5 changed files with 52 additions and 41 deletions
Download Patch File Download Diff File Expand all files Collapse all files
commons/pom.xml
-6
View File
@@ -182,12 +182,6 @@
<scope>runtime</scope>
</dependency>
<dependency>
<groupId>com.google.guava</groupId>
<artifactId>guava</artifactId>
<version>r08</version>
</dependency>
<dependency>
<groupId>net.java.dev.jna</groupId>
<artifactId>jna</artifactId>
commons/src/main/java/org/archive/util/ObjectIdentityBdbManualCache.java
+30 -14
View File
@@ -33,8 +33,9 @@ import java.util.logging.Logger;
import org.archive.bdb.KryoBinding;
import com.google.common.collect.MapEvictionListener;
import com.google.common.collect.MapMaker;
import com.google.common.cache.CacheBuilder;
import com.google.common.cache.RemovalListener;
import com.google.common.cache.RemovalNotification;
import com.sleepycat.bind.EntryBinding;
import com.sleepycat.bind.serial.StoredClassCatalog;
import com.sleepycat.bind.tuple.TupleBinding;
@@ -64,8 +65,9 @@ import com.sleepycat.je.Environment;
* @author paul baclace (conversion to ConcurrentMap)
*
*/
public class ObjectIdentityBdbManualCache<V extends IdentityCacheable>
implements ObjectIdentityCache<V>, Closeable, Serializable, MapEvictionListener<String, V> {
@SuppressWarnings("ALL")
public class ObjectIdentityBdbManualCache<V extends IdentityCacheable>
implements ObjectIdentityCache<V>, Closeable, Serializable {
private static final long serialVersionUID = 1L;
private static final Logger logger =
Logger.getLogger(ObjectIdentityBdbManualCache.class.getName());
@@ -110,6 +112,18 @@ implements ObjectIdentityCache<V>, Closeable, Serializable, MapEvictionListener<
*/
public ObjectIdentityBdbManualCache() {
super();
dirtyItems = CacheBuilder.newBuilder()
.maximumSize(10000)
.expireAfterWrite(5, TimeUnit.MINUTES)
.removalListener(new RemovalListener<String, V>() {
@Override
public void onRemoval(RemovalNotification<String, V> stringVRemovalNotification) {
evictions.incrementAndGet();
diskMap.put(stringVRemovalNotification.getKey(), stringVRemovalNotification.getValue());
}
})
.<String, V>build()
.asMap();
}
/**
@@ -127,16 +141,18 @@ implements ObjectIdentityCache<V>, Closeable, Serializable, MapEvictionListener<
public void initialize(final Environment env, String dbName,
final Class valueClass, final StoredClassCatalog classCatalog)
throws DatabaseException {
// TODO: tune capacity for actual threads, expected size of key caches?
this.memMap = new MapMaker().concurrencyLevel(64).initialCapacity(8192).softValues().makeMap();
// TODO: tune capacity for actual threads, expected size of key caches?
this.memMap = CacheBuilder.newBuilder()
.concurrencyLevel(64)
.initialCapacity(8192)
.softValues()
.<String, V>build()
.asMap();
this.db = openDatabase(env, dbName);
this.diskMap = createDiskMap(this.db, classCatalog, valueClass);
// keep a record of items that must be persisted; auto-persist if
// unchanged after 5 minutes, or more than 10K would collect
this.dirtyItems = new MapMaker().concurrencyLevel(64)
.maximumSize(10000).expireAfterWrite(5,TimeUnit.MINUTES)
.evictionListener(this).makeMap();
this.count = new AtomicLong(diskMap.size());
}
@@ -363,9 +379,9 @@ implements ObjectIdentityCache<V>, Closeable, Serializable, MapEvictionListener<
dirtyItems.put(key,val);
}
@Override
public void onEviction(String key, V val) {
/*@Override
public void onRemoval(RemovalNotification<String, V> stringVRemovalNotification) {
evictions.incrementAndGet();
diskMap.put(key, val);
}
diskMap.put(stringVRemovalNotification.getKey(), stringVRemovalNotification.getValue());
}*/
}
commons/src/main/java/org/archive/util/TestUtils.java
+1 -1
View File
@@ -127,7 +127,7 @@ public class TestUtils {
}
cname = cname.replace(File.separatorChar, '.');
cname = cname.substring(0, cname.length() - 5);
suite.addTestSuite(Class.forName(cname));
suite.addTestSuite((Class<? extends TestCase>) Class.forName(cname));
}
}
}
engine/src/main/java/org/archive/crawler/util/TopNSet.java
+2 -3
View File
@@ -25,10 +25,9 @@ import java.util.SortedSet;
import java.util.TreeSet;
import java.util.concurrent.ConcurrentMap;
import com.google.common.cache.CacheBuilder;
import org.archive.util.Histotable;
import com.google.common.collect.MapMaker;
/**
* Counting Set which only remembers the 'top N' of all String values
* reported (with counts) to it. Precise if counts reported for a
@@ -56,7 +55,7 @@ public class TopNSet implements Serializable {
public TopNSet(int size){
maxsize = size;
set = new MapMaker().concurrencyLevel(64).makeMap();
set = CacheBuilder.newBuilder().concurrencyLevel(64).<String, Long>build().asMap();
}
/**
modules/src/main/java/org/archive/modules/forms/FormLoginProcessor.java
+19 -17
View File
@@ -26,6 +26,9 @@ import java.util.concurrent.atomic.AtomicLong;
import java.util.logging.Level;
import java.util.logging.Logger;
import com.google.common.cache.CacheBuilder;
import com.google.common.cache.CacheLoader;
import com.google.common.cache.LoadingCache;
import org.apache.commons.httpclient.URIException;
import org.apache.commons.lang.StringUtils;
import org.archive.checkpointing.Checkpointable;
@@ -42,9 +45,6 @@ import org.json.JSONException;
import org.json.JSONObject;
import org.springframework.beans.factory.annotation.Autowired;
import com.google.common.base.Function;
import com.google.common.collect.MapMaker;
/**
* A step, post-ExtractorHTMLForms, where a followup CrawlURI to
* attempt a form submission may be synthesized.
@@ -119,22 +119,24 @@ public class FormLoginProcessor extends Processor implements Checkpointable {
Logger.getLogger(FormLoginProcessor.class.getName());
// formProvince (String) -> count
ConcurrentMap<String, AtomicLong> eligibleFormsSeenCount =
new MapMaker().makeComputingMap(new Function<String, AtomicLong>() {
@Override
public AtomicLong apply(String arg0) {
return new AtomicLong(0L);
}
});
ConcurrentMap<String, AtomicLong> eligibleFormsSeenCount =
CacheBuilder.newBuilder()
.<String, AtomicLong>build(
new CacheLoader<String, AtomicLong>() {
public AtomicLong load(String arg0) {
return new AtomicLong(0L);
}
}).asMap();
// formProvince (String) -> count
ConcurrentMap<String, AtomicLong> eligibleFormsAttemptsCount =
new MapMaker().makeComputingMap(new Function<String, AtomicLong>() {
@Override
public AtomicLong apply(String arg0) {
return new AtomicLong(0L);
}
});
ConcurrentMap<String, AtomicLong> eligibleFormsAttemptsCount =
CacheBuilder.newBuilder()
.<String, AtomicLong>build(
new CacheLoader<String, AtomicLong>() {
public AtomicLong load(String arg0) {
return new AtomicLong(0L);
}
}).asMap();
/**
* SURT prefix against which configured username/password is