lee/heritrix3
1
0
Fork 0
mirror of https://github.com/internetarchive/heritrix3.git synced 2026-09-25 07:05:49 +00:00
Code Issues Packages Projects Releases Wiki Activity

Fix exceptions at launch, post-crawl-finish

Browse Source 
* KryoBinding.java
    always default to registrationOptional, for now
* StatisticsTracker.java
    explicitly set valueClass of sourceDistribution to ConcurrentHashMap
* StoredQueue.java
    answer 0 for size() when backing DB closed
This commit is contained in:
gojomo
2011-01-07 01:24:38 +00:00
parent 93cf655bed
commit bdd7f7e1ab
3 changed files with 12 additions and 6 deletions
Download Patch File Download Diff File Expand all files Collapse all files
commons/src/main/java/org/archive/bdb/KryoBinding.java
+2
View File
@@ -53,6 +53,8 @@ public class KryoBinding<K> implements EntryBinding<K> {
public KryoBinding(Class baseClass) {
this.baseClass = baseClass;
kryo.autoregister(baseClass);
// TODO: reevaluate if explicit registration should be required
kryo.setRegistrationOptional(true);
}
public Kryo getKryo() {
commons/src/main/java/org/archive/bdb/StoredQueue.java
+5 -1
View File
@@ -91,7 +91,11 @@ public class StoredQueue<E extends Serializable> extends AbstractQueue<E> {
@Override
public int size() {
return queueMap.size();
try {
return queueMap.size();
} catch (IllegalStateException ise) {
return 0;
}
}
@Override
engine/src/main/java/org/archive/crawler/reporting/StatisticsTracker.java
+5 -5
View File
@@ -332,8 +332,8 @@ public class StatisticsTracker
/** Keep track of URL counts per host per seed */
@SuppressWarnings("unchecked")
protected ObjectIdentityCache<String,ConcurrentMap> sourceHostDistribution =
new ObjectIdentityMemCache<ConcurrentMap>(); // temp dummy;
protected ObjectIdentityCache<String,ConcurrentHashMap> sourceHostDistribution =
new ObjectIdentityMemCache<ConcurrentHashMap>(); // temp dummy;
/* Keep track of 'top' hosts for live reports */
protected TopNSet hostsDistributionTop;
@@ -367,7 +367,7 @@ public class StatisticsTracker
boolean isRecover = (recoveryCheckpoint != null);
try {
this.sourceHostDistribution = bdb.getObjectCache("sourceHostDistribution",
isRecover, ConcurrentMap.class);
isRecover, ConcurrentHashMap.class);
this.hostsDistribution = bdb.getObjectCache("hostsDistribution",
isRecover, AtomicLong.class);
this.hostsBytes = bdb.getObjectCache("hostsBytes",
@@ -851,8 +851,8 @@ public class StatisticsTracker
ConcurrentMap<String,AtomicLong> hostUriCount =
sourceHostDistribution.getOrUse(
source,
new Supplier<ConcurrentMap>() {
public ConcurrentMap<String, AtomicLong> get() {
new Supplier<ConcurrentHashMap>() {
public ConcurrentHashMap<String, AtomicLong> get() {
return new ConcurrentHashMap<String,AtomicLong>();
}});
incrementMapCount(hostUriCount, hostname);