diff --git a/commons/src/main/java/org/archive/util/BloomFilter64bit.java b/commons/src/main/java/org/archive/util/BloomFilter64bit.java
index 34daad3a..f44869d3 100644
--- a/commons/src/main/java/org/archive/util/BloomFilter64bit.java
+++ b/commons/src/main/java/org/archive/util/BloomFilter64bit.java
@@ -41,7 +41,9 @@ import java.security.SecureRandom;
*
Removed dependence on cern.colt MersenneTwister (replaced with
* SecureRandom) and QuickBitVector (replaced with local methods).
* Adapted to allow long bit indices so long as the index/64 (used
- * an array index in bit vector) fits within Integer.MAX_VALUE
+ * an array index in bit vector) fits within Integer.MAX_VALUE. (Thus
+ * it supports filters up to 64*Integer.MAX_VALUE bits in size, or
+ * 16GiB.
*
*
*
diff --git a/engine/src/main/java/org/archive/crawler/util/BloomUriUniqFilter.java b/engine/src/main/java/org/archive/crawler/util/BloomUriUniqFilter.java
index 3c8667aa..2b15f5d2 100644
--- a/engine/src/main/java/org/archive/crawler/util/BloomUriUniqFilter.java
+++ b/engine/src/main/java/org/archive/crawler/util/BloomUriUniqFilter.java
@@ -1,20 +1,20 @@
/* BloomUriUniqFilter
- *
+*
* $Id$
- *
- * Created on June 21, 2005
- *
- * Copyright (C) 2005 Internet Archive.
- *
- * This file is part of the Heritrix web crawler (crawler.archive.org).
- *
+*
+* Created on June 21, 2005
+*
+* Copyright (C) 2005 Internet Archive.
+*
+* This file is part of the Heritrix web crawler (crawler.archive.org).
+*
* Licensed to the Internet Archive (IA) by one or more individual
* contributors.
- *
+*
* The IA licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
- *
+*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
@@ -22,8 +22,7 @@
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
- */
-
+*/
package org.archive.crawler.util;
import java.io.Serializable;
@@ -31,11 +30,12 @@ import java.util.logging.Logger;
import org.archive.modules.CrawlURI;
import org.archive.util.BloomFilter;
-import org.archive.util.BloomFilter32bitSplit;
+import org.archive.util.BloomFilter64bit;
+import org.springframework.beans.factory.InitializingBean;
/**
- * A MG4J BloomFilter-based implementation of an AlreadySeen list.
+ * An implementation of an AlreadySeen list based on the MG4J BloomFilter.
*
* This implementation performs adequately without blowing out
* the heap through to very large numbers of URIs. See
@@ -59,72 +59,64 @@ import org.archive.util.BloomFilter32bitSplit;
*
* ...bytes.
*
- * The default size is very close to the maximum practical size of the
- * Bloom filter implementation, BloomFilter32bitSplit, created in the
- * initialize() method, due to integer arithmetic limits.
+ * The BloomFilter64bit implementation class supports filters up to
+ * 16GiB in size.
*
- * If you need a larger filter, you should edit the initialize
- * method to intantiate a BloomFilter64bit instead.
+ * (If you only need a filter up to 512MiB in size, the
+ * BloomFilter32bitSplit *might* offer better performance, on 32bit
+ * JVMs or with respect to heap-handling of giant arrays. The only
+ * current way to swap in this class is by editing the source.)
*
* @author gojomo
* @version $Date$, $Revision$
*/
public class BloomUriUniqFilter extends SetBasedUriUniqFilter
-implements Serializable {
+implements Serializable, InitializingBean {
private static final long serialVersionUID = 1061526253773091309L;
private static Logger LOGGER =
Logger.getLogger(BloomUriUniqFilter.class.getName());
BloomFilter bloom; // package access for testing convenience
- protected int expected_n; // remember bloom contruction param
-
- protected static final String EXPECTED_SIZE_KEY = ".expected-size";
- protected static final String HASH_COUNT_KEY = ".hash-count";
-
+
// these defaults create a bloom filter that is
// 1.44*125mil*22/8 ~= 495MB in size, and at full
// capacity will give a false contained indication
// 1/(2^22) ~= 1 in every 4 million probes
- private static final int DEFAULT_EXPECTED_SIZE = 125000000; // 125 million
- private static final int DEFAULT_HASH_COUNT = 22; // 1 in 4 million false pos
+ protected int expectedInserts= 125000000; // default 125 million;
+ public int getExpectedInserts() {
+ return expectedInserts;
+ }
+ public void setExpectedInserts(int expectedInserts) {
+ this.expectedInserts = expectedInserts;
+ }
+ protected int hashCount = 22; // 1 in 4 million false pos
+ public int getHashCount() {
+ return hashCount;
+ }
+ public void setHashCount(int hashCount) {
+ this.hashCount = hashCount;
+ }
+
+
/**
* Default constructor
*/
public BloomUriUniqFilter() {
super();
- String ns = System.getProperty(this.getClass().getName() + EXPECTED_SIZE_KEY);
- int n = (ns == null) ? DEFAULT_EXPECTED_SIZE : Integer.parseInt(ns);
- String ds = System.getProperty(this.getClass().getName() + HASH_COUNT_KEY);
- int d = (ds == null) ? DEFAULT_HASH_COUNT : Integer.parseInt(ds);
- initialize(n,d);
}
/**
- * Constructor.
+ * Initializer.
*
* @param n the expected number of elements.
* @param d the number of hash functions; if the filter adds not more
* than n elements, false positives will happen with
* probability 2-d.
*/
- public BloomUriUniqFilter( final int n, final int d ) {
- super();
- initialize(n, d);
- }
-
- /**
- * Initializer shared by constructors.
- *
- * @param n the expected number of elements.
- * @param d the number of hash functions; if the filter adds not more
- * than n elements, false positives will happen with
- * probability 2-d.
- */
- protected void initialize(final int n, final int d) {
- this.expected_n = n;
- bloom = new BloomFilter32bitSplit(n,d);
+ public void afterPropertiesSet() {
+ bloom = new BloomFilter64bit(expectedInserts,hashCount);
}
public void forget(String canonical, CrawlURI item) {
@@ -132,13 +124,12 @@ implements Serializable {
LOGGER.severe("forget(\""+canonical+"\",CrawlURI) not supported");
}
-
protected boolean setAdd(CharSequence uri) {
boolean added = bloom.add(uri);
// warn if bloom has reached its expected size (and its false-pos
// rate will now exceed the theoretical/designed level)
- if( added && (count() == expected_n)) {
- LOGGER.warning("Bloom has reached expected limit "+expected_n);
+ if( added && (count() == expectedInserts)) {
+ LOGGER.warning("Bloom has reached expected limit "+expectedInserts);
}
return added;
}
diff --git a/engine/src/test/java/org/archive/crawler/util/BloomUriUniqFilterTest.java b/engine/src/test/java/org/archive/crawler/util/BloomUriUniqFilterTest.java
index d852f7b4..e00511e4 100644
--- a/engine/src/test/java/org/archive/crawler/util/BloomUriUniqFilterTest.java
+++ b/engine/src/test/java/org/archive/crawler/util/BloomUriUniqFilterTest.java
@@ -52,7 +52,10 @@ implements UriUniqFilter.CrawlUriReceiver {
protected void setUp() throws Exception {
super.setUp();
- this.filter = new BloomUriUniqFilter(2000,24);
+ this.filter = new BloomUriUniqFilter();
+ this.filter.setExpectedInserts(2000);
+ this.filter.setHashCount(24);
+ this.filter.afterPropertiesSet();
this.filter.setDestination(this);
}