diff --git a/commons/src/main/java/org/archive/util/BloomFilter64bit.java b/commons/src/main/java/org/archive/util/BloomFilter64bit.java
index f44869d3..97b0d06e 100644
--- a/commons/src/main/java/org/archive/util/BloomFilter64bit.java
+++ b/commons/src/main/java/org/archive/util/BloomFilter64bit.java
@@ -43,7 +43,7 @@ import java.security.SecureRandom;
*
Adapted to allow long bit indices so long as the index/64 (used
* an array index in bit vector) fits within Integer.MAX_VALUE. (Thus
* it supports filters up to 64*Integer.MAX_VALUE bits in size, or
- * 16GiB.
+ * 16GiB.)
*
*
*
@@ -80,10 +80,10 @@ public class BloomFilter64bit implements Serializable, BloomFilter {
final public long m;
/** The number of hash functions used by this filter. */
final public int d;
- /** The underlying bit vector. */
- final private long[] bits;
+ /** The underlying bit vector. package access for testing */
+ final long[] bits;
/** The random integers used to generate the hash functions. */
- final private long[][] weight;
+ final long[][] weight;
/** The number of elements currently in the filter. It may be
* smaller than the actual number of additions of distinct character
@@ -107,7 +107,7 @@ public class BloomFilter64bit implements Serializable, BloomFilter {
int len = (int)Math.ceil( ( (long)n * (long)d / NATURAL_LOG_OF_2 ) / 64L );
if ( len/64 > Integer.MAX_VALUE ) throw new IllegalArgumentException( "This filter would require " + len * 64L + " bits" );
bits = new long[ len ];
- m = bits.length * 64;
+ m = bits.length * 64L;
if ( DEBUG ) System.err.println( "Number of bits: " + m );
@@ -179,8 +179,10 @@ public class BloomFilter64bit implements Serializable, BloomFilter {
long h;
while( i-- != 0 ) {
h = hash( s, l, i );
- if ( ! getBit( h ) ) result = true;
- setBit( h );
+ if ( ! getBit( h ) ) {
+ result = true;
+ setBit( h );
+ }
}
if ( result ) size++;
return result;
diff --git a/commons/src/test/java/org/archive/util/BloomFilter64bitTest.java b/commons/src/test/java/org/archive/util/BloomFilter64bitTest.java
new file mode 100644
index 00000000..e3ba2f5a
--- /dev/null
+++ b/commons/src/test/java/org/archive/util/BloomFilter64bitTest.java
@@ -0,0 +1,59 @@
+/*
+ * This file is part of the Heritrix web crawler (crawler.archive.org).
+ *
+ * Licensed to the Internet Archive (IA) by one or more individual
+ * contributors.
+ *
+ * The IA licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.archive.util;
+
+
+
+/**
+ * BloomFilter64 tests
+ *
+ * @contributor gojomo
+ * @version $Date: 2009-11-19 14:39:53 -0800 (Thu, 19 Nov 2009) $, $Revision: 6674 $
+ */
+public class BloomFilter64bitTest extends BloomFilterTest {
+
+ protected void setUp() throws Exception {
+ // test at default size of BloomUriUniqFilter -- but don't depend on that
+ // 'engine'-subproject class for values
+ bloom = new BloomFilter64bit(125000000,22);
+ }
+
+ public void testDistributionOfSetBits() {
+ // prelaod
+ testBasics();
+
+ BloomFilter64bit bloom64 = (BloomFilter64bit)bloom;
+ for(int i = 0; i0) {
+ assertTrue("set bits not as expected in early positions",(i/(double)bloom64.bits.length)<0.2d);
+ break;
+ }
+ }
+ for(int i = bloom64.bits.length-1; i>=0; i--) {
+ // verify that first set bit is in first 20% of bitfield
+ if(bloom64.bits[i]>0) {
+ assertTrue("set bits not as expected in late positions",(i/(double)bloom64.bits.length)>0.8d);
+ break;
+ }
+ }
+
+ }
+}
diff --git a/commons/src/test/java/org/archive/util/BloomFilterTest.java b/commons/src/test/java/org/archive/util/BloomFilterTest.java
new file mode 100644
index 00000000..7fb5f6a4
--- /dev/null
+++ b/commons/src/test/java/org/archive/util/BloomFilterTest.java
@@ -0,0 +1,51 @@
+/*
+ * This file is part of the Heritrix web crawler (crawler.archive.org).
+ *
+ * Licensed to the Internet Archive (IA) by one or more individual
+ * contributors.
+ *
+ * The IA licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.archive.util;
+
+import junit.framework.TestCase;
+
+
+/**
+ * BloomFilter tests
+ *
+ * @contributor gojomo
+ * @version $Date: 2009-11-19 14:39:53 -0800 (Thu, 19 Nov 2009) $, $Revision: 6674 $
+ */
+public abstract class BloomFilterTest extends TestCase {
+ protected BloomFilter bloom;
+
+ protected abstract void setUp() throws Exception;
+
+ public void testBasics() {
+ // require initial additions to return 'true' (for 'added')
+ assertTrue(bloom.add("abracadabra"));
+ assertTrue(bloom.add("foobar"));
+ assertTrue(bloom.add("rumplestiltskin"));
+ assertTrue(bloom.add("buckaroobanzai"));
+ assertTrue(bloom.add("scheherazade"));
+
+ // require readdition to return 'false' (not added because already present)
+ assertFalse(bloom.add("abracadabra"));
+ assertFalse(bloom.add("foobar"));
+ assertFalse(bloom.add("rumplestiltskin"));
+ assertFalse(bloom.add("buckaroobanzai"));
+ assertFalse(bloom.add("scheherazade"));
+ }
+}