From e9a11e56c6b2491a2525f94bacb8317e9f39057e Mon Sep 17 00:00:00 2001 From: gojomo Date: Tue, 8 Jun 2010 02:39:35 +0000 Subject: [PATCH] HER-1783 BloomFilter64bit bit-length bug prevents full bitfield from being used; premature saturation * BloomFilter64bit.java fix long arithmetic calculating bit-length m in constructor slight optimization of add() * BloomFilterTest.java abstract superclass for bloom impl class tests * BloomFilter64bitTest.java sanity check that set bits, after a handful of adds into default-sized filter, occupy top and bottom 20% of range (which would have caught this bug earlier) --- .../org/archive/util/BloomFilter64bit.java | 16 ++--- .../archive/util/BloomFilter64bitTest.java | 59 +++++++++++++++++++ .../org/archive/util/BloomFilterTest.java | 51 ++++++++++++++++ 3 files changed, 119 insertions(+), 7 deletions(-) create mode 100644 commons/src/test/java/org/archive/util/BloomFilter64bitTest.java create mode 100644 commons/src/test/java/org/archive/util/BloomFilterTest.java diff --git a/commons/src/main/java/org/archive/util/BloomFilter64bit.java b/commons/src/main/java/org/archive/util/BloomFilter64bit.java index f44869d3..97b0d06e 100644 --- a/commons/src/main/java/org/archive/util/BloomFilter64bit.java +++ b/commons/src/main/java/org/archive/util/BloomFilter64bit.java @@ -43,7 +43,7 @@ import java.security.SecureRandom; *
  • Adapted to allow long bit indices so long as the index/64 (used * an array index in bit vector) fits within Integer.MAX_VALUE. (Thus * it supports filters up to 64*Integer.MAX_VALUE bits in size, or - * 16GiB.
    • + * 16GiB.) * * *
    @@ -80,10 +80,10 @@ public class BloomFilter64bit implements Serializable, BloomFilter { final public long m; /** The number of hash functions used by this filter. */ final public int d; - /** The underlying bit vector. */ - final private long[] bits; + /** The underlying bit vector. package access for testing */ + final long[] bits; /** The random integers used to generate the hash functions. */ - final private long[][] weight; + final long[][] weight; /** The number of elements currently in the filter. It may be * smaller than the actual number of additions of distinct character @@ -107,7 +107,7 @@ public class BloomFilter64bit implements Serializable, BloomFilter { int len = (int)Math.ceil( ( (long)n * (long)d / NATURAL_LOG_OF_2 ) / 64L ); if ( len/64 > Integer.MAX_VALUE ) throw new IllegalArgumentException( "This filter would require " + len * 64L + " bits" ); bits = new long[ len ]; - m = bits.length * 64; + m = bits.length * 64L; if ( DEBUG ) System.err.println( "Number of bits: " + m ); @@ -179,8 +179,10 @@ public class BloomFilter64bit implements Serializable, BloomFilter { long h; while( i-- != 0 ) { h = hash( s, l, i ); - if ( ! getBit( h ) ) result = true; - setBit( h ); + if ( ! getBit( h ) ) { + result = true; + setBit( h ); + } } if ( result ) size++; return result; diff --git a/commons/src/test/java/org/archive/util/BloomFilter64bitTest.java b/commons/src/test/java/org/archive/util/BloomFilter64bitTest.java new file mode 100644 index 00000000..e3ba2f5a --- /dev/null +++ b/commons/src/test/java/org/archive/util/BloomFilter64bitTest.java @@ -0,0 +1,59 @@ +/* + * This file is part of the Heritrix web crawler (crawler.archive.org). + * + * Licensed to the Internet Archive (IA) by one or more individual + * contributors. + * + * The IA licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.archive.util; + + + +/** + * BloomFilter64 tests + * + * @contributor gojomo + * @version $Date: 2009-11-19 14:39:53 -0800 (Thu, 19 Nov 2009) $, $Revision: 6674 $ + */ +public class BloomFilter64bitTest extends BloomFilterTest { + + protected void setUp() throws Exception { + // test at default size of BloomUriUniqFilter -- but don't depend on that + // 'engine'-subproject class for values + bloom = new BloomFilter64bit(125000000,22); + } + + public void testDistributionOfSetBits() { + // prelaod + testBasics(); + + BloomFilter64bit bloom64 = (BloomFilter64bit)bloom; + for(int i = 0; i0) { + assertTrue("set bits not as expected in early positions",(i/(double)bloom64.bits.length)<0.2d); + break; + } + } + for(int i = bloom64.bits.length-1; i>=0; i--) { + // verify that first set bit is in first 20% of bitfield + if(bloom64.bits[i]>0) { + assertTrue("set bits not as expected in late positions",(i/(double)bloom64.bits.length)>0.8d); + break; + } + } + + } +} diff --git a/commons/src/test/java/org/archive/util/BloomFilterTest.java b/commons/src/test/java/org/archive/util/BloomFilterTest.java new file mode 100644 index 00000000..7fb5f6a4 --- /dev/null +++ b/commons/src/test/java/org/archive/util/BloomFilterTest.java @@ -0,0 +1,51 @@ +/* + * This file is part of the Heritrix web crawler (crawler.archive.org). + * + * Licensed to the Internet Archive (IA) by one or more individual + * contributors. + * + * The IA licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.archive.util; + +import junit.framework.TestCase; + + +/** + * BloomFilter tests + * + * @contributor gojomo + * @version $Date: 2009-11-19 14:39:53 -0800 (Thu, 19 Nov 2009) $, $Revision: 6674 $ + */ +public abstract class BloomFilterTest extends TestCase { + protected BloomFilter bloom; + + protected abstract void setUp() throws Exception; + + public void testBasics() { + // require initial additions to return 'true' (for 'added') + assertTrue(bloom.add("abracadabra")); + assertTrue(bloom.add("foobar")); + assertTrue(bloom.add("rumplestiltskin")); + assertTrue(bloom.add("buckaroobanzai")); + assertTrue(bloom.add("scheherazade")); + + // require readdition to return 'false' (not added because already present) + assertFalse(bloom.add("abracadabra")); + assertFalse(bloom.add("foobar")); + assertFalse(bloom.add("rumplestiltskin")); + assertFalse(bloom.add("buckaroobanzai")); + assertFalse(bloom.add("scheherazade")); + } +}