mirror of
https://github.com/internetarchive/heritrix3.git
synced 2026-11-15 10:20:47 +00:00
HER-1783 BloomFilter64bit bit-length bug prevents full bitfield from being used; premature saturation
* BloomFilter64bit.java
fix long arithmetic calculating bit-length m in constructor
slight optimization of add()
* BloomFilterTest.java
abstract superclass for bloom impl class tests
* BloomFilter64bitTest.java
sanity check that set bits, after a handful of adds into default-sized filter, occupy top and bottom 20% of range (which would have caught this bug earlier)
This commit is contained in:
1 parent
e673c7d781
commit
e9a11e56c6
3 files changed
+119
-7
No files matched your search
@@ -43,7 +43,7 @@ import java.security.SecureRandom;
|
||||
* <li>Adapted to allow long bit indices so long as the index/64 (used
|
||||
* an array index in bit vector) fits within Integer.MAX_VALUE. (Thus
|
||||
* it supports filters up to 64*Integer.MAX_VALUE bits in size, or
|
||||
* 16GiB.</li>
|
||||
* 16GiB.)</li>
|
||||
* </ul>
|
||||
*
|
||||
* <hr>
|
||||
@@ -80,10 +80,10 @@ public class BloomFilter64bit implements Serializable, BloomFilter {
|
||||
final public long m;
|
||||
/** The number of hash functions used by this filter. */
|
||||
final public int d;
|
||||
/** The underlying bit vector. */
|
||||
final private long[] bits;
|
||||
/** The underlying bit vector. package access for testing */
|
||||
final long[] bits;
|
||||
/** The random integers used to generate the hash functions. */
|
||||
final private long[][] weight;
|
||||
final long[][] weight;
|
||||
|
||||
/** The number of elements currently in the filter. It may be
|
||||
* smaller than the actual number of additions of distinct character
|
||||
@@ -107,7 +107,7 @@ public class BloomFilter64bit implements Serializable, BloomFilter {
|
||||
int len = (int)Math.ceil( ( (long)n * (long)d / NATURAL_LOG_OF_2 ) / 64L );
|
||||
if ( len/64 > Integer.MAX_VALUE ) throw new IllegalArgumentException( "This filter would require " + len * 64L + " bits" );
|
||||
bits = new long[ len ];
|
||||
m = bits.length * 64;
|
||||
m = bits.length * 64L;
|
||||
|
||||
if ( DEBUG ) System.err.println( "Number of bits: " + m );
|
||||
|
||||
@@ -179,8 +179,10 @@ public class BloomFilter64bit implements Serializable, BloomFilter {
|
||||
long h;
|
||||
while( i-- != 0 ) {
|
||||
h = hash( s, l, i );
|
||||
if ( ! getBit( h ) ) result = true;
|
||||
setBit( h );
|
||||
if ( ! getBit( h ) ) {
|
||||
result = true;
|
||||
setBit( h );
|
||||
}
|
||||
}
|
||||
if ( result ) size++;
|
||||
return result;
|
||||
|
||||
Reference in new issue
Block a user