From 6be649b286d7b3e2a1193cbe808f8a46fa1ae4c4 Mon Sep 17 00:00:00 2001 From: gojomo Date: Tue, 15 Jun 2010 21:30:34 +0000 Subject: [PATCH] [HER-1783] BloomFilter64bit bit-length bug prevents full bitfield from being used; premature saturation * BloomFilter64bit.java include the split-to-subarrays (for larger bitfields) and round-up-to-power-of-2 (for performance) options previously in largely-redundant classes fit a number of problems with int/long overflow and bitwise ops add methods for reporting/testing * BloomFilter.java add methods for reporting/testing * BloomFilterTest.java, BloomFilter64bitTest.java more extensive tests, including two lengthy tests of default/oversized blooms usually disabled by renaming * BloomFilter32bit.java, BloomFilter32bitSplit.java, BloomFilter32bp2.java, BloomFilter32bp2Split.java deleted as buggy or redundant * BenchmarkBlooms.java move to test source dir * BloomUriUniqFilter.java change to accept filter instance (rather than parameters) for added configuration flexibility fix comments * BloomUriUniqFilterTest.java supply filter not paramters --- .../java/org/archive/util/BloomFilter.java | 33 ++- .../org/archive/util/BloomFilter32bit.java | 223 --------------- .../archive/util/BloomFilter32bitSplit.java | 251 ----------------- .../org/archive/util/BloomFilter32bp2.java | 235 ---------------- .../archive/util/BloomFilter32bp2Split.java | 262 ------------------ .../org/archive/util/BloomFilter64bit.java | 221 +++++++++++---- .../org/archive/util/BenchmarkBlooms.java | 67 +++-- .../archive/util/BloomFilter64bitTest.java | 32 +-- .../org/archive/util/BloomFilterTest.java | 158 +++++++++-- .../crawler/util/BloomUriUniqFilter.java | 66 ++--- .../crawler/util/BloomUriUniqFilterTest.java | 4 +- 11 files changed, 390 insertions(+), 1162 deletions(-) delete mode 100644 commons/src/main/java/org/archive/util/BloomFilter32bit.java delete mode 100644 commons/src/main/java/org/archive/util/BloomFilter32bitSplit.java delete mode 100644 commons/src/main/java/org/archive/util/BloomFilter32bp2.java delete mode 100644 commons/src/main/java/org/archive/util/BloomFilter32bp2Split.java rename commons/src/{main => test}/java/org/archive/util/BenchmarkBlooms.java (55%) diff --git a/commons/src/main/java/org/archive/util/BloomFilter.java b/commons/src/main/java/org/archive/util/BloomFilter.java index c83fbc5e..17de30d2 100644 --- a/commons/src/main/java/org/archive/util/BloomFilter.java +++ b/commons/src/main/java/org/archive/util/BloomFilter.java @@ -1,15 +1,11 @@ /* BloomFilter * -* $Id$ -* -* Created on Jun 30, 2005 -* -* Copyright (C) 2005 Internet Archive; an adaptation of +* Copyright (C) 2010 Internet Archive; an adaptation of * LGPL work (C) Sebastiano Vigna * * This file is part of the Heritrix web crawler (crawler.archive.org). * -* Heritrix is free software; you can redistribute it and/or modify +* This class is free software; you can redistribute it and/or modify * it under the terms of the GNU Lesser Public License as published by * the Free Software Foundation; either version 2.1 of the License, or * any later version. @@ -27,13 +23,13 @@ package org.archive.util; /** - * Common interface for different Bloom filter - * implementations + * Common interface for different Bloom filter implementations * * @author Gordon Mohr */ public interface BloomFilter { - /** The number of character sequences in the filter. + /** The number of character sequences in the filter (considered to be the + * number of add()s that returned 'true') * * @return the number of character sequences in the filter (but see {@link #contains(CharSequence)}). */ @@ -67,4 +63,23 @@ public interface BloomFilter { * @return memory used by bloom bitfield, in bytes */ public abstract long getSizeBytes(); + + /** + * Report the number of expected inserts used at instantiation time to + * calculate the bitfield size. + * + * @return long number of inserts expected at instantiation + */ + public abstract long getExpectedInserts(); + + /** + * Report the number of internal independent hash function (and thus the + * number of bits set/checked for each item presented). + * + * @return long count of hash functions + */ + public abstract long getHashCount(); + + // public for white-box unit testing + public boolean getBit(long bitIndex); } \ No newline at end of file diff --git a/commons/src/main/java/org/archive/util/BloomFilter32bit.java b/commons/src/main/java/org/archive/util/BloomFilter32bit.java deleted file mode 100644 index 7a52ce39..00000000 --- a/commons/src/main/java/org/archive/util/BloomFilter32bit.java +++ /dev/null @@ -1,223 +0,0 @@ -/* BloomFilter32bit -* -* $Id$ -* -* Created on Jun 21, 2005 -* -* Copyright (C) 2005 Internet Archive; a slight adaptation of -* LGPL work (C) Sebastiano Vigna -* -* This file is part of the Heritrix web crawler (crawler.archive.org). -* -* Heritrix is free software; you can redistribute it and/or modify -* it under the terms of the GNU Lesser Public License as published by -* the Free Software Foundation; either version 2.1 of the License, or -* any later version. -* -* Heritrix is distributed in the hope that it will be useful, -* but WITHOUT ANY WARRANTY; without even the implied warranty of -* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -* GNU Lesser Public License for more details. -* -* You should have received a copy of the GNU Lesser Public License -* along with Heritrix; if not, write to the Free Software -* Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA -*/ - -package org.archive.util; - -import java.io.Serializable; -import java.security.SecureRandom; - -/** A Bloom filter. - * - * SLIGHTLY ADAPTED VERSION OF MG4J it.unimi.dsi.mg4j.util.BloomFilter - * - *

KEY CHANGES: - * - *

- * - *
- * - *

Instances of this class represent a set of character sequences (with false positives) - * using a Bloom filter. Because of the way Bloom filters work, - * you cannot remove elements. - * - *

Bloom filters have an expected error rate, depending on the number - * of hash functions used, on the filter size and on the number of elements in the filter. This implementation - * uses a variable optimal number of hash functions, depending on the expected - * number of elements. More precisely, a Bloom - * filter for n character sequences with d hash functions will use - * ln 2 dn ≈ 1.44 dn bits; - * false positives will happen with probability 2-d. - * - *

Hash functions are generated at creation time using universal hashing. Each hash function - * uses {@link #NUMBER_OF_WEIGHTS} random integers, which are cyclically multiplied by - * the character codes in a character sequence. The resulting integers are XOR-ed together. - * - *

This class exports access methods that are very similar to those of {@link java.util.Set}, - * but it does not implement that interface, as too many non-optional methods - * would be unimplementable (e.g., iterators). - * - * @author Sebastiano Vigna - */ -public class BloomFilter32bit implements Serializable, BloomFilter { - - private static final long serialVersionUID = -1567837798979475689L; - - /** The number of weights used to create hash functions. */ - final public static int NUMBER_OF_WEIGHTS = 2083; // CHANGED FROM 16 - /** The number of bits in this filter. */ - final public long m; - /** The number of hash functions used by this filter. */ - final public int d; - /** The underlying bit vectorS. */ - final private int[] bits; - /** The random integers used to generate the hash functions. */ - final private int[][] weight; - - /** The number of elements currently in the filter. It may be - * smaller than the actual number of additions of distinct character - * sequences because of false positives. - */ - private int size; - - /** The natural logarithm of 2, used in the computation of the number of bits. */ - private final static double NATURAL_LOG_OF_2 = Math.log( 2 ); - - private final static boolean DEBUG = false; - - /** Creates a new Bloom filter with given number of hash functions and expected number of elements. - * - * @param n the expected number of elements. - * @param d the number of hash functions; if the filter add not more than n elements, - * false positives will happen with probability 2-d. - */ - public BloomFilter32bit( final int n, final int d ) { - this.d = d; - int len = - (int)Math.ceil( ( (long)n * (long)d / NATURAL_LOG_OF_2 ) / 32 ); - this.m = len*32L; - if ( m >= 1L<<32 ) { - throw new IllegalArgumentException( "This filter would require " + m + " bits" ); - } - bits = new int[ len ]; - - if ( DEBUG ) System.err.println( "Number of bits: " + m ); - - // seeded for reproduceable behavior in repeated runs; BUT: - // SecureRandom's default implementation (as of 1.5) - // seems to mix in its own seeding. - final SecureRandom random = new SecureRandom(new byte[] {19,96}); - weight = new int[ d ][]; - for( int i = 0; i < d; i++ ) { - weight[ i ] = new int[ NUMBER_OF_WEIGHTS ]; - for( int j = 0; j < NUMBER_OF_WEIGHTS; j++ ) - weight[ i ][ j ] = random.nextInt(); - } - } - - /** The number of character sequences in the filter. - * - * @return the number of character sequences in the filter (but see {@link #contains(CharSequence)}). - */ - - public int size() { - return size; - } - - /** Hashes the given sequence with the given hash function. - * - * @param s a character sequence. - * @param l the length of s. - * @param k a hash function index (smaller than {@link #d}). - * @return the position in the filter corresponding to s for the hash function k. - */ - private long hash( final CharSequence s, final int l, final int k ) { - final int[] w = weight[ k ]; - int h = 0, i = l; - while( i-- != 0 ) h ^= s.charAt( i ) * w[ i % NUMBER_OF_WEIGHTS ]; - return ((long)h-Integer.MIN_VALUE) % m; - } - - /** Checks whether the given character sequence is in this filter. - * - *

Note that this method may return true on a character sequence that is has - * not been added to the filter. This will happen with probability 2-d, - * where d is the number of hash functions specified at creation time, if - * the number of the elements in the filter is less than n, the number - * of expected elements specified at creation time. - * - * @param s a character sequence. - * @return true if the sequence is in the filter (or if a sequence with the - * same hash sequence is in the filter). - */ - - public boolean contains( final CharSequence s ) { - int i = d, l = s.length(); - while( i-- != 0 ) if ( ! getBit( hash( s, l, i ) ) ) return false; - return true; - } - - /** Adds a character sequence to the filter. - * - * @param s a character sequence. - * @return true if the character sequence was not in the filter (but see {@link #contains(CharSequence)}). - */ - - public boolean add( final CharSequence s ) { - boolean result = false; - int i = d, l = s.length(); - long h; - while( i-- != 0 ) { - h = hash( s, l, i ); - if ( ! getBit( h ) ) result = true; - setBit( h ); - } - if ( result ) size++; - return result; - } - - protected final static long ADDRESS_BITS_PER_UNIT = 5; // 32=2^5 - protected final static long BIT_INDEX_MASK = 31; // = BITS_PER_UNIT - 1; - - /** - * Returns from the local bitvector the value of the bit with - * the specified index. The value is true if the bit - * with the index bitIndex is currently set; otherwise, - * returns false. - * - * (adapted from cern.colt.bitvector.QuickBitVector) - * - * @param bitIndex the bit index. - * @return the value of the bit with the specified index. - */ - protected boolean getBit(long bitIndex) { - return ((bits[(int)(bitIndex >> ADDRESS_BITS_PER_UNIT)] & (1 << (bitIndex & BIT_INDEX_MASK))) != 0); - } - - /** - * Changes the bit with index bitIndex in local bitvector. - * - * (adapted from cern.colt.bitvector.QuickBitVector) - * - * @param bitIndex the index of the bit to be set. - */ - protected void setBit(long bitIndex) { - bits[(int)(bitIndex >> ADDRESS_BITS_PER_UNIT)] |= 1 << (bitIndex & BIT_INDEX_MASK); - } - - /* (non-Javadoc) - * @see org.archive.util.BloomFilter#getSizeBytes() - */ - public long getSizeBytes() { - return bits.length*4; - } -} diff --git a/commons/src/main/java/org/archive/util/BloomFilter32bitSplit.java b/commons/src/main/java/org/archive/util/BloomFilter32bitSplit.java deleted file mode 100644 index fd71c884..00000000 --- a/commons/src/main/java/org/archive/util/BloomFilter32bitSplit.java +++ /dev/null @@ -1,251 +0,0 @@ -/* BloomFilter32bit -* -* $Id$ -* -* Created on Jun 21, 2005 -* -* Copyright (C) 2005 Internet Archive; a slight adaptation of -* LGPL work (C) Sebastiano Vigna -* -* This file is part of the Heritrix web crawler (crawler.archive.org). -* -* Heritrix is free software; you can redistribute it and/or modify -* it under the terms of the GNU Lesser Public License as published by -* the Free Software Foundation; either version 2.1 of the License, or -* any later version. -* -* Heritrix is distributed in the hope that it will be useful, -* but WITHOUT ANY WARRANTY; without even the implied warranty of -* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -* GNU Lesser Public License for more details. -* -* You should have received a copy of the GNU Lesser Public License -* along with Heritrix; if not, write to the Free Software -* Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA -*/ - -package org.archive.util; - -import java.io.Serializable; -import java.security.SecureRandom; - -/** A Bloom filter. - * - * SLIGHTLY ADAPTED VERSION OF MG4J it.unimi.dsi.mg4j.util.BloomFilter - * - *

KEY CHANGES: - * - *

- * - *
- * - *

Instances of this class represent a set of character sequences (with false positives) - * using a Bloom filter. Because of the way Bloom filters work, - * you cannot remove elements. - * - *

Bloom filters have an expected error rate, depending on the number - * of hash functions used, on the filter size and on the number of elements in the filter. This implementation - * uses a variable optimal number of hash functions, depending on the expected - * number of elements. More precisely, a Bloom - * filter for n character sequences with d hash functions will use - * ln 2 dn ≈ 1.44 dn bits; - * false positives will happen with probability 2-d. - * - *

Hash functions are generated at creation time using universal hashing. Each hash function - * uses {@link #NUMBER_OF_WEIGHTS} random integers, which are cyclically multiplied by - * the character codes in a character sequence. The resulting integers are XOR-ed together. - * - *

This class exports access methods that are very similar to those of {@link java.util.Set}, - * but it does not implement that interface, as too many non-optional methods - * would be unimplementable (e.g., iterators). - * - * @author Sebastiano Vigna - */ -public class BloomFilter32bitSplit implements Serializable, BloomFilter { - - private static final long serialVersionUID = -164106965277863971L; - - /** The number of weights used to create hash functions. */ - final public static int NUMBER_OF_WEIGHTS = 2083; // CHANGED FROM 16 - /** The number of bits in this filter. */ - final public long m; - /** The number of hash functions used by this filter. */ - final public int d; - /** The underlying bit vectorS. */ -// final private int[] bits; - final private int[][] bits; - /** The random integers used to generate the hash functions. */ - final private int[][] weight; - - /** The number of elements currently in the filter. It may be - * smaller than the actual number of additions of distinct character - * sequences because of false positives. - */ - private int size; - - /** The natural logarithm of 2, used in the computation of the number of bits. */ - private final static double NATURAL_LOG_OF_2 = Math.log( 2 ); - - /** number of ints in 1MB. */ - private final static int ONE_MB_INTS = 1 << 18; // - - private final static boolean DEBUG = false; - - /** Creates a new Bloom filter with given number of hash functions and expected number of elements. - * - * @param n the expected number of elements. - * @param d the number of hash functions; if the filter add not more than n elements, - * false positives will happen with probability 2-d. - */ - public BloomFilter32bitSplit( final int n, final int d ) { - this.d = d; - int len = - (int)Math.ceil( ( (long)n * (long)d / NATURAL_LOG_OF_2 ) / 32 ); - // round up to ensure divisible into 1MiB chunks - len = ((len / ONE_MB_INTS)+1)*ONE_MB_INTS; - this.m = len*32L; - if ( m >= 1L<<54 ) { - throw new IllegalArgumentException( "This filter would require " + m + " bits" ); - } -// bits = new int[ len ]; - bits = new int[ len/ONE_MB_INTS ][ONE_MB_INTS]; - - if ( DEBUG ) System.err.println( "Number of bits: " + m ); - - // seeded for reproduceable behavior in repeated runs; BUT: - // SecureRandom's default implementation (as of 1.5) - // seems to mix in its own seeding. - final SecureRandom random = new SecureRandom(new byte[] {19,96}); - weight = new int[ d ][]; - for( int i = 0; i < d; i++ ) { - weight[ i ] = new int[ NUMBER_OF_WEIGHTS ]; - for( int j = 0; j < NUMBER_OF_WEIGHTS; j++ ) - weight[ i ][ j ] = random.nextInt(); - } - } - - /** The number of character sequences in the filter. - * - * @return the number of character sequences in the filter (but see {@link #contains(CharSequence)}). - */ - - public int size() { - return size; - } - - /** Hashes the given sequence with the given hash function. - * - * @param s a character sequence. - * @param l the length of s. - * @param k a hash function index (smaller than {@link #d}). - * @return the position in the filter corresponding to s for the hash function k. - */ - private long hash( final CharSequence s, final int l, final int k ) { - final int[] w = weight[ k ]; - int h = 0, i = l; - while( i-- != 0 ) h ^= s.charAt( i ) * w[ i % NUMBER_OF_WEIGHTS ]; - return ((long)h-Integer.MIN_VALUE) % m; - } - - /** Checks whether the given character sequence is in this filter. - * - *

Note that this method may return true on a character sequence that is has - * not been added to the filter. This will happen with probability 2-d, - * where d is the number of hash functions specified at creation time, if - * the number of the elements in the filter is less than n, the number - * of expected elements specified at creation time. - * - * @param s a character sequence. - * @return true if the sequence is in the filter (or if a sequence with the - * same hash sequence is in the filter). - */ - - public boolean contains( final CharSequence s ) { - int i = d, l = s.length(); - while( i-- != 0 ) if ( ! getBit( hash( s, l, i ) ) ) return false; - return true; - } - - /** Adds a character sequence to the filter. - * - * @param s a character sequence. - * @return true if the character sequence was not in the filter (but see {@link #contains(CharSequence)}). - */ - - public boolean add( final CharSequence s ) { - boolean result = false; - int i = d, l = s.length(); - long h; - while( i-- != 0 ) { - h = hash( s, l, i ); - if ( ! setGetBit( h ) ) result = true; - } - if ( result ) size++; - return result; - } - - protected final static long ADDRESS_BITS_PER_UNIT = 5; // 32=2^5 - protected final static long BIT_INDEX_MASK = 31; // = BITS_PER_UNIT - 1; - - /** - * Returns from the local bitvector the value of the bit with - * the specified index. The value is true if the bit - * with the index bitIndex is currently set; otherwise, - * returns false. - * - * (adapted from cern.colt.bitvector.QuickBitVector) - * - * @param bitIndex the bit index. - * @return the value of the bit with the specified index. - */ - protected boolean getBit(long bitIndex) { - long intIndex = (bitIndex >>> ADDRESS_BITS_PER_UNIT); - return ((bits[(int)(intIndex / ONE_MB_INTS)][(int)(intIndex % ONE_MB_INTS)] - & (1 << (bitIndex & BIT_INDEX_MASK))) != 0); - } - - /** - * Changes the bit with index bitIndex in local bitvector. - * - * (adapted from cern.colt.bitvector.QuickBitVector) - * - * @param bitIndex the index of the bit to be set. - */ - protected void setBit(long bitIndex) { - long intIndex = (bitIndex >>> ADDRESS_BITS_PER_UNIT); - bits[(int)(intIndex / ONE_MB_INTS)][(int)(intIndex % ONE_MB_INTS)] - |= 1 << (bitIndex & BIT_INDEX_MASK); - } - - /** - * Sets the bit with index bitIndex in local bitvector -- - * returning the old value. - * - * (adapted from cern.colt.bitvector.QuickBitVector) - * - * @param bitIndex the index of the bit to be set. - */ - protected boolean setGetBit(long bitIndex) { - long intIndex = (int) (bitIndex >>> ADDRESS_BITS_PER_UNIT); - int a = (int)(intIndex / ONE_MB_INTS); - int b = (int)(intIndex % ONE_MB_INTS); - int mask = 1 << (bitIndex & BIT_INDEX_MASK); - boolean ret = ((bits[a][b] & (mask)) != 0); - bits[a][b] |= mask; - return ret; - } - - /* (non-Javadoc) - * @see org.archive.util.BloomFilter#getSizeBytes() - */ - public long getSizeBytes() { - return bits.length*bits[0].length*4; - } -} diff --git a/commons/src/main/java/org/archive/util/BloomFilter32bp2.java b/commons/src/main/java/org/archive/util/BloomFilter32bp2.java deleted file mode 100644 index ffa64d66..00000000 --- a/commons/src/main/java/org/archive/util/BloomFilter32bp2.java +++ /dev/null @@ -1,235 +0,0 @@ -/* BloomFilter -* -* $Id$ -* -* Created on Jun 21, 2005 -* -* Copyright (C) 2005 Internet Archive; a slight adaptation of -* LGPL work (C) Sebastiano Vigna -* -* This file is part of the Heritrix web crawler (crawler.archive.org). -* -* Heritrix is free software; you can redistribute it and/or modify -* it under the terms of the GNU Lesser Public License as published by -* the Free Software Foundation; either version 2.1 of the License, or -* any later version. -* -* Heritrix is distributed in the hope that it will be useful, -* but WITHOUT ANY WARRANTY; without even the implied warranty of -* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -* GNU Lesser Public License for more details. -* -* You should have received a copy of the GNU Lesser Public License -* along with Heritrix; if not, write to the Free Software -* Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA -*/ - -package org.archive.util; - -import java.io.Serializable; -import java.security.SecureRandom; - -/** A Bloom filter. - * - * SLIGHTLY ADAPTED VERSION OF MG4J it.unimi.dsi.mg4j.util.BloomFilter - * - *

KEY CHANGES: - * - *

- * - *
- * - *

Instances of this class represent a set of character sequences (with false positives) - * using a Bloom filter. Because of the way Bloom filters work, - * you cannot remove elements. - * - *

Bloom filters have an expected error rate, depending on the number - * of hash functions used, on the filter size and on the number of elements in the filter. This implementation - * uses a variable optimal number of hash functions, depending on the expected - * number of elements. More precisely, a Bloom - * filter for n character sequences with d hash functions will use - * ln 2 dn ≈ 1.44 dn bits; - * false positives will happen with probability 2-d. - * - *

Hash functions are generated at creation time using universal hashing. Each hash function - * uses {@link #NUMBER_OF_WEIGHTS} random integers, which are cyclically multiplied by - * the character codes in a character sequence. The resulting integers are XOR-ed together. - * - *

This class exports access methods that are very similar to those of {@link java.util.Set}, - * but it does not implement that interface, as too many non-optional methods - * would be unimplementable (e.g., iterators). - * - * @author Sebastiano Vigna - */ -public class BloomFilter32bp2 implements Serializable, BloomFilter { - - private static final long serialVersionUID = -2292902803681146635L; - - /** The number of weights used to create hash functions. */ - final public static int NUMBER_OF_WEIGHTS = 2083; // CHANGED FROM 16 - /** The number of bits in this filter. */ - final public long m; - /** the power-of-two that m is */ - final public long power; // 1<n elements, - * false positives will happen with probability 2-d. - */ - public BloomFilter32bp2( final int n, final int d ) { - this.d = d; - long minBits = (long) ((long)n * (long)d / NATURAL_LOG_OF_2); - long pow = 0; - while((1L< 1L<<32 ) { - throw new IllegalArgumentException( "This filter would require " + m + " bits" ); - } - System.out.println("power "+power+" bits "+m+" len "+len); - - bits = new int[ len ]; - - if ( DEBUG ) System.err.println( "Number of bits: " + m ); - - // seeded for reproduceable behavior in repeated runs; BUT: - // SecureRandom's default implementation (as of 1.5) - // seems to mix in its own seeding. - final SecureRandom random = new SecureRandom(new byte[] {19,96}); - weight = new int[ d ][]; - for( int i = 0; i < d; i++ ) { - weight[ i ] = new int[ NUMBER_OF_WEIGHTS ]; - for( int j = 0; j < NUMBER_OF_WEIGHTS; j++ ) - weight[ i ][ j ] = random.nextInt(); - } - } - - /** The number of character sequences in the filter. - * - * @return the number of character sequences in the filter (but see {@link #contains(CharSequence)}). - */ - - public int size() { - return size; - } - - /** Hashes the given sequence with the given hash function. - * - * @param s a character sequence. - * @param l the length of s. - * @param k a hash function index (smaller than {@link #d}). - * @return the position in the filter corresponding to s for the hash function k. - */ - private int hash( final CharSequence s, final int l, final int k ) { - final int[] w = weight[ k ]; - int h = 0, i = l; - while( i-- != 0 ) h ^= s.charAt( i ) * w[ i % NUMBER_OF_WEIGHTS ]; - return h >>> (32-power); - } - - /** Checks whether the given character sequence is in this filter. - * - *

Note that this method may return true on a character sequence that is has - * not been added to the filter. This will happen with probability 2-d, - * where d is the number of hash functions specified at creation time, if - * the number of the elements in the filter is less than n, the number - * of expected elements specified at creation time. - * - * @param s a character sequence. - * @return true if the sequence is in the filter (or if a sequence with the - * same hash sequence is in the filter). - */ - - public boolean contains( final CharSequence s ) { - int i = d, l = s.length(); - while( i-- != 0 ) if ( ! getBit( hash( s, l, i ) ) ) return false; - return true; - } - - /** Adds a character sequence to the filter. - * - * @param s a character sequence. - * @return true if the character sequence was not in the filter (but see {@link #contains(CharSequence)}). - */ - - public boolean add( final CharSequence s ) { - boolean result = false; - int i = d, l = s.length(); - int h; - while( i-- != 0 ) { - h = hash( s, l, i ); - if ( ! getBit( h ) ) result = true; - setBit( h ); - } - if ( result ) size++; - return result; - } - - protected final static int ADDRESS_BITS_PER_UNIT = 5; // 32=2^5 - protected final static int BIT_INDEX_MASK = 31; // = BITS_PER_UNIT - 1; - - /** - * Returns from the local bitvector the value of the bit with - * the specified index. The value is true if the bit - * with the index bitIndex is currently set; otherwise, - * returns false. - * - * (adapted from cern.colt.bitvector.QuickBitVector) - * - * @param bitIndex the bit index. - * @return the value of the bit with the specified index. - */ - protected boolean getBit(int bitIndex) { - return ((bits[(int)(bitIndex >>> ADDRESS_BITS_PER_UNIT)] & (1 << (bitIndex & BIT_INDEX_MASK))) != 0); - } - - /** - * Changes the bit with index bitIndex in local bitvector. - * - * (adapted from cern.colt.bitvector.QuickBitVector) - * - * @param bitIndex the index of the bit to be set. - */ - protected void setBit(int bitIndex) { - bits[(int)(bitIndex >>> ADDRESS_BITS_PER_UNIT)] |= 1 << (bitIndex & BIT_INDEX_MASK); - } - - /* (non-Javadoc) - * @see org.archive.util.BloomFilter#getSizeBytes() - */ - public long getSizeBytes() { - return bits.length*4; - } -} diff --git a/commons/src/main/java/org/archive/util/BloomFilter32bp2Split.java b/commons/src/main/java/org/archive/util/BloomFilter32bp2Split.java deleted file mode 100644 index aba45f75..00000000 --- a/commons/src/main/java/org/archive/util/BloomFilter32bp2Split.java +++ /dev/null @@ -1,262 +0,0 @@ -/* BloomFilter -* -* $Id$ -* -* Created on Jun 21, 2005 -* -* Copyright (C) 2005 Internet Archive; a slight adaptation of -* LGPL work (C) Sebastiano Vigna -* -* This file is part of the Heritrix web crawler (crawler.archive.org). -* -* Heritrix is free software; you can redistribute it and/or modify -* it under the terms of the GNU Lesser Public License as published by -* the Free Software Foundation; either version 2.1 of the License, or -* any later version. -* -* Heritrix is distributed in the hope that it will be useful, -* but WITHOUT ANY WARRANTY; without even the implied warranty of -* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -* GNU Lesser Public License for more details. -* -* You should have received a copy of the GNU Lesser Public License -* along with Heritrix; if not, write to the Free Software -* Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA -*/ - -package org.archive.util; - -import java.io.Serializable; -import java.security.SecureRandom; - -/** A Bloom filter. - * - * SLIGHTLY ADAPTED VERSION OF MG4J it.unimi.dsi.mg4j.util.BloomFilter - * - *

KEY CHANGES: - * - *

- * - *
- * - *

Instances of this class represent a set of character sequences (with false positives) - * using a Bloom filter. Because of the way Bloom filters work, - * you cannot remove elements. - * - *

Bloom filters have an expected error rate, depending on the number - * of hash functions used, on the filter size and on the number of elements in the filter. This implementation - * uses a variable optimal number of hash functions, depending on the expected - * number of elements. More precisely, a Bloom - * filter for n character sequences with d hash functions will use - * ln 2 dn ≈ 1.44 dn bits; - * false positives will happen with probability 2-d. - * - *

Hash functions are generated at creation time using universal hashing. Each hash function - * uses {@link #NUMBER_OF_WEIGHTS} random integers, which are cyclically multiplied by - * the character codes in a character sequence. The resulting integers are XOR-ed together. - * - *

This class exports access methods that are very similar to those of {@link java.util.Set}, - * but it does not implement that interface, as too many non-optional methods - * would be unimplementable (e.g., iterators). - * - * @author Sebastiano Vigna - */ -public class BloomFilter32bp2Split implements Serializable, BloomFilter { - - private static final long serialVersionUID = -1504889954381695129L; - - /** The number of weights used to create hash functions. */ - final public static int NUMBER_OF_WEIGHTS = 2083; // CHANGED FROM 16 - /** The number of bits in this filter. */ - final public long m; - /** the power-of-two that m is */ - final public long power; // 1<n elements, - * false positives will happen with probability 2-d. - */ - public BloomFilter32bp2Split( final int n, final int d ) { - this.d = d; - long minBits = (long) ((long)n * (long)d / NATURAL_LOG_OF_2); - long pow = 0; - while((1L< 1L<<32 ) { - throw new IllegalArgumentException( "This filter would require " + m + " bits" ); - } - - aShift = (int) (pow - ADDRESS_BITS_PER_UNIT - 8); - bMask = (1<s. - * @param k a hash function index (smaller than {@link #d}). - * @return the position in the filter corresponding to s for the hash function k. - */ - private int hash( final CharSequence s, final int l, final int k ) { - final int[] w = weight[ k ]; - int h = 0, i = l; - while( i-- != 0 ) h ^= s.charAt( i ) * w[ i % NUMBER_OF_WEIGHTS ]; - return h >>> (32-power); - } - - /** Checks whether the given character sequence is in this filter. - * - *

Note that this method may return true on a character sequence that is has - * not been added to the filter. This will happen with probability 2-d, - * where d is the number of hash functions specified at creation time, if - * the number of the elements in the filter is less than n, the number - * of expected elements specified at creation time. - * - * @param s a character sequence. - * @return true if the sequence is in the filter (or if a sequence with the - * same hash sequence is in the filter). - */ - - public boolean contains( final CharSequence s ) { - int i = d, l = s.length(); - while( i-- != 0 ) if ( ! getBit( hash( s, l, i ) ) ) return false; - return true; - } - - /** Adds a character sequence to the filter. - * - * @param s a character sequence. - * @return true if the character sequence was not in the filter (but see {@link #contains(CharSequence)}). - */ - - public boolean add( final CharSequence s ) { - boolean result = false; - int i = d, l = s.length(); - int h; - while( i-- != 0 ) { - h = hash( s, l, i ); - if ( ! setGetBit( h ) ) result = true; - } - if ( result ) size++; - return result; - } - - protected final static int ADDRESS_BITS_PER_UNIT = 5; // 32=2^5 - protected final static int BIT_INDEX_MASK = 31; // = BITS_PER_UNIT - 1; - - /** - * Returns from the local bitvector the value of the bit with - * the specified index. The value is true if the bit - * with the index bitIndex is currently set; otherwise, - * returns false. - * - * (adapted from cern.colt.bitvector.QuickBitVector) - * - * @param bitIndex the bit index. - * @return the value of the bit with the specified index. - */ - protected boolean getBit(int bitIndex) { - int intIndex = (int)(bitIndex >>> ADDRESS_BITS_PER_UNIT); - return ((bits[intIndex>>>aShift][intIndex&bMask] & (1 << (bitIndex & BIT_INDEX_MASK))) != 0); - } - - /** - * Changes the bit with index bitIndex in local bitvector. - * - * (adapted from cern.colt.bitvector.QuickBitVector) - * - * @param bitIndex the index of the bit to be set. - */ - protected void setBit(int bitIndex) { - int intIndex = (int)(bitIndex >>> ADDRESS_BITS_PER_UNIT); - bits[intIndex>>>aShift][intIndex&bMask] |= 1 << (bitIndex & BIT_INDEX_MASK); - } - - /** - * Sets the bit with index bitIndex in local bitvector -- - * returning the old value. - * - * (adapted from cern.colt.bitvector.QuickBitVector) - * - * @param bitIndex the index of the bit to be set. - */ - protected boolean setGetBit(int bitIndex) { - int intIndex = (int)(bitIndex >>> ADDRESS_BITS_PER_UNIT); - int a = intIndex>>>aShift; - int b = intIndex&bMask; - int mask = 1 << (bitIndex & BIT_INDEX_MASK); - boolean ret = ((bits[a][b] & (mask)) != 0); - bits[a][b] |= mask; - return ret; - } - - /* (non-Javadoc) - * @see org.archive.util.BloomFilter#getSizeBytes() - */ - public long getSizeBytes() { - return bits.length*bits[0].length*4; - } -} diff --git a/commons/src/main/java/org/archive/util/BloomFilter64bit.java b/commons/src/main/java/org/archive/util/BloomFilter64bit.java index 97b0d06e..aeca26ed 100644 --- a/commons/src/main/java/org/archive/util/BloomFilter64bit.java +++ b/commons/src/main/java/org/archive/util/BloomFilter64bit.java @@ -28,104 +28,165 @@ package org.archive.util; import java.io.Serializable; import java.security.SecureRandom; +import java.util.Random; /** A Bloom filter. * - * SLIGHTLY ADAPTED VERSION OF MG4J it.unimi.dsi.mg4j.util.BloomFilter + * ADAPTED/IMPROVED VERSION OF MG4J it.unimi.dsi.mg4j.util.BloomFilter * *

KEY CHANGES: * *

* *
* - *

Instances of this class represent a set of character sequences (with false positives) - * using a Bloom filter. Because of the way Bloom filters work, + *

Instances of this class represent a set of character sequences (with + * false positives) using a Bloom filter. Because of the way Bloom filters work, * you cannot remove elements. * *

Bloom filters have an expected error rate, depending on the number - * of hash functions used, on the filter size and on the number of elements in the filter. This implementation - * uses a variable optimal number of hash functions, depending on the expected - * number of elements. More precisely, a Bloom - * filter for n character sequences with d hash functions will use - * ln 2 dn ≈ 1.44 dn bits; - * false positives will happen with probability 2-d. + * of hash functions used, on the filter size and on the number of elements in + * the filter. This implementation uses a variable optimal number of hash + * functions, depending on the expected number of elements. More precisely, a + * Bloom filter for n character sequences with d hash + * functions will use ln 2 dn ≈ + * 1.44 dn bits; false positives will happen with + * probability 2-d. * - *

Hash functions are generated at creation time using universal hashing. Each hash function - * uses {@link #NUMBER_OF_WEIGHTS} random integers, which are cyclically multiplied by - * the character codes in a character sequence. The resulting integers are XOR-ed together. + *

Hash functions are generated at creation time using universal hashing. + * Each hash function uses {@link #NUMBER_OF_WEIGHTS} random integers, which + * are cyclically multiplied by the character codes in a character sequence. + * The resulting integers are XOR-ed together. * - *

This class exports access methods that are very similar to those of {@link java.util.Set}, - * but it does not implement that interface, as too many non-optional methods - * would be unimplementable (e.g., iterators). + *

This class exports access methods that are very similar to those of + * {@link java.util.Set}, but it does not implement that interface, as too + * many non-optional methods would be unimplementable (e.g., iterators). * * @author Sebastiano Vigna + * @contributor Gordon Mohr */ public class BloomFilter64bit implements Serializable, BloomFilter { - - private static final long serialVersionUID = 2317000663009608403L; + private static final long serialVersionUID = 2L; /** The number of weights used to create hash functions. */ - final public static int NUMBER_OF_WEIGHTS = 2083; // CHANGED FROM 16 + final static int NUMBER_OF_WEIGHTS = 2083; // CHANGED FROM 16 /** The number of bits in this filter. */ - final public long m; + final protected long m; + /** if bitfield is an exact power of 2 in length, it is this power */ + protected int power = -1; + /** The expected number of inserts; determines calculated size */ + final protected long expectedInserts; /** The number of hash functions used by this filter. */ - final public int d; - /** The underlying bit vector. package access for testing */ - final long[] bits; + final protected int d; + /** The underlying bit vector */ + final protected long[][] bits; /** The random integers used to generate the hash functions. */ - final long[][] weight; + final protected long[][] weight; /** The number of elements currently in the filter. It may be * smaller than the actual number of additions of distinct character * sequences because of false positives. */ - private int size; + int size; /** The natural logarithm of 2, used in the computation of the number of bits. */ - private final static double NATURAL_LOG_OF_2 = Math.log( 2 ); + final static double NATURAL_LOG_OF_2 = Math.log( 2 ); - private final static boolean DEBUG = false; + /** power-of-two to use as maximum size of bitfield subarrays */ + protected final static int SUBARRAY_POWER_OF_TWO = 26; // 512MiB of longs + /** number of longs in one subarray */ + protected final static int SUBARRAY_LENGTH_IN_LONGS = 1 << SUBARRAY_POWER_OF_TWO; + /** mask for lowest SUBARRAY_POWER_OF_TWO bits */ + protected final static int SUBARRAY_MASK = SUBARRAY_LENGTH_IN_LONGS - 1; //0x0FFFFFFF - /** Creates a new Bloom filter with given number of hash functions and expected number of elements. + final static boolean DEBUG = false; + + /** Creates a new Bloom filter with given number of hash functions and + * expected number of elements. * * @param n the expected number of elements. - * @param d the number of hash functions; if the filter add not more than n elements, - * false positives will happen with probability 2-d. + * @param d the number of hash functions; if the filter add not more + * than n elements, false positives will happen with + * probability 2-d. */ - public BloomFilter64bit( final int n, final int d ) { + public BloomFilter64bit( final long n, final int d) { + this(n,d, new SecureRandom(), false); + } + + public BloomFilter64bit( final long n, final int d, boolean roundUp) { + this(n,d, new SecureRandom(), roundUp); + } + + /** Creates a new Bloom filter with given number of hash functions and + * expected number of elements. + * + * @param n the expected number of elements. + * @param d the number of hash functions; if the filter add not more + * than n elements, false positives will happen with + * probability 2-d. + * @param Random weightsGenerator may provide a seeded Random for reproducible + * internal universal hash function weighting + * @param roundUp if true, round bit size up to next-nearest-power-of-2 + */ + public BloomFilter64bit( final long n, final int d, Random weightsGenerator, boolean roundUp ) { + this.expectedInserts = n; this.d = d; - int len = (int)Math.ceil( ( (long)n * (long)d / NATURAL_LOG_OF_2 ) / 64L ); - if ( len/64 > Integer.MAX_VALUE ) throw new IllegalArgumentException( "This filter would require " + len * 64L + " bits" ); - bits = new long[ len ]; - m = bits.length * 64L; + long lenInLongs = (long)Math.ceil( ( (long)n * (long)d / NATURAL_LOG_OF_2 ) / 64L ); + if ( lenInLongs > (1L<<48) ) { + throw new IllegalArgumentException( + "This filter would require " + lenInLongs + " longs, " + + "greater than this classes maximum of 2^48 longs (2PiB)." ); + } + long lenInBits = lenInLongs * 64L; + + if(roundUp) { + int pow = 0; + while((1L<s for the hash function k. */ - - private long hash( final CharSequence s, final int l, final int k ) { + protected long hash( final CharSequence s, final int l, final int k ) { final long[] w = weight[ k ]; long h = 0; int i = l; while( i-- != 0 ) h ^= s.charAt( i ) * w[ i % NUMBER_OF_WEIGHTS ]; - return ( h & 0x7FFFFFFFFFFFFFFFL ) % m; + long retVal; + if(power>0) { + retVal = h >>> (64-power); + } else { + // ####----####---- + retVal = ( h & 0x7FFFFFFFFFFFFFFFL ) % m; + } + return retVal; } - + + public long[] bitIndexesFor(CharSequence s) { + long[] ret = new long[d]; + for(int i = 0; i < d; i++) { + ret[i] = hash(s,s.length(),i); + } + return ret; + } + /** Checks whether the given character sequence is in this filter. * *

Note that this method may return true on a character sequence that is has @@ -179,9 +254,8 @@ public class BloomFilter64bit implements Serializable, BloomFilter { long h; while( i-- != 0 ) { h = hash( s, l, i ); - if ( ! getBit( h ) ) { + if ( ! setGetBit( h ) ) { result = true; - setBit( h ); } } if ( result ) size++; @@ -189,7 +263,7 @@ public class BloomFilter64bit implements Serializable, BloomFilter { } protected final static long ADDRESS_BITS_PER_UNIT = 6; // 64=2^6 - protected final static long BIT_INDEX_MASK = 63; // = BITS_PER_UNIT - 1; + protected final static long BIT_INDEX_MASK = (1<<6)-1; // = 63 = 2^BITS_PER_UNIT - 1; /** * Returns from the local bitvector the value of the bit with @@ -202,8 +276,11 @@ public class BloomFilter64bit implements Serializable, BloomFilter { * @param bitIndex the bit index. * @return the value of the bit with the specified index. */ - protected boolean getBit(long bitIndex) { - return ((bits[(int)(bitIndex >> ADDRESS_BITS_PER_UNIT)] & (1L << (bitIndex & BIT_INDEX_MASK))) != 0); + public boolean getBit(long bitIndex) { + long longIndex = bitIndex >>> ADDRESS_BITS_PER_UNIT; + int arrayIndex = (int) (longIndex >>> SUBARRAY_POWER_OF_TWO); + int subarrayIndex = (int) (longIndex & SUBARRAY_MASK); + return ((bits[arrayIndex][subarrayIndex] & (1L << (bitIndex & BIT_INDEX_MASK))) != 0); } /** @@ -214,13 +291,45 @@ public class BloomFilter64bit implements Serializable, BloomFilter { * @param bitIndex the index of the bit to be set. */ protected void setBit( long bitIndex) { - bits[(int)(bitIndex >> ADDRESS_BITS_PER_UNIT)] |= 1L << (bitIndex & BIT_INDEX_MASK); + long longIndex = bitIndex >>> ADDRESS_BITS_PER_UNIT; + int arrayIndex = (int) (longIndex >>> SUBARRAY_POWER_OF_TWO); + int subarrayIndex = (int) (longIndex & SUBARRAY_MASK); + bits[arrayIndex][subarrayIndex] |= (1L << (bitIndex & BIT_INDEX_MASK)); + } + + /** + * Sets the bit with index bitIndex in local bitvector -- + * returning the old value. + * + * (adapted from cern.colt.bitvector.QuickBitVector) + * + * @param bitIndex the index of the bit to be set. + */ + protected boolean setGetBit( long bitIndex) { + long longIndex = bitIndex >>> ADDRESS_BITS_PER_UNIT; + int arrayIndex = (int) (longIndex >>> SUBARRAY_POWER_OF_TWO); + int subarrayIndex = (int) (longIndex & SUBARRAY_MASK); + long mask = 1L << (bitIndex & BIT_INDEX_MASK); + boolean ret = (bits[arrayIndex][subarrayIndex] & mask)!=0; + bits[arrayIndex][subarrayIndex] |= mask; + return ret; } /* (non-Javadoc) * @see org.archive.util.BloomFilter#getSizeBytes() */ public long getSizeBytes() { - return bits.length*8; + // account for ragged-sized last array + return 8*(((bits.length-1)*bits[0].length)+bits[bits.length-1].length); } + + @Override + public long getExpectedInserts() { + return expectedInserts; + } + + @Override + public long getHashCount() { + return d; + } } diff --git a/commons/src/main/java/org/archive/util/BenchmarkBlooms.java b/commons/src/test/java/org/archive/util/BenchmarkBlooms.java similarity index 55% rename from commons/src/main/java/org/archive/util/BenchmarkBlooms.java rename to commons/src/test/java/org/archive/util/BenchmarkBlooms.java index 535564f9..b3f20678 100644 --- a/commons/src/main/java/org/archive/util/BenchmarkBlooms.java +++ b/commons/src/test/java/org/archive/util/BenchmarkBlooms.java @@ -49,35 +49,34 @@ public class BenchmarkBlooms { int d_hashes = (args.length > 2) ? Integer.parseInt(args[2]) : 22; int adds = - (args.length > 3) ? Integer.parseInt(args[3]) : 5000000; + (args.length > 3) ? Integer.parseInt(args[3]) : 10000000; + int contains = + (args.length > 4) ? Integer.parseInt(args[4]) : 8000000; String prefix = - (args.length > 4) ? args[4] : "http://www.archive.org/"; + (args.length > 5) ? args[5] : "http://www.archive.org/"; System.out.println( "reps="+reps+" n_expected="+n_expected+ - " d_hashes="+d_hashes+" adds="+adds+" prefix="+prefix); + " d_hashes="+d_hashes+" adds="+adds+ + " contains="+contains+" prefix="+prefix); - BloomFilter bloom64; - BloomFilter bloom32; - BloomFilter bloom32split; - BloomFilter bloom32p2; - BloomFilter bloom32p2split; + BloomFilter64bit bloom64; +// BloomFilter bloom32; +// BloomFilter bloom32split; for (int r=0;r0) { - assertTrue("set bits not as expected in early positions",(i/(double)bloom64.bits.length)<0.2d); - break; - } - } - for(int i = bloom64.bits.length-1; i>=0; i--) { - // verify that first set bit is in first 20% of bitfield - if(bloom64.bits[i]>0) { - assertTrue("set bits not as expected in late positions",(i/(double)bloom64.bits.length)>0.8d); - break; - } - } - + @Override + BloomFilter createBloom(long n, int d, Random weightsGenerator) { + return new BloomFilter64bit(n, d, weightsGenerator, false); } } diff --git a/commons/src/test/java/org/archive/util/BloomFilterTest.java b/commons/src/test/java/org/archive/util/BloomFilterTest.java index d61ecfae..769b8cef 100644 --- a/commons/src/test/java/org/archive/util/BloomFilterTest.java +++ b/commons/src/test/java/org/archive/util/BloomFilterTest.java @@ -19,39 +19,155 @@ package org.archive.util; +import java.security.SecureRandom; +import java.util.Random; + import junit.framework.TestCase; /** - * BloomFilter tests + * BloomFilter tests. * * @contributor gojomo * @version $Date: 2009-11-19 14:39:53 -0800 (Thu, 19 Nov 2009) $, $Revision: 6674 $ */ public abstract class BloomFilterTest extends TestCase { - protected BloomFilter bloom; - protected abstract void setUp() throws Exception; - - public void testBasics() { - // require initial additions to return 'true' (for 'added') - assertTrue(bloom.add("abracadabra")); - assertTrue(bloom.add("foobar")); - assertTrue(bloom.add("rumplestiltskin")); - assertTrue(bloom.add("buckaroobanzai")); - assertTrue(bloom.add("scheherazade")); + abstract BloomFilter createBloom(long n, int d, Random random); + + protected void trialWithParameters(long targetSize, int hashCount, long addCount, long containsCount) { + BloomFilter bloom = createBloom(targetSize,hashCount,new Random(1996L)); - // require readdition to return 'false' (not added because already present) - assertFalse(bloom.add("abracadabra")); - assertFalse(bloom.add("foobar")); - assertFalse(bloom.add("rumplestiltskin")); - assertFalse(bloom.add("buckaroobanzai")); - assertFalse(bloom.add("scheherazade")); + int addFalsePositives = checkAdds(bloom,addCount); + checkDistribution(bloom); + // this is a *very* rough and *very* lenient upper bound for adds <= targetSize + long maxTolerableDuringAdds = addCount / (1<"+maxTolerableDuringAdds+") during adds", + addFalsePositives<10); + + if(containsCount==0) { + return; + } + int containsFalsePositives = checkContains(bloom,containsCount); + // expect at least 0 if bloom wasn't saturated in add phase + // if was saturated, expect at least 1/4th of the theoretical 1-in-every-(2<"+maxTolerableDuringContains+") during contains", + containsFalsePositives<=maxTolerableDuringContains); // no more than double expected 1-in-4mil + assertTrue( + "missing false positives ("+containsFalsePositives+"<"+minTolerableDuringContains+") during contains", + containsFalsePositives>=minTolerableDuringContains); // should be at least a couple + } + + /** + * Test very-large (almost 800MB, spanning more than Integer.MAX_VALUE bit + * indexes) bloom at saturation for expected behavior and level of + * false-positives. + * + * Renamed to non-'test' name so not automatically run, because can + * take 15+ minutes to complete. + */ + public void testOversized() { + trialWithParameters(200000000,22,200000000,32000000); } - @Override - protected void tearDown() throws Exception { - super.tearDown(); - bloom = null; + /** + * Test large (495MB), default-sized bloom at saturation for + * expected behavior and level of false-positives. + * + * Renamed to non-'test' name so not automatically run, because can + * take 15+ minutes to complete. + */ + public void testDefaultFull() { + trialWithParameters(125000000,22,125000000,34000000); + } + + public void testDefaultAbbreviated() { + trialWithParameters(125000000,22,17000000,0); + } + + public void testSmall() { + trialWithParameters(10000000, 20, 10000000, 10000000); + } + + /** + * Check that the given filter behaves properly as a large number of + * constructed unique strings are added: responding positively to + * contains, and negatively to redundant adds. Assuming that the filter + * was empty before it was called, any add()s that report the string was + * already present are false-positives; report the total of same so the + * caller can evaluate if that level was suspiciously out of the expected + * error rate. + * + * @param bloom BloomFilter to check + * @param count int number of unique strings to check + * @return + */ + protected int checkAdds(BloomFilter bloom, long count) { + int falsePositives = 0; + for(int i = 0; i < count; i++) { + String str = "add"+Integer.toString(i); + if(!bloom.add(str)) { + falsePositives++; + } + assertTrue(bloom.contains(str)); + assertFalse(str+" not present on re-add",bloom.add(str)); + } + return falsePositives; + } + + /** + * Check if the given filter contains any of the given constructed + * strings. Since the previously-added strings (of checkAdds) were + * different from these, *any* positive contains results are + * false-positives. Return the total count so that the calling method + * can determine if the false-positive rate is outside the expected + * range. + * + * @param bloom BloomFilter to check + * @param count int number of unique strings to check + * @return + */ + protected int checkContains(BloomFilter bloom, long count) { + int falsePositives = 0; + for(int i = 0; i < count; i++) { + String str = "contains"+Integer.toString(i); + if(bloom.contains(str)) { + falsePositives++; + } + } + return falsePositives; + } + + /** + * Check that the given bloom filter, assumed to have already had a + * significant number of items added, has bits set in the lower and upper + * 10% of its bit field. + * + * (This would have caught previous int/long bugs in the filter hashing + * or conversion of bit indexes into array indexes and bit masks.) + * + * @param bloom BloomFilter to check + */ + public void checkDistribution(BloomFilter bloom) { + long bitLength = bloom.getSizeBytes() * 8L; + for(long i = 0; i=0; i--) { + // verify that first set bit is in first 20% of bitfield + if(bloom.getBit(i)) { + assertTrue("set bits not as expected in late positions",(i/(double)bitLength)>0.1d); + break; + } + } } } diff --git a/engine/src/main/java/org/archive/crawler/util/BloomUriUniqFilter.java b/engine/src/main/java/org/archive/crawler/util/BloomUriUniqFilter.java index 2b15f5d2..98120a6e 100644 --- a/engine/src/main/java/org/archive/crawler/util/BloomUriUniqFilter.java +++ b/engine/src/main/java/org/archive/crawler/util/BloomUriUniqFilter.java @@ -47,25 +47,8 @@ import org.springframework.beans.factory.InitializingBean; * through 125 million unique inserts, which creates a filter structure * about 495MB in size. * - * You may use the following system properties to tune the size and - * false-positive rate of the bloom filter structure used by this class: - * - * org.archive.crawler.util.BloomUriUniqFilter.expected-size (default 125000000) - * org.archive.crawler.util.BloomUriUniqFilter.hash-count (default 22) - * - * The resulting filter will take up approximately... - * - * 1.44 * expected-size * hash-count / 8 - * - * ...bytes. - * - * The BloomFilter64bit implementation class supports filters up to - * 16GiB in size. - * - * (If you only need a filter up to 512MiB in size, the - * BloomFilter32bitSplit *might* offer better performance, on 32bit - * JVMs or with respect to heap-handling of giant arrays. The only - * current way to swap in this class is by editing the source.) + * You may swap in an differently-configured BloomFilter class to alter + * these tradeoffs. * * @author gojomo * @version $Date$, $Revision$ @@ -78,28 +61,13 @@ implements Serializable, InitializingBean { Logger.getLogger(BloomUriUniqFilter.class.getName()); BloomFilter bloom; // package access for testing convenience - - // these defaults create a bloom filter that is - // 1.44*125mil*22/8 ~= 495MB in size, and at full - // capacity will give a false contained indication - // 1/(2^22) ~= 1 in every 4 million probes - protected int expectedInserts= 125000000; // default 125 million; - public int getExpectedInserts() { - return expectedInserts; + public BloomFilter getBloomFilter() { + return bloom; } - public void setExpectedInserts(int expectedInserts) { - this.expectedInserts = expectedInserts; + public void setBloomFilter(BloomFilter filter) { + bloom = filter; } - protected int hashCount = 22; // 1 in 4 million false pos - public int getHashCount() { - return hashCount; - } - public void setHashCount(int hashCount) { - this.hashCount = hashCount; - } - - /** * Default constructor */ @@ -109,14 +77,17 @@ implements Serializable, InitializingBean { /** * Initializer. - * - * @param n the expected number of elements. - * @param d the number of hash functions; if the filter adds not more - * than n elements, false positives will happen with - * probability 2-d. */ public void afterPropertiesSet() { - bloom = new BloomFilter64bit(expectedInserts,hashCount); + if(bloom==null) { + // configure default bloom filter if operator hasn't already + + // these defaults create a bloom filter that is + // 1.44*125mil*22/8 ~= 495MB in size, and at full + // capacity will give a false contained indication + // 1/(2^22) ~= 1 in every 4 million probes + bloom = new BloomFilter64bit(125000000,22); + } } public void forget(String canonical, CrawlURI item) { @@ -128,8 +99,11 @@ implements Serializable, InitializingBean { boolean added = bloom.add(uri); // warn if bloom has reached its expected size (and its false-pos // rate will now exceed the theoretical/designed level) - if( added && (count() == expectedInserts)) { - LOGGER.warning("Bloom has reached expected limit "+expectedInserts); + if( added && (count() == bloom.getExpectedInserts())) { + LOGGER.warning( + "Bloom has reached expected limit "+bloom.getExpectedInserts()+ + "; false-positive rate will now rise above goal of "+ + "1-in-(2^"+bloom.getHashCount()); } return added; } diff --git a/engine/src/test/java/org/archive/crawler/util/BloomUriUniqFilterTest.java b/engine/src/test/java/org/archive/crawler/util/BloomUriUniqFilterTest.java index e00511e4..28390704 100644 --- a/engine/src/test/java/org/archive/crawler/util/BloomUriUniqFilterTest.java +++ b/engine/src/test/java/org/archive/crawler/util/BloomUriUniqFilterTest.java @@ -32,6 +32,7 @@ import org.archive.crawler.datamodel.UriUniqFilter; import org.archive.modules.CrawlURI; import org.archive.net.UURI; import org.archive.net.UURIFactory; +import org.archive.util.BloomFilter64bit; /** @@ -53,8 +54,7 @@ implements UriUniqFilter.CrawlUriReceiver { protected void setUp() throws Exception { super.setUp(); this.filter = new BloomUriUniqFilter(); - this.filter.setExpectedInserts(2000); - this.filter.setHashCount(24); + this.filter.setBloomFilter(new BloomFilter64bit(2000, 24)); this.filter.afterPropertiesSet(); this.filter.setDestination(this); }