KEY CHANGES: - * - *
Instances of this class represent a set of character sequences (with false positives) - * using a Bloom filter. Because of the way Bloom filters work, - * you cannot remove elements. - * - *
Bloom filters have an expected error rate, depending on the number - * of hash functions used, on the filter size and on the number of elements in the filter. This implementation - * uses a variable optimal number of hash functions, depending on the expected - * number of elements. More precisely, a Bloom - * filter for n character sequences with d hash functions will use - * ln 2 dn ≈ 1.44 dn bits; - * false positives will happen with probability 2-d. - * - *
Hash functions are generated at creation time using universal hashing. Each hash function - * uses {@link #NUMBER_OF_WEIGHTS} random integers, which are cyclically multiplied by - * the character codes in a character sequence. The resulting integers are XOR-ed together. - * - *
This class exports access methods that are very similar to those of {@link java.util.Set},
- * but it does not implement that interface, as too many non-optional methods
- * would be unimplementable (e.g., iterators).
- *
- * @author Sebastiano Vigna
- */
-public class BloomFilter32bit implements Serializable, BloomFilter {
-
- private static final long serialVersionUID = -1567837798979475689L;
-
- /** The number of weights used to create hash functions. */
- final public static int NUMBER_OF_WEIGHTS = 2083; // CHANGED FROM 16
- /** The number of bits in this filter. */
- final public long m;
- /** The number of hash functions used by this filter. */
- final public int d;
- /** The underlying bit vectorS. */
- final private int[] bits;
- /** The random integers used to generate the hash functions. */
- final private int[][] weight;
-
- /** The number of elements currently in the filter. It may be
- * smaller than the actual number of additions of distinct character
- * sequences because of false positives.
- */
- private int size;
-
- /** The natural logarithm of 2, used in the computation of the number of bits. */
- private final static double NATURAL_LOG_OF_2 = Math.log( 2 );
-
- private final static boolean DEBUG = false;
-
- /** Creates a new Bloom filter with given number of hash functions and expected number of elements.
- *
- * @param n the expected number of elements.
- * @param d the number of hash functions; if the filter add not more than n elements,
- * false positives will happen with probability 2-d.
- */
- public BloomFilter32bit( final int n, final int d ) {
- this.d = d;
- int len =
- (int)Math.ceil( ( (long)n * (long)d / NATURAL_LOG_OF_2 ) / 32 );
- this.m = len*32L;
- if ( m >= 1L<<32 ) {
- throw new IllegalArgumentException( "This filter would require " + m + " bits" );
- }
- bits = new int[ len ];
-
- if ( DEBUG ) System.err.println( "Number of bits: " + m );
-
- // seeded for reproduceable behavior in repeated runs; BUT:
- // SecureRandom's default implementation (as of 1.5)
- // seems to mix in its own seeding.
- final SecureRandom random = new SecureRandom(new byte[] {19,96});
- weight = new int[ d ][];
- for( int i = 0; i < d; i++ ) {
- weight[ i ] = new int[ NUMBER_OF_WEIGHTS ];
- for( int j = 0; j < NUMBER_OF_WEIGHTS; j++ )
- weight[ i ][ j ] = random.nextInt();
- }
- }
-
- /** The number of character sequences in the filter.
- *
- * @return the number of character sequences in the filter (but see {@link #contains(CharSequence)}).
- */
-
- public int size() {
- return size;
- }
-
- /** Hashes the given sequence with the given hash function.
- *
- * @param s a character sequence.
- * @param l the length of s.
- * @param k a hash function index (smaller than {@link #d}).
- * @return the position in the filter corresponding to s for the hash function k.
- */
- private long hash( final CharSequence s, final int l, final int k ) {
- final int[] w = weight[ k ];
- int h = 0, i = l;
- while( i-- != 0 ) h ^= s.charAt( i ) * w[ i % NUMBER_OF_WEIGHTS ];
- return ((long)h-Integer.MIN_VALUE) % m;
- }
-
- /** Checks whether the given character sequence is in this filter.
- *
- *
Note that this method may return true on a character sequence that is has - * not been added to the filter. This will happen with probability 2-d, - * where d is the number of hash functions specified at creation time, if - * the number of the elements in the filter is less than n, the number - * of expected elements specified at creation time. - * - * @param s a character sequence. - * @return true if the sequence is in the filter (or if a sequence with the - * same hash sequence is in the filter). - */ - - public boolean contains( final CharSequence s ) { - int i = d, l = s.length(); - while( i-- != 0 ) if ( ! getBit( hash( s, l, i ) ) ) return false; - return true; - } - - /** Adds a character sequence to the filter. - * - * @param s a character sequence. - * @return true if the character sequence was not in the filter (but see {@link #contains(CharSequence)}). - */ - - public boolean add( final CharSequence s ) { - boolean result = false; - int i = d, l = s.length(); - long h; - while( i-- != 0 ) { - h = hash( s, l, i ); - if ( ! getBit( h ) ) result = true; - setBit( h ); - } - if ( result ) size++; - return result; - } - - protected final static long ADDRESS_BITS_PER_UNIT = 5; // 32=2^5 - protected final static long BIT_INDEX_MASK = 31; // = BITS_PER_UNIT - 1; - - /** - * Returns from the local bitvector the value of the bit with - * the specified index. The value is true if the bit - * with the index bitIndex is currently set; otherwise, - * returns false. - * - * (adapted from cern.colt.bitvector.QuickBitVector) - * - * @param bitIndex the bit index. - * @return the value of the bit with the specified index. - */ - protected boolean getBit(long bitIndex) { - return ((bits[(int)(bitIndex >> ADDRESS_BITS_PER_UNIT)] & (1 << (bitIndex & BIT_INDEX_MASK))) != 0); - } - - /** - * Changes the bit with index bitIndex in local bitvector. - * - * (adapted from cern.colt.bitvector.QuickBitVector) - * - * @param bitIndex the index of the bit to be set. - */ - protected void setBit(long bitIndex) { - bits[(int)(bitIndex >> ADDRESS_BITS_PER_UNIT)] |= 1 << (bitIndex & BIT_INDEX_MASK); - } - - /* (non-Javadoc) - * @see org.archive.util.BloomFilter#getSizeBytes() - */ - public long getSizeBytes() { - return bits.length*4; - } -} diff --git a/commons/src/main/java/org/archive/util/BloomFilter32bitSplit.java b/commons/src/main/java/org/archive/util/BloomFilter32bitSplit.java deleted file mode 100644 index fd71c884..00000000 --- a/commons/src/main/java/org/archive/util/BloomFilter32bitSplit.java +++ /dev/null @@ -1,251 +0,0 @@ -/* BloomFilter32bit -* -* $Id$ -* -* Created on Jun 21, 2005 -* -* Copyright (C) 2005 Internet Archive; a slight adaptation of -* LGPL work (C) Sebastiano Vigna -* -* This file is part of the Heritrix web crawler (crawler.archive.org). -* -* Heritrix is free software; you can redistribute it and/or modify -* it under the terms of the GNU Lesser Public License as published by -* the Free Software Foundation; either version 2.1 of the License, or -* any later version. -* -* Heritrix is distributed in the hope that it will be useful, -* but WITHOUT ANY WARRANTY; without even the implied warranty of -* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -* GNU Lesser Public License for more details. -* -* You should have received a copy of the GNU Lesser Public License -* along with Heritrix; if not, write to the Free Software -* Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA -*/ - -package org.archive.util; - -import java.io.Serializable; -import java.security.SecureRandom; - -/** A Bloom filter. - * - * SLIGHTLY ADAPTED VERSION OF MG4J it.unimi.dsi.mg4j.util.BloomFilter - * - *
KEY CHANGES: - * - *
Instances of this class represent a set of character sequences (with false positives) - * using a Bloom filter. Because of the way Bloom filters work, - * you cannot remove elements. - * - *
Bloom filters have an expected error rate, depending on the number - * of hash functions used, on the filter size and on the number of elements in the filter. This implementation - * uses a variable optimal number of hash functions, depending on the expected - * number of elements. More precisely, a Bloom - * filter for n character sequences with d hash functions will use - * ln 2 dn ≈ 1.44 dn bits; - * false positives will happen with probability 2-d. - * - *
Hash functions are generated at creation time using universal hashing. Each hash function - * uses {@link #NUMBER_OF_WEIGHTS} random integers, which are cyclically multiplied by - * the character codes in a character sequence. The resulting integers are XOR-ed together. - * - *
This class exports access methods that are very similar to those of {@link java.util.Set},
- * but it does not implement that interface, as too many non-optional methods
- * would be unimplementable (e.g., iterators).
- *
- * @author Sebastiano Vigna
- */
-public class BloomFilter32bitSplit implements Serializable, BloomFilter {
-
- private static final long serialVersionUID = -164106965277863971L;
-
- /** The number of weights used to create hash functions. */
- final public static int NUMBER_OF_WEIGHTS = 2083; // CHANGED FROM 16
- /** The number of bits in this filter. */
- final public long m;
- /** The number of hash functions used by this filter. */
- final public int d;
- /** The underlying bit vectorS. */
-// final private int[] bits;
- final private int[][] bits;
- /** The random integers used to generate the hash functions. */
- final private int[][] weight;
-
- /** The number of elements currently in the filter. It may be
- * smaller than the actual number of additions of distinct character
- * sequences because of false positives.
- */
- private int size;
-
- /** The natural logarithm of 2, used in the computation of the number of bits. */
- private final static double NATURAL_LOG_OF_2 = Math.log( 2 );
-
- /** number of ints in 1MB. */
- private final static int ONE_MB_INTS = 1 << 18; //
-
- private final static boolean DEBUG = false;
-
- /** Creates a new Bloom filter with given number of hash functions and expected number of elements.
- *
- * @param n the expected number of elements.
- * @param d the number of hash functions; if the filter add not more than n elements,
- * false positives will happen with probability 2-d.
- */
- public BloomFilter32bitSplit( final int n, final int d ) {
- this.d = d;
- int len =
- (int)Math.ceil( ( (long)n * (long)d / NATURAL_LOG_OF_2 ) / 32 );
- // round up to ensure divisible into 1MiB chunks
- len = ((len / ONE_MB_INTS)+1)*ONE_MB_INTS;
- this.m = len*32L;
- if ( m >= 1L<<54 ) {
- throw new IllegalArgumentException( "This filter would require " + m + " bits" );
- }
-// bits = new int[ len ];
- bits = new int[ len/ONE_MB_INTS ][ONE_MB_INTS];
-
- if ( DEBUG ) System.err.println( "Number of bits: " + m );
-
- // seeded for reproduceable behavior in repeated runs; BUT:
- // SecureRandom's default implementation (as of 1.5)
- // seems to mix in its own seeding.
- final SecureRandom random = new SecureRandom(new byte[] {19,96});
- weight = new int[ d ][];
- for( int i = 0; i < d; i++ ) {
- weight[ i ] = new int[ NUMBER_OF_WEIGHTS ];
- for( int j = 0; j < NUMBER_OF_WEIGHTS; j++ )
- weight[ i ][ j ] = random.nextInt();
- }
- }
-
- /** The number of character sequences in the filter.
- *
- * @return the number of character sequences in the filter (but see {@link #contains(CharSequence)}).
- */
-
- public int size() {
- return size;
- }
-
- /** Hashes the given sequence with the given hash function.
- *
- * @param s a character sequence.
- * @param l the length of s.
- * @param k a hash function index (smaller than {@link #d}).
- * @return the position in the filter corresponding to s for the hash function k.
- */
- private long hash( final CharSequence s, final int l, final int k ) {
- final int[] w = weight[ k ];
- int h = 0, i = l;
- while( i-- != 0 ) h ^= s.charAt( i ) * w[ i % NUMBER_OF_WEIGHTS ];
- return ((long)h-Integer.MIN_VALUE) % m;
- }
-
- /** Checks whether the given character sequence is in this filter.
- *
- *
Note that this method may return true on a character sequence that is has - * not been added to the filter. This will happen with probability 2-d, - * where d is the number of hash functions specified at creation time, if - * the number of the elements in the filter is less than n, the number - * of expected elements specified at creation time. - * - * @param s a character sequence. - * @return true if the sequence is in the filter (or if a sequence with the - * same hash sequence is in the filter). - */ - - public boolean contains( final CharSequence s ) { - int i = d, l = s.length(); - while( i-- != 0 ) if ( ! getBit( hash( s, l, i ) ) ) return false; - return true; - } - - /** Adds a character sequence to the filter. - * - * @param s a character sequence. - * @return true if the character sequence was not in the filter (but see {@link #contains(CharSequence)}). - */ - - public boolean add( final CharSequence s ) { - boolean result = false; - int i = d, l = s.length(); - long h; - while( i-- != 0 ) { - h = hash( s, l, i ); - if ( ! setGetBit( h ) ) result = true; - } - if ( result ) size++; - return result; - } - - protected final static long ADDRESS_BITS_PER_UNIT = 5; // 32=2^5 - protected final static long BIT_INDEX_MASK = 31; // = BITS_PER_UNIT - 1; - - /** - * Returns from the local bitvector the value of the bit with - * the specified index. The value is true if the bit - * with the index bitIndex is currently set; otherwise, - * returns false. - * - * (adapted from cern.colt.bitvector.QuickBitVector) - * - * @param bitIndex the bit index. - * @return the value of the bit with the specified index. - */ - protected boolean getBit(long bitIndex) { - long intIndex = (bitIndex >>> ADDRESS_BITS_PER_UNIT); - return ((bits[(int)(intIndex / ONE_MB_INTS)][(int)(intIndex % ONE_MB_INTS)] - & (1 << (bitIndex & BIT_INDEX_MASK))) != 0); - } - - /** - * Changes the bit with index bitIndex in local bitvector. - * - * (adapted from cern.colt.bitvector.QuickBitVector) - * - * @param bitIndex the index of the bit to be set. - */ - protected void setBit(long bitIndex) { - long intIndex = (bitIndex >>> ADDRESS_BITS_PER_UNIT); - bits[(int)(intIndex / ONE_MB_INTS)][(int)(intIndex % ONE_MB_INTS)] - |= 1 << (bitIndex & BIT_INDEX_MASK); - } - - /** - * Sets the bit with index bitIndex in local bitvector -- - * returning the old value. - * - * (adapted from cern.colt.bitvector.QuickBitVector) - * - * @param bitIndex the index of the bit to be set. - */ - protected boolean setGetBit(long bitIndex) { - long intIndex = (int) (bitIndex >>> ADDRESS_BITS_PER_UNIT); - int a = (int)(intIndex / ONE_MB_INTS); - int b = (int)(intIndex % ONE_MB_INTS); - int mask = 1 << (bitIndex & BIT_INDEX_MASK); - boolean ret = ((bits[a][b] & (mask)) != 0); - bits[a][b] |= mask; - return ret; - } - - /* (non-Javadoc) - * @see org.archive.util.BloomFilter#getSizeBytes() - */ - public long getSizeBytes() { - return bits.length*bits[0].length*4; - } -} diff --git a/commons/src/main/java/org/archive/util/BloomFilter32bp2.java b/commons/src/main/java/org/archive/util/BloomFilter32bp2.java deleted file mode 100644 index ffa64d66..00000000 --- a/commons/src/main/java/org/archive/util/BloomFilter32bp2.java +++ /dev/null @@ -1,235 +0,0 @@ -/* BloomFilter -* -* $Id$ -* -* Created on Jun 21, 2005 -* -* Copyright (C) 2005 Internet Archive; a slight adaptation of -* LGPL work (C) Sebastiano Vigna -* -* This file is part of the Heritrix web crawler (crawler.archive.org). -* -* Heritrix is free software; you can redistribute it and/or modify -* it under the terms of the GNU Lesser Public License as published by -* the Free Software Foundation; either version 2.1 of the License, or -* any later version. -* -* Heritrix is distributed in the hope that it will be useful, -* but WITHOUT ANY WARRANTY; without even the implied warranty of -* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -* GNU Lesser Public License for more details. -* -* You should have received a copy of the GNU Lesser Public License -* along with Heritrix; if not, write to the Free Software -* Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA -*/ - -package org.archive.util; - -import java.io.Serializable; -import java.security.SecureRandom; - -/** A Bloom filter. - * - * SLIGHTLY ADAPTED VERSION OF MG4J it.unimi.dsi.mg4j.util.BloomFilter - * - *
KEY CHANGES: - * - *
Instances of this class represent a set of character sequences (with false positives) - * using a Bloom filter. Because of the way Bloom filters work, - * you cannot remove elements. - * - *
Bloom filters have an expected error rate, depending on the number - * of hash functions used, on the filter size and on the number of elements in the filter. This implementation - * uses a variable optimal number of hash functions, depending on the expected - * number of elements. More precisely, a Bloom - * filter for n character sequences with d hash functions will use - * ln 2 dn ≈ 1.44 dn bits; - * false positives will happen with probability 2-d. - * - *
Hash functions are generated at creation time using universal hashing. Each hash function - * uses {@link #NUMBER_OF_WEIGHTS} random integers, which are cyclically multiplied by - * the character codes in a character sequence. The resulting integers are XOR-ed together. - * - *
This class exports access methods that are very similar to those of {@link java.util.Set},
- * but it does not implement that interface, as too many non-optional methods
- * would be unimplementable (e.g., iterators).
- *
- * @author Sebastiano Vigna
- */
-public class BloomFilter32bp2 implements Serializable, BloomFilter {
-
- private static final long serialVersionUID = -2292902803681146635L;
-
- /** The number of weights used to create hash functions. */
- final public static int NUMBER_OF_WEIGHTS = 2083; // CHANGED FROM 16
- /** The number of bits in this filter. */
- final public long m;
- /** the power-of-two that m is */
- final public long power; // 1< Note that this method may return true on a character sequence that is has
- * not been added to the filter. This will happen with probability 2-d,
- * where d is the number of hash functions specified at creation time, if
- * the number of the elements in the filter is less than n, the number
- * of expected elements specified at creation time.
- *
- * @param s a character sequence.
- * @return true if the sequence is in the filter (or if a sequence with the
- * same hash sequence is in the filter).
- */
-
- public boolean contains( final CharSequence s ) {
- int i = d, l = s.length();
- while( i-- != 0 ) if ( ! getBit( hash( s, l, i ) ) ) return false;
- return true;
- }
-
- /** Adds a character sequence to the filter.
- *
- * @param s a character sequence.
- * @return true if the character sequence was not in the filter (but see {@link #contains(CharSequence)}).
- */
-
- public boolean add( final CharSequence s ) {
- boolean result = false;
- int i = d, l = s.length();
- int h;
- while( i-- != 0 ) {
- h = hash( s, l, i );
- if ( ! getBit( h ) ) result = true;
- setBit( h );
- }
- if ( result ) size++;
- return result;
- }
-
- protected final static int ADDRESS_BITS_PER_UNIT = 5; // 32=2^5
- protected final static int BIT_INDEX_MASK = 31; // = BITS_PER_UNIT - 1;
-
- /**
- * Returns from the local bitvector the value of the bit with
- * the specified index. The value is true if the bit
- * with the index bitIndex is currently set; otherwise,
- * returns false.
- *
- * (adapted from cern.colt.bitvector.QuickBitVector)
- *
- * @param bitIndex the bit index.
- * @return the value of the bit with the specified index.
- */
- protected boolean getBit(int bitIndex) {
- return ((bits[(int)(bitIndex >>> ADDRESS_BITS_PER_UNIT)] & (1 << (bitIndex & BIT_INDEX_MASK))) != 0);
- }
-
- /**
- * Changes the bit with index bitIndex in local bitvector.
- *
- * (adapted from cern.colt.bitvector.QuickBitVector)
- *
- * @param bitIndex the index of the bit to be set.
- */
- protected void setBit(int bitIndex) {
- bits[(int)(bitIndex >>> ADDRESS_BITS_PER_UNIT)] |= 1 << (bitIndex & BIT_INDEX_MASK);
- }
-
- /* (non-Javadoc)
- * @see org.archive.util.BloomFilter#getSizeBytes()
- */
- public long getSizeBytes() {
- return bits.length*4;
- }
-}
diff --git a/commons/src/main/java/org/archive/util/BloomFilter32bp2Split.java b/commons/src/main/java/org/archive/util/BloomFilter32bp2Split.java
deleted file mode 100644
index aba45f75..00000000
--- a/commons/src/main/java/org/archive/util/BloomFilter32bp2Split.java
+++ /dev/null
@@ -1,262 +0,0 @@
-/* BloomFilter
-*
-* $Id$
-*
-* Created on Jun 21, 2005
-*
-* Copyright (C) 2005 Internet Archive; a slight adaptation of
-* LGPL work (C) Sebastiano Vigna
-*
-* This file is part of the Heritrix web crawler (crawler.archive.org).
-*
-* Heritrix is free software; you can redistribute it and/or modify
-* it under the terms of the GNU Lesser Public License as published by
-* the Free Software Foundation; either version 2.1 of the License, or
-* any later version.
-*
-* Heritrix is distributed in the hope that it will be useful,
-* but WITHOUT ANY WARRANTY; without even the implied warranty of
-* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-* GNU Lesser Public License for more details.
-*
-* You should have received a copy of the GNU Lesser Public License
-* along with Heritrix; if not, write to the Free Software
-* Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
-*/
-
-package org.archive.util;
-
-import java.io.Serializable;
-import java.security.SecureRandom;
-
-/** A Bloom filter.
- *
- * SLIGHTLY ADAPTED VERSION OF MG4J it.unimi.dsi.mg4j.util.BloomFilter
- *
- * KEY CHANGES:
- *
- * Instances of this class represent a set of character sequences (with false positives)
- * using a Bloom filter. Because of the way Bloom filters work,
- * you cannot remove elements.
- *
- * Bloom filters have an expected error rate, depending on the number
- * of hash functions used, on the filter size and on the number of elements in the filter. This implementation
- * uses a variable optimal number of hash functions, depending on the expected
- * number of elements. More precisely, a Bloom
- * filter for n character sequences with d hash functions will use
- * ln 2 dn ≈ 1.44 dn bits;
- * false positives will happen with probability 2-d.
- *
- * Hash functions are generated at creation time using universal hashing. Each hash function
- * uses {@link #NUMBER_OF_WEIGHTS} random integers, which are cyclically multiplied by
- * the character codes in a character sequence. The resulting integers are XOR-ed together.
- *
- * This class exports access methods that are very similar to those of {@link java.util.Set},
- * but it does not implement that interface, as too many non-optional methods
- * would be unimplementable (e.g., iterators).
- *
- * @author Sebastiano Vigna
- */
-public class BloomFilter32bp2Split implements Serializable, BloomFilter {
-
- private static final long serialVersionUID = -1504889954381695129L;
-
- /** The number of weights used to create hash functions. */
- final public static int NUMBER_OF_WEIGHTS = 2083; // CHANGED FROM 16
- /** The number of bits in this filter. */
- final public long m;
- /** the power-of-two that m is */
- final public long power; // 1< Note that this method may return true on a character sequence that is has
- * not been added to the filter. This will happen with probability 2-d,
- * where d is the number of hash functions specified at creation time, if
- * the number of the elements in the filter is less than n, the number
- * of expected elements specified at creation time.
- *
- * @param s a character sequence.
- * @return true if the sequence is in the filter (or if a sequence with the
- * same hash sequence is in the filter).
- */
-
- public boolean contains( final CharSequence s ) {
- int i = d, l = s.length();
- while( i-- != 0 ) if ( ! getBit( hash( s, l, i ) ) ) return false;
- return true;
- }
-
- /** Adds a character sequence to the filter.
- *
- * @param s a character sequence.
- * @return true if the character sequence was not in the filter (but see {@link #contains(CharSequence)}).
- */
-
- public boolean add( final CharSequence s ) {
- boolean result = false;
- int i = d, l = s.length();
- int h;
- while( i-- != 0 ) {
- h = hash( s, l, i );
- if ( ! setGetBit( h ) ) result = true;
- }
- if ( result ) size++;
- return result;
- }
-
- protected final static int ADDRESS_BITS_PER_UNIT = 5; // 32=2^5
- protected final static int BIT_INDEX_MASK = 31; // = BITS_PER_UNIT - 1;
-
- /**
- * Returns from the local bitvector the value of the bit with
- * the specified index. The value is true if the bit
- * with the index bitIndex is currently set; otherwise,
- * returns false.
- *
- * (adapted from cern.colt.bitvector.QuickBitVector)
- *
- * @param bitIndex the bit index.
- * @return the value of the bit with the specified index.
- */
- protected boolean getBit(int bitIndex) {
- int intIndex = (int)(bitIndex >>> ADDRESS_BITS_PER_UNIT);
- return ((bits[intIndex>>>aShift][intIndex&bMask] & (1 << (bitIndex & BIT_INDEX_MASK))) != 0);
- }
-
- /**
- * Changes the bit with index bitIndex in local bitvector.
- *
- * (adapted from cern.colt.bitvector.QuickBitVector)
- *
- * @param bitIndex the index of the bit to be set.
- */
- protected void setBit(int bitIndex) {
- int intIndex = (int)(bitIndex >>> ADDRESS_BITS_PER_UNIT);
- bits[intIndex>>>aShift][intIndex&bMask] |= 1 << (bitIndex & BIT_INDEX_MASK);
- }
-
- /**
- * Sets the bit with index bitIndex in local bitvector --
- * returning the old value.
- *
- * (adapted from cern.colt.bitvector.QuickBitVector)
- *
- * @param bitIndex the index of the bit to be set.
- */
- protected boolean setGetBit(int bitIndex) {
- int intIndex = (int)(bitIndex >>> ADDRESS_BITS_PER_UNIT);
- int a = intIndex>>>aShift;
- int b = intIndex&bMask;
- int mask = 1 << (bitIndex & BIT_INDEX_MASK);
- boolean ret = ((bits[a][b] & (mask)) != 0);
- bits[a][b] |= mask;
- return ret;
- }
-
- /* (non-Javadoc)
- * @see org.archive.util.BloomFilter#getSizeBytes()
- */
- public long getSizeBytes() {
- return bits.length*bits[0].length*4;
- }
-}
diff --git a/commons/src/main/java/org/archive/util/BloomFilter64bit.java b/commons/src/main/java/org/archive/util/BloomFilter64bit.java
index 97b0d06e..aeca26ed 100644
--- a/commons/src/main/java/org/archive/util/BloomFilter64bit.java
+++ b/commons/src/main/java/org/archive/util/BloomFilter64bit.java
@@ -28,104 +28,165 @@ package org.archive.util;
import java.io.Serializable;
import java.security.SecureRandom;
+import java.util.Random;
/** A Bloom filter.
*
- * SLIGHTLY ADAPTED VERSION OF MG4J it.unimi.dsi.mg4j.util.BloomFilter
+ * ADAPTED/IMPROVED VERSION OF MG4J it.unimi.dsi.mg4j.util.BloomFilter
*
* KEY CHANGES:
*
* Instances of this class represent a set of character sequences (with false positives)
- * using a Bloom filter. Because of the way Bloom filters work,
+ * Instances of this class represent a set of character sequences (with
+ * false positives) using a Bloom filter. Because of the way Bloom filters work,
* you cannot remove elements.
*
* Bloom filters have an expected error rate, depending on the number
- * of hash functions used, on the filter size and on the number of elements in the filter. This implementation
- * uses a variable optimal number of hash functions, depending on the expected
- * number of elements. More precisely, a Bloom
- * filter for n character sequences with d hash functions will use
- * ln 2 dn ≈ 1.44 dn bits;
- * false positives will happen with probability 2-d.
+ * of hash functions used, on the filter size and on the number of elements in
+ * the filter. This implementation uses a variable optimal number of hash
+ * functions, depending on the expected number of elements. More precisely, a
+ * Bloom filter for n character sequences with d hash
+ * functions will use ln 2 dn ≈
+ * 1.44 dn bits; false positives will happen with
+ * probability 2-d.
*
- * Hash functions are generated at creation time using universal hashing. Each hash function
- * uses {@link #NUMBER_OF_WEIGHTS} random integers, which are cyclically multiplied by
- * the character codes in a character sequence. The resulting integers are XOR-ed together.
+ * Hash functions are generated at creation time using universal hashing.
+ * Each hash function uses {@link #NUMBER_OF_WEIGHTS} random integers, which
+ * are cyclically multiplied by the character codes in a character sequence.
+ * The resulting integers are XOR-ed together.
*
- * This class exports access methods that are very similar to those of {@link java.util.Set},
- * but it does not implement that interface, as too many non-optional methods
- * would be unimplementable (e.g., iterators).
+ * This class exports access methods that are very similar to those of
+ * {@link java.util.Set}, but it does not implement that interface, as too
+ * many non-optional methods would be unimplementable (e.g., iterators).
*
* @author Sebastiano Vigna
+ * @contributor Gordon Mohr
*/
public class BloomFilter64bit implements Serializable, BloomFilter {
-
- private static final long serialVersionUID = 2317000663009608403L;
+ private static final long serialVersionUID = 2L;
/** The number of weights used to create hash functions. */
- final public static int NUMBER_OF_WEIGHTS = 2083; // CHANGED FROM 16
+ final static int NUMBER_OF_WEIGHTS = 2083; // CHANGED FROM 16
/** The number of bits in this filter. */
- final public long m;
+ final protected long m;
+ /** if bitfield is an exact power of 2 in length, it is this power */
+ protected int power = -1;
+ /** The expected number of inserts; determines calculated size */
+ final protected long expectedInserts;
/** The number of hash functions used by this filter. */
- final public int d;
- /** The underlying bit vector. package access for testing */
- final long[] bits;
+ final protected int d;
+ /** The underlying bit vector */
+ final protected long[][] bits;
/** The random integers used to generate the hash functions. */
- final long[][] weight;
+ final protected long[][] weight;
/** The number of elements currently in the filter. It may be
* smaller than the actual number of additions of distinct character
* sequences because of false positives.
*/
- private int size;
+ int size;
/** The natural logarithm of 2, used in the computation of the number of bits. */
- private final static double NATURAL_LOG_OF_2 = Math.log( 2 );
+ final static double NATURAL_LOG_OF_2 = Math.log( 2 );
- private final static boolean DEBUG = false;
+ /** power-of-two to use as maximum size of bitfield subarrays */
+ protected final static int SUBARRAY_POWER_OF_TWO = 26; // 512MiB of longs
+ /** number of longs in one subarray */
+ protected final static int SUBARRAY_LENGTH_IN_LONGS = 1 << SUBARRAY_POWER_OF_TWO;
+ /** mask for lowest SUBARRAY_POWER_OF_TWO bits */
+ protected final static int SUBARRAY_MASK = SUBARRAY_LENGTH_IN_LONGS - 1; //0x0FFFFFFF
- /** Creates a new Bloom filter with given number of hash functions and expected number of elements.
+ final static boolean DEBUG = false;
+
+ /** Creates a new Bloom filter with given number of hash functions and
+ * expected number of elements.
*
* @param n the expected number of elements.
- * @param d the number of hash functions; if the filter add not more than Note that this method may return true on a character sequence that is has
@@ -179,9 +254,8 @@ public class BloomFilter64bit implements Serializable, BloomFilter {
long h;
while( i-- != 0 ) {
h = hash( s, l, i );
- if ( ! getBit( h ) ) {
+ if ( ! setGetBit( h ) ) {
result = true;
- setBit( h );
}
}
if ( result ) size++;
@@ -189,7 +263,7 @@ public class BloomFilter64bit implements Serializable, BloomFilter {
}
protected final static long ADDRESS_BITS_PER_UNIT = 6; // 64=2^6
- protected final static long BIT_INDEX_MASK = 63; // = BITS_PER_UNIT - 1;
+ protected final static long BIT_INDEX_MASK = (1<<6)-1; // = 63 = 2^BITS_PER_UNIT - 1;
/**
* Returns from the local bitvector the value of the bit with
@@ -202,8 +276,11 @@ public class BloomFilter64bit implements Serializable, BloomFilter {
* @param bitIndex the bit index.
* @return the value of the bit with the specified index.
*/
- protected boolean getBit(long bitIndex) {
- return ((bits[(int)(bitIndex >> ADDRESS_BITS_PER_UNIT)] & (1L << (bitIndex & BIT_INDEX_MASK))) != 0);
+ public boolean getBit(long bitIndex) {
+ long longIndex = bitIndex >>> ADDRESS_BITS_PER_UNIT;
+ int arrayIndex = (int) (longIndex >>> SUBARRAY_POWER_OF_TWO);
+ int subarrayIndex = (int) (longIndex & SUBARRAY_MASK);
+ return ((bits[arrayIndex][subarrayIndex] & (1L << (bitIndex & BIT_INDEX_MASK))) != 0);
}
/**
@@ -214,13 +291,45 @@ public class BloomFilter64bit implements Serializable, BloomFilter {
* @param bitIndex the index of the bit to be set.
*/
protected void setBit( long bitIndex) {
- bits[(int)(bitIndex >> ADDRESS_BITS_PER_UNIT)] |= 1L << (bitIndex & BIT_INDEX_MASK);
+ long longIndex = bitIndex >>> ADDRESS_BITS_PER_UNIT;
+ int arrayIndex = (int) (longIndex >>> SUBARRAY_POWER_OF_TWO);
+ int subarrayIndex = (int) (longIndex & SUBARRAY_MASK);
+ bits[arrayIndex][subarrayIndex] |= (1L << (bitIndex & BIT_INDEX_MASK));
+ }
+
+ /**
+ * Sets the bit with index bitIndex in local bitvector --
+ * returning the old value.
+ *
+ * (adapted from cern.colt.bitvector.QuickBitVector)
+ *
+ * @param bitIndex the index of the bit to be set.
+ */
+ protected boolean setGetBit( long bitIndex) {
+ long longIndex = bitIndex >>> ADDRESS_BITS_PER_UNIT;
+ int arrayIndex = (int) (longIndex >>> SUBARRAY_POWER_OF_TWO);
+ int subarrayIndex = (int) (longIndex & SUBARRAY_MASK);
+ long mask = 1L << (bitIndex & BIT_INDEX_MASK);
+ boolean ret = (bits[arrayIndex][subarrayIndex] & mask)!=0;
+ bits[arrayIndex][subarrayIndex] |= mask;
+ return ret;
}
/* (non-Javadoc)
* @see org.archive.util.BloomFilter#getSizeBytes()
*/
public long getSizeBytes() {
- return bits.length*8;
+ // account for ragged-sized last array
+ return 8*(((bits.length-1)*bits[0].length)+bits[bits.length-1].length);
}
+
+ @Override
+ public long getExpectedInserts() {
+ return expectedInserts;
+ }
+
+ @Override
+ public long getHashCount() {
+ return d;
+ }
}
diff --git a/commons/src/main/java/org/archive/util/BenchmarkBlooms.java b/commons/src/test/java/org/archive/util/BenchmarkBlooms.java
similarity index 55%
rename from commons/src/main/java/org/archive/util/BenchmarkBlooms.java
rename to commons/src/test/java/org/archive/util/BenchmarkBlooms.java
index 535564f9..b3f20678 100644
--- a/commons/src/main/java/org/archive/util/BenchmarkBlooms.java
+++ b/commons/src/test/java/org/archive/util/BenchmarkBlooms.java
@@ -49,35 +49,34 @@ public class BenchmarkBlooms {
int d_hashes =
(args.length > 2) ? Integer.parseInt(args[2]) : 22;
int adds =
- (args.length > 3) ? Integer.parseInt(args[3]) : 5000000;
+ (args.length > 3) ? Integer.parseInt(args[3]) : 10000000;
+ int contains =
+ (args.length > 4) ? Integer.parseInt(args[4]) : 8000000;
String prefix =
- (args.length > 4) ? args[4] : "http://www.archive.org/";
+ (args.length > 5) ? args[5] : "http://www.archive.org/";
System.out.println(
"reps="+reps+" n_expected="+n_expected+
- " d_hashes="+d_hashes+" adds="+adds+" prefix="+prefix);
+ " d_hashes="+d_hashes+" adds="+adds+
+ " contains="+contains+" prefix="+prefix);
- BloomFilter bloom64;
- BloomFilter bloom32;
- BloomFilter bloom32split;
- BloomFilter bloom32p2;
- BloomFilter bloom32p2split;
+ BloomFilter64bit bloom64;
+// BloomFilter bloom32;
+// BloomFilter bloom32split;
for (int r=0;rs.
- * @param k a hash function index (smaller than {@link #d}).
- * @return the position in the filter corresponding to s for the hash function k.
- */
- private int hash( final CharSequence s, final int l, final int k ) {
- final int[] w = weight[ k ];
- int h = 0, i = l;
- while( i-- != 0 ) h ^= s.charAt( i ) * w[ i % NUMBER_OF_WEIGHTS ];
- return h >>> (32-power);
- }
-
- /** Checks whether the given character sequence is in this filter.
- *
- *
- *
- *
- *
- *
- * s for the hash function k.
- */
- private int hash( final CharSequence s, final int l, final int k ) {
- final int[] w = weight[ k ];
- int h = 0, i = l;
- while( i-- != 0 ) h ^= s.charAt( i ) * w[ i % NUMBER_OF_WEIGHTS ];
- return h >>> (32-power);
- }
-
- /** Checks whether the given character sequence is in this filter.
- *
- *
*
*
*
*
- * n elements,
- * false positives will happen with probability 2-d.
+ * @param d the number of hash functions; if the filter add not more
+ * than n elements, false positives will happen with
+ * probability 2-d.
*/
- public BloomFilter64bit( final int n, final int d ) {
+ public BloomFilter64bit( final long n, final int d) {
+ this(n,d, new SecureRandom(), false);
+ }
+
+ public BloomFilter64bit( final long n, final int d, boolean roundUp) {
+ this(n,d, new SecureRandom(), roundUp);
+ }
+
+ /** Creates a new Bloom filter with given number of hash functions and
+ * expected number of elements.
+ *
+ * @param n the expected number of elements.
+ * @param d the number of hash functions; if the filter add not more
+ * than n elements, false positives will happen with
+ * probability 2-d.
+ * @param Random weightsGenerator may provide a seeded Random for reproducible
+ * internal universal hash function weighting
+ * @param roundUp if true, round bit size up to next-nearest-power-of-2
+ */
+ public BloomFilter64bit( final long n, final int d, Random weightsGenerator, boolean roundUp ) {
+ this.expectedInserts = n;
this.d = d;
- int len = (int)Math.ceil( ( (long)n * (long)d / NATURAL_LOG_OF_2 ) / 64L );
- if ( len/64 > Integer.MAX_VALUE ) throw new IllegalArgumentException( "This filter would require " + len * 64L + " bits" );
- bits = new long[ len ];
- m = bits.length * 64L;
+ long lenInLongs = (long)Math.ceil( ( (long)n * (long)d / NATURAL_LOG_OF_2 ) / 64L );
+ if ( lenInLongs > (1L<<48) ) {
+ throw new IllegalArgumentException(
+ "This filter would require " + lenInLongs + " longs, " +
+ "greater than this classes maximum of 2^48 longs (2PiB)." );
+ }
+ long lenInBits = lenInLongs * 64L;
+
+ if(roundUp) {
+ int pow = 0;
+ while((1L<k.
*/
-
- private long hash( final CharSequence s, final int l, final int k ) {
+ protected long hash( final CharSequence s, final int l, final int k ) {
final long[] w = weight[ k ];
long h = 0;
int i = l;
while( i-- != 0 ) h ^= s.charAt( i ) * w[ i % NUMBER_OF_WEIGHTS ];
- return ( h & 0x7FFFFFFFFFFFFFFFL ) % m;
+ long retVal;
+ if(power>0) {
+ retVal = h >>> (64-power);
+ } else {
+ // ####----####----
+ retVal = ( h & 0x7FFFFFFFFFFFFFFFL ) % m;
+ }
+ return retVal;
}
-
+
+ public long[] bitIndexesFor(CharSequence s) {
+ long[] ret = new long[d];
+ for(int i = 0; i < d; i++) {
+ ret[i] = hash(s,s.length(),i);
+ }
+ return ret;
+ }
+
/** Checks whether the given character sequence is in this filter.
*
* n elements, false positives will happen with
- * probability 2-d.
*/
public void afterPropertiesSet() {
- bloom = new BloomFilter64bit(expectedInserts,hashCount);
+ if(bloom==null) {
+ // configure default bloom filter if operator hasn't already
+
+ // these defaults create a bloom filter that is
+ // 1.44*125mil*22/8 ~= 495MB in size, and at full
+ // capacity will give a false contained indication
+ // 1/(2^22) ~= 1 in every 4 million probes
+ bloom = new BloomFilter64bit(125000000,22);
+ }
}
public void forget(String canonical, CrawlURI item) {
@@ -128,8 +99,11 @@ implements Serializable, InitializingBean {
boolean added = bloom.add(uri);
// warn if bloom has reached its expected size (and its false-pos
// rate will now exceed the theoretical/designed level)
- if( added && (count() == expectedInserts)) {
- LOGGER.warning("Bloom has reached expected limit "+expectedInserts);
+ if( added && (count() == bloom.getExpectedInserts())) {
+ LOGGER.warning(
+ "Bloom has reached expected limit "+bloom.getExpectedInserts()+
+ "; false-positive rate will now rise above goal of "+
+ "1-in-(2^"+bloom.getHashCount());
}
return added;
}
diff --git a/engine/src/test/java/org/archive/crawler/util/BloomUriUniqFilterTest.java b/engine/src/test/java/org/archive/crawler/util/BloomUriUniqFilterTest.java
index e00511e4..28390704 100644
--- a/engine/src/test/java/org/archive/crawler/util/BloomUriUniqFilterTest.java
+++ b/engine/src/test/java/org/archive/crawler/util/BloomUriUniqFilterTest.java
@@ -32,6 +32,7 @@ import org.archive.crawler.datamodel.UriUniqFilter;
import org.archive.modules.CrawlURI;
import org.archive.net.UURI;
import org.archive.net.UURIFactory;
+import org.archive.util.BloomFilter64bit;
/**
@@ -53,8 +54,7 @@ implements UriUniqFilter.CrawlUriReceiver {
protected void setUp() throws Exception {
super.setUp();
this.filter = new BloomUriUniqFilter();
- this.filter.setExpectedInserts(2000);
- this.filter.setHashCount(24);
+ this.filter.setBloomFilter(new BloomFilter64bit(2000, 24));
this.filter.afterPropertiesSet();
this.filter.setDestination(this);
}