HER-1783 BloomFilter64bit bit-length bug prevents full bitfield from being used; premature saturation

* BloomFilter64bit.java
    fix long arithmetic calculating bit-length m in constructor
    slight optimization of add()
* BloomFilterTest.java
    abstract superclass for bloom impl class tests
* BloomFilter64bitTest.java
    sanity check that set bits, after a handful of adds into default-sized filter, occupy top and bottom 20% of range (which would have caught this bug earlier)

commons/src/main/java/org/archive/util/BloomFilter64bit.java

@@ -43,7 +43,7 @@ import java.security.SecureRandom;
  * <li>Adapted to allow long bit indices so long as the index/64 (used 
  * an array index in bit vector) fits within Integer.MAX_VALUE. (Thus
  * it supports filters up to 64*Integer.MAX_VALUE bits in size, or 
- * 16GiB.</li>
+ * 16GiB.)</li>
  * </ul>
  * 
  * <hr>
@@ -80,10 +80,10 @@ public class BloomFilter64bit implements Serializable, BloomFilter {
     final public long m;
     /** The number of hash functions used by this filter. */
     final public int d;
-    /** The underlying bit vector. */
-    final private long[] bits;
+    /** The underlying bit vector. package access for testing */
+    final long[] bits;
     /** The random integers used to generate the hash functions. */
-    final private long[][] weight;
+    final long[][] weight;
 
     /** The number of elements currently in the filter. It may be
      * smaller than the actual number of additions of distinct character
@@ -107,7 +107,7 @@ public class BloomFilter64bit implements Serializable, BloomFilter {
         int len = (int)Math.ceil( ( (long)n * (long)d / NATURAL_LOG_OF_2 ) / 64L );
         if ( len/64 > Integer.MAX_VALUE ) throw new IllegalArgumentException( "This filter would require " + len * 64L + " bits" );
         bits = new long[ len ];
-        m = bits.length * 64;
+        m = bits.length * 64L;
 
         if ( DEBUG ) System.err.println( "Number of bits: " + m );
 
@@ -179,8 +179,10 @@ public class BloomFilter64bit implements Serializable, BloomFilter {
         long h;
         while( i-- != 0 ) {
             h = hash( s, l, i );
-            if ( ! getBit( h ) ) result = true;
-            setBit( h );
+            if ( ! getBit( h ) ) {
+                result = true;
+                setBit( h );
+            }
         }
         if ( result ) size++;
         return result;

commons/src/test/java/org/archive/util/BloomFilter64bitTest.java

@@ -0,0 +1,59 @@
+/*
+ *  This file is part of the Heritrix web crawler (crawler.archive.org).
+ *
+ *  Licensed to the Internet Archive (IA) by one or more individual 
+ *  contributors. 
+ *
+ *  The IA licenses this file to You under the Apache License, Version 2.0
+ *  (the "License"); you may not use this file except in compliance with
+ *  the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ */
+
+package org.archive.util;
+
+
+
+/**
+ * BloomFilter64 tests
+ * 
+ * @contributor gojomo
+ * @version $Date: 2009-11-19 14:39:53 -0800 (Thu, 19 Nov 2009) $, $Revision: 6674 $
+ */
+public class BloomFilter64bitTest extends BloomFilterTest {
+    
+    protected void setUp() throws Exception {
+        // test at default size of BloomUriUniqFilter -- but don't depend on that 
+        // 'engine'-subproject class for values
+        bloom = new BloomFilter64bit(125000000,22); 
+    }
+    
+    public void testDistributionOfSetBits() {
+        // prelaod
+        testBasics(); 
+        
+        BloomFilter64bit bloom64 = (BloomFilter64bit)bloom; 
+        for(int i = 0; i<bloom64.bits.length; i++) {
+            // verify that first set bit is in first 20% of bitfield
+            if(bloom64.bits[i]>0) {
+                assertTrue("set bits not as expected in early positions",(i/(double)bloom64.bits.length)<0.2d); 
+                break; 
+            }
+        }
+        for(int i = bloom64.bits.length-1; i>=0; i--) {
+            // verify that first set bit is in first 20% of bitfield
+            if(bloom64.bits[i]>0) {
+                assertTrue("set bits not as expected in late positions",(i/(double)bloom64.bits.length)>0.8d); 
+                break; 
+            }
+        }
+
+    }
+}

commons/src/test/java/org/archive/util/BloomFilterTest.java

@@ -0,0 +1,51 @@
+/*
+ *  This file is part of the Heritrix web crawler (crawler.archive.org).
+ *
+ *  Licensed to the Internet Archive (IA) by one or more individual 
+ *  contributors. 
+ *
+ *  The IA licenses this file to You under the Apache License, Version 2.0
+ *  (the "License"); you may not use this file except in compliance with
+ *  the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ */
+
+package org.archive.util;
+
+import junit.framework.TestCase;
+
+
+/**
+ * BloomFilter tests
+ * 
+ * @contributor gojomo
+ * @version $Date: 2009-11-19 14:39:53 -0800 (Thu, 19 Nov 2009) $, $Revision: 6674 $
+ */
+public abstract class BloomFilterTest extends TestCase {
+    protected BloomFilter bloom; 
+    
+    protected abstract void setUp() throws Exception;
+    
+    public void testBasics() {
+        // require initial additions to return 'true' (for 'added')
+        assertTrue(bloom.add("abracadabra"));
+        assertTrue(bloom.add("foobar"));
+        assertTrue(bloom.add("rumplestiltskin"));
+        assertTrue(bloom.add("buckaroobanzai"));
+        assertTrue(bloom.add("scheherazade"));
+        
+        // require readdition to return 'false' (not added because already present)
+        assertFalse(bloom.add("abracadabra"));
+        assertFalse(bloom.add("foobar"));
+        assertFalse(bloom.add("rumplestiltskin"));
+        assertFalse(bloom.add("buckaroobanzai"));
+        assertFalse(bloom.add("scheherazade"));
+    }
+}