mirror of
https://github.com/internetarchive/heritrix3.git
synced 2026-08-19 13:06:39 +00:00
[HER-1783] BloomFilter64bit bit-length bug prevents full bitfield from being used; premature saturation
* BloomFilter64bit.java
include the split-to-subarrays (for larger bitfields) and round-up-to-power-of-2 (for performance) options previously in largely-redundant classes
fit a number of problems with int/long overflow and bitwise ops
add methods for reporting/testing
* BloomFilter.java
add methods for reporting/testing
* BloomFilterTest.java, BloomFilter64bitTest.java
more extensive tests, including two lengthy tests of default/oversized blooms usually disabled by renaming
* BloomFilter32bit.java, BloomFilter32bitSplit.java, BloomFilter32bp2.java, BloomFilter32bp2Split.java
deleted as buggy or redundant
* BenchmarkBlooms.java
move to test source dir
* BloomUriUniqFilter.java
change to accept filter instance (rather than parameters) for added configuration flexibility
fix comments
* BloomUriUniqFilterTest.java
supply filter not paramters
This commit is contained in:
@@ -32,6 +32,7 @@ import org.archive.crawler.datamodel.UriUniqFilter;
|
||||
import org.archive.modules.CrawlURI;
|
||||
import org.archive.net.UURI;
|
||||
import org.archive.net.UURIFactory;
|
||||
import org.archive.util.BloomFilter64bit;
|
||||
|
||||
|
||||
/**
|
||||
@@ -53,8 +54,7 @@ implements UriUniqFilter.CrawlUriReceiver {
|
||||
protected void setUp() throws Exception {
|
||||
super.setUp();
|
||||
this.filter = new BloomUriUniqFilter();
|
||||
this.filter.setExpectedInserts(2000);
|
||||
this.filter.setHashCount(24);
|
||||
this.filter.setBloomFilter(new BloomFilter64bit(2000, 24));
|
||||
this.filter.afterPropertiesSet();
|
||||
this.filter.setDestination(this);
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user