mirror of
https://github.com/internetarchive/heritrix3.git
synced 2026-11-16 18:57:01 +00:00
Work for [HER-1656] add multiple-queues-per-host ('parallelQueues') capabilities to QueueAssignmentPolicies
* LongToIntConsistentHash.java + Test
consistent-hashing utility class
* URIAuthorityBasedQueueAssignmentPolicy.java
shared superclass for the hostname and surtauthority QAPs
setting for 'deferToPrevious' -- avoid changing assignments
setting for 'parallelQueues' -- when > 1, consistent-hash URIs across that many separate queues (with numerical suffix)
* SurtAuthorityQueueAssignmentPolicy.java, HostnameQueueAssignmentPolicy.java
refactor to derive from URIAuthorityBasedQueueAssignmentPolicy
* QueueAssignmentPolicy.java
add Apache license
* Hop.java
add String representation for convenience
This commit is contained in:
1 parent
d2a82228a3
commit
5bb8596fab
7 files changed
+375
-188
No files matched your search
@@ -0,0 +1,54 @@
|
||||
package org.archive.util;
|
||||
|
||||
import junit.framework.TestCase;
|
||||
|
||||
import org.apache.commons.lang.math.RandomUtils;
|
||||
|
||||
import st.ata.util.FPGenerator;
|
||||
|
||||
public class LongToIntConsistentHashTest extends TestCase {
|
||||
LongToIntConsistentHash conhash;
|
||||
|
||||
@Override
|
||||
protected void setUp() throws Exception {
|
||||
super.setUp();
|
||||
conhash = new LongToIntConsistentHash();
|
||||
}
|
||||
|
||||
public void testRange() {
|
||||
for(long in = 0; in < 10000; in++) {
|
||||
long longHash = FPGenerator.std64.fp(""+in);
|
||||
int upTo = RandomUtils.nextInt(32)+1;
|
||||
int bucket = conhash.bucketFor(longHash, upTo);
|
||||
assertTrue("bucket returned >= upTo",bucket < upTo);
|
||||
}
|
||||
}
|
||||
|
||||
public void testConsistencyUp() {
|
||||
int initialUpTo = 10;
|
||||
int changedCount = 0;
|
||||
for(long in = 0; in < 10000; in++) {
|
||||
long longHash = FPGenerator.std64.fp(""+in);
|
||||
int firstBucket = conhash.bucketFor(longHash, initialUpTo);
|
||||
int secondBucket = conhash.bucketFor(longHash, initialUpTo+1);
|
||||
if(secondBucket!=firstBucket) {
|
||||
changedCount++;
|
||||
}
|
||||
}
|
||||
assertTrue("excessive changes",changedCount < 2000);
|
||||
}
|
||||
|
||||
public void testConsistencyDown() {
|
||||
int initialUpTo = 10;
|
||||
int changedCount = 0;
|
||||
for(long in = 0; in < 10000; in++) {
|
||||
long longHash = FPGenerator.std64.fp(""+in);
|
||||
int firstBucket = conhash.bucketFor(longHash, initialUpTo);
|
||||
int secondBucket = conhash.bucketFor(longHash, initialUpTo-1);
|
||||
if(secondBucket!=firstBucket) {
|
||||
changedCount++;
|
||||
}
|
||||
}
|
||||
assertTrue("excessive changes",changedCount < 2000);
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user