mirror of
https://github.com/internetarchive/heritrix3.git
synced 2026-11-16 18:57:01 +00:00
Work for [HER-1656] add multiple-queues-per-host ('parallelQueues') capabilities to QueueAssignmentPolicies
* LongToIntConsistentHash.java + Test
expand range/distribution tests
correct problems with phony '-1' bucket being created/returned
This commit is contained in:
1 parent
5bb8596fab
commit
1c984753f0
2 files changed
+70
-14
No files matched your search
@@ -1,3 +1,22 @@
|
||||
/*
|
||||
* This file is part of the Heritrix web crawler (crawler.archive.org).
|
||||
*
|
||||
* Licensed to the Internet Archive (IA) by one or more individual
|
||||
* contributors.
|
||||
*
|
||||
* The IA licenses this file to You under the Apache License, Version 2.0
|
||||
* (the "License"); you may not use this file except in compliance with
|
||||
* the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package org.archive.util;
|
||||
|
||||
import junit.framework.TestCase;
|
||||
@@ -21,6 +40,23 @@ public class LongToIntConsistentHashTest extends TestCase {
|
||||
int upTo = RandomUtils.nextInt(32)+1;
|
||||
int bucket = conhash.bucketFor(longHash, upTo);
|
||||
assertTrue("bucket returned >= upTo",bucket < upTo);
|
||||
assertTrue("bucket returned < 0: "+bucket,bucket >= 0);
|
||||
|
||||
}
|
||||
}
|
||||
|
||||
public void testTwoWayDistribution() {
|
||||
// SecureRandom rand = new SecureRandom("foobar".getBytes());
|
||||
for(int p = 0; p < 20; p++) {
|
||||
int[] landings = new int[2];
|
||||
for(long in = 0; in < 100000; in++) {
|
||||
long longHash = FPGenerator.std64.fp(p+"a"+in);
|
||||
// long longHash = rand.nextLong();
|
||||
// long longHash = ArchiveUtils.doubleMurmur((p+":"+in).getBytes());
|
||||
landings[conhash.bucketFor(longHash, 2)]++;
|
||||
}
|
||||
// System.out.println(landings[0]+","+landings[1]);
|
||||
assertTrue("excessive changes",Math.abs(landings[0]-landings[1]) < 2000);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -35,7 +71,7 @@ public class LongToIntConsistentHashTest extends TestCase {
|
||||
changedCount++;
|
||||
}
|
||||
}
|
||||
assertTrue("excessive changes",changedCount < 2000);
|
||||
assertTrue("excessive changes: "+changedCount,changedCount < 2000);
|
||||
}
|
||||
|
||||
public void testConsistencyDown() {
|
||||
@@ -49,6 +85,6 @@ public class LongToIntConsistentHashTest extends TestCase {
|
||||
changedCount++;
|
||||
}
|
||||
}
|
||||
assertTrue("excessive changes",changedCount < 2000);
|
||||
assertTrue("excessive changes: "+changedCount,changedCount < 2000);
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user