lee/heritrix3
1
0
Fork 0
mirror of https://github.com/internetarchive/heritrix3.git synced 2026-09-24 06:36:12 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #405 from internetarchive/faster-tests

Browse Source 
Speed up test suite
This commit is contained in:
Alex Osborne
2021-06-25 11:52:40 +09:00
committed by GitHub
parent 04f958e987 c0cb68f754
commit 6f32d018be
6 changed files with 56 additions and 35 deletions
Download Patch File Download Diff File Expand all files Collapse all files
commons/src/test/java/org/archive/util/BloomFilterTest.java
+18 -4
View File
@@ -21,7 +21,11 @@ package org.archive.util;
import java.util.Random;
import junit.framework.TestCase;
import org.junit.Ignore;
import org.junit.Test;
import static org.junit.Assert.*;
import static org.junit.Assume.assumeTrue;
/**
@@ -30,7 +34,7 @@ import junit.framework.TestCase;
* @author gojomo
* @version $Date: 2009-11-19 14:39:53 -0800 (Thu, 19 Nov 2009) $, $Revision: 6674 $
*/
public abstract class BloomFilterTest extends TestCase {
public abstract class BloomFilterTest {
abstract BloomFilter createBloom(long n, int d, Random random);
@@ -70,6 +74,8 @@ public abstract class BloomFilterTest extends TestCase {
* Renamed to non-'test' name so not automatically run, because can
* take 15+ minutes to complete.
*/
@Test
@Ignore
public void xestOversized() {
trialWithParameters(200000000,22,200000000,32000000);
}
@@ -81,15 +87,23 @@ public abstract class BloomFilterTest extends TestCase {
* Renamed to non-'test' name so not automatically run, because can
* take 15+ minutes to complete.
*/
@Test
@Ignore
public void xestDefaultFull() {
trialWithParameters(125000000,22,125000000,34000000);
}
@Test
public void testDefaultAbbreviated() {
assumeTrue("use -DrunSlowTests=true to enable this test (it takes about 25 seconds)",
"true".equals(System.getProperty("runSlowTests")));
trialWithParameters(125000000,22,17000000,0);
}
@Test
public void testSmall() {
assumeTrue("use -DrunSlowTests=true to enable this test (it takes about 20 seconds)",
"true".equals(System.getProperty("runSlowTests")));
trialWithParameters(10000000, 20, 10000000, 10000000);
}
commons/src/test/java/org/archive/util/ObjectIdentityBdbManualCacheTest.java
+29 -21
View File
@@ -18,25 +18,32 @@
*/
package org.archive.util;
import java.io.File;
import java.util.HashMap;
import java.util.concurrent.atomic.AtomicInteger;
import org.apache.commons.io.FileUtils;
import org.archive.util.bdbje.EnhancedEnvironment;
import java.io.File;
import java.util.HashMap;
import java.util.concurrent.atomic.AtomicInteger;
import org.apache.commons.io.FileUtils;
import org.archive.util.bdbje.EnhancedEnvironment;
import org.junit.*;
import org.junit.rules.TemporaryFolder;
import static org.junit.Assert.*;
import static org.junit.Assume.assumeTrue;
/**
* @author stack
* @author gojomo
* @version $Date: 2009-08-03 23:50:43 -0700 (Mon, 03 Aug 2009) $, $Revision: 6434 $
*/
public class ObjectIdentityBdbManualCacheTest extends TmpDirTestCase {
public class ObjectIdentityBdbManualCacheTest {
@Rule
public TemporaryFolder tmpFolder = new TemporaryFolder();
EnhancedEnvironment env;
private ObjectIdentityBdbManualCache<IdentityCacheableWrapper<HashMap<String,String>>> cache;
protected void setUp() throws Exception {
super.setUp();
File envDir = new File(getTmpDir(),"ObjectIdentityBdbCacheTest");
@Before
public void setUp() throws Exception {
File envDir = tmpFolder.newFolder("ObjectIdentityBdbCacheTest");
org.archive.util.FileUtils.ensureWriteableDirectory(envDir);
FileUtils.deleteDirectory(envDir);
org.archive.util.FileUtils.ensureWriteableDirectory(envDir);
@@ -44,16 +51,19 @@ public class ObjectIdentityBdbManualCacheTest extends TmpDirTestCase {
this.cache = new ObjectIdentityBdbManualCache<IdentityCacheableWrapper<HashMap<String,String>>>();
this.cache.initialize(env,"setUpCache",IdentityCacheableWrapper.class, env.getClassCatalog());
}
protected void tearDown() throws Exception {
@After
public void tearDown() throws Exception {
this.cache.close();
File envDir = env.getHome();
env.close();
FileUtils.deleteDirectory(envDir);
super.tearDown();
}
@Test
public void testReadConsistencyUnderLoad() throws Exception {
assumeTrue("use -DrunSlowTests=true to enable this test (it takes about 1 minute)",
"true".equals(System.getProperty("runSlowTests")));
final ObjectIdentityBdbManualCache<IdentityCacheableWrapper<AtomicInteger>> cbdbmap =
new ObjectIdentityBdbManualCache<>();
cbdbmap.initialize(env,
@@ -95,7 +105,8 @@ public class ObjectIdentityBdbManualCacheTest extends TmpDirTestCase {
}
// SUCCESS
}
@Test
public void testBackingDbGetsUpdated() {
// Set up values.
final String value = "value";
@@ -129,6 +140,8 @@ public class ObjectIdentityBdbManualCacheTest extends TmpDirTestCase {
* expunged of otherwise unreferenced entries as expected.
* @throws InterruptedException
*/
@Test
@Ignore
public void xestMemMapCleared() throws InterruptedException {
TestUtils.forceScarceMemory();
System.gc(); // minimize effects of earlier test heap use
@@ -160,9 +173,4 @@ public class ObjectIdentityBdbManualCacheTest extends TmpDirTestCase {
System.out.println(cache.size()+","+cache.memMap.size()+","+cache.memMap.keySet().size()+","+cache.memMap.values().size()+","+countNonNull);
assertEquals("memMap not cleared", 0, cache.memMap.size());
}
public static void main(String [] args) {
junit.textui.TestRunner.run(ObjectIdentityBdbManualCacheTest.class);
}
}
engine/src/main/java/org/archive/crawler/framework/CrawlJob.java
+2 -3
View File
@@ -408,7 +408,7 @@ public class CrawlJob implements Comparable<CrawlJob>, ApplicationListener<Appli
* (Note the crawl may have been configured to start in a 'paused'
* state.)
*/
public synchronized void launch() {
public void launch() {
if (isProfile()) {
throw new IllegalArgumentException("Can't launch profile" + this);
}
@@ -449,9 +449,8 @@ public class CrawlJob implements Comparable<CrawlJob>, ApplicationListener<Appli
getJobLogger().log(Level.INFO,"Job launched");
scanJobLog();
launcher.start();
// look busy (and give startContext/crawlStart a chance)
try {
Thread.sleep(1500);
launcher.join();
} catch (InterruptedException e) {
// do nothing
}
engine/src/main/java/org/archive/crawler/framework/Engine.java
+3 -3
View File
@@ -317,7 +317,7 @@ public class Engine {
return true;
}
try {
Thread.sleep(500);
Thread.sleep(250);
} catch (InterruptedException e) {
break;
}
@@ -329,8 +329,8 @@ public class Engine {
break;
}
try {
// wait an extra second for good measure
Thread.sleep(1000);
// wait an extra quarter second for good measure
Thread.sleep(250);
} catch (InterruptedException e) {
// ignore
}
engine/src/main/java/org/archive/crawler/frontier/AbstractFrontier.java
+3 -3
View File
@@ -364,7 +364,7 @@ public abstract class AbstractFrontier
}
reachedState(reachedState);
Thread.sleep(1000);
Thread.sleep(250);
if(isEmpty()&&targetState==State.RUN) {
requestState(State.EMPTY);
@@ -384,7 +384,7 @@ public abstract class AbstractFrontier
reachedState(State.PAUSE);
}
Thread.sleep(1000);
Thread.sleep(250);
}
break;
case FINISH:
@@ -393,7 +393,7 @@ public abstract class AbstractFrontier
outboundLock.writeLock().lock();
// process all inbound
while (getInProcessCount()>0) {
Thread.sleep(1000);
Thread.sleep(250);
}
logger.fine("0 urls in process, running final tasks");
finalTasks();
engine/src/main/java/org/archive/crawler/frontier/WorkQueueFrontier.java
+1 -1
View File
@@ -727,7 +727,7 @@ implements Closeable,
// next time
if(getTotalEligibleInactiveQueues()==0) {
try {
Thread.sleep(1000);
Thread.sleep(250);
} catch (InterruptedException e) {
//
}