lee/heritrix3
1
0
Fork 0
mirror of https://github.com/internetarchive/heritrix3.git synced 2026-09-26 07:35:39 +00:00
Code Issues Packages Projects Releases Wiki Activity

* FetchHTTPTest.java

Browse Source 
shut down test servers in tearDown() (server startup/shutdown happens on every test, inefficient but it doesn't take all that long anyway)
* ContentDigestHistoryTest.java
    shut down test servers in finally block
This commit is contained in:
Noah Levitt
2012-09-26 19:29:11 -07:00
parent e6fbe45774
commit 7a92e1ff7c
2 changed files with 101 additions and 70 deletions
Download Patch File Download Diff File Expand all files Collapse all files
modules/src/test/java/org/archive/modules/fetcher/FetchHTTPTest.java
+35 -12
View File
@@ -83,7 +83,7 @@ import sun.security.tools.KeyTool;
public class FetchHTTPTest extends ProcessorTestBase {
// private static Logger logger = Logger.getLogger(FetchHTTPTest.class.getName());
// private static Logger logger = Logger.getLogger(FetchHTTPTest.class.getName());
/* uncomment this for detailed logging from jetty and heritrix */
// static {
@@ -460,6 +460,7 @@ public class FetchHTTPTest extends ProcessorTestBase {
CrawlURI curi = makeCrawlURI("http://localhost:7777/");
fetcher().process(curi);
// logger.info('\n' + httpRequestString(curi) + "\n\n" + rawResponseString(curi));
runDefaultChecks(curi, "acceptHeaders");
// special checks for this test
@@ -961,17 +962,19 @@ public class FetchHTTPTest extends ProcessorTestBase {
NoResponseServer noResponseServer = new NoResponseServer("localhost", 7780);
noResponseServer.start();
// CrawlURI curi = makeCrawlURI("http://stats.bbc.co.uk/robots.txt");
CrawlURI curi = makeCrawlURI("http://localhost:7780");
fetcher().process(curi);
assertEquals(1, curi.getNonFatalFailures().size());
assertTrue(curi.getNonFatalFailures().toArray()[0] instanceof NoHttpResponseException);
// assertEquals(FetchStatusCodes.S_CONNECT_FAILED, curi.getFetchStatus());
assertEquals(FetchStatusCodes.S_CONNECT_LOST, curi.getFetchStatus());
assertEquals(0, curi.getFetchCompletedTime());
noResponseServer.beDone();
noResponseServer.join();
try {
// CrawlURI curi = makeCrawlURI("http://stats.bbc.co.uk/robots.txt");
CrawlURI curi = makeCrawlURI("http://localhost:7780");
fetcher().process(curi);
assertEquals(1, curi.getNonFatalFailures().size());
assertTrue(curi.getNonFatalFailures().toArray()[0] instanceof NoHttpResponseException);
// assertEquals(FetchStatusCodes.S_CONNECT_FAILED, curi.getFetchStatus());
assertEquals(FetchStatusCodes.S_CONNECT_LOST, curi.getFetchStatus());
assertEquals(0, curi.getFetchCompletedTime());
} finally {
noResponseServer.beDone();
noResponseServer.join();
}
}
@Override
@@ -1000,4 +1003,24 @@ public class FetchHTTPTest extends ProcessorTestBase {
return header.getValue();
}
}
@Override
protected void tearDown() throws Exception {
super.tearDown();
if (httpServers != null) {
for (Server server: httpServers.values()) {
server.stop();
server.destroy();
}
httpServers = null;
}
if (fetcher != null) {
fetcher.stop();
fetcher = null;
}
}
}
modules/src/test/java/org/archive/modules/recrawl/ContentDigestHistoryTest.java
+66 -58
View File
@@ -159,72 +159,80 @@ public class ContentDigestHistoryTest extends TmpDirTestCase {
return curi;
}
/*
* fetches two different urls with same content, writes warc records, checks results
*/
public void testWarcDedupe() throws Exception {
historyStore().store.clear();
assertTrue(historyStore().store.isEmpty());
startHttpServer();
Server server = newHttpServer();
FetchHTTP fetcher = FetchHTTPTest.newTestFetchHttp(getClass().getName());
WARCWriterProcessor warcWriter = WARCWriterProcessorTest.newTestWarcWriter(getClass().getName());
warcWriter.setServerCache(fetcher.getServerCache());
warcWriter.start();
fetcher.start();
CrawlURI curi1 = makeCrawlURI("http://127.0.0.1:7777/url1");
CrawlURI curi2 = makeCrawlURI("http://127.0.0.1:7777/url2");
fetcher.process(curi1);
assertEquals(200, curi1.getFetchStatus());
assertEquals(141, curi1.getContentSize());
assertEquals("sha1:TQ5R6YVOZLTQENRIIENVGXHOPX3YCRNJ", curi1.getContentDigestSchemeString());
assertFalse(curi1.hasContentDigestHistory());
loader().process(curi1);
assertTrue(curi1.hasContentDigestHistory());
assertTrue(curi1.getContentDigestHistory().isEmpty());
try {
server.start();
warcWriter.start();
fetcher.start();
warcWriter.process(curi1);
assertEquals(curi1.getUURI().toString(), curi1.getContentDigestHistory().get(A_ORIGINAL_URL));
assertEquals(1, curi1.getContentDigestHistory().get(A_CONTENT_DIGEST_COUNT));
String report = warcWriter.report();
assertTrue(report.contains("Total CrawlURIs: 1\n"));
assertTrue(report.contains("Revisit records: 0\n"));
storer().process(curi1);
assertEquals(1, historyStore().store.size());
assertNotNull(historyStore().store.get("sha1:TQ5R6YVOZLTQENRIIENVGXHOPX3YCRNJ"));
assertEquals(curi1.getUURI().toString(), historyStore().store.get("sha1:TQ5R6YVOZLTQENRIIENVGXHOPX3YCRNJ").get(A_ORIGINAL_URL));
assertEquals(1, historyStore().store.get("sha1:TQ5R6YVOZLTQENRIIENVGXHOPX3YCRNJ").get(A_CONTENT_DIGEST_COUNT));
CrawlURI curi1 = makeCrawlURI("http://127.0.0.1:7777/url1");
CrawlURI curi2 = makeCrawlURI("http://127.0.0.1:7777/url2");
fetcher.process(curi2);
assertEquals(200, curi1.getFetchStatus());
assertEquals(141, curi1.getContentSize());
assertEquals("sha1:TQ5R6YVOZLTQENRIIENVGXHOPX3YCRNJ", curi1.getContentDigestSchemeString());
assertFalse(curi2.hasContentDigestHistory());
fetcher.process(curi1);
assertEquals(200, curi1.getFetchStatus());
assertEquals(141, curi1.getContentSize());
assertEquals("sha1:TQ5R6YVOZLTQENRIIENVGXHOPX3YCRNJ", curi1.getContentDigestSchemeString());
assertFalse(curi1.hasContentDigestHistory());
loader().process(curi2);
assertTrue(curi2.hasContentDigestHistory());
assertEquals(curi1.getUURI().toString(), curi2.getContentDigestHistory().get(A_ORIGINAL_URL));
assertNotSame(curi2.getUURI().toString(), curi2.getContentDigestHistory().get(A_ORIGINAL_URL));
assertEquals(1, curi2.getContentDigestHistory().get(A_CONTENT_DIGEST_COUNT));
warcWriter.process(curi2);
assertEquals(curi1.getUURI().toString(), curi2.getContentDigestHistory().get(A_ORIGINAL_URL));
assertNotSame(curi2.getUURI().toString(), curi2.getContentDigestHistory().get(A_ORIGINAL_URL));
assertEquals(2, curi2.getContentDigestHistory().get(A_CONTENT_DIGEST_COUNT));
report = warcWriter.report();
assertTrue(report.contains("Total CrawlURIs: 2\n"));
assertTrue(report.contains("Revisit records: 1\n"));
storer().process(curi2);
assertEquals(1, historyStore().store.size());
assertNotNull(historyStore().store.get("sha1:TQ5R6YVOZLTQENRIIENVGXHOPX3YCRNJ"));
assertEquals(curi1.getUURI().toString(), historyStore().store.get("sha1:TQ5R6YVOZLTQENRIIENVGXHOPX3YCRNJ").get(A_ORIGINAL_URL));
assertEquals(2, historyStore().store.get("sha1:TQ5R6YVOZLTQENRIIENVGXHOPX3YCRNJ").get(A_CONTENT_DIGEST_COUNT));
warcWriter.stop();
fetcher.stop();
loader().process(curi1);
assertTrue(curi1.hasContentDigestHistory());
assertTrue(curi1.getContentDigestHistory().isEmpty());
warcWriter.process(curi1);
assertEquals(curi1.getUURI().toString(), curi1.getContentDigestHistory().get(A_ORIGINAL_URL));
assertEquals(1, curi1.getContentDigestHistory().get(A_CONTENT_DIGEST_COUNT));
String report = warcWriter.report();
assertTrue(report.contains("Total CrawlURIs: 1\n"));
assertTrue(report.contains("Revisit records: 0\n"));
storer().process(curi1);
assertEquals(1, historyStore().store.size());
assertNotNull(historyStore().store.get("sha1:TQ5R6YVOZLTQENRIIENVGXHOPX3YCRNJ"));
assertEquals(curi1.getUURI().toString(), historyStore().store.get("sha1:TQ5R6YVOZLTQENRIIENVGXHOPX3YCRNJ").get(A_ORIGINAL_URL));
assertEquals(1, historyStore().store.get("sha1:TQ5R6YVOZLTQENRIIENVGXHOPX3YCRNJ").get(A_CONTENT_DIGEST_COUNT));
fetcher.process(curi2);
assertEquals(200, curi1.getFetchStatus());
assertEquals(141, curi1.getContentSize());
assertEquals("sha1:TQ5R6YVOZLTQENRIIENVGXHOPX3YCRNJ", curi1.getContentDigestSchemeString());
assertFalse(curi2.hasContentDigestHistory());
loader().process(curi2);
assertTrue(curi2.hasContentDigestHistory());
assertEquals(curi1.getUURI().toString(), curi2.getContentDigestHistory().get(A_ORIGINAL_URL));
assertNotSame(curi2.getUURI().toString(), curi2.getContentDigestHistory().get(A_ORIGINAL_URL));
assertEquals(1, curi2.getContentDigestHistory().get(A_CONTENT_DIGEST_COUNT));
warcWriter.process(curi2);
assertEquals(curi1.getUURI().toString(), curi2.getContentDigestHistory().get(A_ORIGINAL_URL));
assertNotSame(curi2.getUURI().toString(), curi2.getContentDigestHistory().get(A_ORIGINAL_URL));
assertEquals(2, curi2.getContentDigestHistory().get(A_CONTENT_DIGEST_COUNT));
report = warcWriter.report();
assertTrue(report.contains("Total CrawlURIs: 2\n"));
assertTrue(report.contains("Revisit records: 1\n"));
storer().process(curi2);
assertEquals(1, historyStore().store.size());
assertNotNull(historyStore().store.get("sha1:TQ5R6YVOZLTQENRIIENVGXHOPX3YCRNJ"));
assertEquals(curi1.getUURI().toString(), historyStore().store.get("sha1:TQ5R6YVOZLTQENRIIENVGXHOPX3YCRNJ").get(A_ORIGINAL_URL));
assertEquals(2, historyStore().store.get("sha1:TQ5R6YVOZLTQENRIIENVGXHOPX3YCRNJ").get(A_CONTENT_DIGEST_COUNT));
} finally {
warcWriter.stop();
fetcher.stop();
server.stop();
}
}
@@ -239,7 +247,7 @@ public class ContentDigestHistoryTest extends TmpDirTestCase {
}
protected static final String DEFAULT_PAYLOAD_STRING = "abcdefghijklmnopqrstuvwxyz0123456789\n";
protected void startHttpServer() throws Exception {
protected Server newHttpServer() throws Exception {
HandlerCollection handlers = new HandlerCollection();
handlers.addHandler(new SessionHandler(){
@Override
@@ -262,6 +270,6 @@ public class ContentDigestHistoryTest extends TmpDirTestCase {
sc.setPort(7777);
server.addConnector(sc);
server.start();
return server;
}
}