From 1f315a7cc7c1fb8f51baba7d0ed01bc06eefb3a3 Mon Sep 17 00:00:00 2001 From: Noah Levitt Date: Fri, 29 Jun 2012 02:04:39 -0700 Subject: [PATCH] unit tests for FetchHTTP and FetchHTTP2 --- .classpath | 6 +-- .../org/archive/io/ReplayInputStream.java | 15 +++--- .../modules/fetcher/FetchHTTPTest.java | 28 ++++++----- .../modules/fetcher/FetchHTTPTestBase.java | 48 +++++++++++++++++-- 4 files changed, 68 insertions(+), 29 deletions(-) diff --git a/.classpath b/.classpath index e01a6f62..5c8f48b9 100644 --- a/.classpath +++ b/.classpath @@ -23,7 +23,7 @@ - + @@ -55,7 +55,7 @@ - - + + diff --git a/commons/src/main/java/org/archive/io/ReplayInputStream.java b/commons/src/main/java/org/archive/io/ReplayInputStream.java index fccf5fd3..35ea8175 100644 --- a/commons/src/main/java/org/archive/io/ReplayInputStream.java +++ b/commons/src/main/java/org/archive/io/ReplayInputStream.java @@ -192,11 +192,15 @@ public class ReplayInputStream extends SeekInputStream } public void readFullyTo(OutputStream os) throws IOException { + readFullyTo(this, os); + } + + public static void readFullyTo(InputStream in, OutputStream os) throws IOException { byte[] buf = new byte[4096]; - int c = read(buf); + int c = in.read(buf); while (c != -1) { os.write(buf,0,c); - c = read(buf); + c = in.read(buf); } } @@ -218,12 +222,7 @@ public class ReplayInputStream extends SeekInputStream */ public void readContentTo(OutputStream os) throws IOException { setToResponseBodyStart(); - byte[] buf = new byte[4096]; - int c = read(buf); - while (c != -1) { - os.write(buf,0,c); - c = read(buf); - } + readFullyTo(os); } /** diff --git a/modules/src/test/java/org/archive/modules/fetcher/FetchHTTPTest.java b/modules/src/test/java/org/archive/modules/fetcher/FetchHTTPTest.java index e3407597..9c075b66 100644 --- a/modules/src/test/java/org/archive/modules/fetcher/FetchHTTPTest.java +++ b/modules/src/test/java/org/archive/modules/fetcher/FetchHTTPTest.java @@ -19,27 +19,29 @@ package org.archive.modules.fetcher; +import java.io.IOException; + +import org.archive.modules.CrawlMetadata; import org.archive.modules.Processor; -import org.archive.modules.ProcessorTestBase; -import org.archive.modules.fetcher.FetchHTTP; /** * */ -public class FetchHTTPTest extends ProcessorTestBase { +public class FetchHTTPTest extends FetchHTTPTestBase { @Override - protected Processor makeModule() { - FetchHTTP result = new FetchHTTP(); - // FIXME: Set up server cache, credential store... - // Use SimpleCookieStorage for FetchHTTP test, even though BDB is - // actual default - result.setCookieStorage(new SimpleCookieStorage()); - result.start(); - return result; + protected Processor makeModule() throws IOException { + FetchHTTP fetchHttp = new FetchHTTP(); + fetchHttp.setCookieStorage(new SimpleCookieStorage()); + fetchHttp.setServerCache(new DefaultServerCache()); + CrawlMetadata uap = new CrawlMetadata(); + uap.setUserAgentTemplate(getUserAgentString()); + fetchHttp.setUserAgentProvider(new CrawlMetadata()); + + fetchHttp.start(); + return fetchHttp; } - - // TODO TESTME! + } diff --git a/modules/src/test/java/org/archive/modules/fetcher/FetchHTTPTestBase.java b/modules/src/test/java/org/archive/modules/fetcher/FetchHTTPTestBase.java index 9c0252ad..45e8d55a 100644 --- a/modules/src/test/java/org/archive/modules/fetcher/FetchHTTPTestBase.java +++ b/modules/src/test/java/org/archive/modules/fetcher/FetchHTTPTestBase.java @@ -2,19 +2,24 @@ package org.archive.modules.fetcher; import java.io.IOException; import java.util.logging.Logger; +import java.util.regex.Pattern; +import org.apache.commons.io.IOUtils; import org.archive.modules.CrawlURI; import org.archive.modules.Processor; import org.archive.modules.ProcessorTestBase; import org.archive.net.UURI; import org.archive.net.UURIFactory; +import org.archive.util.Base32; import org.archive.util.Recorder; +import org.archive.util.TmpDirTestCase; import org.mortbay.jetty.Handler; import org.mortbay.jetty.Server; import org.mortbay.jetty.bio.SocketConnector; import org.mortbay.jetty.handler.DefaultHandler; import org.mortbay.jetty.handler.HandlerList; import org.mortbay.jetty.handler.ResourceHandler; +import org.mortbay.log.Log; public abstract class FetchHTTPTestBase extends ProcessorTestBase { @@ -27,6 +32,8 @@ public abstract class FetchHTTPTestBase extends ProcessorTestBase { abstract protected Processor makeModule() throws IOException; public Server startHttpFileServer(String path) throws Exception { + System.setProperty("org.mortbay.LEVEL", "DEBUG"); + Log.getLogger(Server.class.getCanonicalName()).setDebugEnabled(true); Server server = new Server(); SocketConnector sc = new SocketConnector(); sc.setHost("127.0.0.1"); @@ -34,6 +41,7 @@ public abstract class FetchHTTPTestBase extends ProcessorTestBase { server.addConnector(sc); ResourceHandler rhandler = new ResourceHandler(); rhandler.setResourceBase(path); + logger.info("serving files from " + path); HandlerList handlers = new HandlerList(); handlers.setHandlers(new Handler[] {rhandler, new DefaultHandler()}); @@ -60,14 +68,44 @@ public abstract class FetchHTTPTestBase extends ProcessorTestBase { public void testSomething() throws Exception { ensureHttpServer(); + fetcher = getFetcher(); - UURI uuri = UURIFactory.getInstance("http://localhost:7777/"); + UURI uuri = UURIFactory.getInstance("http://localhost:7777/test.txt"); CrawlURI curi = new CrawlURI(uuri); - Recorder.setHttpRecorder(null) - curi.setRecorder(Recorder.getHttpRecorder()); - + curi.setRecorder(getRecorder()); getFetcher().process(curi); + + byte[] buf = IOUtils.toByteArray(getRecorder().getRecordedOutput().getReplayInputStream()); + // curi.getRecorder().getRecordedOutput().getReplayInputStream().readFullyTo(buf); + String requestString = new String(buf, "US-ASCII"); + assertTrue(requestString.startsWith("GET /test.txt HTTP/1.0\r\n")); + // assertTrue(requestString.matches("(?is).*User-Agent: " + Pattern.quote(getUserAgentString()) + "\r\n.*")); + assertTrue(requestString.matches("(?is).*Connection: close\r\n.*")); + assertTrue(requestString.matches("(?is).*Accept: " + Pattern.quote("text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8") + "\r\n.*")); + assertTrue(requestString.matches("(?is).*Host: localhost:7777\r\n.*")); + assertTrue(requestString.endsWith("\r\n\r\n")); - logger.info("data=" + curi.getData()); + buf = IOUtils.toByteArray(curi.getRecorder().getEntityReplayInputStream()); + String entityString = new String(buf, "US-ASCII"); + assertTrue(entityString.equals("I am an ascii text file 39 bytes long.\n")); + + assertEquals(curi.getContentLength(), 39); + assertEquals(Base32.encode(curi.getContentDigest()), "Y6G7VXZWY52LQA774YOVJ7TPZXMOMUY7"); + } + + protected String getUserAgentString() { + return getClass().getName(); + } + + protected Recorder getRecorder() throws IOException { + if (Recorder.getHttpRecorder() == null) { + Recorder httpRecorder = new Recorder( + TmpDirTestCase.tmpDir(), + "tt12345http", 16 * 1024, 512 * 1024); + + Recorder.setHttpRecorder(httpRecorder); + } + + return Recorder.getHttpRecorder(); } }