diff --git a/.classpath b/.classpath
index e01a6f62..5c8f48b9 100644
--- a/.classpath
+++ b/.classpath
@@ -23,7 +23,7 @@
-
+
@@ -55,7 +55,7 @@
-
-
+
+
diff --git a/commons/src/main/java/org/archive/io/ReplayInputStream.java b/commons/src/main/java/org/archive/io/ReplayInputStream.java
index fccf5fd3..35ea8175 100644
--- a/commons/src/main/java/org/archive/io/ReplayInputStream.java
+++ b/commons/src/main/java/org/archive/io/ReplayInputStream.java
@@ -192,11 +192,15 @@ public class ReplayInputStream extends SeekInputStream
}
public void readFullyTo(OutputStream os) throws IOException {
+ readFullyTo(this, os);
+ }
+
+ public static void readFullyTo(InputStream in, OutputStream os) throws IOException {
byte[] buf = new byte[4096];
- int c = read(buf);
+ int c = in.read(buf);
while (c != -1) {
os.write(buf,0,c);
- c = read(buf);
+ c = in.read(buf);
}
}
@@ -218,12 +222,7 @@ public class ReplayInputStream extends SeekInputStream
*/
public void readContentTo(OutputStream os) throws IOException {
setToResponseBodyStart();
- byte[] buf = new byte[4096];
- int c = read(buf);
- while (c != -1) {
- os.write(buf,0,c);
- c = read(buf);
- }
+ readFullyTo(os);
}
/**
diff --git a/modules/src/test/java/org/archive/modules/fetcher/FetchHTTPTest.java b/modules/src/test/java/org/archive/modules/fetcher/FetchHTTPTest.java
index e3407597..9c075b66 100644
--- a/modules/src/test/java/org/archive/modules/fetcher/FetchHTTPTest.java
+++ b/modules/src/test/java/org/archive/modules/fetcher/FetchHTTPTest.java
@@ -19,27 +19,29 @@
package org.archive.modules.fetcher;
+import java.io.IOException;
+
+import org.archive.modules.CrawlMetadata;
import org.archive.modules.Processor;
-import org.archive.modules.ProcessorTestBase;
-import org.archive.modules.fetcher.FetchHTTP;
/**
*
*/
-public class FetchHTTPTest extends ProcessorTestBase {
+public class FetchHTTPTest extends FetchHTTPTestBase {
@Override
- protected Processor makeModule() {
- FetchHTTP result = new FetchHTTP();
- // FIXME: Set up server cache, credential store...
- // Use SimpleCookieStorage for FetchHTTP test, even though BDB is
- // actual default
- result.setCookieStorage(new SimpleCookieStorage());
- result.start();
- return result;
+ protected Processor makeModule() throws IOException {
+ FetchHTTP fetchHttp = new FetchHTTP();
+ fetchHttp.setCookieStorage(new SimpleCookieStorage());
+ fetchHttp.setServerCache(new DefaultServerCache());
+ CrawlMetadata uap = new CrawlMetadata();
+ uap.setUserAgentTemplate(getUserAgentString());
+ fetchHttp.setUserAgentProvider(new CrawlMetadata());
+
+ fetchHttp.start();
+ return fetchHttp;
}
-
- // TODO TESTME!
+
}
diff --git a/modules/src/test/java/org/archive/modules/fetcher/FetchHTTPTestBase.java b/modules/src/test/java/org/archive/modules/fetcher/FetchHTTPTestBase.java
index 9c0252ad..45e8d55a 100644
--- a/modules/src/test/java/org/archive/modules/fetcher/FetchHTTPTestBase.java
+++ b/modules/src/test/java/org/archive/modules/fetcher/FetchHTTPTestBase.java
@@ -2,19 +2,24 @@ package org.archive.modules.fetcher;
import java.io.IOException;
import java.util.logging.Logger;
+import java.util.regex.Pattern;
+import org.apache.commons.io.IOUtils;
import org.archive.modules.CrawlURI;
import org.archive.modules.Processor;
import org.archive.modules.ProcessorTestBase;
import org.archive.net.UURI;
import org.archive.net.UURIFactory;
+import org.archive.util.Base32;
import org.archive.util.Recorder;
+import org.archive.util.TmpDirTestCase;
import org.mortbay.jetty.Handler;
import org.mortbay.jetty.Server;
import org.mortbay.jetty.bio.SocketConnector;
import org.mortbay.jetty.handler.DefaultHandler;
import org.mortbay.jetty.handler.HandlerList;
import org.mortbay.jetty.handler.ResourceHandler;
+import org.mortbay.log.Log;
public abstract class FetchHTTPTestBase extends ProcessorTestBase {
@@ -27,6 +32,8 @@ public abstract class FetchHTTPTestBase extends ProcessorTestBase {
abstract protected Processor makeModule() throws IOException;
public Server startHttpFileServer(String path) throws Exception {
+ System.setProperty("org.mortbay.LEVEL", "DEBUG");
+ Log.getLogger(Server.class.getCanonicalName()).setDebugEnabled(true);
Server server = new Server();
SocketConnector sc = new SocketConnector();
sc.setHost("127.0.0.1");
@@ -34,6 +41,7 @@ public abstract class FetchHTTPTestBase extends ProcessorTestBase {
server.addConnector(sc);
ResourceHandler rhandler = new ResourceHandler();
rhandler.setResourceBase(path);
+ logger.info("serving files from " + path);
HandlerList handlers = new HandlerList();
handlers.setHandlers(new Handler[] {rhandler, new DefaultHandler()});
@@ -60,14 +68,44 @@ public abstract class FetchHTTPTestBase extends ProcessorTestBase {
public void testSomething() throws Exception {
ensureHttpServer();
+ fetcher = getFetcher();
- UURI uuri = UURIFactory.getInstance("http://localhost:7777/");
+ UURI uuri = UURIFactory.getInstance("http://localhost:7777/test.txt");
CrawlURI curi = new CrawlURI(uuri);
- Recorder.setHttpRecorder(null)
- curi.setRecorder(Recorder.getHttpRecorder());
-
+ curi.setRecorder(getRecorder());
getFetcher().process(curi);
+
+ byte[] buf = IOUtils.toByteArray(getRecorder().getRecordedOutput().getReplayInputStream());
+ // curi.getRecorder().getRecordedOutput().getReplayInputStream().readFullyTo(buf);
+ String requestString = new String(buf, "US-ASCII");
+ assertTrue(requestString.startsWith("GET /test.txt HTTP/1.0\r\n"));
+ // assertTrue(requestString.matches("(?is).*User-Agent: " + Pattern.quote(getUserAgentString()) + "\r\n.*"));
+ assertTrue(requestString.matches("(?is).*Connection: close\r\n.*"));
+ assertTrue(requestString.matches("(?is).*Accept: " + Pattern.quote("text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8") + "\r\n.*"));
+ assertTrue(requestString.matches("(?is).*Host: localhost:7777\r\n.*"));
+ assertTrue(requestString.endsWith("\r\n\r\n"));
- logger.info("data=" + curi.getData());
+ buf = IOUtils.toByteArray(curi.getRecorder().getEntityReplayInputStream());
+ String entityString = new String(buf, "US-ASCII");
+ assertTrue(entityString.equals("I am an ascii text file 39 bytes long.\n"));
+
+ assertEquals(curi.getContentLength(), 39);
+ assertEquals(Base32.encode(curi.getContentDigest()), "Y6G7VXZWY52LQA774YOVJ7TPZXMOMUY7");
+ }
+
+ protected String getUserAgentString() {
+ return getClass().getName();
+ }
+
+ protected Recorder getRecorder() throws IOException {
+ if (Recorder.getHttpRecorder() == null) {
+ Recorder httpRecorder = new Recorder(
+ TmpDirTestCase.tmpDir(),
+ "tt12345http", 16 * 1024, 512 * 1024);
+
+ Recorder.setHttpRecorder(httpRecorder);
+ }
+
+ return Recorder.getHttpRecorder();
}
}