lee/heritrix3
1
0
Fork 0
mirror of https://github.com/internetarchive/heritrix3.git synced 2026-09-25 07:05:49 +00:00
Code Issues Packages Projects Releases Wiki Activity

unit tests for FetchHTTP and FetchHTTP2

Browse Source
This commit is contained in:
Noah Levitt
2012-06-29 02:04:39 -07:00
parent 2653583cdd
commit 1f315a7cc7
4 changed files with 68 additions and 29 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.classpath
+3 -3
View File
@@ -23,7 +23,7 @@
<classpathentry kind="var" path="M2_REPO/commons-codec/commons-codec/1.3/commons-codec-1.3.jar" sourcepath="M2_REPO/commons-codec/commons-codec/1.3/commons-codec-1.3-sources.jar"/>
<classpathentry kind="var" path="M2_REPO/fastutil/fastutil/5.0.7/fastutil-5.0.7.jar"/>
<classpathentry kind="var" path="M2_REPO/net/java/dev/jets3t/jets3t/0.5.0/jets3t-0.5.0.jar"/>
<classpathentry kind="var" path="M2_REPO/org/mortbay/jetty/jetty-util/6.1.14/jetty-util-6.1.14.jar"/>
<classpathentry kind="var" path="M2_REPO/org/mortbay/jetty/jetty-util/6.1.14/jetty-util-6.1.14.jar" sourcepath="/M2_REPO/org/mortbay/jetty/jetty-util/6.1.14/jetty-util-6.1.14-sources.jar"/>
<classpathentry kind="var" path="M2_REPO/commons-net/commons-net/2.0/commons-net-2.0.jar" sourcepath="/M2_REPO/commons-net/commons-net/2.0/commons-net-2.0-sources.jar"/>
<classpathentry kind="var" path="M2_REPO/junit/junit/3.8.2/junit-3.8.2.jar" sourcepath="M2_REPO/junit/junit/3.8.2/junit-3.8.2-sources.jar"/>
<classpathentry kind="var" path="M2_REPO/poi/poi-scratchpad/2.5.1-final-20040804/poi-scratchpad-2.5.1-final-20040804.jar"/>
@@ -55,7 +55,7 @@
<classpathentry kind="var" path="M2_REPO/com/esotericsoftware/reflectasm/0.8/reflectasm-0.8.jar"/>
<classpathentry kind="var" path="M2_REPO/com/google/guava/guava/r08/guava-r08.jar" sourcepath="/M2_REPO/com/google/guava/guava/r08/guava-r08-sources.jar"/>
<classpathentry kind="var" path="M2_REPO/net/java/dev/jna/jna/3.2.3/jna-3.2.3.jar" sourcepath="/M2_REPO/net/java/dev/jna/jna/3.2.3/jna-3.2.3-sources.jar"/>
<classpathentry kind="var" path="M2_REPO/org/apache/httpcomponents/httpclient/4.2/httpclient-4.2.jar" sourcepath="/M2_REPO/org/apache/httpcomponents/httpclient/4.2/httpclient-4.2-sources.jar" />
<classpathentry kind="var" path="M2_REPO/org/apache/httpcomponents/httpcore/4.2/httpcore-4.2.jar" sourcepath="/M2_REPO/org/apache/httpcomponents/httpcore/4.2/httpcore-4.2-sources.jar" />
<classpathentry kind="var" path="M2_REPO/org/apache/httpcomponents/httpclient/4.2/httpclient-4.2.jar" sourcepath="/M2_REPO/org/apache/httpcomponents/httpclient/4.2/httpclient-4.2-sources.jar"/>
<classpathentry kind="var" path="M2_REPO/org/apache/httpcomponents/httpcore/4.2/httpcore-4.2.jar" sourcepath="/M2_REPO/org/apache/httpcomponents/httpcore/4.2/httpcore-4.2-sources.jar"/>
<classpathentry kind="output" path="bin"/>
</classpath>
commons/src/main/java/org/archive/io/ReplayInputStream.java
+7 -8
View File
@@ -192,11 +192,15 @@ public class ReplayInputStream extends SeekInputStream
}
public void readFullyTo(OutputStream os) throws IOException {
readFullyTo(this, os);
}
public static void readFullyTo(InputStream in, OutputStream os) throws IOException {
byte[] buf = new byte[4096];
int c = read(buf);
int c = in.read(buf);
while (c != -1) {
os.write(buf,0,c);
c = read(buf);
c = in.read(buf);
}
}
@@ -218,12 +222,7 @@ public class ReplayInputStream extends SeekInputStream
*/
public void readContentTo(OutputStream os) throws IOException {
setToResponseBodyStart();
byte[] buf = new byte[4096];
int c = read(buf);
while (c != -1) {
os.write(buf,0,c);
c = read(buf);
}
readFullyTo(os);
}
/**
modules/src/test/java/org/archive/modules/fetcher/FetchHTTPTest.java
+15 -13
View File
@@ -19,27 +19,29 @@
package org.archive.modules.fetcher;
import java.io.IOException;
import org.archive.modules.CrawlMetadata;
import org.archive.modules.Processor;
import org.archive.modules.ProcessorTestBase;
import org.archive.modules.fetcher.FetchHTTP;
/**
*
*/
public class FetchHTTPTest extends ProcessorTestBase {
public class FetchHTTPTest extends FetchHTTPTestBase {
@Override
protected Processor makeModule() {
FetchHTTP result = new FetchHTTP();
// FIXME: Set up server cache, credential store...
// Use SimpleCookieStorage for FetchHTTP test, even though BDB is
// actual default
result.setCookieStorage(new SimpleCookieStorage());
result.start();
return result;
protected Processor makeModule() throws IOException {
FetchHTTP fetchHttp = new FetchHTTP();
fetchHttp.setCookieStorage(new SimpleCookieStorage());
fetchHttp.setServerCache(new DefaultServerCache());
CrawlMetadata uap = new CrawlMetadata();
uap.setUserAgentTemplate(getUserAgentString());
fetchHttp.setUserAgentProvider(new CrawlMetadata());
fetchHttp.start();
return fetchHttp;
}
// TODO TESTME!
}
modules/src/test/java/org/archive/modules/fetcher/FetchHTTPTestBase.java
+43 -5
View File
@@ -2,19 +2,24 @@ package org.archive.modules.fetcher;
import java.io.IOException;
import java.util.logging.Logger;
import java.util.regex.Pattern;
import org.apache.commons.io.IOUtils;
import org.archive.modules.CrawlURI;
import org.archive.modules.Processor;
import org.archive.modules.ProcessorTestBase;
import org.archive.net.UURI;
import org.archive.net.UURIFactory;
import org.archive.util.Base32;
import org.archive.util.Recorder;
import org.archive.util.TmpDirTestCase;
import org.mortbay.jetty.Handler;
import org.mortbay.jetty.Server;
import org.mortbay.jetty.bio.SocketConnector;
import org.mortbay.jetty.handler.DefaultHandler;
import org.mortbay.jetty.handler.HandlerList;
import org.mortbay.jetty.handler.ResourceHandler;
import org.mortbay.log.Log;
public abstract class FetchHTTPTestBase extends ProcessorTestBase {
@@ -27,6 +32,8 @@ public abstract class FetchHTTPTestBase extends ProcessorTestBase {
abstract protected Processor makeModule() throws IOException;
public Server startHttpFileServer(String path) throws Exception {
System.setProperty("org.mortbay.LEVEL", "DEBUG");
Log.getLogger(Server.class.getCanonicalName()).setDebugEnabled(true);
Server server = new Server();
SocketConnector sc = new SocketConnector();
sc.setHost("127.0.0.1");
@@ -34,6 +41,7 @@ public abstract class FetchHTTPTestBase extends ProcessorTestBase {
server.addConnector(sc);
ResourceHandler rhandler = new ResourceHandler();
rhandler.setResourceBase(path);
logger.info("serving files from " + path);
HandlerList handlers = new HandlerList();
handlers.setHandlers(new Handler[] {rhandler, new DefaultHandler()});
@@ -60,14 +68,44 @@ public abstract class FetchHTTPTestBase extends ProcessorTestBase {
public void testSomething() throws Exception {
ensureHttpServer();
fetcher = getFetcher();
UURI uuri = UURIFactory.getInstance("http://localhost:7777/");
UURI uuri = UURIFactory.getInstance("http://localhost:7777/test.txt");
CrawlURI curi = new CrawlURI(uuri);
Recorder.setHttpRecorder(null)
curi.setRecorder(Recorder.getHttpRecorder());
curi.setRecorder(getRecorder());
getFetcher().process(curi);
byte[] buf = IOUtils.toByteArray(getRecorder().getRecordedOutput().getReplayInputStream());
// curi.getRecorder().getRecordedOutput().getReplayInputStream().readFullyTo(buf);
String requestString = new String(buf, "US-ASCII");
assertTrue(requestString.startsWith("GET /test.txt HTTP/1.0\r\n"));
// assertTrue(requestString.matches("(?is).*User-Agent: " + Pattern.quote(getUserAgentString()) + "\r\n.*"));
assertTrue(requestString.matches("(?is).*Connection: close\r\n.*"));
assertTrue(requestString.matches("(?is).*Accept: " + Pattern.quote("text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8") + "\r\n.*"));
assertTrue(requestString.matches("(?is).*Host: localhost:7777\r\n.*"));
assertTrue(requestString.endsWith("\r\n\r\n"));
logger.info("data=" + curi.getData());
buf = IOUtils.toByteArray(curi.getRecorder().getEntityReplayInputStream());
String entityString = new String(buf, "US-ASCII");
assertTrue(entityString.equals("I am an ascii text file 39 bytes long.\n"));
assertEquals(curi.getContentLength(), 39);
assertEquals(Base32.encode(curi.getContentDigest()), "Y6G7VXZWY52LQA774YOVJ7TPZXMOMUY7");
}
protected String getUserAgentString() {
return getClass().getName();
}
protected Recorder getRecorder() throws IOException {
if (Recorder.getHttpRecorder() == null) {
Recorder httpRecorder = new Recorder(
TmpDirTestCase.tmpDir(),
"tt12345http", 16 * 1024, 512 * 1024);
Recorder.setHttpRecorder(httpRecorder);
}
return Recorder.getHttpRecorder();
}
}