lee/heritrix3
1
0
Fork 0
mirror of https://github.com/internetarchive/heritrix3.git synced 2026-07-17 04:57:11 +00:00
Code Issues Packages Projects Releases Wiki Activity

Bunch of unit tests for FetchHTTP

Browse Source 
* FetchHTTPTest.java
    a bunch of new tests
* FetchHTTP.java
    make sslTrustLevel work correctly (bug found by test; even with fix, only respects value at startup, no overrides)
* modules/pom.xml, .classpath
    dependencies for testing
* engine/pom.xml
    update jetty to 6.1.26 since we require that for the new tests
This commit is contained in:
Noah Levitt
2012-09-26 14:41:49 -07:00
parent 3612396ff3
commit b9cbbbd248
5 changed files with 1019 additions and 29 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.classpath
+11 -7
View File
@@ -8,8 +8,8 @@
<classpathentry kind="src" path="commons/src/test/java"/>
<classpathentry kind="src" path="dist/src/test/java"/>
<classpathentry kind="con" path="org.eclipse.jdt.launching.JRE_CONTAINER"/>
<classpathentry kind="con" path="org.eclipse.jdt.junit.JUNIT_CONTAINER/3"/>
<classpathentry kind="var" path="M2_REPO/org/beanshell/bsh/2.0b5/bsh-2.0b5.jar"/>
<classpathentry kind="var" path="M2_REPO/org/mortbay/jetty/jetty/6.1.14/jetty-6.1.14.jar" sourcepath="/M2_REPO/org/mortbay/jetty/jetty/6.1.14/jetty-6.1.14-sources.jar"/>
<classpathentry kind="var" path="M2_REPO/commons-collections/commons-collections/3.1/commons-collections-3.1.jar" sourcepath="M2_REPO/commons-collections/commons-collections/3.1/commons-collections-3.1-sources.jar"/>
<classpathentry kind="var" path="M2_REPO/commons-logging/commons-logging/1.0.4/commons-logging-1.0.4.jar" sourcepath="M2_REPO/commons-logging/commons-logging/1.0.4/commons-logging-1.0.4-sources.jar"/>
<classpathentry kind="var" path="M2_REPO/poi/poi/2.5.1-final-20040804/poi-2.5.1-final-20040804.jar" sourcepath="M2_REPO/poi/poi/2.5.1-final-20040804/poi-2.5.1-final-20040804-sources.jar"/>
@@ -20,12 +20,9 @@
<classpathentry kind="var" path="M2_REPO/com/lowagie/itext/1.3/itext-1.3.jar" sourcepath="M2_REPO/com/lowagie/itext/1.3/itext-1.3-sources.jar"/>
<classpathentry kind="var" path="M2_REPO/com/anotherbigidea/javaswf/CVS-SNAPSHOT-1/javaswf-CVS-SNAPSHOT-1.jar"/>
<classpathentry kind="var" path="M2_REPO/it/unimi/dsi/mg4j/1.0.1/mg4j-1.0.1.jar"/>
<classpathentry kind="var" path="M2_REPO/commons-codec/commons-codec/1.3/commons-codec-1.3.jar" sourcepath="M2_REPO/commons-codec/commons-codec/1.3/commons-codec-1.3-sources.jar"/>
<classpathentry kind="var" path="M2_REPO/fastutil/fastutil/5.0.7/fastutil-5.0.7.jar"/>
<classpathentry kind="var" path="M2_REPO/net/java/dev/jets3t/jets3t/0.5.0/jets3t-0.5.0.jar"/>
<classpathentry kind="var" path="M2_REPO/org/mortbay/jetty/jetty-util/6.1.14/jetty-util-6.1.14.jar"/>
<classpathentry kind="var" path="M2_REPO/commons-net/commons-net/2.0/commons-net-2.0.jar" sourcepath="/M2_REPO/commons-net/commons-net/2.0/commons-net-2.0-sources.jar"/>
<classpathentry kind="var" path="M2_REPO/junit/junit/3.8.2/junit-3.8.2.jar" sourcepath="M2_REPO/junit/junit/3.8.2/junit-3.8.2-sources.jar"/>
<classpathentry kind="var" path="M2_REPO/poi/poi-scratchpad/2.5.1-final-20040804/poi-scratchpad-2.5.1-final-20040804.jar"/>
<classpathentry kind="var" path="M2_REPO/com/sleepycat/je/4.1.6/je-4.1.6.jar" sourcepath="/M2_REPO/com/sleepycat/je/4.1.6/je-4.1.6-sources.jar"/>
<classpathentry kind="var" path="M2_REPO/commons-io/commons-io/1.4/commons-io-1.4.jar" sourcepath="/M2_REPO/commons-io/commons-io/1.4/commons-io-1.4-sources.jar"/>
@@ -38,9 +35,6 @@
<classpathentry kind="var" path="M2_REPO/org/springframework/spring-expression/3.0.5.RELEASE/spring-expression-3.0.5.RELEASE.jar"/>
<classpathentry kind="var" path="M2_REPO/org/springframework/spring-aop/3.0.5.RELEASE/spring-aop-3.0.5.RELEASE.jar"/>
<classpathentry kind="var" path="M2_REPO/joda-time/joda-time/1.6/joda-time-1.6.jar" sourcepath="/M2_REPO/joda-time/joda-time/1.6/joda-time-1.6-sources.jar"/>
<classpathentry kind="var" path="M2_REPO/org/mortbay/jetty/jetty-ajp/6.1.11/jetty-ajp-6.1.11.jar"/>
<classpathentry kind="var" path="M2_REPO/org/mortbay/jetty/jetty-sslengine/6.1.14/jetty-sslengine-6.1.14.jar"/>
<classpathentry kind="var" path="M2_REPO/org/mortbay/jetty/servlet-api-2.5/6.1.14/servlet-api-2.5-6.1.14.jar"/>
<classpathentry kind="var" path="M2_REPO/org/restlet/org.restlet/1.1.10/org.restlet-1.1.10.jar" sourcepath="/M2_REPO/org/restlet/org.restlet/1.1.10/org.restlet-1.1.10-sources.jar"/>
<classpathentry kind="var" path="M2_REPO/com/noelios/restlet/com.noelios.restlet/1.1.10/com.noelios.restlet-1.1.10.jar" sourcepath="/M2_REPO/com/noelios/restlet/com.noelios.restlet/1.1.10/com.noelios.restlet-1.1.10-sources.jar"/>
<classpathentry kind="var" path="M2_REPO/com/noelios/restlet/com.noelios.restlet.ext.jetty/1.1.10/com.noelios.restlet.ext.jetty-1.1.10.jar"/>
@@ -55,5 +49,15 @@
<classpathentry kind="var" path="M2_REPO/com/esotericsoftware/reflectasm/0.8/reflectasm-0.8.jar"/>
<classpathentry kind="var" path="M2_REPO/com/google/guava/guava/r08/guava-r08.jar" sourcepath="/M2_REPO/com/google/guava/guava/r08/guava-r08-sources.jar"/>
<classpathentry kind="var" path="M2_REPO/net/java/dev/jna/jna/3.2.3/jna-3.2.3.jar" sourcepath="/M2_REPO/net/java/dev/jna/jna/3.2.3/jna-3.2.3-sources.jar"/>
<classpathentry kind="var" path="M2_REPO/org/littleshoot/littleproxy/0.4/littleproxy-0.4.jar" sourcepath="/M2_REPO/org/littleshoot/littleproxy/0.4/littleproxy-0.4-sources.jar"/>
<classpathentry kind="var" path="M2_REPO/org/jboss/netty/netty/3.2.4.Final/netty-3.2.4.Final.jar" sourcepath="M2_REPO/org/jboss/netty/netty/3.2.4.Final/netty-3.2.4.Final-sources.jar"/>
<classpathentry kind="var" path="M2_REPO/org/slf4j/slf4j-api/1.6.6/slf4j-api-1.6.6.jar" sourcepath="/M2_REPO/org/slf4j/slf4j-api/1.6.6/slf4j-api-1.6.6-sources.jar"/>
<classpathentry kind="var" path="M2_REPO/org/slf4j/slf4j-jdk14/1.6.6/slf4j-jdk14-1.6.6.jar" sourcepath="/M2_REPO/org/slf4j/slf4j-jdk14/1.6.6/slf4j-jdk14-1.6.6-sources.jar"/>
<classpathentry kind="var" path="M2_REPO/commons-codec/commons-codec/1.6/commons-codec-1.6.jar" sourcepath="/M2_REPO/commons-codec/commons-codec/1.6/commons-codec-1.6-sources.jar"/>
<classpathentry kind="var" path="M2_REPO/org/mortbay/jetty/jetty-util/6.1.26/jetty-util-6.1.26.jar" sourcepath="/M2_REPO/org/mortbay/jetty/jetty-util/6.1.26/jetty-util-6.1.26-sources.jar"/>
<classpathentry kind="var" path="M2_REPO/org/mortbay/jetty/jetty-ajp/6.1.26/jetty-ajp-6.1.26.jar"/>
<classpathentry kind="var" path="M2_REPO/org/mortbay/jetty/jetty-sslengine/6.1.26/jetty-sslengine-6.1.26.jar"/>
<classpathentry kind="var" path="M2_REPO/org/mortbay/jetty/servlet-api/2.5-20081211/servlet-api-2.5-20081211.jar"/>
<classpathentry kind="var" path="M2_REPO/org/mortbay/jetty/jetty/6.1.26/jetty-6.1.26.jar" sourcepath="/M2_REPO/org/mortbay/jetty/jetty/6.1.26/jetty-6.1.26-sources.jar"/>
<classpathentry kind="output" path="bin"/>
</classpath>
engine/pom.xml
+4 -4
View File
@@ -46,25 +46,25 @@
<dependency>
<groupId>org.mortbay.jetty</groupId>
<artifactId>jetty</artifactId>
<version>6.1.14</version>
<version>6.1.26</version>
<scope>compile</scope>
</dependency>
<dependency>
<groupId>org.mortbay.jetty</groupId>
<artifactId>jetty-util</artifactId>
<version>6.1.14</version>
<version>6.1.26</version>
<scope>compile</scope>
</dependency>
<dependency>
<groupId>org.mortbay.jetty</groupId>
<artifactId>jetty-sslengine</artifactId>
<version>6.1.14</version>
<version>6.1.26</version>
<scope>compile</scope>
</dependency>
<dependency>
<groupId>org.mortbay.jetty</groupId>
<artifactId>servlet-api-2.5</artifactId>
<version>6.1.14</version>
<version>6.1.26</version>
<scope>compile</scope>
</dependency>
<dependency>
modules/pom.xml
+33
View File
@@ -55,6 +55,39 @@
<artifactId>groovy-all</artifactId>
<version>1.6.3</version>
</dependency>
<!-- testing -->
<dependency>
<groupId>org.mortbay.jetty</groupId>
<artifactId>jetty</artifactId>
<version>6.1.26</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.mortbay.jetty</groupId>
<artifactId>jetty-sslengine</artifactId>
<version>6.1.26</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.littleshoot</groupId>
<artifactId>littleproxy</artifactId>
<version>0.4</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.slf4j</groupId>
<artifactId>slf4j-api</artifactId>
<version>1.6.6</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.slf4j</groupId>
<artifactId>slf4j-jdk14</artifactId>
<version>1.6.6</version>
<scope>test</scope>
</dependency>
</dependencies>
<build>
<plugins>
modules/src/main/java/org/archive/modules/fetcher/FetchHTTP.java
+2 -3
View File
@@ -316,7 +316,7 @@ public class FetchHTTP extends Processor implements Lifecycle {
setSslTrustLevel(TrustLevel.OPEN);
}
public TrustLevel getSslTrustLevel() {
return (TrustLevel) kp.get("trustLevel");
return (TrustLevel) kp.get("sslTrustLevel");
}
public void setSslTrustLevel(TrustLevel trustLevel) {
kp.put("sslTrustLevel",trustLevel);
@@ -1359,14 +1359,13 @@ public class FetchHTTP extends Processor implements Lifecycle {
}
super.start();
setSSLFactory();
configureHttp();
if (cookieStorage != null) {
cookieStorage.start();
http.getState().setCookiesMap(cookieStorage.getCookiesMap());
}
setSSLFactory();
}
public boolean isRunning() {
modules/src/test/java/org/archive/modules/fetcher/FetchHTTPTest.java
+969 -15
View File
@@ -18,28 +18,982 @@
*/
package org.archive.modules.fetcher;
import java.io.File;
import java.io.IOException;
import java.io.UnsupportedEncodingException;
import java.net.Inet4Address;
import java.net.ServerSocket;
import java.net.Socket;
import java.net.SocketTimeoutException;
import java.nio.charset.Charset;
import java.util.Arrays;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
import org.archive.modules.Processor;
import javax.net.ssl.SSLHandshakeException;
import javax.servlet.ServletException;
import javax.servlet.http.HttpServletRequest;
import javax.servlet.http.HttpServletResponse;
import org.apache.commons.httpclient.Header;
import org.apache.commons.httpclient.NoHttpResponseException;
import org.apache.commons.httpclient.URIException;
import org.apache.commons.io.IOUtils;
import org.archive.httpclient.ConfigurableX509TrustManager.TrustLevel;
import org.archive.modules.CrawlMetadata;
import org.archive.modules.CrawlURI;
import org.archive.modules.CrawlURI.FetchType;
import org.archive.modules.ProcessorTestBase;
import org.archive.modules.fetcher.FetchHTTP;
import org.archive.modules.credential.HttpAuthenticationCredential;
import org.archive.modules.deciderules.RejectDecideRule;
import org.archive.modules.recrawl.FetchHistoryProcessor;
import org.archive.net.UURI;
import org.archive.net.UURIFactory;
import org.archive.util.Recorder;
import org.archive.util.TmpDirTestCase;
import org.jboss.netty.handler.codec.http.HttpRequest;
import org.littleshoot.proxy.DefaultHttpProxyServer;
import org.littleshoot.proxy.HttpFilter;
import org.littleshoot.proxy.HttpRequestFilter;
import org.littleshoot.proxy.ProxyAuthorizationHandler;
import org.mortbay.jetty.NCSARequestLog;
import org.mortbay.jetty.Request;
import org.mortbay.jetty.Response;
import org.mortbay.jetty.Server;
import org.mortbay.jetty.bio.SocketConnector;
import org.mortbay.jetty.handler.HandlerCollection;
import org.mortbay.jetty.handler.RequestLogHandler;
import org.mortbay.jetty.security.Authenticator;
import org.mortbay.jetty.security.BasicAuthenticator;
import org.mortbay.jetty.security.Constraint;
import org.mortbay.jetty.security.ConstraintMapping;
import org.mortbay.jetty.security.DigestAuthenticator;
import org.mortbay.jetty.security.FormAuthenticator;
import org.mortbay.jetty.security.HashUserRealm;
import org.mortbay.jetty.security.SecurityHandler;
import org.mortbay.jetty.security.SslSocketConnector;
import org.mortbay.jetty.servlet.HashSessionManager;
import org.mortbay.jetty.servlet.SessionHandler;
import org.mortbay.log.Log;
import sun.security.tools.KeyTool;
/**
*
*/
public class FetchHTTPTest extends ProcessorTestBase {
@Override
protected Processor makeModule() {
FetchHTTP result = new FetchHTTP();
// FIXME: Set up server cache, credential store...
// Use SimpleCookieStorage for FetchHTTP test, even though BDB is
// actual default
result.setCookieStorage(new SimpleCookieStorage());
result.start();
return result;
// private static Logger logger = Logger.getLogger(FetchHTTPTest.class.getName());
/* uncomment this for detailed logging from jetty and heritrix */
// static {
// Logger.getLogger("").setLevel(Level.FINE);
// for (java.util.logging.Handler h: Logger.getLogger("").getHandlers()) {
// h.setLevel(Level.ALL);
// h.setFormatter(new OneLineSimpleLogger());
// }
// }
protected static final String BASIC_AUTH_REALM = "basic-auth-realm";
protected static final String BASIC_AUTH_ROLE = "basic-auth-role";
protected static final String BASIC_AUTH_LOGIN = "basic-auth-login";
protected static final String BASIC_AUTH_PASSWORD = "basic-auth-password";
protected static final String DIGEST_AUTH_REALM = "digest-auth-realm";
protected static final String DIGEST_AUTH_ROLE = "digest-auth-role";
protected static final String DIGEST_AUTH_LOGIN = "digest-auth-login";
protected static final String DIGEST_AUTH_PASSWORD = "digest-auth-password";
protected static final String FORM_AUTH_REALM = "form-auth-realm";
protected static final String FORM_AUTH_ROLE = "form-auth-role";
protected static final String FORM_AUTH_LOGIN = "form-auth-login";
protected static final String FORM_AUTH_PASSWORD = "form-auth-password";
protected static final String ETAG_TEST_VALUE = "An ETag is an opaque identifier assigned by a web server to a specific version of a resource found at a URL!";
protected static final String DEFAULT_PAYLOAD_STRING = "abcdefghijklmnopqrstuvwxyz0123456789\n";
protected static final byte[] DEFAULT_GZIPPED_PAYLOAD = { 31, -117, 8, 0,
-69, 25, 60, 80, 0, 3, 75, 76, 74, 78, 73, 77, 75, -49, -56, -52,
-54, -50, -55, -51, -53, 47, 40, 44, 42, 46, 41, 45, 43, -81, -88,
-84, 50, 48, 52, 50, 54, 49, 53, 51, -73, -80, -28, 2, 0, -43, 104,
-33, -11, 37, 0, 0, 0 };
protected static final byte[] EIGHTY_BYTE_LINE = "1234567890123456789012345678901234567890123456789012345678901234567890123456789\n".getBytes();
protected static final String LOGIN_HTML =
"<html>" +
"<head><title>Log In</title></head>" +
"<body>" +
"<form action='/j_security_check' method='post'>" +
"<div> username: <input name='j_username' type='text'/> </div>" +
"<div> password: <input name='j_password' type='password'/> </div>" +
"<div> <input type='submit' /> </div>" +
"</form>" +
"</body>" +
"</html>";
protected static class TestHandler extends SessionHandler {
public TestHandler() {
super();
}
@Override
public void handle(String target, HttpServletRequest request,
HttpServletResponse response, int dispatch) throws IOException,
ServletException {
if (target.endsWith("/set-cookie")) {
response.addCookie(new javax.servlet.http.Cookie("test-cookie-name", "test-cookie-value"));
}
if (target.equals("/login.html")) {
response.setContentType("text/html;charset=US-ASCII");
response.setStatus(HttpServletResponse.SC_OK);
response.getOutputStream().write(LOGIN_HTML.getBytes("US-ASCII"));
((Request)request).setHandled(true);
} else if (target.equals("/200k")) {
response.setContentType("text/plain;charset=US-ASCII");
response.setStatus(HttpServletResponse.SC_OK);
assertTrue(EIGHTY_BYTE_LINE.length == 80);
for (int i = 0; i < 200000 / EIGHTY_BYTE_LINE.length; i++) {
response.getOutputStream().write(EIGHTY_BYTE_LINE);
}
((Request)request).setHandled(true);
} else if (target.equals("/slow.txt")) {
response.setContentType("text/plain;charset=US-ASCII");
response.setStatus(HttpServletResponse.SC_OK);
for (int i = 0; i < 60; i++) {
response.getOutputStream().write(EIGHTY_BYTE_LINE);
response.getOutputStream().flush();
try {
Thread.sleep(500);
} catch (InterruptedException e) {
}
}
((Request)request).setHandled(true);
} else if (target.equals("/chunked.txt")) {
response.setContentType("text/plain;charset=US-ASCII");
response.setStatus(HttpServletResponse.SC_OK);
// response.setContentLength(HttpTokens.CHUNKED_CONTENT);
response.getOutputStream().write(DEFAULT_PAYLOAD_STRING.getBytes("US-ASCII"));
response.getOutputStream().flush();
((Request)request).setHandled(true);
} else if (request.getHeader("Accept-Encoding") != null
&& request.getHeader("Accept-Encoding").contains("gzip")) {
response.setHeader("Content-Encoding", "gzip");
response.setContentType("text/plain;charset=US-ASCII");
response.setStatus(HttpServletResponse.SC_OK);
response.getOutputStream().write(DEFAULT_GZIPPED_PAYLOAD);
((Request)request).setHandled(true);
} else {
response.setContentType("text/plain;charset=US-ASCII");
response.setDateHeader("Last-Modified", 0);
response.setHeader("ETag", ETAG_TEST_VALUE);
response.setStatus(HttpServletResponse.SC_OK);
response.getOutputStream().write(DEFAULT_PAYLOAD_STRING.getBytes("US-ASCII"));
((Request)request).setHandled(true);
}
}
}
protected static Map<Integer, Server> httpServers;
protected static Request lastRequest = null;
protected static Response lastResponse = null;
protected FetchHTTP fetcher;
protected static SecurityHandler makeAuthWrapper(Authenticator authenticator,
final String role, String realm, final String login,
final String password) {
Constraint constraint = new Constraint();
constraint.setRoles(new String[] { role });
constraint.setAuthenticate(true);
ConstraintMapping constraintMapping = new ConstraintMapping();
constraintMapping.setConstraint(constraint);
constraintMapping.setPathSpec("/auth/*");
SecurityHandler authWrapper = new SecurityHandler();
authWrapper.setAuthenticator(authenticator);
authWrapper.setConstraintMappings(new ConstraintMapping[] {constraintMapping});
authWrapper.setUserRealm(new HashUserRealm(realm) {
{
put(login, password);
addUserToRole(login, role);
}
});
return authWrapper;
}
// TODO TESTME!
// can't easily have the same Server do different types of auth for
// different paths, so we have multiple servers
/**
* @return map(port->server)
*/
public static Map<Integer,Server> startHttpServers() throws Exception {
Log.getLogger(Server.class.getCanonicalName()).setDebugEnabled(true);
HashMap<Integer, Server> servers = new HashMap<Integer,Server>();
HandlerCollection handlers = new HandlerCollection();
handlers.addHandler(new TestHandler());
RequestLogHandler requestLogHandler = new RequestLogHandler();
NCSARequestLog requestLog = new NCSARequestLog() {
@Override
public void log(Request request, Response response) {
super.log(request, response);
lastRequest = request;
lastResponse = response;
}
};
requestLogHandler.setRequestLog(requestLog);
handlers.addHandler(requestLogHandler);
// server for basic auth
Server server = new Server();
SocketConnector sc = new SocketConnector();
sc.setHost("127.0.0.1");
sc.setPort(7777);
server.addConnector(sc);
SecurityHandler authWrapper = makeAuthWrapper(new BasicAuthenticator(),
BASIC_AUTH_ROLE, BASIC_AUTH_REALM, BASIC_AUTH_LOGIN,
BASIC_AUTH_PASSWORD);
authWrapper.setHandler(handlers);
server.setHandler(authWrapper);
servers.put(sc.getPort(), server);
File keystoreFile = new File(TmpDirTestCase.tmpDir(), "keystore");
if (keystoreFile.exists()) {
keystoreFile.delete();
}
final String KEYSTORE_PASSWORD = "keystore-password";
KeyTool.main(new String[] {
"-keystore", keystoreFile.getPath(),
"-storepass", KEYSTORE_PASSWORD,
"-keypass", KEYSTORE_PASSWORD,
"-alias", "jetty",
"-genkey",
"-keyalg", "RSA",
"-dname", "CN=127.0.0.1",
"-validity","3650"}); // 10 yr validity
SslSocketConnector ssc = new SslSocketConnector();
ssc.setHost("127.0.0.1");
ssc.setPort(7443);
ssc.setKeyPassword(KEYSTORE_PASSWORD);
ssc.setKeystore(keystoreFile.getPath());
server.addConnector(ssc);
server.start();
// server for digest auth
server = new Server();
sc = new SocketConnector();
sc.setHost("127.0.0.1");
sc.setPort(7778);
server.addConnector(sc);
authWrapper = makeAuthWrapper(new DigestAuthenticator(),
DIGEST_AUTH_ROLE, DIGEST_AUTH_REALM, DIGEST_AUTH_LOGIN,
DIGEST_AUTH_PASSWORD);
authWrapper.setHandler(handlers);
server.setHandler(authWrapper);
server.start();
servers.put(sc.getPort(), server);
// server for form auth
server = new Server();
sc = new SocketConnector();
sc.setHost("127.0.0.1");
sc.setPort(7779);
server.addConnector(sc);
FormAuthenticator formAuthenticatrix = new FormAuthenticator();
formAuthenticatrix.setLoginPage("/login.html");
authWrapper = makeAuthWrapper(formAuthenticatrix,
FORM_AUTH_ROLE, FORM_AUTH_REALM, FORM_AUTH_LOGIN,
FORM_AUTH_PASSWORD);
authWrapper.setHandler(handlers);
SessionHandler sessionHandler = new SessionHandler();
sessionHandler.setSessionManager(new HashSessionManager());
sessionHandler.setHandler(authWrapper);
server.setHandler(sessionHandler);
server.start();
servers.put(sc.getPort(), server);
return servers;
}
protected static void ensureHttpServers() throws Exception {
if (httpServers == null) {
httpServers = startHttpServers();
}
}
protected FetchHTTP fetcher() throws IOException {
if (fetcher == null) {
fetcher = makeModule();
}
return fetcher;
}
protected String getUserAgentString() {
return getClass().getName();
}
protected Recorder getRecorder() throws IOException {
if (Recorder.getHttpRecorder() == null) {
Recorder httpRecorder = new Recorder(TmpDirTestCase.tmpDir(),
getClass().getName(), 16 * 1024, 512 * 1024);
Recorder.setHttpRecorder(httpRecorder);
}
return Recorder.getHttpRecorder();
}
protected CrawlURI makeCrawlURI(String uri) throws URIException,
IOException {
UURI uuri = UURIFactory.getInstance(uri);
CrawlURI curi = new CrawlURI(uuri);
curi.setSeed(true);
curi.setRecorder(getRecorder());
return curi;
}
protected void runDefaultChecks(CrawlURI curi, String... exclusionsArray)
throws IOException, UnsupportedEncodingException {
Set<String> exclusions = new HashSet<String>(Arrays.asList(exclusionsArray));
String requestString = httpRequestString(curi);
if (!exclusions.contains("requestLine")) {
assertTrue(requestString.startsWith("GET / HTTP/1.0\r\n"));
}
assertTrue(requestString.contains("User-Agent: " + getUserAgentString() + "\r\n"));
assertTrue(requestString.matches("(?s).*Connection: [Cc]lose\r\n.*"));
if (!exclusions.contains("acceptHeaders")) {
assertTrue(requestString.contains("Accept: text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8\r\n"));
}
if (!exclusions.contains("hostHeader")) {
assertTrue(requestString.contains("Host: localhost:7777\r\n"));
}
assertTrue(requestString.endsWith("\r\n\r\n"));
// check sizes
assertEquals(DEFAULT_PAYLOAD_STRING.length(), curi.getContentLength());
assertEquals(curi.getContentSize(), curi.getRecordedSize());
// check various
assertEquals("sha1:TQ5R6YVOZLTQENRIIENVGXHOPX3YCRNJ", curi.getContentDigestSchemeString());
assertEquals("text/plain;charset=US-ASCII", curi.getContentType());
assertEquals(Charset.forName("US-ASCII"), curi.getRecorder().getCharset());
assertTrue(curi.getCredentials().isEmpty());
assertTrue(curi.getFetchDuration() >= 0);
assertTrue(curi.getFetchStatus() == 200);
assertTrue(curi.getFetchType() == FetchType.HTTP_GET);
// check message body, i.e. "raw, possibly chunked-transfer-encoded message contents not including the leading headers"
assertEquals(DEFAULT_PAYLOAD_STRING, messageBodyString(curi));
// check entity, i.e. "message-body after any (usually-unnecessary) transfer-decoding but before any content-encoding (eg gzip) decoding"
assertEquals(DEFAULT_PAYLOAD_STRING, entityString(curi));
// check content, i.e. message-body after possibly tranfer-decoding and after content-encoding (eg gzip) decoding
assertEquals(DEFAULT_PAYLOAD_STRING, contentString(curi));
assertEquals(DEFAULT_PAYLOAD_STRING.substring(0, 10), curi.getRecorder().getContentReplayPrefixString(10));
assertEquals(DEFAULT_PAYLOAD_STRING, curi.getRecorder().getContentReplayCharSequence().toString());
if (!exclusions.contains("httpBindAddress")) {
assertEquals("127.0.0.1", lastRequest.getRemoteAddr());
}
assertTrue(curi.getNonFatalFailures().isEmpty());
}
// convenience methods to get strings from raw recorded i/o
protected String rawResponseString(CrawlURI curi) throws IOException, UnsupportedEncodingException {
byte[] buf = IOUtils.toByteArray(curi.getRecorder().getReplayInputStream());
return new String(buf, "US-ASCII");
}
protected String contentString(CrawlURI curi) throws IOException, UnsupportedEncodingException {
byte[] buf = IOUtils.toByteArray(curi.getRecorder().getContentReplayInputStream());
return new String(buf, "US-ASCII");
}
protected String entityString(CrawlURI curi) throws IOException, UnsupportedEncodingException {
byte[] buf = IOUtils.toByteArray(curi.getRecorder().getEntityReplayInputStream());
return new String(buf, "US-ASCII");
}
protected String messageBodyString(CrawlURI curi) throws IOException, UnsupportedEncodingException {
byte[] buf = IOUtils.toByteArray(curi.getRecorder().getMessageBodyReplayInputStream());
return new String(buf, "US-ASCII");
}
protected String httpRequestString(CrawlURI curi) throws IOException, UnsupportedEncodingException {
byte[] buf = IOUtils.toByteArray(curi.getRecorder().getRecordedOutput().getReplayInputStream());
return new String(buf, "US-ASCII");
}
public void testDefaults() throws Exception {
ensureHttpServers();
CrawlURI curi = makeCrawlURI("http://localhost:7777/");
fetcher().process(curi);
runDefaultChecks(curi);
}
public void testAcceptHeaders() throws Exception {
ensureHttpServers();
List<String> headers = Arrays.asList("header1: value1", "header2: value2");
fetcher().setAcceptHeaders(headers);
CrawlURI curi = makeCrawlURI("http://localhost:7777/");
fetcher().process(curi);
runDefaultChecks(curi, "acceptHeaders");
// special checks for this test
String requestString = httpRequestString(curi);
assertFalse(requestString.contains("Accept:"));
for (String h: headers) {
assertTrue(requestString.contains(h));
}
}
public void testCookies() throws Exception {
ensureHttpServers();
checkSetCookieURI();
// second request to see if cookie is sent
CrawlURI curi = makeCrawlURI("http://localhost:7777/");
fetcher().process(curi);
runDefaultChecks(curi);
String requestString = httpRequestString(curi);
assertTrue(requestString.contains("Cookie: test-cookie-name=test-cookie-value\r\n"));
}
public void testIgnoreCookies() throws Exception {
ensureHttpServers();
fetcher().setIgnoreCookies(true);
checkSetCookieURI();
// second request to see if cookie is NOT sent
CrawlURI curi = makeCrawlURI("http://localhost:7777/");
fetcher().process(curi);
runDefaultChecks(curi);
String requestString = httpRequestString(curi);
assertFalse(requestString.contains("Cookie:"));
}
public void testBasicAuth() throws Exception {
ensureHttpServers();
HttpAuthenticationCredential basicAuthCredential = new HttpAuthenticationCredential();
basicAuthCredential.setRealm(BASIC_AUTH_REALM);
basicAuthCredential.setDomain("localhost:7777");
basicAuthCredential.setLogin(BASIC_AUTH_LOGIN);
basicAuthCredential.setPassword(BASIC_AUTH_PASSWORD);
fetcher().getCredentialStore().getCredentials().put("basic-auth-credential",
basicAuthCredential);
CrawlURI curi = makeCrawlURI("http://localhost:7777/auth/1");
fetcher().process(curi);
// check that we got the expected response and the fetcher did its thing
assertEquals(401, curi.getFetchStatus());
assertTrue(curi.getCredentials().contains(basicAuthCredential));
assertTrue(curi.getHttpAuthChallenges() != null && curi.getHttpAuthChallenges().containsKey("basic"));
// fetch again with the credentials
fetcher().process(curi);
String httpRequestString = httpRequestString(curi);
// logger.info('\n' + httpRequestString + contentString(curi));
assertTrue(httpRequestString.contains("Authorization: Basic YmFzaWMtYXV0aC1sb2dpbjpiYXNpYy1hdXRoLXBhc3N3b3Jk\r\n"));
// otherwise should be a normal 200 response
runDefaultChecks(curi, "requestLine");
// fetch a fresh uri to make sure auth info was cached and we don't get another 401
curi = makeCrawlURI("http://localhost:7777/auth/2");
fetcher().process(curi);
httpRequestString = httpRequestString(curi);
assertTrue(httpRequestString.contains("Authorization: Basic YmFzaWMtYXV0aC1sb2dpbjpiYXNpYy1hdXRoLXBhc3N3b3Jk\r\n"));
// otherwise should be a normal 200 response
runDefaultChecks(curi, "requestLine");
}
// server for digest auth is at localhost:7778
public void testDigestAuth() throws Exception {
ensureHttpServers();
HttpAuthenticationCredential digestAuthCred = new HttpAuthenticationCredential();
digestAuthCred.setRealm(DIGEST_AUTH_REALM);
digestAuthCred.setDomain("localhost:7778");
digestAuthCred.setLogin(DIGEST_AUTH_LOGIN);
digestAuthCred.setPassword(DIGEST_AUTH_PASSWORD);
fetcher().getCredentialStore().getCredentials().put("digest-auth-credential",
digestAuthCred);
CrawlURI curi = makeCrawlURI("http://localhost:7778/auth/1");
fetcher().process(curi);
// check that we got the expected response and the fetcher did its thing
assertEquals(401, curi.getFetchStatus());
assertTrue(curi.getCredentials().contains(digestAuthCred));
assertTrue(curi.getHttpAuthChallenges() != null && curi.getHttpAuthChallenges().containsKey("digest"));
// stick a basic auth 401 in there to check it doesn't mess with the digest auth we're working on
CrawlURI interferingUri = makeCrawlURI("http://localhost:7777/auth/basic");
fetcher().process(interferingUri);
assertEquals(401, interferingUri.getFetchStatus());
// logger.info('\n' + httpRequestString(interferingUri) + "\n\n" + rawResponseString(interferingUri));
// fetch original again with the credentials
fetcher().process(curi);
String httpRequestString = httpRequestString(curi);
// logger.info('\n' + httpRequestString + "\n\n" + rawResponseString(interferingUri));
assertTrue(httpRequestString.contains("Authorization: Digest"));
// otherwise should be a normal 200 response
runDefaultChecks(curi, "requestLine", "hostHeader");
// fetch a fresh uri to make sure auth info was cached and we don't get another 401
curi = makeCrawlURI("http://localhost:7778/auth/2");
fetcher().process(curi);
httpRequestString = httpRequestString(curi);
assertTrue(httpRequestString.contains("Authorization: Digest"));
// otherwise should be a normal 200 response
runDefaultChecks(curi, "requestLine", "hostHeader");
}
protected void checkSetCookieURI() throws URIException, IOException,
InterruptedException, UnsupportedEncodingException {
CrawlURI curi = makeCrawlURI("http://localhost:7777/set-cookie");
fetcher().process(curi);
runDefaultChecks(curi, "requestLine");
// check for set-cookie header
byte[] buf = IOUtils.toByteArray(curi.getRecorder().getReplayInputStream());
String rawResponseString = new String(buf, "US-ASCII");
assertTrue(rawResponseString.contains("Set-Cookie: test-cookie-name=test-cookie-value\r\n"));
}
public void testAcceptCompression() throws Exception {
ensureHttpServers();
CrawlURI curi = makeCrawlURI("http://localhost:7777/");
fetcher().setAcceptCompression(true);
fetcher().process(curi);
String httpRequestString = httpRequestString(curi);
// logger.info('\n' + httpRequestString + "\n\n" + rawResponseString(curi));
// logger.info("\n----- begin contentString -----\n" + contentString(curi));
// logger.info("\n----- begin entityString -----\n" + entityString(curi));
// logger.info("\n----- begin messageBodyString -----\n" + messageBodyString(curi));
assertTrue(httpRequestString.contains("Accept-Encoding: gzip,deflate\r\n"));
assertEquals(DEFAULT_GZIPPED_PAYLOAD.length, curi.getContentLength());
assertEquals(curi.getContentSize(), curi.getRecordedSize());
// check various
assertEquals("text/plain;charset=US-ASCII", curi.getContentType());
assertEquals(Charset.forName("US-ASCII"), curi.getRecorder().getCharset());
assertTrue(curi.getCredentials().isEmpty());
assertTrue(curi.getFetchDuration() >= 0);
assertTrue(curi.getFetchStatus() == 200);
assertTrue(curi.getFetchType() == FetchType.HTTP_GET);
// check message body, i.e. "raw, possibly chunked-transfer-encoded message contents not including the leading headers"
assertTrue(Arrays.equals(DEFAULT_GZIPPED_PAYLOAD, IOUtils.toByteArray(curi.getRecorder().getMessageBodyReplayInputStream())));
// check entity, i.e. "message-body after any (usually-unnecessary) transfer-decoding but before any content-encoding (eg gzip) decoding"
assertTrue(Arrays.equals(DEFAULT_GZIPPED_PAYLOAD, IOUtils.toByteArray(curi.getRecorder().getEntityReplayInputStream())));
// check content, i.e. message-body after possibly tranfer-decoding and after content-encoding (eg gzip) decoding
assertEquals(DEFAULT_PAYLOAD_STRING, contentString(curi));
assertEquals("sha1:6HXUWMO6VPBHU4SIPOVJ3OPMCSN6JJW4", curi.getContentDigestSchemeString());
}
// Binding to 127.0.0.2 only works on some systems. Fails on my mac. We
// could skip the test in that case, but better to leave it in I think, so
// we notice if it starts failing on the build box and we can do something
// about it.
public void testHttpBindAddress() throws Exception {
ensureHttpServers();
CrawlURI curi = makeCrawlURI("http://localhost:7777/");
fetcher().setHttpBindAddress("127.0.0.2");
fetcher().process(curi);
// the client bind address isn't recorded anywhere in heritrix as
// far as i can tell, so we get it this way...
assertEquals("127.0.0.2", lastRequest.getRemoteAddr());
runDefaultChecks(curi, "httpBindAddress");
}
protected static class ProxiedRequestRememberer implements HttpRequestFilter {
protected HttpRequest lastProxiedRequest = null;
public HttpRequest getLastProxiedRequest() {
return lastProxiedRequest;
}
@Override
public void filter(HttpRequest httpRequest) {
lastProxiedRequest = httpRequest;
}
public void clear() {
lastProxiedRequest = null;
}
}
public void testHttpProxy() throws Exception {
ensureHttpServers();
ProxiedRequestRememberer proxiedRequestRememberer = new ProxiedRequestRememberer();
DefaultHttpProxyServer httpProxyServer = new DefaultHttpProxyServer(7877, proxiedRequestRememberer, new HashMap<String, HttpFilter>());
httpProxyServer.start(true, false);
try {
fetcher().setHttpProxyHost("localhost");
fetcher().setHttpProxyPort(7877);
CrawlURI curi = makeCrawlURI("http://localhost:7777/");
fetcher().process(curi);
// logger.info('\n' + httpRequestString(curi) + "\n\n" + rawResponseString(curi));
String requestString = httpRequestString(curi);
assertTrue(requestString.startsWith("GET http://localhost:7777/ HTTP/1.0\r\n"));
assertNotNull(getHttpResponseHeader(curi, "Via"));
assertTrue(requestString.contains("Proxy-Connection: close\r\n"));
// check that our little proxy server really handled a request
assertNotNull(proxiedRequestRememberer.getLastProxiedRequest());
runDefaultChecks(curi, "requestLine");
} finally {
httpProxyServer.stop();
}
}
// XXX test disabled because proxy auth doesn't work correctly
public void xestHttpProxyAuth() throws Exception {
ensureHttpServers();
ProxiedRequestRememberer proxiedRequestRememberer = new ProxiedRequestRememberer();
DefaultHttpProxyServer httpProxyServer = new DefaultHttpProxyServer(7877, proxiedRequestRememberer, new HashMap<String, HttpFilter>());
httpProxyServer.addProxyAuthenticationHandler(new ProxyAuthorizationHandler() {
@Override
public boolean authenticate(String userName, String password) {
// logger.info("username=" + userName + " password=" + password);
return "http-proxy-user".equals(userName) && "http-proxy-password".equals(password);
}
});
httpProxyServer.start(true, false);
try {
fetcher().setHttpProxyHost("localhost");
fetcher().setHttpProxyPort(7877);
fetcher().setHttpProxyUser("http-proxy-user");
fetcher().setHttpProxyPassword("http-proxy-password");
fetcher().setUseHTTP11(true); // proxy auth is a http 1.1 feature
CrawlURI curi = makeCrawlURI("http://localhost:7777/");
fetcher().process(curi);
// logger.info('\n' + httpRequestString(curi) + "\n\n" + rawResponseString(curi));
String requestString = httpRequestString(curi);
assertTrue(requestString.startsWith("GET http://localhost:7777/ HTTP/1.1\r\n"));
assertTrue(requestString.contains("Proxy-Connection: close\r\n"));
assertNull(proxiedRequestRememberer.getLastProxiedRequest()); // request didn't make it this far
assertEquals(407, curi.getFetchStatus());
// fetch original again now that credentials should be populated
proxiedRequestRememberer.clear();
curi = makeCrawlURI("http://localhost:7777/");
fetcher().process(curi);
// logger.info('\n' + httpRequestString(curi) + "\n\n" + rawResponseString(curi));
requestString = httpRequestString(curi);
assertTrue(requestString.startsWith("GET http://localhost:7777/ HTTP/1.0\r\n"));
assertTrue(requestString.contains("Proxy-Connection: close\r\n"));
assertNotNull(getHttpResponseHeader(curi, "Via"));
assertNotNull(proxiedRequestRememberer.getLastProxiedRequest());
runDefaultChecks(curi, "requestLine");
} finally {
httpProxyServer.stop();
}
}
public void testMaxFetchKBSec() throws Exception {
ensureHttpServers();
CrawlURI curi = makeCrawlURI("http://localhost:7777/200k");
fetcher().setMaxFetchKBSec(100);
fetcher().process(curi);
assertEquals(200000, curi.getContentLength());
assertTrue(curi.getFetchDuration() > 1800 && curi.getFetchDuration() < 2200);
}
public void testMaxLengthBytes() throws Exception {
ensureHttpServers();
CrawlURI curi = makeCrawlURI("http://localhost:7777/200k");
fetcher().setMaxLengthBytes(50000);
fetcher().process(curi);
assertEquals(50001, curi.getRecordedSize());
}
public void testSendRange() throws Exception {
ensureHttpServers();
CrawlURI curi = makeCrawlURI("http://localhost:7777/200k");
fetcher().setMaxLengthBytes(50000);
fetcher().setSendRange(true);
fetcher().process(curi);
// logger.info("\n" + httpRequestString(curi));
assertTrue(httpRequestString(curi).contains("Range: bytes=0-49999\r\n"));
// XXX make server honor range and inspect response?
// assertEquals(50000, curi.getRecordedSize());
}
public void testSendIfModifiedSince() throws Exception {
ensureHttpServers();
fetcher().setSendIfModifiedSince(true);
CrawlURI curi = makeCrawlURI("http://localhost:7777/");
fetcher().process(curi);
assertFalse(httpRequestString(curi).toLowerCase().contains("if-modified-since"));
assertEquals("Thu, 01 Jan 1970 00:00:00 GMT", getHttpResponseHeader(curi, "last-modified"));
runDefaultChecks(curi);
// logger.info("before FetchHistoryProcessor fetchHistory=" + Arrays.toString(curi.getFetchHistory()));
FetchHistoryProcessor fetchHistoryProcessor = new FetchHistoryProcessor();
fetchHistoryProcessor.process(curi);
// logger.info("after FetchHistoryProcessor fetchHistory=" + Arrays.toString(curi.getFetchHistory()));
fetcher().process(curi);
// logger.info("\n" + httpRequestString(curi));
assertTrue(httpRequestString(curi).contains("If-Modified-Since: Thu, 01 Jan 1970 00:00:00 GMT\r\n"));
runDefaultChecks(curi);
// XXX make server send 304 not-modified and check for it here?
}
public void testSendIfNoneMatch() throws Exception {
ensureHttpServers();
fetcher().setSendIfNoneMatch(true);
CrawlURI curi = makeCrawlURI("http://localhost:7777/");
fetcher().process(curi);
assertFalse(httpRequestString(curi).toLowerCase().contains("if-none-match"));
assertTrue(getHttpResponseHeader(curi, "etag").equals(ETAG_TEST_VALUE));
runDefaultChecks(curi);
FetchHistoryProcessor fetchHistoryProcessor = new FetchHistoryProcessor();
fetchHistoryProcessor.process(curi);
fetcher().process(curi);
assertTrue(httpRequestString(curi).contains("If-None-Match: " + ETAG_TEST_VALUE + "\r\n"));
runDefaultChecks(curi);
// XXX make server send 304 not-modified and check for it here?
}
public void testShouldFetchBodyRule() throws Exception {
ensureHttpServers();
CrawlURI curi = makeCrawlURI("http://localhost:7777/");
fetcher().setShouldFetchBodyRule(new RejectDecideRule());
fetcher().process(curi);
assertTrue(httpRequestString(curi).startsWith("GET / HTTP/1.0\r\n"));
assertEquals("text/plain;charset=US-ASCII", curi.getContentType());
assertTrue(curi.getCredentials().isEmpty());
assertTrue(curi.getFetchDuration() >= 0);
assertTrue(curi.getFetchStatus() == 200);
assertTrue(curi.getFetchType() == FetchType.HTTP_GET);
// check for empty body
assertEquals(0, curi.getContentLength());
assertEquals(curi.getContentSize(), curi.getRecordedSize());
assertEquals("", messageBodyString(curi));
assertEquals("", entityString(curi));
}
public void testFetchTimeout() throws Exception {
ensureHttpServers();
CrawlURI curi = makeCrawlURI("http://localhost:7777/slow.txt");
fetcher().setTimeoutSeconds(2);
fetcher().process(curi);
// logger.info('\n' + httpRequestString(curi) + "\n\n" + rawResponseString(curi));
assertTrue(curi.getAnnotations().contains("timeTrunc"));
assertTrue(curi.getFetchDuration() >= 2000 && curi.getFetchDuration() < 2200);
}
// see http://stackoverflow.com/questions/100841/artificially-create-a-connection-timeout-error
public void testConnectionTimeout() throws Exception {
CrawlURI curi = makeCrawlURI("http://10.255.255.1/");
fetcher().stop();
fetcher().setSoTimeoutMs(300);
fetcher().start();
long start = System.currentTimeMillis();
fetcher().process(curi);
long elapsed = System.currentTimeMillis() - start;
assertTrue(elapsed >= 300 && elapsed < 400);
assertEquals(1, curi.getNonFatalFailures().size());
assertTrue(curi.getNonFatalFailures().toArray()[0] instanceof SocketTimeoutException);
assertTrue(curi.getNonFatalFailures().toArray()[0].toString().matches("(?i).*connect.*timed out.*"));
assertEquals(FetchStatusCodes.S_CONNECT_FAILED, curi.getFetchStatus());
assertEquals(0, curi.getFetchCompletedTime());
}
// XXX testSocketTimeout() (the other kind) - how to simulate?
public void testSslTrustLevel() throws Exception {
// default "open" trust level
ensureHttpServers();
CrawlURI curi = makeCrawlURI("https://localhost:7443/");
fetcher().process(curi);
runDefaultChecks(curi, "hostHeader");
// "normal" trust level
curi = makeCrawlURI("https://localhost:7443/");
fetcher().stop();
fetcher().setSslTrustLevel(TrustLevel.NORMAL);
fetcher().start();
fetcher().process(curi);
assertEquals(1, curi.getNonFatalFailures().size());
assertTrue(curi.getNonFatalFailures().toArray()[0] instanceof SSLHandshakeException);
assertEquals(FetchStatusCodes.S_CONNECT_FAILED, curi.getFetchStatus());
assertEquals(0, curi.getFetchCompletedTime());
}
public void testHttp11() throws Exception {
ensureHttpServers();
CrawlURI curi = makeCrawlURI("http://localhost:7777/");
fetcher().setUseHTTP11(true);
fetcher().process(curi);
assertTrue(httpRequestString(curi).startsWith("GET / HTTP/1.1\r\n"));
// what else?
runDefaultChecks(curi, "requestLine");
}
public void testChunked() throws Exception {
ensureHttpServers();
/* XXX Server expects us to close the connection apparently. But we
* don't detect end of chunked transfer. With these small timeouts we
* can finish quickly. A couple of SocketTimeoutExceptions will happen
* within RecordingInputStream.readFullyOrUntil().
*/
fetcher().stop();
fetcher().setSoTimeoutMs(500);
fetcher().setTimeoutSeconds(1);
fetcher().start();
CrawlURI curi = makeCrawlURI("http://localhost:7777/chunked.txt");
fetcher().setUseHTTP11(true);
fetcher().setSendConnectionClose(false);
fetcher().process(curi);
// logger.info('\n' + httpRequestString(curi) + "\n\n" + rawResponseString(curi));
// logger.info("\n----- rawResponseString -----\n" + rawResponseString(curi));
// logger.info("\n----- contentString -----\n" + contentString(curi));
// logger.info("\n----- entityString -----\n" + entityString(curi));
// logger.info("\n----- messageBodyString -----\n" + messageBodyString(curi));
assertEquals("chunked", getHttpResponseHeader(curi, "transfer-encoding"));
assertEquals("25\r\n" + DEFAULT_PAYLOAD_STRING + "\r\n0\r\n\r\n", messageBodyString(curi));
assertEquals(DEFAULT_PAYLOAD_STRING, entityString(curi));
assertEquals(DEFAULT_PAYLOAD_STRING, contentString(curi));
}
protected static class NoResponseServer extends Thread {
protected String listenAddress;
protected int listenPort;
protected boolean isTimeToBeDone = false;
public NoResponseServer(String address, int port) {
this.listenAddress = address;
this.listenPort = port;
}
@Override
public void run() {
try {
ServerSocket listeningSocket = new ServerSocket(listenPort, 0, Inet4Address.getByName(listenAddress));
listeningSocket.setSoTimeout(600);
while (!isTimeToBeDone) {
try {
Socket connectionSocket = listeningSocket.accept();
connectionSocket.shutdownOutput();
} catch (SocketTimeoutException e) {
}
}
} catch (Exception e) {
}
}
public void beDone() {
isTimeToBeDone = true;
}
}
public void testNoResponse() throws Exception {
NoResponseServer noResponseServer = new NoResponseServer("localhost", 7780);
noResponseServer.start();
// CrawlURI curi = makeCrawlURI("http://stats.bbc.co.uk/robots.txt");
CrawlURI curi = makeCrawlURI("http://localhost:7780");
fetcher().process(curi);
assertEquals(1, curi.getNonFatalFailures().size());
assertTrue(curi.getNonFatalFailures().toArray()[0] instanceof NoHttpResponseException);
// assertEquals(FetchStatusCodes.S_CONNECT_FAILED, curi.getFetchStatus());
assertEquals(FetchStatusCodes.S_CONNECT_LOST, curi.getFetchStatus());
assertEquals(0, curi.getFetchCompletedTime());
noResponseServer.beDone();
noResponseServer.join();
}
@Override
protected FetchHTTP makeModule() throws IOException {
FetchHTTP fetchHttp = new FetchHTTP();
// fetchHttp.setCookieStore(new SimpleCookieStore());
fetchHttp.setCookieStorage(new SimpleCookieStorage());
fetchHttp.setServerCache(new DefaultServerCache());
CrawlMetadata uap = new CrawlMetadata();
uap.setUserAgentTemplate(getUserAgentString());
fetchHttp.setUserAgentProvider(uap);
fetchHttp.start();
return fetchHttp;
}
protected String getHttpResponseHeader(CrawlURI curi, String name) {
Header header = curi.getHttpMethod().getResponseHeader(name);
if (header == null) {
return null;
} else {
return header.getValue();
}
}
}