From 25234898ef8d0f650f1abc4f8fdf7ad22025c16e Mon Sep 17 00:00:00 2001 From: Noah Levitt Date: Sun, 30 Dec 2012 22:35:28 -0800 Subject: [PATCH] Somewhat redundant form auth integration test restored, passes --- .../archive/modules/fetcher/FormAuthTest.java | 342 ++++++++++++++++++ httpcomponents.diff | 28 +- .../modules/fetcher/FetchHTTPRequest.java | 11 +- .../modules/fetcher/SimpleCookieStore.java | 89 +++++ .../modules/fetcher/FetchHTTPTest.java | 49 +-- .../modules/fetcher/FetchHTTPTests.java | 65 ---- 6 files changed, 466 insertions(+), 118 deletions(-) create mode 100644 engine/src/test/java/org/archive/modules/fetcher/FormAuthTest.java create mode 100644 modules/src/main/java/org/archive/modules/fetcher/SimpleCookieStore.java diff --git a/engine/src/test/java/org/archive/modules/fetcher/FormAuthTest.java b/engine/src/test/java/org/archive/modules/fetcher/FormAuthTest.java new file mode 100644 index 00000000..7d3ce576 --- /dev/null +++ b/engine/src/test/java/org/archive/modules/fetcher/FormAuthTest.java @@ -0,0 +1,342 @@ +/* + * This file is part of the Heritrix web crawler (crawler.archive.org). + * + * Licensed to the Internet Archive (IA) by one or more individual + * contributors. + * + * The IA licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.archive.modules.fetcher; + +import java.io.IOException; +import java.io.UnsupportedEncodingException; +import java.nio.charset.Charset; +import java.util.Arrays; +import java.util.HashMap; +import java.util.HashSet; +import java.util.Set; +import java.util.logging.Logger; + +import javax.servlet.ServletException; +import javax.servlet.http.HttpServletRequest; +import javax.servlet.http.HttpServletResponse; + +import junit.framework.TestCase; + +import org.apache.commons.httpclient.URIException; +import org.apache.commons.io.IOUtils; +import org.archive.crawler.prefetch.PreconditionEnforcer; +import org.archive.modules.CrawlMetadata; +import org.archive.modules.CrawlURI; +import org.archive.modules.CrawlURI.FetchType; +import org.archive.modules.credential.HtmlFormCredential; +import org.archive.net.UURI; +import org.archive.net.UURIFactory; +import org.archive.util.Recorder; +import org.archive.util.TmpDirTestCase; +import org.mortbay.jetty.NCSARequestLog; +import org.mortbay.jetty.Request; +import org.mortbay.jetty.Server; +import org.mortbay.jetty.bio.SocketConnector; +import org.mortbay.jetty.handler.HandlerCollection; +import org.mortbay.jetty.handler.RequestLogHandler; +import org.mortbay.jetty.security.Authenticator; +import org.mortbay.jetty.security.Constraint; +import org.mortbay.jetty.security.ConstraintMapping; +import org.mortbay.jetty.security.FormAuthenticator; +import org.mortbay.jetty.security.HashUserRealm; +import org.mortbay.jetty.security.SecurityHandler; +import org.mortbay.jetty.servlet.HashSessionManager; +import org.mortbay.jetty.servlet.SessionHandler; + +/* Somewhat redundant to org.archive.crawler.selftest.FormAuthSelfTest, but + * the code is written, it's easier to run in eclipse, and no doubt tests + * somewhat different stuff. */ +public class FormAuthTest extends TestCase { + + private static Logger logger = Logger.getLogger(FormAuthTest.class.getName()); + + protected static final String DEFAULT_PAYLOAD_STRING = "abcdefghijklmnopqrstuvwxyz0123456789\n"; + + protected static final String FORM_AUTH_REALM = "form-auth-realm"; + protected static final String FORM_AUTH_ROLE = "form-auth-role"; + protected static final String FORM_AUTH_LOGIN = "form-auth-login"; + protected static final String FORM_AUTH_PASSWORD = "form-auth-password"; + + protected FetchHTTP fetchHttp; + + protected FetchHTTP getFetcher() throws IOException { + if (fetchHttp == null) { + fetchHttp = new FetchHTTP(); + fetchHttp.setCookieStore(new SimpleCookieStore()); + fetchHttp.setServerCache(new DefaultServerCache()); + CrawlMetadata uap = new CrawlMetadata(); + uap.setUserAgentTemplate(getClass().getName()); + fetchHttp.setUserAgentProvider(uap); + + fetchHttp.start(); + } + + return fetchHttp; + } + + protected Recorder getRecorder() throws IOException { + if (Recorder.getHttpRecorder() == null) { + Recorder httpRecorder = new Recorder(TmpDirTestCase.tmpDir(), + getClass().getName(), 16 * 1024, 512 * 1024); + Recorder.setHttpRecorder(httpRecorder); + } + + return Recorder.getHttpRecorder(); + } + + protected CrawlURI makeCrawlURI(String uri) throws URIException, + IOException { + UURI uuri = UURIFactory.getInstance(uri); + CrawlURI curi = new CrawlURI(uuri); + curi.setSeed(true); + curi.setRecorder(getRecorder()); + return curi; + } + + // convenience methods to get strings from raw recorded i/o + /** + * Raw response including headers. + */ + protected String rawResponseString(CrawlURI curi) throws IOException, UnsupportedEncodingException { + byte[] buf = IOUtils.toByteArray(curi.getRecorder().getReplayInputStream()); + return new String(buf, "US-ASCII"); + } + /** + * Raw message body, before any unchunking or content-decoding. + */ + protected String messageBodyString(CrawlURI curi) throws IOException, UnsupportedEncodingException { + byte[] buf = IOUtils.toByteArray(curi.getRecorder().getMessageBodyReplayInputStream()); + return new String(buf, "US-ASCII"); + } + /** + * Message body after unchunking but before content-decoding. + */ + protected String entityString(CrawlURI curi) throws IOException, UnsupportedEncodingException { + byte[] buf = IOUtils.toByteArray(curi.getRecorder().getEntityReplayInputStream()); + return new String(buf, "US-ASCII"); + } + /** + * Unchunked, content-decoded message body. + */ + protected String contentString(CrawlURI curi) throws IOException, UnsupportedEncodingException { + byte[] buf = IOUtils.toByteArray(curi.getRecorder().getContentReplayInputStream()); + return new String(buf, "US-ASCII"); + } + protected String httpRequestString(CrawlURI curi) throws IOException, UnsupportedEncodingException { + byte[] buf = IOUtils.toByteArray(curi.getRecorder().getRecordedOutput().getReplayInputStream()); + return new String(buf, "US-ASCII"); + } + + public void testFormAuth() throws Exception { + startHttpServers(); + + HtmlFormCredential cred = new HtmlFormCredential(); + cred.setDomain("localhost:7779"); + cred.setLoginUri("/j_security_check"); + HashMap formItems = new HashMap(); + formItems.put("j_username", FORM_AUTH_LOGIN); + formItems.put("j_password", FORM_AUTH_PASSWORD); + cred.setFormItems(formItems); + + getFetcher().getCredentialStore().getCredentials().put("form-auth-credential", + cred); + + CrawlURI curi = makeCrawlURI("http://localhost:7779/"); + getFetcher().process(curi); + logger.info('\n' + httpRequestString(curi) + contentString(curi)); + runDefaultChecks(curi, "hostHeader"); + + // jetty needs us to hit a restricted url so it can redirect to the + // login page and remember where to redirect back to after successful + // login (if not we get a NPE within jetty) + curi = makeCrawlURI("http://localhost:7779/auth/1"); + getFetcher().process(curi); + logger.info('\n' + httpRequestString(curi) + "\n\n" + rawResponseString(curi)); + assertEquals(302, curi.getFetchStatus()); + assertTrue(curi.getHttpResponseHeader("Location").startsWith("http://localhost:7779/login.html")); + + PreconditionEnforcer preconditionEnforcer = new PreconditionEnforcer(); + preconditionEnforcer.setServerCache(getFetcher().getServerCache()); + preconditionEnforcer.setCredentialStore(getFetcher().getCredentialStore()); + boolean result = preconditionEnforcer.credentialPrecondition(curi); + assertTrue(result); + + CrawlURI loginUri = curi.getPrerequisiteUri(); + assertEquals("http://localhost:7779/j_security_check", loginUri.toString()); + + // there's some special logic with side effects in here for the login uri itself + result = preconditionEnforcer.credentialPrecondition(loginUri); + assertFalse(result); + + loginUri.setRecorder(getRecorder()); + getFetcher().process(loginUri); + logger.info('\n' + httpRequestString(loginUri) + "\n\n" + rawResponseString(loginUri)); + assertEquals(302, loginUri.getFetchStatus()); // 302 on successful login + assertEquals("http://localhost:7779/auth/1", loginUri.getHttpResponseHeader("location")); + + curi = makeCrawlURI("http://localhost:7779/auth/1"); + getFetcher().process(curi); + logger.info('\n' + httpRequestString(curi) + contentString(curi)); + runDefaultChecks(curi, "hostHeader", "requestLine"); + } + + protected static final String LOGIN_HTML = + "" + + "Log In" + + "" + + "
" + + "
username:
" + + "
password:
" + + "
" + "
" + "" + + ""; + + protected static class FormAuthTestHandler extends SessionHandler { + + public FormAuthTestHandler() { + super(); + } + + @Override + public void handle(String target, HttpServletRequest request, + HttpServletResponse response, int dispatch) throws IOException, + ServletException { + + if (target.endsWith("/set-cookie")) { + response.addCookie(new javax.servlet.http.Cookie("test-cookie-name", "test-cookie-value")); + } + + if (target.equals("/login.html")) { + response.setContentType("text/html;charset=US-ASCII"); + response.setStatus(HttpServletResponse.SC_OK); + response.getOutputStream().write(LOGIN_HTML.getBytes("US-ASCII")); + ((Request)request).setHandled(true); + } else { + response.setContentType("text/plain;charset=US-ASCII"); + response.setDateHeader("Last-Modified", 0); + response.setStatus(HttpServletResponse.SC_OK); + response.getOutputStream().write(DEFAULT_PAYLOAD_STRING.getBytes("US-ASCII")); + ((Request)request).setHandled(true); + } + } + } + + protected static SecurityHandler makeAuthWrapper(Authenticator authenticator, + final String role, String realm, final String login, + final String password) { + Constraint constraint = new Constraint(); + constraint.setRoles(new String[] { role }); + constraint.setAuthenticate(true); + + ConstraintMapping constraintMapping = new ConstraintMapping(); + constraintMapping.setConstraint(constraint); + constraintMapping.setPathSpec("/auth/*"); + + SecurityHandler authWrapper = new SecurityHandler(); + authWrapper.setAuthenticator(authenticator); + + authWrapper.setConstraintMappings(new ConstraintMapping[] {constraintMapping}); + authWrapper.setUserRealm(new HashUserRealm(realm) { + { + put(login, password); + addUserToRole(login, role); + } + }); + + return authWrapper; + } + + protected void startHttpServers() throws Exception { + // server for form auth + Server server = new Server(); + + SocketConnector sc = new SocketConnector(); + sc.setHost("127.0.0.1"); + sc.setPort(7779); + server.addConnector(sc); + + HandlerCollection handlers = new HandlerCollection(); + handlers.addHandler(new FormAuthTestHandler()); + RequestLogHandler requestLogHandler = new RequestLogHandler(); + NCSARequestLog requestLog = new NCSARequestLog(); + requestLogHandler.setRequestLog(requestLog); + handlers.addHandler(requestLogHandler); + + FormAuthenticator formAuthenticatrix = new FormAuthenticator(); + formAuthenticatrix.setLoginPage("/login.html"); + + SecurityHandler authWrapper = makeAuthWrapper(formAuthenticatrix, + FORM_AUTH_ROLE, FORM_AUTH_REALM, FORM_AUTH_LOGIN, + FORM_AUTH_PASSWORD); + authWrapper.setHandler(handlers); + + SessionHandler sessionHandler = new SessionHandler(); + sessionHandler.setSessionManager(new HashSessionManager()); + sessionHandler.setHandler(authWrapper); + server.setHandler(sessionHandler); + + server.start(); + } + + protected void runDefaultChecks(CrawlURI curi, String... exclusionsArray) + throws IOException, UnsupportedEncodingException { + + Set exclusions = new HashSet(Arrays.asList(exclusionsArray)); + + String requestString = httpRequestString(curi); + if (!exclusions.contains("requestLine")) { + assertTrue(requestString.startsWith("GET / HTTP/1.0\r\n")); + } + assertTrue(requestString.contains("User-Agent: " + getClass().getName() + "\r\n")); + assertTrue(requestString.matches("(?s).*Connection: [Cc]lose\r\n.*")); + if (!exclusions.contains("acceptHeaders")) { + assertTrue(requestString.contains("Accept: text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8\r\n")); + } + if (!exclusions.contains("hostHeader")) { + assertTrue(requestString.contains("Host: localhost:7777\r\n")); + } + assertTrue(requestString.endsWith("\r\n\r\n")); + + // check sizes + assertEquals(DEFAULT_PAYLOAD_STRING.length(), curi.getContentLength()); + assertEquals(curi.getContentSize(), curi.getRecordedSize()); + + // check various + assertEquals("sha1:TQ5R6YVOZLTQENRIIENVGXHOPX3YCRNJ", curi.getContentDigestSchemeString()); + assertEquals("text/plain;charset=US-ASCII", curi.getContentType()); + assertEquals(Charset.forName("US-ASCII"), curi.getRecorder().getCharset()); + assertTrue(curi.getCredentials().isEmpty()); + assertTrue(curi.getFetchDuration() >= 0); + assertTrue(curi.getFetchStatus() == 200); + assertTrue(curi.getFetchType() == FetchType.HTTP_GET); + + // check message body, i.e. "raw, possibly chunked-transfer-encoded message contents not including the leading headers" + assertEquals(DEFAULT_PAYLOAD_STRING, messageBodyString(curi)); + + // check entity, i.e. "message-body after any (usually-unnecessary) transfer-decoding but before any content-encoding (eg gzip) decoding" + assertEquals(DEFAULT_PAYLOAD_STRING, entityString(curi)); + + // check content, i.e. message-body after possibly tranfer-decoding and after content-encoding (eg gzip) decoding + assertEquals(DEFAULT_PAYLOAD_STRING, contentString(curi)); + assertEquals(DEFAULT_PAYLOAD_STRING.substring(0, 10), curi.getRecorder().getContentReplayPrefixString(10)); + assertEquals(DEFAULT_PAYLOAD_STRING, curi.getRecorder().getContentReplayCharSequence().toString()); + + assertTrue(curi.getNonFatalFailures().isEmpty()); + } + +} diff --git a/httpcomponents.diff b/httpcomponents.diff index 3865ca66..051c55ae 100644 --- a/httpcomponents.diff +++ b/httpcomponents.diff @@ -181,7 +181,33 @@ Index: httpclient/httpclient/src/main/java/org/apache/http/client/methods/BasicA =================================================================== --- httpclient/httpclient/src/main/java/org/apache/http/client/methods/BasicAbortableHttpRequest.java (revision 0) +++ httpclient/httpclient/src/main/java/org/apache/http/client/methods/BasicAbortableHttpRequest.java (working copy) -@@ -0,0 +1,85 @@ +@@ -0,0 +1,111 @@ ++/* ++ * ==================================================================== ++ * Licensed to the Apache Software Foundation (ASF) under one ++ * or more contributor license agreements. See the NOTICE file ++ * distributed with this work for additional information ++ * regarding copyright ownership. The ASF licenses this file ++ * to you under the Apache License, Version 2.0 (the ++ * "License"); you may not use this file except in compliance ++ * with the License. You may obtain a copy of the License at ++ * ++ * http://www.apache.org/licenses/LICENSE-2.0 ++ * ++ * Unless required by applicable law or agreed to in writing, ++ * software distributed under the License is distributed on an ++ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY ++ * KIND, either express or implied. See the License for the ++ * specific language governing permissions and limitations ++ * under the License. ++ * ==================================================================== ++ * ++ * This software consists of voluntary contributions made by many ++ * individuals on behalf of the Apache Software Foundation. For more ++ * information on the Apache Software Foundation, please see ++ * . ++ * ++ */ +package org.apache.http.client.methods; + +import org.apache.http.HttpVersion; diff --git a/modules/src/main/java/org/archive/modules/fetcher/FetchHTTPRequest.java b/modules/src/main/java/org/archive/modules/fetcher/FetchHTTPRequest.java index 9a9932ed..cc27c7e4 100644 --- a/modules/src/main/java/org/archive/modules/fetcher/FetchHTTPRequest.java +++ b/modules/src/main/java/org/archive/modules/fetcher/FetchHTTPRequest.java @@ -37,6 +37,7 @@ import java.util.logging.Logger; import org.apache.commons.httpclient.URIException; import org.apache.commons.lang.StringUtils; +import org.apache.http.HttpEntityEnclosingRequest; import org.apache.http.HttpException; import org.apache.http.HttpHeaders; import org.apache.http.HttpHost; @@ -55,8 +56,8 @@ import org.apache.http.client.config.RequestConfig; import org.apache.http.client.config.RequestConfig.Builder; import org.apache.http.client.entity.UrlEncodedFormEntity; import org.apache.http.client.methods.AbortableHttpRequestBase; +import org.apache.http.client.methods.BasicAbortableHttpEntityEnclosingRequest; import org.apache.http.client.methods.BasicAbortableHttpRequest; -import org.apache.http.client.methods.HttpPost; import org.apache.http.client.protocol.HttpClientContext; import org.apache.http.config.MessageConstraints; import org.apache.http.config.Registry; @@ -214,7 +215,9 @@ public class FetchHTTPRequest { } if (curi.getFetchType() == FetchType.HTTP_POST) { - throw new RuntimeException("fetch type " + FetchType.HTTP_POST + " not implemented"); + request = new BasicAbortableHttpEntityEnclosingRequest("POST", + requestLineUri, + httpVersion); } else { request = new BasicAbortableHttpRequest("GET", requestLineUri, @@ -444,8 +447,8 @@ public class FetchHTTPRequest { // XXX should it get charset from somewhere? UrlEncodedFormEntity entity = new UrlEncodedFormEntity(formParams, HTTP.DEF_CONTENT_CHARSET); - HttpPost postRequest = (HttpPost) request; - postRequest.setEntity(entity); + HttpEntityEnclosingRequest entityEnclosingRequest = (HttpEntityEnclosingRequest) request; + entityEnclosingRequest.setEntity(entity); return true; } diff --git a/modules/src/main/java/org/archive/modules/fetcher/SimpleCookieStore.java b/modules/src/main/java/org/archive/modules/fetcher/SimpleCookieStore.java new file mode 100644 index 00000000..cabd23f9 --- /dev/null +++ b/modules/src/main/java/org/archive/modules/fetcher/SimpleCookieStore.java @@ -0,0 +1,89 @@ +/* + * This file is part of the Heritrix web crawler (crawler.archive.org). + * + * Licensed to the Internet Archive (IA) by one or more individual + * contributors. + * + * The IA licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.archive.modules.fetcher; + +import java.io.IOException; +import java.io.Reader; +import java.util.Date; +import java.util.List; + +import org.apache.http.client.CookieStore; +import org.apache.http.cookie.Cookie; +import org.apache.http.impl.client.BasicCookieStore; +import org.archive.checkpointing.Checkpoint; + +/** In-memory cookie store, mostly for testing. */ +public class SimpleCookieStore extends AbstractCookieStore { + protected CookieStore basicCookieStore = new BasicCookieStore(); + + @Override + public void startCheckpoint(Checkpoint checkpointInProgress) { + throw new RuntimeException("not implemented"); + } + + @Override + public void setRecoveryCheckpoint(Checkpoint recoveryCheckpoint) { + throw new RuntimeException("not implemented"); + } + + @Override + public void finishCheckpoint(Checkpoint checkpointInProgress) { + throw new RuntimeException("not implemented"); + } + + @Override + public void doCheckpoint(Checkpoint checkpointInProgress) + throws IOException { + throw new RuntimeException("not implemented"); + } + + @Override + public List getCookies() { + return basicCookieStore.getCookies(); + } + + @Override + public boolean clearExpired(Date date) { + return basicCookieStore.clearExpired(date); + } + + @Override + public void clear() { + basicCookieStore.clear(); + } + + @Override + public void addCookie(Cookie cookie) { + basicCookieStore.addCookie(cookie); + } + + @Override + protected void saveCookies(String absolutePath) { + throw new RuntimeException("not implemented"); + } + + @Override + protected void prepare() { + } + + @Override + protected void loadCookies(Reader reader) { + throw new RuntimeException("not implemented"); + } +} \ No newline at end of file diff --git a/modules/src/test/java/org/archive/modules/fetcher/FetchHTTPTest.java b/modules/src/test/java/org/archive/modules/fetcher/FetchHTTPTest.java index 9e68e43f..3fdbf03b 100644 --- a/modules/src/test/java/org/archive/modules/fetcher/FetchHTTPTest.java +++ b/modules/src/test/java/org/archive/modules/fetcher/FetchHTTPTest.java @@ -45,11 +45,9 @@ import org.mortbay.jetty.security.BasicAuthenticator; import org.mortbay.jetty.security.Constraint; import org.mortbay.jetty.security.ConstraintMapping; import org.mortbay.jetty.security.DigestAuthenticator; -import org.mortbay.jetty.security.FormAuthenticator; import org.mortbay.jetty.security.HashUserRealm; import org.mortbay.jetty.security.SecurityHandler; import org.mortbay.jetty.security.SslSocketConnector; -import org.mortbay.jetty.servlet.HashSessionManager; import org.mortbay.jetty.servlet.SessionHandler; import org.mortbay.log.Log; @@ -76,11 +74,6 @@ public class FetchHTTPTest extends ProcessorTestBase { protected static final String DIGEST_AUTH_LOGIN = "digest-auth-login"; protected static final String DIGEST_AUTH_PASSWORD = "digest-auth-password"; - protected static final String FORM_AUTH_REALM = "form-auth-realm"; - protected static final String FORM_AUTH_ROLE = "form-auth-role"; - protected static final String FORM_AUTH_LOGIN = "form-auth-login"; - protected static final String FORM_AUTH_PASSWORD = "form-auth-password"; - protected static final String ETAG_TEST_VALUE = "An ETag is an opaque identifier assigned by a web server to a specific version of a resource found at a URL!"; protected static final String DEFAULT_PAYLOAD_STRING = "abcdefghijklmnopqrstuvwxyz0123456789\n"; @@ -93,18 +86,6 @@ public class FetchHTTPTest extends ProcessorTestBase { protected static final byte[] EIGHTY_BYTE_LINE = "1234567890123456789012345678901234567890123456789012345678901234567890123456789\n".getBytes(); - protected static final String LOGIN_HTML = - "" + - "Log In" + - "" + - "
" + - "
username:
" + - "
password:
" + - "
" + - "
" + - "" + - ""; - protected static class TestHandler extends SessionHandler { public TestHandler() { @@ -120,12 +101,7 @@ public class FetchHTTPTest extends ProcessorTestBase { response.addCookie(new javax.servlet.http.Cookie("test-cookie-name", "test-cookie-value")); } - if (target.equals("/login.html")) { - response.setContentType("text/html;charset=US-ASCII"); - response.setStatus(HttpServletResponse.SC_OK); - response.getOutputStream().write(LOGIN_HTML.getBytes("US-ASCII")); - ((Request)request).setHandled(true); - } else if (target.equals("/200k")) { + if (target.equals("/200k")) { response.setContentType("text/plain;charset=US-ASCII"); response.setStatus(HttpServletResponse.SC_OK); assertTrue(EIGHTY_BYTE_LINE.length == 80); @@ -281,29 +257,6 @@ public class FetchHTTPTest extends ProcessorTestBase { server.start(); servers.put(sc.getPort(), server); - // server for form auth - server = new Server(); - - sc = new SocketConnector(); - sc.setHost("127.0.0.1"); - sc.setPort(7779); - server.addConnector(sc); - - FormAuthenticator formAuthenticatrix = new FormAuthenticator(); - formAuthenticatrix.setLoginPage("/login.html"); - - authWrapper = makeAuthWrapper(formAuthenticatrix, - FORM_AUTH_ROLE, FORM_AUTH_REALM, FORM_AUTH_LOGIN, - FORM_AUTH_PASSWORD); - authWrapper.setHandler(handlers); - SessionHandler sessionHandler = new SessionHandler(); - sessionHandler.setSessionManager(new HashSessionManager()); - sessionHandler.setHandler(authWrapper); - server.setHandler(sessionHandler); - - server.start(); - servers.put(sc.getPort(), server); - return servers; } diff --git a/modules/src/test/java/org/archive/modules/fetcher/FetchHTTPTests.java b/modules/src/test/java/org/archive/modules/fetcher/FetchHTTPTests.java index 1883c483..22b90c6c 100644 --- a/modules/src/test/java/org/archive/modules/fetcher/FetchHTTPTests.java +++ b/modules/src/test/java/org/archive/modules/fetcher/FetchHTTPTests.java @@ -30,7 +30,6 @@ import static org.archive.modules.fetcher.FetchHTTPTest.ETAG_TEST_VALUE; import java.io.IOException; import java.io.InterruptedIOException; -import java.io.Reader; import java.io.UnsupportedEncodingException; import java.net.Inet4Address; import java.net.InetAddress; @@ -42,7 +41,6 @@ import java.nio.charset.Charset; import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; -import java.util.Date; import java.util.HashMap; import java.util.HashSet; import java.util.List; @@ -55,10 +53,6 @@ import javax.net.ssl.SSLException; import org.apache.commons.httpclient.URIException; import org.apache.commons.io.IOUtils; import org.apache.http.NoHttpResponseException; -import org.apache.http.client.CookieStore; -import org.apache.http.cookie.Cookie; -import org.apache.http.impl.client.BasicCookieStore; -import org.archive.checkpointing.Checkpoint; import org.archive.httpclient.ConfigurableX509TrustManager.TrustLevel; import org.archive.modules.CrawlMetadata; import org.archive.modules.CrawlURI; @@ -434,65 +428,6 @@ public class FetchHTTPTests extends ProcessorTestBase { } } - protected static class SimpleCookieStore extends AbstractCookieStore { - protected CookieStore basicCookieStore = new BasicCookieStore(); - - @Override - public void startCheckpoint(Checkpoint checkpointInProgress) { - throw new RuntimeException("not implemented"); - } - - @Override - public void setRecoveryCheckpoint(Checkpoint recoveryCheckpoint) { - throw new RuntimeException("not implemented"); - } - - @Override - public void finishCheckpoint(Checkpoint checkpointInProgress) { - throw new RuntimeException("not implemented"); - } - - @Override - public void doCheckpoint(Checkpoint checkpointInProgress) - throws IOException { - throw new RuntimeException("not implemented"); - } - - @Override - public List getCookies() { - return basicCookieStore.getCookies(); - } - - @Override - public boolean clearExpired(Date date) { - return basicCookieStore.clearExpired(date); - } - - @Override - public void clear() { - basicCookieStore.clear(); - } - - @Override - public void addCookie(Cookie cookie) { - basicCookieStore.addCookie(cookie); - } - - @Override - protected void saveCookies(String absolutePath) { - throw new RuntimeException("not implemented"); - } - - @Override - protected void prepare() { - } - - @Override - protected void loadCookies(Reader reader) { - throw new RuntimeException("not implemented"); - } - } - public void testHttpProxy() throws Exception { ProxiedRequestRememberer proxiedRequestRememberer = new ProxiedRequestRememberer(); DefaultHttpProxyServer httpProxyServer = new DefaultHttpProxyServer(7877, proxiedRequestRememberer, new HashMap());