diff --git a/modules/src/main/java/org/archive/modules/fetcher/AbstractCookieStore.java b/modules/src/main/java/org/archive/modules/fetcher/AbstractCookieStore.java index 1a43a14a..7b599dd5 100644 --- a/modules/src/main/java/org/archive/modules/fetcher/AbstractCookieStore.java +++ b/modules/src/main/java/org/archive/modules/fetcher/AbstractCookieStore.java @@ -232,7 +232,8 @@ abstract public class AbstractCookieStore implements Lifecycle, Checkpointable, @Override public boolean clearExpired(Date date) { - throw new RuntimeException("not implemented"); + logger.warning("clearExpired not implemented"); + return false; } @Override diff --git a/modules/src/main/java/org/archive/modules/fetcher/BdbCookieStore.java b/modules/src/main/java/org/archive/modules/fetcher/BdbCookieStore.java index 0f047bbc..d900363d 100644 --- a/modules/src/main/java/org/archive/modules/fetcher/BdbCookieStore.java +++ b/modules/src/main/java/org/archive/modules/fetcher/BdbCookieStore.java @@ -20,6 +20,7 @@ package org.archive.modules.fetcher; import java.io.IOException; import java.io.UnsupportedEncodingException; +import java.util.Arrays; import java.util.Collection; import java.util.Date; import java.util.Iterator; @@ -78,7 +79,7 @@ public class BdbCookieStore extends AbstractCookieStore implements @Override public int size() { return wrapped.size(); } @Override public boolean isEmpty() { throw new RuntimeException("not implemented"); } @Override public boolean contains(Object o) { throw new RuntimeException("not implemented"); } - @Override public Iterator iterator() { throw new RuntimeException("not implemented"); } + @Override public Iterator iterator() { return (Iterator) Arrays.asList(wrapped.toArray()).iterator(); } @Override public Object[] toArray() { return wrapped.toArray(); } @SuppressWarnings("hiding") @Override public T[] toArray(T[] a) { return wrapped.toArray(a); } @Override public boolean add(T e) { throw new RuntimeException("immutable list"); } diff --git a/modules/src/main/java/org/archive/modules/fetcher/FetchHTTPRequest.java b/modules/src/main/java/org/archive/modules/fetcher/FetchHTTPRequest.java index 3ee303ee..d9feaa3d 100644 --- a/modules/src/main/java/org/archive/modules/fetcher/FetchHTTPRequest.java +++ b/modules/src/main/java/org/archive/modules/fetcher/FetchHTTPRequest.java @@ -60,7 +60,6 @@ import org.apache.http.ProtocolVersion; import org.apache.http.auth.AuthScheme; import org.apache.http.auth.AuthScope; import org.apache.http.auth.UsernamePasswordCredentials; -import org.apache.http.client.AuthCache; import org.apache.http.client.ClientProtocolException; import org.apache.http.client.CookieStore; import org.apache.http.client.HttpClient; @@ -88,7 +87,6 @@ import org.apache.http.entity.ContentType; import org.apache.http.entity.mime.HttpMultipartMode; import org.apache.http.entity.mime.MultipartEntityBuilder; import org.apache.http.impl.DefaultBHttpClientConnection; -import org.apache.http.impl.client.BasicAuthCache; import org.apache.http.impl.client.BasicCredentialsProvider; import org.apache.http.impl.client.HttpClientBuilder; import org.apache.http.impl.conn.BasicHttpClientConnectionManager; @@ -400,7 +398,7 @@ public class FetchHTTPRequest { if (fetcher.getIgnoreCookies()) { requestConfigBuilder.setCookieSpec(CookieSpecs.IGNORE_COOKIES); } else { - requestConfigBuilder.setCookieSpec(CookieSpecs.BROWSER_COMPATIBILITY); + requestConfigBuilder.setCookieSpec(CookieSpecs.DEFAULT); } requestConfigBuilder.setConnectionRequestTimeout(fetcher.getSoTimeoutMs()); @@ -524,13 +522,6 @@ public class FetchHTTPRequest { protected void populateHttpCredential(HttpHost host, AuthScheme authScheme, String user, String password) { UsernamePasswordCredentials credentials = new UsernamePasswordCredentials(user, password); - AuthCache authCache = httpClientContext.getAuthCache(); - if (authCache == null) { - authCache = new BasicAuthCache(); - httpClientContext.setAuthCache(authCache); - } - authCache.put(host, authScheme); - if (httpClientContext.getCredentialsProvider() == null) { httpClientContext.setCredentialsProvider(new BasicCredentialsProvider()); } diff --git a/modules/src/test/java/org/archive/modules/fetcher/CookieFetchHTTPIntegrationTest.java b/modules/src/test/java/org/archive/modules/fetcher/CookieFetchHTTPIntegrationTest.java index 257c8502..9f606803 100644 --- a/modules/src/test/java/org/archive/modules/fetcher/CookieFetchHTTPIntegrationTest.java +++ b/modules/src/test/java/org/archive/modules/fetcher/CookieFetchHTTPIntegrationTest.java @@ -43,17 +43,23 @@ import org.archive.spring.ConfigFile; import org.archive.spring.ConfigPath; import org.archive.util.KeyTool; import org.archive.util.TmpDirTestCase; +import org.eclipse.jetty.http.HttpVersion; +import org.eclipse.jetty.server.HttpConfiguration; +import org.eclipse.jetty.server.HttpConnectionFactory; +import org.eclipse.jetty.server.Request; +import org.eclipse.jetty.server.SecureRequestCustomizer; +import org.eclipse.jetty.server.Server; +import org.eclipse.jetty.server.ServerConnector; +import org.eclipse.jetty.server.SslConnectionFactory; +import org.eclipse.jetty.server.session.SessionHandler; +import org.eclipse.jetty.util.log.Log; +import org.eclipse.jetty.util.ssl.SslContextFactory; import com.google.common.io.Files; import junit.extensions.TestSetup; import junit.framework.Test; import junit.framework.TestSuite; -import org.eclipse.jetty.http.HttpVersion; -import org.eclipse.jetty.server.*; -import org.eclipse.jetty.server.session.SessionHandler; -import org.eclipse.jetty.util.log.Log; -import org.eclipse.jetty.util.ssl.SslContextFactory; public class CookieFetchHTTPIntegrationTest extends ProcessorTestBase { @@ -325,15 +331,6 @@ public class CookieFetchHTTPIntegrationTest extends ProcessorTestBase { assertFalse(FetchHTTPTests.httpRequestString(curi).toLowerCase().contains("cookie:")); assertFalse(FetchHTTPTests.rawResponseString(curi).toLowerCase().contains("set-cookie:")); - /* - * XXX I think browsers differ on this behavior. This is what - * org.apache.http.impl.cookie.BrowserCompatSpec does. - */ - curi = makeCrawlURI("http://SUBDOMAIN.example.com:7777/"); - fetcher().process(curi); - assertTrue(FetchHTTPTests.httpRequestString(curi).contains("Cookie: foo=bar\r\n")); - assertFalse(FetchHTTPTests.rawResponseString(curi).toLowerCase().contains("set-cookie:")); - assertEquals(1, cookieStore.getCookies().size()); } diff --git a/pom.xml b/pom.xml index d2ace0bf..460a09ed 100644 --- a/pom.xml +++ b/pom.xml @@ -136,12 +136,12 @@ http://maven.apache.org/guides/mini/guide-m1-m2.html org.apache.httpcomponents httpclient - 4.3.6 + 4.5.13 org.apache.httpcomponents httpmime - 4.3.6 + 4.5.13 junit