From 81fa60d870dc1814524f67baa76b0d46fbbe28f2 Mon Sep 17 00:00:00 2001 From: Andrew Jackson Date: Mon, 14 Jun 2021 23:35:46 +0100 Subject: [PATCH 01/11] Upgrade HTTPClient to 4.5.x for #245 --- .../modules/fetcher/AbstractCookieStore.java | 3 ++- .../modules/fetcher/BdbCookieStore.java | 3 ++- .../modules/fetcher/FetchHTTPRequest.java | 11 +------- .../CookieFetchHTTPIntegrationTest.java | 25 ++++++++----------- pom.xml | 4 +-- 5 files changed, 18 insertions(+), 28 deletions(-) diff --git a/modules/src/main/java/org/archive/modules/fetcher/AbstractCookieStore.java b/modules/src/main/java/org/archive/modules/fetcher/AbstractCookieStore.java index 1a43a14a..7b599dd5 100644 --- a/modules/src/main/java/org/archive/modules/fetcher/AbstractCookieStore.java +++ b/modules/src/main/java/org/archive/modules/fetcher/AbstractCookieStore.java @@ -232,7 +232,8 @@ abstract public class AbstractCookieStore implements Lifecycle, Checkpointable, @Override public boolean clearExpired(Date date) { - throw new RuntimeException("not implemented"); + logger.warning("clearExpired not implemented"); + return false; } @Override diff --git a/modules/src/main/java/org/archive/modules/fetcher/BdbCookieStore.java b/modules/src/main/java/org/archive/modules/fetcher/BdbCookieStore.java index 0f047bbc..d900363d 100644 --- a/modules/src/main/java/org/archive/modules/fetcher/BdbCookieStore.java +++ b/modules/src/main/java/org/archive/modules/fetcher/BdbCookieStore.java @@ -20,6 +20,7 @@ package org.archive.modules.fetcher; import java.io.IOException; import java.io.UnsupportedEncodingException; +import java.util.Arrays; import java.util.Collection; import java.util.Date; import java.util.Iterator; @@ -78,7 +79,7 @@ public class BdbCookieStore extends AbstractCookieStore implements @Override public int size() { return wrapped.size(); } @Override public boolean isEmpty() { throw new RuntimeException("not implemented"); } @Override public boolean contains(Object o) { throw new RuntimeException("not implemented"); } - @Override public Iterator iterator() { throw new RuntimeException("not implemented"); } + @Override public Iterator iterator() { return (Iterator) Arrays.asList(wrapped.toArray()).iterator(); } @Override public Object[] toArray() { return wrapped.toArray(); } @SuppressWarnings("hiding") @Override public T[] toArray(T[] a) { return wrapped.toArray(a); } @Override public boolean add(T e) { throw new RuntimeException("immutable list"); } diff --git a/modules/src/main/java/org/archive/modules/fetcher/FetchHTTPRequest.java b/modules/src/main/java/org/archive/modules/fetcher/FetchHTTPRequest.java index 3ee303ee..d9feaa3d 100644 --- a/modules/src/main/java/org/archive/modules/fetcher/FetchHTTPRequest.java +++ b/modules/src/main/java/org/archive/modules/fetcher/FetchHTTPRequest.java @@ -60,7 +60,6 @@ import org.apache.http.ProtocolVersion; import org.apache.http.auth.AuthScheme; import org.apache.http.auth.AuthScope; import org.apache.http.auth.UsernamePasswordCredentials; -import org.apache.http.client.AuthCache; import org.apache.http.client.ClientProtocolException; import org.apache.http.client.CookieStore; import org.apache.http.client.HttpClient; @@ -88,7 +87,6 @@ import org.apache.http.entity.ContentType; import org.apache.http.entity.mime.HttpMultipartMode; import org.apache.http.entity.mime.MultipartEntityBuilder; import org.apache.http.impl.DefaultBHttpClientConnection; -import org.apache.http.impl.client.BasicAuthCache; import org.apache.http.impl.client.BasicCredentialsProvider; import org.apache.http.impl.client.HttpClientBuilder; import org.apache.http.impl.conn.BasicHttpClientConnectionManager; @@ -400,7 +398,7 @@ public class FetchHTTPRequest { if (fetcher.getIgnoreCookies()) { requestConfigBuilder.setCookieSpec(CookieSpecs.IGNORE_COOKIES); } else { - requestConfigBuilder.setCookieSpec(CookieSpecs.BROWSER_COMPATIBILITY); + requestConfigBuilder.setCookieSpec(CookieSpecs.DEFAULT); } requestConfigBuilder.setConnectionRequestTimeout(fetcher.getSoTimeoutMs()); @@ -524,13 +522,6 @@ public class FetchHTTPRequest { protected void populateHttpCredential(HttpHost host, AuthScheme authScheme, String user, String password) { UsernamePasswordCredentials credentials = new UsernamePasswordCredentials(user, password); - AuthCache authCache = httpClientContext.getAuthCache(); - if (authCache == null) { - authCache = new BasicAuthCache(); - httpClientContext.setAuthCache(authCache); - } - authCache.put(host, authScheme); - if (httpClientContext.getCredentialsProvider() == null) { httpClientContext.setCredentialsProvider(new BasicCredentialsProvider()); } diff --git a/modules/src/test/java/org/archive/modules/fetcher/CookieFetchHTTPIntegrationTest.java b/modules/src/test/java/org/archive/modules/fetcher/CookieFetchHTTPIntegrationTest.java index 257c8502..9f606803 100644 --- a/modules/src/test/java/org/archive/modules/fetcher/CookieFetchHTTPIntegrationTest.java +++ b/modules/src/test/java/org/archive/modules/fetcher/CookieFetchHTTPIntegrationTest.java @@ -43,17 +43,23 @@ import org.archive.spring.ConfigFile; import org.archive.spring.ConfigPath; import org.archive.util.KeyTool; import org.archive.util.TmpDirTestCase; +import org.eclipse.jetty.http.HttpVersion; +import org.eclipse.jetty.server.HttpConfiguration; +import org.eclipse.jetty.server.HttpConnectionFactory; +import org.eclipse.jetty.server.Request; +import org.eclipse.jetty.server.SecureRequestCustomizer; +import org.eclipse.jetty.server.Server; +import org.eclipse.jetty.server.ServerConnector; +import org.eclipse.jetty.server.SslConnectionFactory; +import org.eclipse.jetty.server.session.SessionHandler; +import org.eclipse.jetty.util.log.Log; +import org.eclipse.jetty.util.ssl.SslContextFactory; import com.google.common.io.Files; import junit.extensions.TestSetup; import junit.framework.Test; import junit.framework.TestSuite; -import org.eclipse.jetty.http.HttpVersion; -import org.eclipse.jetty.server.*; -import org.eclipse.jetty.server.session.SessionHandler; -import org.eclipse.jetty.util.log.Log; -import org.eclipse.jetty.util.ssl.SslContextFactory; public class CookieFetchHTTPIntegrationTest extends ProcessorTestBase { @@ -325,15 +331,6 @@ public class CookieFetchHTTPIntegrationTest extends ProcessorTestBase { assertFalse(FetchHTTPTests.httpRequestString(curi).toLowerCase().contains("cookie:")); assertFalse(FetchHTTPTests.rawResponseString(curi).toLowerCase().contains("set-cookie:")); - /* - * XXX I think browsers differ on this behavior. This is what - * org.apache.http.impl.cookie.BrowserCompatSpec does. - */ - curi = makeCrawlURI("http://SUBDOMAIN.example.com:7777/"); - fetcher().process(curi); - assertTrue(FetchHTTPTests.httpRequestString(curi).contains("Cookie: foo=bar\r\n")); - assertFalse(FetchHTTPTests.rawResponseString(curi).toLowerCase().contains("set-cookie:")); - assertEquals(1, cookieStore.getCookies().size()); } diff --git a/pom.xml b/pom.xml index d2ace0bf..460a09ed 100644 --- a/pom.xml +++ b/pom.xml @@ -136,12 +136,12 @@ http://maven.apache.org/guides/mini/guide-m1-m2.html org.apache.httpcomponents httpclient - 4.3.6 + 4.5.13 org.apache.httpcomponents httpmime - 4.3.6 + 4.5.13 junit From d892026731d436445b0d249716e19c0c01831101 Mon Sep 17 00:00:00 2001 From: Andrew Jackson Date: Tue, 15 Jun 2021 00:31:04 +0100 Subject: [PATCH 02/11] Expire cookies when requested. --- .../modules/fetcher/AbstractCookieStore.java | 27 ++++++++++++++----- .../modules/fetcher/BdbCookieStore.java | 16 +++++++++++ .../modules/fetcher/SimpleCookieStore.java | 10 +++++++ 3 files changed, 47 insertions(+), 6 deletions(-) diff --git a/modules/src/main/java/org/archive/modules/fetcher/AbstractCookieStore.java b/modules/src/main/java/org/archive/modules/fetcher/AbstractCookieStore.java index 7b599dd5..fa74035f 100644 --- a/modules/src/main/java/org/archive/modules/fetcher/AbstractCookieStore.java +++ b/modules/src/main/java/org/archive/modules/fetcher/AbstractCookieStore.java @@ -18,8 +18,6 @@ */ package org.archive.modules.fetcher; -import it.unimi.dsi.mg4j.util.MutableString; - import java.io.BufferedReader; import java.io.File; import java.io.FileOutputStream; @@ -47,6 +45,8 @@ import org.archive.spring.ConfigFile; import org.archive.spring.ConfigPath; import org.springframework.context.Lifecycle; +import it.unimi.dsi.mg4j.util.MutableString; + abstract public class AbstractCookieStore implements Lifecycle, Checkpointable, CookieStore, FetchHTTPCookieStore { @@ -219,7 +219,7 @@ abstract public class AbstractCookieStore implements Lifecycle, Checkpointable, } protected class LimitedCookieStoreFacade implements CookieStore { - private final List cookies; + private List cookies; protected LimitedCookieStoreFacade(List cookies) { this.cookies = cookies; @@ -232,8 +232,22 @@ abstract public class AbstractCookieStore implements Lifecycle, Checkpointable, @Override public boolean clearExpired(Date date) { - logger.warning("clearExpired not implemented"); - return false; + List validCookies = new ArrayList(); + for( Cookie c : cookies) { + boolean expired = AbstractCookieStore.this.expireCookie(c, date); + if( !expired ) { + validCookies.add(c); + } + } + // Replace the cookie list, if any expired: + if( validCookies.size() != this.cookies.size() ) { + this.cookies = validCookies; + logger.info("Cleared expired cookies for date: " + date); + return true; + } else { + logger.info("Cleared no expired cookies for date: " + date); + return false; + } } @Override @@ -299,7 +313,8 @@ abstract public class AbstractCookieStore implements Lifecycle, Checkpointable, addCookieImpl(cookie); } - + + abstract public boolean expireCookie(Cookie cookie, Date date); abstract protected void addCookieImpl(Cookie cookie); abstract public void clear(); abstract protected void prepare(); diff --git a/modules/src/main/java/org/archive/modules/fetcher/BdbCookieStore.java b/modules/src/main/java/org/archive/modules/fetcher/BdbCookieStore.java index d900363d..bef77876 100644 --- a/modules/src/main/java/org/archive/modules/fetcher/BdbCookieStore.java +++ b/modules/src/main/java/org/archive/modules/fetcher/BdbCookieStore.java @@ -144,6 +144,22 @@ public class BdbCookieStore extends AbstractCookieStore implements cookies.remove(key); } } + + public boolean expireCookie(Cookie cookie, Date date) { + byte[] key; + try { + key = sortableKey(cookie).getBytes("UTF-8"); + } catch (UnsupportedEncodingException e) { + throw new RuntimeException(e); // impossible + } + + if (cookie.isExpired(date)) { + cookies.remove(key); + return true; + } else { + return false; + } + } protected Collection hostSubset(String host) { try { diff --git a/modules/src/main/java/org/archive/modules/fetcher/SimpleCookieStore.java b/modules/src/main/java/org/archive/modules/fetcher/SimpleCookieStore.java index 89e0a302..1316778b 100644 --- a/modules/src/main/java/org/archive/modules/fetcher/SimpleCookieStore.java +++ b/modules/src/main/java/org/archive/modules/fetcher/SimpleCookieStore.java @@ -77,6 +77,16 @@ public class SimpleCookieStore extends AbstractCookieStore implements CookieStor cookies.addCookie(cookie); } + @Override + public boolean expireCookie(Cookie cookie, Date date) { + if( cookie.isExpired(date)) { + cookies.getCookies().remove(cookie); + return true; + } else { + return false; + } + } + @Override public void clear() { cookies.clear(); From 4cd416a5e4857a1d13ffbce95ec7ef37bb8b016f Mon Sep 17 00:00:00 2001 From: Andrew Jackson Date: Thu, 17 Jun 2021 14:01:04 +0100 Subject: [PATCH 03/11] Update complier plugin and use compatible flip(). --- modules/src/main/java/org/archive/modules/net/Robotstxt.java | 5 ++++- pom.xml | 2 +- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/modules/src/main/java/org/archive/modules/net/Robotstxt.java b/modules/src/main/java/org/archive/modules/net/Robotstxt.java index c11bd34f..5c5a860a 100644 --- a/modules/src/main/java/org/archive/modules/net/Robotstxt.java +++ b/modules/src/main/java/org/archive/modules/net/Robotstxt.java @@ -21,6 +21,7 @@ package org.archive.modules.net; import java.io.IOException; import java.io.Reader; import java.io.Serializable; +import java.nio.Buffer; import java.nio.CharBuffer; import java.util.HashMap; import java.util.LinkedList; @@ -89,7 +90,9 @@ public class Robotstxt implements Serializable { protected void initializeFromReader(Reader reader) throws IOException { CharBuffer buffer = CharBuffer.allocate(MAX_SIZE); while (buffer.hasRemaining() && reader.read(buffer) >= 0) ; - buffer.flip(); + //buffer.flip(); + // Explicit cast as per https://stackoverflow.com/questions/61267495/exception-in-thread-main-java-lang-nosuchmethoderror-java-nio-bytebuffer-flip + ((Buffer) buffer).flip(); String[] lines = LINE_SEPARATOR.split(buffer); if (buffer.limit() == buffer.capacity()) { diff --git a/pom.xml b/pom.xml index 460a09ed..5740bd88 100644 --- a/pom.xml +++ b/pom.xml @@ -385,7 +385,7 @@ http://maven.apache.org/guides/mini/guide-m1-m2.html org.apache.maven.plugins maven-compiler-plugin - 3.3 + 3.8.1 1.8 1.8 From 1d1d2f5abeea78ec1ed9329d8bf30df1198a62c1 Mon Sep 17 00:00:00 2001 From: Andrew Jackson Date: Thu, 17 Jun 2021 14:19:12 +0100 Subject: [PATCH 04/11] Avoid two slf4j implementations. --- modules/pom.xml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/modules/pom.xml b/modules/pom.xml index 93bf30e1..7932f766 100644 --- a/modules/pom.xml +++ b/modules/pom.xml @@ -49,6 +49,12 @@ littleproxy 0.4 test + + + org.slf4j + slf4j-log4j12 + + org.slf4j From 2d64e005cb539464acea0c44b8badb6fa3bb5b90 Mon Sep 17 00:00:00 2001 From: Andrew Jackson Date: Thu, 17 Jun 2021 14:23:20 +0100 Subject: [PATCH 05/11] Switch to standard cookie handling. --- .../main/java/org/archive/modules/fetcher/FetchHTTPRequest.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/src/main/java/org/archive/modules/fetcher/FetchHTTPRequest.java b/modules/src/main/java/org/archive/modules/fetcher/FetchHTTPRequest.java index d9feaa3d..b60f2f1a 100644 --- a/modules/src/main/java/org/archive/modules/fetcher/FetchHTTPRequest.java +++ b/modules/src/main/java/org/archive/modules/fetcher/FetchHTTPRequest.java @@ -398,7 +398,7 @@ public class FetchHTTPRequest { if (fetcher.getIgnoreCookies()) { requestConfigBuilder.setCookieSpec(CookieSpecs.IGNORE_COOKIES); } else { - requestConfigBuilder.setCookieSpec(CookieSpecs.DEFAULT); + requestConfigBuilder.setCookieSpec(CookieSpecs.STANDARD); } requestConfigBuilder.setConnectionRequestTimeout(fetcher.getSoTimeoutMs()); From e07b1bb938493b1861b358835f56bf8f15770855 Mon Sep 17 00:00:00 2001 From: Andrew Jackson Date: Thu, 17 Jun 2021 14:23:42 +0100 Subject: [PATCH 06/11] Support cookie expiration and extend testing to cover that case. --- .../modules/fetcher/AbstractCookieStore.java | 13 ++++++------ .../CookieFetchHTTPIntegrationTest.java | 20 ++++++++++++++++--- 2 files changed, 23 insertions(+), 10 deletions(-) diff --git a/modules/src/main/java/org/archive/modules/fetcher/AbstractCookieStore.java b/modules/src/main/java/org/archive/modules/fetcher/AbstractCookieStore.java index fa74035f..59834f5c 100644 --- a/modules/src/main/java/org/archive/modules/fetcher/AbstractCookieStore.java +++ b/modules/src/main/java/org/archive/modules/fetcher/AbstractCookieStore.java @@ -232,20 +232,19 @@ abstract public class AbstractCookieStore implements Lifecycle, Checkpointable, @Override public boolean clearExpired(Date date) { - List validCookies = new ArrayList(); + int expiredCount = 0; for( Cookie c : cookies) { boolean expired = AbstractCookieStore.this.expireCookie(c, date); - if( !expired ) { - validCookies.add(c); + if( expired ) { + logger.fine("Expired cookie: " + c + " for date: " + date); + expiredCount++; } } // Replace the cookie list, if any expired: - if( validCookies.size() != this.cookies.size() ) { - this.cookies = validCookies; - logger.info("Cleared expired cookies for date: " + date); + if( expiredCount > 0 ) { + logger.fine("Expired " + expiredCount + " cookies for date: " + date); return true; } else { - logger.info("Cleared no expired cookies for date: " + date); return false; } } diff --git a/modules/src/test/java/org/archive/modules/fetcher/CookieFetchHTTPIntegrationTest.java b/modules/src/test/java/org/archive/modules/fetcher/CookieFetchHTTPIntegrationTest.java index 9f606803..e786d6ea 100644 --- a/modules/src/test/java/org/archive/modules/fetcher/CookieFetchHTTPIntegrationTest.java +++ b/modules/src/test/java/org/archive/modules/fetcher/CookieFetchHTTPIntegrationTest.java @@ -23,6 +23,7 @@ import java.io.IOException; import java.net.InetAddress; import java.net.UnknownHostException; import java.util.Set; +import java.util.concurrent.TimeUnit; import javax.servlet.ServletException; import javax.servlet.http.Cookie; @@ -291,18 +292,31 @@ public class CookieFetchHTTPIntegrationTest extends ProcessorTestBase { cookieStore.clear(); fetcher().setCookieStore(cookieStore); - CrawlURI curi = makeCrawlURI("http://example.com:7777/?name=foo&value=bar"); + CrawlURI curi = makeCrawlURI("http://example.com:7777/?name=foo&value=bar&maxAge=2"); fetcher().process(curi); assertFalse(FetchHTTPTests.httpRequestString(curi).toLowerCase().contains("cookie:")); - assertTrue(FetchHTTPTests.rawResponseString(curi).contains("Set-Cookie: foo=bar\r\n")); + //assertTrue(FetchHTTPTests.rawResponseString(curi).contains("Set-Cookie: foo=bar\r\n")); + assertTrue(FetchHTTPTests.rawResponseString(curi).contains("Set-Cookie: foo=bar")); // check second fetch has expected cookie curi = makeCrawlURI("http://example.com:7777/"); fetcher().process(curi); - assertTrue(FetchHTTPTests.httpRequestString(curi).contains("Cookie: foo=bar\r\n")); + assertTrue(FetchHTTPTests.httpRequestString(curi).contains("Cookie: foo=bar")); + //assertTrue(FetchHTTPTests.httpRequestString(curi).contains("Cookie: foo=bar\r\n")); assertFalse(FetchHTTPTests.rawResponseString(curi).toLowerCase().contains("set-cookie:")); assertEquals(1, cookieStore.getCookies().size()); + + // Wait for cookie to expire + TimeUnit.SECONDS.sleep(2); + + // check third fetch has no cookies + curi = makeCrawlURI("http://example.com:7777/"); + fetcher().process(curi); + assertFalse(FetchHTTPTests.httpRequestString(curi).toLowerCase().contains("cookie:")); + assertFalse(FetchHTTPTests.rawResponseString(curi).toLowerCase().contains("set-cookie:")); + + assertEquals(0, cookieStore.getCookies().size()); } public void testBasics() throws URIException, IOException, InterruptedException { From df4a328048d23fa4b27408cc3a72ae8133d0c85b Mon Sep 17 00:00:00 2001 From: Andrew Jackson Date: Sun, 20 Jun 2021 23:16:42 +0100 Subject: [PATCH 07/11] Fix imports. --- .../java/org/archive/modules/fetcher/AbstractCookieStore.java | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/modules/src/main/java/org/archive/modules/fetcher/AbstractCookieStore.java b/modules/src/main/java/org/archive/modules/fetcher/AbstractCookieStore.java index 889cd813..66b898e9 100644 --- a/modules/src/main/java/org/archive/modules/fetcher/AbstractCookieStore.java +++ b/modules/src/main/java/org/archive/modules/fetcher/AbstractCookieStore.java @@ -18,6 +18,7 @@ */ package org.archive.modules.fetcher; +import java.io.*; import java.nio.file.Files; import java.nio.file.Paths; import java.util.ArrayList; @@ -42,8 +43,6 @@ import org.archive.spring.ConfigFile; import org.archive.spring.ConfigPath; import org.springframework.context.Lifecycle; -import it.unimi.dsi.mg4j.util.MutableString; - abstract public class AbstractCookieStore implements Lifecycle, Checkpointable, CookieStore, FetchHTTPCookieStore { From bd6b4f1e0fbbc65c4ebaea9854a52dd7821fad6d Mon Sep 17 00:00:00 2001 From: Andy Jackson Date: Mon, 21 Jun 2021 13:42:13 +0100 Subject: [PATCH 08/11] Switch to StandardCharsets to avoid try/catch. Co-authored-by: Alex Osborne --- .../java/org/archive/modules/fetcher/BdbCookieStore.java | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/modules/src/main/java/org/archive/modules/fetcher/BdbCookieStore.java b/modules/src/main/java/org/archive/modules/fetcher/BdbCookieStore.java index bef77876..29cd46cb 100644 --- a/modules/src/main/java/org/archive/modules/fetcher/BdbCookieStore.java +++ b/modules/src/main/java/org/archive/modules/fetcher/BdbCookieStore.java @@ -146,12 +146,7 @@ public class BdbCookieStore extends AbstractCookieStore implements } public boolean expireCookie(Cookie cookie, Date date) { - byte[] key; - try { - key = sortableKey(cookie).getBytes("UTF-8"); - } catch (UnsupportedEncodingException e) { - throw new RuntimeException(e); // impossible - } + byte[] key = sortableKey(cookie).getBytes(StandardCharsets.UTF_8); if (cookie.isExpired(date)) { cookies.remove(key); From 595c8d1293cf86655bb7a6813add8722b8c94fe0 Mon Sep 17 00:00:00 2001 From: Andrew Jackson Date: Mon, 21 Jun 2021 13:49:21 +0100 Subject: [PATCH 09/11] Resolve issues raised during review by @ato. --- .../org/archive/modules/fetcher/AbstractCookieStore.java | 6 ++++-- .../java/org/archive/modules/fetcher/BdbCookieStore.java | 9 ++------- 2 files changed, 6 insertions(+), 9 deletions(-) diff --git a/modules/src/main/java/org/archive/modules/fetcher/AbstractCookieStore.java b/modules/src/main/java/org/archive/modules/fetcher/AbstractCookieStore.java index 66b898e9..9096889b 100644 --- a/modules/src/main/java/org/archive/modules/fetcher/AbstractCookieStore.java +++ b/modules/src/main/java/org/archive/modules/fetcher/AbstractCookieStore.java @@ -18,7 +18,10 @@ */ package org.archive.modules.fetcher; -import java.io.*; +import java.io.BufferedReader; +import java.io.BufferedWriter; +import java.io.IOException; +import java.io.Reader; import java.nio.file.Files; import java.nio.file.Paths; import java.util.ArrayList; @@ -229,7 +232,6 @@ abstract public class AbstractCookieStore implements Lifecycle, Checkpointable, expiredCount++; } } - // Replace the cookie list, if any expired: if( expiredCount > 0 ) { logger.fine("Expired " + expiredCount + " cookies for date: " + date); return true; diff --git a/modules/src/main/java/org/archive/modules/fetcher/BdbCookieStore.java b/modules/src/main/java/org/archive/modules/fetcher/BdbCookieStore.java index bef77876..417dcdee 100644 --- a/modules/src/main/java/org/archive/modules/fetcher/BdbCookieStore.java +++ b/modules/src/main/java/org/archive/modules/fetcher/BdbCookieStore.java @@ -39,7 +39,6 @@ import com.google.common.net.InternetDomainName; import com.sleepycat.bind.ByteArrayBinding; import com.sleepycat.bind.serial.SerialBinding; import com.sleepycat.bind.serial.StoredClassCatalog; -import com.sleepycat.collections.StoredCollection; import com.sleepycat.collections.StoredSortedMap; import com.sleepycat.je.Database; import com.sleepycat.je.DatabaseException; @@ -66,12 +65,8 @@ public class BdbCookieStore extends AbstractCookieStore implements * *

* This class is "restricted" in the sense that it is immutable, and also - * because some methods throw {@link RuntimeException} for other reasons. - * For example, {@link #iterator()} is not implemented, because we use this - * class to wrap a bdb {@link StoredCollection}, and iterators from that - * class need to be explicitly closed. Since this class hides the fact that - * a StoredCollection underlies it, we simply prevent {@link #iterator()} - * from being used. + * because some methods throw {@link RuntimeException} because they don't + * make sense in this context. */ public static class RestrictedCollectionWrappedList implements List { private Collection wrapped; From d372aacc335308d0a207d9fe0dc6a1049fd1f815 Mon Sep 17 00:00:00 2001 From: Andrew Jackson Date: Mon, 21 Jun 2021 13:50:58 +0100 Subject: [PATCH 10/11] Fix merge error. --- .../main/java/org/archive/modules/fetcher/BdbCookieStore.java | 1 + 1 file changed, 1 insertion(+) diff --git a/modules/src/main/java/org/archive/modules/fetcher/BdbCookieStore.java b/modules/src/main/java/org/archive/modules/fetcher/BdbCookieStore.java index 0932f202..05036685 100644 --- a/modules/src/main/java/org/archive/modules/fetcher/BdbCookieStore.java +++ b/modules/src/main/java/org/archive/modules/fetcher/BdbCookieStore.java @@ -20,6 +20,7 @@ package org.archive.modules.fetcher; import java.io.IOException; import java.io.UnsupportedEncodingException; +import java.nio.charset.StandardCharsets; import java.util.Arrays; import java.util.Collection; import java.util.Date; From 0640cae4a91d910746e2c0322e642702f8fbb6bb Mon Sep 17 00:00:00 2001 From: Andrew Jackson Date: Mon, 5 Jul 2021 21:29:16 +0100 Subject: [PATCH 11/11] Enabled AuthCache but not for DigestScheme. --- .../archive/modules/fetcher/FetchHTTPRequest.java | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/modules/src/main/java/org/archive/modules/fetcher/FetchHTTPRequest.java b/modules/src/main/java/org/archive/modules/fetcher/FetchHTTPRequest.java index b60f2f1a..ab98eac3 100644 --- a/modules/src/main/java/org/archive/modules/fetcher/FetchHTTPRequest.java +++ b/modules/src/main/java/org/archive/modules/fetcher/FetchHTTPRequest.java @@ -60,6 +60,7 @@ import org.apache.http.ProtocolVersion; import org.apache.http.auth.AuthScheme; import org.apache.http.auth.AuthScope; import org.apache.http.auth.UsernamePasswordCredentials; +import org.apache.http.client.AuthCache; import org.apache.http.client.ClientProtocolException; import org.apache.http.client.CookieStore; import org.apache.http.client.HttpClient; @@ -87,6 +88,7 @@ import org.apache.http.entity.ContentType; import org.apache.http.entity.mime.HttpMultipartMode; import org.apache.http.entity.mime.MultipartEntityBuilder; import org.apache.http.impl.DefaultBHttpClientConnection; +import org.apache.http.impl.client.BasicAuthCache; import org.apache.http.impl.client.BasicCredentialsProvider; import org.apache.http.impl.client.HttpClientBuilder; import org.apache.http.impl.conn.BasicHttpClientConnectionManager; @@ -522,6 +524,17 @@ public class FetchHTTPRequest { protected void populateHttpCredential(HttpHost host, AuthScheme authScheme, String user, String password) { UsernamePasswordCredentials credentials = new UsernamePasswordCredentials(user, password); + AuthCache authCache = httpClientContext.getAuthCache(); + if (authCache == null) { + authCache = new BasicAuthCache(); + httpClientContext.setAuthCache(authCache); + } + // Do not attempt to cache DIGEST auth: + // See https://github.com/internetarchive/heritrix3/pull/397 + if( !(authScheme instanceof org.apache.http.impl.auth.DigestScheme) ) { + authCache.put(host, authScheme); + } + if (httpClientContext.getCredentialsProvider() == null) { httpClientContext.setCredentialsProvider(new BasicCredentialsProvider()); }