diff --git a/modules/pom.xml b/modules/pom.xml index d92a622d..27a1155b 100644 --- a/modules/pom.xml +++ b/modules/pom.xml @@ -54,6 +54,12 @@ littleproxy 0.4 test + + + org.slf4j + slf4j-log4j12 + + org.slf4j diff --git a/modules/src/main/java/org/archive/modules/fetcher/AbstractCookieStore.java b/modules/src/main/java/org/archive/modules/fetcher/AbstractCookieStore.java index 8aaeab05..9096889b 100644 --- a/modules/src/main/java/org/archive/modules/fetcher/AbstractCookieStore.java +++ b/modules/src/main/java/org/archive/modules/fetcher/AbstractCookieStore.java @@ -18,7 +18,10 @@ */ package org.archive.modules.fetcher; -import java.io.*; +import java.io.BufferedReader; +import java.io.BufferedWriter; +import java.io.IOException; +import java.io.Reader; import java.nio.file.Files; import java.nio.file.Paths; import java.util.ArrayList; @@ -208,7 +211,7 @@ abstract public class AbstractCookieStore implements Lifecycle, Checkpointable, } protected class LimitedCookieStoreFacade implements CookieStore { - private final List cookies; + private List cookies; protected LimitedCookieStoreFacade(List cookies) { this.cookies = cookies; @@ -221,7 +224,20 @@ abstract public class AbstractCookieStore implements Lifecycle, Checkpointable, @Override public boolean clearExpired(Date date) { - throw new RuntimeException("not implemented"); + int expiredCount = 0; + for( Cookie c : cookies) { + boolean expired = AbstractCookieStore.this.expireCookie(c, date); + if( expired ) { + logger.fine("Expired cookie: " + c + " for date: " + date); + expiredCount++; + } + } + if( expiredCount > 0 ) { + logger.fine("Expired " + expiredCount + " cookies for date: " + date); + return true; + } else { + return false; + } } @Override @@ -287,7 +303,8 @@ abstract public class AbstractCookieStore implements Lifecycle, Checkpointable, addCookieImpl(cookie); } - + + abstract public boolean expireCookie(Cookie cookie, Date date); abstract protected void addCookieImpl(Cookie cookie); abstract public void clear(); abstract protected void prepare(); diff --git a/modules/src/main/java/org/archive/modules/fetcher/BdbCookieStore.java b/modules/src/main/java/org/archive/modules/fetcher/BdbCookieStore.java index 0f047bbc..05036685 100644 --- a/modules/src/main/java/org/archive/modules/fetcher/BdbCookieStore.java +++ b/modules/src/main/java/org/archive/modules/fetcher/BdbCookieStore.java @@ -20,6 +20,8 @@ package org.archive.modules.fetcher; import java.io.IOException; import java.io.UnsupportedEncodingException; +import java.nio.charset.StandardCharsets; +import java.util.Arrays; import java.util.Collection; import java.util.Date; import java.util.Iterator; @@ -38,7 +40,6 @@ import com.google.common.net.InternetDomainName; import com.sleepycat.bind.ByteArrayBinding; import com.sleepycat.bind.serial.SerialBinding; import com.sleepycat.bind.serial.StoredClassCatalog; -import com.sleepycat.collections.StoredCollection; import com.sleepycat.collections.StoredSortedMap; import com.sleepycat.je.Database; import com.sleepycat.je.DatabaseException; @@ -65,12 +66,8 @@ public class BdbCookieStore extends AbstractCookieStore implements * *

* This class is "restricted" in the sense that it is immutable, and also - * because some methods throw {@link RuntimeException} for other reasons. - * For example, {@link #iterator()} is not implemented, because we use this - * class to wrap a bdb {@link StoredCollection}, and iterators from that - * class need to be explicitly closed. Since this class hides the fact that - * a StoredCollection underlies it, we simply prevent {@link #iterator()} - * from being used. + * because some methods throw {@link RuntimeException} because they don't + * make sense in this context. */ public static class RestrictedCollectionWrappedList implements List { private Collection wrapped; @@ -78,7 +75,7 @@ public class BdbCookieStore extends AbstractCookieStore implements @Override public int size() { return wrapped.size(); } @Override public boolean isEmpty() { throw new RuntimeException("not implemented"); } @Override public boolean contains(Object o) { throw new RuntimeException("not implemented"); } - @Override public Iterator iterator() { throw new RuntimeException("not implemented"); } + @Override public Iterator iterator() { return (Iterator) Arrays.asList(wrapped.toArray()).iterator(); } @Override public Object[] toArray() { return wrapped.toArray(); } @SuppressWarnings("hiding") @Override public T[] toArray(T[] a) { return wrapped.toArray(a); } @Override public boolean add(T e) { throw new RuntimeException("immutable list"); } @@ -143,6 +140,17 @@ public class BdbCookieStore extends AbstractCookieStore implements cookies.remove(key); } } + + public boolean expireCookie(Cookie cookie, Date date) { + byte[] key = sortableKey(cookie).getBytes(StandardCharsets.UTF_8); + + if (cookie.isExpired(date)) { + cookies.remove(key); + return true; + } else { + return false; + } + } protected Collection hostSubset(String host) { try { diff --git a/modules/src/main/java/org/archive/modules/fetcher/FetchHTTPRequest.java b/modules/src/main/java/org/archive/modules/fetcher/FetchHTTPRequest.java index 36be6f72..120b2e57 100644 --- a/modules/src/main/java/org/archive/modules/fetcher/FetchHTTPRequest.java +++ b/modules/src/main/java/org/archive/modules/fetcher/FetchHTTPRequest.java @@ -400,7 +400,7 @@ public class FetchHTTPRequest { if (fetcher.getIgnoreCookies()) { requestConfigBuilder.setCookieSpec(CookieSpecs.IGNORE_COOKIES); } else { - requestConfigBuilder.setCookieSpec(CookieSpecs.BROWSER_COMPATIBILITY); + requestConfigBuilder.setCookieSpec(CookieSpecs.STANDARD); } requestConfigBuilder.setConnectionRequestTimeout(fetcher.getSoTimeoutMs()); @@ -529,8 +529,12 @@ public class FetchHTTPRequest { authCache = new BasicAuthCache(); httpClientContext.setAuthCache(authCache); } - authCache.put(host, authScheme); - + // Do not attempt to cache DIGEST auth: + // See https://github.com/internetarchive/heritrix3/pull/397 + if( !(authScheme instanceof org.apache.http.impl.auth.DigestScheme) ) { + authCache.put(host, authScheme); + } + if (httpClientContext.getCredentialsProvider() == null) { httpClientContext.setCredentialsProvider(new BasicCredentialsProvider()); } diff --git a/modules/src/main/java/org/archive/modules/fetcher/SimpleCookieStore.java b/modules/src/main/java/org/archive/modules/fetcher/SimpleCookieStore.java index 89e0a302..1316778b 100644 --- a/modules/src/main/java/org/archive/modules/fetcher/SimpleCookieStore.java +++ b/modules/src/main/java/org/archive/modules/fetcher/SimpleCookieStore.java @@ -77,6 +77,16 @@ public class SimpleCookieStore extends AbstractCookieStore implements CookieStor cookies.addCookie(cookie); } + @Override + public boolean expireCookie(Cookie cookie, Date date) { + if( cookie.isExpired(date)) { + cookies.getCookies().remove(cookie); + return true; + } else { + return false; + } + } + @Override public void clear() { cookies.clear(); diff --git a/modules/src/main/java/org/archive/modules/net/Robotstxt.java b/modules/src/main/java/org/archive/modules/net/Robotstxt.java index c11bd34f..5c5a860a 100644 --- a/modules/src/main/java/org/archive/modules/net/Robotstxt.java +++ b/modules/src/main/java/org/archive/modules/net/Robotstxt.java @@ -21,6 +21,7 @@ package org.archive.modules.net; import java.io.IOException; import java.io.Reader; import java.io.Serializable; +import java.nio.Buffer; import java.nio.CharBuffer; import java.util.HashMap; import java.util.LinkedList; @@ -89,7 +90,9 @@ public class Robotstxt implements Serializable { protected void initializeFromReader(Reader reader) throws IOException { CharBuffer buffer = CharBuffer.allocate(MAX_SIZE); while (buffer.hasRemaining() && reader.read(buffer) >= 0) ; - buffer.flip(); + //buffer.flip(); + // Explicit cast as per https://stackoverflow.com/questions/61267495/exception-in-thread-main-java-lang-nosuchmethoderror-java-nio-bytebuffer-flip + ((Buffer) buffer).flip(); String[] lines = LINE_SEPARATOR.split(buffer); if (buffer.limit() == buffer.capacity()) { diff --git a/modules/src/test/java/org/archive/modules/fetcher/CookieFetchHTTPIntegrationTest.java b/modules/src/test/java/org/archive/modules/fetcher/CookieFetchHTTPIntegrationTest.java index 257c8502..e786d6ea 100644 --- a/modules/src/test/java/org/archive/modules/fetcher/CookieFetchHTTPIntegrationTest.java +++ b/modules/src/test/java/org/archive/modules/fetcher/CookieFetchHTTPIntegrationTest.java @@ -23,6 +23,7 @@ import java.io.IOException; import java.net.InetAddress; import java.net.UnknownHostException; import java.util.Set; +import java.util.concurrent.TimeUnit; import javax.servlet.ServletException; import javax.servlet.http.Cookie; @@ -43,17 +44,23 @@ import org.archive.spring.ConfigFile; import org.archive.spring.ConfigPath; import org.archive.util.KeyTool; import org.archive.util.TmpDirTestCase; +import org.eclipse.jetty.http.HttpVersion; +import org.eclipse.jetty.server.HttpConfiguration; +import org.eclipse.jetty.server.HttpConnectionFactory; +import org.eclipse.jetty.server.Request; +import org.eclipse.jetty.server.SecureRequestCustomizer; +import org.eclipse.jetty.server.Server; +import org.eclipse.jetty.server.ServerConnector; +import org.eclipse.jetty.server.SslConnectionFactory; +import org.eclipse.jetty.server.session.SessionHandler; +import org.eclipse.jetty.util.log.Log; +import org.eclipse.jetty.util.ssl.SslContextFactory; import com.google.common.io.Files; import junit.extensions.TestSetup; import junit.framework.Test; import junit.framework.TestSuite; -import org.eclipse.jetty.http.HttpVersion; -import org.eclipse.jetty.server.*; -import org.eclipse.jetty.server.session.SessionHandler; -import org.eclipse.jetty.util.log.Log; -import org.eclipse.jetty.util.ssl.SslContextFactory; public class CookieFetchHTTPIntegrationTest extends ProcessorTestBase { @@ -285,18 +292,31 @@ public class CookieFetchHTTPIntegrationTest extends ProcessorTestBase { cookieStore.clear(); fetcher().setCookieStore(cookieStore); - CrawlURI curi = makeCrawlURI("http://example.com:7777/?name=foo&value=bar"); + CrawlURI curi = makeCrawlURI("http://example.com:7777/?name=foo&value=bar&maxAge=2"); fetcher().process(curi); assertFalse(FetchHTTPTests.httpRequestString(curi).toLowerCase().contains("cookie:")); - assertTrue(FetchHTTPTests.rawResponseString(curi).contains("Set-Cookie: foo=bar\r\n")); + //assertTrue(FetchHTTPTests.rawResponseString(curi).contains("Set-Cookie: foo=bar\r\n")); + assertTrue(FetchHTTPTests.rawResponseString(curi).contains("Set-Cookie: foo=bar")); // check second fetch has expected cookie curi = makeCrawlURI("http://example.com:7777/"); fetcher().process(curi); - assertTrue(FetchHTTPTests.httpRequestString(curi).contains("Cookie: foo=bar\r\n")); + assertTrue(FetchHTTPTests.httpRequestString(curi).contains("Cookie: foo=bar")); + //assertTrue(FetchHTTPTests.httpRequestString(curi).contains("Cookie: foo=bar\r\n")); assertFalse(FetchHTTPTests.rawResponseString(curi).toLowerCase().contains("set-cookie:")); assertEquals(1, cookieStore.getCookies().size()); + + // Wait for cookie to expire + TimeUnit.SECONDS.sleep(2); + + // check third fetch has no cookies + curi = makeCrawlURI("http://example.com:7777/"); + fetcher().process(curi); + assertFalse(FetchHTTPTests.httpRequestString(curi).toLowerCase().contains("cookie:")); + assertFalse(FetchHTTPTests.rawResponseString(curi).toLowerCase().contains("set-cookie:")); + + assertEquals(0, cookieStore.getCookies().size()); } public void testBasics() throws URIException, IOException, InterruptedException { @@ -325,15 +345,6 @@ public class CookieFetchHTTPIntegrationTest extends ProcessorTestBase { assertFalse(FetchHTTPTests.httpRequestString(curi).toLowerCase().contains("cookie:")); assertFalse(FetchHTTPTests.rawResponseString(curi).toLowerCase().contains("set-cookie:")); - /* - * XXX I think browsers differ on this behavior. This is what - * org.apache.http.impl.cookie.BrowserCompatSpec does. - */ - curi = makeCrawlURI("http://SUBDOMAIN.example.com:7777/"); - fetcher().process(curi); - assertTrue(FetchHTTPTests.httpRequestString(curi).contains("Cookie: foo=bar\r\n")); - assertFalse(FetchHTTPTests.rawResponseString(curi).toLowerCase().contains("set-cookie:")); - assertEquals(1, cookieStore.getCookies().size()); } diff --git a/pom.xml b/pom.xml index dbfe9a88..338f2ebb 100644 --- a/pom.xml +++ b/pom.xml @@ -132,12 +132,12 @@ http://maven.apache.org/guides/mini/guide-m1-m2.html org.apache.httpcomponents httpclient - 4.3.6 + 4.5.13 org.apache.httpcomponents httpmime - 4.3.6 + 4.5.13 junit