lee/heritrix3
1
0
Fork 0
mirror of https://github.com/internetarchive/heritrix3.git synced 2026-09-22 13:46:01 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #397 from ukwa/upgrade-httpclient-4-5

Browse Source 
Upgrade httpclient to 4.5
This commit is contained in:
Andy Jackson
2021-08-02 09:19:30 +01:00
committed by GitHub
parent d72d847d92 0640cae4a9
commit 4d5b2dc3fe
8 changed files with 94 additions and 35 deletions
Download Patch File Download Diff File Expand all files Collapse all files
modules/pom.xml
+6
View File
@@ -54,6 +54,12 @@
<artifactId>littleproxy</artifactId>
<version>0.4</version>
<scope>test</scope>
<exclusions>
<exclusion>
<groupId>org.slf4j</groupId>
<artifactId>slf4j-log4j12</artifactId>
</exclusion>
</exclusions>
</dependency>
<dependency>
<groupId>org.slf4j</groupId>
modules/src/main/java/org/archive/modules/fetcher/AbstractCookieStore.java
+21 -4
View File
@@ -18,7 +18,10 @@
*/
package org.archive.modules.fetcher;
import java.io.*;
import java.io.BufferedReader;
import java.io.BufferedWriter;
import java.io.IOException;
import java.io.Reader;
import java.nio.file.Files;
import java.nio.file.Paths;
import java.util.ArrayList;
@@ -208,7 +211,7 @@ abstract public class AbstractCookieStore implements Lifecycle, Checkpointable,
}
protected class LimitedCookieStoreFacade implements CookieStore {
private final List<Cookie> cookies;
private List<Cookie> cookies;
protected LimitedCookieStoreFacade(List<Cookie> cookies) {
this.cookies = cookies;
@@ -221,7 +224,20 @@ abstract public class AbstractCookieStore implements Lifecycle, Checkpointable,
@Override
public boolean clearExpired(Date date) {
throw new RuntimeException("not implemented");
int expiredCount = 0;
for( Cookie c : cookies) {
boolean expired = AbstractCookieStore.this.expireCookie(c, date);
if( expired ) {
logger.fine("Expired cookie: " + c + " for date: " + date);
expiredCount++;
}
}
if( expiredCount > 0 ) {
logger.fine("Expired " + expiredCount + " cookies for date: " + date);
return true;
} else {
return false;
}
}
@Override
@@ -287,7 +303,8 @@ abstract public class AbstractCookieStore implements Lifecycle, Checkpointable,
addCookieImpl(cookie);
}
abstract public boolean expireCookie(Cookie cookie, Date date);
abstract protected void addCookieImpl(Cookie cookie);
abstract public void clear();
abstract protected void prepare();
modules/src/main/java/org/archive/modules/fetcher/BdbCookieStore.java
+16 -8
View File
@@ -20,6 +20,8 @@ package org.archive.modules.fetcher;
import java.io.IOException;
import java.io.UnsupportedEncodingException;
import java.nio.charset.StandardCharsets;
import java.util.Arrays;
import java.util.Collection;
import java.util.Date;
import java.util.Iterator;
@@ -38,7 +40,6 @@ import com.google.common.net.InternetDomainName;
import com.sleepycat.bind.ByteArrayBinding;
import com.sleepycat.bind.serial.SerialBinding;
import com.sleepycat.bind.serial.StoredClassCatalog;
import com.sleepycat.collections.StoredCollection;
import com.sleepycat.collections.StoredSortedMap;
import com.sleepycat.je.Database;
import com.sleepycat.je.DatabaseException;
@@ -65,12 +66,8 @@ public class BdbCookieStore extends AbstractCookieStore implements
*
* <p>
* This class is "restricted" in the sense that it is immutable, and also
* because some methods throw {@link RuntimeException} for other reasons.
* For example, {@link #iterator()} is not implemented, because we use this
* class to wrap a bdb {@link StoredCollection}, and iterators from that
* class need to be explicitly closed. Since this class hides the fact that
* a StoredCollection underlies it, we simply prevent {@link #iterator()}
* from being used.
* because some methods throw {@link RuntimeException} because they don't
* make sense in this context.
*/
public static class RestrictedCollectionWrappedList<T> implements List<T> {
private Collection<T> wrapped;
@@ -78,7 +75,7 @@ public class BdbCookieStore extends AbstractCookieStore implements
@Override public int size() { return wrapped.size(); }
@Override public boolean isEmpty() { throw new RuntimeException("not implemented"); }
@Override public boolean contains(Object o) { throw new RuntimeException("not implemented"); }
@Override public Iterator<T> iterator() { throw new RuntimeException("not implemented"); }
@Override public Iterator<T> iterator() { return (Iterator<T>) Arrays.asList(wrapped.toArray()).iterator(); }
@Override public Object[] toArray() { return wrapped.toArray(); }
@SuppressWarnings("hiding") @Override public <T> T[] toArray(T[] a) { return wrapped.toArray(a); }
@Override public boolean add(T e) { throw new RuntimeException("immutable list"); }
@@ -143,6 +140,17 @@ public class BdbCookieStore extends AbstractCookieStore implements
cookies.remove(key);
}
}
public boolean expireCookie(Cookie cookie, Date date) {
byte[] key = sortableKey(cookie).getBytes(StandardCharsets.UTF_8);
if (cookie.isExpired(date)) {
cookies.remove(key);
return true;
} else {
return false;
}
}
protected Collection<Cookie> hostSubset(String host) {
try {
modules/src/main/java/org/archive/modules/fetcher/FetchHTTPRequest.java
+7 -3
View File
@@ -400,7 +400,7 @@ public class FetchHTTPRequest {
if (fetcher.getIgnoreCookies()) {
requestConfigBuilder.setCookieSpec(CookieSpecs.IGNORE_COOKIES);
} else {
requestConfigBuilder.setCookieSpec(CookieSpecs.BROWSER_COMPATIBILITY);
requestConfigBuilder.setCookieSpec(CookieSpecs.STANDARD);
}
requestConfigBuilder.setConnectionRequestTimeout(fetcher.getSoTimeoutMs());
@@ -529,8 +529,12 @@ public class FetchHTTPRequest {
authCache = new BasicAuthCache();
httpClientContext.setAuthCache(authCache);
}
authCache.put(host, authScheme);
// Do not attempt to cache DIGEST auth:
// See https://github.com/internetarchive/heritrix3/pull/397
if( !(authScheme instanceof org.apache.http.impl.auth.DigestScheme) ) {
authCache.put(host, authScheme);
}
if (httpClientContext.getCredentialsProvider() == null) {
httpClientContext.setCredentialsProvider(new BasicCredentialsProvider());
}
modules/src/main/java/org/archive/modules/fetcher/SimpleCookieStore.java
+10
View File
@@ -77,6 +77,16 @@ public class SimpleCookieStore extends AbstractCookieStore implements CookieStor
cookies.addCookie(cookie);
}
@Override
public boolean expireCookie(Cookie cookie, Date date) {
if( cookie.isExpired(date)) {
cookies.getCookies().remove(cookie);
return true;
} else {
return false;
}
}
@Override
public void clear() {
cookies.clear();
modules/src/main/java/org/archive/modules/net/Robotstxt.java
+4 -1
View File
@@ -21,6 +21,7 @@ package org.archive.modules.net;
import java.io.IOException;
import java.io.Reader;
import java.io.Serializable;
import java.nio.Buffer;
import java.nio.CharBuffer;
import java.util.HashMap;
import java.util.LinkedList;
@@ -89,7 +90,9 @@ public class Robotstxt implements Serializable {
protected void initializeFromReader(Reader reader) throws IOException {
CharBuffer buffer = CharBuffer.allocate(MAX_SIZE);
while (buffer.hasRemaining() && reader.read(buffer) >= 0) ;
buffer.flip();
//buffer.flip();
// Explicit cast as per https://stackoverflow.com/questions/61267495/exception-in-thread-main-java-lang-nosuchmethoderror-java-nio-bytebuffer-flip
((Buffer) buffer).flip();
String[] lines = LINE_SEPARATOR.split(buffer);
if (buffer.limit() == buffer.capacity()) {
modules/src/test/java/org/archive/modules/fetcher/CookieFetchHTTPIntegrationTest.java
+28 -17
View File
@@ -23,6 +23,7 @@ import java.io.IOException;
import java.net.InetAddress;
import java.net.UnknownHostException;
import java.util.Set;
import java.util.concurrent.TimeUnit;
import javax.servlet.ServletException;
import javax.servlet.http.Cookie;
@@ -43,17 +44,23 @@ import org.archive.spring.ConfigFile;
import org.archive.spring.ConfigPath;
import org.archive.util.KeyTool;
import org.archive.util.TmpDirTestCase;
import org.eclipse.jetty.http.HttpVersion;
import org.eclipse.jetty.server.HttpConfiguration;
import org.eclipse.jetty.server.HttpConnectionFactory;
import org.eclipse.jetty.server.Request;
import org.eclipse.jetty.server.SecureRequestCustomizer;
import org.eclipse.jetty.server.Server;
import org.eclipse.jetty.server.ServerConnector;
import org.eclipse.jetty.server.SslConnectionFactory;
import org.eclipse.jetty.server.session.SessionHandler;
import org.eclipse.jetty.util.log.Log;
import org.eclipse.jetty.util.ssl.SslContextFactory;
import com.google.common.io.Files;
import junit.extensions.TestSetup;
import junit.framework.Test;
import junit.framework.TestSuite;
import org.eclipse.jetty.http.HttpVersion;
import org.eclipse.jetty.server.*;
import org.eclipse.jetty.server.session.SessionHandler;
import org.eclipse.jetty.util.log.Log;
import org.eclipse.jetty.util.ssl.SslContextFactory;
public class CookieFetchHTTPIntegrationTest extends ProcessorTestBase {
@@ -285,18 +292,31 @@ public class CookieFetchHTTPIntegrationTest extends ProcessorTestBase {
cookieStore.clear();
fetcher().setCookieStore(cookieStore);
CrawlURI curi = makeCrawlURI("http://example.com:7777/?name=foo&value=bar");
CrawlURI curi = makeCrawlURI("http://example.com:7777/?name=foo&value=bar&maxAge=2");
fetcher().process(curi);
assertFalse(FetchHTTPTests.httpRequestString(curi).toLowerCase().contains("cookie:"));
assertTrue(FetchHTTPTests.rawResponseString(curi).contains("Set-Cookie: foo=bar\r\n"));
//assertTrue(FetchHTTPTests.rawResponseString(curi).contains("Set-Cookie: foo=bar\r\n"));
assertTrue(FetchHTTPTests.rawResponseString(curi).contains("Set-Cookie: foo=bar"));
// check second fetch has expected cookie
curi = makeCrawlURI("http://example.com:7777/");
fetcher().process(curi);
assertTrue(FetchHTTPTests.httpRequestString(curi).contains("Cookie: foo=bar\r\n"));
assertTrue(FetchHTTPTests.httpRequestString(curi).contains("Cookie: foo=bar"));
//assertTrue(FetchHTTPTests.httpRequestString(curi).contains("Cookie: foo=bar\r\n"));
assertFalse(FetchHTTPTests.rawResponseString(curi).toLowerCase().contains("set-cookie:"));
assertEquals(1, cookieStore.getCookies().size());
// Wait for cookie to expire
TimeUnit.SECONDS.sleep(2);
// check third fetch has no cookies
curi = makeCrawlURI("http://example.com:7777/");
fetcher().process(curi);
assertFalse(FetchHTTPTests.httpRequestString(curi).toLowerCase().contains("cookie:"));
assertFalse(FetchHTTPTests.rawResponseString(curi).toLowerCase().contains("set-cookie:"));
assertEquals(0, cookieStore.getCookies().size());
}
public void testBasics() throws URIException, IOException, InterruptedException {
@@ -325,15 +345,6 @@ public class CookieFetchHTTPIntegrationTest extends ProcessorTestBase {
assertFalse(FetchHTTPTests.httpRequestString(curi).toLowerCase().contains("cookie:"));
assertFalse(FetchHTTPTests.rawResponseString(curi).toLowerCase().contains("set-cookie:"));
/*
* XXX I think browsers differ on this behavior. This is what
* org.apache.http.impl.cookie.BrowserCompatSpec does.
*/
curi = makeCrawlURI("http://SUBDOMAIN.example.com:7777/");
fetcher().process(curi);
assertTrue(FetchHTTPTests.httpRequestString(curi).contains("Cookie: foo=bar\r\n"));
assertFalse(FetchHTTPTests.rawResponseString(curi).toLowerCase().contains("set-cookie:"));
assertEquals(1, cookieStore.getCookies().size());
}
pom.xml
+2 -2
View File
@@ -132,12 +132,12 @@ http://maven.apache.org/guides/mini/guide-m1-m2.html
<dependency>
<groupId>org.apache.httpcomponents</groupId>
<artifactId>httpclient</artifactId>
<version>4.3.6</version>
<version>4.5.13</version>
</dependency>
<dependency>
<groupId>org.apache.httpcomponents</groupId>
<artifactId>httpmime</artifactId>
<version>4.3.6</version>
<version>4.5.13</version>
</dependency>
<dependency>
<groupId>junit</groupId>