lee/heritrix3
1
0
Fork 0
mirror of https://github.com/internetarchive/heritrix3.git synced 2026-09-23 06:05:46 +00:00
Code Issues Packages Projects Releases Wiki Activity

Upgrade HTTPClient to 4.5.x for #245

Browse Source
This commit is contained in:
Andrew Jackson
2021-06-14 23:35:46 +01:00
parent d0221642c3
commit 81fa60d870
5 changed files with 18 additions and 28 deletions
Download Patch File Download Diff File Expand all files Collapse all files
modules/src/main/java/org/archive/modules/fetcher/AbstractCookieStore.java
+2 -1
View File
@@ -232,7 +232,8 @@ abstract public class AbstractCookieStore implements Lifecycle, Checkpointable,
@Override
public boolean clearExpired(Date date) {
throw new RuntimeException("not implemented");
logger.warning("clearExpired not implemented");
return false;
}
@Override
modules/src/main/java/org/archive/modules/fetcher/BdbCookieStore.java
+2 -1
View File
@@ -20,6 +20,7 @@ package org.archive.modules.fetcher;
import java.io.IOException;
import java.io.UnsupportedEncodingException;
import java.util.Arrays;
import java.util.Collection;
import java.util.Date;
import java.util.Iterator;
@@ -78,7 +79,7 @@ public class BdbCookieStore extends AbstractCookieStore implements
@Override public int size() { return wrapped.size(); }
@Override public boolean isEmpty() { throw new RuntimeException("not implemented"); }
@Override public boolean contains(Object o) { throw new RuntimeException("not implemented"); }
@Override public Iterator<T> iterator() { throw new RuntimeException("not implemented"); }
@Override public Iterator<T> iterator() { return (Iterator<T>) Arrays.asList(wrapped.toArray()).iterator(); }
@Override public Object[] toArray() { return wrapped.toArray(); }
@SuppressWarnings("hiding") @Override public <T> T[] toArray(T[] a) { return wrapped.toArray(a); }
@Override public boolean add(T e) { throw new RuntimeException("immutable list"); }
modules/src/main/java/org/archive/modules/fetcher/FetchHTTPRequest.java
+1 -10
View File
@@ -60,7 +60,6 @@ import org.apache.http.ProtocolVersion;
import org.apache.http.auth.AuthScheme;
import org.apache.http.auth.AuthScope;
import org.apache.http.auth.UsernamePasswordCredentials;
import org.apache.http.client.AuthCache;
import org.apache.http.client.ClientProtocolException;
import org.apache.http.client.CookieStore;
import org.apache.http.client.HttpClient;
@@ -88,7 +87,6 @@ import org.apache.http.entity.ContentType;
import org.apache.http.entity.mime.HttpMultipartMode;
import org.apache.http.entity.mime.MultipartEntityBuilder;
import org.apache.http.impl.DefaultBHttpClientConnection;
import org.apache.http.impl.client.BasicAuthCache;
import org.apache.http.impl.client.BasicCredentialsProvider;
import org.apache.http.impl.client.HttpClientBuilder;
import org.apache.http.impl.conn.BasicHttpClientConnectionManager;
@@ -400,7 +398,7 @@ public class FetchHTTPRequest {
if (fetcher.getIgnoreCookies()) {
requestConfigBuilder.setCookieSpec(CookieSpecs.IGNORE_COOKIES);
} else {
requestConfigBuilder.setCookieSpec(CookieSpecs.BROWSER_COMPATIBILITY);
requestConfigBuilder.setCookieSpec(CookieSpecs.DEFAULT);
}
requestConfigBuilder.setConnectionRequestTimeout(fetcher.getSoTimeoutMs());
@@ -524,13 +522,6 @@ public class FetchHTTPRequest {
protected void populateHttpCredential(HttpHost host, AuthScheme authScheme, String user, String password) {
UsernamePasswordCredentials credentials = new UsernamePasswordCredentials(user, password);
AuthCache authCache = httpClientContext.getAuthCache();
if (authCache == null) {
authCache = new BasicAuthCache();
httpClientContext.setAuthCache(authCache);
}
authCache.put(host, authScheme);
if (httpClientContext.getCredentialsProvider() == null) {
httpClientContext.setCredentialsProvider(new BasicCredentialsProvider());
}
modules/src/test/java/org/archive/modules/fetcher/CookieFetchHTTPIntegrationTest.java
+11 -14
View File
@@ -43,17 +43,23 @@ import org.archive.spring.ConfigFile;
import org.archive.spring.ConfigPath;
import org.archive.util.KeyTool;
import org.archive.util.TmpDirTestCase;
import org.eclipse.jetty.http.HttpVersion;
import org.eclipse.jetty.server.HttpConfiguration;
import org.eclipse.jetty.server.HttpConnectionFactory;
import org.eclipse.jetty.server.Request;
import org.eclipse.jetty.server.SecureRequestCustomizer;
import org.eclipse.jetty.server.Server;
import org.eclipse.jetty.server.ServerConnector;
import org.eclipse.jetty.server.SslConnectionFactory;
import org.eclipse.jetty.server.session.SessionHandler;
import org.eclipse.jetty.util.log.Log;
import org.eclipse.jetty.util.ssl.SslContextFactory;
import com.google.common.io.Files;
import junit.extensions.TestSetup;
import junit.framework.Test;
import junit.framework.TestSuite;
import org.eclipse.jetty.http.HttpVersion;
import org.eclipse.jetty.server.*;
import org.eclipse.jetty.server.session.SessionHandler;
import org.eclipse.jetty.util.log.Log;
import org.eclipse.jetty.util.ssl.SslContextFactory;
public class CookieFetchHTTPIntegrationTest extends ProcessorTestBase {
@@ -325,15 +331,6 @@ public class CookieFetchHTTPIntegrationTest extends ProcessorTestBase {
assertFalse(FetchHTTPTests.httpRequestString(curi).toLowerCase().contains("cookie:"));
assertFalse(FetchHTTPTests.rawResponseString(curi).toLowerCase().contains("set-cookie:"));
/*
* XXX I think browsers differ on this behavior. This is what
* org.apache.http.impl.cookie.BrowserCompatSpec does.
*/
curi = makeCrawlURI("http://SUBDOMAIN.example.com:7777/");
fetcher().process(curi);
assertTrue(FetchHTTPTests.httpRequestString(curi).contains("Cookie: foo=bar\r\n"));
assertFalse(FetchHTTPTests.rawResponseString(curi).toLowerCase().contains("set-cookie:"));
assertEquals(1, cookieStore.getCookies().size());
}
pom.xml
+2 -2
View File
@@ -136,12 +136,12 @@ http://maven.apache.org/guides/mini/guide-m1-m2.html
<dependency>
<groupId>org.apache.httpcomponents</groupId>
<artifactId>httpclient</artifactId>
<version>4.3.6</version>
<version>4.5.13</version>
</dependency>
<dependency>
<groupId>org.apache.httpcomponents</groupId>
<artifactId>httpmime</artifactId>
<version>4.3.6</version>
<version>4.5.13</version>
</dependency>
<dependency>
<groupId>junit</groupId>