diff --git a/commons/src/main/java/org/archive/net/http/RecordingHttpClient.java b/commons/src/main/java/org/archive/net/http/RecordingHttpClient.java index 43f032b6..ca8dece4 100644 --- a/commons/src/main/java/org/archive/net/http/RecordingHttpClient.java +++ b/commons/src/main/java/org/archive/net/http/RecordingHttpClient.java @@ -26,6 +26,8 @@ import org.apache.http.ConnectionReuseStrategy; import org.apache.http.HttpException; import org.apache.http.HttpResponse; import org.apache.http.client.HttpRequestRetryHandler; +import org.apache.http.client.params.CookiePolicy; +import org.apache.http.client.params.HttpClientParams; import org.apache.http.conn.ClientConnectionManager; import org.apache.http.conn.ClientConnectionOperator; import org.apache.http.conn.OperatedClientConnection; @@ -45,7 +47,7 @@ public class RecordingHttpClient extends DefaultHttpClient { public RecordingHttpClient() { super(); - // never retry (heritrix handles this elsewhere) + // never retry requests (heritrix handles this elsewhere) setHttpRequestRetryHandler(new HttpRequestRetryHandler() { @Override public boolean retryRequest(IOException exception, int executionCount, @@ -54,12 +56,17 @@ public class RecordingHttpClient extends DefaultHttpClient { } }); + // never reuse (no keep-alive) setReuseStrategy(new ConnectionReuseStrategy() { @Override public boolean keepAlive(HttpResponse response, HttpContext context) { return false; } }); + + // browser compatible cookie policy + HttpClientParams.setCookiePolicy(getParams(), + CookiePolicy.BROWSER_COMPATIBILITY); } @Override diff --git a/modules/src/main/java/org/archive/modules/fetcher/FetchHTTP2.java b/modules/src/main/java/org/archive/modules/fetcher/FetchHTTP2.java index d62d7556..49dbcaf2 100644 --- a/modules/src/main/java/org/archive/modules/fetcher/FetchHTTP2.java +++ b/modules/src/main/java/org/archive/modules/fetcher/FetchHTTP2.java @@ -33,9 +33,7 @@ import java.util.Map; import org.apache.commons.httpclient.HttpStatus; import org.apache.commons.lang.StringUtils; import org.apache.http.Header; -import org.apache.http.HeaderElement; import org.apache.http.HttpResponse; -import org.apache.http.ParseException; import org.apache.http.client.ClientProtocolException; import org.apache.http.client.HttpClient; import org.apache.http.client.methods.HttpGet; @@ -43,6 +41,7 @@ import org.apache.http.client.methods.HttpPost; import org.apache.http.client.methods.HttpRequestBase; import org.apache.http.impl.client.DefaultHttpClient; import org.apache.http.message.BasicHeader; +import org.apache.http.protocol.HTTP; import org.archive.io.RecorderLengthExceededException; import org.archive.io.RecorderTimeoutException; import org.archive.modules.CrawlURI; @@ -118,7 +117,8 @@ public class FetchHTTP2 extends Processor implements Lifecycle { kp.put("sendConnectionClose",sendClose); } - protected static final Header HEADER_SEND_CONNECTION_CLOSE = new BasicHeader("Connection", "close"); + protected static final Header HEADER_SEND_CONNECTION_CLOSE = new BasicHeader( + HTTP.CONN_DIRECTIVE, HTTP.CONN_CLOSE); /** * Can this processor fetch the given CrawlURI. May set a fetch status @@ -243,7 +243,7 @@ public class FetchHTTP2 extends Processor implements Lifecycle { userAgent = getUserAgentProvider().getUserAgent(); } - request.setHeader("User-Agent", userAgent); + request.setHeader(HTTP.USER_AGENT, userAgent); if(StringUtils.isNotBlank(from)) { request.setHeader("From", from); }