lee/heritrix3
1
0
Fork 0
mirror of https://github.com/internetarchive/heritrix3.git synced 2026-07-20 22:41:41 +00:00
Code Issues Packages Projects Releases Wiki Activity

browser-compatibility mode for cookie matching; that's what old FetchHTTP does, and also avoids sending weird Cookie2:$Version1

Browse Source
This commit is contained in:
Noah Levitt
2012-06-18 17:49:09 -07:00
parent 3c7bd8afd6
commit de152eaa9e
2 changed files with 12 additions and 5 deletions
Download Patch File Download Diff File Expand all files Collapse all files
commons/src/main/java/org/archive/net/http/RecordingHttpClient.java
+8 -1
View File
@@ -26,6 +26,8 @@ import org.apache.http.ConnectionReuseStrategy;
import org.apache.http.HttpException;
import org.apache.http.HttpResponse;
import org.apache.http.client.HttpRequestRetryHandler;
import org.apache.http.client.params.CookiePolicy;
import org.apache.http.client.params.HttpClientParams;
import org.apache.http.conn.ClientConnectionManager;
import org.apache.http.conn.ClientConnectionOperator;
import org.apache.http.conn.OperatedClientConnection;
@@ -45,7 +47,7 @@ public class RecordingHttpClient extends DefaultHttpClient {
public RecordingHttpClient() {
super();
// never retry (heritrix handles this elsewhere)
// never retry requests (heritrix handles this elsewhere)
setHttpRequestRetryHandler(new HttpRequestRetryHandler() {
@Override
public boolean retryRequest(IOException exception, int executionCount,
@@ -54,12 +56,17 @@ public class RecordingHttpClient extends DefaultHttpClient {
}
});
// never reuse (no keep-alive)
setReuseStrategy(new ConnectionReuseStrategy() {
@Override
public boolean keepAlive(HttpResponse response, HttpContext context) {
return false;
}
});
// browser compatible cookie policy
HttpClientParams.setCookiePolicy(getParams(),
CookiePolicy.BROWSER_COMPATIBILITY);
}
@Override
modules/src/main/java/org/archive/modules/fetcher/FetchHTTP2.java
+4 -4
View File
@@ -33,9 +33,7 @@ import java.util.Map;
import org.apache.commons.httpclient.HttpStatus;
import org.apache.commons.lang.StringUtils;
import org.apache.http.Header;
import org.apache.http.HeaderElement;
import org.apache.http.HttpResponse;
import org.apache.http.ParseException;
import org.apache.http.client.ClientProtocolException;
import org.apache.http.client.HttpClient;
import org.apache.http.client.methods.HttpGet;
@@ -43,6 +41,7 @@ import org.apache.http.client.methods.HttpPost;
import org.apache.http.client.methods.HttpRequestBase;
import org.apache.http.impl.client.DefaultHttpClient;
import org.apache.http.message.BasicHeader;
import org.apache.http.protocol.HTTP;
import org.archive.io.RecorderLengthExceededException;
import org.archive.io.RecorderTimeoutException;
import org.archive.modules.CrawlURI;
@@ -118,7 +117,8 @@ public class FetchHTTP2 extends Processor implements Lifecycle {
kp.put("sendConnectionClose",sendClose);
}
protected static final Header HEADER_SEND_CONNECTION_CLOSE = new BasicHeader("Connection", "close");
protected static final Header HEADER_SEND_CONNECTION_CLOSE = new BasicHeader(
HTTP.CONN_DIRECTIVE, HTTP.CONN_CLOSE);
/**
* Can this processor fetch the given CrawlURI. May set a fetch status
@@ -243,7 +243,7 @@ public class FetchHTTP2 extends Processor implements Lifecycle {
userAgent = getUserAgentProvider().getUserAgent();
}
request.setHeader("User-Agent", userAgent);
request.setHeader(HTTP.USER_AGENT, userAgent);
if(StringUtils.isNotBlank(from)) {
request.setHeader("From", from);
}