lee/heritrix3
1
0
Fork 0
mirror of https://github.com/internetarchive/heritrix3.git synced 2026-09-20 20:55:49 +00:00
Code Issues Packages Projects Releases Wiki Activity

fix HER-2063 - omit port in Host request header when it is default for the scheme

Browse Source
This commit is contained in:
Noah Levitt
2014-02-03 17:18:40 -08:00
parent 10ec391da1
commit edd62f3de0
2 changed files with 47 additions and 2 deletions
Download Patch File Download Diff File Expand all files Collapse all files
modules/src/main/java/org/archive/modules/fetcher/FetchHTTPRequest.java
+36
View File
@@ -46,6 +46,7 @@ import javax.net.ssl.SSLSocket;
import org.apache.commons.httpclient.URIException;
import org.apache.commons.lang.StringUtils;
import org.apache.http.HttpEntityEnclosingRequest;
import org.apache.http.HttpException;
import org.apache.http.HttpHeaders;
import org.apache.http.HttpHost;
import org.apache.http.HttpRequest;
@@ -72,6 +73,7 @@ import org.apache.http.conn.DnsResolver;
import org.apache.http.conn.HttpClientConnectionManager;
import org.apache.http.conn.ManagedHttpClientConnection;
import org.apache.http.conn.routing.HttpRoute;
import org.apache.http.conn.routing.HttpRoutePlanner;
import org.apache.http.conn.socket.ConnectionSocketFactory;
import org.apache.http.conn.socket.PlainConnectionSocketFactory;
import org.apache.http.conn.ssl.AllowAllHostnameVerifier;
@@ -89,6 +91,8 @@ import org.apache.http.io.HttpMessageParserFactory;
import org.apache.http.io.HttpMessageWriterFactory;
import org.apache.http.message.BasicNameValuePair;
import org.apache.http.protocol.HTTP;
import org.apache.http.protocol.HttpContext;
import org.apache.http.util.Args;
import org.archive.modules.CrawlURI;
import org.archive.modules.CrawlURI.FetchType;
import org.archive.modules.Processor;
@@ -555,6 +559,36 @@ class FetchHTTPRequest {
}
}
protected static final HttpRoutePlanner ROUTE_PLANNER = new HttpRoutePlanner() {
@Override
public HttpRoute determineRoute(HttpHost host, HttpRequest request,
HttpContext context) throws HttpException {
Args.notNull(host, "Target host");
Args.notNull(request, "Request");
final HttpClientContext clientContext = HttpClientContext.adapt(context);
final RequestConfig config = clientContext.getRequestConfig();
final InetAddress local = config.getLocalAddress();
HttpHost proxy = config.getProxy();
final HttpHost target;
if (host.getPort() > 0
&& (host.getSchemeName().equalsIgnoreCase("http") && host.getPort() == 80
|| host.getSchemeName().equalsIgnoreCase("https") && host.getPort() == 443)) {
target = new HttpHost(host.getHostName(), -1, host.getSchemeName());
} else {
target = host;
}
final boolean secure = target.getSchemeName().equalsIgnoreCase("https");
if (proxy == null) {
return new HttpRoute(target, local, secure);
} else {
return new HttpRoute(target, local, proxy, secure);
}
}
};
protected void initHttpClientBuilder() {
httpClientBuilder = HttpClientBuilder.create();
@@ -565,6 +599,8 @@ class FetchHTTPRequest {
// we handle redirects manually
httpClientBuilder.disableRedirectHandling();
httpClientBuilder.setRoutePlanner(ROUTE_PLANNER);
}
public HttpResponse execute() throws ClientProtocolException, IOException {
modules/src/test/java/org/archive/modules/fetcher/FetchHTTPTests.java
+11 -2
View File
@@ -52,10 +52,8 @@ import javax.net.ssl.SSLException;
import org.apache.commons.httpclient.URIException;
import org.apache.commons.io.IOUtils;
import org.apache.commons.lang.StringEscapeUtils;
import org.apache.http.NoHttpResponseException;
import org.archive.httpclient.ConfigurableX509TrustManager.TrustLevel;
import org.archive.io.ReplayCharSequence;
import org.archive.modules.CrawlMetadata;
import org.archive.modules.CrawlURI;
import org.archive.modules.CrawlURI.FetchType;
@@ -821,6 +819,17 @@ public class FetchHTTPTests extends ProcessorTestBase {
runDefaultChecks(curi, "requestLine", "contentType");
}
// see https://webarchive.jira.com/browse/HER-2063
public void testHostHeaderDefaultPort() throws Exception {
CrawlURI curi = makeCrawlURI("http://example.com/");
fetcher().process(curi);
assertTrue(httpRequestString(curi).contains("Host: example.com\r\n"));
curi = makeCrawlURI("https://example.com/");
fetcher().process(curi);
assertTrue(httpRequestString(curi).contains("Host: example.com\r\n"));
}
@Override
protected FetchHTTP makeModule() throws IOException {
FetchHTTP fetchHttp = newTestFetchHttp(getUserAgentString());