diff --git a/modules/src/main/java/org/archive/modules/fetcher/FetchHTTPRequest.java b/modules/src/main/java/org/archive/modules/fetcher/FetchHTTPRequest.java index 120b2e57..684847e4 100644 --- a/modules/src/main/java/org/archive/modules/fetcher/FetchHTTPRequest.java +++ b/modules/src/main/java/org/archive/modules/fetcher/FetchHTTPRequest.java @@ -189,7 +189,12 @@ public class FetchHTTPRequest { if (StringUtils.isNotEmpty(proxyHostname) && proxyPort != null) { this.proxyHost = new HttpHost(proxyHostname, proxyPort); this.requestConfigBuilder.setProxy(this.proxyHost); - requestLineUri = curi.getUURI().toString(); + if ("https".equalsIgnoreCase(curi.getUURI().getScheme())) { + // with SSL connections the hostname is already send with the CONNECT + requestLineUri = curi.getUURI().getEscapedPathQuery(); + } else { + requestLineUri = curi.getUURI().toString(); + } } else { requestLineUri = curi.getUURI().getEscapedPathQuery(); } @@ -604,7 +609,7 @@ public class FetchHTTPRequest { DEFAULT_BUFSIZE, chardecoder, charencoder, cconfig.getMessageConstraints(), null, null, DefaultHttpRequestWriterFactory.INSTANCE, - DefaultHttpResponseParserFactory.INSTANCE, curi); + DefaultHttpResponseParserFactory.INSTANCE, proxyHost, curi); } }; BasicHttpClientConnectionManager connMan = new BasicHttpClientConnectionManager( @@ -623,6 +628,9 @@ public class FetchHTTPRequest { private static final AtomicLong COUNTER = new AtomicLong(); private String id; private final CrawlURI curi; + private final boolean isProxyConnect; + private boolean shouldWrapInput = true; + private boolean shouldWrapOutput = true; public RecordingHttpClientConnection( final int buffersize, @@ -633,21 +641,32 @@ public class FetchHTTPRequest { final ContentLengthStrategy incomingContentStrategy, final ContentLengthStrategy outgoingContentStrategy, final HttpMessageWriterFactory requestWriterFactory, - final HttpMessageParserFactory responseParserFactory, CrawlURI curi) { + final HttpMessageParserFactory responseParserFactory, + final HttpHost proxy, CrawlURI curi) { super(buffersize, fragmentSizeHint, chardecoder, charencoder, constraints, incomingContentStrategy, outgoingContentStrategy, requestWriterFactory, responseParserFactory); id = "recording-http-connection-" + Long.toString(COUNTER.getAndIncrement()); this.curi = curi; + // if we send HTTPS over a proxy, then the first connection should not be recorded, + // as it is only the "CONNECT" to open the SSL-tunnel for the actual connection + isProxyConnect = (proxy != null && "https".equalsIgnoreCase(curi.getBaseURI().getScheme())); + if (isProxyConnect) { + shouldWrapInput = shouldWrapOutput = false; + } } @Override protected InputStream getSocketInputStream(final Socket socket) throws IOException { curi.setServerIP(socket.getInetAddress().getHostAddress()); Recorder recorder = Recorder.getHttpRecorder(); - if (recorder != null) { // XXX || (isSecure() && isProxied())) { + + if (shouldWrapInput && recorder != null) { // means: !(isSecure() && isProxied()) { return recorder.inputWrap(super.getSocketInputStream(socket)); } else { + if (isProxyConnect) { + shouldWrapInput = true; + } return super.getSocketInputStream(socket); } } @@ -655,9 +674,14 @@ public class FetchHTTPRequest { @Override protected OutputStream getSocketOutputStream(final Socket socket) throws IOException { Recorder recorder = Recorder.getHttpRecorder(); - if (recorder != null) { // XXX || (isSecure() && isProxied())) { + + if (shouldWrapOutput && recorder != null) { // means: !(isSecure() && isProxied()) { return recorder.outputWrap(super.getSocketOutputStream(socket)); } else { + // for the next connection we want to record the contents + if (isProxyConnect) { + shouldWrapOutput = true; + } return super.getSocketOutputStream(socket); } } diff --git a/modules/src/test/java/org/archive/modules/fetcher/FetchHTTPTests.java b/modules/src/test/java/org/archive/modules/fetcher/FetchHTTPTests.java index 0295c14b..25e387d7 100644 --- a/modules/src/test/java/org/archive/modules/fetcher/FetchHTTPTests.java +++ b/modules/src/test/java/org/archive/modules/fetcher/FetchHTTPTests.java @@ -501,6 +501,28 @@ public class FetchHTTPTests extends ProcessorTestBase { httpProxyServer.stop(); } } + + public void testHttpsProxy() throws Exception { + DefaultHttpProxyServer httpProxyServer = new DefaultHttpProxyServer(7877); + httpProxyServer.start(true, false); + + try { + fetcher().setHttpProxyHost("localhost"); + fetcher().setHttpProxyPort(7877); + + CrawlURI curi = makeCrawlURI("https://localhost:7443/"); + fetcher().process(curi); + + String requestString = httpRequestString(curi); + assertTrue(requestString.contains("Host: localhost:7443\r\n")); + // in case of HTTPS we do not have a "Via" header in the recorded request + // as this is only present in the "CONNECT" that we do not record + assertNull(curi.getHttpResponseHeader("Via")); + runDefaultChecks(curi, "hostHeader"); + } finally { + httpProxyServer.stop(); + } + } public void testMaxFetchKBSec() throws Exception { CrawlURI curi = makeCrawlURI("http://localhost:7777/200k");