lee/heritrix3
1
0
Fork 0
mirror of https://github.com/internetarchive/heritrix3.git synced 2026-09-20 20:55:49 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #457 from ClemensRobbenhaar/issues191-https-over-proxy

Browse Source 
Fix issue#191: "RIS already open for ToeThread..." exception during https pages crawl over proxy
This commit is contained in:
Alex Osborne
2022-01-17 16:18:13 +09:00
committed by GitHub
parent c4edadf1a8 261fa318b4
commit aa4f0f9933
2 changed files with 51 additions and 5 deletions
Download Patch File Download Diff File Expand all files Collapse all files
modules/src/main/java/org/archive/modules/fetcher/FetchHTTPRequest.java
+29 -5
View File
@@ -189,7 +189,12 @@ public class FetchHTTPRequest {
if (StringUtils.isNotEmpty(proxyHostname) && proxyPort != null) {
this.proxyHost = new HttpHost(proxyHostname, proxyPort);
this.requestConfigBuilder.setProxy(this.proxyHost);
requestLineUri = curi.getUURI().toString();
if ("https".equalsIgnoreCase(curi.getUURI().getScheme())) {
// with SSL connections the hostname is already send with the CONNECT
requestLineUri = curi.getUURI().getEscapedPathQuery();
} else {
requestLineUri = curi.getUURI().toString();
}
} else {
requestLineUri = curi.getUURI().getEscapedPathQuery();
}
@@ -604,7 +609,7 @@ public class FetchHTTPRequest {
DEFAULT_BUFSIZE, chardecoder, charencoder,
cconfig.getMessageConstraints(), null, null,
DefaultHttpRequestWriterFactory.INSTANCE,
DefaultHttpResponseParserFactory.INSTANCE, curi);
DefaultHttpResponseParserFactory.INSTANCE, proxyHost, curi);
}
};
BasicHttpClientConnectionManager connMan = new BasicHttpClientConnectionManager(
@@ -623,6 +628,9 @@ public class FetchHTTPRequest {
private static final AtomicLong COUNTER = new AtomicLong();
private String id;
private final CrawlURI curi;
private final boolean isProxyConnect;
private boolean shouldWrapInput = true;
private boolean shouldWrapOutput = true;
public RecordingHttpClientConnection(
final int buffersize,
@@ -633,21 +641,32 @@ public class FetchHTTPRequest {
final ContentLengthStrategy incomingContentStrategy,
final ContentLengthStrategy outgoingContentStrategy,
final HttpMessageWriterFactory<HttpRequest> requestWriterFactory,
final HttpMessageParserFactory<HttpResponse> responseParserFactory, CrawlURI curi) {
final HttpMessageParserFactory<HttpResponse> responseParserFactory,
final HttpHost proxy, CrawlURI curi) {
super(buffersize, fragmentSizeHint, chardecoder, charencoder,
constraints, incomingContentStrategy, outgoingContentStrategy,
requestWriterFactory, responseParserFactory);
id = "recording-http-connection-" + Long.toString(COUNTER.getAndIncrement());
this.curi = curi;
// if we send HTTPS over a proxy, then the first connection should not be recorded,
// as it is only the "CONNECT" to open the SSL-tunnel for the actual connection
isProxyConnect = (proxy != null && "https".equalsIgnoreCase(curi.getBaseURI().getScheme()));
if (isProxyConnect) {
shouldWrapInput = shouldWrapOutput = false;
}
}
@Override
protected InputStream getSocketInputStream(final Socket socket) throws IOException {
curi.setServerIP(socket.getInetAddress().getHostAddress());
Recorder recorder = Recorder.getHttpRecorder();
if (recorder != null) { // XXX || (isSecure() && isProxied())) {
if (shouldWrapInput && recorder != null) { // means: !(isSecure() && isProxied()) {
return recorder.inputWrap(super.getSocketInputStream(socket));
} else {
if (isProxyConnect) {
shouldWrapInput = true;
}
return super.getSocketInputStream(socket);
}
}
@@ -655,9 +674,14 @@ public class FetchHTTPRequest {
@Override
protected OutputStream getSocketOutputStream(final Socket socket) throws IOException {
Recorder recorder = Recorder.getHttpRecorder();
if (recorder != null) { // XXX || (isSecure() && isProxied())) {
if (shouldWrapOutput && recorder != null) { // means: !(isSecure() && isProxied()) {
return recorder.outputWrap(super.getSocketOutputStream(socket));
} else {
// for the next connection we want to record the contents
if (isProxyConnect) {
shouldWrapOutput = true;
}
return super.getSocketOutputStream(socket);
}
}
modules/src/test/java/org/archive/modules/fetcher/FetchHTTPTests.java
+22
View File
@@ -501,6 +501,28 @@ public class FetchHTTPTests extends ProcessorTestBase {
httpProxyServer.stop();
}
}
public void testHttpsProxy() throws Exception {
DefaultHttpProxyServer httpProxyServer = new DefaultHttpProxyServer(7877);
httpProxyServer.start(true, false);
try {
fetcher().setHttpProxyHost("localhost");
fetcher().setHttpProxyPort(7877);
CrawlURI curi = makeCrawlURI("https://localhost:7443/");
fetcher().process(curi);
String requestString = httpRequestString(curi);
assertTrue(requestString.contains("Host: localhost:7443\r\n"));
// in case of HTTPS we do not have a "Via" header in the recorded request
// as this is only present in the "CONNECT" that we do not record
assertNull(curi.getHttpResponseHeader("Via"));
runDefaultChecks(curi, "hostHeader");
} finally {
httpProxyServer.stop();
}
}
public void testMaxFetchKBSec() throws Exception {
CrawlURI curi = makeCrawlURI("http://localhost:7777/200k");