lee/heritrix3
1
0
Fork 0
mirror of https://github.com/internetarchive/heritrix3.git synced 2026-08-25 16:06:33 +00:00
Code Issues Packages Projects Releases Wiki Activity

BrowserProcessor: use FetchHTTP2's configured proxy as an upstream proxy

Browse Source
This commit is contained in:
Alex Osborne
2025-06-10 09:01:02 +09:00
parent cc50af72be
commit c3f9afe64b
4 changed files with 63 additions and 5 deletions
Download Patch File Download Diff File Expand all files Collapse all files
commons/src/main/java/org/archive/net/MitmProxy.java
+22 -1
View File
@@ -20,6 +20,8 @@
package org.archive.net;
import org.eclipse.jetty.client.HttpClient;
import org.eclipse.jetty.client.HttpProxy;
import org.eclipse.jetty.client.ProxyConfiguration;
import org.eclipse.jetty.http.HttpHeader;
import org.eclipse.jetty.http.HttpURI;
import org.eclipse.jetty.io.Content;
@@ -42,6 +44,7 @@ import static org.eclipse.jetty.http.HttpHeader.ACCEPT_ENCODING;
* An HTTP proxy server which intercepts TLS and records or replays responses.
*/
public class MitmProxy {
private static final String UPSTREAM_PROXY = MitmProxy.class.getName() + ".upstreamProxy";
private final SslConnectionFactory sslConnectionFactory = new SslConnectionFactory();
private final Server server = new Server(0);
private final RequestHandler requestHandler;
@@ -63,7 +66,6 @@ public class MitmProxy {
public void start() throws Exception {
sslConnectionFactory.start();
server.setHandler(new Handler.Sequence(
new SslConnectHandler(),
new MitmProxyHandler()));
@@ -90,6 +92,10 @@ public class MitmProxy {
body.transferTo(Content.Sink.asOutputStream(response));
callback().succeeded();
}
public void setUpstreamProxy(ProxyConfiguration.Proxy proxy) {
request.setAttribute(UPSTREAM_PROXY, proxy);
}
}
public interface RequestHandler {
@@ -156,8 +162,16 @@ public class MitmProxy {
// Host header is not allowed in HTTP/2
headers.remove(HttpHeader.HOST);
});
ProxyConfiguration.Proxy upstreamProxy = (HttpProxy)clientToProxyRequest.getAttribute(UPSTREAM_PROXY);
if (upstreamProxy != null) {
addUpstreamProxyIfAbsent(upstreamProxy);
proxyToServerRequest.tag(upstreamProxy);
}
var listener = (ExchangeListener)clientToProxyRequest.getAttribute(ExchangeListener.class.getName());
if (listener != null) {
proxyToServerRequest.onRequestBegin(listener);
proxyToServerRequest.onRequestHeaders(listener);
proxyToServerRequest.onRequestContent(listener);
proxyToServerRequest.onResponseHeaders(listener);
@@ -165,6 +179,13 @@ public class MitmProxy {
proxyToServerRequest.onComplete(listener);
}
}
private void addUpstreamProxyIfAbsent(ProxyConfiguration.Proxy proxy) {
for (var existingProxy : getHttpClient().getProxyConfiguration().getProxies()) {
if (existingProxy == proxy) return;
}
getHttpClient().getProxyConfiguration().addProxy(proxy);
}
}
/**
engine/src/main/java/org/archive/crawler/processor/BrowserProcessor.java
+5 -2
View File
@@ -42,6 +42,7 @@ import org.archive.net.webdriver.*;
import org.archive.spring.KeyedProperties;
import org.archive.util.IdleBarrier;
import org.archive.util.Recorder;
import org.eclipse.jetty.client.ProxyConfiguration;
import org.eclipse.jetty.client.Result;
import org.json.JSONException;
import org.json.JSONObject;
@@ -168,7 +169,7 @@ public class BrowserProcessor extends Processor {
String pageId = UUID.randomUUID().toString();
var tab = webdriver.browsingContext().create(BrowsingContext.CreateType.tab).context();
try {
BrowserPage page = new BrowserPage(curi, new IdleBarrier(), webdriver, tab);
BrowserPage page = new BrowserPage(curi, new IdleBarrier(), webdriver, tab, fetcher.getProxy());
pages.put(pageId, page);
pageIdsByContext.put(tab, pageId);
webdriver.network().addIntercept(List.of(Network.InterceptPhase.beforeRequestSent), List.of(tab),
@@ -293,6 +294,7 @@ public class BrowserProcessor extends Processor {
} else {
// Record exchange as a subresource
proxyRequest.setListener(new SubresourceRecorder(page, proxyRequest.url()));
proxyRequest.setUpstreamProxy(page.proxy);
}
}
@@ -477,7 +479,8 @@ public class BrowserProcessor extends Processor {
record BrowserPage(CrawlURI curi,
IdleBarrier networkActivity,
WebDriverBiDi webdriver,
BrowsingContext.Context context) implements Page {
BrowsingContext.Context context,
ProxyConfiguration.Proxy proxy) implements Page {
/**
* Evaluates JavaScript and returns the result as simple Java objects (numbers, strings, maps, lists).
engine/src/test/java/org/archive/crawler/processor/BrowserProcessorTest.java
+34
View File
@@ -8,11 +8,15 @@ import org.archive.modules.fetcher.FetchHTTP2;
import org.archive.net.UURIFactory;
import org.archive.url.URIException;
import org.archive.util.Recorder;
import org.eclipse.jetty.proxy.ProxyHandler;
import org.eclipse.jetty.server.Server;
import org.eclipse.jetty.server.ServerConnector;
import org.junit.jupiter.api.*;
import org.junit.jupiter.api.condition.EnabledIfSystemProperty;
import org.junit.jupiter.api.io.TempDir;
import java.io.IOException;
import java.net.Inet4Address;
import java.net.InetAddress;
import java.net.InetSocketAddress;
import java.nio.file.Path;
@@ -65,6 +69,33 @@ class BrowserProcessorTest {
assertFalse(crawlURI.getAnnotations().contains("browser"), "navigation should have aborted");
}
@Test
public void testHttpProxy() throws Exception {
InetAddress localhost = Inet4Address.getLoopbackAddress();
Server proxyServer = new Server(new InetSocketAddress(localhost, 0));
proxyServer.setHandler(new ProxyHandler.Forward());
proxyServer.start();
try {
fetcher.setHttpProxyHost(localhost.getHostAddress());
fetcher.setHttpProxyPort(((ServerConnector)proxyServer.getConnectors()[0]).getLocalPort());
CrawlURI crawlURI = newCrawlURI(baseUrl);
fetcher.process(crawlURI);
assertEquals(200, crawlURI.getFetchStatus());
browserProcessor.innerProcess(crawlURI);
var outLinks = new ArrayList<>(crawlURI.getOutLinks());
assertEquals("/link", outLinks.get(0).getUURI().getPath());
assertTrue(crawlURI.getAnnotations().contains("browser"));
assertEquals("true", crawlURI.getHttpResponseHeader("Used-Proxy"));
assertEquals("true", subrequests.get(0).getHttpResponseHeader("Used-Proxy"));
logger.log(DEBUG, "Subrequests: {0}", subrequests);
} finally {
proxyServer.stop();
}
}
private CrawlURI newCrawlURI(String uri) throws URIException {
CrawlURI curi = new CrawlURI(UURIFactory.getInstance(uri));
Recorder recorder = new Recorder(tempDir.toFile(), "fetcher");
@@ -114,6 +145,9 @@ class BrowserProcessorTest {
}
default -> status = 404;
}
if (exchange.getRequestHeaders().containsKey("Via")) {
exchange.getResponseHeaders().add("Used-Proxy", "true");
}
exchange.getResponseHeaders().add("Content-Type", contentType);
exchange.sendResponseHeaders(status, 0);
exchange.getResponseBody().write(body.getBytes());
modules/src/main/java/org/archive/modules/fetcher/FetchHTTP2.java
+2 -2
View File
@@ -205,7 +205,7 @@ public class FetchHTTP2 extends Processor implements Lifecycle, InitializingBean
.timeout(getTimeoutSeconds(), TimeUnit.SECONDS)
.method(curi.getFetchType() == CrawlURI.FetchType.HTTP_POST ? HttpMethod.POST : HttpMethod.GET)
.agent(getUserAgentProvider().getUserAgent())
.tag(getHttpProxy());
.tag(getProxy());
if (!curi.getUURI().getScheme().equals("https")) {
request.version(HttpVersion.HTTP_1_1);
} else if (useHTTP3 && curi.getFetchAttempts() == 0) {
@@ -269,7 +269,7 @@ public class FetchHTTP2 extends Processor implements Lifecycle, InitializingBean
kp.put("httpProxyPort", port);
}
private HttpProxy getHttpProxy() {
public ProxyConfiguration.Proxy getProxy() {
String host = getHttpProxyHost();
Integer port = getHttpProxyPort();
if (host == null || port == null) return null;