diff --git a/commons/src/main/java/org/archive/net/MitmProxy.java b/commons/src/main/java/org/archive/net/MitmProxy.java index 88d2477b..786d0321 100644 --- a/commons/src/main/java/org/archive/net/MitmProxy.java +++ b/commons/src/main/java/org/archive/net/MitmProxy.java @@ -20,6 +20,8 @@ package org.archive.net; import org.eclipse.jetty.client.HttpClient; +import org.eclipse.jetty.client.HttpProxy; +import org.eclipse.jetty.client.ProxyConfiguration; import org.eclipse.jetty.http.HttpHeader; import org.eclipse.jetty.http.HttpURI; import org.eclipse.jetty.io.Content; @@ -42,6 +44,7 @@ import static org.eclipse.jetty.http.HttpHeader.ACCEPT_ENCODING; * An HTTP proxy server which intercepts TLS and records or replays responses. */ public class MitmProxy { + private static final String UPSTREAM_PROXY = MitmProxy.class.getName() + ".upstreamProxy"; private final SslConnectionFactory sslConnectionFactory = new SslConnectionFactory(); private final Server server = new Server(0); private final RequestHandler requestHandler; @@ -63,7 +66,6 @@ public class MitmProxy { public void start() throws Exception { sslConnectionFactory.start(); - server.setHandler(new Handler.Sequence( new SslConnectHandler(), new MitmProxyHandler())); @@ -90,6 +92,10 @@ public class MitmProxy { body.transferTo(Content.Sink.asOutputStream(response)); callback().succeeded(); } + + public void setUpstreamProxy(ProxyConfiguration.Proxy proxy) { + request.setAttribute(UPSTREAM_PROXY, proxy); + } } public interface RequestHandler { @@ -156,8 +162,16 @@ public class MitmProxy { // Host header is not allowed in HTTP/2 headers.remove(HttpHeader.HOST); }); + + ProxyConfiguration.Proxy upstreamProxy = (HttpProxy)clientToProxyRequest.getAttribute(UPSTREAM_PROXY); + if (upstreamProxy != null) { + addUpstreamProxyIfAbsent(upstreamProxy); + proxyToServerRequest.tag(upstreamProxy); + } + var listener = (ExchangeListener)clientToProxyRequest.getAttribute(ExchangeListener.class.getName()); if (listener != null) { + proxyToServerRequest.onRequestBegin(listener); proxyToServerRequest.onRequestHeaders(listener); proxyToServerRequest.onRequestContent(listener); proxyToServerRequest.onResponseHeaders(listener); @@ -165,6 +179,13 @@ public class MitmProxy { proxyToServerRequest.onComplete(listener); } } + + private void addUpstreamProxyIfAbsent(ProxyConfiguration.Proxy proxy) { + for (var existingProxy : getHttpClient().getProxyConfiguration().getProxies()) { + if (existingProxy == proxy) return; + } + getHttpClient().getProxyConfiguration().addProxy(proxy); + } } /** diff --git a/engine/src/main/java/org/archive/crawler/processor/BrowserProcessor.java b/engine/src/main/java/org/archive/crawler/processor/BrowserProcessor.java index 84b2c112..6ae7b982 100644 --- a/engine/src/main/java/org/archive/crawler/processor/BrowserProcessor.java +++ b/engine/src/main/java/org/archive/crawler/processor/BrowserProcessor.java @@ -42,6 +42,7 @@ import org.archive.net.webdriver.*; import org.archive.spring.KeyedProperties; import org.archive.util.IdleBarrier; import org.archive.util.Recorder; +import org.eclipse.jetty.client.ProxyConfiguration; import org.eclipse.jetty.client.Result; import org.json.JSONException; import org.json.JSONObject; @@ -168,7 +169,7 @@ public class BrowserProcessor extends Processor { String pageId = UUID.randomUUID().toString(); var tab = webdriver.browsingContext().create(BrowsingContext.CreateType.tab).context(); try { - BrowserPage page = new BrowserPage(curi, new IdleBarrier(), webdriver, tab); + BrowserPage page = new BrowserPage(curi, new IdleBarrier(), webdriver, tab, fetcher.getProxy()); pages.put(pageId, page); pageIdsByContext.put(tab, pageId); webdriver.network().addIntercept(List.of(Network.InterceptPhase.beforeRequestSent), List.of(tab), @@ -293,6 +294,7 @@ public class BrowserProcessor extends Processor { } else { // Record exchange as a subresource proxyRequest.setListener(new SubresourceRecorder(page, proxyRequest.url())); + proxyRequest.setUpstreamProxy(page.proxy); } } @@ -477,7 +479,8 @@ public class BrowserProcessor extends Processor { record BrowserPage(CrawlURI curi, IdleBarrier networkActivity, WebDriverBiDi webdriver, - BrowsingContext.Context context) implements Page { + BrowsingContext.Context context, + ProxyConfiguration.Proxy proxy) implements Page { /** * Evaluates JavaScript and returns the result as simple Java objects (numbers, strings, maps, lists). diff --git a/engine/src/test/java/org/archive/crawler/processor/BrowserProcessorTest.java b/engine/src/test/java/org/archive/crawler/processor/BrowserProcessorTest.java index a98a1923..1e485e8b 100644 --- a/engine/src/test/java/org/archive/crawler/processor/BrowserProcessorTest.java +++ b/engine/src/test/java/org/archive/crawler/processor/BrowserProcessorTest.java @@ -8,11 +8,15 @@ import org.archive.modules.fetcher.FetchHTTP2; import org.archive.net.UURIFactory; import org.archive.url.URIException; import org.archive.util.Recorder; +import org.eclipse.jetty.proxy.ProxyHandler; +import org.eclipse.jetty.server.Server; +import org.eclipse.jetty.server.ServerConnector; import org.junit.jupiter.api.*; import org.junit.jupiter.api.condition.EnabledIfSystemProperty; import org.junit.jupiter.api.io.TempDir; import java.io.IOException; +import java.net.Inet4Address; import java.net.InetAddress; import java.net.InetSocketAddress; import java.nio.file.Path; @@ -65,6 +69,33 @@ class BrowserProcessorTest { assertFalse(crawlURI.getAnnotations().contains("browser"), "navigation should have aborted"); } + @Test + public void testHttpProxy() throws Exception { + InetAddress localhost = Inet4Address.getLoopbackAddress(); + Server proxyServer = new Server(new InetSocketAddress(localhost, 0)); + proxyServer.setHandler(new ProxyHandler.Forward()); + proxyServer.start(); + try { + fetcher.setHttpProxyHost(localhost.getHostAddress()); + fetcher.setHttpProxyPort(((ServerConnector)proxyServer.getConnectors()[0]).getLocalPort()); + + CrawlURI crawlURI = newCrawlURI(baseUrl); + fetcher.process(crawlURI); + assertEquals(200, crawlURI.getFetchStatus()); + browserProcessor.innerProcess(crawlURI); + + var outLinks = new ArrayList<>(crawlURI.getOutLinks()); + assertEquals("/link", outLinks.get(0).getUURI().getPath()); + assertTrue(crawlURI.getAnnotations().contains("browser")); + assertEquals("true", crawlURI.getHttpResponseHeader("Used-Proxy")); + assertEquals("true", subrequests.get(0).getHttpResponseHeader("Used-Proxy")); + + logger.log(DEBUG, "Subrequests: {0}", subrequests); + } finally { + proxyServer.stop(); + } + } + private CrawlURI newCrawlURI(String uri) throws URIException { CrawlURI curi = new CrawlURI(UURIFactory.getInstance(uri)); Recorder recorder = new Recorder(tempDir.toFile(), "fetcher"); @@ -114,6 +145,9 @@ class BrowserProcessorTest { } default -> status = 404; } + if (exchange.getRequestHeaders().containsKey("Via")) { + exchange.getResponseHeaders().add("Used-Proxy", "true"); + } exchange.getResponseHeaders().add("Content-Type", contentType); exchange.sendResponseHeaders(status, 0); exchange.getResponseBody().write(body.getBytes()); diff --git a/modules/src/main/java/org/archive/modules/fetcher/FetchHTTP2.java b/modules/src/main/java/org/archive/modules/fetcher/FetchHTTP2.java index 2efa4257..66e89f16 100644 --- a/modules/src/main/java/org/archive/modules/fetcher/FetchHTTP2.java +++ b/modules/src/main/java/org/archive/modules/fetcher/FetchHTTP2.java @@ -205,7 +205,7 @@ public class FetchHTTP2 extends Processor implements Lifecycle, InitializingBean .timeout(getTimeoutSeconds(), TimeUnit.SECONDS) .method(curi.getFetchType() == CrawlURI.FetchType.HTTP_POST ? HttpMethod.POST : HttpMethod.GET) .agent(getUserAgentProvider().getUserAgent()) - .tag(getHttpProxy()); + .tag(getProxy()); if (!curi.getUURI().getScheme().equals("https")) { request.version(HttpVersion.HTTP_1_1); } else if (useHTTP3 && curi.getFetchAttempts() == 0) { @@ -269,7 +269,7 @@ public class FetchHTTP2 extends Processor implements Lifecycle, InitializingBean kp.put("httpProxyPort", port); } - private HttpProxy getHttpProxy() { + public ProxyConfiguration.Proxy getProxy() { String host = getHttpProxyHost(); Integer port = getHttpProxyPort(); if (host == null || port == null) return null;