diff --git a/CHANGELOG.md b/CHANGELOG.md index 910e720d..d095baf2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,9 @@ [Full Changelog](https://github.com/internetarchive/heritrix3/compare/3.13.0...HEAD) +### New features +* **FetchHTTP2:** SOCKS5 proxy support + ## [3.13.0](https://github.com/internetarchive/heritrix3/releases/tag/3.13.0) (2025-12-11) [Download distribution zip](https://repo1.maven.org/maven2/org/archive/heritrix/heritrix/3.13.0/heritrix-3.13.0-dist.zip) (or [tar.gz](https://repo1.maven.org/maven2/org/archive/heritrix/heritrix/3.13.0/heritrix-3.13.0-dist.tar.gz)) diff --git a/modules/src/main/java/org/archive/modules/fetcher/FetchHTTP2.java b/modules/src/main/java/org/archive/modules/fetcher/FetchHTTP2.java index 4bf0896f..dbbfb5c0 100644 --- a/modules/src/main/java/org/archive/modules/fetcher/FetchHTTP2.java +++ b/modules/src/main/java/org/archive/modules/fetcher/FetchHTTP2.java @@ -34,6 +34,7 @@ import org.archive.modules.net.ServerCache; import org.archive.util.Recorder; import org.eclipse.jetty.alpn.client.ALPNClientConnection; import org.eclipse.jetty.client.*; +import org.eclipse.jetty.client.ProxyConfiguration.Proxy; import org.eclipse.jetty.client.transport.HttpClientConnectionFactory; import org.eclipse.jetty.client.transport.HttpClientTransportDynamic; import org.eclipse.jetty.http.*; @@ -93,7 +94,7 @@ public class FetchHTTP2 extends Processor implements Lifecycle, InitializingBean protected String digestAlgorithm = "sha1"; protected boolean useHTTP2 = true; protected boolean useHTTP3 = false; - private final Map httpProxies = new ConcurrentHashMap<>(); + private final Map proxies = new ConcurrentHashMap<>(); public FetchHTTP2(@Autowired ServerCache serverCache, @Autowired(required = false) AbstractCookieStore cookieStore) { this.serverCache = serverCache; @@ -280,25 +281,105 @@ public class FetchHTTP2 extends Processor implements Lifecycle, InitializingBean kp.put("httpProxyPort", port); } - public ProxyConfiguration.Proxy getProxy() { - String host = getHttpProxyHost(); - Integer port = getHttpProxyPort(); - if (host == null || port == null) return null; - return httpProxies.computeIfAbsent(new HttpProxySettings(host, port), this::createHttpProxy); + public String getSocksProxyHost() { + return (String) kp.get("socksProxyHost"); } - private HttpProxy createHttpProxy(HttpProxySettings settings) { - HttpProxy proxy = new HttpProxy(settings.host(), settings.port()) { - @Override - public boolean matches(Origin origin) { - return origin.getTag() == this; - } - }; + /** + * Sets a SOCKS5 proxy host to use. This will override any set HTTP proxy. + */ + public void setSocksProxyHost(String socksProxyHost) { + kp.put("socksProxyHost", socksProxyHost); + } + + public Integer getSocksProxyPort() { + return (Integer) kp.get("socksProxyPort"); + } + /** + * Sets a SOCKS5 proxy port to use. + */ + public void setSocksProxyPort(Integer socksProxyPort) { + kp.put("socksProxyPort", socksProxyPort); + } + + public String getSocksProxyUsername() { + return (String) kp.get("socksProxyUsername"); + } + + /** + * Sets a SOCKS5 proxy username to use (enables username/password authentication). + */ + public void setSocksProxyUsername(String socksProxyUsername) { + kp.put("socksProxyUsername", socksProxyUsername); + } + + public String getSocksProxyPassword() { + return (String) kp.get("socksProxyPassword"); + } + + /** + * Sets a SOCKS5 proxy password to use (enables username/password authentication). + */ + public void setSocksProxyPassword(String socksProxyPassword) { + kp.put("socksProxyPassword", socksProxyPassword); + } + + public Proxy getProxy() { + ProxySettings settings = getProxySettings(); + if (settings == null) return null; + return proxies.computeIfAbsent(settings, this::createProxy); + } + + private ProxySettings getProxySettings() { + String socksHost = getSocksProxyHost(); + Integer socksPort = getSocksProxyPort(); + if (socksHost != null && socksPort != null) { + return new SocksProxySettings(socksHost, socksPort, getSocksProxyUsername(), getSocksProxyPassword()); + } + String host = getHttpProxyHost(); + Integer port = getHttpProxyPort(); + if (host != null && port != null) { + return new HttpProxySettings(host, port); + } + return null; + } + + private Proxy createProxy(ProxySettings settings) { + Proxy proxy = settings.createProxy(); httpClient.getProxyConfiguration().addProxy(proxy); return proxy; } - private record HttpProxySettings(String host, int port) { + private interface ProxySettings { + Proxy createProxy(); + } + + private record HttpProxySettings(String host, int port) implements ProxySettings { + @Override + public Proxy createProxy() { + return new HttpProxy(host(), port()) { + @Override + public boolean matches(Origin origin) { + return origin.getTag() == this; + } + }; + } + } + + private record SocksProxySettings(String host, int port, String username, String password) implements ProxySettings { + @Override + public Proxy createProxy() { + Socks5Proxy proxy = new Socks5Proxy(host(), port()) { + @Override + public boolean matches(Origin origin) { + return origin.getTag() == this; + } + }; + if (username() != null && password() != null) { + proxy.putAuthenticationFactory(new Socks5.UsernamePasswordAuthenticationFactory(username(), password())); + } + return proxy; + } } /** @@ -423,7 +504,7 @@ public class FetchHTTP2 extends Processor implements Lifecycle, InitializingBean // Server IP address var socketAddress = (InetSocketAddress) response.getRequest().getConnection().getRemoteSocketAddress(); - if (socketAddress != null) { + if (socketAddress != null && !(response.getRequest().getTag() instanceof Proxy)) { curi.setServerIP(socketAddress.getAddress().getHostAddress()); } @@ -511,7 +592,7 @@ public class FetchHTTP2 extends Processor implements Lifecycle, InitializingBean throw new RuntimeException(e); } httpClient = null; - httpProxies.clear(); + proxies.clear(); } public UserAgentProvider getUserAgentProvider() { diff --git a/modules/src/test/java/org/archive/modules/fetcher/FetchHTTP2Test.java b/modules/src/test/java/org/archive/modules/fetcher/FetchHTTP2Test.java index 9248b17b..f22d6d5e 100644 --- a/modules/src/test/java/org/archive/modules/fetcher/FetchHTTP2Test.java +++ b/modules/src/test/java/org/archive/modules/fetcher/FetchHTTP2Test.java @@ -25,9 +25,12 @@ import org.archive.modules.CrawlURI; import org.archive.net.UURIFactory; import org.archive.spring.ConfigPath; import org.archive.util.Recorder; +import org.bbottema.javasocksproxyserver.SocksServer; +import org.bbottema.javasocksproxyserver.auth.UsernamePasswordAuthenticator; import org.eclipse.jetty.proxy.ProxyHandler; import org.eclipse.jetty.server.Server; import org.eclipse.jetty.server.ServerConnector; +import org.eclipse.jetty.util.Promise; import org.junit.jupiter.api.*; import org.junit.jupiter.api.io.TempDir; @@ -35,10 +38,17 @@ import java.io.IOException; import java.net.Inet4Address; import java.net.InetAddress; import java.net.InetSocketAddress; +import java.net.ServerSocket; +import java.net.UnknownHostException; import java.nio.file.Files; import java.nio.file.Path; +import java.util.List; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.TimeUnit; import java.util.zip.GZIPOutputStream; +import javax.net.ServerSocketFactory; + import static org.junit.jupiter.api.Assertions.*; public class FetchHTTP2Test { @@ -148,11 +158,89 @@ public class FetchHTTP2Test { assertEquals("true", curi.getHttpResponseHeader("Used-Proxy")); assertEquals(200, curi.getFetchStatus()); assertEquals("Hello World!", curi.getRecorder().getContentReplayPrefixString(100)); + assertNull(curi.getServerIP()); } finally { proxyServer.stop(); } } + @Test + public void testSocksProxy() throws Exception { + int socksPort; + try (ServerSocket socket = new ServerSocket(0, 0, Inet4Address.getLoopbackAddress())) { + socksPort = socket.getLocalPort(); + } + SocksServer socksServer = new SocksServer(socksPort); + socksServer.setAuthenticator(new UsernamePasswordAuthenticator(false) { + @Override + public boolean validate(String username, String password) { + return "user".equals(username) && "pass".equals(password); + } + }); + + CompletableFuture socksServerStartedFuture = new CompletableFuture<>(); + socksServer.setFactory(new ServerSocketFactory() { + final ServerSocketFactory defaultSocketFactory = ServerSocketFactory.getDefault(); + + @Override + public ServerSocket createServerSocket(int port) throws IOException { + return createServerSocket(port, -1); + } + + @Override + public ServerSocket createServerSocket(int port, int backlog) throws IOException { + return createServerSocket(port, backlog, null); + } + + @Override + public ServerSocket createServerSocket(int port, int backlog, InetAddress ifAddress) throws IOException { + try { + ServerSocket serverSocket = defaultSocketFactory.createServerSocket(port, backlog, ifAddress); + socksServerStartedFuture.complete(null); + return serverSocket; + } catch (Throwable e) { + socksServerStartedFuture.completeExceptionally(e); + throw e; + } + } + }); + socksServer.start(); + socksServerStartedFuture.get(30, TimeUnit.SECONDS); + + try { + String socksHost = Inet4Address.getLoopbackAddress().getHostAddress(); + var serverCache = new DefaultServerCache(); + fetcher.stop(); + fetcher = new FetchHTTP2(serverCache, cookieStore) { + @Override + protected void resolveSocketAddress(String host, int port, Promise> promise) { + if (socksHost.equals(host) && socksPort == port) { + promise.succeeded(List.of(new InetSocketAddress(Inet4Address.getLoopbackAddress(), port))); + } else { + promise.failed(new UnknownHostException("Blocked resolution for " + host + ":" + port)); + } + } + }; + fetcher.setUserAgentProvider(new CrawlMetadata()); + + fetcher.setSocksProxyHost(socksHost); + fetcher.setSocksProxyPort(socksPort); + fetcher.setSocksProxyUsername("user"); + fetcher.setSocksProxyPassword("pass"); + fetcher.start(); + + var curi = new CrawlURI(UURIFactory.getInstance("http://localhost:" + server.getAddress().getPort() + "/")); + curi.setRecorder(recorder); + fetcher.innerProcess(curi); + + assertEquals(200, curi.getFetchStatus()); + assertEquals("Hello World!", curi.getRecorder().getContentReplayPrefixString(100)); + assertNull(curi.getServerIP()); + } finally { + socksServer.stop(); + } + } + @Test public void testGzipEncoding() throws Exception { fetcher.start();