Merge pull request #710 from internetarchive/ato/fetchhttp2-socks

FetchHTTP2: Add SOCKS5 proxy support

CHANGELOG.md

@@ -4,6 +4,9 @@
 
 [Full Changelog](https://github.com/internetarchive/heritrix3/compare/3.13.0...HEAD)
 
+### New features
+* **FetchHTTP2:** SOCKS5 proxy support
+
 ## [3.13.0](https://github.com/internetarchive/heritrix3/releases/tag/3.13.0)  (2025-12-11)
 
 [Download distribution zip](https://repo1.maven.org/maven2/org/archive/heritrix/heritrix/3.13.0/heritrix-3.13.0-dist.zip) (or [tar.gz](https://repo1.maven.org/maven2/org/archive/heritrix/heritrix/3.13.0/heritrix-3.13.0-dist.tar.gz))

modules/src/main/java/org/archive/modules/fetcher/FetchHTTP2.java

@@ -34,6 +34,7 @@ import org.archive.modules.net.ServerCache;
 import org.archive.util.Recorder;
 import org.eclipse.jetty.alpn.client.ALPNClientConnection;
 import org.eclipse.jetty.client.*;
+import org.eclipse.jetty.client.ProxyConfiguration.Proxy;
 import org.eclipse.jetty.client.transport.HttpClientConnectionFactory;
 import org.eclipse.jetty.client.transport.HttpClientTransportDynamic;
 import org.eclipse.jetty.http.*;
@@ -93,7 +94,7 @@ public class FetchHTTP2 extends Processor implements Lifecycle, InitializingBean
     protected String digestAlgorithm = "sha1";
     protected boolean useHTTP2 = true;
     protected boolean useHTTP3 = false;
-    private final Map<HttpProxySettings, HttpProxy> httpProxies = new ConcurrentHashMap<>();
+    private final Map<ProxySettings, Proxy> proxies = new ConcurrentHashMap<>();
 
     public FetchHTTP2(@Autowired ServerCache serverCache, @Autowired(required = false) AbstractCookieStore cookieStore) {
         this.serverCache = serverCache;
@@ -280,25 +281,105 @@ public class FetchHTTP2 extends Processor implements Lifecycle, InitializingBean
         kp.put("httpProxyPort", port);
     }
 
-    public ProxyConfiguration.Proxy getProxy() {
-        String host = getHttpProxyHost();
-        Integer port = getHttpProxyPort();
-        if (host == null || port == null) return null;
-        return httpProxies.computeIfAbsent(new HttpProxySettings(host, port), this::createHttpProxy);
+    public String getSocksProxyHost() {
+        return (String) kp.get("socksProxyHost");
     }
 
-    private HttpProxy createHttpProxy(HttpProxySettings settings) {
-        HttpProxy proxy = new HttpProxy(settings.host(), settings.port()) {
-            @Override
-            public boolean matches(Origin origin) {
-                return origin.getTag() == this;
-            }
-        };
+    /**
+     * Sets a SOCKS5 proxy host to use. This will override any set HTTP proxy.
+     */
+    public void setSocksProxyHost(String socksProxyHost) {
+        kp.put("socksProxyHost", socksProxyHost);
+    }
+
+    public Integer getSocksProxyPort() {
+        return (Integer) kp.get("socksProxyPort");
+    }
+    /**
+     * Sets a SOCKS5 proxy port to use.
+     */
+    public void setSocksProxyPort(Integer socksProxyPort) {
+        kp.put("socksProxyPort", socksProxyPort);
+    }
+
+    public String getSocksProxyUsername() {
+        return (String) kp.get("socksProxyUsername");
+    }
+
+    /**
+     * Sets a SOCKS5 proxy username to use (enables username/password authentication).
+     */
+    public void setSocksProxyUsername(String socksProxyUsername) {
+        kp.put("socksProxyUsername", socksProxyUsername);
+    }
+
+    public String getSocksProxyPassword() {
+        return (String) kp.get("socksProxyPassword");
+    }
+
+    /**
+     * Sets a SOCKS5 proxy password to use (enables username/password authentication).
+     */
+    public void setSocksProxyPassword(String socksProxyPassword) {
+        kp.put("socksProxyPassword", socksProxyPassword);
+    }
+
+    public Proxy getProxy() {
+        ProxySettings settings = getProxySettings();
+        if (settings == null) return null;
+        return proxies.computeIfAbsent(settings, this::createProxy);
+    }
+
+    private ProxySettings getProxySettings() {
+        String socksHost = getSocksProxyHost();
+        Integer socksPort = getSocksProxyPort();
+        if (socksHost != null && socksPort != null) {
+            return new SocksProxySettings(socksHost, socksPort, getSocksProxyUsername(), getSocksProxyPassword());
+        }
+        String host = getHttpProxyHost();
+        Integer port = getHttpProxyPort();
+        if (host != null && port != null) {
+            return new HttpProxySettings(host, port);
+        }
+        return null;
+    }
+
+    private Proxy createProxy(ProxySettings settings) {
+        Proxy proxy = settings.createProxy();
         httpClient.getProxyConfiguration().addProxy(proxy);
         return proxy;
     }
 
-    private record HttpProxySettings(String host, int port) {
+    private interface ProxySettings {
+        Proxy createProxy();
+    }
+
+    private record HttpProxySettings(String host, int port) implements ProxySettings {
+        @Override
+        public Proxy createProxy() {
+            return new HttpProxy(host(), port()) {
+                @Override
+                public boolean matches(Origin origin) {
+                    return origin.getTag() == this;
+                }
+            };
+        }
+    }
+
+    private record SocksProxySettings(String host, int port, String username, String password) implements ProxySettings {
+        @Override
+        public Proxy createProxy() {
+            Socks5Proxy proxy = new Socks5Proxy(host(), port()) {
+                @Override
+                public boolean matches(Origin origin) {
+                    return origin.getTag() == this;
+                }
+            };
+            if (username() != null && password() != null) {
+                proxy.putAuthenticationFactory(new Socks5.UsernamePasswordAuthenticationFactory(username(), password()));
+            }
+            return proxy;
+        }
     }
 
     /**
@@ -423,7 +504,7 @@ public class FetchHTTP2 extends Processor implements Lifecycle, InitializingBean
 
         // Server IP address
         var socketAddress = (InetSocketAddress) response.getRequest().getConnection().getRemoteSocketAddress();
-        if (socketAddress != null) {
+        if (socketAddress != null && !(response.getRequest().getTag() instanceof Proxy)) {
             curi.setServerIP(socketAddress.getAddress().getHostAddress());
         }
 
@@ -511,7 +592,7 @@ public class FetchHTTP2 extends Processor implements Lifecycle, InitializingBean
             throw new RuntimeException(e);
         }
         httpClient = null;
-        httpProxies.clear();
+        proxies.clear();
     }
 
     public UserAgentProvider getUserAgentProvider() {

modules/src/test/java/org/archive/modules/fetcher/FetchHTTP2Test.java

@@ -25,9 +25,12 @@ import org.archive.modules.CrawlURI;
 import org.archive.net.UURIFactory;
 import org.archive.spring.ConfigPath;
 import org.archive.util.Recorder;
+import org.bbottema.javasocksproxyserver.SocksServer;
+import org.bbottema.javasocksproxyserver.auth.UsernamePasswordAuthenticator;
 import org.eclipse.jetty.proxy.ProxyHandler;
 import org.eclipse.jetty.server.Server;
 import org.eclipse.jetty.server.ServerConnector;
+import org.eclipse.jetty.util.Promise;
 import org.junit.jupiter.api.*;
 import org.junit.jupiter.api.io.TempDir;
 
@@ -35,10 +38,17 @@ import java.io.IOException;
 import java.net.Inet4Address;
 import java.net.InetAddress;
 import java.net.InetSocketAddress;
+import java.net.ServerSocket;
+import java.net.UnknownHostException;
 import java.nio.file.Files;
 import java.nio.file.Path;
+import java.util.List;
+import java.util.concurrent.CompletableFuture;
+import java.util.concurrent.TimeUnit;
 import java.util.zip.GZIPOutputStream;
 
+import javax.net.ServerSocketFactory;
+
 import static org.junit.jupiter.api.Assertions.*;
 
 public class FetchHTTP2Test {
@@ -148,11 +158,89 @@ public class FetchHTTP2Test {
             assertEquals("true", curi.getHttpResponseHeader("Used-Proxy"));
             assertEquals(200, curi.getFetchStatus());
             assertEquals("Hello World!", curi.getRecorder().getContentReplayPrefixString(100));
+            assertNull(curi.getServerIP());
         } finally {
             proxyServer.stop();
         }
     }
 
+    @Test
+    public void testSocksProxy() throws Exception {
+        int socksPort;
+        try (ServerSocket socket = new ServerSocket(0, 0, Inet4Address.getLoopbackAddress())) {
+            socksPort = socket.getLocalPort();
+        }
+        SocksServer socksServer = new SocksServer(socksPort);
+        socksServer.setAuthenticator(new UsernamePasswordAuthenticator(false) {
+            @Override
+            public boolean validate(String username, String password) {
+                return "user".equals(username) && "pass".equals(password);
+            }
+        });
+
+        CompletableFuture<Void> socksServerStartedFuture = new CompletableFuture<>();
+        socksServer.setFactory(new ServerSocketFactory() {
+            final ServerSocketFactory defaultSocketFactory = ServerSocketFactory.getDefault();
+
+            @Override
+            public ServerSocket createServerSocket(int port) throws IOException {
+                return createServerSocket(port, -1);
+            }
+
+            @Override
+            public ServerSocket createServerSocket(int port, int backlog) throws IOException {
+                return createServerSocket(port, backlog, null);
+            }
+
+            @Override
+            public ServerSocket createServerSocket(int port, int backlog, InetAddress ifAddress) throws IOException {
+                try {
+                    ServerSocket serverSocket = defaultSocketFactory.createServerSocket(port, backlog, ifAddress);
+                    socksServerStartedFuture.complete(null);
+                    return serverSocket;
+                } catch (Throwable e) {
+                    socksServerStartedFuture.completeExceptionally(e);
+                    throw e;
+                }
+            }
+        });
+        socksServer.start();
+        socksServerStartedFuture.get(30, TimeUnit.SECONDS);
+
+        try {
+            String socksHost = Inet4Address.getLoopbackAddress().getHostAddress();
+            var serverCache = new DefaultServerCache();
+            fetcher.stop();
+            fetcher = new FetchHTTP2(serverCache, cookieStore) {
+                @Override
+                protected void resolveSocketAddress(String host, int port, Promise<List<InetSocketAddress>> promise) {
+                    if (socksHost.equals(host) && socksPort == port) {
+                        promise.succeeded(List.of(new InetSocketAddress(Inet4Address.getLoopbackAddress(), port)));
+                    } else {
+                        promise.failed(new UnknownHostException("Blocked resolution for " + host + ":" + port));
+                    }
+                }
+            };
+            fetcher.setUserAgentProvider(new CrawlMetadata());
+
+            fetcher.setSocksProxyHost(socksHost);
+            fetcher.setSocksProxyPort(socksPort);
+            fetcher.setSocksProxyUsername("user");
+            fetcher.setSocksProxyPassword("pass");
+            fetcher.start();
+
+            var curi = new CrawlURI(UURIFactory.getInstance("http://localhost:" + server.getAddress().getPort() + "/"));
+            curi.setRecorder(recorder);
+            fetcher.innerProcess(curi);
+
+            assertEquals(200, curi.getFetchStatus());
+            assertEquals("Hello World!", curi.getRecorder().getContentReplayPrefixString(100));
+            assertNull(curi.getServerIP());
+        } finally {
+            socksServer.stop();
+        }
+    }
+
     @Test
     public void testGzipEncoding() throws Exception {
         fetcher.start();