From cb725bb38ed5e73ccf45745648ca9164a63afcff Mon Sep 17 00:00:00 2001
From: Noah Levitt <nlevitt@archive.org>
Date: Sun, 30 Dec 2012 19:05:55 -0800
Subject: [PATCH] Fix socket timeout

---
 .../archive/modules/fetcher/FetchHTTP.java    |  9 ++------
 .../modules/fetcher/FetchHTTPRequest.java     | 21 +++++++++++++++----
 2 files changed, 19 insertions(+), 11 deletions(-)

diff --git a/modules/src/main/java/org/archive/modules/fetcher/FetchHTTP.java b/modules/src/main/java/org/archive/modules/fetcher/FetchHTTP.java
index fd361a4e..6731d7f8 100644
--- a/modules/src/main/java/org/archive/modules/fetcher/FetchHTTP.java
+++ b/modules/src/main/java/org/archive/modules/fetcher/FetchHTTP.java
@@ -640,13 +640,8 @@ public class FetchHTTP extends Processor implements Lifecycle {
             return;
         }
         
-        // set hardMax on bytes (if set by operator)
-        long hardMax = getMaxLengthBytes();
-        // set overall timeout (if set by operator)
-        long timeoutMs = 1000 * getTimeoutSeconds();
-        // Get max fetch rate (bytes/ms). It comes in in KB/sec
-        long maxRateKBps = getMaxFetchKBSec();
-        rec.getRecordedInput().setLimits(hardMax, timeoutMs, maxRateKBps);
+        rec.getRecordedInput().setLimits(getMaxLengthBytes(),
+                1000l * (long) getTimeoutSeconds(), (long) getMaxFetchKBSec());
 
         HttpResponse response = null;
         try {
diff --git a/modules/src/main/java/org/archive/modules/fetcher/FetchHTTPRequest.java b/modules/src/main/java/org/archive/modules/fetcher/FetchHTTPRequest.java
index 2f2097d5..72d04ee7 100644
--- a/modules/src/main/java/org/archive/modules/fetcher/FetchHTTPRequest.java
+++ b/modules/src/main/java/org/archive/modules/fetcher/FetchHTTPRequest.java
@@ -61,6 +61,7 @@ import org.apache.http.client.protocol.HttpClientContext;
 import org.apache.http.config.MessageConstraints;
 import org.apache.http.config.Registry;
 import org.apache.http.config.RegistryBuilder;
+import org.apache.http.config.SocketConfig;
 import org.apache.http.conn.DnsResolver;
 import org.apache.http.conn.HttpClientConnectionManager;
 import org.apache.http.conn.SocketClientConnection;
@@ -311,6 +312,12 @@ public class FetchHTTPRequest {
 
         configBuilder.setConnectionRequestTimeout(fetcher.getSoTimeoutMs());
         configBuilder.setConnectTimeout(fetcher.getSoTimeoutMs());
+
+        /*
+         * XXX This socket timeout seems to be ignored. The one on the
+         * socketConfig on the PoolingHttpClientConnectionManager in the
+         * HttpClientBuilder is respected.
+         */
         configBuilder.setSocketTimeout(fetcher.getSoTimeoutMs());        
 
         // local bind address
@@ -455,11 +462,11 @@ public class FetchHTTPRequest {
         
         httpClientBuilder.setCookieStore(fetcher.getCookieStore());
         
-        HttpClientConnectionManager connManager = makeConnectionManager();
+        HttpClientConnectionManager connManager = buildConnectionManager();
         httpClientBuilder.setConnectionManager(connManager);
     }
 
-    protected HttpClientConnectionManager makeConnectionManager() {
+    protected HttpClientConnectionManager buildConnectionManager() {
         Registry<ConnectionSocketFactory> socketFactoryRegistry = RegistryBuilder.<ConnectionSocketFactory>create()
                 .register("http", PlainSocketFactory.getSocketFactory())
                 .register("https", new SSLSocketFactory(fetcher.sslContext(), new AllowAllHostnameVerifier()))
@@ -479,9 +486,15 @@ public class FetchHTTPRequest {
 
         DnsResolver dnsResolver = new ServerCacheResolver(fetcher.getServerCache());
         
-        return new PoolingHttpClientConnectionManager(socketFactoryRegistry,
+        PoolingHttpClientConnectionManager connMan = new PoolingHttpClientConnectionManager(socketFactoryRegistry,
                 connFactory, null, dnsResolver, -1, TimeUnit.MILLISECONDS);
-    }    
+        
+        SocketConfig.Builder socketConfigBuilder = SocketConfig.custom();
+        socketConfigBuilder.setSoTimeout(fetcher.getSoTimeoutMs());
+        connMan.setDefaultSocketConfig(socketConfigBuilder.build());
+        
+        return connMan;
+    }
     
     protected void initHttpClientBuilder() {
         httpClientBuilder = HttpClientBuilder.create();