From 948d641b1460c0d5a1e66cd4c9ac00a12bb9e831 Mon Sep 17 00:00:00 2001
From: Noah Levitt <nlevitt@archive.org>
Date: Thu, 6 Sep 2012 12:39:44 -0700
Subject: [PATCH] socket/connection timeout

---
 .../modules/fetcher/FetchHTTPTestBase.java    | 30 +++++++++++++++++--
 .../archive/modules/fetcher/FetchHTTP2.java   |  9 ++++--
 2 files changed, 35 insertions(+), 4 deletions(-)

diff --git a/engine/src/test/java/org/archive/modules/fetcher/FetchHTTPTestBase.java b/engine/src/test/java/org/archive/modules/fetcher/FetchHTTPTestBase.java
index 41270c45..a9ceadde 100644
--- a/engine/src/test/java/org/archive/modules/fetcher/FetchHTTPTestBase.java
+++ b/engine/src/test/java/org/archive/modules/fetcher/FetchHTTPTestBase.java
@@ -19,6 +19,7 @@
 package org.archive.modules.fetcher;
 
 import java.io.IOException;
+import java.io.InterruptedIOException;
 import java.io.UnsupportedEncodingException;
 import java.nio.charset.Charset;
 import java.util.Arrays;
@@ -852,8 +853,33 @@ public abstract class FetchHTTPTestBase extends ProcessorTestBase {
         getFetcher().setTimeoutSeconds(2);
         getFetcher().process(curi);
         
-        logger.info('\n' + httpRequestString(curi) + "\n\n" + rawResponseString(curi));
+        // logger.info('\n' + httpRequestString(curi) + "\n\n" + rawResponseString(curi));
         assertTrue(curi.getAnnotations().contains("timeTrunc"));
-        assertTrue(curi.getFetchDuration() > 2000 && curi.getFetchDuration() < 2200);
+        assertTrue(curi.getFetchDuration() >= 2000 && curi.getFetchDuration() < 2200);
     }
+
+    // see http://stackoverflow.com/questions/100841/artificially-create-a-connection-timeout-error
+    public void testConnectionTimeout() throws Exception {
+        CrawlURI curi = makeCrawlURI("http://10.255.255.1/");
+        getFetcher().setSoTimeoutMs(300);
+        
+        long start = System.currentTimeMillis();
+        getFetcher().process(curi);
+        long elapsed = System.currentTimeMillis() - start;
+        
+        assertTrue(elapsed >= 300 && elapsed < 400);
+        
+        // Httpcomponents throws org.apache.http.conn.ConnectTimeoutException,
+        // commons-httpclient throws java.net.SocketTimeoutException. Both are
+        // instances of InterruptedIOException
+        assertEquals(1, curi.getNonFatalFailures().size());
+        assertTrue(curi.getNonFatalFailures().toArray()[0] instanceof InterruptedIOException);
+        assertTrue(curi.getNonFatalFailures().toArray()[0].toString().matches("(?i).*connect.*timed out.*"));
+
+        assertEquals(FetchStatusCodes.S_CONNECT_FAILED, curi.getFetchStatus());
+        
+        assertEquals(0, curi.getFetchCompletedTime());
+    }
+    
+    // XXX testSocketTimeout() (the other kind) - how to simulate?
 }
diff --git a/modules/src/main/java/org/archive/modules/fetcher/FetchHTTP2.java b/modules/src/main/java/org/archive/modules/fetcher/FetchHTTP2.java
index c3ab78cf..a20357da 100644
--- a/modules/src/main/java/org/archive/modules/fetcher/FetchHTTP2.java
+++ b/modules/src/main/java/org/archive/modules/fetcher/FetchHTTP2.java
@@ -75,6 +75,7 @@ import org.apache.http.entity.ContentType;
 import org.apache.http.impl.client.BasicAuthCache;
 import org.apache.http.message.BasicHeader;
 import org.apache.http.message.BasicNameValuePair;
+import org.apache.http.params.HttpConnectionParams;
 import org.apache.http.params.HttpProtocolParams;
 import org.apache.http.protocol.BasicHttpContext;
 import org.apache.http.protocol.HTTP;
@@ -1099,6 +1100,9 @@ public class FetchHTTP2 extends AbstractFetchHTTP implements Lifecycle {
                     A_ETAG_HEADER, "If-None-Match");
         }
 
+        HttpConnectionParams.setConnectionTimeout(request.getParams(), getSoTimeoutMs());
+        HttpConnectionParams.setSoTimeout(request.getParams(), getSoTimeoutMs());
+
         // TODO: What happens if below method adds a header already
         // added above: e.g. Connection, Range, or Referer?
         configureAcceptHeaders(request);
@@ -1297,9 +1301,10 @@ public class FetchHTTP2 extends AbstractFetchHTTP implements Lifecycle {
      */
     protected void cleanup(final CrawlURI curi, final Exception exception,
             final String message, final int status) {
-        logger.log(Level.WARNING, message, exception);
+        if (logger.isLoggable(Level.FINE)) {
+            logger.fine(message + ": " + exception);
+        }
         
-        // message ignored!
         curi.getNonFatalFailures().add(exception);
         curi.setFetchStatus(status);
         curi.getRecorder().close();