From 6b72ac9705ba73825871f4aa8003ce58a218329c Mon Sep 17 00:00:00 2001
From: Noah Levitt <nlevitt@archive.org>
Date: Fri, 24 Jan 2014 16:07:04 -0800
Subject: [PATCH] Switch to BasicHttpClientConnectionManager instead of
 PoolingHttpClientConnectionManager, and make the connection manager a member
 variable of HttpRequest. We had been seeing a strange bug where once in every
 few thousand urls or so, the connection manager would get gc'd while the http
 request was being sent, and RecordingOutputStream.flush() would get called,
 as in the stack trace below.  The ToeThread was using it at the same time, so
 if conditions were right, the recorded stream would get mangled. This commit
 makes the connection manager a member of FetchHTTPRequest so it doesn't get
 gc'd prematurely. Also switches to BasicHttpClientConnectionManager, since
 we're not doing any pooling.

  it.unimi.dsi.fastutil.io.FastBufferedOutputStream.dumpBuffer(boolean) line: 133
  it.unimi.dsi.fastutil.io.FastBufferedOutputStream.flush() line: 166
  org.archive.io.RecordingOutputStream.flush() line: 469
  org.apache.http.impl.io.SessionOutputBufferImpl.flushStream() line: 128
  org.apache.http.impl.io.SessionOutputBufferImpl.flush() line: 143
  org.archive.modules.fetcher.FetchHTTPRequest$RecordingHttpClientConnection(org.apache.http.impl.BHttpConnectionBase).close() line: 314
  org.archive.modules.fetcher.FetchHTTPRequest$RecordingHttpClientConnection.close() line: 520
  org.apache.http.impl.conn.CPoolEntry.closeConnection() line: 69
  org.apache.http.impl.conn.CPoolEntry.close() line: 95
  org.apache.http.impl.conn.CPool(org.apache.http.pool.AbstractConnPool<T,C,E>).shutdown() line: 130
  org.apache.http.impl.conn.PoolingHttpClientConnectionManager.shutdown() line: 347
  org.apache.http.impl.conn.PoolingHttpClientConnectionManager.finalize() line: 168
  java.lang.ref.Finalizer.invokeFinalizeMethod(java.lang.Object) line: not available [native method]
---
 .../archive/modules/fetcher/FetchHTTPRequest.java   | 13 ++++++++-----
 1 file changed, 8 insertions(+), 5 deletions(-)

diff --git a/modules/src/main/java/org/archive/modules/fetcher/FetchHTTPRequest.java b/modules/src/main/java/org/archive/modules/fetcher/FetchHTTPRequest.java
index b1b5f745..b5318067 100644
--- a/modules/src/main/java/org/archive/modules/fetcher/FetchHTTPRequest.java
+++ b/modules/src/main/java/org/archive/modules/fetcher/FetchHTTPRequest.java
@@ -81,9 +81,9 @@ import org.apache.http.impl.DefaultBHttpClientConnection;
 import org.apache.http.impl.client.BasicAuthCache;
 import org.apache.http.impl.client.BasicCredentialsProvider;
 import org.apache.http.impl.client.HttpClientBuilder;
+import org.apache.http.impl.conn.BasicHttpClientConnectionManager;
 import org.apache.http.impl.conn.DefaultHttpResponseParserFactory;
 import org.apache.http.impl.conn.ManagedHttpClientConnectionFactory;
-import org.apache.http.impl.conn.PoolingHttpClientConnectionManager;
 import org.apache.http.impl.io.DefaultHttpRequestWriterFactory;
 import org.apache.http.io.HttpMessageParserFactory;
 import org.apache.http.io.HttpMessageWriterFactory;
@@ -144,6 +144,8 @@ class FetchHTTPRequest {
     protected HttpHost targetHost;
     protected boolean addedCredentials;
     protected HttpHost proxyHost;
+    // make this a member variable so it doesn't get gc'd prematurely
+    protected HttpClientConnectionManager connMan;
 
     public FetchHTTPRequest(FetchHTTP fetcher, CrawlURI curi) throws URIException {
         this.fetcher = fetcher;
@@ -423,8 +425,8 @@ class FetchHTTPRequest {
         
         httpClientBuilder.setDefaultCookieStore(fetcher.getCookieStore());
         
-        HttpClientConnectionManager connManager = buildConnectionManager();
-        httpClientBuilder.setConnectionManager(connManager);
+        connMan = buildConnectionManager();
+        httpClientBuilder.setConnectionManager(connMan);
     }
 
     protected HttpClientConnectionManager buildConnectionManager() {
@@ -464,11 +466,12 @@ class FetchHTTPRequest {
                         DefaultHttpResponseParserFactory.INSTANCE);
             }
         };
-        PoolingHttpClientConnectionManager connMan = new PoolingHttpClientConnectionManager(socketFactoryRegistry, connFactory, dnsResolver);
+        BasicHttpClientConnectionManager connMan = new BasicHttpClientConnectionManager(
+                socketFactoryRegistry, connFactory, null, dnsResolver);
         
         SocketConfig.Builder socketConfigBuilder = SocketConfig.custom();
         socketConfigBuilder.setSoTimeout(fetcher.getSoTimeoutMs());
-        connMan.setDefaultSocketConfig(socketConfigBuilder.build());
+        connMan.setSocketConfig(socketConfigBuilder.build());
         
         return connMan;
     }