Close recorders on failure to avoid exception from retry cycle within httpclient- now all the FetchHTTPTests pass

modules/src/main/java/org/archive/modules/fetcher/FetchHTTP.java

@@ -648,10 +648,10 @@ public class FetchHTTP extends Processor implements Lifecycle {
             response = req.execute();
             addResponseContent(response, curi);
         } catch (ClientProtocolException e) {
-            failedExecuteCleanup(req.request, curi, e);
+            failedExecuteCleanup(curi, e);
             return;
         } catch (IOException e) {
-            failedExecuteCleanup(req.request, curi, e);
+            failedExecuteCleanup(curi, e);
             return;
         }
         
@@ -684,9 +684,6 @@ public class FetchHTTP extends Processor implements Lifecycle {
             rec.close();
             // ensure recording has stopped
             rec.closeRecorders();
-            if (!req.request.isAborted()) {
-                req.request.reset();
-            }
             // Note completion time
             curi.setFetchCompletedTime(System.currentTimeMillis());
             
@@ -991,17 +988,14 @@ public class FetchHTTP extends Processor implements Lifecycle {
      * 
      * @param curi
      *            CrawlURI we failed on.
-     * @param request
-     *            Method we failed on.
      * @param exception
      *            Exception we failed with.
      */
-    protected void failedExecuteCleanup(final AbortableHttpRequestBase request,
-            final CrawlURI curi, final Exception exception) {
+    protected void failedExecuteCleanup(final CrawlURI curi,
+            final Exception exception) {
         cleanup(curi, exception, "executeMethod", S_CONNECT_FAILED);
-        request.reset();
     }
-    
+
     /**
      * Cleanup after a failed method execute.
      * 

modules/src/main/java/org/archive/modules/fetcher/FetchHTTPRequest.java

@@ -134,6 +134,22 @@ public class FetchHTTPRequest {
                 super.receiveResponseEntity(response);
             }
         }
+        
+        @Override
+        public void shutdown() throws IOException {
+            super.shutdown();
+
+            /*
+             * Need to do this to avoid "java.io.IOException: RIS already open"
+             * on urls that are retried within httpcomponents. Exercised by
+             * FetchHTTPTests.testNoResponse()
+             */
+            Recorder recorder = Recorder.getHttpRecorder();
+            if (recorder != null) {
+                recorder.close();
+                recorder.closeRecorders();
+            }
+        }
     }
     
     /**

modules/src/test/java/org/archive/modules/fetcher/FetchHTTPTests.java

@@ -787,6 +787,7 @@ public class FetchHTTPTests extends ProcessorTestBase {
         }
     }
 
+    // Implicitly tests the retry cycle within httpcomponents
     public void testNoResponse() throws Exception {
         NoResponseServer noResponseServer = new NoResponseServer("localhost", 7780);
         noResponseServer.start();