From 24e813c85892fb1093acf623222b316fc81766ca Mon Sep 17 00:00:00 2001
From: Noah Levitt <nlevitt@archive.org>
Date: Fri, 7 Sep 2012 13:28:41 -0700
Subject: [PATCH] test chunked transfer encoding

---
 .../modules/fetcher/FetchHTTPTestBase.java    | 36 ++++++++++++++++++-
 .../modules/fetcher/AbstractFetchHTTP.java    |  2 ++
 .../archive/modules/fetcher/FetchHTTP2.java   |  2 --
 3 files changed, 37 insertions(+), 3 deletions(-)

diff --git a/engine/src/test/java/org/archive/modules/fetcher/FetchHTTPTestBase.java b/engine/src/test/java/org/archive/modules/fetcher/FetchHTTPTestBase.java
index f2f35bc0..07afeea5 100644
--- a/engine/src/test/java/org/archive/modules/fetcher/FetchHTTPTestBase.java
+++ b/engine/src/test/java/org/archive/modules/fetcher/FetchHTTPTestBase.java
@@ -171,6 +171,13 @@ public abstract class FetchHTTPTestBase extends ProcessorTestBase {
                     }
                 }
                 ((Request)request).setHandled(true);
+            } else if (target.equals("/chunked.txt")) {
+                response.setContentType("text/plain;charset=US-ASCII");
+                response.setStatus(HttpServletResponse.SC_OK);
+                // response.setContentLength(HttpTokens.CHUNKED_CONTENT);
+                response.getOutputStream().write(DEFAULT_PAYLOAD_STRING.getBytes("US-ASCII"));
+                response.getOutputStream().flush();
+                ((Request)request).setHandled(true);
             } else if (request.getHeader("Accept-Encoding") != null
                     && request.getHeader("Accept-Encoding").contains("gzip")) {
                 response.setHeader("Content-Encoding", "gzip");
@@ -941,5 +948,32 @@ public abstract class FetchHTTPTestBase extends ProcessorTestBase {
         // what else?
         runDefaultChecks(curi, "requestLine");
     }
-    
+
+    public void testChunked() throws Exception {
+        ensureHttpServers();
+        CrawlURI curi = makeCrawlURI("http://localhost:7777/chunked.txt");
+        getFetcher().setUseHTTP11(true);
+        getFetcher().setSendConnectionClose(false);
+        
+        /* XXX Server expects us to close the connection apparently. But we
+         * don't detect end of chunked transfer. With these small timeouts we
+         * can finish quickly. A couple of SocketTimeoutExceptions will happen
+         * within RecordingInputStream.readFullyOrUntil().
+         */
+        getFetcher().setSoTimeoutMs(500);
+        getFetcher().setTimeoutSeconds(1);
+        
+        getFetcher().process(curi);
+        
+        //        logger.info('\n' + httpRequestString(curi) + "\n\n" + rawResponseString(curi));
+        //        logger.info("\n----- rawResponseString -----\n" + rawResponseString(curi));
+        //        logger.info("\n----- contentString -----\n" + contentString(curi));
+        //        logger.info("\n----- entityString -----\n" + entityString(curi));
+        //        logger.info("\n----- messageBodyString -----\n" + messageBodyString(curi));
+        
+        assertEquals("chunked", curi.getHttpResponseHeader("transfer-encoding"));
+        assertEquals("25\r\n" + DEFAULT_PAYLOAD_STRING + "\r\n0\r\n\r\n", messageBodyString(curi));
+        assertEquals(DEFAULT_PAYLOAD_STRING, entityString(curi));
+        assertEquals(DEFAULT_PAYLOAD_STRING, contentString(curi));
+    }
 }
diff --git a/modules/src/main/java/org/archive/modules/fetcher/AbstractFetchHTTP.java b/modules/src/main/java/org/archive/modules/fetcher/AbstractFetchHTTP.java
index 4d5c315a..99731e16 100644
--- a/modules/src/main/java/org/archive/modules/fetcher/AbstractFetchHTTP.java
+++ b/modules/src/main/java/org/archive/modules/fetcher/AbstractFetchHTTP.java
@@ -48,4 +48,6 @@ abstract public class AbstractFetchHTTP extends Processor {
     abstract public void setTimeoutSeconds(int timeout);
     abstract public void setSslTrustLevel(TrustLevel trustLevel);
     abstract public void setUseHTTP11(boolean useHTTP11);
+    abstract public void setUserAgentProvider(UserAgentProvider provider);
+    abstract public void setSendConnectionClose(boolean sendClose);
 }
diff --git a/modules/src/main/java/org/archive/modules/fetcher/FetchHTTP2.java b/modules/src/main/java/org/archive/modules/fetcher/FetchHTTP2.java
index df8b620d..2e44d811 100644
--- a/modules/src/main/java/org/archive/modules/fetcher/FetchHTTP2.java
+++ b/modules/src/main/java/org/archive/modules/fetcher/FetchHTTP2.java
@@ -699,8 +699,6 @@ public class FetchHTTP2 extends AbstractFetchHTTP implements Lifecycle {
             if (!request.isAborted()) {
                 // Force read-to-end, so that any socket hangs occur here,
                 // not in later modules.
-                
-                // XXX does it matter that we're circumventing the library here? response.getEntity().getContent()
                 rec.getRecordedInput().readFullyOrUntil(softMax); 
             }
         } catch (RecorderTimeoutException ex) {