From baaf4f8092defd36bcedc359b6ad123e79aa14f8 Mon Sep 17 00:00:00 2001
From: Noah Levitt <nlevitt@archive.org>
Date: Tue, 4 Sep 2012 13:25:51 -0700
Subject: [PATCH] testShouldFetchBodyRule

---
 .../modules/fetcher/FetchHTTPTestBase.java    | 23 +++++++++++++++++++
 .../modules/fetcher/AbstractFetchHTTP.java    |  3 +++
 2 files changed, 26 insertions(+)

diff --git a/engine/src/test/java/org/archive/modules/fetcher/FetchHTTPTestBase.java b/engine/src/test/java/org/archive/modules/fetcher/FetchHTTPTestBase.java
index ea8b883a..79a23006 100644
--- a/engine/src/test/java/org/archive/modules/fetcher/FetchHTTPTestBase.java
+++ b/engine/src/test/java/org/archive/modules/fetcher/FetchHTTPTestBase.java
@@ -43,6 +43,7 @@ import org.archive.modules.CrawlURI.FetchType;
 import org.archive.modules.ProcessorTestBase;
 import org.archive.modules.credential.HtmlFormCredential;
 import org.archive.modules.credential.HttpAuthenticationCredential;
+import org.archive.modules.deciderules.RejectDecideRule;
 import org.archive.modules.recrawl.FetchHistoryProcessor;
 import org.archive.net.UURI;
 import org.archive.net.UURIFactory;
@@ -810,4 +811,26 @@ public abstract class FetchHTTPTestBase extends ProcessorTestBase {
         runDefaultChecks(curi);
         // XXX make server send 304 not-modified and check for it here?
     }
+    
+    public void testShouldFetchBodyRule() throws Exception {
+        ensureHttpServers();
+        CrawlURI curi = makeCrawlURI("http://localhost:7777/");
+        getFetcher().setShouldFetchBodyRule(new RejectDecideRule());
+        getFetcher().process(curi);
+        logger.info('\n' + httpRequestString(curi) + "\n\n" + rawResponseString(curi));
+
+        assertTrue(httpRequestString(curi).startsWith("GET / HTTP/1.0\r\n"));
+        assertEquals("text/plain;charset=US-ASCII", curi.getContentType());
+        assertTrue(curi.getCredentials().isEmpty());
+        assertTrue(curi.getFetchDuration() >= 0);
+        assertTrue(curi.getFetchStatus() == 200);
+        assertTrue(curi.getFetchType() == FetchType.HTTP_GET);
+        
+        // check for empty body
+        assertEquals(0, curi.getContentLength());
+        assertEquals(curi.getContentSize(), curi.getRecordedSize());
+        assertEquals("", messageBodyString(curi));
+        assertEquals("", entityString(curi));
+    }
+
 }
diff --git a/modules/src/main/java/org/archive/modules/fetcher/AbstractFetchHTTP.java b/modules/src/main/java/org/archive/modules/fetcher/AbstractFetchHTTP.java
index 0e9aeb43..5db90641 100644
--- a/modules/src/main/java/org/archive/modules/fetcher/AbstractFetchHTTP.java
+++ b/modules/src/main/java/org/archive/modules/fetcher/AbstractFetchHTTP.java
@@ -22,6 +22,7 @@ import java.util.List;
 
 import org.archive.modules.Processor;
 import org.archive.modules.credential.CredentialStore;
+import org.archive.modules.deciderules.DecideRule;
 import org.archive.modules.net.ServerCache;
 
 abstract public class AbstractFetchHTTP extends Processor {
@@ -40,4 +41,6 @@ abstract public class AbstractFetchHTTP extends Processor {
     abstract public void setSendRange(boolean sendRange);
     abstract public void setSendIfModifiedSince(boolean sendIfModifiedSince);
     abstract public void setSendIfNoneMatch(boolean sendIfNoneMatch);
+    abstract public void setSendReferer(boolean sendReferer);
+    abstract public void setShouldFetchBodyRule(DecideRule rule);
 }