From 26f34c1b8c1ea46c3e348da648441621c0a9bc4a Mon Sep 17 00:00:00 2001
From: nlevitt <nlevitt@daa5b2f2-a927-0410-8b2d-f5f262fa301a>
Date: Mon, 25 Apr 2011 20:43:17 +0000
Subject: [PATCH] HER-1547 some sites may require an 'Accept' header *
 FetchHTTP.java     "Accept:
 text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8"     by
 default - this is what my firefox 4.0 for mac sends *
 profile-crawler-beans.cxml     commented-out default value

---
 .../archive/crawler/restlet/profile-crawler-beans.cxml |  1 +
 .../java/org/archive/modules/fetcher/FetchHTTP.java    | 10 ++++++----
 2 files changed, 7 insertions(+), 4 deletions(-)

diff --git a/engine/src/main/resources/org/archive/crawler/restlet/profile-crawler-beans.cxml b/engine/src/main/resources/org/archive/crawler/restlet/profile-crawler-beans.cxml
index 242b07b2..5fa2e30a 100644
--- a/engine/src/main/resources/org/archive/crawler/restlet/profile-crawler-beans.cxml
+++ b/engine/src/main/resources/org/archive/crawler/restlet/profile-crawler-beans.cxml
@@ -269,6 +269,7 @@ http://example.example/example
   <!-- <property name="sslTrustLevel" value="OPEN" /> -->
   <!-- <property name="acceptHeaders"> 
         <list>
+         <value>Accept: text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8</value>
         </list>
        </property>
   -->
diff --git a/modules/src/main/java/org/archive/modules/fetcher/FetchHTTP.java b/modules/src/main/java/org/archive/modules/fetcher/FetchHTTP.java
index 9e769c29..e3c435de 100644
--- a/modules/src/main/java/org/archive/modules/fetcher/FetchHTTP.java
+++ b/modules/src/main/java/org/archive/modules/fetcher/FetchHTTP.java
@@ -42,9 +42,9 @@ import java.security.KeyManagementException;
 import java.security.KeyStoreException;
 import java.security.MessageDigest;
 import java.security.NoSuchAlgorithmException;
+import java.util.Arrays;
 import java.util.HashSet;
 import java.util.Iterator;
-import java.util.LinkedList;
 import java.util.List;
 import java.util.Map;
 import java.util.Set;
@@ -195,11 +195,13 @@ public class FetchHTTP extends Processor implements Lifecycle {
 
     /**
      * Accept Headers to include in each request. Each must be the complete
-     * header, e.g., 'Accept-Language: en'. (Thus, this can also be used to 
-     * other headers not beginning 'Accept-' as well.)
+     * header, e.g., 'Accept-Language: en'. (Thus, this can also be used to
+     * other headers not beginning 'Accept-' as well.) By default heritrix sends
+     * an Accept header similar to what a typical browser would send (the value
+     * comes from Firefox 4.0).
      */
     {
-        setAcceptHeaders(new LinkedList<String>());
+        setAcceptHeaders(Arrays.asList("Accept: text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8"));
     }
     @SuppressWarnings("unchecked")
     public List<String> getAcceptHeaders() {