diff --git a/engine/src/main/resources/org/archive/crawler/restlet/profile-crawler-beans.cxml b/engine/src/main/resources/org/archive/crawler/restlet/profile-crawler-beans.cxml index 242b07b2..5fa2e30a 100644 --- a/engine/src/main/resources/org/archive/crawler/restlet/profile-crawler-beans.cxml +++ b/engine/src/main/resources/org/archive/crawler/restlet/profile-crawler-beans.cxml @@ -269,6 +269,7 @@ http://example.example/example diff --git a/modules/src/main/java/org/archive/modules/fetcher/FetchHTTP.java b/modules/src/main/java/org/archive/modules/fetcher/FetchHTTP.java index 9e769c29..e3c435de 100644 --- a/modules/src/main/java/org/archive/modules/fetcher/FetchHTTP.java +++ b/modules/src/main/java/org/archive/modules/fetcher/FetchHTTP.java @@ -42,9 +42,9 @@ import java.security.KeyManagementException; import java.security.KeyStoreException; import java.security.MessageDigest; import java.security.NoSuchAlgorithmException; +import java.util.Arrays; import java.util.HashSet; import java.util.Iterator; -import java.util.LinkedList; import java.util.List; import java.util.Map; import java.util.Set; @@ -195,11 +195,13 @@ public class FetchHTTP extends Processor implements Lifecycle { /** * Accept Headers to include in each request. Each must be the complete - * header, e.g., 'Accept-Language: en'. (Thus, this can also be used to - * other headers not beginning 'Accept-' as well.) + * header, e.g., 'Accept-Language: en'. (Thus, this can also be used to + * other headers not beginning 'Accept-' as well.) By default heritrix sends + * an Accept header similar to what a typical browser would send (the value + * comes from Firefox 4.0). */ { - setAcceptHeaders(new LinkedList()); + setAcceptHeaders(Arrays.asList("Accept: text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8")); } @SuppressWarnings("unchecked") public List getAcceptHeaders() {