HER-1547 some sites may require an 'Accept' header

* FetchHTTP.java
    "Accept: text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8"
    by default - this is what my firefox 4.0 for mac sends
* profile-crawler-beans.cxml
    commented-out default value
This commit is contained in:
nlevitt
2011-04-25 20:43:17 +00:00
parent d159b4e106
commit 26f34c1b8c
2 changed files with 7 additions and 4 deletions
@@ -269,6 +269,7 @@ http://example.example/example
<!-- <property name="sslTrustLevel" value="OPEN" /> -->
<!-- <property name="acceptHeaders">
<list>
<value>Accept: text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8</value>
</list>
</property>
-->
@@ -42,9 +42,9 @@ import java.security.KeyManagementException;
import java.security.KeyStoreException;
import java.security.MessageDigest;
import java.security.NoSuchAlgorithmException;
import java.util.Arrays;
import java.util.HashSet;
import java.util.Iterator;
import java.util.LinkedList;
import java.util.List;
import java.util.Map;
import java.util.Set;
@@ -195,11 +195,13 @@ public class FetchHTTP extends Processor implements Lifecycle {
/**
* Accept Headers to include in each request. Each must be the complete
* header, e.g., 'Accept-Language: en'. (Thus, this can also be used to
* other headers not beginning 'Accept-' as well.)
* header, e.g., 'Accept-Language: en'. (Thus, this can also be used to
* other headers not beginning 'Accept-' as well.) By default heritrix sends
* an Accept header similar to what a typical browser would send (the value
* comes from Firefox 4.0).
*/
{
setAcceptHeaders(new LinkedList<String>());
setAcceptHeaders(Arrays.asList("Accept: text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8"));
}
@SuppressWarnings("unchecked")
public List<String> getAcceptHeaders() {