diff --git a/contrib/src/main/java/org/archive/crawler/frontier/AMQPUrlReceiver.java b/contrib/src/main/java/org/archive/crawler/frontier/AMQPUrlReceiver.java index 880d770f..859d716d 100644 --- a/contrib/src/main/java/org/archive/crawler/frontier/AMQPUrlReceiver.java +++ b/contrib/src/main/java/org/archive/crawler/frontier/AMQPUrlReceiver.java @@ -21,6 +21,7 @@ package org.archive.crawler.frontier; import java.io.IOException; import java.io.UnsupportedEncodingException; +import java.util.Arrays; import java.util.HashMap; import java.util.HashSet; import java.util.Map; @@ -131,6 +132,14 @@ public class AMQPUrlReceiver implements Lifecycle, ApplicationListener REQUEST_HEADER_BLACKLIST = new HashSet(Arrays.asList( + "accept-encoding", "upgrade-insecure-requests", "host", "connection")); // XXX should we be using QueueingConsumer because of possible blocking in // frontier.schedule()? @@ -377,9 +389,11 @@ public class AMQPUrlReceiver implements Lifecycle, ApplicationListener customHttpRequestHeaders = new HashMap(); - for (Object key : joHeaders.keySet()) { - customHttpRequestHeaders.put(key.toString(), - joHeaders.getString(key.toString())); + for (Object key: joHeaders.keySet()) { + String k = key.toString(); + if (!k.startsWith(":") && !REQUEST_HEADER_BLACKLIST.contains(k)) { + customHttpRequestHeaders.put(k, joHeaders.getString(key.toString())); + } } curi.getData().put("customHttpRequestHeaders", customHttpRequestHeaders); @@ -392,7 +406,7 @@ public class AMQPUrlReceiver implements Lifecycle, ApplicationListener getExtraInfo() { + return (Map) kp.get("extraInfo"); + } + /** + * Arbitrary additional information to include in the json payload. + */ + public void setExtraInfo(Map extraInfo) { + kp.put("extraInfo", extraInfo); } /** @@ -94,6 +108,12 @@ public class AMQPPublishProcessor extends AMQPProducerProcessor implements Seria message.put("clientId", getClientId()); } + if (getExtraInfo() != null) { + for (String k: getExtraInfo().keySet()) { + message.put(k, getExtraInfo().get(k)); + } + } + HashMap metadata = new HashMap(); metadata.put("pathFromSeed", curi.getPathFromSeed()); diff --git a/contrib/src/main/java/org/archive/modules/postprocessor/AMQPCrawlLogFeed.java b/contrib/src/main/java/org/archive/modules/postprocessor/AMQPCrawlLogFeed.java index 77f45cd2..2448ca08 100644 --- a/contrib/src/main/java/org/archive/modules/postprocessor/AMQPCrawlLogFeed.java +++ b/contrib/src/main/java/org/archive/modules/postprocessor/AMQPCrawlLogFeed.java @@ -86,8 +86,8 @@ public class AMQPCrawlLogFeed extends AMQPProducerProcessor implements Lifecycle public AMQPCrawlLogFeed() { // set default values - exchange = "heritrix.realTimeFeed"; - routingKey = "crawlLog"; + setExchange("heritrix.realTimeFeed"); + setRoutingKey("crawlLog"); } @Override diff --git a/modules/src/main/java/org/archive/modules/fetcher/FetchHTTPRequest.java b/modules/src/main/java/org/archive/modules/fetcher/FetchHTTPRequest.java index ea4c047f..b4af96aa 100644 --- a/modules/src/main/java/org/archive/modules/fetcher/FetchHTTPRequest.java +++ b/modules/src/main/java/org/archive/modules/fetcher/FetchHTTPRequest.java @@ -53,6 +53,7 @@ import org.apache.http.HttpException; import org.apache.http.HttpHeaders; import org.apache.http.HttpHost; import org.apache.http.HttpRequest; +import org.apache.http.HttpRequestInterceptor; import org.apache.http.HttpResponse; import org.apache.http.NameValuePair; import org.apache.http.ProtocolVersion; @@ -347,13 +348,25 @@ public class FetchHTTPRequest { request.addHeader("X-Requested-With", "XMLHttpRequest"); } - @SuppressWarnings("unchecked") - Map uriCustomHeaders = (Map) curi.getData().get("customHttpRequestHeaders"); - if (uriCustomHeaders != null) { - for (Entry h: uriCustomHeaders.entrySet()) { - request.setHeader(h.getKey(), h.getValue()); + + /* + * set custom request headers in last interceptor, so they override + * anything else (this could just as well belong in + * configureHttpClientBuilder()) + */ + httpClientBuilder.addInterceptorLast(new HttpRequestInterceptor() { + @Override + public void process(HttpRequest request, HttpContext context) throws HttpException, IOException { + @SuppressWarnings("unchecked") + Map uriCustomHeaders = (Map) curi.getData().get("customHttpRequestHeaders"); + if (uriCustomHeaders != null) { + for (Entry h: uriCustomHeaders.entrySet()) { + request.setHeader(h.getKey(), h.getValue()); + } + } } - } + }); + } /**