diff --git a/contrib/src/main/java/org/archive/modules/extractor/ExtractorChrome.java b/contrib/src/main/java/org/archive/modules/extractor/ExtractorChrome.java index 4d7c3a54..d76fbec0 100644 --- a/contrib/src/main/java/org/archive/modules/extractor/ExtractorChrome.java +++ b/contrib/src/main/java/org/archive/modules/extractor/ExtractorChrome.java @@ -29,6 +29,7 @@ import org.archive.modules.ProcessorChain; import org.archive.net.chrome.*; import org.archive.spring.KeyedProperties; import org.archive.util.Recorder; +import org.archive.util.UriUtils; import org.json.JSONArray; import org.springframework.context.ApplicationEventPublisher; @@ -216,11 +217,17 @@ public class ExtractorChrome extends ContentExtractor { } Map headers = (Map) curi.getData().get(A_HTTP_RESPONSE_HEADERS); + if (headers == null) { + logger.log(WARNING, "Response headers unavailable in CrawlURI. Letting the browser " + + "refetch {0}", curi.getURI()); + interceptedRequest.continueNormally(); + return; + } interceptedRequest.fulfill(curi.getFetchStatus(), headers.entrySet(), body); } private void handleCapturedRequest(CrawlURI via, ChromeRequest request) { - if (request.isResponseFulfilledByInterception()) { + if (request.isResponseFulfilledByInterception() || UriUtils.isDataUri(request.getUrl())) { return; } diff --git a/contrib/src/main/java/org/archive/net/chrome/ChromeWindow.java b/contrib/src/main/java/org/archive/net/chrome/ChromeWindow.java index e54b4583..13a35b1a 100644 --- a/contrib/src/main/java/org/archive/net/chrome/ChromeWindow.java +++ b/contrib/src/main/java/org/archive/net/chrome/ChromeWindow.java @@ -162,7 +162,9 @@ public class ChromeWindow implements Closeable { // it seems this event can arrive both before and after requestWillBeSent so we need to cope with that String requestId = params.getString("requestId"); ChromeRequest request = requestMap.computeIfAbsent(requestId, id -> new ChromeRequest(this, id)); - request.setRawRequestHeaders(params.getJSONObject("headers")); + if (params.has("headers")) { + request.setRawRequestHeaders(params.getJSONObject("headers")); + } } private void handleResponseReceived(JSONObject params) { @@ -180,8 +182,12 @@ public class ChromeWindow implements Closeable { logger.log(WARNING, "Got responseReceivedExtraInfo event without corresponding requestWillBeSent"); return; } - request.setRawResponseHeaders(params.getJSONObject("headers")); - request.setResponseHeadersText(params.getString("headersText")); + if (params.has("headers")) { + request.setRawResponseHeaders(params.getJSONObject("headers")); + } + if (params.has("headersText")) { + request.setResponseHeadersText(params.getString("headersText")); + } } private void handleLoadingFinished(JSONObject params) { diff --git a/contrib/src/test/java/org/archive/modules/extractor/ExtractorChromeTest.java b/contrib/src/test/java/org/archive/modules/extractor/ExtractorChromeTest.java index 78c8e157..f3a39166 100644 --- a/contrib/src/test/java/org/archive/modules/extractor/ExtractorChromeTest.java +++ b/contrib/src/test/java/org/archive/modules/extractor/ExtractorChromeTest.java @@ -67,6 +67,7 @@ public class ExtractorChromeTest { response.setContentType("text/html"); response.getWriter().write("link" + "" + + "" + "" + ""); baseRequest.setHandled(true);