From 6b19daf1d28a8f4848bfb537b2cb45bb36dc7ea7 Mon Sep 17 00:00:00 2001 From: Alex Osborne Date: Tue, 17 Aug 2021 09:07:04 +0900 Subject: [PATCH 1/3] ExtractorChrome: Don't capture data: URIs They are already captured as part of their containing document. There's no need to record them separately. #430 --- .../java/org/archive/modules/extractor/ExtractorChrome.java | 3 ++- .../org/archive/modules/extractor/ExtractorChromeTest.java | 1 + 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/contrib/src/main/java/org/archive/modules/extractor/ExtractorChrome.java b/contrib/src/main/java/org/archive/modules/extractor/ExtractorChrome.java index 4d7c3a54..c1b38398 100644 --- a/contrib/src/main/java/org/archive/modules/extractor/ExtractorChrome.java +++ b/contrib/src/main/java/org/archive/modules/extractor/ExtractorChrome.java @@ -29,6 +29,7 @@ import org.archive.modules.ProcessorChain; import org.archive.net.chrome.*; import org.archive.spring.KeyedProperties; import org.archive.util.Recorder; +import org.archive.util.UriUtils; import org.json.JSONArray; import org.springframework.context.ApplicationEventPublisher; @@ -220,7 +221,7 @@ public class ExtractorChrome extends ContentExtractor { } private void handleCapturedRequest(CrawlURI via, ChromeRequest request) { - if (request.isResponseFulfilledByInterception()) { + if (request.isResponseFulfilledByInterception() || UriUtils.isDataUri(request.getUrl())) { return; } diff --git a/contrib/src/test/java/org/archive/modules/extractor/ExtractorChromeTest.java b/contrib/src/test/java/org/archive/modules/extractor/ExtractorChromeTest.java index 78c8e157..f3a39166 100644 --- a/contrib/src/test/java/org/archive/modules/extractor/ExtractorChromeTest.java +++ b/contrib/src/test/java/org/archive/modules/extractor/ExtractorChromeTest.java @@ -67,6 +67,7 @@ public class ExtractorChromeTest { response.setContentType("text/html"); response.getWriter().write("link" + "" + + "" + "" + ""); baseRequest.setHandled(true); From e93e457e192ad017683c927acb84d5d1544f6ae3 Mon Sep 17 00:00:00 2001 From: Alex Osborne Date: Tue, 17 Aug 2021 09:14:59 +0900 Subject: [PATCH 2/3] ChromeWindow: Handle raw headers or headersText being unavailable Colin encountered headersText being unavailable. (Maybe HTTP/2?) --- .../java/org/archive/net/chrome/ChromeWindow.java | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/contrib/src/main/java/org/archive/net/chrome/ChromeWindow.java b/contrib/src/main/java/org/archive/net/chrome/ChromeWindow.java index e54b4583..13a35b1a 100644 --- a/contrib/src/main/java/org/archive/net/chrome/ChromeWindow.java +++ b/contrib/src/main/java/org/archive/net/chrome/ChromeWindow.java @@ -162,7 +162,9 @@ public class ChromeWindow implements Closeable { // it seems this event can arrive both before and after requestWillBeSent so we need to cope with that String requestId = params.getString("requestId"); ChromeRequest request = requestMap.computeIfAbsent(requestId, id -> new ChromeRequest(this, id)); - request.setRawRequestHeaders(params.getJSONObject("headers")); + if (params.has("headers")) { + request.setRawRequestHeaders(params.getJSONObject("headers")); + } } private void handleResponseReceived(JSONObject params) { @@ -180,8 +182,12 @@ public class ChromeWindow implements Closeable { logger.log(WARNING, "Got responseReceivedExtraInfo event without corresponding requestWillBeSent"); return; } - request.setRawResponseHeaders(params.getJSONObject("headers")); - request.setResponseHeadersText(params.getString("headersText")); + if (params.has("headers")) { + request.setRawResponseHeaders(params.getJSONObject("headers")); + } + if (params.has("headersText")) { + request.setResponseHeadersText(params.getString("headersText")); + } } private void handleLoadingFinished(JSONObject params) { From 3346428bf2c730930666dc3cb7fc6c708e99f746 Mon Sep 17 00:00:00 2001 From: Alex Osborne Date: Tue, 17 Aug 2021 09:20:35 +0900 Subject: [PATCH 3/3] ExtractorChrome: Warn instead of throwing when response headers missing Colin reported this exception. I'm uncertain how this can occur though as FetchHTTP should populate the response headers. Perhaps a different Fetch module was used? --- .../java/org/archive/modules/extractor/ExtractorChrome.java | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/contrib/src/main/java/org/archive/modules/extractor/ExtractorChrome.java b/contrib/src/main/java/org/archive/modules/extractor/ExtractorChrome.java index c1b38398..d76fbec0 100644 --- a/contrib/src/main/java/org/archive/modules/extractor/ExtractorChrome.java +++ b/contrib/src/main/java/org/archive/modules/extractor/ExtractorChrome.java @@ -217,6 +217,12 @@ public class ExtractorChrome extends ContentExtractor { } Map headers = (Map) curi.getData().get(A_HTTP_RESPONSE_HEADERS); + if (headers == null) { + logger.log(WARNING, "Response headers unavailable in CrawlURI. Letting the browser " + + "refetch {0}", curi.getURI()); + interceptedRequest.continueNormally(); + return; + } interceptedRequest.fulfill(curi.getFetchStatus(), headers.entrySet(), body); }