Merge pull request #431 from internetarchive/extractor-chrome-bug-fixes

ExtractorChrome bug fixes

contrib/src/main/java/org/archive/modules/extractor/ExtractorChrome.java

@@ -29,6 +29,7 @@ import org.archive.modules.ProcessorChain;
 import org.archive.net.chrome.*;
 import org.archive.spring.KeyedProperties;
 import org.archive.util.Recorder;
+import org.archive.util.UriUtils;
 import org.json.JSONArray;
 import org.springframework.context.ApplicationEventPublisher;
 
@@ -216,11 +217,17 @@ public class ExtractorChrome extends ContentExtractor {
         }
 
         Map<String,String> headers = (Map<String, String>) curi.getData().get(A_HTTP_RESPONSE_HEADERS);
+        if (headers == null) {
+            logger.log(WARNING, "Response headers unavailable in CrawlURI. Letting the browser " +
+                    "refetch {0}", curi.getURI());
+            interceptedRequest.continueNormally();
+            return;
+        }
         interceptedRequest.fulfill(curi.getFetchStatus(), headers.entrySet(), body);
     }
 
     private void handleCapturedRequest(CrawlURI via, ChromeRequest request) {
-        if (request.isResponseFulfilledByInterception()) {
+        if (request.isResponseFulfilledByInterception() || UriUtils.isDataUri(request.getUrl())) {
             return;
         }
 

contrib/src/main/java/org/archive/net/chrome/ChromeWindow.java

@@ -162,7 +162,9 @@ public class ChromeWindow implements Closeable {
         // it seems this event can arrive both before and after requestWillBeSent so we need to cope with that
         String requestId = params.getString("requestId");
         ChromeRequest request = requestMap.computeIfAbsent(requestId, id -> new ChromeRequest(this, id));
-        request.setRawRequestHeaders(params.getJSONObject("headers"));
+        if (params.has("headers")) {
+            request.setRawRequestHeaders(params.getJSONObject("headers"));
+        }
     }
 
     private void handleResponseReceived(JSONObject params) {
@@ -180,8 +182,12 @@ public class ChromeWindow implements Closeable {
             logger.log(WARNING, "Got responseReceivedExtraInfo event without corresponding requestWillBeSent");
             return;
         }
-        request.setRawResponseHeaders(params.getJSONObject("headers"));
-        request.setResponseHeadersText(params.getString("headersText"));
+        if (params.has("headers")) {
+            request.setRawResponseHeaders(params.getJSONObject("headers"));
+        }
+        if (params.has("headersText")) {
+            request.setResponseHeadersText(params.getString("headersText"));
+        }
     }
 
     private void handleLoadingFinished(JSONObject params) {

contrib/src/test/java/org/archive/modules/extractor/ExtractorChromeTest.java

@@ -67,6 +67,7 @@ public class ExtractorChromeTest {
                         response.setContentType("text/html");
                         response.getWriter().write("<a href=http://example.org/page2.html>link</a>" +
                                 "<img src=/blue.png>" +
+                                "<img src='data:image/gif;base64,R0lGODlhAQABAAAAACH5BAEKAAEALAAAAAABAAEAAAICTAEAOw=='>" +
                                 "<script>fetch('/post', {method: 'POST', body: 'hello'});</script>" +
                                 "<link rel=stylesheet href=style.css>");
                         baseRequest.setHandled(true);