From aa265855ef7ebdf2e55df944e81c1e40c0b2de4b Mon Sep 17 00:00:00 2001
From: Alex Osborne <aosborne@nla.gov.au>
Date: Tue, 6 Jul 2021 00:49:20 +0900
Subject: [PATCH] ExtractorChrome: Capture requests made by the browser

This adds a `captureRequests` flag to ExtractorChrome which is enabled
by default and causes requests made by the browser to be captured via
the devtools Network domain. Captured browser requests are sent to the
disposition chain for WARC writing and also to statistics tracker and
crawl log.

Browser requests are given the annotation "browser" so they can be
easily distinguished in the log from normal requests.

There are a quite a few limitations that will be addressed in followup
work:

* The frontier is entirely unaware of browser requests. This means they
  bypass quotas, ignore scope rules, politeness and don't count towards
  the statistics tracked by the frontier itself.

* There's no replay of previously saved resources so duplicate requests
  for the same URL end up being made.

* Heritrix's extractors do not currently process browser requests.

* Various error and failure cases likely need improving.
---
 .../modules/extractor/ExtractorChrome.java    | 128 ++++++++++++++-
 .../org/archive/net/chrome/ChromeClient.java  |  20 +--
 .../org/archive/net/chrome/ChromeRequest.java | 154 ++++++++++++++++++
 .../org/archive/net/chrome/ChromeWindow.java  | 114 ++++++++++++-
 .../extractor/ExtractorChromeTest.java        | 120 +++++++++++++-
 5 files changed, 512 insertions(+), 24 deletions(-)
 create mode 100644 contrib/src/main/java/org/archive/net/chrome/ChromeRequest.java

diff --git a/contrib/src/main/java/org/archive/modules/extractor/ExtractorChrome.java b/contrib/src/main/java/org/archive/modules/extractor/ExtractorChrome.java
index 42379adf..8bc74da5 100644
--- a/contrib/src/main/java/org/archive/modules/extractor/ExtractorChrome.java
+++ b/contrib/src/main/java/org/archive/modules/extractor/ExtractorChrome.java
@@ -19,17 +19,39 @@
 
 package org.archive.modules.extractor;
 
+import org.archive.crawler.event.CrawlURIDispositionEvent;
+import org.archive.crawler.framework.CrawlController;
+import org.archive.crawler.framework.Frontier;
 import org.archive.modules.CrawlURI;
 import org.archive.net.chrome.ChromeClient;
 import org.archive.net.chrome.ChromeProcess;
+import org.archive.net.chrome.ChromeRequest;
 import org.archive.net.chrome.ChromeWindow;
+import org.archive.spring.KeyedProperties;
+import org.archive.util.Recorder;
 import org.json.JSONArray;
+import org.springframework.context.ApplicationEventPublisher;
 
+import java.io.ByteArrayInputStream;
 import java.io.IOException;
+import java.io.InputStream;
+import java.io.SequenceInputStream;
+import java.util.Arrays;
 import java.util.concurrent.ExecutionException;
 import java.util.concurrent.Semaphore;
 import java.util.concurrent.TimeUnit;
 import java.util.concurrent.TimeoutException;
+import java.util.logging.Logger;
+import java.util.regex.Pattern;
+
+import static java.nio.charset.StandardCharsets.US_ASCII;
+import static java.util.Collections.enumeration;
+import static java.util.logging.Level.INFO;
+import static java.util.logging.Level.WARNING;
+import static java.util.regex.Pattern.CASE_INSENSITIVE;
+import static org.archive.crawler.event.CrawlURIDispositionEvent.Disposition.FAILED;
+import static org.archive.crawler.event.CrawlURIDispositionEvent.Disposition.SUCCEEDED;
+import static org.archive.modules.CrawlURI.FetchType.*;
 
 /**
  * Extracts links using a web browser via the Chrome Devtools Protocol.
@@ -37,6 +59,7 @@ import java.util.concurrent.TimeoutException;
  * To use, first define this as a top-level bean:
  * <pre>
  * &lt;bean id=&quot;extractorChrome&quot; class=&quot;org.archive.modules.extractor.ExtractorChrome&quot;&gt;
+ *   &lt;!-- &lt;property name=&quot;captureRequests&quot; value=&quot;true&quot; /&gt; --&gt;
  *   &lt;!-- &lt;property name=&quot;devtoolsUrl&quot; value=&quot;ws://127.0.0.1:1234/devtools/browser/2bc831e8-6c02-4c9b-affd-14c93b8579d7&quot; /&gt; --&gt;
  *   &lt;!-- &lt;property name=&quot;executable&quot; value=&quot;chromium-browser&quot; /&gt; --&gt;
  *   &lt;!-- &lt;property name=&quot;loadTimeoutSeconds&quot; value=&quot;30&quot; /&gt; --&gt;
@@ -52,6 +75,10 @@ import java.util.concurrent.TimeoutException;
  * <code>--headless --remote-debugging-port=1234</code>).
  */
 public class ExtractorChrome extends ContentExtractor {
+    private static final Logger logger = Logger.getLogger(ExtractorChrome.class.getName());
+
+    private static final Pattern TRANSFER_ENCODING_RE = Pattern.compile("\r\nTransfer-Encoding:[^\n\r]+", CASE_INSENSITIVE);
+
     /**
      * The maximum number of browser windows that are allowed to be opened simultaneously. Feel free to increase this
      * if you have lots of RAM available.
@@ -83,10 +110,23 @@ public class ExtractorChrome extends ContentExtractor {
      */
     private int loadTimeoutSeconds = 30;
 
+    /**
+     * Capture requests made by the browser.
+     */
+    private boolean captureRequests = true;
+
     private Semaphore openWindowsSemaphore = null;
     private ChromeProcess process = null;
     private ChromeClient client = null;
 
+    private final CrawlController controller;
+    private final ApplicationEventPublisher eventPublisher;
+
+    public ExtractorChrome(CrawlController controller, ApplicationEventPublisher eventPublisher) {
+        this.controller = controller;
+        this.eventPublisher = eventPublisher;
+    }
+
     @Override
     protected boolean shouldExtract(CrawlURI uri) {
         return uri.getContentType().startsWith("text/html");
@@ -107,25 +147,105 @@ public class ExtractorChrome extends ContentExtractor {
         return false;
     }
 
-    private void visit(CrawlURI uri) throws InterruptedException {
+    private void visit(CrawlURI curi) throws InterruptedException {
         try (ChromeWindow window = client.createWindow(windowWidth, windowHeight)) {
+            if (captureRequests) {
+                window.captureRequests(request -> handleCapturedRequest(curi, request));
+            }
+
             try {
-                window.navigateAsync(uri.getURI()).get(loadTimeoutSeconds, TimeUnit.SECONDS);
+                window.navigateAsync(curi.getURI()).get(loadTimeoutSeconds, TimeUnit.SECONDS);
             } catch (ExecutionException e) {
                 throw new RuntimeException(e.getCause());
             } catch (TimeoutException e) {
-                throw new RuntimeException("Timed out navigating to " + uri.getURI());
+                throw new RuntimeException("Timed out navigating to " + curi.getURI());
             }
 
             JSONArray links = window.eval("Array.from(document.querySelectorAll('a[href], area[href]'))" +
                     ".map(link => link.protocol + '//' + link.host + link.pathname + link.search + link.hash)")
                     .getJSONArray("value");
             for (int i = 0; i < links.length(); i++) {
-                addOutlink(uri, links.getString(i), LinkContext.NAVLINK_MISC, Hop.NAVLINK);
+                addOutlink(curi, links.getString(i), LinkContext.NAVLINK_MISC, Hop.NAVLINK);
             }
         }
     }
 
+    private void handleCapturedRequest(CrawlURI via, ChromeRequest request) {
+        Recorder recorder = new Recorder(controller.getScratchDir().getFile(),
+                controller.getRecorderOutBufferBytes(),
+                controller.getRecorderInBufferBytes());
+        try {
+            String digestAlgorithm = "sha1";
+            recorder.getRecordedInput().setDigest(digestAlgorithm);
+            recorder.getRecordedOutput().write(request.getRequestHeader().getBytes(US_ASCII));
+            recorder.getRecordedOutput().write(request.getRequestBody());
+
+            // strip the Transfer-Encoding header since we only have access to the decoded body
+            String responseHeader = TRANSFER_ENCODING_RE.matcher(request.getResponseHeader()).replaceAll("");
+
+            recorder.inputWrap(new SequenceInputStream(enumeration(Arrays.asList(
+                    new ByteArrayInputStream(responseHeader.getBytes(US_ASCII)),
+                    new InputStream() {
+                        public int read() {
+                            recorder.markContentBegin();
+                            return -1;
+                        }
+                    },
+                    new ByteArrayInputStream(request.getResponseBody())))));
+            recorder.getRecordedInput().readFully();
+            recorder.closeRecorders();
+
+            CrawlURI curi = via.createCrawlURI(request.getUrl(), LinkContext.EMBED_MISC, Hop.EMBED);
+            curi.getAnnotations().add("browser");
+            curi.setContentDigest(digestAlgorithm, recorder.getRecordedInput().getDigestValue());
+            curi.setContentSize(recorder.getRecordedInput().getSize());
+            curi.setContentType(request.getResponseContentType());
+            curi.setFetchBeginTime(request.getBeginTime());
+            curi.setFetchCompletedTime(System.currentTimeMillis());
+            curi.setFetchStatus(request.getStatus());
+            curi.setRecorder(recorder);
+            curi.setServerIP(request.getRemoteIPAddress());
+            curi.setThreadNumber(via.getThreadNumber());
+
+            switch (request.getMethod()) {
+                case "GET":
+                    curi.setFetchType(HTTP_GET);
+                    break;
+                case "POST":
+                    curi.setFetchType(HTTP_POST);
+                    break;
+                default:
+                    curi.setFetchType(UNKNOWN);
+                    break;
+            }
+
+            // send it to the disposition chain to invoke the warc writer etc
+            Frontier frontier = controller.getFrontier(); // allowed to be null to simplify unit tests
+            curi.getOverlayNames(); // for side-effect of creating the overlayNames list
+            KeyedProperties.loadOverridesFrom(curi);
+            try {
+                frontier.beginDisposition(curi);
+                controller.getDispositionChain().process(curi,null);
+            } finally {
+                KeyedProperties.clearOverridesFrom(curi);
+            }
+
+            curi.aboutToLog();
+            controller.getLoggerModule().getUriProcessing().log(INFO, curi.getUURI().toString(), curi);
+
+            if (curi.isSuccess()) {
+                eventPublisher.publishEvent(new CrawlURIDispositionEvent(this, curi, SUCCEEDED));
+            } else {
+                eventPublisher.publishEvent(new CrawlURIDispositionEvent(this, curi, FAILED));
+            }
+            frontier.endDisposition();
+        } catch (Exception e) {
+            logger.log(WARNING, "Exception handling subrequest " + request.getUrl(), e);
+        } finally {
+            recorder.cleanup();
+        }
+    }
+
     @Override
     public void start() {
         if (isRunning) return;
diff --git a/contrib/src/main/java/org/archive/net/chrome/ChromeClient.java b/contrib/src/main/java/org/archive/net/chrome/ChromeClient.java
index 7de68b3a..cc81598b 100644
--- a/contrib/src/main/java/org/archive/net/chrome/ChromeClient.java
+++ b/contrib/src/main/java/org/archive/net/chrome/ChromeClient.java
@@ -29,6 +29,7 @@ import java.util.Map;
 import java.util.concurrent.*;
 import java.util.concurrent.atomic.AtomicLong;
 import java.util.function.Consumer;
+import java.util.logging.Level;
 import java.util.logging.Logger;
 
 import static java.util.logging.Level.WARNING;
@@ -43,7 +44,6 @@ public class ChromeClient implements Closeable {
     private final DevtoolsSocket devtoolsSocket;
     private final AtomicLong nextMessageId = new AtomicLong(0);
     private final Map<Long, CompletableFuture<JSONObject>> responseFutures = new ConcurrentHashMap<>();
-    private final ExecutorService threadPool = Executors.newCachedThreadPool();
     final ConcurrentHashMap<String, Consumer<JSONObject>> sessionEventHandlers = new ConcurrentHashMap<>();
 
     public ChromeClient(String devtoolsUrl) {
@@ -110,16 +110,7 @@ public class ChromeClient implements Closeable {
             String sessionId = message.getString("sessionId");
             Consumer<JSONObject> handler = sessionEventHandlers.get(sessionId);
             if (handler != null) {
-                // Run event handlers on a different thread so we don't block the websocket receiving thread.
-                // That would cause a deadlock if an event handler itself made an RPC call as the response could
-                // never be processed.
-                threadPool.submit(() -> {
-                    try {
-                        handler.accept(message);
-                    } catch (Throwable t) {
-                        logger.log(WARNING, "Exception handling browser event " + message, t);
-                    }
-                });
+                handler.accept(message);
             } else {
                 logger.log(WARNING, "Received event for unknown session {0}", sessionId);
             }
@@ -164,13 +155,14 @@ public class ChromeClient implements Closeable {
         }
 
         @Override
-        public void onClose(int i, String s, boolean b) {
-
+        public void onClose(int code, String reason, boolean remote) {
+            if (!remote) return;
+            logger.log(WARNING, "Websocket closed by browser: " + reason);
         }
 
         @Override
         public void onError(Exception e) {
-
+            logger.log(Level.SEVERE, "Websocket error", e);
         }
     }
 }
diff --git a/contrib/src/main/java/org/archive/net/chrome/ChromeRequest.java b/contrib/src/main/java/org/archive/net/chrome/ChromeRequest.java
new file mode 100644
index 00000000..68ce072b
--- /dev/null
+++ b/contrib/src/main/java/org/archive/net/chrome/ChromeRequest.java
@@ -0,0 +1,154 @@
+/*
+ *  This file is part of the Heritrix web crawler (crawler.archive.org).
+ *
+ *  Licensed to the Internet Archive (IA) by one or more individual
+ *  contributors.
+ *
+ *  The IA licenses this file to You under the Apache License, Version 2.0
+ *  (the "License"); you may not use this file except in compliance with
+ *  the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ */
+
+package org.archive.net.chrome;
+
+import org.json.JSONObject;
+
+import java.nio.charset.StandardCharsets;
+import java.util.Base64;
+
+public class ChromeRequest {
+    private final ChromeWindow window;
+    private final String id;
+    private JSONObject requestJson;
+    private JSONObject rawRequestHeaders;
+    private JSONObject responseJson;
+    private JSONObject rawResponseHeaders;
+    private String responseHeadersText;
+    private final long beginTime = System.currentTimeMillis();
+
+    public ChromeRequest(ChromeWindow window, String id) {
+        this.window = window;
+        this.id = id;
+    }
+
+    void setRawRequestHeaders(JSONObject rawRequestHeaders) {
+        this.rawRequestHeaders = rawRequestHeaders;
+    }
+
+    void setResponseJson(JSONObject responseJson) {
+        this.responseJson = responseJson;
+    }
+
+    void setRawResponseHeaders(JSONObject headers) {
+        this.rawResponseHeaders = headers;
+    }
+
+    void setResponseHeadersText(String responseHeadersText) {
+        this.responseHeadersText = responseHeadersText;
+    }
+
+    public String getUrl() {
+        return requestJson.getString("url");
+    }
+
+    public byte[] getResponseBody() {
+        JSONObject reply = window.call("Network.getResponseBody", "requestId", id);
+        byte[] body;
+        if (reply.getBoolean("base64Encoded")) {
+            body = Base64.getDecoder().decode(reply.getString("body"));
+        } else {
+            body = reply.getString("body").getBytes(StandardCharsets.UTF_8);
+        }
+        return body;
+    }
+
+    public String getRequestHeader() {
+        if (responseJson != null && responseJson.has("requestHeadersText")) {
+            return responseJson.getString("requestHeadersText");
+        }
+        StringBuilder builder = new StringBuilder();
+        builder.append(requestJson.getString("method"));
+        builder.append(' ');
+        builder.append(getUrl());
+        builder.append(" HTTP/1.1\r\n");
+        formatHeaders(builder, requestJson.getJSONObject("headers"), rawRequestHeaders);
+        return builder.toString();
+    }
+
+    public byte[] getRequestBody() {
+        if (requestJson.has("postData")) {
+            return requestJson.getString("postData").getBytes(StandardCharsets.UTF_8);
+        } else {
+            return new byte[0];
+        }
+    }
+
+    public String getResponseHeader() {
+        if (responseHeadersText != null) {
+            return responseHeadersText;
+        } else if (responseJson.has("headersText")) {
+            return responseJson.getString("headersText");
+        }
+        StringBuilder builder = new StringBuilder();
+        if (responseJson.getString("protocol").equals("http/1.0")) {
+            builder.append("HTTP/1.0");
+        } else {
+            builder.append("HTTP/1.1");
+        }
+        builder.append(getStatus());
+        builder.append(" ");
+        builder.append(responseJson.getString("statusText"));
+        builder.append("\r\n");
+        formatHeaders(builder, responseJson.getJSONObject("headers"), rawResponseHeaders);
+        return builder.toString();
+    }
+
+    private void formatHeaders(StringBuilder builder, JSONObject headers, JSONObject rawHeaders) {
+        if (rawHeaders != null) {
+            headers = rawHeaders;
+        }
+        for (Object key : headers.keySet()) {
+            builder.append(key);
+            builder.append(": ");
+            builder.append(headers.getString((String) key));
+            builder.append("\r\n");
+        }
+        builder.append("\r\n");
+    }
+
+    public String getMethod() {
+        return requestJson.getString("method");
+    }
+
+    public int getStatus() {
+        return responseJson.getInt("status");
+    }
+
+    public String getResponseContentType() {
+        return responseJson.getString("mimeType");
+    }
+
+    public long getBeginTime() {
+        return beginTime;
+    }
+
+    public String getRemoteIPAddress() {
+        if (responseJson.has("remoteIPAddress")) {
+            return responseJson.getString("remoteIPAddress");
+        } else {
+            return null;
+        }
+    }
+
+    void setRequestJson(JSONObject requestJson) {
+        this.requestJson = requestJson;
+    }
+}
diff --git a/contrib/src/main/java/org/archive/net/chrome/ChromeWindow.java b/contrib/src/main/java/org/archive/net/chrome/ChromeWindow.java
index 59e8f2e7..a65fa8cc 100644
--- a/contrib/src/main/java/org/archive/net/chrome/ChromeWindow.java
+++ b/contrib/src/main/java/org/archive/net/chrome/ChromeWindow.java
@@ -22,10 +22,13 @@ package org.archive.net.chrome;
 import org.json.JSONObject;
 
 import java.io.Closeable;
-import java.util.concurrent.CompletableFuture;
+import java.util.Map;
+import java.util.concurrent.*;
+import java.util.function.Consumer;
 import java.util.logging.Logger;
 
 import static java.util.logging.Level.FINE;
+import static java.util.logging.Level.WARNING;
 
 /**
  * A browser window or tab.
@@ -38,12 +41,17 @@ public class ChromeWindow implements Closeable {
     private final String sessionId;
     private boolean closed;
     private CompletableFuture<Void> loadEventFuture;
+    private final Map<String,ChromeRequest> requestMap = new ConcurrentHashMap<>();
+    private Consumer<ChromeRequest> requestConsumer;
+    private final ExecutorService eventExecutor;
 
     public ChromeWindow(ChromeClient client, String targetId) {
         this.client = client;
         this.targetId = targetId;
         this.sessionId = client.call("Target.attachToTarget", "targetId", targetId,
                 "flatten", true).getString("sessionId");
+        eventExecutor = Executors.newSingleThreadExecutor(runnable ->
+                new Thread(runnable, "ChromeWindow (sessionId=" + sessionId +")"));
         client.sessionEventHandlers.put(sessionId, this::handleEvent);
         call("Page.enable"); // for loadEventFired
         call("Page.setLifecycleEventsEnabled", "enabled", true); // for networkidle
@@ -78,9 +86,47 @@ public class ChromeWindow implements Closeable {
     }
 
     private void handleEvent(JSONObject message) {
+        // Run event handlers on a different thread so we don't block the websocket receiving thread.
+        // That would cause a deadlock if an event handler itself made an RPC call as the response could
+        // never be processed.
+        // We use a single thread per window though as the order events are processed is important.
+        eventExecutor.submit(() -> {
+            try {
+                handleEventOnEventThread(message);
+            } catch (Throwable t) {
+                logger.log(WARNING, "Exception handling browser event " + message, t);
+            }
+        });
+    }
+
+    private void handleEventOnEventThread(JSONObject message) {
+        JSONObject params = message.getJSONObject("params");
         switch (message.getString("method")) {
+            case "Fetch.requestPaused":
+                handlePausedRequest(params);
+                break;
+            case "Network.requestWillBeSent":
+                handleRequestWillBeSent(params);
+                break;
+            case "Network.requestWillBeSentExtraInfo":
+                handleRequestWillBeSentExtraInfo(params);
+                break;
+            case "Network.responseReceived":
+                handleResponseReceived(params);
+                break;
+            case "Network.responseReceivedExtraInfo":
+                handleResponseReceivedExtraInfo(params);
+                break;
+            case "Network.loadingFinished":
+                handleLoadingFinished(params);
+                break;
             case "Page.loadEventFired":
                 if (loadEventFuture != null) {
+                    try {
+                        Thread.sleep(1000);
+                    } catch (InterruptedException e) {
+                        Thread.currentThread().interrupt();
+                    }
                     loadEventFuture.complete(null);
                 }
                 break;
@@ -90,11 +136,77 @@ public class ChromeWindow implements Closeable {
         }
     }
 
+    private void handleRequestWillBeSent(JSONObject params) {
+        String requestId = params.getString("requestId");
+        ChromeRequest request = requestMap.computeIfAbsent(requestId, id -> new ChromeRequest(this, id));
+        request.setRequestJson(params.getJSONObject("request"));
+    }
+
+    private void handleRequestWillBeSentExtraInfo(JSONObject params) {
+        // it seems this event can arrive both before and after requestWillBeSent so we need to cope with that
+        String requestId = params.getString("requestId");
+        ChromeRequest request = requestMap.computeIfAbsent(requestId, id -> new ChromeRequest(this, id));
+        request.setRawRequestHeaders(params.getJSONObject("headers"));
+    }
+
+    private void handleResponseReceived(JSONObject params) {
+        ChromeRequest request = requestMap.get(params.getString("requestId"));
+        if (request == null) {
+            logger.log(WARNING, "Got responseReceived event without corresponding requestWillBeSent");
+            return;
+        }
+        request.setResponseJson(params.getJSONObject("response"));
+    }
+
+    private void handleResponseReceivedExtraInfo(JSONObject params) {
+        ChromeRequest request = requestMap.get(params.getString("requestId"));
+        if (request == null) {
+            logger.log(WARNING, "Got responseReceivedExtraInfo event without corresponding requestWillBeSent");
+            return;
+        }
+        request.setRawResponseHeaders(params.getJSONObject("headers"));
+        request.setResponseHeadersText(params.getString("headersText"));
+    }
+
+    private void handleLoadingFinished(JSONObject params) {
+        ChromeRequest request = requestMap.get(params.getString("requestId"));
+        if (request == null) {
+            logger.log(WARNING, "Got loadingFinished event without corresponding requestWillBeSent");
+            return;
+        }
+        if (requestConsumer != null) {
+            requestConsumer.accept(request);
+        }
+    }
+
     @Override
     public void close() {
         if (closed) return;
         closed = true;
+        eventExecutor.shutdown();
+        try {
+            eventExecutor.awaitTermination(1, TimeUnit.SECONDS);
+        } catch (InterruptedException e) {
+            Thread.currentThread().interrupt();
+        }
         client.call("Target.closeTarget", "targetId", targetId);
         client.sessionEventHandlers.remove(sessionId);
     }
+
+    public void captureRequests(Consumer<ChromeRequest> requestConsumer) {
+        this.requestConsumer = requestConsumer;
+        call("Network.enable");
+    }
+
+    private void handlePausedRequest(JSONObject params) {
+        if (params.has("responseStatusCode")) {
+            String stream = call("Fetch.takeResponseBodyAsStream", "requestId",
+                    params.getString("requestId")).getString("stream");
+            System.out.println(call("IO.read", "handle", stream));
+            System.out.println("stream " + stream);
+            call("IO.close", "handle", stream);
+        } else {
+            call("Fetch.continueRequest", "requestId", params.getString("requestId"));
+        }
+    }
 }
\ No newline at end of file
diff --git a/contrib/src/test/java/org/archive/modules/extractor/ExtractorChromeTest.java b/contrib/src/test/java/org/archive/modules/extractor/ExtractorChromeTest.java
index ba77fa21..17a9db14 100644
--- a/contrib/src/test/java/org/archive/modules/extractor/ExtractorChromeTest.java
+++ b/contrib/src/test/java/org/archive/modules/extractor/ExtractorChromeTest.java
@@ -19,34 +19,144 @@
 
 package org.archive.modules.extractor;
 
-import org.apache.commons.httpclient.URIException;
+import org.archive.crawler.framework.CrawlController;
+import org.archive.crawler.framework.Frontier;
+import org.archive.crawler.reporting.CrawlerLoggerModule;
 import org.archive.modules.CrawlURI;
+import org.archive.modules.DispositionChain;
+import org.archive.modules.Processor;
 import org.archive.net.UURIFactory;
+import org.eclipse.jetty.server.Request;
+import org.eclipse.jetty.server.Server;
+import org.eclipse.jetty.server.handler.AbstractHandler;
+import org.junit.AfterClass;
+import org.junit.BeforeClass;
+import org.junit.Rule;
 import org.junit.Test;
+import org.junit.rules.TemporaryFolder;
 
+import javax.servlet.ServletException;
+import javax.servlet.http.HttpServletRequest;
+import javax.servlet.http.HttpServletResponse;
+import java.io.IOException;
+import java.net.InetSocketAddress;
+import java.util.ArrayList;
+import java.util.Arrays;
 import java.util.Collections;
 import java.util.List;
+import java.util.logging.Level;
+import java.util.logging.Logger;
 
 import static java.util.stream.Collectors.toList;
-import static org.junit.Assert.assertEquals;
+import static org.archive.modules.CrawlURI.FetchType.HTTP_GET;
+import static org.archive.modules.CrawlURI.FetchType.HTTP_POST;
+import static org.easymock.EasyMock.*;
+import static org.junit.Assert.*;
 import static org.junit.Assume.assumeNoException;
 
 public class ExtractorChromeTest {
+    private static Server server;
+
+    @BeforeClass
+    public static void startServer() throws Exception {
+        server = new Server(InetSocketAddress.createUnresolved("127.0.0.1", 7778));
+        server.setHandler(new AbstractHandler() {
+            @Override
+            public void handle(String target, Request baseRequest, HttpServletRequest request, HttpServletResponse response) throws IOException, ServletException {
+                switch (target) {
+                    case "/":
+                        response.setContentType("text/html");
+                        response.getWriter().write("<a href=http://example.org/page2.html>link</a>" +
+                                "<img src=/blue.png>" +
+                                "<script>fetch('/post', {method: 'POST', body: 'hello'});</script>");
+                        baseRequest.setHandled(true);
+                        break;
+                    case "/blue.png":
+                        response.setContentType("image/png");
+                        response.getWriter().write("bogus png");
+                        baseRequest.setHandled(true);
+                        break;
+                    case "/post":
+                        response.setContentType("plain/text");
+                        response.getWriter().write("method=" + request.getMethod());
+                        baseRequest.setHandled(true);
+                        break;
+                    default:
+                        System.err.println("Unhandled target: " + target);
+                        break;
+                }
+
+            }
+        });
+        server.start();
+    }
+
+    @AfterClass
+    public static void stopServer() throws Exception {
+        server.stop();
+    }
+
+    @Rule
+    public TemporaryFolder tempFolder = new TemporaryFolder();
+
     @Test
-    public void test() throws URIException {
-        ExtractorChrome extractor = new ExtractorChrome();
+    public void test() throws IOException {
+        List<CrawlURI> processedURIs = Collections.synchronizedList(new ArrayList<>());
+        CrawlController controller = new CrawlController();
+        DispositionChain dispositionChain = new DispositionChain();
+        dispositionChain.setProcessors(Arrays.asList(new Processor() {
+            @Override
+            protected boolean shouldProcess(CrawlURI uri) {
+                return true;
+            }
+
+            @Override
+            protected void innerProcess(CrawlURI uri) {
+                processedURIs.add(uri);
+            }
+        }));
+        controller.setDispositionChain(dispositionChain);
+        controller.setLoggerModule(new CrawlerLoggerModule() {
+            @Override
+            public Logger getUriProcessing() {
+                Logger logger = Logger.getAnonymousLogger();
+                logger.setLevel(Level.WARNING);
+                return logger;
+            }
+        });
+        Frontier frontier = createMock(Frontier.class);
+        frontier.considerIncluded(anyObject());
+        expectLastCall().anyTimes();
+        frontier.beginDisposition(anyObject());
+        expectLastCall().anyTimes();
+        frontier.endDisposition();
+        expectLastCall().anyTimes();
+        frontier.finished(anyObject());
+        expectLastCall().anyTimes();
+        replay(frontier);
+        controller.setFrontier(frontier);
+
+        ExtractorChrome extractor = new ExtractorChrome(controller, event -> { /* ignored */ });
         try {
             extractor.start();
         } catch (RuntimeException e) {
             assumeNoException("Unable to start Chrome", e);
         }
         try {
-            CrawlURI curi = new CrawlURI(UURIFactory.getInstance("data:text/html,<a href=http://example.org/page2.html>link</a>"));
+            CrawlURI curi = new CrawlURI(UURIFactory.getInstance("http://127.0.0.1:7778/"));
             extractor.innerExtract(curi);
             List<String> outLinks = curi.getOutLinks().stream().map(CrawlURI::toString).sorted().collect(toList());
             assertEquals(Collections.singletonList("http://example.org/page2.html"), outLinks);
         } finally {
             extractor.stop();
         }
+
+        assertEquals(3, processedURIs.size());
+        for (CrawlURI curi: processedURIs) {
+            assertEquals(200, curi.getFetchStatus());
+            assertEquals(curi.getUURI().getPath().equals("/post") ? HTTP_POST : HTTP_GET, curi.getFetchType());
+            assertNotNull(curi.getContentDigest());
+            assertTrue(curi.getContentSize() > 0);
+        }
     }
 }
\ No newline at end of file