diff --git a/contrib/src/main/java/org/archive/modules/extractor/ExtractorChrome.java b/contrib/src/main/java/org/archive/modules/extractor/ExtractorChrome.java index 42379adf..8bc74da5 100644 --- a/contrib/src/main/java/org/archive/modules/extractor/ExtractorChrome.java +++ b/contrib/src/main/java/org/archive/modules/extractor/ExtractorChrome.java @@ -19,17 +19,39 @@ package org.archive.modules.extractor; +import org.archive.crawler.event.CrawlURIDispositionEvent; +import org.archive.crawler.framework.CrawlController; +import org.archive.crawler.framework.Frontier; import org.archive.modules.CrawlURI; import org.archive.net.chrome.ChromeClient; import org.archive.net.chrome.ChromeProcess; +import org.archive.net.chrome.ChromeRequest; import org.archive.net.chrome.ChromeWindow; +import org.archive.spring.KeyedProperties; +import org.archive.util.Recorder; import org.json.JSONArray; +import org.springframework.context.ApplicationEventPublisher; +import java.io.ByteArrayInputStream; import java.io.IOException; +import java.io.InputStream; +import java.io.SequenceInputStream; +import java.util.Arrays; import java.util.concurrent.ExecutionException; import java.util.concurrent.Semaphore; import java.util.concurrent.TimeUnit; import java.util.concurrent.TimeoutException; +import java.util.logging.Logger; +import java.util.regex.Pattern; + +import static java.nio.charset.StandardCharsets.US_ASCII; +import static java.util.Collections.enumeration; +import static java.util.logging.Level.INFO; +import static java.util.logging.Level.WARNING; +import static java.util.regex.Pattern.CASE_INSENSITIVE; +import static org.archive.crawler.event.CrawlURIDispositionEvent.Disposition.FAILED; +import static org.archive.crawler.event.CrawlURIDispositionEvent.Disposition.SUCCEEDED; +import static org.archive.modules.CrawlURI.FetchType.*; /** * Extracts links using a web browser via the Chrome Devtools Protocol. @@ -37,6 +59,7 @@ import java.util.concurrent.TimeoutException; * To use, first define this as a top-level bean: *
  * <bean id="extractorChrome" class="org.archive.modules.extractor.ExtractorChrome">
+ *   <!-- <property name="captureRequests" value="true" /> -->
  *   <!-- <property name="devtoolsUrl" value="ws://127.0.0.1:1234/devtools/browser/2bc831e8-6c02-4c9b-affd-14c93b8579d7" /> -->
  *   <!-- <property name="executable" value="chromium-browser" /> -->
  *   <!-- <property name="loadTimeoutSeconds" value="30" /> -->
@@ -52,6 +75,10 @@ import java.util.concurrent.TimeoutException;
  * --headless --remote-debugging-port=1234).
  */
 public class ExtractorChrome extends ContentExtractor {
+    private static final Logger logger = Logger.getLogger(ExtractorChrome.class.getName());
+
+    private static final Pattern TRANSFER_ENCODING_RE = Pattern.compile("\r\nTransfer-Encoding:[^\n\r]+", CASE_INSENSITIVE);
+
     /**
      * The maximum number of browser windows that are allowed to be opened simultaneously. Feel free to increase this
      * if you have lots of RAM available.
@@ -83,10 +110,23 @@ public class ExtractorChrome extends ContentExtractor {
      */
     private int loadTimeoutSeconds = 30;
 
+    /**
+     * Capture requests made by the browser.
+     */
+    private boolean captureRequests = true;
+
     private Semaphore openWindowsSemaphore = null;
     private ChromeProcess process = null;
     private ChromeClient client = null;
 
+    private final CrawlController controller;
+    private final ApplicationEventPublisher eventPublisher;
+
+    public ExtractorChrome(CrawlController controller, ApplicationEventPublisher eventPublisher) {
+        this.controller = controller;
+        this.eventPublisher = eventPublisher;
+    }
+
     @Override
     protected boolean shouldExtract(CrawlURI uri) {
         return uri.getContentType().startsWith("text/html");
@@ -107,25 +147,105 @@ public class ExtractorChrome extends ContentExtractor {
         return false;
     }
 
-    private void visit(CrawlURI uri) throws InterruptedException {
+    private void visit(CrawlURI curi) throws InterruptedException {
         try (ChromeWindow window = client.createWindow(windowWidth, windowHeight)) {
+            if (captureRequests) {
+                window.captureRequests(request -> handleCapturedRequest(curi, request));
+            }
+
             try {
-                window.navigateAsync(uri.getURI()).get(loadTimeoutSeconds, TimeUnit.SECONDS);
+                window.navigateAsync(curi.getURI()).get(loadTimeoutSeconds, TimeUnit.SECONDS);
             } catch (ExecutionException e) {
                 throw new RuntimeException(e.getCause());
             } catch (TimeoutException e) {
-                throw new RuntimeException("Timed out navigating to " + uri.getURI());
+                throw new RuntimeException("Timed out navigating to " + curi.getURI());
             }
 
             JSONArray links = window.eval("Array.from(document.querySelectorAll('a[href], area[href]'))" +
                     ".map(link => link.protocol + '//' + link.host + link.pathname + link.search + link.hash)")
                     .getJSONArray("value");
             for (int i = 0; i < links.length(); i++) {
-                addOutlink(uri, links.getString(i), LinkContext.NAVLINK_MISC, Hop.NAVLINK);
+                addOutlink(curi, links.getString(i), LinkContext.NAVLINK_MISC, Hop.NAVLINK);
             }
         }
     }
 
+    private void handleCapturedRequest(CrawlURI via, ChromeRequest request) {
+        Recorder recorder = new Recorder(controller.getScratchDir().getFile(),
+                controller.getRecorderOutBufferBytes(),
+                controller.getRecorderInBufferBytes());
+        try {
+            String digestAlgorithm = "sha1";
+            recorder.getRecordedInput().setDigest(digestAlgorithm);
+            recorder.getRecordedOutput().write(request.getRequestHeader().getBytes(US_ASCII));
+            recorder.getRecordedOutput().write(request.getRequestBody());
+
+            // strip the Transfer-Encoding header since we only have access to the decoded body
+            String responseHeader = TRANSFER_ENCODING_RE.matcher(request.getResponseHeader()).replaceAll("");
+
+            recorder.inputWrap(new SequenceInputStream(enumeration(Arrays.asList(
+                    new ByteArrayInputStream(responseHeader.getBytes(US_ASCII)),
+                    new InputStream() {
+                        public int read() {
+                            recorder.markContentBegin();
+                            return -1;
+                        }
+                    },
+                    new ByteArrayInputStream(request.getResponseBody())))));
+            recorder.getRecordedInput().readFully();
+            recorder.closeRecorders();
+
+            CrawlURI curi = via.createCrawlURI(request.getUrl(), LinkContext.EMBED_MISC, Hop.EMBED);
+            curi.getAnnotations().add("browser");
+            curi.setContentDigest(digestAlgorithm, recorder.getRecordedInput().getDigestValue());
+            curi.setContentSize(recorder.getRecordedInput().getSize());
+            curi.setContentType(request.getResponseContentType());
+            curi.setFetchBeginTime(request.getBeginTime());
+            curi.setFetchCompletedTime(System.currentTimeMillis());
+            curi.setFetchStatus(request.getStatus());
+            curi.setRecorder(recorder);
+            curi.setServerIP(request.getRemoteIPAddress());
+            curi.setThreadNumber(via.getThreadNumber());
+
+            switch (request.getMethod()) {
+                case "GET":
+                    curi.setFetchType(HTTP_GET);
+                    break;
+                case "POST":
+                    curi.setFetchType(HTTP_POST);
+                    break;
+                default:
+                    curi.setFetchType(UNKNOWN);
+                    break;
+            }
+
+            // send it to the disposition chain to invoke the warc writer etc
+            Frontier frontier = controller.getFrontier(); // allowed to be null to simplify unit tests
+            curi.getOverlayNames(); // for side-effect of creating the overlayNames list
+            KeyedProperties.loadOverridesFrom(curi);
+            try {
+                frontier.beginDisposition(curi);
+                controller.getDispositionChain().process(curi,null);
+            } finally {
+                KeyedProperties.clearOverridesFrom(curi);
+            }
+
+            curi.aboutToLog();
+            controller.getLoggerModule().getUriProcessing().log(INFO, curi.getUURI().toString(), curi);
+
+            if (curi.isSuccess()) {
+                eventPublisher.publishEvent(new CrawlURIDispositionEvent(this, curi, SUCCEEDED));
+            } else {
+                eventPublisher.publishEvent(new CrawlURIDispositionEvent(this, curi, FAILED));
+            }
+            frontier.endDisposition();
+        } catch (Exception e) {
+            logger.log(WARNING, "Exception handling subrequest " + request.getUrl(), e);
+        } finally {
+            recorder.cleanup();
+        }
+    }
+
     @Override
     public void start() {
         if (isRunning) return;
diff --git a/contrib/src/main/java/org/archive/net/chrome/ChromeClient.java b/contrib/src/main/java/org/archive/net/chrome/ChromeClient.java
index 7de68b3a..cc81598b 100644
--- a/contrib/src/main/java/org/archive/net/chrome/ChromeClient.java
+++ b/contrib/src/main/java/org/archive/net/chrome/ChromeClient.java
@@ -29,6 +29,7 @@ import java.util.Map;
 import java.util.concurrent.*;
 import java.util.concurrent.atomic.AtomicLong;
 import java.util.function.Consumer;
+import java.util.logging.Level;
 import java.util.logging.Logger;
 
 import static java.util.logging.Level.WARNING;
@@ -43,7 +44,6 @@ public class ChromeClient implements Closeable {
     private final DevtoolsSocket devtoolsSocket;
     private final AtomicLong nextMessageId = new AtomicLong(0);
     private final Map> responseFutures = new ConcurrentHashMap<>();
-    private final ExecutorService threadPool = Executors.newCachedThreadPool();
     final ConcurrentHashMap> sessionEventHandlers = new ConcurrentHashMap<>();
 
     public ChromeClient(String devtoolsUrl) {
@@ -110,16 +110,7 @@ public class ChromeClient implements Closeable {
             String sessionId = message.getString("sessionId");
             Consumer handler = sessionEventHandlers.get(sessionId);
             if (handler != null) {
-                // Run event handlers on a different thread so we don't block the websocket receiving thread.
-                // That would cause a deadlock if an event handler itself made an RPC call as the response could
-                // never be processed.
-                threadPool.submit(() -> {
-                    try {
-                        handler.accept(message);
-                    } catch (Throwable t) {
-                        logger.log(WARNING, "Exception handling browser event " + message, t);
-                    }
-                });
+                handler.accept(message);
             } else {
                 logger.log(WARNING, "Received event for unknown session {0}", sessionId);
             }
@@ -164,13 +155,14 @@ public class ChromeClient implements Closeable {
         }
 
         @Override
-        public void onClose(int i, String s, boolean b) {
-
+        public void onClose(int code, String reason, boolean remote) {
+            if (!remote) return;
+            logger.log(WARNING, "Websocket closed by browser: " + reason);
         }
 
         @Override
         public void onError(Exception e) {
-
+            logger.log(Level.SEVERE, "Websocket error", e);
         }
     }
 }
diff --git a/contrib/src/main/java/org/archive/net/chrome/ChromeRequest.java b/contrib/src/main/java/org/archive/net/chrome/ChromeRequest.java
new file mode 100644
index 00000000..68ce072b
--- /dev/null
+++ b/contrib/src/main/java/org/archive/net/chrome/ChromeRequest.java
@@ -0,0 +1,154 @@
+/*
+ *  This file is part of the Heritrix web crawler (crawler.archive.org).
+ *
+ *  Licensed to the Internet Archive (IA) by one or more individual
+ *  contributors.
+ *
+ *  The IA licenses this file to You under the Apache License, Version 2.0
+ *  (the "License"); you may not use this file except in compliance with
+ *  the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ */
+
+package org.archive.net.chrome;
+
+import org.json.JSONObject;
+
+import java.nio.charset.StandardCharsets;
+import java.util.Base64;
+
+public class ChromeRequest {
+    private final ChromeWindow window;
+    private final String id;
+    private JSONObject requestJson;
+    private JSONObject rawRequestHeaders;
+    private JSONObject responseJson;
+    private JSONObject rawResponseHeaders;
+    private String responseHeadersText;
+    private final long beginTime = System.currentTimeMillis();
+
+    public ChromeRequest(ChromeWindow window, String id) {
+        this.window = window;
+        this.id = id;
+    }
+
+    void setRawRequestHeaders(JSONObject rawRequestHeaders) {
+        this.rawRequestHeaders = rawRequestHeaders;
+    }
+
+    void setResponseJson(JSONObject responseJson) {
+        this.responseJson = responseJson;
+    }
+
+    void setRawResponseHeaders(JSONObject headers) {
+        this.rawResponseHeaders = headers;
+    }
+
+    void setResponseHeadersText(String responseHeadersText) {
+        this.responseHeadersText = responseHeadersText;
+    }
+
+    public String getUrl() {
+        return requestJson.getString("url");
+    }
+
+    public byte[] getResponseBody() {
+        JSONObject reply = window.call("Network.getResponseBody", "requestId", id);
+        byte[] body;
+        if (reply.getBoolean("base64Encoded")) {
+            body = Base64.getDecoder().decode(reply.getString("body"));
+        } else {
+            body = reply.getString("body").getBytes(StandardCharsets.UTF_8);
+        }
+        return body;
+    }
+
+    public String getRequestHeader() {
+        if (responseJson != null && responseJson.has("requestHeadersText")) {
+            return responseJson.getString("requestHeadersText");
+        }
+        StringBuilder builder = new StringBuilder();
+        builder.append(requestJson.getString("method"));
+        builder.append(' ');
+        builder.append(getUrl());
+        builder.append(" HTTP/1.1\r\n");
+        formatHeaders(builder, requestJson.getJSONObject("headers"), rawRequestHeaders);
+        return builder.toString();
+    }
+
+    public byte[] getRequestBody() {
+        if (requestJson.has("postData")) {
+            return requestJson.getString("postData").getBytes(StandardCharsets.UTF_8);
+        } else {
+            return new byte[0];
+        }
+    }
+
+    public String getResponseHeader() {
+        if (responseHeadersText != null) {
+            return responseHeadersText;
+        } else if (responseJson.has("headersText")) {
+            return responseJson.getString("headersText");
+        }
+        StringBuilder builder = new StringBuilder();
+        if (responseJson.getString("protocol").equals("http/1.0")) {
+            builder.append("HTTP/1.0");
+        } else {
+            builder.append("HTTP/1.1");
+        }
+        builder.append(getStatus());
+        builder.append(" ");
+        builder.append(responseJson.getString("statusText"));
+        builder.append("\r\n");
+        formatHeaders(builder, responseJson.getJSONObject("headers"), rawResponseHeaders);
+        return builder.toString();
+    }
+
+    private void formatHeaders(StringBuilder builder, JSONObject headers, JSONObject rawHeaders) {
+        if (rawHeaders != null) {
+            headers = rawHeaders;
+        }
+        for (Object key : headers.keySet()) {
+            builder.append(key);
+            builder.append(": ");
+            builder.append(headers.getString((String) key));
+            builder.append("\r\n");
+        }
+        builder.append("\r\n");
+    }
+
+    public String getMethod() {
+        return requestJson.getString("method");
+    }
+
+    public int getStatus() {
+        return responseJson.getInt("status");
+    }
+
+    public String getResponseContentType() {
+        return responseJson.getString("mimeType");
+    }
+
+    public long getBeginTime() {
+        return beginTime;
+    }
+
+    public String getRemoteIPAddress() {
+        if (responseJson.has("remoteIPAddress")) {
+            return responseJson.getString("remoteIPAddress");
+        } else {
+            return null;
+        }
+    }
+
+    void setRequestJson(JSONObject requestJson) {
+        this.requestJson = requestJson;
+    }
+}
diff --git a/contrib/src/main/java/org/archive/net/chrome/ChromeWindow.java b/contrib/src/main/java/org/archive/net/chrome/ChromeWindow.java
index 59e8f2e7..a65fa8cc 100644
--- a/contrib/src/main/java/org/archive/net/chrome/ChromeWindow.java
+++ b/contrib/src/main/java/org/archive/net/chrome/ChromeWindow.java
@@ -22,10 +22,13 @@ package org.archive.net.chrome;
 import org.json.JSONObject;
 
 import java.io.Closeable;
-import java.util.concurrent.CompletableFuture;
+import java.util.Map;
+import java.util.concurrent.*;
+import java.util.function.Consumer;
 import java.util.logging.Logger;
 
 import static java.util.logging.Level.FINE;
+import static java.util.logging.Level.WARNING;
 
 /**
  * A browser window or tab.
@@ -38,12 +41,17 @@ public class ChromeWindow implements Closeable {
     private final String sessionId;
     private boolean closed;
     private CompletableFuture loadEventFuture;
+    private final Map requestMap = new ConcurrentHashMap<>();
+    private Consumer requestConsumer;
+    private final ExecutorService eventExecutor;
 
     public ChromeWindow(ChromeClient client, String targetId) {
         this.client = client;
         this.targetId = targetId;
         this.sessionId = client.call("Target.attachToTarget", "targetId", targetId,
                 "flatten", true).getString("sessionId");
+        eventExecutor = Executors.newSingleThreadExecutor(runnable ->
+                new Thread(runnable, "ChromeWindow (sessionId=" + sessionId +")"));
         client.sessionEventHandlers.put(sessionId, this::handleEvent);
         call("Page.enable"); // for loadEventFired
         call("Page.setLifecycleEventsEnabled", "enabled", true); // for networkidle
@@ -78,9 +86,47 @@ public class ChromeWindow implements Closeable {
     }
 
     private void handleEvent(JSONObject message) {
+        // Run event handlers on a different thread so we don't block the websocket receiving thread.
+        // That would cause a deadlock if an event handler itself made an RPC call as the response could
+        // never be processed.
+        // We use a single thread per window though as the order events are processed is important.
+        eventExecutor.submit(() -> {
+            try {
+                handleEventOnEventThread(message);
+            } catch (Throwable t) {
+                logger.log(WARNING, "Exception handling browser event " + message, t);
+            }
+        });
+    }
+
+    private void handleEventOnEventThread(JSONObject message) {
+        JSONObject params = message.getJSONObject("params");
         switch (message.getString("method")) {
+            case "Fetch.requestPaused":
+                handlePausedRequest(params);
+                break;
+            case "Network.requestWillBeSent":
+                handleRequestWillBeSent(params);
+                break;
+            case "Network.requestWillBeSentExtraInfo":
+                handleRequestWillBeSentExtraInfo(params);
+                break;
+            case "Network.responseReceived":
+                handleResponseReceived(params);
+                break;
+            case "Network.responseReceivedExtraInfo":
+                handleResponseReceivedExtraInfo(params);
+                break;
+            case "Network.loadingFinished":
+                handleLoadingFinished(params);
+                break;
             case "Page.loadEventFired":
                 if (loadEventFuture != null) {
+                    try {
+                        Thread.sleep(1000);
+                    } catch (InterruptedException e) {
+                        Thread.currentThread().interrupt();
+                    }
                     loadEventFuture.complete(null);
                 }
                 break;
@@ -90,11 +136,77 @@ public class ChromeWindow implements Closeable {
         }
     }
 
+    private void handleRequestWillBeSent(JSONObject params) {
+        String requestId = params.getString("requestId");
+        ChromeRequest request = requestMap.computeIfAbsent(requestId, id -> new ChromeRequest(this, id));
+        request.setRequestJson(params.getJSONObject("request"));
+    }
+
+    private void handleRequestWillBeSentExtraInfo(JSONObject params) {
+        // it seems this event can arrive both before and after requestWillBeSent so we need to cope with that
+        String requestId = params.getString("requestId");
+        ChromeRequest request = requestMap.computeIfAbsent(requestId, id -> new ChromeRequest(this, id));
+        request.setRawRequestHeaders(params.getJSONObject("headers"));
+    }
+
+    private void handleResponseReceived(JSONObject params) {
+        ChromeRequest request = requestMap.get(params.getString("requestId"));
+        if (request == null) {
+            logger.log(WARNING, "Got responseReceived event without corresponding requestWillBeSent");
+            return;
+        }
+        request.setResponseJson(params.getJSONObject("response"));
+    }
+
+    private void handleResponseReceivedExtraInfo(JSONObject params) {
+        ChromeRequest request = requestMap.get(params.getString("requestId"));
+        if (request == null) {
+            logger.log(WARNING, "Got responseReceivedExtraInfo event without corresponding requestWillBeSent");
+            return;
+        }
+        request.setRawResponseHeaders(params.getJSONObject("headers"));
+        request.setResponseHeadersText(params.getString("headersText"));
+    }
+
+    private void handleLoadingFinished(JSONObject params) {
+        ChromeRequest request = requestMap.get(params.getString("requestId"));
+        if (request == null) {
+            logger.log(WARNING, "Got loadingFinished event without corresponding requestWillBeSent");
+            return;
+        }
+        if (requestConsumer != null) {
+            requestConsumer.accept(request);
+        }
+    }
+
     @Override
     public void close() {
         if (closed) return;
         closed = true;
+        eventExecutor.shutdown();
+        try {
+            eventExecutor.awaitTermination(1, TimeUnit.SECONDS);
+        } catch (InterruptedException e) {
+            Thread.currentThread().interrupt();
+        }
         client.call("Target.closeTarget", "targetId", targetId);
         client.sessionEventHandlers.remove(sessionId);
     }
+
+    public void captureRequests(Consumer requestConsumer) {
+        this.requestConsumer = requestConsumer;
+        call("Network.enable");
+    }
+
+    private void handlePausedRequest(JSONObject params) {
+        if (params.has("responseStatusCode")) {
+            String stream = call("Fetch.takeResponseBodyAsStream", "requestId",
+                    params.getString("requestId")).getString("stream");
+            System.out.println(call("IO.read", "handle", stream));
+            System.out.println("stream " + stream);
+            call("IO.close", "handle", stream);
+        } else {
+            call("Fetch.continueRequest", "requestId", params.getString("requestId"));
+        }
+    }
 }
\ No newline at end of file
diff --git a/contrib/src/test/java/org/archive/modules/extractor/ExtractorChromeTest.java b/contrib/src/test/java/org/archive/modules/extractor/ExtractorChromeTest.java
index ba77fa21..17a9db14 100644
--- a/contrib/src/test/java/org/archive/modules/extractor/ExtractorChromeTest.java
+++ b/contrib/src/test/java/org/archive/modules/extractor/ExtractorChromeTest.java
@@ -19,34 +19,144 @@
 
 package org.archive.modules.extractor;
 
-import org.apache.commons.httpclient.URIException;
+import org.archive.crawler.framework.CrawlController;
+import org.archive.crawler.framework.Frontier;
+import org.archive.crawler.reporting.CrawlerLoggerModule;
 import org.archive.modules.CrawlURI;
+import org.archive.modules.DispositionChain;
+import org.archive.modules.Processor;
 import org.archive.net.UURIFactory;
+import org.eclipse.jetty.server.Request;
+import org.eclipse.jetty.server.Server;
+import org.eclipse.jetty.server.handler.AbstractHandler;
+import org.junit.AfterClass;
+import org.junit.BeforeClass;
+import org.junit.Rule;
 import org.junit.Test;
+import org.junit.rules.TemporaryFolder;
 
+import javax.servlet.ServletException;
+import javax.servlet.http.HttpServletRequest;
+import javax.servlet.http.HttpServletResponse;
+import java.io.IOException;
+import java.net.InetSocketAddress;
+import java.util.ArrayList;
+import java.util.Arrays;
 import java.util.Collections;
 import java.util.List;
+import java.util.logging.Level;
+import java.util.logging.Logger;
 
 import static java.util.stream.Collectors.toList;
-import static org.junit.Assert.assertEquals;
+import static org.archive.modules.CrawlURI.FetchType.HTTP_GET;
+import static org.archive.modules.CrawlURI.FetchType.HTTP_POST;
+import static org.easymock.EasyMock.*;
+import static org.junit.Assert.*;
 import static org.junit.Assume.assumeNoException;
 
 public class ExtractorChromeTest {
+    private static Server server;
+
+    @BeforeClass
+    public static void startServer() throws Exception {
+        server = new Server(InetSocketAddress.createUnresolved("127.0.0.1", 7778));
+        server.setHandler(new AbstractHandler() {
+            @Override
+            public void handle(String target, Request baseRequest, HttpServletRequest request, HttpServletResponse response) throws IOException, ServletException {
+                switch (target) {
+                    case "/":
+                        response.setContentType("text/html");
+                        response.getWriter().write("link" +
+                                "" +
+                                "");
+                        baseRequest.setHandled(true);
+                        break;
+                    case "/blue.png":
+                        response.setContentType("image/png");
+                        response.getWriter().write("bogus png");
+                        baseRequest.setHandled(true);
+                        break;
+                    case "/post":
+                        response.setContentType("plain/text");
+                        response.getWriter().write("method=" + request.getMethod());
+                        baseRequest.setHandled(true);
+                        break;
+                    default:
+                        System.err.println("Unhandled target: " + target);
+                        break;
+                }
+
+            }
+        });
+        server.start();
+    }
+
+    @AfterClass
+    public static void stopServer() throws Exception {
+        server.stop();
+    }
+
+    @Rule
+    public TemporaryFolder tempFolder = new TemporaryFolder();
+
     @Test
-    public void test() throws URIException {
-        ExtractorChrome extractor = new ExtractorChrome();
+    public void test() throws IOException {
+        List processedURIs = Collections.synchronizedList(new ArrayList<>());
+        CrawlController controller = new CrawlController();
+        DispositionChain dispositionChain = new DispositionChain();
+        dispositionChain.setProcessors(Arrays.asList(new Processor() {
+            @Override
+            protected boolean shouldProcess(CrawlURI uri) {
+                return true;
+            }
+
+            @Override
+            protected void innerProcess(CrawlURI uri) {
+                processedURIs.add(uri);
+            }
+        }));
+        controller.setDispositionChain(dispositionChain);
+        controller.setLoggerModule(new CrawlerLoggerModule() {
+            @Override
+            public Logger getUriProcessing() {
+                Logger logger = Logger.getAnonymousLogger();
+                logger.setLevel(Level.WARNING);
+                return logger;
+            }
+        });
+        Frontier frontier = createMock(Frontier.class);
+        frontier.considerIncluded(anyObject());
+        expectLastCall().anyTimes();
+        frontier.beginDisposition(anyObject());
+        expectLastCall().anyTimes();
+        frontier.endDisposition();
+        expectLastCall().anyTimes();
+        frontier.finished(anyObject());
+        expectLastCall().anyTimes();
+        replay(frontier);
+        controller.setFrontier(frontier);
+
+        ExtractorChrome extractor = new ExtractorChrome(controller, event -> { /* ignored */ });
         try {
             extractor.start();
         } catch (RuntimeException e) {
             assumeNoException("Unable to start Chrome", e);
         }
         try {
-            CrawlURI curi = new CrawlURI(UURIFactory.getInstance("data:text/html,link"));
+            CrawlURI curi = new CrawlURI(UURIFactory.getInstance("http://127.0.0.1:7778/"));
             extractor.innerExtract(curi);
             List outLinks = curi.getOutLinks().stream().map(CrawlURI::toString).sorted().collect(toList());
             assertEquals(Collections.singletonList("http://example.org/page2.html"), outLinks);
         } finally {
             extractor.stop();
         }
+
+        assertEquals(3, processedURIs.size());
+        for (CrawlURI curi: processedURIs) {
+            assertEquals(200, curi.getFetchStatus());
+            assertEquals(curi.getUURI().getPath().equals("/post") ? HTTP_POST : HTTP_GET, curi.getFetchType());
+            assertNotNull(curi.getContentDigest());
+            assertTrue(curi.getContentSize() > 0);
+        }
     }
 }
\ No newline at end of file