Merge pull request #411 from internetarchive/chrome-request-capturing

ExtractorChrome: Capture requests made by the browser
2026-09-25 07:05:49 +00:00 · 2021-07-12 17:49:31 +09:00
parent c54f18e3ff aa265855ef
commit 812ea0f720
5 changed files with 512 additions and 24 deletions
@@ -19,17 +19,39 @@

 package org.archive.modules.extractor;

+import org.archive.crawler.event.CrawlURIDispositionEvent;
+import org.archive.crawler.framework.CrawlController;
+import org.archive.crawler.framework.Frontier;
 import org.archive.modules.CrawlURI;
 import org.archive.net.chrome.ChromeClient;
 import org.archive.net.chrome.ChromeProcess;
+import org.archive.net.chrome.ChromeRequest;
 import org.archive.net.chrome.ChromeWindow;
+import org.archive.spring.KeyedProperties;
+import org.archive.util.Recorder;
 import org.json.JSONArray;
+import org.springframework.context.ApplicationEventPublisher;

+import java.io.ByteArrayInputStream;
 import java.io.IOException;
+import java.io.InputStream;
+import java.io.SequenceInputStream;
+import java.util.Arrays;
 import java.util.concurrent.ExecutionException;
 import java.util.concurrent.Semaphore;
 import java.util.concurrent.TimeUnit;
 import java.util.concurrent.TimeoutException;
+import java.util.logging.Logger;
+import java.util.regex.Pattern;
+
+import static java.nio.charset.StandardCharsets.US_ASCII;
+import static java.util.Collections.enumeration;
+import static java.util.logging.Level.INFO;
+import static java.util.logging.Level.WARNING;
+import static java.util.regex.Pattern.CASE_INSENSITIVE;
+import static org.archive.crawler.event.CrawlURIDispositionEvent.Disposition.FAILED;
+import static org.archive.crawler.event.CrawlURIDispositionEvent.Disposition.SUCCEEDED;
+import static org.archive.modules.CrawlURI.FetchType.*;

 /**
 * Extracts links using a web browser via the Chrome Devtools Protocol.
@@ -37,6 +59,7 @@ import java.util.concurrent.TimeoutException;
 * To use, first define this as a top-level bean:
 * <pre>
 * &lt;bean id=&quot;extractorChrome&quot; class=&quot;org.archive.modules.extractor.ExtractorChrome&quot;&gt;
+ *   &lt;!-- &lt;property name=&quot;captureRequests&quot; value=&quot;true&quot; /&gt; --&gt;
 *   &lt;!-- &lt;property name=&quot;devtoolsUrl&quot; value=&quot;ws://127.0.0.1:1234/devtools/browser/2bc831e8-6c02-4c9b-affd-14c93b8579d7&quot; /&gt; --&gt;
 *   &lt;!-- &lt;property name=&quot;executable&quot; value=&quot;chromium-browser&quot; /&gt; --&gt;
 *   &lt;!-- &lt;property name=&quot;loadTimeoutSeconds&quot; value=&quot;30&quot; /&gt; --&gt;
@@ -52,6 +75,10 @@ import java.util.concurrent.TimeoutException;
 * <code>--headless --remote-debugging-port=1234</code>).
 */
 public class ExtractorChrome extends ContentExtractor {
+    private static final Logger logger = Logger.getLogger(ExtractorChrome.class.getName());
+
+    private static final Pattern TRANSFER_ENCODING_RE = Pattern.compile("\r\nTransfer-Encoding:[^\n\r]+", CASE_INSENSITIVE);
+
    /**
     * The maximum number of browser windows that are allowed to be opened simultaneously. Feel free to increase this
     * if you have lots of RAM available.
@@ -83,10 +110,23 @@ public class ExtractorChrome extends ContentExtractor {
     */
    private int loadTimeoutSeconds = 30;

+    /**
+     * Capture requests made by the browser.
+     */
+    private boolean captureRequests = true;
+
    private Semaphore openWindowsSemaphore = null;
    private ChromeProcess process = null;
    private ChromeClient client = null;

+    private final CrawlController controller;
+    private final ApplicationEventPublisher eventPublisher;
+
+    public ExtractorChrome(CrawlController controller, ApplicationEventPublisher eventPublisher) {
+        this.controller = controller;
+        this.eventPublisher = eventPublisher;
+    }
+
    @Override
    protected boolean shouldExtract(CrawlURI uri) {
        return uri.getContentType().startsWith("text/html");
@@ -107,25 +147,105 @@ public class ExtractorChrome extends ContentExtractor {
        return false;
    }

-    private void visit(CrawlURI uri) throws InterruptedException {
+    private void visit(CrawlURI curi) throws InterruptedException {
        try (ChromeWindow window = client.createWindow(windowWidth, windowHeight)) {
+            if (captureRequests) {
+                window.captureRequests(request -> handleCapturedRequest(curi, request));
+            }
+
            try {
-                window.navigateAsync(uri.getURI()).get(loadTimeoutSeconds, TimeUnit.SECONDS);
+                window.navigateAsync(curi.getURI()).get(loadTimeoutSeconds, TimeUnit.SECONDS);
            } catch (ExecutionException e) {
                throw new RuntimeException(e.getCause());
            } catch (TimeoutException e) {
-                throw new RuntimeException("Timed out navigating to " + uri.getURI());
+                throw new RuntimeException("Timed out navigating to " + curi.getURI());
            }

            JSONArray links = window.eval("Array.from(document.querySelectorAll('a[href], area[href]'))" +
                    ".map(link => link.protocol + '//' + link.host + link.pathname + link.search + link.hash)")
                    .getJSONArray("value");
            for (int i = 0; i < links.length(); i++) {
-                addOutlink(uri, links.getString(i), LinkContext.NAVLINK_MISC, Hop.NAVLINK);
+                addOutlink(curi, links.getString(i), LinkContext.NAVLINK_MISC, Hop.NAVLINK);
            }
        }
    }

+    private void handleCapturedRequest(CrawlURI via, ChromeRequest request) {
+        Recorder recorder = new Recorder(controller.getScratchDir().getFile(),
+                controller.getRecorderOutBufferBytes(),
+                controller.getRecorderInBufferBytes());
+        try {
+            String digestAlgorithm = "sha1";
+            recorder.getRecordedInput().setDigest(digestAlgorithm);
+            recorder.getRecordedOutput().write(request.getRequestHeader().getBytes(US_ASCII));
+            recorder.getRecordedOutput().write(request.getRequestBody());
+
+            // strip the Transfer-Encoding header since we only have access to the decoded body
+            String responseHeader = TRANSFER_ENCODING_RE.matcher(request.getResponseHeader()).replaceAll("");
+
+            recorder.inputWrap(new SequenceInputStream(enumeration(Arrays.asList(
+                    new ByteArrayInputStream(responseHeader.getBytes(US_ASCII)),
+                    new InputStream() {
+                        public int read() {
+                            recorder.markContentBegin();
+                            return -1;
+                        }
+                    },
+                    new ByteArrayInputStream(request.getResponseBody())))));
+            recorder.getRecordedInput().readFully();
+            recorder.closeRecorders();
+
+            CrawlURI curi = via.createCrawlURI(request.getUrl(), LinkContext.EMBED_MISC, Hop.EMBED);
+            curi.getAnnotations().add("browser");
+            curi.setContentDigest(digestAlgorithm, recorder.getRecordedInput().getDigestValue());
+            curi.setContentSize(recorder.getRecordedInput().getSize());
+            curi.setContentType(request.getResponseContentType());
+            curi.setFetchBeginTime(request.getBeginTime());
+            curi.setFetchCompletedTime(System.currentTimeMillis());
+            curi.setFetchStatus(request.getStatus());
+            curi.setRecorder(recorder);
+            curi.setServerIP(request.getRemoteIPAddress());
+            curi.setThreadNumber(via.getThreadNumber());
+
+            switch (request.getMethod()) {
+                case "GET":
+                    curi.setFetchType(HTTP_GET);
+                    break;
+                case "POST":
+                    curi.setFetchType(HTTP_POST);
+                    break;
+                default:
+                    curi.setFetchType(UNKNOWN);
+                    break;
+            }
+
+            // send it to the disposition chain to invoke the warc writer etc
+            Frontier frontier = controller.getFrontier(); // allowed to be null to simplify unit tests
+            curi.getOverlayNames(); // for side-effect of creating the overlayNames list
+            KeyedProperties.loadOverridesFrom(curi);
+            try {
+                frontier.beginDisposition(curi);
+                controller.getDispositionChain().process(curi,null);
+            } finally {
+                KeyedProperties.clearOverridesFrom(curi);
+            }
+
+            curi.aboutToLog();
+            controller.getLoggerModule().getUriProcessing().log(INFO, curi.getUURI().toString(), curi);
+
+            if (curi.isSuccess()) {
+                eventPublisher.publishEvent(new CrawlURIDispositionEvent(this, curi, SUCCEEDED));
+            } else {
+                eventPublisher.publishEvent(new CrawlURIDispositionEvent(this, curi, FAILED));
+            }
+            frontier.endDisposition();
+        } catch (Exception e) {
+            logger.log(WARNING, "Exception handling subrequest " + request.getUrl(), e);
+        } finally {
+            recorder.cleanup();
+        }
+    }
+
    @Override
    public void start() {
        if (isRunning) return;
@@ -29,6 +29,7 @@ import java.util.Map;
 import java.util.concurrent.*;
 import java.util.concurrent.atomic.AtomicLong;
 import java.util.function.Consumer;
+import java.util.logging.Level;
 import java.util.logging.Logger;

 import static java.util.logging.Level.WARNING;
@@ -43,7 +44,6 @@ public class ChromeClient implements Closeable {
    private final DevtoolsSocket devtoolsSocket;
    private final AtomicLong nextMessageId = new AtomicLong(0);
    private final Map<Long, CompletableFuture<JSONObject>> responseFutures = new ConcurrentHashMap<>();
-    private final ExecutorService threadPool = Executors.newCachedThreadPool();
    final ConcurrentHashMap<String, Consumer<JSONObject>> sessionEventHandlers = new ConcurrentHashMap<>();

    public ChromeClient(String devtoolsUrl) {
@@ -110,16 +110,7 @@ public class ChromeClient implements Closeable {
            String sessionId = message.getString("sessionId");
            Consumer<JSONObject> handler = sessionEventHandlers.get(sessionId);
            if (handler != null) {
-                // Run event handlers on a different thread so we don't block the websocket receiving thread.
-                // That would cause a deadlock if an event handler itself made an RPC call as the response could
-                // never be processed.
-                threadPool.submit(() -> {
-                    try {
-                        handler.accept(message);
-                    } catch (Throwable t) {
-                        logger.log(WARNING, "Exception handling browser event " + message, t);
-                    }
-                });
+                handler.accept(message);
            } else {
                logger.log(WARNING, "Received event for unknown session {0}", sessionId);
            }
@@ -164,13 +155,14 @@ public class ChromeClient implements Closeable {
        }

        @Override
-        public void onClose(int i, String s, boolean b) {
-
+        public void onClose(int code, String reason, boolean remote) {
+            if (!remote) return;
+            logger.log(WARNING, "Websocket closed by browser: " + reason);
        }

        @Override
        public void onError(Exception e) {
-
+            logger.log(Level.SEVERE, "Websocket error", e);
        }
    }
 }
@@ -0,0 +1,154 @@
+/*
+ *  This file is part of the Heritrix web crawler (crawler.archive.org).
+ *
+ *  Licensed to the Internet Archive (IA) by one or more individual
+ *  contributors.
+ *
+ *  The IA licenses this file to You under the Apache License, Version 2.0
+ *  (the "License"); you may not use this file except in compliance with
+ *  the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ */
+
+package org.archive.net.chrome;
+
+import org.json.JSONObject;
+
+import java.nio.charset.StandardCharsets;
+import java.util.Base64;
+
+public class ChromeRequest {
+    private final ChromeWindow window;
+    private final String id;
+    private JSONObject requestJson;
+    private JSONObject rawRequestHeaders;
+    private JSONObject responseJson;
+    private JSONObject rawResponseHeaders;
+    private String responseHeadersText;
+    private final long beginTime = System.currentTimeMillis();
+
+    public ChromeRequest(ChromeWindow window, String id) {
+        this.window = window;
+        this.id = id;
+    }
+
+    void setRawRequestHeaders(JSONObject rawRequestHeaders) {
+        this.rawRequestHeaders = rawRequestHeaders;
+    }
+
+    void setResponseJson(JSONObject responseJson) {
+        this.responseJson = responseJson;
+    }
+
+    void setRawResponseHeaders(JSONObject headers) {
+        this.rawResponseHeaders = headers;
+    }
+
+    void setResponseHeadersText(String responseHeadersText) {
+        this.responseHeadersText = responseHeadersText;
+    }
+
+    public String getUrl() {
+        return requestJson.getString("url");
+    }
+
+    public byte[] getResponseBody() {
+        JSONObject reply = window.call("Network.getResponseBody", "requestId", id);
+        byte[] body;
+        if (reply.getBoolean("base64Encoded")) {
+            body = Base64.getDecoder().decode(reply.getString("body"));
+        } else {
+            body = reply.getString("body").getBytes(StandardCharsets.UTF_8);
+        }
+        return body;
+    }
+
+    public String getRequestHeader() {
+        if (responseJson != null && responseJson.has("requestHeadersText")) {
+            return responseJson.getString("requestHeadersText");
+        }
+        StringBuilder builder = new StringBuilder();
+        builder.append(requestJson.getString("method"));
+        builder.append(' ');
+        builder.append(getUrl());
+        builder.append(" HTTP/1.1\r\n");
+        formatHeaders(builder, requestJson.getJSONObject("headers"), rawRequestHeaders);
+        return builder.toString();
+    }
+
+    public byte[] getRequestBody() {
+        if (requestJson.has("postData")) {
+            return requestJson.getString("postData").getBytes(StandardCharsets.UTF_8);
+        } else {
+            return new byte[0];
+        }
+    }
+
+    public String getResponseHeader() {
+        if (responseHeadersText != null) {
+            return responseHeadersText;
+        } else if (responseJson.has("headersText")) {
+            return responseJson.getString("headersText");
+        }
+        StringBuilder builder = new StringBuilder();
+        if (responseJson.getString("protocol").equals("http/1.0")) {
+            builder.append("HTTP/1.0");
+        } else {
+            builder.append("HTTP/1.1");
+        }
+        builder.append(getStatus());
+        builder.append(" ");
+        builder.append(responseJson.getString("statusText"));
+        builder.append("\r\n");
+        formatHeaders(builder, responseJson.getJSONObject("headers"), rawResponseHeaders);
+        return builder.toString();
+    }
+
+    private void formatHeaders(StringBuilder builder, JSONObject headers, JSONObject rawHeaders) {
+        if (rawHeaders != null) {
+            headers = rawHeaders;
+        }
+        for (Object key : headers.keySet()) {
+            builder.append(key);
+            builder.append(": ");
+            builder.append(headers.getString((String) key));
+            builder.append("\r\n");
+        }
+        builder.append("\r\n");
+    }
+
+    public String getMethod() {
+        return requestJson.getString("method");
+    }
+
+    public int getStatus() {
+        return responseJson.getInt("status");
+    }
+
+    public String getResponseContentType() {
+        return responseJson.getString("mimeType");
+    }
+
+    public long getBeginTime() {
+        return beginTime;
+    }
+
+    public String getRemoteIPAddress() {
+        if (responseJson.has("remoteIPAddress")) {
+            return responseJson.getString("remoteIPAddress");
+        } else {
+            return null;
+        }
+    }
+
+    void setRequestJson(JSONObject requestJson) {
+        this.requestJson = requestJson;
+    }
+}
@@ -22,10 +22,13 @@ package org.archive.net.chrome;
 import org.json.JSONObject;

 import java.io.Closeable;
-import java.util.concurrent.CompletableFuture;
+import java.util.Map;
+import java.util.concurrent.*;
+import java.util.function.Consumer;
 import java.util.logging.Logger;

 import static java.util.logging.Level.FINE;
+import static java.util.logging.Level.WARNING;

 /**
 * A browser window or tab.
@@ -38,12 +41,17 @@ public class ChromeWindow implements Closeable {
    private final String sessionId;
    private boolean closed;
    private CompletableFuture<Void> loadEventFuture;
+    private final Map<String,ChromeRequest> requestMap = new ConcurrentHashMap<>();
+    private Consumer<ChromeRequest> requestConsumer;
+    private final ExecutorService eventExecutor;

    public ChromeWindow(ChromeClient client, String targetId) {
        this.client = client;
        this.targetId = targetId;
        this.sessionId = client.call("Target.attachToTarget", "targetId", targetId,
                "flatten", true).getString("sessionId");
+        eventExecutor = Executors.newSingleThreadExecutor(runnable ->
+                new Thread(runnable, "ChromeWindow (sessionId=" + sessionId +")"));
        client.sessionEventHandlers.put(sessionId, this::handleEvent);
        call("Page.enable"); // for loadEventFired
        call("Page.setLifecycleEventsEnabled", "enabled", true); // for networkidle
@@ -78,9 +86,47 @@ public class ChromeWindow implements Closeable {
    }

    private void handleEvent(JSONObject message) {
+        // Run event handlers on a different thread so we don't block the websocket receiving thread.
+        // That would cause a deadlock if an event handler itself made an RPC call as the response could
+        // never be processed.
+        // We use a single thread per window though as the order events are processed is important.
+        eventExecutor.submit(() -> {
+            try {
+                handleEventOnEventThread(message);
+            } catch (Throwable t) {
+                logger.log(WARNING, "Exception handling browser event " + message, t);
+            }
+        });
+    }
+
+    private void handleEventOnEventThread(JSONObject message) {
+        JSONObject params = message.getJSONObject("params");
        switch (message.getString("method")) {
+            case "Fetch.requestPaused":
+                handlePausedRequest(params);
+                break;
+            case "Network.requestWillBeSent":
+                handleRequestWillBeSent(params);
+                break;
+            case "Network.requestWillBeSentExtraInfo":
+                handleRequestWillBeSentExtraInfo(params);
+                break;
+            case "Network.responseReceived":
+                handleResponseReceived(params);
+                break;
+            case "Network.responseReceivedExtraInfo":
+                handleResponseReceivedExtraInfo(params);
+                break;
+            case "Network.loadingFinished":
+                handleLoadingFinished(params);
+                break;
            case "Page.loadEventFired":
                if (loadEventFuture != null) {
+                    try {
+                        Thread.sleep(1000);
+                    } catch (InterruptedException e) {
+                        Thread.currentThread().interrupt();
+                    }
                    loadEventFuture.complete(null);
                }
                break;
@@ -90,11 +136,77 @@ public class ChromeWindow implements Closeable {
        }
    }

+    private void handleRequestWillBeSent(JSONObject params) {
+        String requestId = params.getString("requestId");
+        ChromeRequest request = requestMap.computeIfAbsent(requestId, id -> new ChromeRequest(this, id));
+        request.setRequestJson(params.getJSONObject("request"));
+    }
+
+    private void handleRequestWillBeSentExtraInfo(JSONObject params) {
+        // it seems this event can arrive both before and after requestWillBeSent so we need to cope with that
+        String requestId = params.getString("requestId");
+        ChromeRequest request = requestMap.computeIfAbsent(requestId, id -> new ChromeRequest(this, id));
+        request.setRawRequestHeaders(params.getJSONObject("headers"));
+    }
+
+    private void handleResponseReceived(JSONObject params) {
+        ChromeRequest request = requestMap.get(params.getString("requestId"));
+        if (request == null) {
+            logger.log(WARNING, "Got responseReceived event without corresponding requestWillBeSent");
+            return;
+        }
+        request.setResponseJson(params.getJSONObject("response"));
+    }
+
+    private void handleResponseReceivedExtraInfo(JSONObject params) {
+        ChromeRequest request = requestMap.get(params.getString("requestId"));
+        if (request == null) {
+            logger.log(WARNING, "Got responseReceivedExtraInfo event without corresponding requestWillBeSent");
+            return;
+        }
+        request.setRawResponseHeaders(params.getJSONObject("headers"));
+        request.setResponseHeadersText(params.getString("headersText"));
+    }
+
+    private void handleLoadingFinished(JSONObject params) {
+        ChromeRequest request = requestMap.get(params.getString("requestId"));
+        if (request == null) {
+            logger.log(WARNING, "Got loadingFinished event without corresponding requestWillBeSent");
+            return;
+        }
+        if (requestConsumer != null) {
+            requestConsumer.accept(request);
+        }
+    }
+
    @Override
    public void close() {
        if (closed) return;
        closed = true;
+        eventExecutor.shutdown();
+        try {
+            eventExecutor.awaitTermination(1, TimeUnit.SECONDS);
+        } catch (InterruptedException e) {
+            Thread.currentThread().interrupt();
+        }
        client.call("Target.closeTarget", "targetId", targetId);
        client.sessionEventHandlers.remove(sessionId);
    }
+
+    public void captureRequests(Consumer<ChromeRequest> requestConsumer) {
+        this.requestConsumer = requestConsumer;
+        call("Network.enable");
+    }
+
+    private void handlePausedRequest(JSONObject params) {
+        if (params.has("responseStatusCode")) {
+            String stream = call("Fetch.takeResponseBodyAsStream", "requestId",
+                    params.getString("requestId")).getString("stream");
+            System.out.println(call("IO.read", "handle", stream));
+            System.out.println("stream " + stream);
+            call("IO.close", "handle", stream);
+        } else {
+            call("Fetch.continueRequest", "requestId", params.getString("requestId"));
+        }
+    }
 }
@@ -19,34 +19,144 @@

 package org.archive.modules.extractor;

-import org.apache.commons.httpclient.URIException;
+import org.archive.crawler.framework.CrawlController;
+import org.archive.crawler.framework.Frontier;
+import org.archive.crawler.reporting.CrawlerLoggerModule;
 import org.archive.modules.CrawlURI;
+import org.archive.modules.DispositionChain;
+import org.archive.modules.Processor;
 import org.archive.net.UURIFactory;
+import org.eclipse.jetty.server.Request;
+import org.eclipse.jetty.server.Server;
+import org.eclipse.jetty.server.handler.AbstractHandler;
+import org.junit.AfterClass;
+import org.junit.BeforeClass;
+import org.junit.Rule;
 import org.junit.Test;
+import org.junit.rules.TemporaryFolder;

+import javax.servlet.ServletException;
+import javax.servlet.http.HttpServletRequest;
+import javax.servlet.http.HttpServletResponse;
+import java.io.IOException;
+import java.net.InetSocketAddress;
+import java.util.ArrayList;
+import java.util.Arrays;
 import java.util.Collections;
 import java.util.List;
+import java.util.logging.Level;
+import java.util.logging.Logger;

 import static java.util.stream.Collectors.toList;
-import static org.junit.Assert.assertEquals;
+import static org.archive.modules.CrawlURI.FetchType.HTTP_GET;
+import static org.archive.modules.CrawlURI.FetchType.HTTP_POST;
+import static org.easymock.EasyMock.*;
+import static org.junit.Assert.*;
 import static org.junit.Assume.assumeNoException;

 public class ExtractorChromeTest {
+    private static Server server;
+
+    @BeforeClass
+    public static void startServer() throws Exception {
+        server = new Server(InetSocketAddress.createUnresolved("127.0.0.1", 7778));
+        server.setHandler(new AbstractHandler() {
+            @Override
+            public void handle(String target, Request baseRequest, HttpServletRequest request, HttpServletResponse response) throws IOException, ServletException {
+                switch (target) {
+                    case "/":
+                        response.setContentType("text/html");
+                        response.getWriter().write("<a href=http://example.org/page2.html>link</a>" +
+                                "<img src=/blue.png>" +
+                                "<script>fetch('/post', {method: 'POST', body: 'hello'});</script>");
+                        baseRequest.setHandled(true);
+                        break;
+                    case "/blue.png":
+                        response.setContentType("image/png");
+                        response.getWriter().write("bogus png");
+                        baseRequest.setHandled(true);
+                        break;
+                    case "/post":
+                        response.setContentType("plain/text");
+                        response.getWriter().write("method=" + request.getMethod());
+                        baseRequest.setHandled(true);
+                        break;
+                    default:
+                        System.err.println("Unhandled target: " + target);
+                        break;
+                }
+
+            }
+        });
+        server.start();
+    }
+
+    @AfterClass
+    public static void stopServer() throws Exception {
+        server.stop();
+    }
+
+    @Rule
+    public TemporaryFolder tempFolder = new TemporaryFolder();
+
    @Test
-    public void test() throws URIException {
-        ExtractorChrome extractor = new ExtractorChrome();
+    public void test() throws IOException {
+        List<CrawlURI> processedURIs = Collections.synchronizedList(new ArrayList<>());
+        CrawlController controller = new CrawlController();
+        DispositionChain dispositionChain = new DispositionChain();
+        dispositionChain.setProcessors(Arrays.asList(new Processor() {
+            @Override
+            protected boolean shouldProcess(CrawlURI uri) {
+                return true;
+            }
+
+            @Override
+            protected void innerProcess(CrawlURI uri) {
+                processedURIs.add(uri);
+            }
+        }));
+        controller.setDispositionChain(dispositionChain);
+        controller.setLoggerModule(new CrawlerLoggerModule() {
+            @Override
+            public Logger getUriProcessing() {
+                Logger logger = Logger.getAnonymousLogger();
+                logger.setLevel(Level.WARNING);
+                return logger;
+            }
+        });
+        Frontier frontier = createMock(Frontier.class);
+        frontier.considerIncluded(anyObject());
+        expectLastCall().anyTimes();
+        frontier.beginDisposition(anyObject());
+        expectLastCall().anyTimes();
+        frontier.endDisposition();
+        expectLastCall().anyTimes();
+        frontier.finished(anyObject());
+        expectLastCall().anyTimes();
+        replay(frontier);
+        controller.setFrontier(frontier);
+
+        ExtractorChrome extractor = new ExtractorChrome(controller, event -> { /* ignored */ });
        try {
            extractor.start();
        } catch (RuntimeException e) {
            assumeNoException("Unable to start Chrome", e);
        }
        try {
-            CrawlURI curi = new CrawlURI(UURIFactory.getInstance("data:text/html,<a href=http://example.org/page2.html>link</a>"));
+            CrawlURI curi = new CrawlURI(UURIFactory.getInstance("http://127.0.0.1:7778/"));
            extractor.innerExtract(curi);
            List<String> outLinks = curi.getOutLinks().stream().map(CrawlURI::toString).sorted().collect(toList());
            assertEquals(Collections.singletonList("http://example.org/page2.html"), outLinks);
        } finally {
            extractor.stop();
        }
+
+        assertEquals(3, processedURIs.size());
+        for (CrawlURI curi: processedURIs) {
+            assertEquals(200, curi.getFetchStatus());
+            assertEquals(curi.getUURI().getPath().equals("/post") ? HTTP_POST : HTTP_GET, curi.getFetchType());
+            assertNotNull(curi.getContentDigest());
+            assertTrue(curi.getContentSize() > 0);
+        }
    }
 }