diff --git a/contrib/src/main/java/org/archive/modules/extractor/ExtractorChrome.java b/contrib/src/main/java/org/archive/modules/extractor/ExtractorChrome.java
new file mode 100644
index 00000000..42379adf
--- /dev/null
+++ b/contrib/src/main/java/org/archive/modules/extractor/ExtractorChrome.java
@@ -0,0 +1,206 @@
+/*
+ * This file is part of the Heritrix web crawler (crawler.archive.org).
+ *
+ * Licensed to the Internet Archive (IA) by one or more individual
+ * contributors.
+ *
+ * The IA licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.archive.modules.extractor;
+
+import org.archive.modules.CrawlURI;
+import org.archive.net.chrome.ChromeClient;
+import org.archive.net.chrome.ChromeProcess;
+import org.archive.net.chrome.ChromeWindow;
+import org.json.JSONArray;
+
+import java.io.IOException;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.Semaphore;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.TimeoutException;
+
+/**
+ * Extracts links using a web browser via the Chrome Devtools Protocol.
+ *
+ * To use, first define this as a top-level bean:
+ *
+ * <bean id="extractorChrome" class="org.archive.modules.extractor.ExtractorChrome">
+ * <!-- <property name="devtoolsUrl" value="ws://127.0.0.1:1234/devtools/browser/2bc831e8-6c02-4c9b-affd-14c93b8579d7" /> -->
+ * <!-- <property name="executable" value="chromium-browser" /> -->
+ * <!-- <property name="loadTimeoutSeconds" value="30" /> -->
+ * <!-- <property name="maxOpenWindows" value="16" /> -->
+ * <!-- <property name="windowWidth" value="1366" /> -->
+ * <!-- <property name="windowWidth" value="768" /> -->
+ * </bean>
+ *
+ * Then add <ref bean="extractorChrome"/> to the fetch chain before extractorHTML.
+ *
+ * By default an instance of the browser will be run as a subprocess for the duration of the crawl. Alternatively set
+ * devtoolsUrl to connect to an existing instance of the browser (run with
+ * --headless --remote-debugging-port=1234).
+ */
+public class ExtractorChrome extends ContentExtractor {
+ /**
+ * The maximum number of browser windows that are allowed to be opened simultaneously. Feel free to increase this
+ * if you have lots of RAM available.
+ */
+ private int maxOpenWindows = 16;
+
+ /**
+ * URL of the devtools server to connect. If null a new browser process will be launched.
+ */
+ private String devtoolsUrl = null;
+
+ /**
+ * The name or path to the browser executable. If null common locations will be searched.
+ */
+ private String executable = null;
+
+ /**
+ * Width of the browser window.
+ */
+ private int windowWidth = 1366;
+
+ /**
+ * Height of the browser window.
+ */
+ private int windowHeight = 768;
+
+ /**
+ * Number of seconds to wait for the page to load.
+ */
+ private int loadTimeoutSeconds = 30;
+
+ private Semaphore openWindowsSemaphore = null;
+ private ChromeProcess process = null;
+ private ChromeClient client = null;
+
+ @Override
+ protected boolean shouldExtract(CrawlURI uri) {
+ return uri.getContentType().startsWith("text/html");
+ }
+
+ @Override
+ protected boolean innerExtract(CrawlURI uri) {
+ try {
+ openWindowsSemaphore.acquire();
+ try {
+ visit(uri);
+ } finally {
+ openWindowsSemaphore.release();
+ }
+ } catch (InterruptedException e) {
+ Thread.currentThread().interrupt();
+ }
+ return false;
+ }
+
+ private void visit(CrawlURI uri) throws InterruptedException {
+ try (ChromeWindow window = client.createWindow(windowWidth, windowHeight)) {
+ try {
+ window.navigateAsync(uri.getURI()).get(loadTimeoutSeconds, TimeUnit.SECONDS);
+ } catch (ExecutionException e) {
+ throw new RuntimeException(e.getCause());
+ } catch (TimeoutException e) {
+ throw new RuntimeException("Timed out navigating to " + uri.getURI());
+ }
+
+ JSONArray links = window.eval("Array.from(document.querySelectorAll('a[href], area[href]'))" +
+ ".map(link => link.protocol + '//' + link.host + link.pathname + link.search + link.hash)")
+ .getJSONArray("value");
+ for (int i = 0; i < links.length(); i++) {
+ addOutlink(uri, links.getString(i), LinkContext.NAVLINK_MISC, Hop.NAVLINK);
+ }
+ }
+ }
+
+ @Override
+ public void start() {
+ if (isRunning) return;
+ super.start();
+ openWindowsSemaphore = new Semaphore(maxOpenWindows);
+ if (devtoolsUrl != null) {
+ client = new ChromeClient(devtoolsUrl);
+ } else {
+ try {
+ process = new ChromeProcess(executable);
+ } catch (IOException e) {
+ throw new RuntimeException("Failed to launch browser process", e);
+ }
+ client = new ChromeClient(process.getDevtoolsUrl());
+ }
+ }
+
+ @Override
+ public void stop() {
+ super.stop();
+ if (client != null) {
+ client.close();
+ client = null;
+ }
+ if (process != null) {
+ process.close();
+ process = null;
+ }
+ }
+
+ public String getExecutable() {
+ return executable;
+ }
+
+ public void setExecutable(String executable) {
+ this.executable = executable;
+ }
+
+ public int getMaxOpenWindows() {
+ return maxOpenWindows;
+ }
+
+ public void setMaxOpenWindows(int maxOpenWindows) {
+ this.maxOpenWindows = maxOpenWindows;
+ }
+
+ public String getDevtoolsUrl() {
+ return devtoolsUrl;
+ }
+
+ public void setDevtoolsUrl(String devtoolsUrl) {
+ this.devtoolsUrl = devtoolsUrl;
+ }
+
+ public int getWindowWidth() {
+ return windowWidth;
+ }
+
+ public void setWindowWidth(int windowWidth) {
+ this.windowWidth = windowWidth;
+ }
+
+ public int getWindowHeight() {
+ return windowHeight;
+ }
+
+ public void setWindowHeight(int windowHeight) {
+ this.windowHeight = windowHeight;
+ }
+
+ public int getLoadTimeoutSeconds() {
+ return loadTimeoutSeconds;
+ }
+
+ public void setLoadTimeoutSeconds(int loadTimeoutSeconds) {
+ this.loadTimeoutSeconds = loadTimeoutSeconds;
+ }
+}
\ No newline at end of file
diff --git a/contrib/src/main/java/org/archive/net/chrome/ChromeClient.java b/contrib/src/main/java/org/archive/net/chrome/ChromeClient.java
new file mode 100644
index 00000000..7de68b3a
--- /dev/null
+++ b/contrib/src/main/java/org/archive/net/chrome/ChromeClient.java
@@ -0,0 +1,176 @@
+/*
+ * This file is part of the Heritrix web crawler (crawler.archive.org).
+ *
+ * Licensed to the Internet Archive (IA) by one or more individual
+ * contributors.
+ *
+ * The IA licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.archive.net.chrome;
+
+import org.java_websocket.client.WebSocketClient;
+import org.java_websocket.handshake.ServerHandshake;
+import org.json.JSONObject;
+
+import java.io.Closeable;
+import java.net.URI;
+import java.util.Map;
+import java.util.concurrent.*;
+import java.util.concurrent.atomic.AtomicLong;
+import java.util.function.Consumer;
+import java.util.logging.Logger;
+
+import static java.util.logging.Level.WARNING;
+
+/**
+ * A client for the Chrome Devtools Protocol.
+ */
+public class ChromeClient implements Closeable {
+ private static final Logger logger = Logger.getLogger(ChromeClient.class.getName());
+ private static final int RPC_TIMEOUT_SECONDS = 10;
+
+ private final DevtoolsSocket devtoolsSocket;
+ private final AtomicLong nextMessageId = new AtomicLong(0);
+ private final Map> responseFutures = new ConcurrentHashMap<>();
+ private final ExecutorService threadPool = Executors.newCachedThreadPool();
+ final ConcurrentHashMap> sessionEventHandlers = new ConcurrentHashMap<>();
+
+ public ChromeClient(String devtoolsUrl) {
+ devtoolsSocket = new DevtoolsSocket(URI.create(devtoolsUrl));
+ try {
+ devtoolsSocket.connectBlocking();
+ } catch (InterruptedException e) {
+ throw new ChromeException("Interrupted while connecting", e);
+ }
+ }
+
+ public JSONObject call(String method, Object... keysAndValues) {
+ return callInSession(null, method, keysAndValues);
+ }
+
+ public JSONObject callInSession(String sessionId, String method, Object... keysAndValues) {
+ JSONObject params = new JSONObject();
+ if (keysAndValues.length % 2 != 0) {
+ throw new IllegalArgumentException("keysAndValues.length must even");
+ }
+ for (int i = 0; i < keysAndValues.length; i += 2) {
+ params.put((String)keysAndValues[i], keysAndValues[i + 1]);
+ }
+ return callInternal(sessionId, method, params);
+ }
+
+ private JSONObject callInternal(String sessionId, String method, JSONObject params) {
+ long id = nextMessageId.getAndIncrement();
+ JSONObject message = new JSONObject();
+ message.put("id", id);
+ if (sessionId != null) {
+ message.put("sessionId", sessionId);
+ }
+ message.put("method", method);
+ message.put("params", params);
+ CompletableFuture future = new CompletableFuture<>();
+ responseFutures.put(id, future);
+ devtoolsSocket.send(message.toString());
+ try {
+ return future.get(RPC_TIMEOUT_SECONDS, TimeUnit.SECONDS);
+ } catch (InterruptedException e) {
+ throw new ChromeException("Call interrupted", e);
+ } catch (TimeoutException e) {
+ throw new ChromeException("Call timed out: " + message, e);
+ } catch (ExecutionException e) {
+ throw new ChromeException("Call failed: " + message + ": " + e.getMessage(), e.getCause());
+ }
+ }
+
+ private void handleResponse(JSONObject message) {
+ long id = message.getLong("id");
+ CompletableFuture future = responseFutures.remove(id);
+ if (future == null) {
+ logger.log(WARNING, "Unexpected RPC response id {0}", id);
+ } else if (message.has("error")) {
+ future.completeExceptionally(new ChromeException(message.getJSONObject("error").getString("message")));
+ } else {
+ future.complete(message.getJSONObject("result"));
+ }
+ }
+
+ private void handleEvent(JSONObject message) {
+ if (message.has("sessionId")) {
+ String sessionId = message.getString("sessionId");
+ Consumer handler = sessionEventHandlers.get(sessionId);
+ if (handler != null) {
+ // Run event handlers on a different thread so we don't block the websocket receiving thread.
+ // That would cause a deadlock if an event handler itself made an RPC call as the response could
+ // never be processed.
+ threadPool.submit(() -> {
+ try {
+ handler.accept(message);
+ } catch (Throwable t) {
+ logger.log(WARNING, "Exception handling browser event " + message, t);
+ }
+ });
+ } else {
+ logger.log(WARNING, "Received event for unknown session {0}", sessionId);
+ }
+ }
+ }
+
+ public ChromeWindow createWindow(int width, int height) {
+ String targetId = call("Target.createTarget", "url", "about:blank",
+ "width", width, "height", height).getString("targetId");
+ return new ChromeWindow(this, targetId);
+ }
+
+ @Override
+ public void close() {
+ devtoolsSocket.close();
+ }
+
+ private class DevtoolsSocket extends WebSocketClient {
+ public DevtoolsSocket(URI uri) {
+ super(uri);
+ setConnectionLostTimeout(-1); // disable pings - Chromium doesn't support them
+ }
+
+ @Override
+ public void onOpen(ServerHandshake serverHandshake) {
+
+ }
+
+ @Override
+ public void onMessage(String messageString) {
+ try {
+ JSONObject message = new JSONObject(messageString);
+ if (message.has("method")) {
+ handleEvent(message);
+ } else {
+ handleResponse(message);
+ }
+ } catch (Throwable e) {
+ logger.log(WARNING, "Exception handling message from Chromium", e);
+ throw e;
+ }
+ }
+
+ @Override
+ public void onClose(int i, String s, boolean b) {
+
+ }
+
+ @Override
+ public void onError(Exception e) {
+
+ }
+ }
+}
diff --git a/contrib/src/main/java/org/archive/net/chrome/ChromeException.java b/contrib/src/main/java/org/archive/net/chrome/ChromeException.java
new file mode 100644
index 00000000..e65dabc0
--- /dev/null
+++ b/contrib/src/main/java/org/archive/net/chrome/ChromeException.java
@@ -0,0 +1,38 @@
+/*
+ * This file is part of the Heritrix web crawler (crawler.archive.org).
+ *
+ * Licensed to the Internet Archive (IA) by one or more individual
+ * contributors.
+ *
+ * The IA licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.archive.net.chrome;
+
+public class ChromeException extends RuntimeException {
+ public ChromeException() {
+ super();
+ }
+
+ public ChromeException(String message) {
+ super(message);
+ }
+
+ public ChromeException(String message, Throwable throwable) {
+ super(message, throwable);
+ }
+
+ public ChromeException(Throwable throwable) {
+ super(throwable);
+ }
+}
diff --git a/contrib/src/main/java/org/archive/net/chrome/ChromeProcess.java b/contrib/src/main/java/org/archive/net/chrome/ChromeProcess.java
new file mode 100644
index 00000000..ca0e85af
--- /dev/null
+++ b/contrib/src/main/java/org/archive/net/chrome/ChromeProcess.java
@@ -0,0 +1,165 @@
+/*
+ * This file is part of the Heritrix web crawler (crawler.archive.org).
+ *
+ * Licensed to the Internet Archive (IA) by one or more individual
+ * contributors.
+ *
+ * The IA licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.archive.net.chrome;
+
+import org.archive.modules.extractor.ExtractorChrome;
+
+import java.io.BufferedReader;
+import java.io.Closeable;
+import java.io.IOException;
+import java.io.InputStreamReader;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.Set;
+import java.util.concurrent.*;
+import java.util.logging.Logger;
+
+import static java.nio.charset.StandardCharsets.ISO_8859_1;
+import static java.util.logging.Level.FINER;
+
+/**
+ * Manages starting and stopping a browser process.
+ */
+public class ChromeProcess implements Closeable {
+ private static final Logger logger = Logger.getLogger(ExtractorChrome.class.getName());
+
+ private static final String[] DEFAULT_EXECUTABLES = {"chromium-browser", "chromium", "google-chrome",
+ "C:\\Program Files (x86)\\Google\\Chrome\\Application\\chrome.exe", "firefox"};
+ private static final int SHUTDOWN_TIMEOUT_SECONDS = 2;
+
+ private static final Set runningProcesses = Collections.newSetFromMap(new ConcurrentHashMap<>());
+ private static Thread shutdownHook;
+
+ private final Process process;
+ private final String devtoolsUrl;
+
+ public ChromeProcess(String executable) throws IOException {
+ process = executable == null ? launchAny() : launch(executable);
+ runningProcesses.add(process);
+ registerShutdownHook();
+ devtoolsUrl = readDevtoolsUriFromStderr(process);
+ }
+
+ private static Process launch(String executable) throws IOException {
+ return new ProcessBuilder(executable, "--headless", "--remote-debugging-port=0").inheritIO()
+ .redirectError(ProcessBuilder.Redirect.PIPE).start();
+ }
+
+ /**
+ * Try to launch the browser process using each of DEFAULT_EXECUTABLES in turn until one succeeds.
+ */
+ private static Process launchAny() throws IOException {
+ IOException lastException = null;
+ for (String executable : DEFAULT_EXECUTABLES) {
+ try {
+ return launch(executable);
+ } catch (IOException e) {
+ lastException = e;
+ }
+ }
+ throw new IOException("Failed to launch any of " + Arrays.asList(DEFAULT_EXECUTABLES), lastException);
+ }
+
+ @Override
+ public void close() {
+ destroyProcess(process);
+ runningProcesses.remove(process);
+ }
+
+ /**
+ * Register a shutdown hook that destroys all running browser processes before exiting in case stop() is never
+ * called. This can happen if the Heritrix exits abnormally.
+ */
+ private static synchronized void registerShutdownHook() {
+ if (shutdownHook != null) return;
+ shutdownHook = new Thread(ChromeProcess::destroyAllRunningProcesses, "ChromiumClient shutdown hook");
+ Runtime.getRuntime().addShutdownHook(shutdownHook);
+ }
+
+ private static void destroyAllRunningProcesses() {
+ for (Process process : runningProcesses) {
+ process.destroy();
+ }
+ for (Process process : runningProcesses) {
+ try {
+ if (!process.waitFor(SHUTDOWN_TIMEOUT_SECONDS, TimeUnit.SECONDS)) {
+ break;
+ }
+ } catch (InterruptedException e) {
+ break;
+ }
+ }
+ for (Process process : runningProcesses) {
+ process.destroyForcibly();
+ }
+ }
+
+ private static void destroyProcess(Process process) {
+ process.destroy();
+ try {
+ process.waitFor(SHUTDOWN_TIMEOUT_SECONDS, TimeUnit.SECONDS);
+ } catch (InterruptedException e) {
+ Thread.currentThread().interrupt();
+ } finally {
+ process.destroyForcibly();
+ }
+ }
+
+ /**
+ * Reads the stderr of a Chromium process and returns the DevTools URI. Once this method
+ * returns stderr will continue to be consumed and logged by a background thread.
+ */
+ private static String readDevtoolsUriFromStderr(Process process) throws IOException {
+ BufferedReader stderr = new BufferedReader(new InputStreamReader(process.getErrorStream(), ISO_8859_1));
+ CompletableFuture future = new CompletableFuture<>();
+ Thread thread = new Thread(() -> {
+ String listenMsg = "DevTools listening on ";
+ try {
+ while (true) {
+ String line = stderr.readLine();
+ if (line == null) break;
+ if (!future.isDone() && line.startsWith(listenMsg)) {
+ future.complete(line.substring(listenMsg.length()));
+ }
+ logger.log(FINER, "Chromium STDERR: {0}", line);
+ }
+ } catch (IOException e) {
+ future.completeExceptionally(e);
+ }
+ });
+ thread.setName("Chromium stderr reader");
+ thread.setDaemon(true);
+ thread.start();
+
+ try {
+ return future.get(10, TimeUnit.SECONDS);
+ } catch (InterruptedException | ExecutionException | TimeoutException e) {
+ // unwrap the exception if we can to cut down on log noise
+ if (e.getCause() instanceof IOException) {
+ throw (IOException) e.getCause();
+ }
+ throw new IOException(e);
+ }
+ }
+
+ public String getDevtoolsUrl() {
+ return devtoolsUrl;
+ }
+}
diff --git a/contrib/src/main/java/org/archive/net/chrome/ChromeWindow.java b/contrib/src/main/java/org/archive/net/chrome/ChromeWindow.java
new file mode 100644
index 00000000..59e8f2e7
--- /dev/null
+++ b/contrib/src/main/java/org/archive/net/chrome/ChromeWindow.java
@@ -0,0 +1,100 @@
+/*
+ * This file is part of the Heritrix web crawler (crawler.archive.org).
+ *
+ * Licensed to the Internet Archive (IA) by one or more individual
+ * contributors.
+ *
+ * The IA licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.archive.net.chrome;
+
+import org.json.JSONObject;
+
+import java.io.Closeable;
+import java.util.concurrent.CompletableFuture;
+import java.util.logging.Logger;
+
+import static java.util.logging.Level.FINE;
+
+/**
+ * A browser window or tab.
+ */
+public class ChromeWindow implements Closeable {
+ private static final Logger logger = Logger.getLogger(ChromeWindow.class.getName());
+
+ private final ChromeClient client;
+ private final String targetId;
+ private final String sessionId;
+ private boolean closed;
+ private CompletableFuture loadEventFuture;
+
+ public ChromeWindow(ChromeClient client, String targetId) {
+ this.client = client;
+ this.targetId = targetId;
+ this.sessionId = client.call("Target.attachToTarget", "targetId", targetId,
+ "flatten", true).getString("sessionId");
+ client.sessionEventHandlers.put(sessionId, this::handleEvent);
+ call("Page.enable"); // for loadEventFired
+ call("Page.setLifecycleEventsEnabled", "enabled", true); // for networkidle
+ call("Runtime.enable"); // required by Firefox for Runtime.evaluate to work
+ }
+
+ /**
+ * Call a devtools method in the session of this window.
+ */
+ public JSONObject call(String method, Object... keysAndValues) {
+ return client.callInSession(sessionId, method, keysAndValues);
+ }
+
+ /**
+ * Evaluate a JavaScript expression.
+ */
+ public JSONObject eval(String expression) {
+ return call("Runtime.evaluate", "expression", expression,
+ "returnByValue", true).getJSONObject("result");
+ }
+
+ /**
+ * Navigate this window to a new URL. Returns a future which will be fulfilled when the page finishes loading.
+ */
+ public CompletableFuture navigateAsync(String url) {
+ if (loadEventFuture != null) {
+ loadEventFuture.cancel(false);
+ }
+ loadEventFuture = new CompletableFuture<>();
+ call("Page.navigate", "url", url);
+ return loadEventFuture;
+ }
+
+ private void handleEvent(JSONObject message) {
+ switch (message.getString("method")) {
+ case "Page.loadEventFired":
+ if (loadEventFuture != null) {
+ loadEventFuture.complete(null);
+ }
+ break;
+ default:
+ logger.log(FINE, "Unhandled event {0}", message);
+ break;
+ }
+ }
+
+ @Override
+ public void close() {
+ if (closed) return;
+ closed = true;
+ client.call("Target.closeTarget", "targetId", targetId);
+ client.sessionEventHandlers.remove(sessionId);
+ }
+}
\ No newline at end of file
diff --git a/contrib/src/test/java/org/archive/modules/extractor/ExtractorChromeTest.java b/contrib/src/test/java/org/archive/modules/extractor/ExtractorChromeTest.java
new file mode 100644
index 00000000..ba77fa21
--- /dev/null
+++ b/contrib/src/test/java/org/archive/modules/extractor/ExtractorChromeTest.java
@@ -0,0 +1,52 @@
+/*
+ * This file is part of the Heritrix web crawler (crawler.archive.org).
+ *
+ * Licensed to the Internet Archive (IA) by one or more individual
+ * contributors.
+ *
+ * The IA licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.archive.modules.extractor;
+
+import org.apache.commons.httpclient.URIException;
+import org.archive.modules.CrawlURI;
+import org.archive.net.UURIFactory;
+import org.junit.Test;
+
+import java.util.Collections;
+import java.util.List;
+
+import static java.util.stream.Collectors.toList;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assume.assumeNoException;
+
+public class ExtractorChromeTest {
+ @Test
+ public void test() throws URIException {
+ ExtractorChrome extractor = new ExtractorChrome();
+ try {
+ extractor.start();
+ } catch (RuntimeException e) {
+ assumeNoException("Unable to start Chrome", e);
+ }
+ try {
+ CrawlURI curi = new CrawlURI(UURIFactory.getInstance("data:text/html,link"));
+ extractor.innerExtract(curi);
+ List outLinks = curi.getOutLinks().stream().map(CrawlURI::toString).sorted().collect(toList());
+ assertEquals(Collections.singletonList("http://example.org/page2.html"), outLinks);
+ } finally {
+ extractor.stop();
+ }
+ }
+}
\ No newline at end of file
diff --git a/contrib/src/test/java/org/archive/net/chrome/ChromeClientTest.java b/contrib/src/test/java/org/archive/net/chrome/ChromeClientTest.java
new file mode 100644
index 00000000..0cff559b
--- /dev/null
+++ b/contrib/src/test/java/org/archive/net/chrome/ChromeClientTest.java
@@ -0,0 +1,49 @@
+/*
+ * This file is part of the Heritrix web crawler (crawler.archive.org).
+ *
+ * Licensed to the Internet Archive (IA) by one or more individual
+ * contributors.
+ *
+ * The IA licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.archive.net.chrome;
+
+import org.json.JSONObject;
+import org.junit.Test;
+
+import java.io.IOException;
+import java.util.concurrent.TimeUnit;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assume.assumeNoException;
+
+public class ChromeClientTest {
+ @Test
+ public void navigate() throws Exception {
+ ChromeProcess triedProcess;
+ try {
+ triedProcess = new ChromeProcess(null);
+ } catch (IOException e) {
+ assumeNoException("Chrome unavailable", e);
+ return;
+ }
+ try (ChromeProcess process = triedProcess;
+ ChromeClient client = new ChromeClient(process.getDevtoolsUrl());
+ ChromeWindow window = client.createWindow(1024, 768)) {
+ window.navigateAsync("data:text/html,hi
").get(10, TimeUnit.SECONDS);
+ JSONObject result = window.eval("document.getElementsByTagName('h1')[0].textContent");
+ assertEquals("hi", result.getString("value"));
+ }
+ }
+}
\ No newline at end of file