From 7ff44208013322e48ad0c58d7fc84cb0aa0c85a6 Mon Sep 17 00:00:00 2001 From: Alex Osborne Date: Mon, 21 Jun 2021 12:56:24 +0900 Subject: [PATCH] Add ExtractorChrome to contrib This adds a link extractor that uses a headless browser via the Chrome Devtools Protocol. It works with Chromium and Google Chrome and while untested likely also other Blink-based browsers such as Brave and Microsoft Edge. It also works with Firefox which implements a subset of the protocol. In addition to ExtractorChrome itself this adds a couple of classes under org.archive.net.chrome for communicating with the browser: * ChromeProcess - manages the lifecycle of the browser child process * ChromeClient - client for the Chrome Devtools Protocol * ChromeWindow - client for controlling a particular browser window Why this extractor when we already have Umbra? The primary goal is to have something that works out of the box without any additional infrastructure beyond having a browser installed. A longer term goal is to be able to take advantage of tighter integration between Heritrix and the browser to do things like recording browser subrequests to WARC files or running ExtractorHTML against the loaded DOM. While this extractor is already usable it is currently quite bare bones and only extracts links from and tags. The intent is to expand it with more functionality over time with features like intercepting subrequests, reusing previously saved resources, taking screenshots or running JavaScript behaviours. --- contrib/pom.xml | 5 + .../modules/extractor/ExtractorChrome.java | 206 ++++++++++++++++++ .../org/archive/net/chrome/ChromeClient.java | 176 +++++++++++++++ .../archive/net/chrome/ChromeException.java | 38 ++++ .../org/archive/net/chrome/ChromeProcess.java | 165 ++++++++++++++ .../org/archive/net/chrome/ChromeWindow.java | 100 +++++++++ .../extractor/ExtractorChromeTest.java | 52 +++++ .../archive/net/chrome/ChromeClientTest.java | 49 +++++ 8 files changed, 791 insertions(+) create mode 100644 contrib/src/main/java/org/archive/modules/extractor/ExtractorChrome.java create mode 100644 contrib/src/main/java/org/archive/net/chrome/ChromeClient.java create mode 100644 contrib/src/main/java/org/archive/net/chrome/ChromeException.java create mode 100644 contrib/src/main/java/org/archive/net/chrome/ChromeProcess.java create mode 100644 contrib/src/main/java/org/archive/net/chrome/ChromeWindow.java create mode 100644 contrib/src/test/java/org/archive/modules/extractor/ExtractorChromeTest.java create mode 100644 contrib/src/test/java/org/archive/net/chrome/ChromeClientTest.java diff --git a/contrib/pom.xml b/contrib/pom.xml index 13c6d3fb..fe9996bd 100644 --- a/contrib/pom.xml +++ b/contrib/pom.xml @@ -83,6 +83,11 @@ gson 2.8.6 + + org.java-websocket + Java-WebSocket + 1.5.2 + diff --git a/contrib/src/main/java/org/archive/modules/extractor/ExtractorChrome.java b/contrib/src/main/java/org/archive/modules/extractor/ExtractorChrome.java new file mode 100644 index 00000000..42379adf --- /dev/null +++ b/contrib/src/main/java/org/archive/modules/extractor/ExtractorChrome.java @@ -0,0 +1,206 @@ +/* + * This file is part of the Heritrix web crawler (crawler.archive.org). + * + * Licensed to the Internet Archive (IA) by one or more individual + * contributors. + * + * The IA licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.archive.modules.extractor; + +import org.archive.modules.CrawlURI; +import org.archive.net.chrome.ChromeClient; +import org.archive.net.chrome.ChromeProcess; +import org.archive.net.chrome.ChromeWindow; +import org.json.JSONArray; + +import java.io.IOException; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.Semaphore; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; + +/** + * Extracts links using a web browser via the Chrome Devtools Protocol. + *

+ * To use, first define this as a top-level bean: + * 

+ * <bean id="extractorChrome" class="org.archive.modules.extractor.ExtractorChrome">
+ *   <!-- <property name="devtoolsUrl" value="ws://127.0.0.1:1234/devtools/browser/2bc831e8-6c02-4c9b-affd-14c93b8579d7" /> -->
+ *   <!-- <property name="executable" value="chromium-browser" /> -->
+ *   <!-- <property name="loadTimeoutSeconds" value="30" /> -->
+ *   <!-- <property name="maxOpenWindows" value="16" /> -->
+ *   <!-- <property name="windowWidth" value="1366" /> -->
+ *   <!-- <property name="windowWidth" value="768" /> -->
+ * </bean>
+ *
+ * Then add <ref bean="extractorChrome"/> to the fetch chain before extractorHTML. + *

+ * By default an instance of the browser will be run as a subprocess for the duration of the crawl. Alternatively set + * devtoolsUrl to connect to an existing instance of the browser (run with + * --headless --remote-debugging-port=1234). + */ +public class ExtractorChrome extends ContentExtractor { + /** + * The maximum number of browser windows that are allowed to be opened simultaneously. Feel free to increase this + * if you have lots of RAM available. + */ + private int maxOpenWindows = 16; + + /** + * URL of the devtools server to connect. If null a new browser process will be launched. + */ + private String devtoolsUrl = null; + + /** + * The name or path to the browser executable. If null common locations will be searched. + */ + private String executable = null; + + /** + * Width of the browser window. + */ + private int windowWidth = 1366; + + /** + * Height of the browser window. + */ + private int windowHeight = 768; + + /** + * Number of seconds to wait for the page to load. + */ + private int loadTimeoutSeconds = 30; + + private Semaphore openWindowsSemaphore = null; + private ChromeProcess process = null; + private ChromeClient client = null; + + @Override + protected boolean shouldExtract(CrawlURI uri) { + return uri.getContentType().startsWith("text/html"); + } + + @Override + protected boolean innerExtract(CrawlURI uri) { + try { + openWindowsSemaphore.acquire(); + try { + visit(uri); + } finally { + openWindowsSemaphore.release(); + } + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + } + return false; + } + + private void visit(CrawlURI uri) throws InterruptedException { + try (ChromeWindow window = client.createWindow(windowWidth, windowHeight)) { + try { + window.navigateAsync(uri.getURI()).get(loadTimeoutSeconds, TimeUnit.SECONDS); + } catch (ExecutionException e) { + throw new RuntimeException(e.getCause()); + } catch (TimeoutException e) { + throw new RuntimeException("Timed out navigating to " + uri.getURI()); + } + + JSONArray links = window.eval("Array.from(document.querySelectorAll('a[href], area[href]'))" + + ".map(link => link.protocol + '//' + link.host + link.pathname + link.search + link.hash)") + .getJSONArray("value"); + for (int i = 0; i < links.length(); i++) { + addOutlink(uri, links.getString(i), LinkContext.NAVLINK_MISC, Hop.NAVLINK); + } + } + } + + @Override + public void start() { + if (isRunning) return; + super.start(); + openWindowsSemaphore = new Semaphore(maxOpenWindows); + if (devtoolsUrl != null) { + client = new ChromeClient(devtoolsUrl); + } else { + try { + process = new ChromeProcess(executable); + } catch (IOException e) { + throw new RuntimeException("Failed to launch browser process", e); + } + client = new ChromeClient(process.getDevtoolsUrl()); + } + } + + @Override + public void stop() { + super.stop(); + if (client != null) { + client.close(); + client = null; + } + if (process != null) { + process.close(); + process = null; + } + } + + public String getExecutable() { + return executable; + } + + public void setExecutable(String executable) { + this.executable = executable; + } + + public int getMaxOpenWindows() { + return maxOpenWindows; + } + + public void setMaxOpenWindows(int maxOpenWindows) { + this.maxOpenWindows = maxOpenWindows; + } + + public String getDevtoolsUrl() { + return devtoolsUrl; + } + + public void setDevtoolsUrl(String devtoolsUrl) { + this.devtoolsUrl = devtoolsUrl; + } + + public int getWindowWidth() { + return windowWidth; + } + + public void setWindowWidth(int windowWidth) { + this.windowWidth = windowWidth; + } + + public int getWindowHeight() { + return windowHeight; + } + + public void setWindowHeight(int windowHeight) { + this.windowHeight = windowHeight; + } + + public int getLoadTimeoutSeconds() { + return loadTimeoutSeconds; + } + + public void setLoadTimeoutSeconds(int loadTimeoutSeconds) { + this.loadTimeoutSeconds = loadTimeoutSeconds; + } +} \ No newline at end of file diff --git a/contrib/src/main/java/org/archive/net/chrome/ChromeClient.java b/contrib/src/main/java/org/archive/net/chrome/ChromeClient.java new file mode 100644 index 00000000..7de68b3a --- /dev/null +++ b/contrib/src/main/java/org/archive/net/chrome/ChromeClient.java @@ -0,0 +1,176 @@ +/* + * This file is part of the Heritrix web crawler (crawler.archive.org). + * + * Licensed to the Internet Archive (IA) by one or more individual + * contributors. + * + * The IA licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.archive.net.chrome; + +import org.java_websocket.client.WebSocketClient; +import org.java_websocket.handshake.ServerHandshake; +import org.json.JSONObject; + +import java.io.Closeable; +import java.net.URI; +import java.util.Map; +import java.util.concurrent.*; +import java.util.concurrent.atomic.AtomicLong; +import java.util.function.Consumer; +import java.util.logging.Logger; + +import static java.util.logging.Level.WARNING; + +/** + * A client for the Chrome Devtools Protocol. + */ +public class ChromeClient implements Closeable { + private static final Logger logger = Logger.getLogger(ChromeClient.class.getName()); + private static final int RPC_TIMEOUT_SECONDS = 10; + + private final DevtoolsSocket devtoolsSocket; + private final AtomicLong nextMessageId = new AtomicLong(0); + private final Map> responseFutures = new ConcurrentHashMap<>(); + private final ExecutorService threadPool = Executors.newCachedThreadPool(); + final ConcurrentHashMap> sessionEventHandlers = new ConcurrentHashMap<>(); + + public ChromeClient(String devtoolsUrl) { + devtoolsSocket = new DevtoolsSocket(URI.create(devtoolsUrl)); + try { + devtoolsSocket.connectBlocking(); + } catch (InterruptedException e) { + throw new ChromeException("Interrupted while connecting", e); + } + } + + public JSONObject call(String method, Object... keysAndValues) { + return callInSession(null, method, keysAndValues); + } + + public JSONObject callInSession(String sessionId, String method, Object... keysAndValues) { + JSONObject params = new JSONObject(); + if (keysAndValues.length % 2 != 0) { + throw new IllegalArgumentException("keysAndValues.length must even"); + } + for (int i = 0; i < keysAndValues.length; i += 2) { + params.put((String)keysAndValues[i], keysAndValues[i + 1]); + } + return callInternal(sessionId, method, params); + } + + private JSONObject callInternal(String sessionId, String method, JSONObject params) { + long id = nextMessageId.getAndIncrement(); + JSONObject message = new JSONObject(); + message.put("id", id); + if (sessionId != null) { + message.put("sessionId", sessionId); + } + message.put("method", method); + message.put("params", params); + CompletableFuture future = new CompletableFuture<>(); + responseFutures.put(id, future); + devtoolsSocket.send(message.toString()); + try { + return future.get(RPC_TIMEOUT_SECONDS, TimeUnit.SECONDS); + } catch (InterruptedException e) { + throw new ChromeException("Call interrupted", e); + } catch (TimeoutException e) { + throw new ChromeException("Call timed out: " + message, e); + } catch (ExecutionException e) { + throw new ChromeException("Call failed: " + message + ": " + e.getMessage(), e.getCause()); + } + } + + private void handleResponse(JSONObject message) { + long id = message.getLong("id"); + CompletableFuture future = responseFutures.remove(id); + if (future == null) { + logger.log(WARNING, "Unexpected RPC response id {0}", id); + } else if (message.has("error")) { + future.completeExceptionally(new ChromeException(message.getJSONObject("error").getString("message"))); + } else { + future.complete(message.getJSONObject("result")); + } + } + + private void handleEvent(JSONObject message) { + if (message.has("sessionId")) { + String sessionId = message.getString("sessionId"); + Consumer handler = sessionEventHandlers.get(sessionId); + if (handler != null) { + // Run event handlers on a different thread so we don't block the websocket receiving thread. + // That would cause a deadlock if an event handler itself made an RPC call as the response could + // never be processed. + threadPool.submit(() -> { + try { + handler.accept(message); + } catch (Throwable t) { + logger.log(WARNING, "Exception handling browser event " + message, t); + } + }); + } else { + logger.log(WARNING, "Received event for unknown session {0}", sessionId); + } + } + } + + public ChromeWindow createWindow(int width, int height) { + String targetId = call("Target.createTarget", "url", "about:blank", + "width", width, "height", height).getString("targetId"); + return new ChromeWindow(this, targetId); + } + + @Override + public void close() { + devtoolsSocket.close(); + } + + private class DevtoolsSocket extends WebSocketClient { + public DevtoolsSocket(URI uri) { + super(uri); + setConnectionLostTimeout(-1); // disable pings - Chromium doesn't support them + } + + @Override + public void onOpen(ServerHandshake serverHandshake) { + + } + + @Override + public void onMessage(String messageString) { + try { + JSONObject message = new JSONObject(messageString); + if (message.has("method")) { + handleEvent(message); + } else { + handleResponse(message); + } + } catch (Throwable e) { + logger.log(WARNING, "Exception handling message from Chromium", e); + throw e; + } + } + + @Override + public void onClose(int i, String s, boolean b) { + + } + + @Override + public void onError(Exception e) { + + } + } +} diff --git a/contrib/src/main/java/org/archive/net/chrome/ChromeException.java b/contrib/src/main/java/org/archive/net/chrome/ChromeException.java new file mode 100644 index 00000000..e65dabc0 --- /dev/null +++ b/contrib/src/main/java/org/archive/net/chrome/ChromeException.java @@ -0,0 +1,38 @@ +/* + * This file is part of the Heritrix web crawler (crawler.archive.org). + * + * Licensed to the Internet Archive (IA) by one or more individual + * contributors. + * + * The IA licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.archive.net.chrome; + +public class ChromeException extends RuntimeException { + public ChromeException() { + super(); + } + + public ChromeException(String message) { + super(message); + } + + public ChromeException(String message, Throwable throwable) { + super(message, throwable); + } + + public ChromeException(Throwable throwable) { + super(throwable); + } +} diff --git a/contrib/src/main/java/org/archive/net/chrome/ChromeProcess.java b/contrib/src/main/java/org/archive/net/chrome/ChromeProcess.java new file mode 100644 index 00000000..ca0e85af --- /dev/null +++ b/contrib/src/main/java/org/archive/net/chrome/ChromeProcess.java @@ -0,0 +1,165 @@ +/* + * This file is part of the Heritrix web crawler (crawler.archive.org). + * + * Licensed to the Internet Archive (IA) by one or more individual + * contributors. + * + * The IA licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.archive.net.chrome; + +import org.archive.modules.extractor.ExtractorChrome; + +import java.io.BufferedReader; +import java.io.Closeable; +import java.io.IOException; +import java.io.InputStreamReader; +import java.util.Arrays; +import java.util.Collections; +import java.util.Set; +import java.util.concurrent.*; +import java.util.logging.Logger; + +import static java.nio.charset.StandardCharsets.ISO_8859_1; +import static java.util.logging.Level.FINER; + +/** + * Manages starting and stopping a browser process. + */ +public class ChromeProcess implements Closeable { + private static final Logger logger = Logger.getLogger(ExtractorChrome.class.getName()); + + private static final String[] DEFAULT_EXECUTABLES = {"chromium-browser", "chromium", "google-chrome", + "C:\\Program Files (x86)\\Google\\Chrome\\Application\\chrome.exe", "firefox"}; + private static final int SHUTDOWN_TIMEOUT_SECONDS = 2; + + private static final Set runningProcesses = Collections.newSetFromMap(new ConcurrentHashMap<>()); + private static Thread shutdownHook; + + private final Process process; + private final String devtoolsUrl; + + public ChromeProcess(String executable) throws IOException { + process = executable == null ? launchAny() : launch(executable); + runningProcesses.add(process); + registerShutdownHook(); + devtoolsUrl = readDevtoolsUriFromStderr(process); + } + + private static Process launch(String executable) throws IOException { + return new ProcessBuilder(executable, "--headless", "--remote-debugging-port=0").inheritIO() + .redirectError(ProcessBuilder.Redirect.PIPE).start(); + } + + /** + * Try to launch the browser process using each of DEFAULT_EXECUTABLES in turn until one succeeds. + */ + private static Process launchAny() throws IOException { + IOException lastException = null; + for (String executable : DEFAULT_EXECUTABLES) { + try { + return launch(executable); + } catch (IOException e) { + lastException = e; + } + } + throw new IOException("Failed to launch any of " + Arrays.asList(DEFAULT_EXECUTABLES), lastException); + } + + @Override + public void close() { + destroyProcess(process); + runningProcesses.remove(process); + } + + /** + * Register a shutdown hook that destroys all running browser processes before exiting in case stop() is never + * called. This can happen if the Heritrix exits abnormally. + */ + private static synchronized void registerShutdownHook() { + if (shutdownHook != null) return; + shutdownHook = new Thread(ChromeProcess::destroyAllRunningProcesses, "ChromiumClient shutdown hook"); + Runtime.getRuntime().addShutdownHook(shutdownHook); + } + + private static void destroyAllRunningProcesses() { + for (Process process : runningProcesses) { + process.destroy(); + } + for (Process process : runningProcesses) { + try { + if (!process.waitFor(SHUTDOWN_TIMEOUT_SECONDS, TimeUnit.SECONDS)) { + break; + } + } catch (InterruptedException e) { + break; + } + } + for (Process process : runningProcesses) { + process.destroyForcibly(); + } + } + + private static void destroyProcess(Process process) { + process.destroy(); + try { + process.waitFor(SHUTDOWN_TIMEOUT_SECONDS, TimeUnit.SECONDS); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + } finally { + process.destroyForcibly(); + } + } + + /** + * Reads the stderr of a Chromium process and returns the DevTools URI. Once this method + * returns stderr will continue to be consumed and logged by a background thread. + */ + private static String readDevtoolsUriFromStderr(Process process) throws IOException { + BufferedReader stderr = new BufferedReader(new InputStreamReader(process.getErrorStream(), ISO_8859_1)); + CompletableFuture future = new CompletableFuture<>(); + Thread thread = new Thread(() -> { + String listenMsg = "DevTools listening on "; + try { + while (true) { + String line = stderr.readLine(); + if (line == null) break; + if (!future.isDone() && line.startsWith(listenMsg)) { + future.complete(line.substring(listenMsg.length())); + } + logger.log(FINER, "Chromium STDERR: {0}", line); + } + } catch (IOException e) { + future.completeExceptionally(e); + } + }); + thread.setName("Chromium stderr reader"); + thread.setDaemon(true); + thread.start(); + + try { + return future.get(10, TimeUnit.SECONDS); + } catch (InterruptedException | ExecutionException | TimeoutException e) { + // unwrap the exception if we can to cut down on log noise + if (e.getCause() instanceof IOException) { + throw (IOException) e.getCause(); + } + throw new IOException(e); + } + } + + public String getDevtoolsUrl() { + return devtoolsUrl; + } +} diff --git a/contrib/src/main/java/org/archive/net/chrome/ChromeWindow.java b/contrib/src/main/java/org/archive/net/chrome/ChromeWindow.java new file mode 100644 index 00000000..59e8f2e7 --- /dev/null +++ b/contrib/src/main/java/org/archive/net/chrome/ChromeWindow.java @@ -0,0 +1,100 @@ +/* + * This file is part of the Heritrix web crawler (crawler.archive.org). + * + * Licensed to the Internet Archive (IA) by one or more individual + * contributors. + * + * The IA licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.archive.net.chrome; + +import org.json.JSONObject; + +import java.io.Closeable; +import java.util.concurrent.CompletableFuture; +import java.util.logging.Logger; + +import static java.util.logging.Level.FINE; + +/** + * A browser window or tab. + */ +public class ChromeWindow implements Closeable { + private static final Logger logger = Logger.getLogger(ChromeWindow.class.getName()); + + private final ChromeClient client; + private final String targetId; + private final String sessionId; + private boolean closed; + private CompletableFuture loadEventFuture; + + public ChromeWindow(ChromeClient client, String targetId) { + this.client = client; + this.targetId = targetId; + this.sessionId = client.call("Target.attachToTarget", "targetId", targetId, + "flatten", true).getString("sessionId"); + client.sessionEventHandlers.put(sessionId, this::handleEvent); + call("Page.enable"); // for loadEventFired + call("Page.setLifecycleEventsEnabled", "enabled", true); // for networkidle + call("Runtime.enable"); // required by Firefox for Runtime.evaluate to work + } + + /** + * Call a devtools method in the session of this window. + */ + public JSONObject call(String method, Object... keysAndValues) { + return client.callInSession(sessionId, method, keysAndValues); + } + + /** + * Evaluate a JavaScript expression. + */ + public JSONObject eval(String expression) { + return call("Runtime.evaluate", "expression", expression, + "returnByValue", true).getJSONObject("result"); + } + + /** + * Navigate this window to a new URL. Returns a future which will be fulfilled when the page finishes loading. + */ + public CompletableFuture navigateAsync(String url) { + if (loadEventFuture != null) { + loadEventFuture.cancel(false); + } + loadEventFuture = new CompletableFuture<>(); + call("Page.navigate", "url", url); + return loadEventFuture; + } + + private void handleEvent(JSONObject message) { + switch (message.getString("method")) { + case "Page.loadEventFired": + if (loadEventFuture != null) { + loadEventFuture.complete(null); + } + break; + default: + logger.log(FINE, "Unhandled event {0}", message); + break; + } + } + + @Override + public void close() { + if (closed) return; + closed = true; + client.call("Target.closeTarget", "targetId", targetId); + client.sessionEventHandlers.remove(sessionId); + } +} \ No newline at end of file diff --git a/contrib/src/test/java/org/archive/modules/extractor/ExtractorChromeTest.java b/contrib/src/test/java/org/archive/modules/extractor/ExtractorChromeTest.java new file mode 100644 index 00000000..ba77fa21 --- /dev/null +++ b/contrib/src/test/java/org/archive/modules/extractor/ExtractorChromeTest.java @@ -0,0 +1,52 @@ +/* + * This file is part of the Heritrix web crawler (crawler.archive.org). + * + * Licensed to the Internet Archive (IA) by one or more individual + * contributors. + * + * The IA licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.archive.modules.extractor; + +import org.apache.commons.httpclient.URIException; +import org.archive.modules.CrawlURI; +import org.archive.net.UURIFactory; +import org.junit.Test; + +import java.util.Collections; +import java.util.List; + +import static java.util.stream.Collectors.toList; +import static org.junit.Assert.assertEquals; +import static org.junit.Assume.assumeNoException; + +public class ExtractorChromeTest { + @Test + public void test() throws URIException { + ExtractorChrome extractor = new ExtractorChrome(); + try { + extractor.start(); + } catch (RuntimeException e) { + assumeNoException("Unable to start Chrome", e); + } + try { + CrawlURI curi = new CrawlURI(UURIFactory.getInstance("data:text/html,link")); + extractor.innerExtract(curi); + List outLinks = curi.getOutLinks().stream().map(CrawlURI::toString).sorted().collect(toList()); + assertEquals(Collections.singletonList("http://example.org/page2.html"), outLinks); + } finally { + extractor.stop(); + } + } +} \ No newline at end of file diff --git a/contrib/src/test/java/org/archive/net/chrome/ChromeClientTest.java b/contrib/src/test/java/org/archive/net/chrome/ChromeClientTest.java new file mode 100644 index 00000000..0cff559b --- /dev/null +++ b/contrib/src/test/java/org/archive/net/chrome/ChromeClientTest.java @@ -0,0 +1,49 @@ +/* + * This file is part of the Heritrix web crawler (crawler.archive.org). + * + * Licensed to the Internet Archive (IA) by one or more individual + * contributors. + * + * The IA licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.archive.net.chrome; + +import org.json.JSONObject; +import org.junit.Test; + +import java.io.IOException; +import java.util.concurrent.TimeUnit; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assume.assumeNoException; + +public class ChromeClientTest { + @Test + public void navigate() throws Exception { + ChromeProcess triedProcess; + try { + triedProcess = new ChromeProcess(null); + } catch (IOException e) { + assumeNoException("Chrome unavailable", e); + return; + } + try (ChromeProcess process = triedProcess; + ChromeClient client = new ChromeClient(process.getDevtoolsUrl()); + ChromeWindow window = client.createWindow(1024, 768)) { + window.navigateAsync("data:text/html,

hi

").get(10, TimeUnit.SECONDS); + JSONObject result = window.eval("document.getElementsByTagName('h1')[0].textContent"); + assertEquals("hi", result.getString("value")); + } + } +} \ No newline at end of file