From 486dffbedf4bb299fc5305cf1d28db14f93160fc Mon Sep 17 00:00:00 2001 From: Alex Osborne Date: Thu, 12 Nov 2026 00:25:38 +0900 Subject: [PATCH] BrowserProcessor: allow per-browser concurrency, executable and options --- docs/bean-reference.rst | 10 +++ .../crawler/processor/BrowserConfig.java | 81 +++++++++++++++++++ .../crawler/processor/BrowserProcessor.java | 60 +++++++------- .../crawler/processor/LocalBrowser.java | 27 +++++++ .../archive/crawler/processor/SshBrowser.java | 57 +++++++++++++ .../restlet/profile-crawler-beans.cxml | 4 + .../processor/BrowserProcessorTest.java | 21 ++++- 7 files changed, 232 insertions(+), 28 deletions(-) create mode 100644 engine/src/main/java/org/archive/crawler/processor/BrowserConfig.java create mode 100644 engine/src/main/java/org/archive/crawler/processor/LocalBrowser.java create mode 100644 engine/src/main/java/org/archive/crawler/processor/SshBrowser.java diff --git a/docs/bean-reference.rst b/docs/bean-reference.rst index c57ddf63..c76aa6a6 100644 --- a/docs/bean-reference.rst +++ b/docs/bean-reference.rst @@ -546,6 +546,16 @@ BrowserProcessor .. bean-doc:: org.archive.crawler.processor.BrowserProcessor +LocalBrowser +~~~~~~~~~~~~ + +.. bean-doc:: org.archive.crawler.processor.LocalBrowser + +SshBrowser +~~~~~~~~~~ + +.. bean-doc:: org.archive.crawler.processor.SshBrowser + ExtractLinksBehavior ~~~~~~~~~~~~~~~~~~~~ diff --git a/engine/src/main/java/org/archive/crawler/processor/BrowserConfig.java b/engine/src/main/java/org/archive/crawler/processor/BrowserConfig.java new file mode 100644 index 00000000..1a3f3f92 --- /dev/null +++ b/engine/src/main/java/org/archive/crawler/processor/BrowserConfig.java @@ -0,0 +1,81 @@ +/* + * This file is part of the Heritrix web crawler (crawler.archive.org). + * + * Licensed to the Internet Archive (IA) by one or more individual + * contributors. + * + * The IA licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.archive.crawler.processor; + +import java.util.List; + +/** + * Settings for one of the {@link BrowserProcessor#setBrowsers browsers} of a BrowserProcessor. + * Settings left unset take the value of the BrowserProcessor's property of the same name. + */ +public abstract class BrowserConfig { + private Integer concurrency; + private String executable; + private List options; + + /** + * Parses a {@link BrowserProcessor#setBrowsers browsers} entry given as a string into a config + * with nothing overridden. + */ + static BrowserConfig parse(String browser) { + if (browser.equals("local")) return new LocalBrowser(); + try { + var ssh = new SshBrowser(); + ssh.setUri(browser); + return ssh; + } catch (IllegalArgumentException e) { + throw new IllegalArgumentException("Unsupported browser '" + browser + + "': expected 'local' or 'ssh://[user@]host[:port]'"); + } + } + + public Integer getConcurrency() { + return concurrency; + } + + /** + * Maximum number of web pages that can be open in this browser at once. + */ + public void setConcurrency(Integer concurrency) { + this.concurrency = concurrency; + } + + public String getExecutable() { + return executable; + } + + /** + * Webdriver executable to launch for this browser. + */ + public void setExecutable(String executable) { + this.executable = executable; + } + + public List getOptions() { + return options; + } + + /** + * Extra command-line options to be passed to this browser's webdriver executable. + */ + public void setOptions(List options) { + this.options = options; + } +} diff --git a/engine/src/main/java/org/archive/crawler/processor/BrowserProcessor.java b/engine/src/main/java/org/archive/crawler/processor/BrowserProcessor.java index 221bd1a0..c5883f09 100644 --- a/engine/src/main/java/org/archive/crawler/processor/BrowserProcessor.java +++ b/engine/src/main/java/org/archive/crawler/processor/BrowserProcessor.java @@ -54,8 +54,6 @@ import org.springframework.context.ApplicationEventPublisher; import java.io.ByteArrayInputStream; import java.io.IOException; import java.io.UncheckedIOException; -import java.net.URI; -import java.net.URISyntaxException; import java.nio.ByteBuffer; import java.nio.channels.Channels; import java.nio.channels.WritableByteChannel; @@ -116,7 +114,7 @@ public class BrowserProcessor extends Processor { protected String executable; protected List options = List.of("--headless"); protected int concurrency = 20; - protected List browsers = List.of("local"); + protected List browsers = List.of("local"); protected int minBrowsers = 1; protected int browserIdleTimeoutSeconds = 300; @@ -315,7 +313,7 @@ public class BrowserProcessor extends Processor { public void start() { if (isRunning) return; if (browsers.isEmpty()) throw new IllegalArgumentException("browsers must not be empty"); - List sshUris = browsers.stream().map(BrowserProcessor::parseBrowser).toList(); + List configs = browsers.stream().map(BrowserProcessor::toConfig).toList(); if (minBrowsers > browsers.size()) { throw new IllegalArgumentException("minBrowsers is " + minBrowsers + " but only " + browsers.size() + " browsers are configured"); @@ -346,7 +344,7 @@ public class BrowserProcessor extends Processor { .setNameFormat("BrowserProcessor-launcher-%d").setDaemon(true).build()); pool.clear(); for (int i = 0; i < browsers.size(); i++) { - pool.add(new Browser(i, sshUris.get(i))); + pool.add(new Browser(i, configs.get(i))); } CompletableFuture.allOf(pool.stream() .map(browser -> CompletableFuture.runAsync(browser::launch, launcher)) @@ -364,20 +362,12 @@ public class BrowserProcessor extends Processor { } } - /** - * Returns the ssh URI of a {@link #setBrowsers browsers} entry, or null for a local browser. - */ - private static URI parseBrowser(String browser) { - if (browser.equals("local")) return null; - try { - URI uri = new URI(browser); - SshBrowserProcess.checkUri(uri); - return uri; - } catch (URISyntaxException | IllegalArgumentException e) { - // fall through + private static BrowserConfig toConfig(Object browser) { + BrowserConfig config = browser instanceof String string ? BrowserConfig.parse(string) : (BrowserConfig) browser; + if (config instanceof SshBrowser ssh && ssh.uri() == null) { + throw new IllegalArgumentException("SshBrowser uri must be set"); } - throw new IllegalArgumentException("Unsupported browser '" + browser + - "': expected 'local' or 'ssh://[user@]host[:port]'"); + return config; } private void handleBeforeRequestSent(WebDriverBiDi driver, Network.BeforeRequestSent event) { @@ -499,13 +489,17 @@ public class BrowserProcessor extends Processor { this.browserIdleTimeoutSeconds = browserIdleTimeoutSeconds; } - public List getBrowsers() { + public List getBrowsers() { return browsers; } /** * Browsers to open pages in, one per entry. Pages from the same queue are opened in the same browser while it * has a free tab, so that a site keeps one browser's cookies and cache. + *

+ * Each entry is a string or a {@link LocalBrowser} or {@link SshBrowser} bean. A bean can override + * {@code concurrency}, {@code executable} and {@code options} for that browser. The strings are shorthand for a + * bean with nothing overridden: *

    *
  • {@code local} launches {@link #setExecutable(String) executable} on this machine.
  • *
  • {@code ssh://[user@]host[:port]} launches it on another machine using the ssh command, which carries the @@ -517,7 +511,7 @@ public class BrowserProcessor extends Processor { * the same port number here.
  • *
*/ - public void setBrowsers(List browsers) { + public void setBrowsers(List browsers) { this.browsers = browsers; } @@ -671,7 +665,7 @@ public class BrowserProcessor extends Processor { private static final long MAX_RETRY_DELAY_SECONDS = 600; protected enum State { UP, IDLE, WAKING, DOWN } protected final int id; - private final URI sshUri; // null for a local browser + private final BrowserConfig config; protected volatile LocalWebDriverBiDi driver; protected volatile State state = State.DOWN; protected volatile String lastFailure; @@ -681,13 +675,25 @@ public class BrowserProcessor extends Processor { private volatile long idleSince; private long retryDelaySeconds = INITIAL_RETRY_DELAY_SECONDS; // guarded by this - Browser(int id, URI sshUri) { + Browser(int id, BrowserConfig config) { this.id = id; - this.sshUri = sshUri; + this.config = config; + } + + int concurrency() { + return config.getConcurrency() != null ? config.getConcurrency() : concurrency; + } + + private String executable() { + return config.getExecutable() != null ? config.getExecutable() : executable; + } + + private List options() { + return config.getOptions() != null ? config.getOptions() : options; } private boolean hasFreeTabSlot() { - return state == State.UP && openTabs < concurrency; + return state == State.UP && openTabs < concurrency(); } /** @@ -773,16 +779,16 @@ public class BrowserProcessor extends Processor { BrowserProcess process; int proxyPort; String profileDir; - if (sshUri == null) { + if (!(config instanceof SshBrowser ssh)) { Path scratchDir = crawlController.getScratchDir().getFile().toPath(); Path localProfileDir = scratchDir.resolve("profile-" + id); Files.createDirectories(localProfileDir); Files.writeString(localProfileDir.resolve("user.js"), USER_JS); - process = BrowserProcess.launchLocal(executable, options, localProfileDir); + process = BrowserProcess.launchLocal(executable(), options(), localProfileDir); proxyPort = proxy.getPort(); profileDir = localProfileDir.toString(); } else { - var sshProcess = SshBrowserProcess.launch("ssh", sshUri, executable, options, proxy.getPort(), + var sshProcess = SshBrowserProcess.launch("ssh", ssh.uri(), executable(), options(), proxy.getPort(), USER_JS); process = sshProcess; proxyPort = sshProcess.proxyPort(); diff --git a/engine/src/main/java/org/archive/crawler/processor/LocalBrowser.java b/engine/src/main/java/org/archive/crawler/processor/LocalBrowser.java new file mode 100644 index 00000000..0ef47499 --- /dev/null +++ b/engine/src/main/java/org/archive/crawler/processor/LocalBrowser.java @@ -0,0 +1,27 @@ +/* + * This file is part of the Heritrix web crawler (crawler.archive.org). + * + * Licensed to the Internet Archive (IA) by one or more individual + * contributors. + * + * The IA licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.archive.crawler.processor; + +/** + * A browser of a {@link BrowserProcessor} launched on this machine. The {@code browsers} entry + * {@code local} is shorthand for this bean with nothing set. + */ +public class LocalBrowser extends BrowserConfig { +} diff --git a/engine/src/main/java/org/archive/crawler/processor/SshBrowser.java b/engine/src/main/java/org/archive/crawler/processor/SshBrowser.java new file mode 100644 index 00000000..11b45f00 --- /dev/null +++ b/engine/src/main/java/org/archive/crawler/processor/SshBrowser.java @@ -0,0 +1,57 @@ +/* + * This file is part of the Heritrix web crawler (crawler.archive.org). + * + * Licensed to the Internet Archive (IA) by one or more individual + * contributors. + * + * The IA licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.archive.crawler.processor; + +import org.archive.net.webdriver.SshBrowserProcess; + +import java.net.URI; +import java.net.URISyntaxException; + +/** + * A browser of a {@link BrowserProcessor} launched on another machine over ssh, as described under + * {@link BrowserProcessor#setBrowsers browsers}. The {@code browsers} entry + * {@code ssh://[user@]host[:port]} is shorthand for this bean with only {@code uri} set. + */ +public class SshBrowser extends BrowserConfig { + private URI uri; + + public String getUri() { + return uri == null ? null : uri.toString(); + } + + /** + * The machine to launch the browser on, as {@code ssh://[user@]host[:port]}. + */ + public void setUri(String uri) { + URI parsed; + try { + parsed = new URI(uri); + } catch (URISyntaxException e) { + throw new IllegalArgumentException( + "Expected ssh://[user@]host[:port] but got '" + uri + "'", e); + } + SshBrowserProcess.checkUri(parsed); + this.uri = parsed; + } + + URI uri() { + return uri; + } +} diff --git a/engine/src/main/resources/org/archive/crawler/restlet/profile-crawler-beans.cxml b/engine/src/main/resources/org/archive/crawler/restlet/profile-crawler-beans.cxml index 4e01f475..cccdefbd 100644 --- a/engine/src/main/resources/org/archive/crawler/restlet/profile-crawler-beans.cxml +++ b/engine/src/main/resources/org/archive/crawler/restlet/profile-crawler-beans.cxml @@ -331,6 +331,10 @@ http://example.example/example local ssh://crawler@browser1.example.org + + + + diff --git a/engine/src/test/java/org/archive/crawler/processor/BrowserProcessorTest.java b/engine/src/test/java/org/archive/crawler/processor/BrowserProcessorTest.java index 4b0b7586..b5c0e2c9 100644 --- a/engine/src/test/java/org/archive/crawler/processor/BrowserProcessorTest.java +++ b/engine/src/test/java/org/archive/crawler/processor/BrowserProcessorTest.java @@ -232,6 +232,25 @@ class BrowserProcessorTest { assertFalse(processor.isRunning()); } + @Test + public void testBrowserBeanOverrides() { + var failing = new LocalBrowser(); + failing.setExecutable("/nonexistent/firefox"); + var narrow = new LocalBrowser(); + narrow.setConcurrency(1); + var processor = newBrowserProcessor("overrides", List.of(failing, narrow, "local")); + processor.start(); + try { + assertEquals(BrowserProcessor.Browser.State.DOWN, processor.pool.get(0).state); + assertTrue(processor.pool.get(0).lastFailure.contains("/nonexistent/firefox"), + processor.pool.get(0).lastFailure); + assertEquals(1, processor.pool.get(1).concurrency()); + assertEquals(processor.getConcurrency(), processor.pool.get(2).concurrency()); + } finally { + processor.stop(); + } + } + @Test public void testRejectsUnsupportedBrowser() { for (String browser : List.of("remote", "ssh:host", "http://host")) { @@ -462,7 +481,7 @@ class BrowserProcessorTest { browserProcessor.start(); } - private static BrowserProcessor newBrowserProcessor(String name, List browsers) { + private static BrowserProcessor newBrowserProcessor(String name, List browsers) { var crawlController = new CrawlController(); crawlController.getScratchDir().setPath(tempDir.resolve(name).toString()); FetchChain fetchChain = new FetchChain();