BrowserProcessor: allow per-browser concurrency, executable and options

This commit is contained in:
Alex Osborne committed 2026-11-13 10:12:38 +09:00
1 parent d7c9a5581e
commit 486dffbedf
7 files changed
+232 -28

No files matched your search

+10
View File
@@ -546,6 +546,16 @@ BrowserProcessor
.. bean-doc:: org.archive.crawler.processor.BrowserProcessor
LocalBrowser
~~~~~~~~~~~~
.. bean-doc:: org.archive.crawler.processor.LocalBrowser
SshBrowser
~~~~~~~~~~
.. bean-doc:: org.archive.crawler.processor.SshBrowser
ExtractLinksBehavior
~~~~~~~~~~~~~~~~~~~~
@@ -0,0 +1,81 @@
/*
* This file is part of the Heritrix web crawler (crawler.archive.org).
*
* Licensed to the Internet Archive (IA) by one or more individual
* contributors.
*
* The IA licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.archive.crawler.processor;
import java.util.List;
/**
* Settings for one of the {@link BrowserProcessor#setBrowsers browsers} of a BrowserProcessor.
* Settings left unset take the value of the BrowserProcessor's property of the same name.
*/
public abstract class BrowserConfig {
private Integer concurrency;
private String executable;
private List<String> options;
/**
* Parses a {@link BrowserProcessor#setBrowsers browsers} entry given as a string into a config
* with nothing overridden.
*/
static BrowserConfig parse(String browser) {
if (browser.equals("local")) return new LocalBrowser();
try {
var ssh = new SshBrowser();
ssh.setUri(browser);
return ssh;
} catch (IllegalArgumentException e) {
throw new IllegalArgumentException("Unsupported browser '" + browser +
"': expected 'local' or 'ssh://[user@]host[:port]'");
}
}
public Integer getConcurrency() {
return concurrency;
}
/**
* Maximum number of web pages that can be open in this browser at once.
*/
public void setConcurrency(Integer concurrency) {
this.concurrency = concurrency;
}
public String getExecutable() {
return executable;
}
/**
* Webdriver executable to launch for this browser.
*/
public void setExecutable(String executable) {
this.executable = executable;
}
public List<String> getOptions() {
return options;
}
/**
* Extra command-line options to be passed to this browser's webdriver executable.
*/
public void setOptions(List<String> options) {
this.options = options;
}
}
@@ -54,8 +54,6 @@ import org.springframework.context.ApplicationEventPublisher;
import java.io.ByteArrayInputStream;
import java.io.IOException;
import java.io.UncheckedIOException;
import java.net.URI;
import java.net.URISyntaxException;
import java.nio.ByteBuffer;
import java.nio.channels.Channels;
import java.nio.channels.WritableByteChannel;
@@ -116,7 +114,7 @@ public class BrowserProcessor extends Processor {
protected String executable;
protected List<String> options = List.of("--headless");
protected int concurrency = 20;
protected List<String> browsers = List.of("local");
protected List<?> browsers = List.of("local");
protected int minBrowsers = 1;
protected int browserIdleTimeoutSeconds = 300;
@@ -315,7 +313,7 @@ public class BrowserProcessor extends Processor {
public void start() {
if (isRunning) return;
if (browsers.isEmpty()) throw new IllegalArgumentException("browsers must not be empty");
List<URI> sshUris = browsers.stream().map(BrowserProcessor::parseBrowser).toList();
List<BrowserConfig> configs = browsers.stream().map(BrowserProcessor::toConfig).toList();
if (minBrowsers > browsers.size()) {
throw new IllegalArgumentException("minBrowsers is " + minBrowsers + " but only " + browsers.size()
+ " browsers are configured");
@@ -346,7 +344,7 @@ public class BrowserProcessor extends Processor {
.setNameFormat("BrowserProcessor-launcher-%d").setDaemon(true).build());
pool.clear();
for (int i = 0; i < browsers.size(); i++) {
pool.add(new Browser(i, sshUris.get(i)));
pool.add(new Browser(i, configs.get(i)));
}
CompletableFuture.allOf(pool.stream()
.map(browser -> CompletableFuture.runAsync(browser::launch, launcher))
@@ -364,20 +362,12 @@ public class BrowserProcessor extends Processor {
}
}
/**
* Returns the ssh URI of a {@link #setBrowsers browsers} entry, or null for a local browser.
*/
private static URI parseBrowser(String browser) {
if (browser.equals("local")) return null;
try {
URI uri = new URI(browser);
SshBrowserProcess.checkUri(uri);
return uri;
} catch (URISyntaxException | IllegalArgumentException e) {
// fall through
private static BrowserConfig toConfig(Object browser) {
BrowserConfig config = browser instanceof String string ? BrowserConfig.parse(string) : (BrowserConfig) browser;
if (config instanceof SshBrowser ssh && ssh.uri() == null) {
throw new IllegalArgumentException("SshBrowser uri must be set");
}
throw new IllegalArgumentException("Unsupported browser '" + browser +
"': expected 'local' or 'ssh://[user@]host[:port]'");
return config;
}
private void handleBeforeRequestSent(WebDriverBiDi driver, Network.BeforeRequestSent event) {
@@ -499,13 +489,17 @@ public class BrowserProcessor extends Processor {
this.browserIdleTimeoutSeconds = browserIdleTimeoutSeconds;
}
public List<String> getBrowsers() {
public List<?> getBrowsers() {
return browsers;
}
/**
* Browsers to open pages in, one per entry. Pages from the same queue are opened in the same browser while it
* has a free tab, so that a site keeps one browser's cookies and cache.
* <p>
* Each entry is a string or a {@link LocalBrowser} or {@link SshBrowser} bean. A bean can override
* {@code concurrency}, {@code executable} and {@code options} for that browser. The strings are shorthand for a
* bean with nothing overridden:
* <ul>
* <li>{@code local} launches {@link #setExecutable(String) executable} on this machine.</li>
* <li>{@code ssh://[user@]host[:port]} launches it on another machine using the ssh command, which carries the
@@ -517,7 +511,7 @@ public class BrowserProcessor extends Processor {
* the same port number here.</li>
* </ul>
*/
public void setBrowsers(List<String> browsers) {
public void setBrowsers(List<?> browsers) {
this.browsers = browsers;
}
@@ -671,7 +665,7 @@ public class BrowserProcessor extends Processor {
private static final long MAX_RETRY_DELAY_SECONDS = 600;
protected enum State { UP, IDLE, WAKING, DOWN }
protected final int id;
private final URI sshUri; // null for a local browser
private final BrowserConfig config;
protected volatile LocalWebDriverBiDi driver;
protected volatile State state = State.DOWN;
protected volatile String lastFailure;
@@ -681,13 +675,25 @@ public class BrowserProcessor extends Processor {
private volatile long idleSince;
private long retryDelaySeconds = INITIAL_RETRY_DELAY_SECONDS; // guarded by this
Browser(int id, URI sshUri) {
Browser(int id, BrowserConfig config) {
this.id = id;
this.sshUri = sshUri;
this.config = config;
}
int concurrency() {
return config.getConcurrency() != null ? config.getConcurrency() : concurrency;
}
private String executable() {
return config.getExecutable() != null ? config.getExecutable() : executable;
}
private List<String> options() {
return config.getOptions() != null ? config.getOptions() : options;
}
private boolean hasFreeTabSlot() {
return state == State.UP && openTabs < concurrency;
return state == State.UP && openTabs < concurrency();
}
/**
@@ -773,16 +779,16 @@ public class BrowserProcessor extends Processor {
BrowserProcess process;
int proxyPort;
String profileDir;
if (sshUri == null) {
if (!(config instanceof SshBrowser ssh)) {
Path scratchDir = crawlController.getScratchDir().getFile().toPath();
Path localProfileDir = scratchDir.resolve("profile-" + id);
Files.createDirectories(localProfileDir);
Files.writeString(localProfileDir.resolve("user.js"), USER_JS);
process = BrowserProcess.launchLocal(executable, options, localProfileDir);
process = BrowserProcess.launchLocal(executable(), options(), localProfileDir);
proxyPort = proxy.getPort();
profileDir = localProfileDir.toString();
} else {
var sshProcess = SshBrowserProcess.launch("ssh", sshUri, executable, options, proxy.getPort(),
var sshProcess = SshBrowserProcess.launch("ssh", ssh.uri(), executable(), options(), proxy.getPort(),
USER_JS);
process = sshProcess;
proxyPort = sshProcess.proxyPort();
@@ -0,0 +1,27 @@
/*
* This file is part of the Heritrix web crawler (crawler.archive.org).
*
* Licensed to the Internet Archive (IA) by one or more individual
* contributors.
*
* The IA licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.archive.crawler.processor;
/**
* A browser of a {@link BrowserProcessor} launched on this machine. The {@code browsers} entry
* {@code local} is shorthand for this bean with nothing set.
*/
public class LocalBrowser extends BrowserConfig {
}
@@ -0,0 +1,57 @@
/*
* This file is part of the Heritrix web crawler (crawler.archive.org).
*
* Licensed to the Internet Archive (IA) by one or more individual
* contributors.
*
* The IA licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.archive.crawler.processor;
import org.archive.net.webdriver.SshBrowserProcess;
import java.net.URI;
import java.net.URISyntaxException;
/**
* A browser of a {@link BrowserProcessor} launched on another machine over ssh, as described under
* {@link BrowserProcessor#setBrowsers browsers}. The {@code browsers} entry
* {@code ssh://[user@]host[:port]} is shorthand for this bean with only {@code uri} set.
*/
public class SshBrowser extends BrowserConfig {
private URI uri;
public String getUri() {
return uri == null ? null : uri.toString();
}
/**
* The machine to launch the browser on, as {@code ssh://[user@]host[:port]}.
*/
public void setUri(String uri) {
URI parsed;
try {
parsed = new URI(uri);
} catch (URISyntaxException e) {
throw new IllegalArgumentException(
"Expected ssh://[user@]host[:port] but got '" + uri + "'", e);
}
SshBrowserProcess.checkUri(parsed);
this.uri = parsed;
}
URI uri() {
return uri;
}
}
@@ -331,6 +331,10 @@ http://example.example/example
<list>
<value>local</value>
<value>ssh://crawler@browser1.example.org</value>
<bean class="org.archive.crawler.processor.SshBrowser">
<property name="uri" value="ssh://crawler@browser2.example.org" />
<property name="concurrency" value="40" />
</bean>
</list>
</property>
<property name="executable" value="firefox" />
@@ -232,6 +232,25 @@ class BrowserProcessorTest {
assertFalse(processor.isRunning());
}
@Test
public void testBrowserBeanOverrides() {
var failing = new LocalBrowser();
failing.setExecutable("/nonexistent/firefox");
var narrow = new LocalBrowser();
narrow.setConcurrency(1);
var processor = newBrowserProcessor("overrides", List.of(failing, narrow, "local"));
processor.start();
try {
assertEquals(BrowserProcessor.Browser.State.DOWN, processor.pool.get(0).state);
assertTrue(processor.pool.get(0).lastFailure.contains("/nonexistent/firefox"),
processor.pool.get(0).lastFailure);
assertEquals(1, processor.pool.get(1).concurrency());
assertEquals(processor.getConcurrency(), processor.pool.get(2).concurrency());
} finally {
processor.stop();
}
}
@Test
public void testRejectsUnsupportedBrowser() {
for (String browser : List.of("remote", "ssh:host", "http://host")) {
@@ -462,7 +481,7 @@ class BrowserProcessorTest {
browserProcessor.start();
}
private static BrowserProcessor newBrowserProcessor(String name, List<String> browsers) {
private static BrowserProcessor newBrowserProcessor(String name, List<?> browsers) {
var crawlController = new CrawlController();
crawlController.getScratchDir().setPath(tempDir.resolve(name).toString());
FetchChain fetchChain = new FetchChain();