mirror of
https://github.com/internetarchive/heritrix3.git
synced 2026-11-22 13:47:02 +00:00
BrowserProcessor: allow per-browser concurrency, executable and options
This commit is contained in:
1 parent
d7c9a5581e
commit
486dffbedf
7 files changed
+232
-28
No files matched your search
@@ -546,6 +546,16 @@ BrowserProcessor
|
||||
|
||||
.. bean-doc:: org.archive.crawler.processor.BrowserProcessor
|
||||
|
||||
LocalBrowser
|
||||
~~~~~~~~~~~~
|
||||
|
||||
.. bean-doc:: org.archive.crawler.processor.LocalBrowser
|
||||
|
||||
SshBrowser
|
||||
~~~~~~~~~~
|
||||
|
||||
.. bean-doc:: org.archive.crawler.processor.SshBrowser
|
||||
|
||||
ExtractLinksBehavior
|
||||
~~~~~~~~~~~~~~~~~~~~
|
||||
|
||||
|
||||
@@ -0,0 +1,81 @@
|
||||
/*
|
||||
* This file is part of the Heritrix web crawler (crawler.archive.org).
|
||||
*
|
||||
* Licensed to the Internet Archive (IA) by one or more individual
|
||||
* contributors.
|
||||
*
|
||||
* The IA licenses this file to You under the Apache License, Version 2.0
|
||||
* (the "License"); you may not use this file except in compliance with
|
||||
* the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package org.archive.crawler.processor;
|
||||
|
||||
import java.util.List;
|
||||
|
||||
/**
|
||||
* Settings for one of the {@link BrowserProcessor#setBrowsers browsers} of a BrowserProcessor.
|
||||
* Settings left unset take the value of the BrowserProcessor's property of the same name.
|
||||
*/
|
||||
public abstract class BrowserConfig {
|
||||
private Integer concurrency;
|
||||
private String executable;
|
||||
private List<String> options;
|
||||
|
||||
/**
|
||||
* Parses a {@link BrowserProcessor#setBrowsers browsers} entry given as a string into a config
|
||||
* with nothing overridden.
|
||||
*/
|
||||
static BrowserConfig parse(String browser) {
|
||||
if (browser.equals("local")) return new LocalBrowser();
|
||||
try {
|
||||
var ssh = new SshBrowser();
|
||||
ssh.setUri(browser);
|
||||
return ssh;
|
||||
} catch (IllegalArgumentException e) {
|
||||
throw new IllegalArgumentException("Unsupported browser '" + browser +
|
||||
"': expected 'local' or 'ssh://[user@]host[:port]'");
|
||||
}
|
||||
}
|
||||
|
||||
public Integer getConcurrency() {
|
||||
return concurrency;
|
||||
}
|
||||
|
||||
/**
|
||||
* Maximum number of web pages that can be open in this browser at once.
|
||||
*/
|
||||
public void setConcurrency(Integer concurrency) {
|
||||
this.concurrency = concurrency;
|
||||
}
|
||||
|
||||
public String getExecutable() {
|
||||
return executable;
|
||||
}
|
||||
|
||||
/**
|
||||
* Webdriver executable to launch for this browser.
|
||||
*/
|
||||
public void setExecutable(String executable) {
|
||||
this.executable = executable;
|
||||
}
|
||||
|
||||
public List<String> getOptions() {
|
||||
return options;
|
||||
}
|
||||
|
||||
/**
|
||||
* Extra command-line options to be passed to this browser's webdriver executable.
|
||||
*/
|
||||
public void setOptions(List<String> options) {
|
||||
this.options = options;
|
||||
}
|
||||
}
|
||||
@@ -54,8 +54,6 @@ import org.springframework.context.ApplicationEventPublisher;
|
||||
import java.io.ByteArrayInputStream;
|
||||
import java.io.IOException;
|
||||
import java.io.UncheckedIOException;
|
||||
import java.net.URI;
|
||||
import java.net.URISyntaxException;
|
||||
import java.nio.ByteBuffer;
|
||||
import java.nio.channels.Channels;
|
||||
import java.nio.channels.WritableByteChannel;
|
||||
@@ -116,7 +114,7 @@ public class BrowserProcessor extends Processor {
|
||||
protected String executable;
|
||||
protected List<String> options = List.of("--headless");
|
||||
protected int concurrency = 20;
|
||||
protected List<String> browsers = List.of("local");
|
||||
protected List<?> browsers = List.of("local");
|
||||
protected int minBrowsers = 1;
|
||||
protected int browserIdleTimeoutSeconds = 300;
|
||||
|
||||
@@ -315,7 +313,7 @@ public class BrowserProcessor extends Processor {
|
||||
public void start() {
|
||||
if (isRunning) return;
|
||||
if (browsers.isEmpty()) throw new IllegalArgumentException("browsers must not be empty");
|
||||
List<URI> sshUris = browsers.stream().map(BrowserProcessor::parseBrowser).toList();
|
||||
List<BrowserConfig> configs = browsers.stream().map(BrowserProcessor::toConfig).toList();
|
||||
if (minBrowsers > browsers.size()) {
|
||||
throw new IllegalArgumentException("minBrowsers is " + minBrowsers + " but only " + browsers.size()
|
||||
+ " browsers are configured");
|
||||
@@ -346,7 +344,7 @@ public class BrowserProcessor extends Processor {
|
||||
.setNameFormat("BrowserProcessor-launcher-%d").setDaemon(true).build());
|
||||
pool.clear();
|
||||
for (int i = 0; i < browsers.size(); i++) {
|
||||
pool.add(new Browser(i, sshUris.get(i)));
|
||||
pool.add(new Browser(i, configs.get(i)));
|
||||
}
|
||||
CompletableFuture.allOf(pool.stream()
|
||||
.map(browser -> CompletableFuture.runAsync(browser::launch, launcher))
|
||||
@@ -364,20 +362,12 @@ public class BrowserProcessor extends Processor {
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns the ssh URI of a {@link #setBrowsers browsers} entry, or null for a local browser.
|
||||
*/
|
||||
private static URI parseBrowser(String browser) {
|
||||
if (browser.equals("local")) return null;
|
||||
try {
|
||||
URI uri = new URI(browser);
|
||||
SshBrowserProcess.checkUri(uri);
|
||||
return uri;
|
||||
} catch (URISyntaxException | IllegalArgumentException e) {
|
||||
// fall through
|
||||
private static BrowserConfig toConfig(Object browser) {
|
||||
BrowserConfig config = browser instanceof String string ? BrowserConfig.parse(string) : (BrowserConfig) browser;
|
||||
if (config instanceof SshBrowser ssh && ssh.uri() == null) {
|
||||
throw new IllegalArgumentException("SshBrowser uri must be set");
|
||||
}
|
||||
throw new IllegalArgumentException("Unsupported browser '" + browser +
|
||||
"': expected 'local' or 'ssh://[user@]host[:port]'");
|
||||
return config;
|
||||
}
|
||||
|
||||
private void handleBeforeRequestSent(WebDriverBiDi driver, Network.BeforeRequestSent event) {
|
||||
@@ -499,13 +489,17 @@ public class BrowserProcessor extends Processor {
|
||||
this.browserIdleTimeoutSeconds = browserIdleTimeoutSeconds;
|
||||
}
|
||||
|
||||
public List<String> getBrowsers() {
|
||||
public List<?> getBrowsers() {
|
||||
return browsers;
|
||||
}
|
||||
|
||||
/**
|
||||
* Browsers to open pages in, one per entry. Pages from the same queue are opened in the same browser while it
|
||||
* has a free tab, so that a site keeps one browser's cookies and cache.
|
||||
* <p>
|
||||
* Each entry is a string or a {@link LocalBrowser} or {@link SshBrowser} bean. A bean can override
|
||||
* {@code concurrency}, {@code executable} and {@code options} for that browser. The strings are shorthand for a
|
||||
* bean with nothing overridden:
|
||||
* <ul>
|
||||
* <li>{@code local} launches {@link #setExecutable(String) executable} on this machine.</li>
|
||||
* <li>{@code ssh://[user@]host[:port]} launches it on another machine using the ssh command, which carries the
|
||||
@@ -517,7 +511,7 @@ public class BrowserProcessor extends Processor {
|
||||
* the same port number here.</li>
|
||||
* </ul>
|
||||
*/
|
||||
public void setBrowsers(List<String> browsers) {
|
||||
public void setBrowsers(List<?> browsers) {
|
||||
this.browsers = browsers;
|
||||
}
|
||||
|
||||
@@ -671,7 +665,7 @@ public class BrowserProcessor extends Processor {
|
||||
private static final long MAX_RETRY_DELAY_SECONDS = 600;
|
||||
protected enum State { UP, IDLE, WAKING, DOWN }
|
||||
protected final int id;
|
||||
private final URI sshUri; // null for a local browser
|
||||
private final BrowserConfig config;
|
||||
protected volatile LocalWebDriverBiDi driver;
|
||||
protected volatile State state = State.DOWN;
|
||||
protected volatile String lastFailure;
|
||||
@@ -681,13 +675,25 @@ public class BrowserProcessor extends Processor {
|
||||
private volatile long idleSince;
|
||||
private long retryDelaySeconds = INITIAL_RETRY_DELAY_SECONDS; // guarded by this
|
||||
|
||||
Browser(int id, URI sshUri) {
|
||||
Browser(int id, BrowserConfig config) {
|
||||
this.id = id;
|
||||
this.sshUri = sshUri;
|
||||
this.config = config;
|
||||
}
|
||||
|
||||
int concurrency() {
|
||||
return config.getConcurrency() != null ? config.getConcurrency() : concurrency;
|
||||
}
|
||||
|
||||
private String executable() {
|
||||
return config.getExecutable() != null ? config.getExecutable() : executable;
|
||||
}
|
||||
|
||||
private List<String> options() {
|
||||
return config.getOptions() != null ? config.getOptions() : options;
|
||||
}
|
||||
|
||||
private boolean hasFreeTabSlot() {
|
||||
return state == State.UP && openTabs < concurrency;
|
||||
return state == State.UP && openTabs < concurrency();
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -773,16 +779,16 @@ public class BrowserProcessor extends Processor {
|
||||
BrowserProcess process;
|
||||
int proxyPort;
|
||||
String profileDir;
|
||||
if (sshUri == null) {
|
||||
if (!(config instanceof SshBrowser ssh)) {
|
||||
Path scratchDir = crawlController.getScratchDir().getFile().toPath();
|
||||
Path localProfileDir = scratchDir.resolve("profile-" + id);
|
||||
Files.createDirectories(localProfileDir);
|
||||
Files.writeString(localProfileDir.resolve("user.js"), USER_JS);
|
||||
process = BrowserProcess.launchLocal(executable, options, localProfileDir);
|
||||
process = BrowserProcess.launchLocal(executable(), options(), localProfileDir);
|
||||
proxyPort = proxy.getPort();
|
||||
profileDir = localProfileDir.toString();
|
||||
} else {
|
||||
var sshProcess = SshBrowserProcess.launch("ssh", sshUri, executable, options, proxy.getPort(),
|
||||
var sshProcess = SshBrowserProcess.launch("ssh", ssh.uri(), executable(), options(), proxy.getPort(),
|
||||
USER_JS);
|
||||
process = sshProcess;
|
||||
proxyPort = sshProcess.proxyPort();
|
||||
|
||||
@@ -0,0 +1,27 @@
|
||||
/*
|
||||
* This file is part of the Heritrix web crawler (crawler.archive.org).
|
||||
*
|
||||
* Licensed to the Internet Archive (IA) by one or more individual
|
||||
* contributors.
|
||||
*
|
||||
* The IA licenses this file to You under the Apache License, Version 2.0
|
||||
* (the "License"); you may not use this file except in compliance with
|
||||
* the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package org.archive.crawler.processor;
|
||||
|
||||
/**
|
||||
* A browser of a {@link BrowserProcessor} launched on this machine. The {@code browsers} entry
|
||||
* {@code local} is shorthand for this bean with nothing set.
|
||||
*/
|
||||
public class LocalBrowser extends BrowserConfig {
|
||||
}
|
||||
@@ -0,0 +1,57 @@
|
||||
/*
|
||||
* This file is part of the Heritrix web crawler (crawler.archive.org).
|
||||
*
|
||||
* Licensed to the Internet Archive (IA) by one or more individual
|
||||
* contributors.
|
||||
*
|
||||
* The IA licenses this file to You under the Apache License, Version 2.0
|
||||
* (the "License"); you may not use this file except in compliance with
|
||||
* the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package org.archive.crawler.processor;
|
||||
|
||||
import org.archive.net.webdriver.SshBrowserProcess;
|
||||
|
||||
import java.net.URI;
|
||||
import java.net.URISyntaxException;
|
||||
|
||||
/**
|
||||
* A browser of a {@link BrowserProcessor} launched on another machine over ssh, as described under
|
||||
* {@link BrowserProcessor#setBrowsers browsers}. The {@code browsers} entry
|
||||
* {@code ssh://[user@]host[:port]} is shorthand for this bean with only {@code uri} set.
|
||||
*/
|
||||
public class SshBrowser extends BrowserConfig {
|
||||
private URI uri;
|
||||
|
||||
public String getUri() {
|
||||
return uri == null ? null : uri.toString();
|
||||
}
|
||||
|
||||
/**
|
||||
* The machine to launch the browser on, as {@code ssh://[user@]host[:port]}.
|
||||
*/
|
||||
public void setUri(String uri) {
|
||||
URI parsed;
|
||||
try {
|
||||
parsed = new URI(uri);
|
||||
} catch (URISyntaxException e) {
|
||||
throw new IllegalArgumentException(
|
||||
"Expected ssh://[user@]host[:port] but got '" + uri + "'", e);
|
||||
}
|
||||
SshBrowserProcess.checkUri(parsed);
|
||||
this.uri = parsed;
|
||||
}
|
||||
|
||||
URI uri() {
|
||||
return uri;
|
||||
}
|
||||
}
|
||||
@@ -331,6 +331,10 @@ http://example.example/example
|
||||
<list>
|
||||
<value>local</value>
|
||||
<value>ssh://crawler@browser1.example.org</value>
|
||||
<bean class="org.archive.crawler.processor.SshBrowser">
|
||||
<property name="uri" value="ssh://crawler@browser2.example.org" />
|
||||
<property name="concurrency" value="40" />
|
||||
</bean>
|
||||
</list>
|
||||
</property>
|
||||
<property name="executable" value="firefox" />
|
||||
|
||||
@@ -232,6 +232,25 @@ class BrowserProcessorTest {
|
||||
assertFalse(processor.isRunning());
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testBrowserBeanOverrides() {
|
||||
var failing = new LocalBrowser();
|
||||
failing.setExecutable("/nonexistent/firefox");
|
||||
var narrow = new LocalBrowser();
|
||||
narrow.setConcurrency(1);
|
||||
var processor = newBrowserProcessor("overrides", List.of(failing, narrow, "local"));
|
||||
processor.start();
|
||||
try {
|
||||
assertEquals(BrowserProcessor.Browser.State.DOWN, processor.pool.get(0).state);
|
||||
assertTrue(processor.pool.get(0).lastFailure.contains("/nonexistent/firefox"),
|
||||
processor.pool.get(0).lastFailure);
|
||||
assertEquals(1, processor.pool.get(1).concurrency());
|
||||
assertEquals(processor.getConcurrency(), processor.pool.get(2).concurrency());
|
||||
} finally {
|
||||
processor.stop();
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testRejectsUnsupportedBrowser() {
|
||||
for (String browser : List.of("remote", "ssh:host", "http://host")) {
|
||||
@@ -462,7 +481,7 @@ class BrowserProcessorTest {
|
||||
browserProcessor.start();
|
||||
}
|
||||
|
||||
private static BrowserProcessor newBrowserProcessor(String name, List<String> browsers) {
|
||||
private static BrowserProcessor newBrowserProcessor(String name, List<?> browsers) {
|
||||
var crawlController = new CrawlController();
|
||||
crawlController.getScratchDir().setPath(tempDir.resolve(name).toString());
|
||||
FetchChain fetchChain = new FetchChain();
|
||||
|
||||
Reference in new issue
Block a user