Merge pull request #742 from internetarchive/ato/BrowserProcessor-fixes

BrowserProcessor fixes
This commit is contained in:
Alex Osborne authored and GitHub committed 2026-06-19 14:29:34 +09:00
commit 6344f8fdcf
4 files changed
+94 -13

No files matched your search

+3 -1
View File
@@ -31,8 +31,10 @@ jobs:
key: ${{ runner.os }}-maven-${{ hashFiles('**/pom.xml') }}
restore-keys: |
${{ runner.os }}-maven-
- name: Show Firefox version
run: firefox --version
- name: Build with Maven
run: mvn -B package --file pom.xml
run: mvn -B package --file pom.xml -DrunBrowserTests=true
env:
# Apparently Azure kills idle http connections after 4 minutes which can cause the build to fail
# with a timeout error. Limit the connection ttl to 2 minutes to workaround this as per
@@ -23,6 +23,8 @@ import org.apache.commons.lang3.StringUtils;
import org.archive.crawler.event.CrawlURIDispositionEvent;
import org.archive.crawler.framework.CrawlController;
import org.archive.crawler.framework.Frontier;
import org.archive.io.RecorderLengthExceededException;
import org.archive.io.RecorderTimeoutException;
import org.archive.io.RecordingInputStream;
import org.archive.modules.CrawlURI;
import org.archive.modules.Processor;
@@ -61,6 +63,7 @@ import java.time.Duration;
import java.util.*;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.Semaphore;
import java.util.concurrent.TimeoutException;
import java.util.concurrent.atomic.AtomicLong;
import java.util.stream.Stream;
@@ -69,6 +72,8 @@ import static java.lang.System.Logger.Level.WARNING;
import static org.archive.crawler.event.CrawlURIDispositionEvent.Disposition.FAILED;
import static org.archive.crawler.event.CrawlURIDispositionEvent.Disposition.SUCCEEDED;
import static org.archive.modules.CoreAttributeConstants.A_HTTP_RESPONSE_HEADERS;
import static org.archive.modules.fetcher.FetchErrors.LENGTH_TRUNC;
import static org.archive.modules.fetcher.FetchErrors.TIMER_TRUNC;
/**
* Opens a web page in a local web browser via WebDriver BiDi and runs {@link Behavior}s to interact with the page.
@@ -171,7 +176,9 @@ public class BrowserProcessor extends Processor {
String pageId = UUID.randomUUID().toString();
var tab = webdriver.browsingContext().create(BrowsingContext.CreateType.tab).context();
try {
BrowserPage page = new BrowserPage(curi, new IdleBarrier(), webdriver, tab, fetcher.getProxy());
String userAgent = curi.getUserAgent();
if (userAgent == null) userAgent = fetcher.getUserAgentProvider().getUserAgent();
BrowserPage page = new BrowserPage(curi, new IdleBarrier(), webdriver, tab, fetcher.getProxy(), userAgent);
pages.put(pageId, page);
pageIdsByContext.put(tab, pageId);
webdriver.network().addIntercept(List.of(Network.InterceptPhase.beforeRequestSent), List.of(tab),
@@ -285,7 +292,17 @@ public class BrowserProcessor extends Processor {
if (event.request().url().startsWith("data:") || !event.isBlocked()) return;
List<Network.Header> requestHeaders = event.request().headers();
String pageId = pageIdsByContext.get(event.context());
if (pageId != null) requestHeaders.add(new Network.Header(PAGE_ID_HEADER, pageId));
if (pageId != null) {
requestHeaders.add(new Network.Header(PAGE_ID_HEADER, pageId));
BrowserPage page = pages.get(pageId);
if (page != null) {
String userAgent = page.userAgent();
if (userAgent != null) {
requestHeaders.removeIf(header -> header.name().equalsIgnoreCase("User-Agent"));
requestHeaders.add(new Network.Header("User-Agent", userAgent));
}
}
}
webdriver.network().continueRequestAsync(event.request().request(), requestHeaders);
}
@@ -371,6 +388,7 @@ public class BrowserProcessor extends Processor {
private final BrowserPage page;
private String method;
private boolean recordingFailed;
private boolean truncated;
public SubresourceRecorder(BrowserPage page, String url) throws IOException {
this.page = page;
@@ -381,7 +399,7 @@ public class BrowserProcessor extends Processor {
crawlController.getRecorderOutBufferBytes(),
crawlController.getRecorderInBufferBytes());
this.curi.setRecorder(recorder);
this.curi.setFetchBeginTime(System.currentTimeMillis());
fetcher.beginRecording(curi, recorder);
this.curi.getAnnotations().add("subresource");
this.requestRecorder = Channels.newChannel(recorder.outputWrap(null));
//noinspection resource
@@ -429,8 +447,15 @@ public class BrowserProcessor extends Processor {
@Override
public void onContent(org.eclipse.jetty.client.Response response, ByteBuffer content) {
if (truncated) return; // already hit a limit; stop recording but treat as success
try {
responseRecorder.write(content.duplicate());
} catch (RecorderLengthExceededException e) {
curi.getAnnotations().add(LENGTH_TRUNC);
truncated = true;
} catch (RecorderTimeoutException e) {
curi.getAnnotations().add(TIMER_TRUNC);
truncated = true;
} catch (Exception e) {
logger.log(ERROR, "Error recording response body of " + curi, e);
recordingFailed = true;
@@ -445,6 +470,11 @@ public class BrowserProcessor extends Processor {
if (recordingFailed) {
curi.setFetchStatus(FetchStatusCodes.S_RUNTIME_EXCEPTION);
} else if (result.isFailed() && !truncated) {
Throwable failure = result.getFailure();
if (failure != null) curi.getNonFatalFailures().add(failure);
curi.setFetchStatus(failure instanceof TimeoutException
? FetchStatusCodes.S_TIMEOUT : FetchStatusCodes.S_CONNECT_FAILED);
} else {
subresourcesRecorded.incrementAndGet();
curi.getOverlayNames(); // for sideeffect of creating the overlayNames list
@@ -492,11 +522,14 @@ public class BrowserProcessor extends Processor {
/**
* A CrawlURI that's currently loaded as a page in a browser.
*/
protected record BrowserPage(CrawlURI curi,
IdleBarrier networkActivity,
WebDriverBiDi webdriver,
BrowsingContext.Context context,
ProxyConfiguration.Proxy proxy) implements Page {
protected record BrowserPage(
CrawlURI curi,
IdleBarrier networkActivity,
WebDriverBiDi webdriver,
BrowsingContext.Context context,
ProxyConfiguration.Proxy proxy,
String userAgent
) implements Page {
/**
* Evaluates JavaScript and returns the result as simple Java objects (numbers, strings, maps, lists).
@@ -26,6 +26,7 @@ import java.util.*;
import java.util.zip.GZIPOutputStream;
import static java.lang.System.Logger.Level.DEBUG;
import static org.archive.modules.fetcher.FetchErrors.LENGTH_TRUNC;
import static org.junit.jupiter.api.Assertions.*;
@EnabledIfSystemProperty(named = "runBrowserTests", matches = "true")
@@ -50,9 +51,13 @@ class BrowserProcessorTest {
assertEquals(200, crawlURI.getFetchStatus());
browserProcessor.innerProcess(crawlURI);
String expectedUserAgent = fetcher.getUserAgentProvider().getUserAgent();
var outLinks = new ArrayList<>(crawlURI.getOutLinks());
assertEquals("/link", outLinks.get(0).getUURI().getPath());
assertTrue(crawlURI.getAnnotations().contains("browser"));
assertEquals(expectedUserAgent, crawlURI.getHttpResponseHeader("Reflected-User-Agent"));
assertNotNull(crawlURI.getContentDigest());
logger.log(DEBUG, "Subrequests: {0}", subrequests);
var subrequestByPath = new HashMap<String,CrawlURI>();
@@ -70,6 +75,8 @@ class BrowserProcessorTest {
assertTrue(postRequest.startsWith("POST /post-json HTTP/1.1\r\n"), postRequest);
assertTrue(postRequest.toLowerCase(Locale.ROOT).contains("content-type: application/json\r\n"), postRequest);
assertTrue(postRequest.endsWith("\r\n\r\n" + JSON_POST_BODY), postRequest);
assertEquals(expectedUserAgent, postJson.getHttpResponseHeader("Reflected-User-Agent"));
assertNotNull(postJson.getContentDigest());
}
@Test
@@ -105,6 +112,30 @@ class BrowserProcessorTest {
assertFalse(crawlURI.getAnnotations().contains("browser"), "navigation should have aborted");
}
@Test
public void testTruncation() throws IOException, InterruptedException {
fetcher.setMaxLengthBytes(10_000);
try {
CrawlURI crawlURI = newCrawlURI(baseUrl + "truncation");
fetcher.process(crawlURI);
assertEquals(200, crawlURI.getFetchStatus());
browserProcessor.innerProcess(crawlURI);
var subrequestByPath = new HashMap<String,CrawlURI>();
for (CrawlURI curi : subrequests) {
subrequestByPath.put(curi.getUURI().getPath(), curi);
}
CrawlURI large = subrequestByPath.get("/large.bin");
assertNotNull(large, "oversized subresource should still have been recorded");
assertTrue(large.getAnnotations().contains(LENGTH_TRUNC),
"oversized subresource should be annotated as length-truncated: " + large.getAnnotations());
assertTrue(large.isSuccess(),
"truncated subresource should still be a successful fetch, was status " + large.getFetchStatus());
} finally {
fetcher.setMaxLengthBytes(0L); // restore default (no limit)
}
}
@Test
public void testHttpProxy() throws Exception {
InetAddress localhost = Inet4Address.getLoopbackAddress();
@@ -174,6 +205,8 @@ class BrowserProcessorTest {
httpServer = HttpServer.create(new InetSocketAddress(InetAddress.getLoopbackAddress(), 0), -1);
httpServer.createContext("/", exchange -> {
logger.log(DEBUG, "Server received request: {0} {1}", exchange.getRequestMethod(), exchange.getRequestURI());
String userAgent = exchange.getRequestHeaders().getFirst("User-Agent");
if (userAgent != null) exchange.getResponseHeaders().add("Reflected-User-Agent", userAgent);
String contentType = "text/html";
int status = 200;
String body = "";
@@ -218,6 +251,11 @@ class BrowserProcessorTest {
body = "sample-download-file";
exchange.getResponseHeaders().add("Content-Disposition", "attachment; filename=heritrix-test.bin");
}
case "/truncation" -> body = "<img src=large.bin>";
case "/large.bin" -> {
body = "x".repeat(100_000);
contentType = "application/octet-stream";
}
case "/gzip" -> {
exchange.getResponseHeaders().add("Content-Encoding", "gzip");
exchange.sendResponseHeaders(200, 0);
@@ -207,10 +207,7 @@ public class FetchHTTP2 extends Processor implements Lifecycle, InitializingBean
var listener = new InputStreamResponseListener();
var recorder = curi.getRecorder();
if (digestAlgorithm != null) recorder.getRecordedInput().setDigest(digestAlgorithm);
recorder.getRecordedInput().setLimits(getMaxLengthBytes(),
1000L * (long) getTimeoutSeconds(), getMaxFetchKBSec());
curi.setFetchBeginTime(System.currentTimeMillis());
beginRecording(curi, recorder);
try {
UURI uuri = curi.getUURI();
@@ -560,6 +557,17 @@ public class FetchHTTP2 extends Processor implements Lifecycle, InitializingBean
}
}
/**
* Prepares the Recorder before a fetch begins: enables content digesting and applies the
* length, time and rate limits. Also records the fetch begin time on the CrawlURI.
*/
public void beginRecording(CrawlURI curi, Recorder recorder) {
if (digestAlgorithm != null) recorder.getRecordedInput().setDigest(digestAlgorithm);
recorder.getRecordedInput().setLimits(getMaxLengthBytes(),
1000L * (long) getTimeoutSeconds(), getMaxFetchKBSec());
curi.setFetchBeginTime(System.currentTimeMillis());
}
/**
* Updates the CrawlURI with details from the Recorder after it is closed.
*/