From ff6f98d38da0d00d8017722473e973d3d6297961 Mon Sep 17 00:00:00 2001 From: Alex Osborne Date: Thu, 18 Jun 2026 23:00:21 +0900 Subject: [PATCH 1/5] BrowserProcessor: set User-Agent header Ensures requests made by the browser use the user-agent string configured for the page in the job config. --- .../crawler/processor/BrowserProcessor.java | 29 ++++++++++++++----- .../processor/BrowserProcessorTest.java | 6 ++++ 2 files changed, 28 insertions(+), 7 deletions(-) diff --git a/engine/src/main/java/org/archive/crawler/processor/BrowserProcessor.java b/engine/src/main/java/org/archive/crawler/processor/BrowserProcessor.java index e79c433f..c021aaff 100644 --- a/engine/src/main/java/org/archive/crawler/processor/BrowserProcessor.java +++ b/engine/src/main/java/org/archive/crawler/processor/BrowserProcessor.java @@ -171,7 +171,9 @@ public class BrowserProcessor extends Processor { String pageId = UUID.randomUUID().toString(); var tab = webdriver.browsingContext().create(BrowsingContext.CreateType.tab).context(); try { - BrowserPage page = new BrowserPage(curi, new IdleBarrier(), webdriver, tab, fetcher.getProxy()); + String userAgent = curi.getUserAgent(); + if (userAgent == null) userAgent = fetcher.getUserAgentProvider().getUserAgent(); + BrowserPage page = new BrowserPage(curi, new IdleBarrier(), webdriver, tab, fetcher.getProxy(), userAgent); pages.put(pageId, page); pageIdsByContext.put(tab, pageId); webdriver.network().addIntercept(List.of(Network.InterceptPhase.beforeRequestSent), List.of(tab), @@ -285,7 +287,17 @@ public class BrowserProcessor extends Processor { if (event.request().url().startsWith("data:") || !event.isBlocked()) return; List requestHeaders = event.request().headers(); String pageId = pageIdsByContext.get(event.context()); - if (pageId != null) requestHeaders.add(new Network.Header(PAGE_ID_HEADER, pageId)); + if (pageId != null) { + requestHeaders.add(new Network.Header(PAGE_ID_HEADER, pageId)); + BrowserPage page = pages.get(pageId); + if (page != null) { + String userAgent = page.userAgent(); + if (userAgent != null) { + requestHeaders.removeIf(header -> header.name().equalsIgnoreCase("User-Agent")); + requestHeaders.add(new Network.Header("User-Agent", userAgent)); + } + } + } webdriver.network().continueRequestAsync(event.request().request(), requestHeaders); } @@ -492,11 +504,14 @@ public class BrowserProcessor extends Processor { /** * A CrawlURI that's currently loaded as a page in a browser. */ - protected record BrowserPage(CrawlURI curi, - IdleBarrier networkActivity, - WebDriverBiDi webdriver, - BrowsingContext.Context context, - ProxyConfiguration.Proxy proxy) implements Page { + protected record BrowserPage( + CrawlURI curi, + IdleBarrier networkActivity, + WebDriverBiDi webdriver, + BrowsingContext.Context context, + ProxyConfiguration.Proxy proxy, + String userAgent + ) implements Page { /** * Evaluates JavaScript and returns the result as simple Java objects (numbers, strings, maps, lists). diff --git a/engine/src/test/java/org/archive/crawler/processor/BrowserProcessorTest.java b/engine/src/test/java/org/archive/crawler/processor/BrowserProcessorTest.java index 05887901..a4163a33 100644 --- a/engine/src/test/java/org/archive/crawler/processor/BrowserProcessorTest.java +++ b/engine/src/test/java/org/archive/crawler/processor/BrowserProcessorTest.java @@ -50,9 +50,12 @@ class BrowserProcessorTest { assertEquals(200, crawlURI.getFetchStatus()); browserProcessor.innerProcess(crawlURI); + String expectedUserAgent = fetcher.getUserAgentProvider().getUserAgent(); + var outLinks = new ArrayList<>(crawlURI.getOutLinks()); assertEquals("/link", outLinks.get(0).getUURI().getPath()); assertTrue(crawlURI.getAnnotations().contains("browser")); + assertEquals(expectedUserAgent, crawlURI.getHttpResponseHeader("Reflected-User-Agent")); logger.log(DEBUG, "Subrequests: {0}", subrequests); var subrequestByPath = new HashMap(); @@ -70,6 +73,7 @@ class BrowserProcessorTest { assertTrue(postRequest.startsWith("POST /post-json HTTP/1.1\r\n"), postRequest); assertTrue(postRequest.toLowerCase(Locale.ROOT).contains("content-type: application/json\r\n"), postRequest); assertTrue(postRequest.endsWith("\r\n\r\n" + JSON_POST_BODY), postRequest); + assertEquals(expectedUserAgent, postJson.getHttpResponseHeader("Reflected-User-Agent")); } @Test @@ -174,6 +178,8 @@ class BrowserProcessorTest { httpServer = HttpServer.create(new InetSocketAddress(InetAddress.getLoopbackAddress(), 0), -1); httpServer.createContext("/", exchange -> { logger.log(DEBUG, "Server received request: {0} {1}", exchange.getRequestMethod(), exchange.getRequestURI()); + String userAgent = exchange.getRequestHeaders().getFirst("User-Agent"); + if (userAgent != null) exchange.getResponseHeaders().add("Reflected-User-Agent", userAgent); String contentType = "text/html"; int status = 200; String body = ""; From 4cb48ec5dccb02486bfce49992527ce2fa6854cc Mon Sep 17 00:00:00 2001 From: Alex Osborne Date: Thu, 18 Jun 2026 23:18:23 +0900 Subject: [PATCH 2/5] BrowserProcessor: fix content digest and recording limits for subresources --- .../crawler/processor/BrowserProcessor.java | 2 +- .../crawler/processor/BrowserProcessorTest.java | 2 ++ .../org/archive/modules/fetcher/FetchHTTP2.java | 16 ++++++++++++---- 3 files changed, 15 insertions(+), 5 deletions(-) diff --git a/engine/src/main/java/org/archive/crawler/processor/BrowserProcessor.java b/engine/src/main/java/org/archive/crawler/processor/BrowserProcessor.java index c021aaff..6410e9b7 100644 --- a/engine/src/main/java/org/archive/crawler/processor/BrowserProcessor.java +++ b/engine/src/main/java/org/archive/crawler/processor/BrowserProcessor.java @@ -393,7 +393,7 @@ public class BrowserProcessor extends Processor { crawlController.getRecorderOutBufferBytes(), crawlController.getRecorderInBufferBytes()); this.curi.setRecorder(recorder); - this.curi.setFetchBeginTime(System.currentTimeMillis()); + fetcher.beginRecording(curi, recorder); this.curi.getAnnotations().add("subresource"); this.requestRecorder = Channels.newChannel(recorder.outputWrap(null)); //noinspection resource diff --git a/engine/src/test/java/org/archive/crawler/processor/BrowserProcessorTest.java b/engine/src/test/java/org/archive/crawler/processor/BrowserProcessorTest.java index a4163a33..790f1d41 100644 --- a/engine/src/test/java/org/archive/crawler/processor/BrowserProcessorTest.java +++ b/engine/src/test/java/org/archive/crawler/processor/BrowserProcessorTest.java @@ -56,6 +56,7 @@ class BrowserProcessorTest { assertEquals("/link", outLinks.get(0).getUURI().getPath()); assertTrue(crawlURI.getAnnotations().contains("browser")); assertEquals(expectedUserAgent, crawlURI.getHttpResponseHeader("Reflected-User-Agent")); + assertNotNull(crawlURI.getContentDigest()); logger.log(DEBUG, "Subrequests: {0}", subrequests); var subrequestByPath = new HashMap(); @@ -74,6 +75,7 @@ class BrowserProcessorTest { assertTrue(postRequest.toLowerCase(Locale.ROOT).contains("content-type: application/json\r\n"), postRequest); assertTrue(postRequest.endsWith("\r\n\r\n" + JSON_POST_BODY), postRequest); assertEquals(expectedUserAgent, postJson.getHttpResponseHeader("Reflected-User-Agent")); + assertNotNull(postJson.getContentDigest()); } @Test diff --git a/modules/src/main/java/org/archive/modules/fetcher/FetchHTTP2.java b/modules/src/main/java/org/archive/modules/fetcher/FetchHTTP2.java index 500742bb..e2d32210 100644 --- a/modules/src/main/java/org/archive/modules/fetcher/FetchHTTP2.java +++ b/modules/src/main/java/org/archive/modules/fetcher/FetchHTTP2.java @@ -207,10 +207,7 @@ public class FetchHTTP2 extends Processor implements Lifecycle, InitializingBean var listener = new InputStreamResponseListener(); var recorder = curi.getRecorder(); - if (digestAlgorithm != null) recorder.getRecordedInput().setDigest(digestAlgorithm); - recorder.getRecordedInput().setLimits(getMaxLengthBytes(), - 1000L * (long) getTimeoutSeconds(), getMaxFetchKBSec()); - curi.setFetchBeginTime(System.currentTimeMillis()); + beginRecording(curi, recorder); try { UURI uuri = curi.getUURI(); @@ -560,6 +557,17 @@ public class FetchHTTP2 extends Processor implements Lifecycle, InitializingBean } } + /** + * Prepares the Recorder before a fetch begins: enables content digesting and applies the + * length, time and rate limits. Also records the fetch begin time on the CrawlURI. + */ + public void beginRecording(CrawlURI curi, Recorder recorder) { + if (digestAlgorithm != null) recorder.getRecordedInput().setDigest(digestAlgorithm); + recorder.getRecordedInput().setLimits(getMaxLengthBytes(), + 1000L * (long) getTimeoutSeconds(), getMaxFetchKBSec()); + curi.setFetchBeginTime(System.currentTimeMillis()); + } + /** * Updates the CrawlURI with details from the Recorder after it is closed. */ From 2c9f167ae3efd3a45a87b7df9da067536b99b6e2 Mon Sep 17 00:00:00 2001 From: Alex Osborne Date: Thu, 18 Jun 2026 23:25:20 +0900 Subject: [PATCH 3/5] BrowserProcessor: handle recording truncation with length and timeout limits --- .../crawler/processor/BrowserProcessor.java | 12 ++++++++ .../processor/BrowserProcessorTest.java | 30 +++++++++++++++++++ 2 files changed, 42 insertions(+) diff --git a/engine/src/main/java/org/archive/crawler/processor/BrowserProcessor.java b/engine/src/main/java/org/archive/crawler/processor/BrowserProcessor.java index 6410e9b7..2eeb6053 100644 --- a/engine/src/main/java/org/archive/crawler/processor/BrowserProcessor.java +++ b/engine/src/main/java/org/archive/crawler/processor/BrowserProcessor.java @@ -23,6 +23,8 @@ import org.apache.commons.lang3.StringUtils; import org.archive.crawler.event.CrawlURIDispositionEvent; import org.archive.crawler.framework.CrawlController; import org.archive.crawler.framework.Frontier; +import org.archive.io.RecorderLengthExceededException; +import org.archive.io.RecorderTimeoutException; import org.archive.io.RecordingInputStream; import org.archive.modules.CrawlURI; import org.archive.modules.Processor; @@ -69,6 +71,8 @@ import static java.lang.System.Logger.Level.WARNING; import static org.archive.crawler.event.CrawlURIDispositionEvent.Disposition.FAILED; import static org.archive.crawler.event.CrawlURIDispositionEvent.Disposition.SUCCEEDED; import static org.archive.modules.CoreAttributeConstants.A_HTTP_RESPONSE_HEADERS; +import static org.archive.modules.fetcher.FetchErrors.LENGTH_TRUNC; +import static org.archive.modules.fetcher.FetchErrors.TIMER_TRUNC; /** * Opens a web page in a local web browser via WebDriver BiDi and runs {@link Behavior}s to interact with the page. @@ -383,6 +387,7 @@ public class BrowserProcessor extends Processor { private final BrowserPage page; private String method; private boolean recordingFailed; + private boolean truncated; public SubresourceRecorder(BrowserPage page, String url) throws IOException { this.page = page; @@ -441,8 +446,15 @@ public class BrowserProcessor extends Processor { @Override public void onContent(org.eclipse.jetty.client.Response response, ByteBuffer content) { + if (truncated) return; // already hit a limit; stop recording but treat as success try { responseRecorder.write(content.duplicate()); + } catch (RecorderLengthExceededException e) { + curi.getAnnotations().add(LENGTH_TRUNC); + truncated = true; + } catch (RecorderTimeoutException e) { + curi.getAnnotations().add(TIMER_TRUNC); + truncated = true; } catch (Exception e) { logger.log(ERROR, "Error recording response body of " + curi, e); recordingFailed = true; diff --git a/engine/src/test/java/org/archive/crawler/processor/BrowserProcessorTest.java b/engine/src/test/java/org/archive/crawler/processor/BrowserProcessorTest.java index 790f1d41..e42184b6 100644 --- a/engine/src/test/java/org/archive/crawler/processor/BrowserProcessorTest.java +++ b/engine/src/test/java/org/archive/crawler/processor/BrowserProcessorTest.java @@ -26,6 +26,7 @@ import java.util.*; import java.util.zip.GZIPOutputStream; import static java.lang.System.Logger.Level.DEBUG; +import static org.archive.modules.fetcher.FetchErrors.LENGTH_TRUNC; import static org.junit.jupiter.api.Assertions.*; @EnabledIfSystemProperty(named = "runBrowserTests", matches = "true") @@ -111,6 +112,30 @@ class BrowserProcessorTest { assertFalse(crawlURI.getAnnotations().contains("browser"), "navigation should have aborted"); } + @Test + public void testTruncation() throws IOException, InterruptedException { + fetcher.setMaxLengthBytes(10_000); + try { + CrawlURI crawlURI = newCrawlURI(baseUrl + "truncation"); + fetcher.process(crawlURI); + assertEquals(200, crawlURI.getFetchStatus()); + browserProcessor.innerProcess(crawlURI); + + var subrequestByPath = new HashMap(); + for (CrawlURI curi : subrequests) { + subrequestByPath.put(curi.getUURI().getPath(), curi); + } + CrawlURI large = subrequestByPath.get("/large.bin"); + assertNotNull(large, "oversized subresource should still have been recorded"); + assertTrue(large.getAnnotations().contains(LENGTH_TRUNC), + "oversized subresource should be annotated as length-truncated: " + large.getAnnotations()); + assertTrue(large.isSuccess(), + "truncated subresource should still be a successful fetch, was status " + large.getFetchStatus()); + } finally { + fetcher.setMaxLengthBytes(0L); // restore default (no limit) + } + } + @Test public void testHttpProxy() throws Exception { InetAddress localhost = Inet4Address.getLoopbackAddress(); @@ -226,6 +251,11 @@ class BrowserProcessorTest { body = "sample-download-file"; exchange.getResponseHeaders().add("Content-Disposition", "attachment; filename=heritrix-test.bin"); } + case "/truncation" -> body = ""; + case "/large.bin" -> { + body = "x".repeat(100_000); + contentType = "application/octet-stream"; + } case "/gzip" -> { exchange.getResponseHeaders().add("Content-Encoding", "gzip"); exchange.sendResponseHeaders(200, 0); From d3da2f9afe7b3b5c3ad59a5a80f05f3d8feaa200 Mon Sep 17 00:00:00 2001 From: Alex Osborne Date: Thu, 18 Jun 2026 23:30:48 +0900 Subject: [PATCH 4/5] BrowserProcessor: handle result.isFailed() in SubresourceRecorder.onComplete() --- .../org/archive/crawler/processor/BrowserProcessor.java | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/engine/src/main/java/org/archive/crawler/processor/BrowserProcessor.java b/engine/src/main/java/org/archive/crawler/processor/BrowserProcessor.java index 2eeb6053..580636a8 100644 --- a/engine/src/main/java/org/archive/crawler/processor/BrowserProcessor.java +++ b/engine/src/main/java/org/archive/crawler/processor/BrowserProcessor.java @@ -63,6 +63,7 @@ import java.time.Duration; import java.util.*; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.Semaphore; +import java.util.concurrent.TimeoutException; import java.util.concurrent.atomic.AtomicLong; import java.util.stream.Stream; @@ -469,6 +470,11 @@ public class BrowserProcessor extends Processor { if (recordingFailed) { curi.setFetchStatus(FetchStatusCodes.S_RUNTIME_EXCEPTION); + } else if (result.isFailed() && !truncated) { + Throwable failure = result.getFailure(); + if (failure != null) curi.getNonFatalFailures().add(failure); + curi.setFetchStatus(failure instanceof TimeoutException + ? FetchStatusCodes.S_TIMEOUT : FetchStatusCodes.S_CONNECT_FAILED); } else { subresourcesRecorded.incrementAndGet(); curi.getOverlayNames(); // for sideeffect of creating the overlayNames list From 1f145f792661f2afdf7165fec27670c9a2bdbbcc Mon Sep 17 00:00:00 2001 From: Alex Osborne Date: Thu, 18 Jun 2026 23:50:34 +0900 Subject: [PATCH 5/5] CI: try running browser tests --- .github/workflows/maven.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/maven.yml b/.github/workflows/maven.yml index 34a2d2b1..121bcfb8 100644 --- a/.github/workflows/maven.yml +++ b/.github/workflows/maven.yml @@ -31,8 +31,10 @@ jobs: key: ${{ runner.os }}-maven-${{ hashFiles('**/pom.xml') }} restore-keys: | ${{ runner.os }}-maven- + - name: Show Firefox version + run: firefox --version - name: Build with Maven - run: mvn -B package --file pom.xml + run: mvn -B package --file pom.xml -DrunBrowserTests=true env: # Apparently Azure kills idle http connections after 4 minutes which can cause the build to fail # with a timeout error. Limit the connection ttl to 2 minutes to workaround this as per