From 4cb48ec5dccb02486bfce49992527ce2fa6854cc Mon Sep 17 00:00:00 2001 From: Alex Osborne Date: Thu, 18 Jun 2026 23:18:23 +0900 Subject: [PATCH] BrowserProcessor: fix content digest and recording limits for subresources --- .../crawler/processor/BrowserProcessor.java | 2 +- .../crawler/processor/BrowserProcessorTest.java | 2 ++ .../org/archive/modules/fetcher/FetchHTTP2.java | 16 ++++++++++++---- 3 files changed, 15 insertions(+), 5 deletions(-) diff --git a/engine/src/main/java/org/archive/crawler/processor/BrowserProcessor.java b/engine/src/main/java/org/archive/crawler/processor/BrowserProcessor.java index c021aaff..6410e9b7 100644 --- a/engine/src/main/java/org/archive/crawler/processor/BrowserProcessor.java +++ b/engine/src/main/java/org/archive/crawler/processor/BrowserProcessor.java @@ -393,7 +393,7 @@ public class BrowserProcessor extends Processor { crawlController.getRecorderOutBufferBytes(), crawlController.getRecorderInBufferBytes()); this.curi.setRecorder(recorder); - this.curi.setFetchBeginTime(System.currentTimeMillis()); + fetcher.beginRecording(curi, recorder); this.curi.getAnnotations().add("subresource"); this.requestRecorder = Channels.newChannel(recorder.outputWrap(null)); //noinspection resource diff --git a/engine/src/test/java/org/archive/crawler/processor/BrowserProcessorTest.java b/engine/src/test/java/org/archive/crawler/processor/BrowserProcessorTest.java index a4163a33..790f1d41 100644 --- a/engine/src/test/java/org/archive/crawler/processor/BrowserProcessorTest.java +++ b/engine/src/test/java/org/archive/crawler/processor/BrowserProcessorTest.java @@ -56,6 +56,7 @@ class BrowserProcessorTest { assertEquals("/link", outLinks.get(0).getUURI().getPath()); assertTrue(crawlURI.getAnnotations().contains("browser")); assertEquals(expectedUserAgent, crawlURI.getHttpResponseHeader("Reflected-User-Agent")); + assertNotNull(crawlURI.getContentDigest()); logger.log(DEBUG, "Subrequests: {0}", subrequests); var subrequestByPath = new HashMap(); @@ -74,6 +75,7 @@ class BrowserProcessorTest { assertTrue(postRequest.toLowerCase(Locale.ROOT).contains("content-type: application/json\r\n"), postRequest); assertTrue(postRequest.endsWith("\r\n\r\n" + JSON_POST_BODY), postRequest); assertEquals(expectedUserAgent, postJson.getHttpResponseHeader("Reflected-User-Agent")); + assertNotNull(postJson.getContentDigest()); } @Test diff --git a/modules/src/main/java/org/archive/modules/fetcher/FetchHTTP2.java b/modules/src/main/java/org/archive/modules/fetcher/FetchHTTP2.java index 500742bb..e2d32210 100644 --- a/modules/src/main/java/org/archive/modules/fetcher/FetchHTTP2.java +++ b/modules/src/main/java/org/archive/modules/fetcher/FetchHTTP2.java @@ -207,10 +207,7 @@ public class FetchHTTP2 extends Processor implements Lifecycle, InitializingBean var listener = new InputStreamResponseListener(); var recorder = curi.getRecorder(); - if (digestAlgorithm != null) recorder.getRecordedInput().setDigest(digestAlgorithm); - recorder.getRecordedInput().setLimits(getMaxLengthBytes(), - 1000L * (long) getTimeoutSeconds(), getMaxFetchKBSec()); - curi.setFetchBeginTime(System.currentTimeMillis()); + beginRecording(curi, recorder); try { UURI uuri = curi.getUURI(); @@ -560,6 +557,17 @@ public class FetchHTTP2 extends Processor implements Lifecycle, InitializingBean } } + /** + * Prepares the Recorder before a fetch begins: enables content digesting and applies the + * length, time and rate limits. Also records the fetch begin time on the CrawlURI. + */ + public void beginRecording(CrawlURI curi, Recorder recorder) { + if (digestAlgorithm != null) recorder.getRecordedInput().setDigest(digestAlgorithm); + recorder.getRecordedInput().setLimits(getMaxLengthBytes(), + 1000L * (long) getTimeoutSeconds(), getMaxFetchKBSec()); + curi.setFetchBeginTime(System.currentTimeMillis()); + } + /** * Updates the CrawlURI with details from the Recorder after it is closed. */