lee/heritrix3
1
0
Fork 0
mirror of https://github.com/internetarchive/heritrix3.git synced 2026-08-28 09:26:54 +00:00
Code Issues Packages Projects Releases Wiki Activity

BrowserProcessor: fix content digest and recording limits for subresources

Browse Source
This commit is contained in:
Alex Osborne
2026-06-18 23:18:23 +09:00
parent ff6f98d38d
commit 4cb48ec5dc
3 changed files with 15 additions and 5 deletions
Download Patch File Download Diff File Expand all files Collapse all files
engine/src/main/java/org/archive/crawler/processor/BrowserProcessor.java
+1 -1
View File
@@ -393,7 +393,7 @@ public class BrowserProcessor extends Processor {
crawlController.getRecorderOutBufferBytes(),
crawlController.getRecorderInBufferBytes());
this.curi.setRecorder(recorder);
this.curi.setFetchBeginTime(System.currentTimeMillis());
fetcher.beginRecording(curi, recorder);
this.curi.getAnnotations().add("subresource");
this.requestRecorder = Channels.newChannel(recorder.outputWrap(null));
//noinspection resource
engine/src/test/java/org/archive/crawler/processor/BrowserProcessorTest.java
+2
View File
@@ -56,6 +56,7 @@ class BrowserProcessorTest {
assertEquals("/link", outLinks.get(0).getUURI().getPath());
assertTrue(crawlURI.getAnnotations().contains("browser"));
assertEquals(expectedUserAgent, crawlURI.getHttpResponseHeader("Reflected-User-Agent"));
assertNotNull(crawlURI.getContentDigest());
logger.log(DEBUG, "Subrequests: {0}", subrequests);
var subrequestByPath = new HashMap<String,CrawlURI>();
@@ -74,6 +75,7 @@ class BrowserProcessorTest {
assertTrue(postRequest.toLowerCase(Locale.ROOT).contains("content-type: application/json\r\n"), postRequest);
assertTrue(postRequest.endsWith("\r\n\r\n" + JSON_POST_BODY), postRequest);
assertEquals(expectedUserAgent, postJson.getHttpResponseHeader("Reflected-User-Agent"));
assertNotNull(postJson.getContentDigest());
}
@Test
modules/src/main/java/org/archive/modules/fetcher/FetchHTTP2.java
+12 -4
View File
@@ -207,10 +207,7 @@ public class FetchHTTP2 extends Processor implements Lifecycle, InitializingBean
var listener = new InputStreamResponseListener();
var recorder = curi.getRecorder();
if (digestAlgorithm != null) recorder.getRecordedInput().setDigest(digestAlgorithm);
recorder.getRecordedInput().setLimits(getMaxLengthBytes(),
1000L * (long) getTimeoutSeconds(), getMaxFetchKBSec());
curi.setFetchBeginTime(System.currentTimeMillis());
beginRecording(curi, recorder);
try {
UURI uuri = curi.getUURI();
@@ -560,6 +557,17 @@ public class FetchHTTP2 extends Processor implements Lifecycle, InitializingBean
}
}
/**
* Prepares the Recorder before a fetch begins: enables content digesting and applies the
* length, time and rate limits. Also records the fetch begin time on the CrawlURI.
*/
public void beginRecording(CrawlURI curi, Recorder recorder) {
if (digestAlgorithm != null) recorder.getRecordedInput().setDigest(digestAlgorithm);
recorder.getRecordedInput().setLimits(getMaxLengthBytes(),
1000L * (long) getTimeoutSeconds(), getMaxFetchKBSec());
curi.setFetchBeginTime(System.currentTimeMillis());
}
/**
* Updates the CrawlURI with details from the Recorder after it is closed.
*/