From 1c27e2e7b0ae3be6bbc29007aeb35a331f2db166 Mon Sep 17 00:00:00 2001 From: Matt Van Horn <455140+mvanhorn@users.noreply.github.com> Date: Mon, 2 Nov 2026 00:32:19 -0800 Subject: [PATCH] fix: disregard HTTP responses rejected before body fetching A rejection by shouldFetchBodyRule now sets S_OUT_OF_SCOPE, causing the frontier to disregard the URI without charging the queue budget or retrying it, and skipping extraction and WARC writing. Tests cover rejected responses and preserve successful handling for accepted or disabled rules, prerequisites, size limits, and timeouts. Fixes #455 --- .../archive/modules/fetcher/FetchHTTP.java | 11 +++-- .../modules/fetcher/FetchHTTPTest.java | 45 ++++++++++++++++++- 2 files changed, 52 insertions(+), 4 deletions(-) diff --git a/modules/src/main/java/org/archive/modules/fetcher/FetchHTTP.java b/modules/src/main/java/org/archive/modules/fetcher/FetchHTTP.java index 52c90d08..645db25c 100644 --- a/modules/src/main/java/org/archive/modules/fetcher/FetchHTTP.java +++ b/modules/src/main/java/org/archive/modules/fetcher/FetchHTTP.java @@ -24,6 +24,7 @@ import static org.archive.modules.fetcher.FetchErrors.TIMER_TRUNC; import static org.archive.modules.fetcher.FetchStatusCodes.S_CONNECT_FAILED; import static org.archive.modules.fetcher.FetchStatusCodes.S_CONNECT_LOST; import static org.archive.modules.fetcher.FetchStatusCodes.S_DOMAIN_PREREQUISITE_FAILURE; +import static org.archive.modules.fetcher.FetchStatusCodes.S_OUT_OF_SCOPE; import static org.archive.modules.fetcher.FetchStatusCodes.S_UNFETCHABLE_URI; import static org.archive.modules.fetcher.FetchStatusCodes.S_SSL_ERROR; import static org.archive.modules.recrawl.RecrawlAttributeConstants.A_REFERENCE_LENGTH; @@ -465,9 +466,12 @@ public class FetchHTTP extends Processor implements Lifecycle { } /** * DecideRules applied after receipt of HTTP response headers but before we - * start to download the body. If any filter returns FALSE, the fetch is - * aborted. Prerequisites such as robots.txt by-pass filtering (i.e. they - * cannot be midfetch aborted. + * start to download the body. If the rule returns REJECT, the fetch is + * aborted and the URI is marked {@link FetchStatusCodes#S_OUT_OF_SCOPE}. + * This makes {@link CrawlURI#isSuccess()} false, skips extraction and WARC + * writing (including header-only captures), and causes the frontier to + * disregard the URI without a queue-budget charge or retry. Prerequisites + * such as robots.txt bypass filtering and cannot be midfetch aborted. */ public void setShouldFetchBodyRule(DecideRule rule) { kp.put("shouldFetchBodyRule", rule); @@ -641,6 +645,7 @@ public class FetchHTTP extends Processor implements Lifecycle { protected boolean maybeMidfetchAbort(CrawlURI curi, AbstractExecutionAwareRequest request) { if (checkMidfetchAbort(curi)) { doAbort(curi, request, "midFetchAbort"); + curi.setFetchStatus(S_OUT_OF_SCOPE); curi.getRecorder().getRecordedInput().chopAtMessageBodyBegin(); return true; } else { diff --git a/modules/src/test/java/org/archive/modules/fetcher/FetchHTTPTest.java b/modules/src/test/java/org/archive/modules/fetcher/FetchHTTPTest.java index 86a96c57..b485bbe1 100644 --- a/modules/src/test/java/org/archive/modules/fetcher/FetchHTTPTest.java +++ b/modules/src/test/java/org/archive/modules/fetcher/FetchHTTPTest.java @@ -578,6 +578,8 @@ public class FetchHTTPTest { fetcher().setMaxLengthBytes(50000); fetcher().process(curi); assertEquals(50001, curi.getRecordedSize()); + assertEquals(200, curi.getFetchStatus()); + assertFalse(curi.getAnnotations().contains("midFetchAbort")); } @Test @@ -661,7 +663,8 @@ public class FetchHTTPTest { assertEquals("text/plain;charset=US-ASCII", curi.getContentType()); assertTrue(curi.getCredentials().isEmpty()); assertTrue(curi.getFetchDuration() >= 0); - assertEquals(200, curi.getFetchStatus()); + assertEquals(FetchStatusCodes.S_OUT_OF_SCOPE, curi.getFetchStatus()); + assertFalse(curi.isSuccess()); assertSame(curi.getFetchType(), FetchType.HTTP_GET); assertEquals(1, curi.getAnnotations().size()); @@ -674,6 +677,44 @@ public class FetchHTTPTest { assertEquals("", entityString(curi)); } + @Test + public void testShouldFetchBodyRuleAccept() throws Exception { + CrawlURI curi = makeCrawlURI("http://localhost:7777/200k"); + fetcher().process(curi); + + assertEquals(200, curi.getFetchStatus()); + assertTrue(curi.isSuccess()); + assertFalse(messageBodyString(curi).isEmpty()); + assertFalse(curi.getAnnotations().contains("midFetchAbort")); + } + + @Test + public void testShouldFetchBodyRuleNone() throws Exception { + CrawlURI curi = makeCrawlURI("http://localhost:7777/200k"); + RejectDecideRule rule = new RejectDecideRule(); + rule.setEnabled(false); + fetcher().setShouldFetchBodyRule(rule); + fetcher().process(curi); + + assertEquals(200, curi.getFetchStatus()); + assertTrue(curi.isSuccess()); + assertFalse(messageBodyString(curi).isEmpty()); + assertFalse(curi.getAnnotations().contains("midFetchAbort")); + } + + @Test + public void testShouldFetchBodyRulePrerequisite() throws Exception { + CrawlURI curi = makeCrawlURI("http://localhost:7777/200k"); + curi.setPrerequisite(true); + fetcher().setShouldFetchBodyRule(new RejectDecideRule()); + fetcher().process(curi); + + assertEquals(200, curi.getFetchStatus()); + assertTrue(curi.isSuccess()); + assertFalse(messageBodyString(curi).isEmpty()); + assertFalse(curi.getAnnotations().contains("midFetchAbort")); + } + @Test public void testFetchTimeout() throws Exception { CrawlURI curi = makeCrawlURI("http://localhost:7777/slow.txt"); @@ -682,6 +723,8 @@ public class FetchHTTPTest { // logger.info('\n' + httpRequestString(curi) + "\n\n" + rawResponseString(curi)); assertTrue(curi.getAnnotations().contains("timeTrunc")); + assertEquals(200, curi.getFetchStatus()); + assertFalse(curi.getAnnotations().contains("midFetchAbort")); assertTrue(curi.getFetchDuration() >= 2000 && curi.getFetchDuration() < 2200); }