From 1c27e2e7b0ae3be6bbc29007aeb35a331f2db166 Mon Sep 17 00:00:00 2001 From: Matt Van Horn <455140+mvanhorn@users.noreply.github.com> Date: Mon, 2 Nov 2026 00:32:19 -0800 Subject: [PATCH 1/2] fix: disregard HTTP responses rejected before body fetching A rejection by shouldFetchBodyRule now sets S_OUT_OF_SCOPE, causing the frontier to disregard the URI without charging the queue budget or retrying it, and skipping extraction and WARC writing. Tests cover rejected responses and preserve successful handling for accepted or disabled rules, prerequisites, size limits, and timeouts. Fixes #455 --- .../archive/modules/fetcher/FetchHTTP.java | 11 +++-- .../modules/fetcher/FetchHTTPTest.java | 45 ++++++++++++++++++- 2 files changed, 52 insertions(+), 4 deletions(-) diff --git a/modules/src/main/java/org/archive/modules/fetcher/FetchHTTP.java b/modules/src/main/java/org/archive/modules/fetcher/FetchHTTP.java index 52c90d08..645db25c 100644 --- a/modules/src/main/java/org/archive/modules/fetcher/FetchHTTP.java +++ b/modules/src/main/java/org/archive/modules/fetcher/FetchHTTP.java @@ -24,6 +24,7 @@ import static org.archive.modules.fetcher.FetchErrors.TIMER_TRUNC; import static org.archive.modules.fetcher.FetchStatusCodes.S_CONNECT_FAILED; import static org.archive.modules.fetcher.FetchStatusCodes.S_CONNECT_LOST; import static org.archive.modules.fetcher.FetchStatusCodes.S_DOMAIN_PREREQUISITE_FAILURE; +import static org.archive.modules.fetcher.FetchStatusCodes.S_OUT_OF_SCOPE; import static org.archive.modules.fetcher.FetchStatusCodes.S_UNFETCHABLE_URI; import static org.archive.modules.fetcher.FetchStatusCodes.S_SSL_ERROR; import static org.archive.modules.recrawl.RecrawlAttributeConstants.A_REFERENCE_LENGTH; @@ -465,9 +466,12 @@ public class FetchHTTP extends Processor implements Lifecycle { } /** * DecideRules applied after receipt of HTTP response headers but before we - * start to download the body. If any filter returns FALSE, the fetch is - * aborted. Prerequisites such as robots.txt by-pass filtering (i.e. they - * cannot be midfetch aborted. + * start to download the body. If the rule returns REJECT, the fetch is + * aborted and the URI is marked {@link FetchStatusCodes#S_OUT_OF_SCOPE}. + * This makes {@link CrawlURI#isSuccess()} false, skips extraction and WARC + * writing (including header-only captures), and causes the frontier to + * disregard the URI without a queue-budget charge or retry. Prerequisites + * such as robots.txt bypass filtering and cannot be midfetch aborted. */ public void setShouldFetchBodyRule(DecideRule rule) { kp.put("shouldFetchBodyRule", rule); @@ -641,6 +645,7 @@ public class FetchHTTP extends Processor implements Lifecycle { protected boolean maybeMidfetchAbort(CrawlURI curi, AbstractExecutionAwareRequest request) { if (checkMidfetchAbort(curi)) { doAbort(curi, request, "midFetchAbort"); + curi.setFetchStatus(S_OUT_OF_SCOPE); curi.getRecorder().getRecordedInput().chopAtMessageBodyBegin(); return true; } else { diff --git a/modules/src/test/java/org/archive/modules/fetcher/FetchHTTPTest.java b/modules/src/test/java/org/archive/modules/fetcher/FetchHTTPTest.java index 86a96c57..b485bbe1 100644 --- a/modules/src/test/java/org/archive/modules/fetcher/FetchHTTPTest.java +++ b/modules/src/test/java/org/archive/modules/fetcher/FetchHTTPTest.java @@ -578,6 +578,8 @@ public class FetchHTTPTest { fetcher().setMaxLengthBytes(50000); fetcher().process(curi); assertEquals(50001, curi.getRecordedSize()); + assertEquals(200, curi.getFetchStatus()); + assertFalse(curi.getAnnotations().contains("midFetchAbort")); } @Test @@ -661,7 +663,8 @@ public class FetchHTTPTest { assertEquals("text/plain;charset=US-ASCII", curi.getContentType()); assertTrue(curi.getCredentials().isEmpty()); assertTrue(curi.getFetchDuration() >= 0); - assertEquals(200, curi.getFetchStatus()); + assertEquals(FetchStatusCodes.S_OUT_OF_SCOPE, curi.getFetchStatus()); + assertFalse(curi.isSuccess()); assertSame(curi.getFetchType(), FetchType.HTTP_GET); assertEquals(1, curi.getAnnotations().size()); @@ -674,6 +677,44 @@ public class FetchHTTPTest { assertEquals("", entityString(curi)); } + @Test + public void testShouldFetchBodyRuleAccept() throws Exception { + CrawlURI curi = makeCrawlURI("http://localhost:7777/200k"); + fetcher().process(curi); + + assertEquals(200, curi.getFetchStatus()); + assertTrue(curi.isSuccess()); + assertFalse(messageBodyString(curi).isEmpty()); + assertFalse(curi.getAnnotations().contains("midFetchAbort")); + } + + @Test + public void testShouldFetchBodyRuleNone() throws Exception { + CrawlURI curi = makeCrawlURI("http://localhost:7777/200k"); + RejectDecideRule rule = new RejectDecideRule(); + rule.setEnabled(false); + fetcher().setShouldFetchBodyRule(rule); + fetcher().process(curi); + + assertEquals(200, curi.getFetchStatus()); + assertTrue(curi.isSuccess()); + assertFalse(messageBodyString(curi).isEmpty()); + assertFalse(curi.getAnnotations().contains("midFetchAbort")); + } + + @Test + public void testShouldFetchBodyRulePrerequisite() throws Exception { + CrawlURI curi = makeCrawlURI("http://localhost:7777/200k"); + curi.setPrerequisite(true); + fetcher().setShouldFetchBodyRule(new RejectDecideRule()); + fetcher().process(curi); + + assertEquals(200, curi.getFetchStatus()); + assertTrue(curi.isSuccess()); + assertFalse(messageBodyString(curi).isEmpty()); + assertFalse(curi.getAnnotations().contains("midFetchAbort")); + } + @Test public void testFetchTimeout() throws Exception { CrawlURI curi = makeCrawlURI("http://localhost:7777/slow.txt"); @@ -682,6 +723,8 @@ public class FetchHTTPTest { // logger.info('\n' + httpRequestString(curi) + "\n\n" + rawResponseString(curi)); assertTrue(curi.getAnnotations().contains("timeTrunc")); + assertEquals(200, curi.getFetchStatus()); + assertFalse(curi.getAnnotations().contains("midFetchAbort")); assertTrue(curi.getFetchDuration() >= 2000 && curi.getFetchDuration() < 2200); } From a769651bf5528b5aea5b60c888e8af9d13cfabec Mon Sep 17 00:00:00 2001 From: Matt Van Horn <455140+mvanhorn@users.noreply.github.com> Date: Mon, 2 Nov 2026 06:01:06 -0800 Subject: [PATCH 2/2] fix: give midfetch aborts their own fetch status code Add S_MIDFETCH_ABORTED (-5005) and set it in FetchHTTP when shouldFetchBodyRule rejects a response, in place of S_OUT_OF_SCOPE. The code means something distinct from a scope rejection, so crawl logs and reports can now tell the two apart. AbstractFrontier.isDisregarded() treats it as disregarded, CrawlURI gives it a readable string and the glossary lists it. --- docs/glossary.rst | 2 ++ .../archive/crawler/frontier/AbstractFrontier.java | 2 ++ .../org/archive/crawler/datamodel/CrawlURITest.java | 7 +++++++ .../archive/crawler/frontier/BdbFrontierTest.java | 13 +++++++++++++ .../src/main/java/org/archive/modules/CrawlURI.java | 2 ++ .../java/org/archive/modules/fetcher/FetchHTTP.java | 6 +++--- .../archive/modules/fetcher/FetchStatusCodes.java | 5 +++++ .../org/archive/modules/fetcher/FetchHTTPTest.java | 2 +- 8 files changed, 35 insertions(+), 4 deletions(-) diff --git a/docs/glossary.rst b/docs/glossary.rst index 2205e94e..c8464c68 100644 --- a/docs/glossary.rst +++ b/docs/glossary.rst @@ -347,6 +347,8 @@ Other Heritrix status codes are listed below. - Blocked due to exceeding an established quota. * - -5004 - Blocked due to exceeding an established runtime + * - -5005 + - The fetch was aborted after receiving the response headers because the response was rejected by ``shouldFetchBodyRule``. * - -6000 - Deleted from Frontier by user. * - -7000 diff --git a/engine/src/main/java/org/archive/crawler/frontier/AbstractFrontier.java b/engine/src/main/java/org/archive/crawler/frontier/AbstractFrontier.java index 42ccfa34..bdee3d7f 100644 --- a/engine/src/main/java/org/archive/crawler/frontier/AbstractFrontier.java +++ b/engine/src/main/java/org/archive/crawler/frontier/AbstractFrontier.java @@ -27,6 +27,7 @@ import static org.archive.modules.fetcher.FetchStatusCodes.S_CONNECT_LOST; import static org.archive.modules.fetcher.FetchStatusCodes.S_DEFERRED; import static org.archive.modules.fetcher.FetchStatusCodes.S_DELETED_BY_USER; import static org.archive.modules.fetcher.FetchStatusCodes.S_DOMAIN_UNRESOLVABLE; +import static org.archive.modules.fetcher.FetchStatusCodes.S_MIDFETCH_ABORTED; import static org.archive.modules.fetcher.FetchStatusCodes.S_OUT_OF_SCOPE; import static org.archive.modules.fetcher.FetchStatusCodes.S_ROBOTS_PRECLUDED; import static org.archive.modules.fetcher.FetchStatusCodes.S_TOO_MANY_EMBED_HOPS; @@ -1072,6 +1073,7 @@ public abstract class AbstractFrontier case S_BLOCKED_BY_CUSTOM_PROCESSOR: case S_OUT_OF_SCOPE: // filtered out by scope case S_BLOCKED_BY_USER: // filtered out by user + case S_MIDFETCH_ABORTED: // rejected by shouldFetchBodyRule case S_TOO_MANY_EMBED_HOPS: // too far from last true link case S_TOO_MANY_LINK_HOPS: // too far from seeds case S_DELETED_BY_USER: // user deleted diff --git a/engine/src/test/java/org/archive/crawler/datamodel/CrawlURITest.java b/engine/src/test/java/org/archive/crawler/datamodel/CrawlURITest.java index 9dbf1f7d..5f2fba68 100644 --- a/engine/src/test/java/org/archive/crawler/datamodel/CrawlURITest.java +++ b/engine/src/test/java/org/archive/crawler/datamodel/CrawlURITest.java @@ -29,6 +29,7 @@ import java.nio.file.Path; import org.archive.url.URIException; import org.archive.modules.CrawlURI; import org.archive.modules.SchedulingConstants; +import org.archive.modules.fetcher.FetchStatusCodes; import org.archive.modules.extractor.LinkContext.SimpleLinkContext; import org.archive.net.UURI; import org.archive.net.UURIFactory; @@ -108,6 +109,12 @@ public class CrawlURITest { assertEquals("value", curi.getData().get("key"), "Didn't find AList item"); } + @Test + public void testMidfetchAbortedStatusString() { + assertEquals("Heritrix(-5005)-Midfetch aborted", + CrawlURI.fetchStatusCodesToString(FetchStatusCodes.S_MIDFETCH_ABORTED)); + } + @Test public void testExtendHopsPath() { assertEquals("L",CrawlURI.extendHopsPath("",'L'), "from empty"); diff --git a/engine/src/test/java/org/archive/crawler/frontier/BdbFrontierTest.java b/engine/src/test/java/org/archive/crawler/frontier/BdbFrontierTest.java index 8025f590..e049efe2 100644 --- a/engine/src/test/java/org/archive/crawler/frontier/BdbFrontierTest.java +++ b/engine/src/test/java/org/archive/crawler/frontier/BdbFrontierTest.java @@ -19,7 +19,13 @@ package org.archive.crawler.frontier; +import static org.junit.jupiter.api.Assertions.assertTrue; + import org.archive.crawler.framework.CrawlerProcessorTestBase; +import org.archive.modules.CrawlURI; +import org.archive.modules.fetcher.FetchStatusCodes; +import org.archive.net.UURIFactory; +import org.junit.jupiter.api.Test; /** @@ -30,5 +36,12 @@ public class BdbFrontierTest extends CrawlerProcessorTestBase { // TODO TESTME + @Test + public void testMidfetchAbortIsDisregarded() throws Exception { + CrawlURI curi = new CrawlURI(UURIFactory.getInstance("http://example.com/")); + curi.setFetchStatus(FetchStatusCodes.S_MIDFETCH_ABORTED); + assertTrue(new BdbFrontier().isDisregarded(curi)); + } + } diff --git a/modules/src/main/java/org/archive/modules/CrawlURI.java b/modules/src/main/java/org/archive/modules/CrawlURI.java index 7ea725aa..a43589d2 100644 --- a/modules/src/main/java/org/archive/modules/CrawlURI.java +++ b/modules/src/main/java/org/archive/modules/CrawlURI.java @@ -360,6 +360,8 @@ implements Reporter, Serializable, OverlayContext, Comparable { + ")-Domain unresolvable"; case S_OUT_OF_SCOPE: return "Heritrix(" + S_OUT_OF_SCOPE + ")-Out of scope"; + case S_MIDFETCH_ABORTED: + return "Heritrix(" + S_MIDFETCH_ABORTED + ")-Midfetch aborted"; case S_DOMAIN_PREREQUISITE_FAILURE: return "Heritrix(" + S_DOMAIN_PREREQUISITE_FAILURE + ")-Domain prerequisite failure"; diff --git a/modules/src/main/java/org/archive/modules/fetcher/FetchHTTP.java b/modules/src/main/java/org/archive/modules/fetcher/FetchHTTP.java index 645db25c..a07c06ee 100644 --- a/modules/src/main/java/org/archive/modules/fetcher/FetchHTTP.java +++ b/modules/src/main/java/org/archive/modules/fetcher/FetchHTTP.java @@ -24,7 +24,7 @@ import static org.archive.modules.fetcher.FetchErrors.TIMER_TRUNC; import static org.archive.modules.fetcher.FetchStatusCodes.S_CONNECT_FAILED; import static org.archive.modules.fetcher.FetchStatusCodes.S_CONNECT_LOST; import static org.archive.modules.fetcher.FetchStatusCodes.S_DOMAIN_PREREQUISITE_FAILURE; -import static org.archive.modules.fetcher.FetchStatusCodes.S_OUT_OF_SCOPE; +import static org.archive.modules.fetcher.FetchStatusCodes.S_MIDFETCH_ABORTED; import static org.archive.modules.fetcher.FetchStatusCodes.S_UNFETCHABLE_URI; import static org.archive.modules.fetcher.FetchStatusCodes.S_SSL_ERROR; import static org.archive.modules.recrawl.RecrawlAttributeConstants.A_REFERENCE_LENGTH; @@ -467,7 +467,7 @@ public class FetchHTTP extends Processor implements Lifecycle { /** * DecideRules applied after receipt of HTTP response headers but before we * start to download the body. If the rule returns REJECT, the fetch is - * aborted and the URI is marked {@link FetchStatusCodes#S_OUT_OF_SCOPE}. + * aborted and the URI is marked {@link FetchStatusCodes#S_MIDFETCH_ABORTED}. * This makes {@link CrawlURI#isSuccess()} false, skips extraction and WARC * writing (including header-only captures), and causes the frontier to * disregard the URI without a queue-budget charge or retry. Prerequisites @@ -645,7 +645,7 @@ public class FetchHTTP extends Processor implements Lifecycle { protected boolean maybeMidfetchAbort(CrawlURI curi, AbstractExecutionAwareRequest request) { if (checkMidfetchAbort(curi)) { doAbort(curi, request, "midFetchAbort"); - curi.setFetchStatus(S_OUT_OF_SCOPE); + curi.setFetchStatus(S_MIDFETCH_ABORTED); curi.getRecorder().getRecordedInput().chopAtMessageBodyBegin(); return true; } else { diff --git a/modules/src/main/java/org/archive/modules/fetcher/FetchStatusCodes.java b/modules/src/main/java/org/archive/modules/fetcher/FetchStatusCodes.java index 20c56038..bb3ede22 100644 --- a/modules/src/main/java/org/archive/modules/fetcher/FetchStatusCodes.java +++ b/modules/src/main/java/org/archive/modules/fetcher/FetchStatusCodes.java @@ -102,6 +102,11 @@ public interface FetchStatusCodes { * TODO: Add to documentation and help page. */ public static final int S_BLOCKED_BY_RUNTIME_LIMIT = -5004; + /** + * Fetch aborted after the response headers were received, because the + * response was rejected by FetchHTTP's shouldFetchBodyRule. + */ + public static final int S_MIDFETCH_ABORTED = -5005; /** deleted from frontier by user */ public static final int S_DELETED_BY_USER = -6000; diff --git a/modules/src/test/java/org/archive/modules/fetcher/FetchHTTPTest.java b/modules/src/test/java/org/archive/modules/fetcher/FetchHTTPTest.java index b485bbe1..a1504ef2 100644 --- a/modules/src/test/java/org/archive/modules/fetcher/FetchHTTPTest.java +++ b/modules/src/test/java/org/archive/modules/fetcher/FetchHTTPTest.java @@ -663,7 +663,7 @@ public class FetchHTTPTest { assertEquals("text/plain;charset=US-ASCII", curi.getContentType()); assertTrue(curi.getCredentials().isEmpty()); assertTrue(curi.getFetchDuration() >= 0); - assertEquals(FetchStatusCodes.S_OUT_OF_SCOPE, curi.getFetchStatus()); + assertEquals(FetchStatusCodes.S_MIDFETCH_ABORTED, curi.getFetchStatus()); assertFalse(curi.isSuccess()); assertSame(curi.getFetchType(), FetchType.HTTP_GET);