fix: give midfetch aborts their own fetch status code

Add S_MIDFETCH_ABORTED (-5005) and set it in FetchHTTP when
shouldFetchBodyRule rejects a response, in place of S_OUT_OF_SCOPE.
The code means something distinct from a scope rejection, so crawl logs
and reports can now tell the two apart. AbstractFrontier.isDisregarded()
treats it as disregarded, CrawlURI gives it a readable string and the
glossary lists it.
This commit is contained in:
Matt Van Horn committed 2026-11-02 06:01:06 -08:00
1 parent 1c27e2e7b0
commit a769651bf5
8 files changed
+35 -4

No files matched your search

+2
View File
@@ -347,6 +347,8 @@ Other Heritrix status codes are listed below.
- Blocked due to exceeding an established quota.
* - -5004
- Blocked due to exceeding an established runtime
* - -5005
- The fetch was aborted after receiving the response headers because the response was rejected by ``shouldFetchBodyRule``.
* - -6000
- Deleted from Frontier by user.
* - -7000
@@ -27,6 +27,7 @@ import static org.archive.modules.fetcher.FetchStatusCodes.S_CONNECT_LOST;
import static org.archive.modules.fetcher.FetchStatusCodes.S_DEFERRED;
import static org.archive.modules.fetcher.FetchStatusCodes.S_DELETED_BY_USER;
import static org.archive.modules.fetcher.FetchStatusCodes.S_DOMAIN_UNRESOLVABLE;
import static org.archive.modules.fetcher.FetchStatusCodes.S_MIDFETCH_ABORTED;
import static org.archive.modules.fetcher.FetchStatusCodes.S_OUT_OF_SCOPE;
import static org.archive.modules.fetcher.FetchStatusCodes.S_ROBOTS_PRECLUDED;
import static org.archive.modules.fetcher.FetchStatusCodes.S_TOO_MANY_EMBED_HOPS;
@@ -1072,6 +1073,7 @@ public abstract class AbstractFrontier
case S_BLOCKED_BY_CUSTOM_PROCESSOR:
case S_OUT_OF_SCOPE: // filtered out by scope
case S_BLOCKED_BY_USER: // filtered out by user
case S_MIDFETCH_ABORTED: // rejected by shouldFetchBodyRule
case S_TOO_MANY_EMBED_HOPS: // too far from last true link
case S_TOO_MANY_LINK_HOPS: // too far from seeds
case S_DELETED_BY_USER: // user deleted
@@ -29,6 +29,7 @@ import java.nio.file.Path;
import org.archive.url.URIException;
import org.archive.modules.CrawlURI;
import org.archive.modules.SchedulingConstants;
import org.archive.modules.fetcher.FetchStatusCodes;
import org.archive.modules.extractor.LinkContext.SimpleLinkContext;
import org.archive.net.UURI;
import org.archive.net.UURIFactory;
@@ -108,6 +109,12 @@ public class CrawlURITest {
assertEquals("value", curi.getData().get("key"), "Didn't find AList item");
}
@Test
public void testMidfetchAbortedStatusString() {
assertEquals("Heritrix(-5005)-Midfetch aborted",
CrawlURI.fetchStatusCodesToString(FetchStatusCodes.S_MIDFETCH_ABORTED));
}
@Test
public void testExtendHopsPath() {
assertEquals("L",CrawlURI.extendHopsPath("",'L'), "from empty");
@@ -19,7 +19,13 @@
package org.archive.crawler.frontier;
import static org.junit.jupiter.api.Assertions.assertTrue;
import org.archive.crawler.framework.CrawlerProcessorTestBase;
import org.archive.modules.CrawlURI;
import org.archive.modules.fetcher.FetchStatusCodes;
import org.archive.net.UURIFactory;
import org.junit.jupiter.api.Test;
/**
@@ -30,5 +36,12 @@ public class BdbFrontierTest extends CrawlerProcessorTestBase {
// TODO TESTME
@Test
public void testMidfetchAbortIsDisregarded() throws Exception {
CrawlURI curi = new CrawlURI(UURIFactory.getInstance("http://example.com/"));
curi.setFetchStatus(FetchStatusCodes.S_MIDFETCH_ABORTED);
assertTrue(new BdbFrontier().isDisregarded(curi));
}
}
@@ -360,6 +360,8 @@ implements Reporter, Serializable, OverlayContext, Comparable<CrawlURI> {
+ ")-Domain unresolvable";
case S_OUT_OF_SCOPE:
return "Heritrix(" + S_OUT_OF_SCOPE + ")-Out of scope";
case S_MIDFETCH_ABORTED:
return "Heritrix(" + S_MIDFETCH_ABORTED + ")-Midfetch aborted";
case S_DOMAIN_PREREQUISITE_FAILURE:
return "Heritrix(" + S_DOMAIN_PREREQUISITE_FAILURE
+ ")-Domain prerequisite failure";
@@ -24,7 +24,7 @@ import static org.archive.modules.fetcher.FetchErrors.TIMER_TRUNC;
import static org.archive.modules.fetcher.FetchStatusCodes.S_CONNECT_FAILED;
import static org.archive.modules.fetcher.FetchStatusCodes.S_CONNECT_LOST;
import static org.archive.modules.fetcher.FetchStatusCodes.S_DOMAIN_PREREQUISITE_FAILURE;
import static org.archive.modules.fetcher.FetchStatusCodes.S_OUT_OF_SCOPE;
import static org.archive.modules.fetcher.FetchStatusCodes.S_MIDFETCH_ABORTED;
import static org.archive.modules.fetcher.FetchStatusCodes.S_UNFETCHABLE_URI;
import static org.archive.modules.fetcher.FetchStatusCodes.S_SSL_ERROR;
import static org.archive.modules.recrawl.RecrawlAttributeConstants.A_REFERENCE_LENGTH;
@@ -467,7 +467,7 @@ public class FetchHTTP extends Processor implements Lifecycle {
/**
* DecideRules applied after receipt of HTTP response headers but before we
* start to download the body. If the rule returns REJECT, the fetch is
* aborted and the URI is marked {@link FetchStatusCodes#S_OUT_OF_SCOPE}.
* aborted and the URI is marked {@link FetchStatusCodes#S_MIDFETCH_ABORTED}.
* This makes {@link CrawlURI#isSuccess()} false, skips extraction and WARC
* writing (including header-only captures), and causes the frontier to
* disregard the URI without a queue-budget charge or retry. Prerequisites
@@ -645,7 +645,7 @@ public class FetchHTTP extends Processor implements Lifecycle {
protected boolean maybeMidfetchAbort(CrawlURI curi, AbstractExecutionAwareRequest request) {
if (checkMidfetchAbort(curi)) {
doAbort(curi, request, "midFetchAbort");
curi.setFetchStatus(S_OUT_OF_SCOPE);
curi.setFetchStatus(S_MIDFETCH_ABORTED);
curi.getRecorder().getRecordedInput().chopAtMessageBodyBegin();
return true;
} else {
@@ -102,6 +102,11 @@ public interface FetchStatusCodes {
* TODO: Add to documentation and help page.
*/
public static final int S_BLOCKED_BY_RUNTIME_LIMIT = -5004;
/**
* Fetch aborted after the response headers were received, because the
* response was rejected by FetchHTTP's shouldFetchBodyRule.
*/
public static final int S_MIDFETCH_ABORTED = -5005;
/** deleted from frontier by user */
public static final int S_DELETED_BY_USER = -6000;
@@ -663,7 +663,7 @@ public class FetchHTTPTest {
assertEquals("text/plain;charset=US-ASCII", curi.getContentType());
assertTrue(curi.getCredentials().isEmpty());
assertTrue(curi.getFetchDuration() >= 0);
assertEquals(FetchStatusCodes.S_OUT_OF_SCOPE, curi.getFetchStatus());
assertEquals(FetchStatusCodes.S_MIDFETCH_ABORTED, curi.getFetchStatus());
assertFalse(curi.isSuccess());
assertSame(curi.getFetchType(), FetchType.HTTP_GET);