Merge pull request #792 from mvanhorn/fix/455-midfetch-abort-statistics

fix: disregard HTTP responses rejected before body fetching
This commit is contained in:
Alex Osborne authored and GitHub committed 2026-11-02 23:38:13 +09:00
commit 0f46340d26
8 files changed
+83 -4

No files matched your search

+2
View File
@@ -347,6 +347,8 @@ Other Heritrix status codes are listed below.
- Blocked due to exceeding an established quota.
* - -5004
- Blocked due to exceeding an established runtime
* - -5005
- The fetch was aborted after receiving the response headers because the response was rejected by ``shouldFetchBodyRule``.
* - -6000
- Deleted from Frontier by user.
* - -7000
@@ -27,6 +27,7 @@ import static org.archive.modules.fetcher.FetchStatusCodes.S_CONNECT_LOST;
import static org.archive.modules.fetcher.FetchStatusCodes.S_DEFERRED;
import static org.archive.modules.fetcher.FetchStatusCodes.S_DELETED_BY_USER;
import static org.archive.modules.fetcher.FetchStatusCodes.S_DOMAIN_UNRESOLVABLE;
import static org.archive.modules.fetcher.FetchStatusCodes.S_MIDFETCH_ABORTED;
import static org.archive.modules.fetcher.FetchStatusCodes.S_OUT_OF_SCOPE;
import static org.archive.modules.fetcher.FetchStatusCodes.S_ROBOTS_PRECLUDED;
import static org.archive.modules.fetcher.FetchStatusCodes.S_TOO_MANY_EMBED_HOPS;
@@ -1072,6 +1073,7 @@ public abstract class AbstractFrontier
case S_BLOCKED_BY_CUSTOM_PROCESSOR:
case S_OUT_OF_SCOPE: // filtered out by scope
case S_BLOCKED_BY_USER: // filtered out by user
case S_MIDFETCH_ABORTED: // rejected by shouldFetchBodyRule
case S_TOO_MANY_EMBED_HOPS: // too far from last true link
case S_TOO_MANY_LINK_HOPS: // too far from seeds
case S_DELETED_BY_USER: // user deleted
@@ -29,6 +29,7 @@ import java.nio.file.Path;
import org.archive.url.URIException;
import org.archive.modules.CrawlURI;
import org.archive.modules.SchedulingConstants;
import org.archive.modules.fetcher.FetchStatusCodes;
import org.archive.modules.extractor.LinkContext.SimpleLinkContext;
import org.archive.net.UURI;
import org.archive.net.UURIFactory;
@@ -108,6 +109,12 @@ public class CrawlURITest {
assertEquals("value", curi.getData().get("key"), "Didn't find AList item");
}
@Test
public void testMidfetchAbortedStatusString() {
assertEquals("Heritrix(-5005)-Midfetch aborted",
CrawlURI.fetchStatusCodesToString(FetchStatusCodes.S_MIDFETCH_ABORTED));
}
@Test
public void testExtendHopsPath() {
assertEquals("L",CrawlURI.extendHopsPath("",'L'), "from empty");
@@ -19,7 +19,13 @@
package org.archive.crawler.frontier;
import static org.junit.jupiter.api.Assertions.assertTrue;
import org.archive.crawler.framework.CrawlerProcessorTestBase;
import org.archive.modules.CrawlURI;
import org.archive.modules.fetcher.FetchStatusCodes;
import org.archive.net.UURIFactory;
import org.junit.jupiter.api.Test;
/**
@@ -30,5 +36,12 @@ public class BdbFrontierTest extends CrawlerProcessorTestBase {
// TODO TESTME
@Test
public void testMidfetchAbortIsDisregarded() throws Exception {
CrawlURI curi = new CrawlURI(UURIFactory.getInstance("http://example.com/"));
curi.setFetchStatus(FetchStatusCodes.S_MIDFETCH_ABORTED);
assertTrue(new BdbFrontier().isDisregarded(curi));
}
}
@@ -360,6 +360,8 @@ implements Reporter, Serializable, OverlayContext, Comparable<CrawlURI> {
+ ")-Domain unresolvable";
case S_OUT_OF_SCOPE:
return "Heritrix(" + S_OUT_OF_SCOPE + ")-Out of scope";
case S_MIDFETCH_ABORTED:
return "Heritrix(" + S_MIDFETCH_ABORTED + ")-Midfetch aborted";
case S_DOMAIN_PREREQUISITE_FAILURE:
return "Heritrix(" + S_DOMAIN_PREREQUISITE_FAILURE
+ ")-Domain prerequisite failure";
@@ -24,6 +24,7 @@ import static org.archive.modules.fetcher.FetchErrors.TIMER_TRUNC;
import static org.archive.modules.fetcher.FetchStatusCodes.S_CONNECT_FAILED;
import static org.archive.modules.fetcher.FetchStatusCodes.S_CONNECT_LOST;
import static org.archive.modules.fetcher.FetchStatusCodes.S_DOMAIN_PREREQUISITE_FAILURE;
import static org.archive.modules.fetcher.FetchStatusCodes.S_MIDFETCH_ABORTED;
import static org.archive.modules.fetcher.FetchStatusCodes.S_UNFETCHABLE_URI;
import static org.archive.modules.fetcher.FetchStatusCodes.S_SSL_ERROR;
import static org.archive.modules.recrawl.RecrawlAttributeConstants.A_REFERENCE_LENGTH;
@@ -465,9 +466,12 @@ public class FetchHTTP extends Processor implements Lifecycle {
}
/**
* DecideRules applied after receipt of HTTP response headers but before we
* start to download the body. If any filter returns FALSE, the fetch is
* aborted. Prerequisites such as robots.txt by-pass filtering (i.e. they
* cannot be midfetch aborted.
* start to download the body. If the rule returns REJECT, the fetch is
* aborted and the URI is marked {@link FetchStatusCodes#S_MIDFETCH_ABORTED}.
* This makes {@link CrawlURI#isSuccess()} false, skips extraction and WARC
* writing (including header-only captures), and causes the frontier to
* disregard the URI without a queue-budget charge or retry. Prerequisites
* such as robots.txt bypass filtering and cannot be midfetch aborted.
*/
public void setShouldFetchBodyRule(DecideRule rule) {
kp.put("shouldFetchBodyRule", rule);
@@ -641,6 +645,7 @@ public class FetchHTTP extends Processor implements Lifecycle {
protected boolean maybeMidfetchAbort(CrawlURI curi, AbstractExecutionAwareRequest request) {
if (checkMidfetchAbort(curi)) {
doAbort(curi, request, "midFetchAbort");
curi.setFetchStatus(S_MIDFETCH_ABORTED);
curi.getRecorder().getRecordedInput().chopAtMessageBodyBegin();
return true;
} else {
@@ -102,6 +102,11 @@ public interface FetchStatusCodes {
* TODO: Add to documentation and help page.
*/
public static final int S_BLOCKED_BY_RUNTIME_LIMIT = -5004;
/**
* Fetch aborted after the response headers were received, because the
* response was rejected by FetchHTTP's shouldFetchBodyRule.
*/
public static final int S_MIDFETCH_ABORTED = -5005;
/** deleted from frontier by user */
public static final int S_DELETED_BY_USER = -6000;
@@ -578,6 +578,8 @@ public class FetchHTTPTest {
fetcher().setMaxLengthBytes(50000);
fetcher().process(curi);
assertEquals(50001, curi.getRecordedSize());
assertEquals(200, curi.getFetchStatus());
assertFalse(curi.getAnnotations().contains("midFetchAbort"));
}
@Test
@@ -661,7 +663,8 @@ public class FetchHTTPTest {
assertEquals("text/plain;charset=US-ASCII", curi.getContentType());
assertTrue(curi.getCredentials().isEmpty());
assertTrue(curi.getFetchDuration() >= 0);
assertEquals(200, curi.getFetchStatus());
assertEquals(FetchStatusCodes.S_MIDFETCH_ABORTED, curi.getFetchStatus());
assertFalse(curi.isSuccess());
assertSame(curi.getFetchType(), FetchType.HTTP_GET);
assertEquals(1, curi.getAnnotations().size());
@@ -674,6 +677,44 @@ public class FetchHTTPTest {
assertEquals("", entityString(curi));
}
@Test
public void testShouldFetchBodyRuleAccept() throws Exception {
CrawlURI curi = makeCrawlURI("http://localhost:7777/200k");
fetcher().process(curi);
assertEquals(200, curi.getFetchStatus());
assertTrue(curi.isSuccess());
assertFalse(messageBodyString(curi).isEmpty());
assertFalse(curi.getAnnotations().contains("midFetchAbort"));
}
@Test
public void testShouldFetchBodyRuleNone() throws Exception {
CrawlURI curi = makeCrawlURI("http://localhost:7777/200k");
RejectDecideRule rule = new RejectDecideRule();
rule.setEnabled(false);
fetcher().setShouldFetchBodyRule(rule);
fetcher().process(curi);
assertEquals(200, curi.getFetchStatus());
assertTrue(curi.isSuccess());
assertFalse(messageBodyString(curi).isEmpty());
assertFalse(curi.getAnnotations().contains("midFetchAbort"));
}
@Test
public void testShouldFetchBodyRulePrerequisite() throws Exception {
CrawlURI curi = makeCrawlURI("http://localhost:7777/200k");
curi.setPrerequisite(true);
fetcher().setShouldFetchBodyRule(new RejectDecideRule());
fetcher().process(curi);
assertEquals(200, curi.getFetchStatus());
assertTrue(curi.isSuccess());
assertFalse(messageBodyString(curi).isEmpty());
assertFalse(curi.getAnnotations().contains("midFetchAbort"));
}
@Test
public void testFetchTimeout() throws Exception {
CrawlURI curi = makeCrawlURI("http://localhost:7777/slow.txt");
@@ -682,6 +723,8 @@ public class FetchHTTPTest {
// logger.info('\n' + httpRequestString(curi) + "\n\n" + rawResponseString(curi));
assertTrue(curi.getAnnotations().contains("timeTrunc"));
assertEquals(200, curi.getFetchStatus());
assertFalse(curi.getAnnotations().contains("midFetchAbort"));
assertTrue(curi.getFetchDuration() >= 2000 && curi.getFetchDuration() < 2200);
}