diff --git a/docs/glossary.rst b/docs/glossary.rst index 2205e94e..c8464c68 100644 --- a/docs/glossary.rst +++ b/docs/glossary.rst @@ -347,6 +347,8 @@ Other Heritrix status codes are listed below. - Blocked due to exceeding an established quota. * - -5004 - Blocked due to exceeding an established runtime + * - -5005 + - The fetch was aborted after receiving the response headers because the response was rejected by ``shouldFetchBodyRule``. * - -6000 - Deleted from Frontier by user. * - -7000 diff --git a/engine/src/main/java/org/archive/crawler/frontier/AbstractFrontier.java b/engine/src/main/java/org/archive/crawler/frontier/AbstractFrontier.java index 42ccfa34..bdee3d7f 100644 --- a/engine/src/main/java/org/archive/crawler/frontier/AbstractFrontier.java +++ b/engine/src/main/java/org/archive/crawler/frontier/AbstractFrontier.java @@ -27,6 +27,7 @@ import static org.archive.modules.fetcher.FetchStatusCodes.S_CONNECT_LOST; import static org.archive.modules.fetcher.FetchStatusCodes.S_DEFERRED; import static org.archive.modules.fetcher.FetchStatusCodes.S_DELETED_BY_USER; import static org.archive.modules.fetcher.FetchStatusCodes.S_DOMAIN_UNRESOLVABLE; +import static org.archive.modules.fetcher.FetchStatusCodes.S_MIDFETCH_ABORTED; import static org.archive.modules.fetcher.FetchStatusCodes.S_OUT_OF_SCOPE; import static org.archive.modules.fetcher.FetchStatusCodes.S_ROBOTS_PRECLUDED; import static org.archive.modules.fetcher.FetchStatusCodes.S_TOO_MANY_EMBED_HOPS; @@ -1072,6 +1073,7 @@ public abstract class AbstractFrontier case S_BLOCKED_BY_CUSTOM_PROCESSOR: case S_OUT_OF_SCOPE: // filtered out by scope case S_BLOCKED_BY_USER: // filtered out by user + case S_MIDFETCH_ABORTED: // rejected by shouldFetchBodyRule case S_TOO_MANY_EMBED_HOPS: // too far from last true link case S_TOO_MANY_LINK_HOPS: // too far from seeds case S_DELETED_BY_USER: // user deleted diff --git a/engine/src/test/java/org/archive/crawler/datamodel/CrawlURITest.java b/engine/src/test/java/org/archive/crawler/datamodel/CrawlURITest.java index 9dbf1f7d..5f2fba68 100644 --- a/engine/src/test/java/org/archive/crawler/datamodel/CrawlURITest.java +++ b/engine/src/test/java/org/archive/crawler/datamodel/CrawlURITest.java @@ -29,6 +29,7 @@ import java.nio.file.Path; import org.archive.url.URIException; import org.archive.modules.CrawlURI; import org.archive.modules.SchedulingConstants; +import org.archive.modules.fetcher.FetchStatusCodes; import org.archive.modules.extractor.LinkContext.SimpleLinkContext; import org.archive.net.UURI; import org.archive.net.UURIFactory; @@ -108,6 +109,12 @@ public class CrawlURITest { assertEquals("value", curi.getData().get("key"), "Didn't find AList item"); } + @Test + public void testMidfetchAbortedStatusString() { + assertEquals("Heritrix(-5005)-Midfetch aborted", + CrawlURI.fetchStatusCodesToString(FetchStatusCodes.S_MIDFETCH_ABORTED)); + } + @Test public void testExtendHopsPath() { assertEquals("L",CrawlURI.extendHopsPath("",'L'), "from empty"); diff --git a/engine/src/test/java/org/archive/crawler/frontier/BdbFrontierTest.java b/engine/src/test/java/org/archive/crawler/frontier/BdbFrontierTest.java index 8025f590..e049efe2 100644 --- a/engine/src/test/java/org/archive/crawler/frontier/BdbFrontierTest.java +++ b/engine/src/test/java/org/archive/crawler/frontier/BdbFrontierTest.java @@ -19,7 +19,13 @@ package org.archive.crawler.frontier; +import static org.junit.jupiter.api.Assertions.assertTrue; + import org.archive.crawler.framework.CrawlerProcessorTestBase; +import org.archive.modules.CrawlURI; +import org.archive.modules.fetcher.FetchStatusCodes; +import org.archive.net.UURIFactory; +import org.junit.jupiter.api.Test; /** @@ -30,5 +36,12 @@ public class BdbFrontierTest extends CrawlerProcessorTestBase { // TODO TESTME + @Test + public void testMidfetchAbortIsDisregarded() throws Exception { + CrawlURI curi = new CrawlURI(UURIFactory.getInstance("http://example.com/")); + curi.setFetchStatus(FetchStatusCodes.S_MIDFETCH_ABORTED); + assertTrue(new BdbFrontier().isDisregarded(curi)); + } + } diff --git a/modules/src/main/java/org/archive/modules/CrawlURI.java b/modules/src/main/java/org/archive/modules/CrawlURI.java index 7ea725aa..a43589d2 100644 --- a/modules/src/main/java/org/archive/modules/CrawlURI.java +++ b/modules/src/main/java/org/archive/modules/CrawlURI.java @@ -360,6 +360,8 @@ implements Reporter, Serializable, OverlayContext, Comparable { + ")-Domain unresolvable"; case S_OUT_OF_SCOPE: return "Heritrix(" + S_OUT_OF_SCOPE + ")-Out of scope"; + case S_MIDFETCH_ABORTED: + return "Heritrix(" + S_MIDFETCH_ABORTED + ")-Midfetch aborted"; case S_DOMAIN_PREREQUISITE_FAILURE: return "Heritrix(" + S_DOMAIN_PREREQUISITE_FAILURE + ")-Domain prerequisite failure"; diff --git a/modules/src/main/java/org/archive/modules/fetcher/FetchHTTP.java b/modules/src/main/java/org/archive/modules/fetcher/FetchHTTP.java index 645db25c..a07c06ee 100644 --- a/modules/src/main/java/org/archive/modules/fetcher/FetchHTTP.java +++ b/modules/src/main/java/org/archive/modules/fetcher/FetchHTTP.java @@ -24,7 +24,7 @@ import static org.archive.modules.fetcher.FetchErrors.TIMER_TRUNC; import static org.archive.modules.fetcher.FetchStatusCodes.S_CONNECT_FAILED; import static org.archive.modules.fetcher.FetchStatusCodes.S_CONNECT_LOST; import static org.archive.modules.fetcher.FetchStatusCodes.S_DOMAIN_PREREQUISITE_FAILURE; -import static org.archive.modules.fetcher.FetchStatusCodes.S_OUT_OF_SCOPE; +import static org.archive.modules.fetcher.FetchStatusCodes.S_MIDFETCH_ABORTED; import static org.archive.modules.fetcher.FetchStatusCodes.S_UNFETCHABLE_URI; import static org.archive.modules.fetcher.FetchStatusCodes.S_SSL_ERROR; import static org.archive.modules.recrawl.RecrawlAttributeConstants.A_REFERENCE_LENGTH; @@ -467,7 +467,7 @@ public class FetchHTTP extends Processor implements Lifecycle { /** * DecideRules applied after receipt of HTTP response headers but before we * start to download the body. If the rule returns REJECT, the fetch is - * aborted and the URI is marked {@link FetchStatusCodes#S_OUT_OF_SCOPE}. + * aborted and the URI is marked {@link FetchStatusCodes#S_MIDFETCH_ABORTED}. * This makes {@link CrawlURI#isSuccess()} false, skips extraction and WARC * writing (including header-only captures), and causes the frontier to * disregard the URI without a queue-budget charge or retry. Prerequisites @@ -645,7 +645,7 @@ public class FetchHTTP extends Processor implements Lifecycle { protected boolean maybeMidfetchAbort(CrawlURI curi, AbstractExecutionAwareRequest request) { if (checkMidfetchAbort(curi)) { doAbort(curi, request, "midFetchAbort"); - curi.setFetchStatus(S_OUT_OF_SCOPE); + curi.setFetchStatus(S_MIDFETCH_ABORTED); curi.getRecorder().getRecordedInput().chopAtMessageBodyBegin(); return true; } else { diff --git a/modules/src/main/java/org/archive/modules/fetcher/FetchStatusCodes.java b/modules/src/main/java/org/archive/modules/fetcher/FetchStatusCodes.java index 20c56038..bb3ede22 100644 --- a/modules/src/main/java/org/archive/modules/fetcher/FetchStatusCodes.java +++ b/modules/src/main/java/org/archive/modules/fetcher/FetchStatusCodes.java @@ -102,6 +102,11 @@ public interface FetchStatusCodes { * TODO: Add to documentation and help page. */ public static final int S_BLOCKED_BY_RUNTIME_LIMIT = -5004; + /** + * Fetch aborted after the response headers were received, because the + * response was rejected by FetchHTTP's shouldFetchBodyRule. + */ + public static final int S_MIDFETCH_ABORTED = -5005; /** deleted from frontier by user */ public static final int S_DELETED_BY_USER = -6000; diff --git a/modules/src/test/java/org/archive/modules/fetcher/FetchHTTPTest.java b/modules/src/test/java/org/archive/modules/fetcher/FetchHTTPTest.java index b485bbe1..a1504ef2 100644 --- a/modules/src/test/java/org/archive/modules/fetcher/FetchHTTPTest.java +++ b/modules/src/test/java/org/archive/modules/fetcher/FetchHTTPTest.java @@ -663,7 +663,7 @@ public class FetchHTTPTest { assertEquals("text/plain;charset=US-ASCII", curi.getContentType()); assertTrue(curi.getCredentials().isEmpty()); assertTrue(curi.getFetchDuration() >= 0); - assertEquals(FetchStatusCodes.S_OUT_OF_SCOPE, curi.getFetchStatus()); + assertEquals(FetchStatusCodes.S_MIDFETCH_ABORTED, curi.getFetchStatus()); assertFalse(curi.isSuccess()); assertSame(curi.getFetchType(), FetchType.HTTP_GET);