diff --git a/modules/src/main/java/org/archive/modules/fetcher/FetchHTTP.java b/modules/src/main/java/org/archive/modules/fetcher/FetchHTTP.java index 52c90d08..645db25c 100644 --- a/modules/src/main/java/org/archive/modules/fetcher/FetchHTTP.java +++ b/modules/src/main/java/org/archive/modules/fetcher/FetchHTTP.java @@ -24,6 +24,7 @@ import static org.archive.modules.fetcher.FetchErrors.TIMER_TRUNC; import static org.archive.modules.fetcher.FetchStatusCodes.S_CONNECT_FAILED; import static org.archive.modules.fetcher.FetchStatusCodes.S_CONNECT_LOST; import static org.archive.modules.fetcher.FetchStatusCodes.S_DOMAIN_PREREQUISITE_FAILURE; +import static org.archive.modules.fetcher.FetchStatusCodes.S_OUT_OF_SCOPE; import static org.archive.modules.fetcher.FetchStatusCodes.S_UNFETCHABLE_URI; import static org.archive.modules.fetcher.FetchStatusCodes.S_SSL_ERROR; import static org.archive.modules.recrawl.RecrawlAttributeConstants.A_REFERENCE_LENGTH; @@ -465,9 +466,12 @@ public class FetchHTTP extends Processor implements Lifecycle { } /** * DecideRules applied after receipt of HTTP response headers but before we - * start to download the body. If any filter returns FALSE, the fetch is - * aborted. Prerequisites such as robots.txt by-pass filtering (i.e. they - * cannot be midfetch aborted. + * start to download the body. If the rule returns REJECT, the fetch is + * aborted and the URI is marked {@link FetchStatusCodes#S_OUT_OF_SCOPE}. + * This makes {@link CrawlURI#isSuccess()} false, skips extraction and WARC + * writing (including header-only captures), and causes the frontier to + * disregard the URI without a queue-budget charge or retry. Prerequisites + * such as robots.txt bypass filtering and cannot be midfetch aborted. */ public void setShouldFetchBodyRule(DecideRule rule) { kp.put("shouldFetchBodyRule", rule); @@ -641,6 +645,7 @@ public class FetchHTTP extends Processor implements Lifecycle { protected boolean maybeMidfetchAbort(CrawlURI curi, AbstractExecutionAwareRequest request) { if (checkMidfetchAbort(curi)) { doAbort(curi, request, "midFetchAbort"); + curi.setFetchStatus(S_OUT_OF_SCOPE); curi.getRecorder().getRecordedInput().chopAtMessageBodyBegin(); return true; } else { diff --git a/modules/src/test/java/org/archive/modules/fetcher/FetchHTTPTest.java b/modules/src/test/java/org/archive/modules/fetcher/FetchHTTPTest.java index 86a96c57..b485bbe1 100644 --- a/modules/src/test/java/org/archive/modules/fetcher/FetchHTTPTest.java +++ b/modules/src/test/java/org/archive/modules/fetcher/FetchHTTPTest.java @@ -578,6 +578,8 @@ public class FetchHTTPTest { fetcher().setMaxLengthBytes(50000); fetcher().process(curi); assertEquals(50001, curi.getRecordedSize()); + assertEquals(200, curi.getFetchStatus()); + assertFalse(curi.getAnnotations().contains("midFetchAbort")); } @Test @@ -661,7 +663,8 @@ public class FetchHTTPTest { assertEquals("text/plain;charset=US-ASCII", curi.getContentType()); assertTrue(curi.getCredentials().isEmpty()); assertTrue(curi.getFetchDuration() >= 0); - assertEquals(200, curi.getFetchStatus()); + assertEquals(FetchStatusCodes.S_OUT_OF_SCOPE, curi.getFetchStatus()); + assertFalse(curi.isSuccess()); assertSame(curi.getFetchType(), FetchType.HTTP_GET); assertEquals(1, curi.getAnnotations().size()); @@ -674,6 +677,44 @@ public class FetchHTTPTest { assertEquals("", entityString(curi)); } + @Test + public void testShouldFetchBodyRuleAccept() throws Exception { + CrawlURI curi = makeCrawlURI("http://localhost:7777/200k"); + fetcher().process(curi); + + assertEquals(200, curi.getFetchStatus()); + assertTrue(curi.isSuccess()); + assertFalse(messageBodyString(curi).isEmpty()); + assertFalse(curi.getAnnotations().contains("midFetchAbort")); + } + + @Test + public void testShouldFetchBodyRuleNone() throws Exception { + CrawlURI curi = makeCrawlURI("http://localhost:7777/200k"); + RejectDecideRule rule = new RejectDecideRule(); + rule.setEnabled(false); + fetcher().setShouldFetchBodyRule(rule); + fetcher().process(curi); + + assertEquals(200, curi.getFetchStatus()); + assertTrue(curi.isSuccess()); + assertFalse(messageBodyString(curi).isEmpty()); + assertFalse(curi.getAnnotations().contains("midFetchAbort")); + } + + @Test + public void testShouldFetchBodyRulePrerequisite() throws Exception { + CrawlURI curi = makeCrawlURI("http://localhost:7777/200k"); + curi.setPrerequisite(true); + fetcher().setShouldFetchBodyRule(new RejectDecideRule()); + fetcher().process(curi); + + assertEquals(200, curi.getFetchStatus()); + assertTrue(curi.isSuccess()); + assertFalse(messageBodyString(curi).isEmpty()); + assertFalse(curi.getAnnotations().contains("midFetchAbort")); + } + @Test public void testFetchTimeout() throws Exception { CrawlURI curi = makeCrawlURI("http://localhost:7777/slow.txt"); @@ -682,6 +723,8 @@ public class FetchHTTPTest { // logger.info('\n' + httpRequestString(curi) + "\n\n" + rawResponseString(curi)); assertTrue(curi.getAnnotations().contains("timeTrunc")); + assertEquals(200, curi.getFetchStatus()); + assertFalse(curi.getAnnotations().contains("midFetchAbort")); assertTrue(curi.getFetchDuration() >= 2000 && curi.getFetchDuration() < 2200); }