mirror of
https://github.com/internetarchive/heritrix3.git
synced 2026-11-22 13:47:02 +00:00
fix: disregard HTTP responses rejected before body fetching
A rejection by shouldFetchBodyRule now sets S_OUT_OF_SCOPE, causing the frontier to disregard the URI without charging the queue budget or retrying it, and skipping extraction and WARC writing. Tests cover rejected responses and preserve successful handling for accepted or disabled rules, prerequisites, size limits, and timeouts. Fixes #455
This commit is contained in:
2 files changed
+52
-4
No files matched your search
@@ -24,6 +24,7 @@ import static org.archive.modules.fetcher.FetchErrors.TIMER_TRUNC;
|
||||
import static org.archive.modules.fetcher.FetchStatusCodes.S_CONNECT_FAILED;
|
||||
import static org.archive.modules.fetcher.FetchStatusCodes.S_CONNECT_LOST;
|
||||
import static org.archive.modules.fetcher.FetchStatusCodes.S_DOMAIN_PREREQUISITE_FAILURE;
|
||||
import static org.archive.modules.fetcher.FetchStatusCodes.S_OUT_OF_SCOPE;
|
||||
import static org.archive.modules.fetcher.FetchStatusCodes.S_UNFETCHABLE_URI;
|
||||
import static org.archive.modules.fetcher.FetchStatusCodes.S_SSL_ERROR;
|
||||
import static org.archive.modules.recrawl.RecrawlAttributeConstants.A_REFERENCE_LENGTH;
|
||||
@@ -465,9 +466,12 @@ public class FetchHTTP extends Processor implements Lifecycle {
|
||||
}
|
||||
/**
|
||||
* DecideRules applied after receipt of HTTP response headers but before we
|
||||
* start to download the body. If any filter returns FALSE, the fetch is
|
||||
* aborted. Prerequisites such as robots.txt by-pass filtering (i.e. they
|
||||
* cannot be midfetch aborted.
|
||||
* start to download the body. If the rule returns REJECT, the fetch is
|
||||
* aborted and the URI is marked {@link FetchStatusCodes#S_OUT_OF_SCOPE}.
|
||||
* This makes {@link CrawlURI#isSuccess()} false, skips extraction and WARC
|
||||
* writing (including header-only captures), and causes the frontier to
|
||||
* disregard the URI without a queue-budget charge or retry. Prerequisites
|
||||
* such as robots.txt bypass filtering and cannot be midfetch aborted.
|
||||
*/
|
||||
public void setShouldFetchBodyRule(DecideRule rule) {
|
||||
kp.put("shouldFetchBodyRule", rule);
|
||||
@@ -641,6 +645,7 @@ public class FetchHTTP extends Processor implements Lifecycle {
|
||||
protected boolean maybeMidfetchAbort(CrawlURI curi, AbstractExecutionAwareRequest request) {
|
||||
if (checkMidfetchAbort(curi)) {
|
||||
doAbort(curi, request, "midFetchAbort");
|
||||
curi.setFetchStatus(S_OUT_OF_SCOPE);
|
||||
curi.getRecorder().getRecordedInput().chopAtMessageBodyBegin();
|
||||
return true;
|
||||
} else {
|
||||
|
||||
@@ -578,6 +578,8 @@ public class FetchHTTPTest {
|
||||
fetcher().setMaxLengthBytes(50000);
|
||||
fetcher().process(curi);
|
||||
assertEquals(50001, curi.getRecordedSize());
|
||||
assertEquals(200, curi.getFetchStatus());
|
||||
assertFalse(curi.getAnnotations().contains("midFetchAbort"));
|
||||
}
|
||||
|
||||
@Test
|
||||
@@ -661,7 +663,8 @@ public class FetchHTTPTest {
|
||||
assertEquals("text/plain;charset=US-ASCII", curi.getContentType());
|
||||
assertTrue(curi.getCredentials().isEmpty());
|
||||
assertTrue(curi.getFetchDuration() >= 0);
|
||||
assertEquals(200, curi.getFetchStatus());
|
||||
assertEquals(FetchStatusCodes.S_OUT_OF_SCOPE, curi.getFetchStatus());
|
||||
assertFalse(curi.isSuccess());
|
||||
assertSame(curi.getFetchType(), FetchType.HTTP_GET);
|
||||
|
||||
assertEquals(1, curi.getAnnotations().size());
|
||||
@@ -674,6 +677,44 @@ public class FetchHTTPTest {
|
||||
assertEquals("", entityString(curi));
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testShouldFetchBodyRuleAccept() throws Exception {
|
||||
CrawlURI curi = makeCrawlURI("http://localhost:7777/200k");
|
||||
fetcher().process(curi);
|
||||
|
||||
assertEquals(200, curi.getFetchStatus());
|
||||
assertTrue(curi.isSuccess());
|
||||
assertFalse(messageBodyString(curi).isEmpty());
|
||||
assertFalse(curi.getAnnotations().contains("midFetchAbort"));
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testShouldFetchBodyRuleNone() throws Exception {
|
||||
CrawlURI curi = makeCrawlURI("http://localhost:7777/200k");
|
||||
RejectDecideRule rule = new RejectDecideRule();
|
||||
rule.setEnabled(false);
|
||||
fetcher().setShouldFetchBodyRule(rule);
|
||||
fetcher().process(curi);
|
||||
|
||||
assertEquals(200, curi.getFetchStatus());
|
||||
assertTrue(curi.isSuccess());
|
||||
assertFalse(messageBodyString(curi).isEmpty());
|
||||
assertFalse(curi.getAnnotations().contains("midFetchAbort"));
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testShouldFetchBodyRulePrerequisite() throws Exception {
|
||||
CrawlURI curi = makeCrawlURI("http://localhost:7777/200k");
|
||||
curi.setPrerequisite(true);
|
||||
fetcher().setShouldFetchBodyRule(new RejectDecideRule());
|
||||
fetcher().process(curi);
|
||||
|
||||
assertEquals(200, curi.getFetchStatus());
|
||||
assertTrue(curi.isSuccess());
|
||||
assertFalse(messageBodyString(curi).isEmpty());
|
||||
assertFalse(curi.getAnnotations().contains("midFetchAbort"));
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testFetchTimeout() throws Exception {
|
||||
CrawlURI curi = makeCrawlURI("http://localhost:7777/slow.txt");
|
||||
@@ -682,6 +723,8 @@ public class FetchHTTPTest {
|
||||
|
||||
// logger.info('\n' + httpRequestString(curi) + "\n\n" + rawResponseString(curi));
|
||||
assertTrue(curi.getAnnotations().contains("timeTrunc"));
|
||||
assertEquals(200, curi.getFetchStatus());
|
||||
assertFalse(curi.getAnnotations().contains("midFetchAbort"));
|
||||
assertTrue(curi.getFetchDuration() >= 2000 && curi.getFetchDuration() < 2200);
|
||||
}
|
||||
|
||||
|
||||
Reference in new issue
Block a user