mirror of
https://github.com/internetarchive/heritrix3.git
synced 2026-11-22 13:47:02 +00:00
Merge pull request #792 from mvanhorn/fix/455-midfetch-abort-statistics
fix: disregard HTTP responses rejected before body fetching
This commit is contained in:
8 files changed
+83
-4
No files matched your search
@@ -347,6 +347,8 @@ Other Heritrix status codes are listed below.
|
||||
- Blocked due to exceeding an established quota.
|
||||
* - -5004
|
||||
- Blocked due to exceeding an established runtime
|
||||
* - -5005
|
||||
- The fetch was aborted after receiving the response headers because the response was rejected by ``shouldFetchBodyRule``.
|
||||
* - -6000
|
||||
- Deleted from Frontier by user.
|
||||
* - -7000
|
||||
|
||||
@@ -27,6 +27,7 @@ import static org.archive.modules.fetcher.FetchStatusCodes.S_CONNECT_LOST;
|
||||
import static org.archive.modules.fetcher.FetchStatusCodes.S_DEFERRED;
|
||||
import static org.archive.modules.fetcher.FetchStatusCodes.S_DELETED_BY_USER;
|
||||
import static org.archive.modules.fetcher.FetchStatusCodes.S_DOMAIN_UNRESOLVABLE;
|
||||
import static org.archive.modules.fetcher.FetchStatusCodes.S_MIDFETCH_ABORTED;
|
||||
import static org.archive.modules.fetcher.FetchStatusCodes.S_OUT_OF_SCOPE;
|
||||
import static org.archive.modules.fetcher.FetchStatusCodes.S_ROBOTS_PRECLUDED;
|
||||
import static org.archive.modules.fetcher.FetchStatusCodes.S_TOO_MANY_EMBED_HOPS;
|
||||
@@ -1072,6 +1073,7 @@ public abstract class AbstractFrontier
|
||||
case S_BLOCKED_BY_CUSTOM_PROCESSOR:
|
||||
case S_OUT_OF_SCOPE: // filtered out by scope
|
||||
case S_BLOCKED_BY_USER: // filtered out by user
|
||||
case S_MIDFETCH_ABORTED: // rejected by shouldFetchBodyRule
|
||||
case S_TOO_MANY_EMBED_HOPS: // too far from last true link
|
||||
case S_TOO_MANY_LINK_HOPS: // too far from seeds
|
||||
case S_DELETED_BY_USER: // user deleted
|
||||
|
||||
@@ -29,6 +29,7 @@ import java.nio.file.Path;
|
||||
import org.archive.url.URIException;
|
||||
import org.archive.modules.CrawlURI;
|
||||
import org.archive.modules.SchedulingConstants;
|
||||
import org.archive.modules.fetcher.FetchStatusCodes;
|
||||
import org.archive.modules.extractor.LinkContext.SimpleLinkContext;
|
||||
import org.archive.net.UURI;
|
||||
import org.archive.net.UURIFactory;
|
||||
@@ -108,6 +109,12 @@ public class CrawlURITest {
|
||||
assertEquals("value", curi.getData().get("key"), "Didn't find AList item");
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testMidfetchAbortedStatusString() {
|
||||
assertEquals("Heritrix(-5005)-Midfetch aborted",
|
||||
CrawlURI.fetchStatusCodesToString(FetchStatusCodes.S_MIDFETCH_ABORTED));
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testExtendHopsPath() {
|
||||
assertEquals("L",CrawlURI.extendHopsPath("",'L'), "from empty");
|
||||
|
||||
@@ -19,7 +19,13 @@
|
||||
|
||||
package org.archive.crawler.frontier;
|
||||
|
||||
import static org.junit.jupiter.api.Assertions.assertTrue;
|
||||
|
||||
import org.archive.crawler.framework.CrawlerProcessorTestBase;
|
||||
import org.archive.modules.CrawlURI;
|
||||
import org.archive.modules.fetcher.FetchStatusCodes;
|
||||
import org.archive.net.UURIFactory;
|
||||
import org.junit.jupiter.api.Test;
|
||||
|
||||
|
||||
/**
|
||||
@@ -30,5 +36,12 @@ public class BdbFrontierTest extends CrawlerProcessorTestBase {
|
||||
|
||||
// TODO TESTME
|
||||
|
||||
@Test
|
||||
public void testMidfetchAbortIsDisregarded() throws Exception {
|
||||
CrawlURI curi = new CrawlURI(UURIFactory.getInstance("http://example.com/"));
|
||||
curi.setFetchStatus(FetchStatusCodes.S_MIDFETCH_ABORTED);
|
||||
assertTrue(new BdbFrontier().isDisregarded(curi));
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
@@ -360,6 +360,8 @@ implements Reporter, Serializable, OverlayContext, Comparable<CrawlURI> {
|
||||
+ ")-Domain unresolvable";
|
||||
case S_OUT_OF_SCOPE:
|
||||
return "Heritrix(" + S_OUT_OF_SCOPE + ")-Out of scope";
|
||||
case S_MIDFETCH_ABORTED:
|
||||
return "Heritrix(" + S_MIDFETCH_ABORTED + ")-Midfetch aborted";
|
||||
case S_DOMAIN_PREREQUISITE_FAILURE:
|
||||
return "Heritrix(" + S_DOMAIN_PREREQUISITE_FAILURE
|
||||
+ ")-Domain prerequisite failure";
|
||||
|
||||
@@ -24,6 +24,7 @@ import static org.archive.modules.fetcher.FetchErrors.TIMER_TRUNC;
|
||||
import static org.archive.modules.fetcher.FetchStatusCodes.S_CONNECT_FAILED;
|
||||
import static org.archive.modules.fetcher.FetchStatusCodes.S_CONNECT_LOST;
|
||||
import static org.archive.modules.fetcher.FetchStatusCodes.S_DOMAIN_PREREQUISITE_FAILURE;
|
||||
import static org.archive.modules.fetcher.FetchStatusCodes.S_MIDFETCH_ABORTED;
|
||||
import static org.archive.modules.fetcher.FetchStatusCodes.S_UNFETCHABLE_URI;
|
||||
import static org.archive.modules.fetcher.FetchStatusCodes.S_SSL_ERROR;
|
||||
import static org.archive.modules.recrawl.RecrawlAttributeConstants.A_REFERENCE_LENGTH;
|
||||
@@ -465,9 +466,12 @@ public class FetchHTTP extends Processor implements Lifecycle {
|
||||
}
|
||||
/**
|
||||
* DecideRules applied after receipt of HTTP response headers but before we
|
||||
* start to download the body. If any filter returns FALSE, the fetch is
|
||||
* aborted. Prerequisites such as robots.txt by-pass filtering (i.e. they
|
||||
* cannot be midfetch aborted.
|
||||
* start to download the body. If the rule returns REJECT, the fetch is
|
||||
* aborted and the URI is marked {@link FetchStatusCodes#S_MIDFETCH_ABORTED}.
|
||||
* This makes {@link CrawlURI#isSuccess()} false, skips extraction and WARC
|
||||
* writing (including header-only captures), and causes the frontier to
|
||||
* disregard the URI without a queue-budget charge or retry. Prerequisites
|
||||
* such as robots.txt bypass filtering and cannot be midfetch aborted.
|
||||
*/
|
||||
public void setShouldFetchBodyRule(DecideRule rule) {
|
||||
kp.put("shouldFetchBodyRule", rule);
|
||||
@@ -641,6 +645,7 @@ public class FetchHTTP extends Processor implements Lifecycle {
|
||||
protected boolean maybeMidfetchAbort(CrawlURI curi, AbstractExecutionAwareRequest request) {
|
||||
if (checkMidfetchAbort(curi)) {
|
||||
doAbort(curi, request, "midFetchAbort");
|
||||
curi.setFetchStatus(S_MIDFETCH_ABORTED);
|
||||
curi.getRecorder().getRecordedInput().chopAtMessageBodyBegin();
|
||||
return true;
|
||||
} else {
|
||||
|
||||
@@ -102,6 +102,11 @@ public interface FetchStatusCodes {
|
||||
* TODO: Add to documentation and help page.
|
||||
*/
|
||||
public static final int S_BLOCKED_BY_RUNTIME_LIMIT = -5004;
|
||||
/**
|
||||
* Fetch aborted after the response headers were received, because the
|
||||
* response was rejected by FetchHTTP's shouldFetchBodyRule.
|
||||
*/
|
||||
public static final int S_MIDFETCH_ABORTED = -5005;
|
||||
/** deleted from frontier by user */
|
||||
public static final int S_DELETED_BY_USER = -6000;
|
||||
|
||||
|
||||
@@ -578,6 +578,8 @@ public class FetchHTTPTest {
|
||||
fetcher().setMaxLengthBytes(50000);
|
||||
fetcher().process(curi);
|
||||
assertEquals(50001, curi.getRecordedSize());
|
||||
assertEquals(200, curi.getFetchStatus());
|
||||
assertFalse(curi.getAnnotations().contains("midFetchAbort"));
|
||||
}
|
||||
|
||||
@Test
|
||||
@@ -661,7 +663,8 @@ public class FetchHTTPTest {
|
||||
assertEquals("text/plain;charset=US-ASCII", curi.getContentType());
|
||||
assertTrue(curi.getCredentials().isEmpty());
|
||||
assertTrue(curi.getFetchDuration() >= 0);
|
||||
assertEquals(200, curi.getFetchStatus());
|
||||
assertEquals(FetchStatusCodes.S_MIDFETCH_ABORTED, curi.getFetchStatus());
|
||||
assertFalse(curi.isSuccess());
|
||||
assertSame(curi.getFetchType(), FetchType.HTTP_GET);
|
||||
|
||||
assertEquals(1, curi.getAnnotations().size());
|
||||
@@ -674,6 +677,44 @@ public class FetchHTTPTest {
|
||||
assertEquals("", entityString(curi));
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testShouldFetchBodyRuleAccept() throws Exception {
|
||||
CrawlURI curi = makeCrawlURI("http://localhost:7777/200k");
|
||||
fetcher().process(curi);
|
||||
|
||||
assertEquals(200, curi.getFetchStatus());
|
||||
assertTrue(curi.isSuccess());
|
||||
assertFalse(messageBodyString(curi).isEmpty());
|
||||
assertFalse(curi.getAnnotations().contains("midFetchAbort"));
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testShouldFetchBodyRuleNone() throws Exception {
|
||||
CrawlURI curi = makeCrawlURI("http://localhost:7777/200k");
|
||||
RejectDecideRule rule = new RejectDecideRule();
|
||||
rule.setEnabled(false);
|
||||
fetcher().setShouldFetchBodyRule(rule);
|
||||
fetcher().process(curi);
|
||||
|
||||
assertEquals(200, curi.getFetchStatus());
|
||||
assertTrue(curi.isSuccess());
|
||||
assertFalse(messageBodyString(curi).isEmpty());
|
||||
assertFalse(curi.getAnnotations().contains("midFetchAbort"));
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testShouldFetchBodyRulePrerequisite() throws Exception {
|
||||
CrawlURI curi = makeCrawlURI("http://localhost:7777/200k");
|
||||
curi.setPrerequisite(true);
|
||||
fetcher().setShouldFetchBodyRule(new RejectDecideRule());
|
||||
fetcher().process(curi);
|
||||
|
||||
assertEquals(200, curi.getFetchStatus());
|
||||
assertTrue(curi.isSuccess());
|
||||
assertFalse(messageBodyString(curi).isEmpty());
|
||||
assertFalse(curi.getAnnotations().contains("midFetchAbort"));
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testFetchTimeout() throws Exception {
|
||||
CrawlURI curi = makeCrawlURI("http://localhost:7777/slow.txt");
|
||||
@@ -682,6 +723,8 @@ public class FetchHTTPTest {
|
||||
|
||||
// logger.info('\n' + httpRequestString(curi) + "\n\n" + rawResponseString(curi));
|
||||
assertTrue(curi.getAnnotations().contains("timeTrunc"));
|
||||
assertEquals(200, curi.getFetchStatus());
|
||||
assertFalse(curi.getAnnotations().contains("midFetchAbort"));
|
||||
assertTrue(curi.getFetchDuration() >= 2000 && curi.getFetchDuration() < 2200);
|
||||
}
|
||||
|
||||
|
||||
Reference in new issue
Block a user