mirror of
https://github.com/internetarchive/heritrix3.git
synced 2026-11-22 13:47:02 +00:00
fix: give midfetch aborts their own fetch status code
Add S_MIDFETCH_ABORTED (-5005) and set it in FetchHTTP when shouldFetchBodyRule rejects a response, in place of S_OUT_OF_SCOPE. The code means something distinct from a scope rejection, so crawl logs and reports can now tell the two apart. AbstractFrontier.isDisregarded() treats it as disregarded, CrawlURI gives it a readable string and the glossary lists it.
This commit is contained in:
8 files changed
+35
-4
No files matched your search
@@ -347,6 +347,8 @@ Other Heritrix status codes are listed below.
|
||||
- Blocked due to exceeding an established quota.
|
||||
* - -5004
|
||||
- Blocked due to exceeding an established runtime
|
||||
* - -5005
|
||||
- The fetch was aborted after receiving the response headers because the response was rejected by ``shouldFetchBodyRule``.
|
||||
* - -6000
|
||||
- Deleted from Frontier by user.
|
||||
* - -7000
|
||||
|
||||
@@ -27,6 +27,7 @@ import static org.archive.modules.fetcher.FetchStatusCodes.S_CONNECT_LOST;
|
||||
import static org.archive.modules.fetcher.FetchStatusCodes.S_DEFERRED;
|
||||
import static org.archive.modules.fetcher.FetchStatusCodes.S_DELETED_BY_USER;
|
||||
import static org.archive.modules.fetcher.FetchStatusCodes.S_DOMAIN_UNRESOLVABLE;
|
||||
import static org.archive.modules.fetcher.FetchStatusCodes.S_MIDFETCH_ABORTED;
|
||||
import static org.archive.modules.fetcher.FetchStatusCodes.S_OUT_OF_SCOPE;
|
||||
import static org.archive.modules.fetcher.FetchStatusCodes.S_ROBOTS_PRECLUDED;
|
||||
import static org.archive.modules.fetcher.FetchStatusCodes.S_TOO_MANY_EMBED_HOPS;
|
||||
@@ -1072,6 +1073,7 @@ public abstract class AbstractFrontier
|
||||
case S_BLOCKED_BY_CUSTOM_PROCESSOR:
|
||||
case S_OUT_OF_SCOPE: // filtered out by scope
|
||||
case S_BLOCKED_BY_USER: // filtered out by user
|
||||
case S_MIDFETCH_ABORTED: // rejected by shouldFetchBodyRule
|
||||
case S_TOO_MANY_EMBED_HOPS: // too far from last true link
|
||||
case S_TOO_MANY_LINK_HOPS: // too far from seeds
|
||||
case S_DELETED_BY_USER: // user deleted
|
||||
|
||||
@@ -29,6 +29,7 @@ import java.nio.file.Path;
|
||||
import org.archive.url.URIException;
|
||||
import org.archive.modules.CrawlURI;
|
||||
import org.archive.modules.SchedulingConstants;
|
||||
import org.archive.modules.fetcher.FetchStatusCodes;
|
||||
import org.archive.modules.extractor.LinkContext.SimpleLinkContext;
|
||||
import org.archive.net.UURI;
|
||||
import org.archive.net.UURIFactory;
|
||||
@@ -108,6 +109,12 @@ public class CrawlURITest {
|
||||
assertEquals("value", curi.getData().get("key"), "Didn't find AList item");
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testMidfetchAbortedStatusString() {
|
||||
assertEquals("Heritrix(-5005)-Midfetch aborted",
|
||||
CrawlURI.fetchStatusCodesToString(FetchStatusCodes.S_MIDFETCH_ABORTED));
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testExtendHopsPath() {
|
||||
assertEquals("L",CrawlURI.extendHopsPath("",'L'), "from empty");
|
||||
|
||||
@@ -19,7 +19,13 @@
|
||||
|
||||
package org.archive.crawler.frontier;
|
||||
|
||||
import static org.junit.jupiter.api.Assertions.assertTrue;
|
||||
|
||||
import org.archive.crawler.framework.CrawlerProcessorTestBase;
|
||||
import org.archive.modules.CrawlURI;
|
||||
import org.archive.modules.fetcher.FetchStatusCodes;
|
||||
import org.archive.net.UURIFactory;
|
||||
import org.junit.jupiter.api.Test;
|
||||
|
||||
|
||||
/**
|
||||
@@ -30,5 +36,12 @@ public class BdbFrontierTest extends CrawlerProcessorTestBase {
|
||||
|
||||
// TODO TESTME
|
||||
|
||||
@Test
|
||||
public void testMidfetchAbortIsDisregarded() throws Exception {
|
||||
CrawlURI curi = new CrawlURI(UURIFactory.getInstance("http://example.com/"));
|
||||
curi.setFetchStatus(FetchStatusCodes.S_MIDFETCH_ABORTED);
|
||||
assertTrue(new BdbFrontier().isDisregarded(curi));
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
@@ -360,6 +360,8 @@ implements Reporter, Serializable, OverlayContext, Comparable<CrawlURI> {
|
||||
+ ")-Domain unresolvable";
|
||||
case S_OUT_OF_SCOPE:
|
||||
return "Heritrix(" + S_OUT_OF_SCOPE + ")-Out of scope";
|
||||
case S_MIDFETCH_ABORTED:
|
||||
return "Heritrix(" + S_MIDFETCH_ABORTED + ")-Midfetch aborted";
|
||||
case S_DOMAIN_PREREQUISITE_FAILURE:
|
||||
return "Heritrix(" + S_DOMAIN_PREREQUISITE_FAILURE
|
||||
+ ")-Domain prerequisite failure";
|
||||
|
||||
@@ -24,7 +24,7 @@ import static org.archive.modules.fetcher.FetchErrors.TIMER_TRUNC;
|
||||
import static org.archive.modules.fetcher.FetchStatusCodes.S_CONNECT_FAILED;
|
||||
import static org.archive.modules.fetcher.FetchStatusCodes.S_CONNECT_LOST;
|
||||
import static org.archive.modules.fetcher.FetchStatusCodes.S_DOMAIN_PREREQUISITE_FAILURE;
|
||||
import static org.archive.modules.fetcher.FetchStatusCodes.S_OUT_OF_SCOPE;
|
||||
import static org.archive.modules.fetcher.FetchStatusCodes.S_MIDFETCH_ABORTED;
|
||||
import static org.archive.modules.fetcher.FetchStatusCodes.S_UNFETCHABLE_URI;
|
||||
import static org.archive.modules.fetcher.FetchStatusCodes.S_SSL_ERROR;
|
||||
import static org.archive.modules.recrawl.RecrawlAttributeConstants.A_REFERENCE_LENGTH;
|
||||
@@ -467,7 +467,7 @@ public class FetchHTTP extends Processor implements Lifecycle {
|
||||
/**
|
||||
* DecideRules applied after receipt of HTTP response headers but before we
|
||||
* start to download the body. If the rule returns REJECT, the fetch is
|
||||
* aborted and the URI is marked {@link FetchStatusCodes#S_OUT_OF_SCOPE}.
|
||||
* aborted and the URI is marked {@link FetchStatusCodes#S_MIDFETCH_ABORTED}.
|
||||
* This makes {@link CrawlURI#isSuccess()} false, skips extraction and WARC
|
||||
* writing (including header-only captures), and causes the frontier to
|
||||
* disregard the URI without a queue-budget charge or retry. Prerequisites
|
||||
@@ -645,7 +645,7 @@ public class FetchHTTP extends Processor implements Lifecycle {
|
||||
protected boolean maybeMidfetchAbort(CrawlURI curi, AbstractExecutionAwareRequest request) {
|
||||
if (checkMidfetchAbort(curi)) {
|
||||
doAbort(curi, request, "midFetchAbort");
|
||||
curi.setFetchStatus(S_OUT_OF_SCOPE);
|
||||
curi.setFetchStatus(S_MIDFETCH_ABORTED);
|
||||
curi.getRecorder().getRecordedInput().chopAtMessageBodyBegin();
|
||||
return true;
|
||||
} else {
|
||||
|
||||
@@ -102,6 +102,11 @@ public interface FetchStatusCodes {
|
||||
* TODO: Add to documentation and help page.
|
||||
*/
|
||||
public static final int S_BLOCKED_BY_RUNTIME_LIMIT = -5004;
|
||||
/**
|
||||
* Fetch aborted after the response headers were received, because the
|
||||
* response was rejected by FetchHTTP's shouldFetchBodyRule.
|
||||
*/
|
||||
public static final int S_MIDFETCH_ABORTED = -5005;
|
||||
/** deleted from frontier by user */
|
||||
public static final int S_DELETED_BY_USER = -6000;
|
||||
|
||||
|
||||
@@ -663,7 +663,7 @@ public class FetchHTTPTest {
|
||||
assertEquals("text/plain;charset=US-ASCII", curi.getContentType());
|
||||
assertTrue(curi.getCredentials().isEmpty());
|
||||
assertTrue(curi.getFetchDuration() >= 0);
|
||||
assertEquals(FetchStatusCodes.S_OUT_OF_SCOPE, curi.getFetchStatus());
|
||||
assertEquals(FetchStatusCodes.S_MIDFETCH_ABORTED, curi.getFetchStatus());
|
||||
assertFalse(curi.isSuccess());
|
||||
assertSame(curi.getFetchType(), FetchType.HTTP_GET);
|
||||
|
||||
|
||||
Reference in new issue
Block a user