mirror of
https://github.com/internetarchive/heritrix3.git
synced 2026-11-22 13:47:02 +00:00
Soft404Detector: only probe servers reached by seeds or links
Servers reached by a redirect or speculative link, such as other sites a crawled site redirects to, got a probe too.
This commit is contained in:
1 parent
82e1c5ad59
commit
c3d062bee2
2 files changed
+17
-10
No files matched your search
@@ -174,8 +174,10 @@ public class Soft404Detector extends Processor {
|
||||
protected void scheduleProbe(CrawlURI curi, CrawlServer server) {
|
||||
int status = curi.getFetchStatus();
|
||||
if (status < 200 || status >= 400) return;
|
||||
// Don't probe servers that are only being fetched from for embeds (e.g. CDNs).
|
||||
if (Hop.EMBED.getHopString().equals(curi.getLastHop())) return;
|
||||
// Only probe servers reached as seeds or by following links, not e.g. CDNs or other
|
||||
// sites that a crawled site redirects to.
|
||||
String hop = curi.getLastHop();
|
||||
if (!hop.isEmpty() && !hop.equals(Hop.NAVLINK.getHopString())) return;
|
||||
if (!server.startSoft404Probe()) return;
|
||||
server.makeDirty();
|
||||
try {
|
||||
|
||||
@@ -35,6 +35,8 @@ import org.archive.util.Recorder;
|
||||
import org.junit.jupiter.api.BeforeEach;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import org.junit.jupiter.api.io.TempDir;
|
||||
import org.junit.jupiter.params.ParameterizedTest;
|
||||
import org.junit.jupiter.params.provider.ValueSource;
|
||||
|
||||
import static org.junit.jupiter.api.Assertions.*;
|
||||
|
||||
@@ -190,15 +192,18 @@ class Soft404DetectorTest {
|
||||
assertFalse(server().isSoft404ProbeScheduled());
|
||||
}
|
||||
|
||||
@Test
|
||||
void doesNotProbeFromEmbeds() throws Exception {
|
||||
CrawlURI image = new CrawlURI(UURIFactory.getInstance("https://example.com/logo.png"),
|
||||
"LE", UURIFactory.getInstance("https://other.example/"), LinkContext.EMBED_MISC);
|
||||
image.setFetchStatus(200);
|
||||
image.setFetchType(CrawlURI.FetchType.HTTP_GET);
|
||||
detector.process(image);
|
||||
@ParameterizedTest
|
||||
@ValueSource(strings = {"LE", "LR", "LX"})
|
||||
void onlyProbesFromSeedsAndNavlinks(String pathFromSeed) throws Exception {
|
||||
CrawlURI page = new CrawlURI(UURIFactory.getInstance("https://example.com/page"),
|
||||
pathFromSeed, UURIFactory.getInstance("https://other.example/"),
|
||||
LinkContext.NAVLINK_MISC);
|
||||
page.setFetchStatus(200);
|
||||
page.setFetchType(CrawlURI.FetchType.HTTP_GET);
|
||||
page.setContentType("text/html");
|
||||
detector.process(page);
|
||||
|
||||
assertTrue(image.getOutLinks().isEmpty());
|
||||
assertTrue(page.getOutLinks().isEmpty());
|
||||
assertFalse(server().isSoft404ProbeScheduled());
|
||||
}
|
||||
|
||||
|
||||
Reference in new issue
Block a user