diff --git a/modules/src/main/java/org/archive/modules/processor/Soft404Detector.java b/modules/src/main/java/org/archive/modules/processor/Soft404Detector.java index d380ac8c..ac3efbdd 100644 --- a/modules/src/main/java/org/archive/modules/processor/Soft404Detector.java +++ b/modules/src/main/java/org/archive/modules/processor/Soft404Detector.java @@ -174,8 +174,10 @@ public class Soft404Detector extends Processor { protected void scheduleProbe(CrawlURI curi, CrawlServer server) { int status = curi.getFetchStatus(); if (status < 200 || status >= 400) return; - // Don't probe servers that are only being fetched from for embeds (e.g. CDNs). - if (Hop.EMBED.getHopString().equals(curi.getLastHop())) return; + // Only probe servers reached as seeds or by following links, not e.g. CDNs or other + // sites that a crawled site redirects to. + String hop = curi.getLastHop(); + if (!hop.isEmpty() && !hop.equals(Hop.NAVLINK.getHopString())) return; if (!server.startSoft404Probe()) return; server.makeDirty(); try { diff --git a/modules/src/test/java/org/archive/modules/processor/Soft404DetectorTest.java b/modules/src/test/java/org/archive/modules/processor/Soft404DetectorTest.java index debac9f1..427aeb30 100644 --- a/modules/src/test/java/org/archive/modules/processor/Soft404DetectorTest.java +++ b/modules/src/test/java/org/archive/modules/processor/Soft404DetectorTest.java @@ -35,6 +35,8 @@ import org.archive.util.Recorder; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.io.TempDir; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.ValueSource; import static org.junit.jupiter.api.Assertions.*; @@ -190,15 +192,18 @@ class Soft404DetectorTest { assertFalse(server().isSoft404ProbeScheduled()); } - @Test - void doesNotProbeFromEmbeds() throws Exception { - CrawlURI image = new CrawlURI(UURIFactory.getInstance("https://example.com/logo.png"), - "LE", UURIFactory.getInstance("https://other.example/"), LinkContext.EMBED_MISC); - image.setFetchStatus(200); - image.setFetchType(CrawlURI.FetchType.HTTP_GET); - detector.process(image); + @ParameterizedTest + @ValueSource(strings = {"LE", "LR", "LX"}) + void onlyProbesFromSeedsAndNavlinks(String pathFromSeed) throws Exception { + CrawlURI page = new CrawlURI(UURIFactory.getInstance("https://example.com/page"), + pathFromSeed, UURIFactory.getInstance("https://other.example/"), + LinkContext.NAVLINK_MISC); + page.setFetchStatus(200); + page.setFetchType(CrawlURI.FetchType.HTTP_GET); + page.setContentType("text/html"); + detector.process(page); - assertTrue(image.getOutLinks().isEmpty()); + assertTrue(page.getOutLinks().isEmpty()); assertFalse(server().isSoft404ProbeScheduled()); }