Soft404Detector: only probe servers reached by seeds or links

Servers reached by a redirect or speculative link, such as other sites a
crawled site redirects to, got a probe too.
This commit is contained in:
Alex Osborne committed 2026-11-17 14:36:46 +09:00
1 parent 82e1c5ad59
commit c3d062bee2
2 files changed
+17 -10

No files matched your search

@@ -174,8 +174,10 @@ public class Soft404Detector extends Processor {
protected void scheduleProbe(CrawlURI curi, CrawlServer server) {
int status = curi.getFetchStatus();
if (status < 200 || status >= 400) return;
// Don't probe servers that are only being fetched from for embeds (e.g. CDNs).
if (Hop.EMBED.getHopString().equals(curi.getLastHop())) return;
// Only probe servers reached as seeds or by following links, not e.g. CDNs or other
// sites that a crawled site redirects to.
String hop = curi.getLastHop();
if (!hop.isEmpty() && !hop.equals(Hop.NAVLINK.getHopString())) return;
if (!server.startSoft404Probe()) return;
server.makeDirty();
try {
@@ -35,6 +35,8 @@ import org.archive.util.Recorder;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.io.TempDir;
import org.junit.jupiter.params.ParameterizedTest;
import org.junit.jupiter.params.provider.ValueSource;
import static org.junit.jupiter.api.Assertions.*;
@@ -190,15 +192,18 @@ class Soft404DetectorTest {
assertFalse(server().isSoft404ProbeScheduled());
}
@Test
void doesNotProbeFromEmbeds() throws Exception {
CrawlURI image = new CrawlURI(UURIFactory.getInstance("https://example.com/logo.png"),
"LE", UURIFactory.getInstance("https://other.example/"), LinkContext.EMBED_MISC);
image.setFetchStatus(200);
image.setFetchType(CrawlURI.FetchType.HTTP_GET);
detector.process(image);
@ParameterizedTest
@ValueSource(strings = {"LE", "LR", "LX"})
void onlyProbesFromSeedsAndNavlinks(String pathFromSeed) throws Exception {
CrawlURI page = new CrawlURI(UURIFactory.getInstance("https://example.com/page"),
pathFromSeed, UURIFactory.getInstance("https://other.example/"),
LinkContext.NAVLINK_MISC);
page.setFetchStatus(200);
page.setFetchType(CrawlURI.FetchType.HTTP_GET);
page.setContentType("text/html");
detector.process(page);
assertTrue(image.getOutLinks().isEmpty());
assertTrue(page.getOutLinks().isEmpty());
assertFalse(server().isSoft404ProbeScheduled());
}