From 5ed6147c3e3f69441b3771c95b4dbd56645e1f06 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kristinn=20Sigur=C3=B0sson?= Date: Tue, 29 Oct 2024 08:53:15 +0000 Subject: [PATCH] Treat manifest hops same as navlink hops Links from manifests (e.g. sitemaps) should not receive the preferential treatment sometimes accorded to "transitive" hops. Most commonly this is about giving priority to discovered (probable) embeds. Manifests should be regarded as more analogous with a directory page. --- modules/src/main/java/org/archive/modules/CrawlURI.java | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/modules/src/main/java/org/archive/modules/CrawlURI.java b/modules/src/main/java/org/archive/modules/CrawlURI.java index 7a1d915c..9e2ebb3d 100644 --- a/modules/src/main/java/org/archive/modules/CrawlURI.java +++ b/modules/src/main/java/org/archive/modules/CrawlURI.java @@ -1441,6 +1441,9 @@ implements Reporter, Serializable, OverlayContext, Comparable { * Tally up the number of transitive (non-simple-link) hops at * the end of this CrawlURI's pathFromSeed. * + * {@link Hop#NAVLINK} and {@link Hop#MANIFEST} are regarded as + * "simple links". All other hop types are regarded as transitive. + * * In some cases, URIs with greater than zero but less than some * threshold such hops are treated specially. * @@ -1453,7 +1456,8 @@ implements Reporter, Serializable, OverlayContext, Comparable { String path = getPathFromSeed(); int transCount = 0; for(int i=path.length()-1;i>=0;i--) { - if(path.charAt(i)==Hop.NAVLINK.getHopChar()) { + if(path.charAt(i)==Hop.NAVLINK.getHopChar() + || path.charAt(i)==Hop.MANIFEST.getHopChar()) { break; } transCount++;