From 81bef22270e53fb93203886cdef041c448eb482e Mon Sep 17 00:00:00 2001 From: Kristinn Sigurdsson Date: Wed, 16 Jun 2021 13:07:18 +0000 Subject: [PATCH] Mark nested sitemap links as sitemaps --- .../org/archive/modules/extractor/ExtractorSitemap.java | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/modules/src/main/java/org/archive/modules/extractor/ExtractorSitemap.java b/modules/src/main/java/org/archive/modules/extractor/ExtractorSitemap.java index 66b64495..7b95a887 100644 --- a/modules/src/main/java/org/archive/modules/extractor/ExtractorSitemap.java +++ b/modules/src/main/java/org/archive/modules/extractor/ExtractorSitemap.java @@ -160,9 +160,15 @@ public class ExtractorSitemap extends ContentExtractor { // Add the URI: // Adding 'regular' URL listed in the sitemap - addRelativeToBase(curi, max, newUri.toString(), + CrawlURI newCuri = addRelativeToBase(curi, max, newUri.toString(), LinkContext.MANIFEST_MISC, Hop.MANIFEST); + if (isSitemap) { + // Annotate as a Site Map: + newCuri.getAnnotations().add( + ExtractorRobotsTxt.ANNOTATION_IS_SITEMAP); + } + // And log about it: LOGGER.fine("Found " + newUri + " from " + curi + " Dated " + lastModified + " and with isSitemap = " + isSitemap);