Merge branch 'master' into upgrade-httpclient-4-5

2026-09-23 06:05:46 +00:00 · 2021-06-17 14:32:43 +01:00
parent e07b1bb938 4b82d6060c
commit 998eaa3ed8
1 changed files with 7 additions and 1 deletions
@@ -160,9 +160,15 @@ public class ExtractorSitemap extends ContentExtractor {

            // Add the URI:
        	// Adding 'regular' URL listed in the sitemap
-            addRelativeToBase(curi, max, newUri.toString(),
+            CrawlURI newCuri = addRelativeToBase(curi, max, newUri.toString(),
                    LinkContext.MANIFEST_MISC, Hop.MANIFEST);

+            if (isSitemap) {
+                // Annotate as a Site Map:
+                newCuri.getAnnotations().add(
+                        ExtractorRobotsTxt.ANNOTATION_IS_SITEMAP);
+            }
+            
            // And log about it:
            LOGGER.fine("Found " + newUri + " from " + curi + " Dated "
                    + lastModified + " and with isSitemap = " + isSitemap);