From 9d79ef73725c3f09d1683660256feee7649e8bb9 Mon Sep 17 00:00:00 2001 From: Alex Osborne Date: Thu, 5 Nov 2026 11:01:18 +0900 Subject: [PATCH 1/3] Add ExtractorSVG and extract links from inline SVG ExtractorSVG parses SVG with the JDK's StAX parser and extracts hrefs from elements like , and " + + "")); + } + + @Test + public void testAnimateHref() throws Exception { + assertEquals(List.of( + "E https://example.org/1.png animate/@values", + "E https://example.org/2.png animate/@values", + "E https://example.org/3.png set/@to", + "E https://example.org/4.png animate/@from", + "E https://example.org/5.png animate/@to"), + extract(SVG_START + + "" + + "" + + "" + + "" + + "" + + "")); + } + + @Test + public void testForeignObject() throws Exception { + assertEquals(List.of( + "E https://example.org/after.png image/@href", + "E https://example.org/inner.png img/@src", + "E https://example.org/nested.png image/@href", + "L https://example.org/inner.html a/@href"), + extract(SVG_START + + "" + + "
" + + "a < bx" + + "" + + "
" + + "" + + "")); + } + + @Test + public void testForeignObjectWithoutExtractorHTML() throws Exception { + ((ExtractorSVG) extractor).setExtractorHTML(null); + assertEquals(List.of( + "E https://example.org/after.png image/@href", + "E https://example.org/nested.png image/@href"), + extract(SVG_START + + "
" + + "" + + "" + + "
" + + "" + + "")); + } + + @Test + public void testNonSvgNamespaceIgnored() throws Exception { + assertEquals(List.of("E https://example.org/b.png image/@href"), + extract(SVG_START + "" + + "")); + } + + @Test + public void testEntitiesFromInternalSubset() throws Exception { + // as written by Adobe Illustrator + assertEquals(List.of("E https://example.org/a.png image/@xlink:href"), + extract("" + + "" + + "" + + "]>" + + "")); + } + + @Test + public void testExternalEntityNotLoaded() throws Exception { + assertEquals(List.of("E https://example.org/a.png image/@href"), + extract("]>" + + SVG_START + "&x;")); + } + + @Test + public void testMalformedKeepsEarlierLinks() throws Exception { + CrawlURI curi = makeCrawlURI(SVG_START + "" + + "", "image/svg+xml"); + extractor.process(curi); + assertEquals(List.of("E https://example.org/a.png image/@href"), links(curi)); + assertTrue(curi.hasBeenLinkExtracted()); + assertEquals(1, curi.getNonFatalFailures().size()); + } + + @Test + public void testMaxSizeToParse() throws Exception { + String start = SVG_START + ""; + ((ExtractorSVG) extractor).setMaxSizeToParse(start.length() + 10); + CrawlURI curi = makeCrawlURI(start + "", "image/svg+xml"); + extractor.process(curi); + assertEquals(List.of("E https://example.org/a.png image/@href"), links(curi)); + assertTrue(curi.hasBeenLinkExtracted()); + assertTrue(curi.getNonFatalFailures().isEmpty(), curi.getNonFatalFailures().toString()); + } + + @Test + public void testCharsetFromContentType() throws Exception { + String svg = "" + SVG_START + + ""; + CrawlURI curi = makeCrawlURI(svg, "image/svg+xml; charset=UTF-8"); + extractor.process(curi); + assertEquals(List.of("E https://example.org/%C3%A9.png image/@href"), links(curi)); + } + + @Test + public void testShouldExtract() throws Exception { + ExtractorSVG svg = (ExtractorSVG) extractor; + String content = SVG_START + ""; + assertTrue(svg.shouldExtract(makeCrawlURI(content, "image/svg+xml"))); + assertTrue(svg.shouldExtract(makeCrawlURI(content, "IMAGE/SVG+XML; charset=utf-8"))); + assertTrue(svg.shouldExtract(makeCrawlURI(content, "text/xml"))); + assertTrue(svg.shouldExtract(makeCrawlURI(content, "application/xml"))); + assertFalse(svg.shouldExtract(makeCrawlURI(content, "text/html"))); + } + + @Test + public void testGenericXmlWithSvgRoot() throws Exception { + CrawlURI curi = makeCrawlURI("https://example.org/icon?id=3", + SVG_START + "", "text/xml"); + extractor.process(curi); + assertEquals(List.of("E https://example.org/a.png image/@href"), links(curi)); + assertTrue(curi.hasBeenLinkExtracted()); + } + + @Test + public void testNonSvgRootLeftToOtherExtractors() throws Exception { + String[][] documents = { + {"application/xml", "" + + "https://example.org/post"}, + {"text/xml", ""}, + {"image/svg+xml", ""}, // no namespace + {"image/svg+xml", "not xml"}, + }; + for (String[] document : documents) { + CrawlURI curi = makeCrawlURI(document[1], document[0]); + extractor.process(curi); + assertEquals(List.of(), links(curi), document[1]); + assertFalse(curi.hasBeenLinkExtracted(), document[1]); + assertTrue(curi.getNonFatalFailures().isEmpty(), document[1]); + } + } + + private List extract(String svg) throws Exception { + CrawlURI curi = makeCrawlURI(svg, "image/svg+xml"); + extractor.process(curi); + assertTrue(curi.getNonFatalFailures().isEmpty(), curi.getNonFatalFailures().toString()); + return links(curi); + } + + private CrawlURI makeCrawlURI(String content, String contentType) throws Exception { + return makeCrawlURI("https://example.org/image.svg", content, contentType); + } + + private CrawlURI makeCrawlURI(String uri, String content, String contentType) throws Exception { + CrawlURI curi = new CrawlURI(UURIFactory.getInstance(uri)); + curi.setContentType(contentType); + curi.setRecorder(createRecorder(content, "UTF-8")); + curi.setContentSize(content.getBytes(StandardCharsets.UTF_8).length); + curi.setFetchStatus(200); + return curi; + } + + /** + * Outlinks formatted as "hop uri context", sorted. + */ + private static List links(CrawlURI curi) { + List links = new ArrayList<>(); + for (CrawlURI link : curi.getOutLinks()) { + links.add(link.getLastHop() + " " + link.getURI() + " " + link.getViaContext()); + } + Collections.sort(links); + return links; + } +} diff --git a/modules/src/test/java/org/archive/modules/extractor/JerichoExtractorHTMLTest.java b/modules/src/test/java/org/archive/modules/extractor/JerichoExtractorHTMLTest.java index e8c52091..c2462668 100644 --- a/modules/src/test/java/org/archive/modules/extractor/JerichoExtractorHTMLTest.java +++ b/modules/src/test/java/org/archive/modules/extractor/JerichoExtractorHTMLTest.java @@ -257,4 +257,9 @@ public class JerichoExtractorHTMLTest extends ExtractorHTMLTest { protected JerichoExtractorHTML getExtractor() { return (JerichoExtractorHTML) extractor; } + + @Override + protected String xlinkHrefAttributeContext() { + return "xlink:href"; + } } \ No newline at end of file From 5838dbf1f9bc56edc26f38982e36899a2c5de30f Mon Sep 17 00:00:00 2001 From: Alex Osborne Date: Thu, 5 Nov 2026 15:36:38 +0900 Subject: [PATCH 2/3] ExtractorSVG: obey ExtractorHTML obeyRelNofollow setting --- .../modules/extractor/ExtractorSVG.java | 23 +++++++++++++++++-- .../modules/extractor/ExtractorSVGTest.java | 20 ++++++++++++++++ 2 files changed, 41 insertions(+), 2 deletions(-) diff --git a/modules/src/main/java/org/archive/modules/extractor/ExtractorSVG.java b/modules/src/main/java/org/archive/modules/extractor/ExtractorSVG.java index 2e41d701..c1c5676d 100644 --- a/modules/src/main/java/org/archive/modules/extractor/ExtractorSVG.java +++ b/modules/src/main/java/org/archive/modules/extractor/ExtractorSVG.java @@ -86,6 +86,10 @@ public class ExtractorSVG extends ContentExtractor { private static final Pattern PSEUDO_ATTRIBUTE_HREF = Pattern.compile("(?:^|\\s)href\\s*=\\s*(?:\"([^\"]*)\"|'([^']*)')"); + /** rel is a set of space-separated, case-insensitive tokens */ + private static final Pattern REL_NOFOLLOW = + Pattern.compile("(?i)(?:^|\\s)nofollow(?:\\s|$)"); + private static final XMLInputFactory INPUT_FACTORY = newInputFactory(); private static final XMLOutputFactory OUTPUT_FACTORY = XMLOutputFactory.newFactory(); @@ -131,7 +135,8 @@ public class ExtractorSVG extends ContentExtractor { /** * HTML extractor to use to process the content of foreignObject elements. Autowired if - * available. If null, links will not be extracted from foreignObject elements. + * available. If null, links will not be extracted from foreignObject elements. Its + * obeyRelNofollow setting also applies to {@code } links in the SVG. */ transient protected ExtractorHTML extractorHTML; public ExtractorHTML getExtractorHTML() { @@ -283,7 +288,9 @@ public class ExtractorSVG extends ContentExtractor { if (name.equals("href") && (noNamespace || namespace.equals(XLINK_NS))) { String context = noNamespace ? "href" : "xlink:href"; if (elementLC.equals("a")) { - addLink(curi, value, elementLC, context, Hop.NAVLINK); + if (!obeyRelNofollow() || !isNofollow(reader.getAttributeValue(null, "rel"))) { + addLink(curi, value, elementLC, context, Hop.NAVLINK); + } } else if (EMBED_HREF_ELEMENTS.contains(elementLC)) { addLink(curi, value, elementLC, context, Hop.EMBED); } @@ -310,6 +317,18 @@ public class ExtractorSVG extends ContentExtractor { } } + /** + * Follows the ExtractorHTML bean's obeyRelNofollow setting, if there is one. + */ + private boolean obeyRelNofollow() { + ExtractorHTML html = getExtractorHTML(); + return html != null && html.getObeyRelNofollow(); + } + + private static boolean isNofollow(String rel) { + return rel != null && REL_NOFOLLOW.matcher(rel).find(); + } + protected void processXmlStylesheet(CrawlURI curi, String data) { if (data == null) { return; diff --git a/modules/src/test/java/org/archive/modules/extractor/ExtractorSVGTest.java b/modules/src/test/java/org/archive/modules/extractor/ExtractorSVGTest.java index 80bb1077..093719d9 100644 --- a/modules/src/test/java/org/archive/modules/extractor/ExtractorSVGTest.java +++ b/modules/src/test/java/org/archive/modules/extractor/ExtractorSVGTest.java @@ -100,6 +100,26 @@ public class ExtractorSVGTest extends ContentExtractorTestBase { "")); } + @Test + public void testObeyRelNofollow() throws Exception { + String svg = SVG_START + + "x" + + "y" + + "" + + ""; + assertEquals(List.of( + "E https://example.org/image.png image/@href", + "L https://example.org/followed.html a/@href", + "L https://example.org/nofollow.html a/@href"), + extract(svg)); + + ((ExtractorSVG) extractor).getExtractorHTML().setObeyRelNofollow(true); + assertEquals(List.of( + "E https://example.org/image.png image/@href", + "L https://example.org/followed.html a/@href"), + extract(svg)); + } + @Test public void testHrefOnOtherElementsIgnored() throws Exception { // href isn't an attribute of , and other namespaces aren't xlink From 832279989dd11fe156e17e1aca7b8e1d3e623a1e Mon Sep 17 00:00:00 2001 From: Alex Osborne Date: Fri, 6 Nov 2026 13:55:52 +0900 Subject: [PATCH 3/3] Add ExtractorSVG to the Groovy default profile --- .../org/archive/crawler/restlet/profile-crawler-beans.groovy | 3 +++ 1 file changed, 3 insertions(+) diff --git a/engine/src/main/resources/org/archive/crawler/restlet/profile-crawler-beans.groovy b/engine/src/main/resources/org/archive/crawler/restlet/profile-crawler-beans.groovy index 8c6bf8da..59b4c622 100644 --- a/engine/src/main/resources/org/archive/crawler/restlet/profile-crawler-beans.groovy +++ b/engine/src/main/resources/org/archive/crawler/restlet/profile-crawler-beans.groovy @@ -272,6 +272,7 @@ http://example.example/example extractorHttp(ExtractorHTTP) extractorRobotsTxt(ExtractorRobotsTxt) extractorSitemap(ExtractorSitemap) + extractorSvg(ExtractorSVG) extractorHtml(ExtractorHTML) { // extractJavascript = true // extractValueAttributes = true @@ -307,6 +308,8 @@ http://example.example/example ref('extractorRobotsTxt'), // ...extract links from sitemaps... ref('extractorSitemap'), + // ...extract outlinks from SVG content... + ref('extractorSvg'), // ...extract outlinks from HTML content... ref('extractorHtml'), // ...extract outlinks from CSS content...