diff --git a/docs/bean-reference.rst b/docs/bean-reference.rst index 666f71da..c5e790f9 100644 --- a/docs/bean-reference.rst +++ b/docs/bean-reference.rst @@ -463,6 +463,11 @@ ExtractorSitemap .. bean-doc:: org.archive.modules.extractor.ExtractorSitemap +ExtractorSVG +~~~~~~~~~~~~ + +.. bean-doc:: org.archive.modules.extractor.ExtractorSVG + ExtractorSWF ~~~~~~~~~~~~ diff --git a/engine/src/main/resources/org/archive/crawler/restlet/profile-crawler-beans.cxml b/engine/src/main/resources/org/archive/crawler/restlet/profile-crawler-beans.cxml index 1b6a53db..a0044b47 100644 --- a/engine/src/main/resources/org/archive/crawler/restlet/profile-crawler-beans.cxml +++ b/engine/src/main/resources/org/archive/crawler/restlet/profile-crawler-beans.cxml @@ -298,6 +298,8 @@ http://example.example/example + + @@ -354,6 +356,8 @@ http://example.example/example + + diff --git a/engine/src/main/resources/org/archive/crawler/restlet/profile-crawler-beans.groovy b/engine/src/main/resources/org/archive/crawler/restlet/profile-crawler-beans.groovy index 8c6bf8da..59b4c622 100644 --- a/engine/src/main/resources/org/archive/crawler/restlet/profile-crawler-beans.groovy +++ b/engine/src/main/resources/org/archive/crawler/restlet/profile-crawler-beans.groovy @@ -272,6 +272,7 @@ http://example.example/example extractorHttp(ExtractorHTTP) extractorRobotsTxt(ExtractorRobotsTxt) extractorSitemap(ExtractorSitemap) + extractorSvg(ExtractorSVG) extractorHtml(ExtractorHTML) { // extractJavascript = true // extractValueAttributes = true @@ -307,6 +308,8 @@ http://example.example/example ref('extractorRobotsTxt'), // ...extract links from sitemaps... ref('extractorSitemap'), + // ...extract outlinks from SVG content... + ref('extractorSvg'), // ...extract outlinks from HTML content... ref('extractorHtml'), // ...extract outlinks from CSS content... diff --git a/modules/src/main/java/org/archive/modules/extractor/ExtractorCSS.java b/modules/src/main/java/org/archive/modules/extractor/ExtractorCSS.java index 3cdb7f5d..f0483309 100644 --- a/modules/src/main/java/org/archive/modules/extractor/ExtractorCSS.java +++ b/modules/src/main/java/org/archive/modules/extractor/ExtractorCSS.java @@ -133,6 +133,11 @@ public class ExtractorCSS extends ContentExtractor { // Remove backslashes when used as escape character in CSS URL cssUri = TextUtils.replaceAll(CSS_BACKSLASH_ESCAPE, cssUri, "$1"); + if (cssUri.startsWith("#")) { + // a reference to an element in the document itself, + // e.g. an SVG gradient: fill: url(#gradient) + continue; + } foundLinks++; int max = ext.getExtractorParameters().getMaxOutlinks(); try { diff --git a/modules/src/main/java/org/archive/modules/extractor/ExtractorHTML.java b/modules/src/main/java/org/archive/modules/extractor/ExtractorHTML.java index 67e3e7c4..be06205b 100644 --- a/modules/src/main/java/org/archive/modules/extractor/ExtractorHTML.java +++ b/modules/src/main/java/org/archive/modules/extractor/ExtractorHTML.java @@ -473,6 +473,12 @@ public class ExtractorHTML extends ContentExtractor implements InitializingBean linkContext = context; } else if ("a[data-remote='true']/@href".equals(context)) { processEmbed(curi, value, context); + } else if (ExtractorSVG.EMBED_HREF_ELEMENTS.contains( + elementStr.toLowerCase(Locale.ROOT))) { + // inline SVG resources such as and + if (!value.toString().startsWith("#")) { + processEmbed(curi, value, context); + } } else { // other HREFs treated as links processLink(curi, value, context); @@ -588,6 +594,11 @@ public class ExtractorHTML extends ContentExtractor implements InitializingBean && elementStr.equalsIgnoreCase(IFRAME)) { // embedded HTML document processSrcdoc(curi, value); + } else if (ExtractorSVG.URL_PRESENTATION_ATTRIBUTES.contains( + attrName.toString().toLowerCase(Locale.ROOT))) { + // inline SVG presentation attribute, e.g. fill="url(...)" + numberOfLinksExtracted.addAndGet(ExtractorCSS.processStyleCode( + this, curi, value)); } // 2023 updates get img or source data attr diff --git a/modules/src/main/java/org/archive/modules/extractor/ExtractorSVG.java b/modules/src/main/java/org/archive/modules/extractor/ExtractorSVG.java new file mode 100644 index 00000000..c1c5676d --- /dev/null +++ b/modules/src/main/java/org/archive/modules/extractor/ExtractorSVG.java @@ -0,0 +1,416 @@ +/* + * This file is part of the Heritrix web crawler (crawler.archive.org). + * + * Licensed to the Internet Archive (IA) by one or more individual + * contributors. + * + * The IA licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.archive.modules.extractor; + +import java.io.ByteArrayInputStream; +import java.io.IOException; +import java.io.InputStream; +import java.io.StringWriter; +import java.util.ArrayList; +import java.util.List; +import java.util.Locale; +import java.util.Set; +import java.util.logging.Level; +import java.util.logging.Logger; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +import javax.xml.stream.XMLInputFactory; +import javax.xml.stream.XMLOutputFactory; +import javax.xml.stream.XMLStreamConstants; +import javax.xml.stream.XMLStreamException; +import javax.xml.stream.XMLStreamReader; +import javax.xml.stream.XMLStreamWriter; + +import org.apache.commons.io.input.BoundedInputStream; +import org.apache.commons.lang3.StringEscapeUtils; +import org.archive.modules.CrawlURI; +import org.archive.url.URIException; +import org.springframework.beans.factory.annotation.Autowired; + +/** + * Extracts links from SVG images. + * + *

Handles responses with the content type {@code image/svg+xml}, and also {@code text/xml} + * and {@code application/xml} responses that turn out to be SVG. Other XML documents are left + * for later extractors, so this extractor should come before any other extractor that handles + * XML, such as ExtractorXML. + * + *

Images, stylesheets, scripts, fonts and other resources the SVG loads are extracted as + * embedded links, and {@code } links as navigation links. Links in inline CSS are extracted + * too. Inline scripts and HTML embedded in the SVG are only processed if the crawl has an + * ExtractorJS and an ExtractorHTML bean, respectively, and use their settings. + * + *

As in browsers, nothing is extracted after an XML syntax error, or from a document without + * the SVG namespace declaration. Only the first {@code maxSizeToParse} bytes are read. + */ +public class ExtractorSVG extends ContentExtractor { + + private static final Logger logger = Logger.getLogger(ExtractorSVG.class.getName()); + + static final String SVG_NS = "http://www.w3.org/2000/svg"; + static final String XLINK_NS = "http://www.w3.org/1999/xlink"; + + /** + * Elements (lower case) whose {@code href} references a resource that's loaded as part of + * rendering the document, rather than navigated to. + */ + static final Set EMBED_HREF_ELEMENTS = Set.of( + "audio", "cursor", "feimage", "filter", "font-face-uri", "image", "lineargradient", + "mpath", "pattern", "radialgradient", "script", "textpath", "tref", "use", + "video"); // audio and video are from SVG Tiny 1.2, which browsers don't implement + + /** + * Presentation attributes (lower case) that can take a CSS {@code url(...)} value. + */ + static final Set URL_PRESENTATION_ATTRIBUTES = Set.of( + "clip-path", "cursor", "fill", "filter", "marker-end", "marker-mid", "marker-start", + "mask", "stroke"); + + private static final Pattern PSEUDO_ATTRIBUTE_HREF = + Pattern.compile("(?:^|\\s)href\\s*=\\s*(?:\"([^\"]*)\"|'([^']*)')"); + + /** rel is a set of space-separated, case-insensitive tokens */ + private static final Pattern REL_NOFOLLOW = + Pattern.compile("(?i)(?:^|\\s)nofollow(?:\\s|$)"); + + private static final XMLInputFactory INPUT_FACTORY = newInputFactory(); + private static final XMLOutputFactory OUTPUT_FACTORY = XMLOutputFactory.newFactory(); + + private static XMLInputFactory newInputFactory() { + XMLInputFactory factory = XMLInputFactory.newFactory(); + factory.setProperty(XMLInputFactory.IS_NAMESPACE_AWARE, true); + factory.setProperty(XMLInputFactory.IS_COALESCING, true); + // the internal DTD subset is still processed, as some editors (e.g. Adobe Illustrator) + // declare entities there for the namespace URIs + factory.setProperty(XMLInputFactory.IS_SUPPORTING_EXTERNAL_ENTITIES, false); + // never fetch the external DTD subset + factory.setXMLResolver((publicId, systemId, baseUri, namespace) -> + new ByteArrayInputStream(new byte[0])); + return factory; + } + + { + setMaxSizeToParse(10 * 1024 * 1024L); // 10MB + } + public long getMaxSizeToParse() { + return (Long) kp.get("maxSizeToParse"); + } + /** + * The maximum number of bytes of an SVG document to parse, after any content encoding such + * as gzip is decoded. Links after this point are not extracted. + */ + public void setMaxSizeToParse(long maxSizeToParse) { + kp.put("maxSizeToParse", maxSizeToParse); + } + + /** + * Javascript extractor to use to process inline scripts. Autowired if available. If null, + * links will not be extracted from inline scripts. + */ + transient protected ExtractorJS extractorJS; + public ExtractorJS getExtractorJS() { + return extractorJS; + } + @Autowired(required = false) + public void setExtractorJS(ExtractorJS extractorJS) { + this.extractorJS = extractorJS; + } + + /** + * HTML extractor to use to process the content of foreignObject elements. Autowired if + * available. If null, links will not be extracted from foreignObject elements. Its + * obeyRelNofollow setting also applies to {@code } links in the SVG. + */ + transient protected ExtractorHTML extractorHTML; + public ExtractorHTML getExtractorHTML() { + return extractorHTML; + } + @Autowired(required = false) + public void setExtractorHTML(ExtractorHTML extractorHTML) { + this.extractorHTML = extractorHTML; + } + + @Override + protected boolean shouldExtract(CrawlURI curi) { + // like browsers, also accept SVG served as generic XML; whether it's actually SVG is + // decided by the root element + return curi.hasContentType("image/svg+xml") + || curi.hasContentType("text/xml") + || curi.hasContentType("application/xml"); + } + + @Override + protected boolean innerExtract(CrawlURI curi) { + boolean isSvg = false; + try (BoundedInputStream in = BoundedInputStream.builder() + .setInputStream(curi.getRecorder().getContentReplayInputStream()) + .setMaxCount(getMaxSizeToParse()) + .get()) { + try { + XMLStreamReader reader; + if (curi.containsContentTypeCharsetDeclaration()) { + // as in browsers, the charset from the HTTP headers wins + reader = INPUT_FACTORY.createXMLStreamReader(in, + curi.getRecorder().getCharset().name()); + } else { + reader = INPUT_FACTORY.createXMLStreamReader(in); + } + try { + isSvg = processRootElement(curi, reader); + if (isSvg) { + extract(curi, reader); + } + } finally { + reader.close(); + } + } catch (XMLStreamException e) { + if (!isSvg) { + // not well-formed up to the root element, so leave it to other extractors + return false; + } + // like browsers, keep whatever came before the error + if (in.getRemaining() == 0) { + if (logger.isLoggable(Level.FINE)) { + logger.fine("Stopped parsing SVG at maxSizeToParse: " + curi); + } + } else { + if (logger.isLoggable(Level.FINE)) { + logger.fine("Error parsing SVG " + curi + ": " + e.getMessage()); + } + curi.getNonFatalFailures().add(e); + } + } + } catch (IOException e) { + logger.log(Level.WARNING, "Failed reading content of " + curi, e); + curi.getNonFatalFailures().add(e); + return false; + } + // if it's not SVG, leave it to other extractors + return isSvg; + } + + /** + * Reads up to the root element. If it's an SVG {@code } element, extracts links from it + * and from any {@code } processing instructions before it. + * + * @return false if the document isn't SVG + */ + protected boolean processRootElement(CrawlURI curi, XMLStreamReader reader) + throws XMLStreamException { + List stylesheets = new ArrayList<>(); + while (reader.hasNext()) { + int event = reader.next(); + if (event == XMLStreamConstants.PROCESSING_INSTRUCTION + && "xml-stylesheet".equals(reader.getPITarget())) { + stylesheets.add(reader.getPIData()); + } else if (event == XMLStreamConstants.START_ELEMENT) { + if (!SVG_NS.equals(reader.getNamespaceURI()) + || !reader.getLocalName().equals("svg")) { + return false; + } + for (String data : stylesheets) { + processXmlStylesheet(curi, data); + } + processAttributes(curi, reader, "svg"); + return true; + } + } + return false; + } + + protected void extract(CrawlURI curi, XMLStreamReader reader) throws XMLStreamException { + StringBuilder text = null; // content of the current ")); + } + + @Test + public void testObeyRelNofollow() throws Exception { + String svg = SVG_START + + "x" + + "y" + + "" + + ""; + assertEquals(List.of( + "E https://example.org/image.png image/@href", + "L https://example.org/followed.html a/@href", + "L https://example.org/nofollow.html a/@href"), + extract(svg)); + + ((ExtractorSVG) extractor).getExtractorHTML().setObeyRelNofollow(true); + assertEquals(List.of( + "E https://example.org/image.png image/@href", + "L https://example.org/followed.html a/@href"), + extract(svg)); + } + + @Test + public void testHrefOnOtherElementsIgnored() throws Exception { + // href isn't an attribute of , and other namespaces aren't xlink + assertEquals(List.of(), + extract(SVG_START + "")); + } + + @Test + public void testXmlStylesheet() throws Exception { + assertEquals(List.of("E https://example.org/style.css?a=1&b=2 ?xml-stylesheet/@href"), + extract("" + + "" + + SVG_START + "")); + } + + @Test + public void testCss() throws Exception { + assertEquals(List.of( + "E https://example.org/clip.svg " + LinkContext.EMBED_MISC, + "E https://example.org/fill.svg " + LinkContext.EMBED_MISC, + "E https://example.org/font.woff " + LinkContext.EMBED_MISC, + "E https://example.org/marker.svg " + LinkContext.EMBED_MISC, + "E https://example.org/style.svg " + LinkContext.EMBED_MISC), + extract(SVG_START + + "" + + "" + + "" + + "")); + } + + @Test + public void testCdataStyleAndScript() throws Exception { + assertEquals(List.of( + "E https://example.org/bg.png " + LinkContext.EMBED_MISC, + "X https://example.org/script.html " + LinkContext.JS_MISC), + extract(SVG_START + + "" + + "" + + "")); + } + + @Test + public void testAnimateHref() throws Exception { + assertEquals(List.of( + "E https://example.org/1.png animate/@values", + "E https://example.org/2.png animate/@values", + "E https://example.org/3.png set/@to", + "E https://example.org/4.png animate/@from", + "E https://example.org/5.png animate/@to"), + extract(SVG_START + + "" + + "" + + "" + + "" + + "" + + "")); + } + + @Test + public void testForeignObject() throws Exception { + assertEquals(List.of( + "E https://example.org/after.png image/@href", + "E https://example.org/inner.png img/@src", + "E https://example.org/nested.png image/@href", + "L https://example.org/inner.html a/@href"), + extract(SVG_START + + "" + + "

" + + "a < bx" + + "" + + "
" + + "" + + "")); + } + + @Test + public void testForeignObjectWithoutExtractorHTML() throws Exception { + ((ExtractorSVG) extractor).setExtractorHTML(null); + assertEquals(List.of( + "E https://example.org/after.png image/@href", + "E https://example.org/nested.png image/@href"), + extract(SVG_START + + "
" + + "" + + "" + + "
" + + "" + + "")); + } + + @Test + public void testNonSvgNamespaceIgnored() throws Exception { + assertEquals(List.of("E https://example.org/b.png image/@href"), + extract(SVG_START + "" + + "")); + } + + @Test + public void testEntitiesFromInternalSubset() throws Exception { + // as written by Adobe Illustrator + assertEquals(List.of("E https://example.org/a.png image/@xlink:href"), + extract("" + + "" + + "" + + "]>" + + "")); + } + + @Test + public void testExternalEntityNotLoaded() throws Exception { + assertEquals(List.of("E https://example.org/a.png image/@href"), + extract("]>" + + SVG_START + "&x;")); + } + + @Test + public void testMalformedKeepsEarlierLinks() throws Exception { + CrawlURI curi = makeCrawlURI(SVG_START + "" + + "", "image/svg+xml"); + extractor.process(curi); + assertEquals(List.of("E https://example.org/a.png image/@href"), links(curi)); + assertTrue(curi.hasBeenLinkExtracted()); + assertEquals(1, curi.getNonFatalFailures().size()); + } + + @Test + public void testMaxSizeToParse() throws Exception { + String start = SVG_START + ""; + ((ExtractorSVG) extractor).setMaxSizeToParse(start.length() + 10); + CrawlURI curi = makeCrawlURI(start + "", "image/svg+xml"); + extractor.process(curi); + assertEquals(List.of("E https://example.org/a.png image/@href"), links(curi)); + assertTrue(curi.hasBeenLinkExtracted()); + assertTrue(curi.getNonFatalFailures().isEmpty(), curi.getNonFatalFailures().toString()); + } + + @Test + public void testCharsetFromContentType() throws Exception { + String svg = "" + SVG_START + + ""; + CrawlURI curi = makeCrawlURI(svg, "image/svg+xml; charset=UTF-8"); + extractor.process(curi); + assertEquals(List.of("E https://example.org/%C3%A9.png image/@href"), links(curi)); + } + + @Test + public void testShouldExtract() throws Exception { + ExtractorSVG svg = (ExtractorSVG) extractor; + String content = SVG_START + ""; + assertTrue(svg.shouldExtract(makeCrawlURI(content, "image/svg+xml"))); + assertTrue(svg.shouldExtract(makeCrawlURI(content, "IMAGE/SVG+XML; charset=utf-8"))); + assertTrue(svg.shouldExtract(makeCrawlURI(content, "text/xml"))); + assertTrue(svg.shouldExtract(makeCrawlURI(content, "application/xml"))); + assertFalse(svg.shouldExtract(makeCrawlURI(content, "text/html"))); + } + + @Test + public void testGenericXmlWithSvgRoot() throws Exception { + CrawlURI curi = makeCrawlURI("https://example.org/icon?id=3", + SVG_START + "", "text/xml"); + extractor.process(curi); + assertEquals(List.of("E https://example.org/a.png image/@href"), links(curi)); + assertTrue(curi.hasBeenLinkExtracted()); + } + + @Test + public void testNonSvgRootLeftToOtherExtractors() throws Exception { + String[][] documents = { + {"application/xml", "" + + "https://example.org/post"}, + {"text/xml", ""}, + {"image/svg+xml", ""}, // no namespace + {"image/svg+xml", "not xml"}, + }; + for (String[] document : documents) { + CrawlURI curi = makeCrawlURI(document[1], document[0]); + extractor.process(curi); + assertEquals(List.of(), links(curi), document[1]); + assertFalse(curi.hasBeenLinkExtracted(), document[1]); + assertTrue(curi.getNonFatalFailures().isEmpty(), document[1]); + } + } + + private List extract(String svg) throws Exception { + CrawlURI curi = makeCrawlURI(svg, "image/svg+xml"); + extractor.process(curi); + assertTrue(curi.getNonFatalFailures().isEmpty(), curi.getNonFatalFailures().toString()); + return links(curi); + } + + private CrawlURI makeCrawlURI(String content, String contentType) throws Exception { + return makeCrawlURI("https://example.org/image.svg", content, contentType); + } + + private CrawlURI makeCrawlURI(String uri, String content, String contentType) throws Exception { + CrawlURI curi = new CrawlURI(UURIFactory.getInstance(uri)); + curi.setContentType(contentType); + curi.setRecorder(createRecorder(content, "UTF-8")); + curi.setContentSize(content.getBytes(StandardCharsets.UTF_8).length); + curi.setFetchStatus(200); + return curi; + } + + /** + * Outlinks formatted as "hop uri context", sorted. + */ + private static List links(CrawlURI curi) { + List links = new ArrayList<>(); + for (CrawlURI link : curi.getOutLinks()) { + links.add(link.getLastHop() + " " + link.getURI() + " " + link.getViaContext()); + } + Collections.sort(links); + return links; + } +} diff --git a/modules/src/test/java/org/archive/modules/extractor/JerichoExtractorHTMLTest.java b/modules/src/test/java/org/archive/modules/extractor/JerichoExtractorHTMLTest.java index e8c52091..c2462668 100644 --- a/modules/src/test/java/org/archive/modules/extractor/JerichoExtractorHTMLTest.java +++ b/modules/src/test/java/org/archive/modules/extractor/JerichoExtractorHTMLTest.java @@ -257,4 +257,9 @@ public class JerichoExtractorHTMLTest extends ExtractorHTMLTest { protected JerichoExtractorHTML getExtractor() { return (JerichoExtractorHTML) extractor; } + + @Override + protected String xlinkHrefAttributeContext() { + return "xlink:href"; + } } \ No newline at end of file