Merge pull request #812 from internetarchive/svg-extractor

Add ExtractorSVG for links in SVG images
This commit is contained in:
Alex Osborne authored and GitHub committed 2026-11-06 13:56:57 +09:00
commit c09f281c0f
11 files changed
+844 -3

No files matched your search

+5
View File
@@ -463,6 +463,11 @@ ExtractorSitemap
.. bean-doc:: org.archive.modules.extractor.ExtractorSitemap
ExtractorSVG
~~~~~~~~~~~~
.. bean-doc:: org.archive.modules.extractor.ExtractorSVG
ExtractorSWF
~~~~~~~~~~~~
@@ -298,6 +298,8 @@ http://example.example/example
<bean id="extractorSitemap" class="org.archive.modules.extractor.ExtractorSitemap">
</bean>
<bean id="extractorSvg" class="org.archive.modules.extractor.ExtractorSVG">
</bean>
<bean id="extractorHtml" class="org.archive.modules.extractor.ExtractorHTML">
<!-- <property name="extractJavascript" value="true" /> -->
<!-- <property name="extractValueAttributes" value="true" /> -->
@@ -354,6 +356,8 @@ http://example.example/example
<ref bean="extractorRobotsTxt"/>
<!-- ...extract links from sitemaps... -->
<ref bean="extractorSitemap"/>
<!-- ...extract outlinks from SVG content... -->
<ref bean="extractorSvg"/>
<!-- ...extract outlinks from HTML content... -->
<ref bean="extractorHtml"/>
<!-- ...extract outlinks from CSS content... -->
@@ -272,6 +272,7 @@ http://example.example/example
extractorHttp(ExtractorHTTP)
extractorRobotsTxt(ExtractorRobotsTxt)
extractorSitemap(ExtractorSitemap)
extractorSvg(ExtractorSVG)
extractorHtml(ExtractorHTML) {
// extractJavascript = true
// extractValueAttributes = true
@@ -307,6 +308,8 @@ http://example.example/example
ref('extractorRobotsTxt'),
// ...extract links from sitemaps...
ref('extractorSitemap'),
// ...extract outlinks from SVG content...
ref('extractorSvg'),
// ...extract outlinks from HTML content...
ref('extractorHtml'),
// ...extract outlinks from CSS content...
@@ -133,6 +133,11 @@ public class ExtractorCSS extends ContentExtractor {
// Remove backslashes when used as escape character in CSS URL
cssUri = TextUtils.replaceAll(CSS_BACKSLASH_ESCAPE, cssUri,
"$1");
if (cssUri.startsWith("#")) {
// a reference to an element in the document itself,
// e.g. an SVG gradient: fill: url(#gradient)
continue;
}
foundLinks++;
int max = ext.getExtractorParameters().getMaxOutlinks();
try {
@@ -473,6 +473,12 @@ public class ExtractorHTML extends ContentExtractor implements InitializingBean
linkContext = context;
} else if ("a[data-remote='true']/@href".equals(context)) {
processEmbed(curi, value, context);
} else if (ExtractorSVG.EMBED_HREF_ELEMENTS.contains(
elementStr.toLowerCase(Locale.ROOT))) {
// inline SVG resources such as <image href> and <use href>
if (!value.toString().startsWith("#")) {
processEmbed(curi, value, context);
}
} else {
// other HREFs treated as links
processLink(curi, value, context);
@@ -588,6 +594,11 @@ public class ExtractorHTML extends ContentExtractor implements InitializingBean
&& elementStr.equalsIgnoreCase(IFRAME)) {
// embedded HTML document
processSrcdoc(curi, value);
} else if (ExtractorSVG.URL_PRESENTATION_ATTRIBUTES.contains(
attrName.toString().toLowerCase(Locale.ROOT))) {
// inline SVG presentation attribute, e.g. fill="url(...)"
numberOfLinksExtracted.addAndGet(ExtractorCSS.processStyleCode(
this, curi, value));
}
// 2023 updates get img or source data attr
@@ -0,0 +1,416 @@
/*
* This file is part of the Heritrix web crawler (crawler.archive.org).
*
* Licensed to the Internet Archive (IA) by one or more individual
* contributors.
*
* The IA licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.archive.modules.extractor;
import java.io.ByteArrayInputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.StringWriter;
import java.util.ArrayList;
import java.util.List;
import java.util.Locale;
import java.util.Set;
import java.util.logging.Level;
import java.util.logging.Logger;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import javax.xml.stream.XMLInputFactory;
import javax.xml.stream.XMLOutputFactory;
import javax.xml.stream.XMLStreamConstants;
import javax.xml.stream.XMLStreamException;
import javax.xml.stream.XMLStreamReader;
import javax.xml.stream.XMLStreamWriter;
import org.apache.commons.io.input.BoundedInputStream;
import org.apache.commons.lang3.StringEscapeUtils;
import org.archive.modules.CrawlURI;
import org.archive.url.URIException;
import org.springframework.beans.factory.annotation.Autowired;
/**
* Extracts links from SVG images.
*
* <p>Handles responses with the content type {@code image/svg+xml}, and also {@code text/xml}
* and {@code application/xml} responses that turn out to be SVG. Other XML documents are left
* for later extractors, so this extractor should come before any other extractor that handles
* XML, such as ExtractorXML.
*
* <p>Images, stylesheets, scripts, fonts and other resources the SVG loads are extracted as
* embedded links, and {@code <a>} links as navigation links. Links in inline CSS are extracted
* too. Inline scripts and HTML embedded in the SVG are only processed if the crawl has an
* ExtractorJS and an ExtractorHTML bean, respectively, and use their settings.
*
* <p>As in browsers, nothing is extracted after an XML syntax error, or from a document without
* the SVG namespace declaration. Only the first {@code maxSizeToParse} bytes are read.
*/
public class ExtractorSVG extends ContentExtractor {
private static final Logger logger = Logger.getLogger(ExtractorSVG.class.getName());
static final String SVG_NS = "http://www.w3.org/2000/svg";
static final String XLINK_NS = "http://www.w3.org/1999/xlink";
/**
* Elements (lower case) whose {@code href} references a resource that's loaded as part of
* rendering the document, rather than navigated to.
*/
static final Set<String> EMBED_HREF_ELEMENTS = Set.of(
"audio", "cursor", "feimage", "filter", "font-face-uri", "image", "lineargradient",
"mpath", "pattern", "radialgradient", "script", "textpath", "tref", "use",
"video"); // audio and video are from SVG Tiny 1.2, which browsers don't implement
/**
* Presentation attributes (lower case) that can take a CSS {@code url(...)} value.
*/
static final Set<String> URL_PRESENTATION_ATTRIBUTES = Set.of(
"clip-path", "cursor", "fill", "filter", "marker-end", "marker-mid", "marker-start",
"mask", "stroke");
private static final Pattern PSEUDO_ATTRIBUTE_HREF =
Pattern.compile("(?:^|\\s)href\\s*=\\s*(?:\"([^\"]*)\"|'([^']*)')");
/** rel is a set of space-separated, case-insensitive tokens */
private static final Pattern REL_NOFOLLOW =
Pattern.compile("(?i)(?:^|\\s)nofollow(?:\\s|$)");
private static final XMLInputFactory INPUT_FACTORY = newInputFactory();
private static final XMLOutputFactory OUTPUT_FACTORY = XMLOutputFactory.newFactory();
private static XMLInputFactory newInputFactory() {
XMLInputFactory factory = XMLInputFactory.newFactory();
factory.setProperty(XMLInputFactory.IS_NAMESPACE_AWARE, true);
factory.setProperty(XMLInputFactory.IS_COALESCING, true);
// the internal DTD subset is still processed, as some editors (e.g. Adobe Illustrator)
// declare entities there for the namespace URIs
factory.setProperty(XMLInputFactory.IS_SUPPORTING_EXTERNAL_ENTITIES, false);
// never fetch the external DTD subset
factory.setXMLResolver((publicId, systemId, baseUri, namespace) ->
new ByteArrayInputStream(new byte[0]));
return factory;
}
{
setMaxSizeToParse(10 * 1024 * 1024L); // 10MB
}
public long getMaxSizeToParse() {
return (Long) kp.get("maxSizeToParse");
}
/**
* The maximum number of bytes of an SVG document to parse, after any content encoding such
* as gzip is decoded. Links after this point are not extracted.
*/
public void setMaxSizeToParse(long maxSizeToParse) {
kp.put("maxSizeToParse", maxSizeToParse);
}
/**
* Javascript extractor to use to process inline scripts. Autowired if available. If null,
* links will not be extracted from inline scripts.
*/
transient protected ExtractorJS extractorJS;
public ExtractorJS getExtractorJS() {
return extractorJS;
}
@Autowired(required = false)
public void setExtractorJS(ExtractorJS extractorJS) {
this.extractorJS = extractorJS;
}
/**
* HTML extractor to use to process the content of foreignObject elements. Autowired if
* available. If null, links will not be extracted from foreignObject elements. Its
* obeyRelNofollow setting also applies to {@code <a>} links in the SVG.
*/
transient protected ExtractorHTML extractorHTML;
public ExtractorHTML getExtractorHTML() {
return extractorHTML;
}
@Autowired(required = false)
public void setExtractorHTML(ExtractorHTML extractorHTML) {
this.extractorHTML = extractorHTML;
}
@Override
protected boolean shouldExtract(CrawlURI curi) {
// like browsers, also accept SVG served as generic XML; whether it's actually SVG is
// decided by the root element
return curi.hasContentType("image/svg+xml")
|| curi.hasContentType("text/xml")
|| curi.hasContentType("application/xml");
}
@Override
protected boolean innerExtract(CrawlURI curi) {
boolean isSvg = false;
try (BoundedInputStream in = BoundedInputStream.builder()
.setInputStream(curi.getRecorder().getContentReplayInputStream())
.setMaxCount(getMaxSizeToParse())
.get()) {
try {
XMLStreamReader reader;
if (curi.containsContentTypeCharsetDeclaration()) {
// as in browsers, the charset from the HTTP headers wins
reader = INPUT_FACTORY.createXMLStreamReader(in,
curi.getRecorder().getCharset().name());
} else {
reader = INPUT_FACTORY.createXMLStreamReader(in);
}
try {
isSvg = processRootElement(curi, reader);
if (isSvg) {
extract(curi, reader);
}
} finally {
reader.close();
}
} catch (XMLStreamException e) {
if (!isSvg) {
// not well-formed up to the root element, so leave it to other extractors
return false;
}
// like browsers, keep whatever came before the error
if (in.getRemaining() == 0) {
if (logger.isLoggable(Level.FINE)) {
logger.fine("Stopped parsing SVG at maxSizeToParse: " + curi);
}
} else {
if (logger.isLoggable(Level.FINE)) {
logger.fine("Error parsing SVG " + curi + ": " + e.getMessage());
}
curi.getNonFatalFailures().add(e);
}
}
} catch (IOException e) {
logger.log(Level.WARNING, "Failed reading content of " + curi, e);
curi.getNonFatalFailures().add(e);
return false;
}
// if it's not SVG, leave it to other extractors
return isSvg;
}
/**
* Reads up to the root element. If it's an SVG {@code <svg>} element, extracts links from it
* and from any {@code <?xml-stylesheet?>} processing instructions before it.
*
* @return false if the document isn't SVG
*/
protected boolean processRootElement(CrawlURI curi, XMLStreamReader reader)
throws XMLStreamException {
List<String> stylesheets = new ArrayList<>();
while (reader.hasNext()) {
int event = reader.next();
if (event == XMLStreamConstants.PROCESSING_INSTRUCTION
&& "xml-stylesheet".equals(reader.getPITarget())) {
stylesheets.add(reader.getPIData());
} else if (event == XMLStreamConstants.START_ELEMENT) {
if (!SVG_NS.equals(reader.getNamespaceURI())
|| !reader.getLocalName().equals("svg")) {
return false;
}
for (String data : stylesheets) {
processXmlStylesheet(curi, data);
}
processAttributes(curi, reader, "svg");
return true;
}
}
return false;
}
protected void extract(CrawlURI curi, XMLStreamReader reader) throws XMLStreamException {
StringBuilder text = null; // content of the current <style> or <script>
while (reader.hasNext()) {
int event = reader.next();
switch (event) {
case XMLStreamConstants.START_ELEMENT:
if (!SVG_NS.equals(reader.getNamespaceURI())) {
break;
}
String element = reader.getLocalName();
if (element.equals("foreignObject")) {
processForeignObject(curi, reader);
break;
}
processAttributes(curi, reader, element);
if (element.equals("style") || element.equals("script")) {
text = new StringBuilder();
}
break;
case XMLStreamConstants.CHARACTERS:
case XMLStreamConstants.CDATA:
if (text != null) {
text.append(reader.getText());
}
break;
case XMLStreamConstants.END_ELEMENT:
if (text != null && SVG_NS.equals(reader.getNamespaceURI())) {
if (reader.getLocalName().equals("style")) {
numberOfLinksExtracted.addAndGet(
ExtractorCSS.processStyleCode(this, curi, text));
} else if (reader.getLocalName().equals("script")) {
processScriptCode(curi, text);
}
text = null;
}
break;
default:
break;
}
}
}
protected void processAttributes(CrawlURI curi, XMLStreamReader reader, String element) {
String elementLC = element.toLowerCase(Locale.ROOT);
boolean animatesHref = false;
for (int i = 0; i < reader.getAttributeCount(); i++) {
String namespace = reader.getAttributeNamespace(i);
String name = reader.getAttributeLocalName(i);
String value = reader.getAttributeValue(i);
boolean noNamespace = namespace == null || namespace.isEmpty();
if (name.equals("href") && (noNamespace || namespace.equals(XLINK_NS))) {
String context = noNamespace ? "href" : "xlink:href";
if (elementLC.equals("a")) {
if (!obeyRelNofollow() || !isNofollow(reader.getAttributeValue(null, "rel"))) {
addLink(curi, value, elementLC, context, Hop.NAVLINK);
}
} else if (EMBED_HREF_ELEMENTS.contains(elementLC)) {
addLink(curi, value, elementLC, context, Hop.EMBED);
}
} else if (!noNamespace) {
// other namespaced attributes aren't of interest
} else if (name.equals("style") || URL_PRESENTATION_ATTRIBUTES.contains(name)) {
numberOfLinksExtracted.addAndGet(ExtractorCSS.processStyleCode(this, curi, value));
} else if (name.equals("attributeName")
&& (value.trim().equals("href") || value.trim().equals("xlink:href"))) {
animatesHref = true;
}
}
if (animatesHref && (elementLC.equals("animate") || elementLC.equals("set"))) {
for (String attribute : new String[] {"from", "to", "values"}) {
String value = reader.getAttributeValue(null, attribute);
if (value == null) {
continue;
}
for (String uri : value.split(";")) {
addLink(curi, uri, elementLC, attribute, Hop.EMBED);
}
}
}
}
/**
* Follows the ExtractorHTML bean's obeyRelNofollow setting, if there is one.
*/
private boolean obeyRelNofollow() {
ExtractorHTML html = getExtractorHTML();
return html != null && html.getObeyRelNofollow();
}
private static boolean isNofollow(String rel) {
return rel != null && REL_NOFOLLOW.matcher(rel).find();
}
protected void processXmlStylesheet(CrawlURI curi, String data) {
if (data == null) {
return;
}
Matcher matcher = PSEUDO_ATTRIBUTE_HREF.matcher(data);
if (matcher.find()) {
String href = matcher.group(1) != null ? matcher.group(1) : matcher.group(2);
addLink(curi, StringEscapeUtils.unescapeXml(href), "?xml-stylesheet", "href",
Hop.EMBED);
}
}
/**
* Serializes the content of a foreignObject element and extracts links from it as HTML.
* Leaves the reader positioned at the element's end tag. Without an HTML extractor, the
* content is left for the caller, which ignores elements outside the SVG namespace.
*/
protected void processForeignObject(CrawlURI curi, XMLStreamReader reader)
throws XMLStreamException {
ExtractorHTML html = getExtractorHTML();
if (html == null) {
return;
}
StringWriter buffer = new StringWriter();
XMLStreamWriter writer = OUTPUT_FACTORY.createXMLStreamWriter(buffer);
int depth = 1;
while (depth > 0) {
int event = reader.next();
switch (event) {
case XMLStreamConstants.START_ELEMENT:
depth++;
writer.writeStartElement(
qualifiedName(reader.getPrefix(), reader.getLocalName()));
for (int i = 0; i < reader.getAttributeCount(); i++) {
writer.writeAttribute(
qualifiedName(reader.getAttributePrefix(i),
reader.getAttributeLocalName(i)),
reader.getAttributeValue(i));
}
break;
case XMLStreamConstants.END_ELEMENT:
depth--;
if (depth > 0) {
writer.writeEndElement();
}
break;
case XMLStreamConstants.CHARACTERS:
case XMLStreamConstants.CDATA:
case XMLStreamConstants.SPACE:
writer.writeCharacters(reader.getText());
break;
default:
break;
}
}
writer.close();
html.processSrcdoc(curi, buffer.toString());
}
private static String qualifiedName(String prefix, String localName) {
return prefix == null || prefix.isEmpty() ? localName : prefix + ":" + localName;
}
protected void processScriptCode(CrawlURI curi, CharSequence code) {
if (getExtractorJS() != null) {
numberOfLinksExtracted.addAndGet(getExtractorJS().considerStrings(this, curi, code));
}
}
protected void addLink(CrawlURI curi, String uri, String element, String attribute, Hop hop) {
uri = uri.trim();
if (uri.isEmpty() || uri.startsWith("#")) {
// a reference to an element in this same document
return;
}
try {
HTMLLinkContext context = HTMLLinkContext.get(
ExtractorHTML.elementContext(element, attribute).toString());
addRelativeToBase(curi, getExtractorParameters().getMaxOutlinks(), uri, context, hop);
numberOfLinksExtracted.incrementAndGet();
} catch (URIException e) {
logUriError(e, curi.getUURI(), uri);
}
}
}
@@ -131,8 +131,9 @@ public class JerichoExtractorHTML extends ExtractorHTML {
final boolean overlyEagerLinkDetection = getExtractValueAttributes();
// HREF
if (((attr = attributes.get("href")) != null) &&
// HREF (or XLINK:HREF in inline SVG)
if ((((attr = attributes.get("href")) != null)
|| ((attr = attributes.get("xlink:href")) != null)) &&
((attrValue = attr.getValue()) != null)) {
CharSequence context = elementContext(elementName, attr
.getKey());
@@ -149,6 +150,11 @@ public class JerichoExtractorHTML extends ExtractorHTML {
} else {
processLink(curi, attrValue, context);
}
} else if (ExtractorSVG.EMBED_HREF_ELEMENTS.contains(elementName)) {
// inline SVG resources such as <image href> and <use href>
if (!attrValue.startsWith("#")) {
processEmbed(curi, attrValue, context);
}
} else {
// other HREFs treated as links
processLink(curi, attrValue, context);
@@ -307,7 +313,16 @@ public class JerichoExtractorHTML extends ExtractorHTML {
numberOfLinksExtracted.addAndGet(ExtractorCSS.processStyleCode(
this, curi, attrValue));
}
// inline SVG presentation attributes, e.g. fill="url(...)"
for (String name : ExtractorSVG.URL_PRESENTATION_ATTRIBUTES) {
if (((attr = attributes.get(name)) != null) &&
((attrValue = attr.getValue()) != null)) {
numberOfLinksExtracted.addAndGet(ExtractorCSS.processStyleCode(
this, curi, attrValue));
}
}
// FLASHVARS
if (((attr = attributes.get("flashvars")) != null) &&
((attrValue = attr.getValue()) != null)) {
@@ -22,6 +22,8 @@ package org.archive.modules.extractor;
import static org.archive.modules.extractor.LinkContext.EMBED_MISC;
import static org.archive.modules.extractor.LinkContext.NAVLINK_MISC;
import static org.junit.jupiter.api.Assertions.assertEquals;
import java.util.Collection;
import java.util.Collections;
@@ -29,6 +31,7 @@ import org.archive.modules.CrawlURI;
import org.archive.net.UURI;
import org.archive.net.UURIFactory;
import org.archive.util.Recorder;
import org.junit.jupiter.api.Test;
/**
@@ -89,4 +92,14 @@ public class ExtractorCSSTest extends StringExtractorTestBase {
return VALID_TEST_DATA;
}
@Test
public void testSameDocumentReferenceIgnored() throws Exception {
CrawlURI curi = new CrawlURI(UURIFactory.getInstance("http://www.archive.org/start/"));
long found = ExtractorCSS.processStyleCode(makeExtractor(), curi,
"rect { fill: url(#gradient) } p { background: url(bg.png) }");
assertEquals(1, found);
assertEquals("http://www.archive.org/start/bg.png",
curi.getOutLinks().iterator().next().getURI());
}
}
@@ -974,6 +974,30 @@ public class ExtractorHTMLTest extends StringExtractorTestBase {
assertTrue(!curi.containsDataKey(ExtractorHTML.A_META_ROBOTS), "meta robots");
}
@Test
public void testInlineSvg() throws URIException {
String html = "<svg><image href=/image.png /><use xlink:href='/icons.svg#a'/>" +
"<use href='#local'/>" +
"<filter><feImage href=\"/fe.png\"/></filter>" +
"<rect fill='URL(/fill.svg#g)' stroke='url(#local)'/>" +
"<a href=/page.html><text>x</text></a></svg>";
assertEquals(List.of(
"E https://www.example.org/fe.png feimage/@href",
"E https://www.example.org/fill.svg " + LinkContext.EMBED_MISC,
"E https://www.example.org/icons.svg use/@" + xlinkHrefAttributeContext(),
"E https://www.example.org/image.png image/@href",
"L https://www.example.org/page.html a/@href"),
extractLinks(html));
}
/**
* The attribute name in the context of links from xlink:href attributes. ExtractorHTML's
* regex matches only the 'href' part.
*/
protected String xlinkHrefAttributeContext() {
return "href";
}
/**
* Runs the extractor over the given HTML and returns the outlinks
* formatted as "hop uri context", sorted.
@@ -0,0 +1,340 @@
/*
* This file is part of the Heritrix web crawler (crawler.archive.org).
*
* Licensed to the Internet Archive (IA) by one or more individual
* contributors.
*
* The IA licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.archive.modules.extractor;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertFalse;
import static org.junit.jupiter.api.Assertions.assertTrue;
import java.nio.charset.StandardCharsets;
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
import org.archive.modules.CrawlMetadata;
import org.archive.modules.CrawlURI;
import org.archive.net.UURIFactory;
import org.junit.jupiter.api.Test;
public class ExtractorSVGTest extends ContentExtractorTestBase {
private static final String SVG_START = "<svg xmlns='http://www.w3.org/2000/svg' " +
"xmlns:xlink='http://www.w3.org/1999/xlink'>";
@Override
protected Extractor makeExtractor() {
ExtractorSVG extractor = new ExtractorSVG();
extractor.setLoggerModule(new UnitTestUriLoggerModule());
extractor.setExtractorJS(new ExtractorJS());
ExtractorHTML extractorHTML = new ExtractorHTML();
extractorHTML.setLoggerModule(new UnitTestUriLoggerModule());
CrawlMetadata metadata = new CrawlMetadata();
metadata.afterPropertiesSet();
extractorHTML.setMetadata(metadata);
extractorHTML.setExtractorJS(new ExtractorJS());
extractorHTML.afterPropertiesSet();
extractor.setExtractorHTML(extractorHTML);
return extractor;
}
@Test
public void testHrefs() throws Exception {
assertEquals(List.of(
"E https://example.org/a.png image/@href",
"E https://example.org/b.svg use/@xlink:href",
"E https://example.org/c.png feimage/@href",
"E https://example.org/d.js script/@href",
"E https://example.org/e.svg lineargradient/@xlink:href",
"E https://example.org/f.svg pattern/@href",
"E https://example.org/g.svg filter/@href",
"E https://example.org/h.svg textpath/@href",
"E https://example.org/i.svg mpath/@href",
"E https://example.org/j.svg font-face-uri/@xlink:href",
"E https://example.org/k.mp3 audio/@xlink:href",
"E https://example.org/l.mp4 video/@xlink:href",
"L https://example.org/page.html a/@href",
"L https://example.org/page2.html a/@xlink:href"),
extract(SVG_START +
"<image href='a.png'/>" +
"<use xlink:href='b.svg#icon'/>" +
"<filter><feImage href='c.png'/></filter>" +
"<script href='d.js'/>" +
"<linearGradient xlink:href='e.svg#grad'/>" +
"<pattern href='f.svg#p'/>" +
"<filter href='g.svg#f'/>" +
"<text><textPath href='h.svg#path'>x</textPath></text>" +
"<animateMotion><mpath href='i.svg#path'/></animateMotion>" +
"<font-face><font-face-src><font-face-uri xlink:href='j.svg#font'/>" +
"</font-face-src></font-face>" +
"<audio xlink:href='k.mp3'/><video xlink:href='l.mp4'/>" +
"<a href='page.html'><text>x</text></a>" +
"<a xlink:href='page2.html'><text>y</text></a>" +
"</svg>"));
}
@Test
public void testSameDocumentReferencesIgnored() throws Exception {
assertEquals(List.of(),
extract(SVG_START +
"<linearGradient id='g'/>" +
"<use href='#icon'/>" +
"<rect fill='url(#g)' style='stroke: url(#g)'/>" +
"<a href='#top'><text>x</text></a>" +
"</svg>"));
}
@Test
public void testObeyRelNofollow() throws Exception {
String svg = SVG_START +
"<a href='followed.html' rel='noopener'><text>x</text></a>" +
"<a href='nofollow.html' rel='author NoFollow'><text>y</text></a>" +
"<image rel='nofollow' href='image.png'/>" +
"</svg>";
assertEquals(List.of(
"E https://example.org/image.png image/@href",
"L https://example.org/followed.html a/@href",
"L https://example.org/nofollow.html a/@href"),
extract(svg));
((ExtractorSVG) extractor).getExtractorHTML().setObeyRelNofollow(true);
assertEquals(List.of(
"E https://example.org/image.png image/@href",
"L https://example.org/followed.html a/@href"),
extract(svg));
}
@Test
public void testHrefOnOtherElementsIgnored() throws Exception {
// href isn't an attribute of <rect>, and other namespaces aren't xlink
assertEquals(List.of(),
extract(SVG_START + "<rect href='a.png' xmlns:x='urn:x' x:href='b.png'/></svg>"));
}
@Test
public void testXmlStylesheet() throws Exception {
assertEquals(List.of("E https://example.org/style.css?a=1&b=2 ?xml-stylesheet/@href"),
extract("<?xml version='1.0'?>" +
"<?xml-stylesheet type='text/css' href='style.css?a=1&amp;b=2'?>" +
SVG_START + "</svg>"));
}
@Test
public void testCss() throws Exception {
assertEquals(List.of(
"E https://example.org/clip.svg " + LinkContext.EMBED_MISC,
"E https://example.org/fill.svg " + LinkContext.EMBED_MISC,
"E https://example.org/font.woff " + LinkContext.EMBED_MISC,
"E https://example.org/marker.svg " + LinkContext.EMBED_MISC,
"E https://example.org/style.svg " + LinkContext.EMBED_MISC),
extract(SVG_START +
"<style>@font-face { src: url(font.woff) }</style>" +
"<rect fill='url(fill.svg#g) red' clip-path='url(\"clip.svg#c\")' " +
" marker-start='URL(marker.svg#m)' style='stroke: url(style.svg#s)'/>" +
"<rect width='url(not-a-presentation-attribute.svg)'/>" +
"</svg>"));
}
@Test
public void testCdataStyleAndScript() throws Exception {
assertEquals(List.of(
"E https://example.org/bg.png " + LinkContext.EMBED_MISC,
"X https://example.org/script.html " + LinkContext.JS_MISC),
extract(SVG_START +
"<style><![CDATA[ rect { fill: url(bg.png) } ]]></style>" +
"<script><![CDATA[ if (a < b) location = '/script.html'; ]]></script>" +
"</svg>"));
}
@Test
public void testAnimateHref() throws Exception {
assertEquals(List.of(
"E https://example.org/1.png animate/@values",
"E https://example.org/2.png animate/@values",
"E https://example.org/3.png set/@to",
"E https://example.org/4.png animate/@from",
"E https://example.org/5.png animate/@to"),
extract(SVG_START +
"<image href='#none'>" +
"<animate attributeName='href' values='1.png; 2.png;#x' dur='1s'/>" +
"<set attributeName='xlink:href' to='3.png'/>" +
"<animate attributeName='href' from='4.png' to='5.png'/>" +
"<animate attributeName='x' from='6.png' to='7.png'/>" +
"</image></svg>"));
}
@Test
public void testForeignObject() throws Exception {
assertEquals(List.of(
"E https://example.org/after.png image/@href",
"E https://example.org/inner.png img/@src",
"E https://example.org/nested.png image/@href",
"L https://example.org/inner.html a/@href"),
extract(SVG_START +
"<foreignObject width='100' height='100'>" +
"<div xmlns='http://www.w3.org/1999/xhtml'>" +
"<img src='inner.png' alt='a &lt; b'/><a href='inner.html'>x</a>" +
"<svg xmlns='http://www.w3.org/2000/svg'><image href='nested.png'/></svg>" +
"</div></foreignObject>" +
"<image href='after.png'/>" +
"</svg>"));
}
@Test
public void testForeignObjectWithoutExtractorHTML() throws Exception {
((ExtractorSVG) extractor).setExtractorHTML(null);
assertEquals(List.of(
"E https://example.org/after.png image/@href",
"E https://example.org/nested.png image/@href"),
extract(SVG_START +
"<foreignObject><div xmlns='http://www.w3.org/1999/xhtml'>" +
"<img src='inner.png'/>" +
"<svg xmlns='http://www.w3.org/2000/svg'><image href='nested.png'/></svg>" +
"</div></foreignObject>" +
"<image href='after.png'/>" +
"</svg>"));
}
@Test
public void testNonSvgNamespaceIgnored() throws Exception {
assertEquals(List.of("E https://example.org/b.png image/@href"),
extract(SVG_START + "<x:image xmlns:x='urn:x' href='a.png'/>" +
"<image href='b.png'/></svg>"));
}
@Test
public void testEntitiesFromInternalSubset() throws Exception {
// as written by Adobe Illustrator
assertEquals(List.of("E https://example.org/a.png image/@xlink:href"),
extract("<?xml version='1.0' encoding='utf-8'?>" +
"<!DOCTYPE svg PUBLIC '-//W3C//DTD SVG 1.1//EN' " +
"'http://www.w3.org/Graphics/SVG/1.1/DTD/svg11.dtd' [" +
"<!ENTITY ns_svg 'http://www.w3.org/2000/svg'>" +
"<!ENTITY ns_xlink 'http://www.w3.org/1999/xlink'>" +
"]><svg xmlns='&ns_svg;' xmlns:xlink='&ns_xlink;'>" +
"<image xlink:href='a.png'/></svg>"));
}
@Test
public void testExternalEntityNotLoaded() throws Exception {
assertEquals(List.of("E https://example.org/a.png image/@href"),
extract("<!DOCTYPE svg [<!ENTITY x SYSTEM 'file:///etc/passwd'>]>" +
SVG_START + "<text>&x;</text><image href='a.png'/></svg>"));
}
@Test
public void testMalformedKeepsEarlierLinks() throws Exception {
CrawlURI curi = makeCrawlURI(SVG_START + "<image href='a.png'/><oops attr=unquoted/>" +
"<image href='b.png'/></svg>", "image/svg+xml");
extractor.process(curi);
assertEquals(List.of("E https://example.org/a.png image/@href"), links(curi));
assertTrue(curi.hasBeenLinkExtracted());
assertEquals(1, curi.getNonFatalFailures().size());
}
@Test
public void testMaxSizeToParse() throws Exception {
String start = SVG_START + "<image href='a.png'/>";
((ExtractorSVG) extractor).setMaxSizeToParse(start.length() + 10);
CrawlURI curi = makeCrawlURI(start + "<image href='b.png'/></svg>", "image/svg+xml");
extractor.process(curi);
assertEquals(List.of("E https://example.org/a.png image/@href"), links(curi));
assertTrue(curi.hasBeenLinkExtracted());
assertTrue(curi.getNonFatalFailures().isEmpty(), curi.getNonFatalFailures().toString());
}
@Test
public void testCharsetFromContentType() throws Exception {
String svg = "<?xml version='1.0' encoding='iso-8859-1'?>" + SVG_START +
"<image href='é.png'/></svg>";
CrawlURI curi = makeCrawlURI(svg, "image/svg+xml; charset=UTF-8");
extractor.process(curi);
assertEquals(List.of("E https://example.org/%C3%A9.png image/@href"), links(curi));
}
@Test
public void testShouldExtract() throws Exception {
ExtractorSVG svg = (ExtractorSVG) extractor;
String content = SVG_START + "</svg>";
assertTrue(svg.shouldExtract(makeCrawlURI(content, "image/svg+xml")));
assertTrue(svg.shouldExtract(makeCrawlURI(content, "IMAGE/SVG+XML; charset=utf-8")));
assertTrue(svg.shouldExtract(makeCrawlURI(content, "text/xml")));
assertTrue(svg.shouldExtract(makeCrawlURI(content, "application/xml")));
assertFalse(svg.shouldExtract(makeCrawlURI(content, "text/html")));
}
@Test
public void testGenericXmlWithSvgRoot() throws Exception {
CrawlURI curi = makeCrawlURI("https://example.org/icon?id=3",
SVG_START + "<image href='a.png'/></svg>", "text/xml");
extractor.process(curi);
assertEquals(List.of("E https://example.org/a.png image/@href"), links(curi));
assertTrue(curi.hasBeenLinkExtracted());
}
@Test
public void testNonSvgRootLeftToOtherExtractors() throws Exception {
String[][] documents = {
{"application/xml", "<?xml-stylesheet href='feed.css'?>" +
"<rss><channel><link>https://example.org/post</link></channel></rss>"},
{"text/xml", "<x:svg xmlns:x='urn:x'><image href='a.png'/></x:svg>"},
{"image/svg+xml", "<svg><image href='a.png'/></svg>"}, // no namespace
{"image/svg+xml", "not xml"},
};
for (String[] document : documents) {
CrawlURI curi = makeCrawlURI(document[1], document[0]);
extractor.process(curi);
assertEquals(List.of(), links(curi), document[1]);
assertFalse(curi.hasBeenLinkExtracted(), document[1]);
assertTrue(curi.getNonFatalFailures().isEmpty(), document[1]);
}
}
private List<String> extract(String svg) throws Exception {
CrawlURI curi = makeCrawlURI(svg, "image/svg+xml");
extractor.process(curi);
assertTrue(curi.getNonFatalFailures().isEmpty(), curi.getNonFatalFailures().toString());
return links(curi);
}
private CrawlURI makeCrawlURI(String content, String contentType) throws Exception {
return makeCrawlURI("https://example.org/image.svg", content, contentType);
}
private CrawlURI makeCrawlURI(String uri, String content, String contentType) throws Exception {
CrawlURI curi = new CrawlURI(UURIFactory.getInstance(uri));
curi.setContentType(contentType);
curi.setRecorder(createRecorder(content, "UTF-8"));
curi.setContentSize(content.getBytes(StandardCharsets.UTF_8).length);
curi.setFetchStatus(200);
return curi;
}
/**
* Outlinks formatted as "hop uri context", sorted.
*/
private static List<String> links(CrawlURI curi) {
List<String> links = new ArrayList<>();
for (CrawlURI link : curi.getOutLinks()) {
links.add(link.getLastHop() + " " + link.getURI() + " " + link.getViaContext());
}
Collections.sort(links);
return links;
}
}
@@ -257,4 +257,9 @@ public class JerichoExtractorHTMLTest extends ExtractorHTMLTest {
protected JerichoExtractorHTML getExtractor() {
return (JerichoExtractorHTML) extractor;
}
@Override
protected String xlinkHrefAttributeContext() {
return "xlink:href";
}
}