mirror of
https://github.com/internetarchive/heritrix3.git
synced 2026-11-22 13:47:02 +00:00
Merge pull request #812 from internetarchive/svg-extractor
Add ExtractorSVG for links in SVG images
This commit is contained in:
11 files changed
+844
-3
No files matched your search
@@ -463,6 +463,11 @@ ExtractorSitemap
|
||||
|
||||
.. bean-doc:: org.archive.modules.extractor.ExtractorSitemap
|
||||
|
||||
ExtractorSVG
|
||||
~~~~~~~~~~~~
|
||||
|
||||
.. bean-doc:: org.archive.modules.extractor.ExtractorSVG
|
||||
|
||||
ExtractorSWF
|
||||
~~~~~~~~~~~~
|
||||
|
||||
|
||||
@@ -298,6 +298,8 @@ http://example.example/example
|
||||
<bean id="extractorSitemap" class="org.archive.modules.extractor.ExtractorSitemap">
|
||||
</bean>
|
||||
|
||||
<bean id="extractorSvg" class="org.archive.modules.extractor.ExtractorSVG">
|
||||
</bean>
|
||||
<bean id="extractorHtml" class="org.archive.modules.extractor.ExtractorHTML">
|
||||
<!-- <property name="extractJavascript" value="true" /> -->
|
||||
<!-- <property name="extractValueAttributes" value="true" /> -->
|
||||
@@ -354,6 +356,8 @@ http://example.example/example
|
||||
<ref bean="extractorRobotsTxt"/>
|
||||
<!-- ...extract links from sitemaps... -->
|
||||
<ref bean="extractorSitemap"/>
|
||||
<!-- ...extract outlinks from SVG content... -->
|
||||
<ref bean="extractorSvg"/>
|
||||
<!-- ...extract outlinks from HTML content... -->
|
||||
<ref bean="extractorHtml"/>
|
||||
<!-- ...extract outlinks from CSS content... -->
|
||||
|
||||
@@ -272,6 +272,7 @@ http://example.example/example
|
||||
extractorHttp(ExtractorHTTP)
|
||||
extractorRobotsTxt(ExtractorRobotsTxt)
|
||||
extractorSitemap(ExtractorSitemap)
|
||||
extractorSvg(ExtractorSVG)
|
||||
extractorHtml(ExtractorHTML) {
|
||||
// extractJavascript = true
|
||||
// extractValueAttributes = true
|
||||
@@ -307,6 +308,8 @@ http://example.example/example
|
||||
ref('extractorRobotsTxt'),
|
||||
// ...extract links from sitemaps...
|
||||
ref('extractorSitemap'),
|
||||
// ...extract outlinks from SVG content...
|
||||
ref('extractorSvg'),
|
||||
// ...extract outlinks from HTML content...
|
||||
ref('extractorHtml'),
|
||||
// ...extract outlinks from CSS content...
|
||||
|
||||
@@ -133,6 +133,11 @@ public class ExtractorCSS extends ContentExtractor {
|
||||
// Remove backslashes when used as escape character in CSS URL
|
||||
cssUri = TextUtils.replaceAll(CSS_BACKSLASH_ESCAPE, cssUri,
|
||||
"$1");
|
||||
if (cssUri.startsWith("#")) {
|
||||
// a reference to an element in the document itself,
|
||||
// e.g. an SVG gradient: fill: url(#gradient)
|
||||
continue;
|
||||
}
|
||||
foundLinks++;
|
||||
int max = ext.getExtractorParameters().getMaxOutlinks();
|
||||
try {
|
||||
|
||||
@@ -473,6 +473,12 @@ public class ExtractorHTML extends ContentExtractor implements InitializingBean
|
||||
linkContext = context;
|
||||
} else if ("a[data-remote='true']/@href".equals(context)) {
|
||||
processEmbed(curi, value, context);
|
||||
} else if (ExtractorSVG.EMBED_HREF_ELEMENTS.contains(
|
||||
elementStr.toLowerCase(Locale.ROOT))) {
|
||||
// inline SVG resources such as <image href> and <use href>
|
||||
if (!value.toString().startsWith("#")) {
|
||||
processEmbed(curi, value, context);
|
||||
}
|
||||
} else {
|
||||
// other HREFs treated as links
|
||||
processLink(curi, value, context);
|
||||
@@ -588,6 +594,11 @@ public class ExtractorHTML extends ContentExtractor implements InitializingBean
|
||||
&& elementStr.equalsIgnoreCase(IFRAME)) {
|
||||
// embedded HTML document
|
||||
processSrcdoc(curi, value);
|
||||
} else if (ExtractorSVG.URL_PRESENTATION_ATTRIBUTES.contains(
|
||||
attrName.toString().toLowerCase(Locale.ROOT))) {
|
||||
// inline SVG presentation attribute, e.g. fill="url(...)"
|
||||
numberOfLinksExtracted.addAndGet(ExtractorCSS.processStyleCode(
|
||||
this, curi, value));
|
||||
}
|
||||
|
||||
// 2023 updates get img or source data attr
|
||||
|
||||
@@ -0,0 +1,416 @@
|
||||
/*
|
||||
* This file is part of the Heritrix web crawler (crawler.archive.org).
|
||||
*
|
||||
* Licensed to the Internet Archive (IA) by one or more individual
|
||||
* contributors.
|
||||
*
|
||||
* The IA licenses this file to You under the Apache License, Version 2.0
|
||||
* (the "License"); you may not use this file except in compliance with
|
||||
* the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
package org.archive.modules.extractor;
|
||||
|
||||
import java.io.ByteArrayInputStream;
|
||||
import java.io.IOException;
|
||||
import java.io.InputStream;
|
||||
import java.io.StringWriter;
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
import java.util.Locale;
|
||||
import java.util.Set;
|
||||
import java.util.logging.Level;
|
||||
import java.util.logging.Logger;
|
||||
import java.util.regex.Matcher;
|
||||
import java.util.regex.Pattern;
|
||||
|
||||
import javax.xml.stream.XMLInputFactory;
|
||||
import javax.xml.stream.XMLOutputFactory;
|
||||
import javax.xml.stream.XMLStreamConstants;
|
||||
import javax.xml.stream.XMLStreamException;
|
||||
import javax.xml.stream.XMLStreamReader;
|
||||
import javax.xml.stream.XMLStreamWriter;
|
||||
|
||||
import org.apache.commons.io.input.BoundedInputStream;
|
||||
import org.apache.commons.lang3.StringEscapeUtils;
|
||||
import org.archive.modules.CrawlURI;
|
||||
import org.archive.url.URIException;
|
||||
import org.springframework.beans.factory.annotation.Autowired;
|
||||
|
||||
/**
|
||||
* Extracts links from SVG images.
|
||||
*
|
||||
* <p>Handles responses with the content type {@code image/svg+xml}, and also {@code text/xml}
|
||||
* and {@code application/xml} responses that turn out to be SVG. Other XML documents are left
|
||||
* for later extractors, so this extractor should come before any other extractor that handles
|
||||
* XML, such as ExtractorXML.
|
||||
*
|
||||
* <p>Images, stylesheets, scripts, fonts and other resources the SVG loads are extracted as
|
||||
* embedded links, and {@code <a>} links as navigation links. Links in inline CSS are extracted
|
||||
* too. Inline scripts and HTML embedded in the SVG are only processed if the crawl has an
|
||||
* ExtractorJS and an ExtractorHTML bean, respectively, and use their settings.
|
||||
*
|
||||
* <p>As in browsers, nothing is extracted after an XML syntax error, or from a document without
|
||||
* the SVG namespace declaration. Only the first {@code maxSizeToParse} bytes are read.
|
||||
*/
|
||||
public class ExtractorSVG extends ContentExtractor {
|
||||
|
||||
private static final Logger logger = Logger.getLogger(ExtractorSVG.class.getName());
|
||||
|
||||
static final String SVG_NS = "http://www.w3.org/2000/svg";
|
||||
static final String XLINK_NS = "http://www.w3.org/1999/xlink";
|
||||
|
||||
/**
|
||||
* Elements (lower case) whose {@code href} references a resource that's loaded as part of
|
||||
* rendering the document, rather than navigated to.
|
||||
*/
|
||||
static final Set<String> EMBED_HREF_ELEMENTS = Set.of(
|
||||
"audio", "cursor", "feimage", "filter", "font-face-uri", "image", "lineargradient",
|
||||
"mpath", "pattern", "radialgradient", "script", "textpath", "tref", "use",
|
||||
"video"); // audio and video are from SVG Tiny 1.2, which browsers don't implement
|
||||
|
||||
/**
|
||||
* Presentation attributes (lower case) that can take a CSS {@code url(...)} value.
|
||||
*/
|
||||
static final Set<String> URL_PRESENTATION_ATTRIBUTES = Set.of(
|
||||
"clip-path", "cursor", "fill", "filter", "marker-end", "marker-mid", "marker-start",
|
||||
"mask", "stroke");
|
||||
|
||||
private static final Pattern PSEUDO_ATTRIBUTE_HREF =
|
||||
Pattern.compile("(?:^|\\s)href\\s*=\\s*(?:\"([^\"]*)\"|'([^']*)')");
|
||||
|
||||
/** rel is a set of space-separated, case-insensitive tokens */
|
||||
private static final Pattern REL_NOFOLLOW =
|
||||
Pattern.compile("(?i)(?:^|\\s)nofollow(?:\\s|$)");
|
||||
|
||||
private static final XMLInputFactory INPUT_FACTORY = newInputFactory();
|
||||
private static final XMLOutputFactory OUTPUT_FACTORY = XMLOutputFactory.newFactory();
|
||||
|
||||
private static XMLInputFactory newInputFactory() {
|
||||
XMLInputFactory factory = XMLInputFactory.newFactory();
|
||||
factory.setProperty(XMLInputFactory.IS_NAMESPACE_AWARE, true);
|
||||
factory.setProperty(XMLInputFactory.IS_COALESCING, true);
|
||||
// the internal DTD subset is still processed, as some editors (e.g. Adobe Illustrator)
|
||||
// declare entities there for the namespace URIs
|
||||
factory.setProperty(XMLInputFactory.IS_SUPPORTING_EXTERNAL_ENTITIES, false);
|
||||
// never fetch the external DTD subset
|
||||
factory.setXMLResolver((publicId, systemId, baseUri, namespace) ->
|
||||
new ByteArrayInputStream(new byte[0]));
|
||||
return factory;
|
||||
}
|
||||
|
||||
{
|
||||
setMaxSizeToParse(10 * 1024 * 1024L); // 10MB
|
||||
}
|
||||
public long getMaxSizeToParse() {
|
||||
return (Long) kp.get("maxSizeToParse");
|
||||
}
|
||||
/**
|
||||
* The maximum number of bytes of an SVG document to parse, after any content encoding such
|
||||
* as gzip is decoded. Links after this point are not extracted.
|
||||
*/
|
||||
public void setMaxSizeToParse(long maxSizeToParse) {
|
||||
kp.put("maxSizeToParse", maxSizeToParse);
|
||||
}
|
||||
|
||||
/**
|
||||
* Javascript extractor to use to process inline scripts. Autowired if available. If null,
|
||||
* links will not be extracted from inline scripts.
|
||||
*/
|
||||
transient protected ExtractorJS extractorJS;
|
||||
public ExtractorJS getExtractorJS() {
|
||||
return extractorJS;
|
||||
}
|
||||
@Autowired(required = false)
|
||||
public void setExtractorJS(ExtractorJS extractorJS) {
|
||||
this.extractorJS = extractorJS;
|
||||
}
|
||||
|
||||
/**
|
||||
* HTML extractor to use to process the content of foreignObject elements. Autowired if
|
||||
* available. If null, links will not be extracted from foreignObject elements. Its
|
||||
* obeyRelNofollow setting also applies to {@code <a>} links in the SVG.
|
||||
*/
|
||||
transient protected ExtractorHTML extractorHTML;
|
||||
public ExtractorHTML getExtractorHTML() {
|
||||
return extractorHTML;
|
||||
}
|
||||
@Autowired(required = false)
|
||||
public void setExtractorHTML(ExtractorHTML extractorHTML) {
|
||||
this.extractorHTML = extractorHTML;
|
||||
}
|
||||
|
||||
@Override
|
||||
protected boolean shouldExtract(CrawlURI curi) {
|
||||
// like browsers, also accept SVG served as generic XML; whether it's actually SVG is
|
||||
// decided by the root element
|
||||
return curi.hasContentType("image/svg+xml")
|
||||
|| curi.hasContentType("text/xml")
|
||||
|| curi.hasContentType("application/xml");
|
||||
}
|
||||
|
||||
@Override
|
||||
protected boolean innerExtract(CrawlURI curi) {
|
||||
boolean isSvg = false;
|
||||
try (BoundedInputStream in = BoundedInputStream.builder()
|
||||
.setInputStream(curi.getRecorder().getContentReplayInputStream())
|
||||
.setMaxCount(getMaxSizeToParse())
|
||||
.get()) {
|
||||
try {
|
||||
XMLStreamReader reader;
|
||||
if (curi.containsContentTypeCharsetDeclaration()) {
|
||||
// as in browsers, the charset from the HTTP headers wins
|
||||
reader = INPUT_FACTORY.createXMLStreamReader(in,
|
||||
curi.getRecorder().getCharset().name());
|
||||
} else {
|
||||
reader = INPUT_FACTORY.createXMLStreamReader(in);
|
||||
}
|
||||
try {
|
||||
isSvg = processRootElement(curi, reader);
|
||||
if (isSvg) {
|
||||
extract(curi, reader);
|
||||
}
|
||||
} finally {
|
||||
reader.close();
|
||||
}
|
||||
} catch (XMLStreamException e) {
|
||||
if (!isSvg) {
|
||||
// not well-formed up to the root element, so leave it to other extractors
|
||||
return false;
|
||||
}
|
||||
// like browsers, keep whatever came before the error
|
||||
if (in.getRemaining() == 0) {
|
||||
if (logger.isLoggable(Level.FINE)) {
|
||||
logger.fine("Stopped parsing SVG at maxSizeToParse: " + curi);
|
||||
}
|
||||
} else {
|
||||
if (logger.isLoggable(Level.FINE)) {
|
||||
logger.fine("Error parsing SVG " + curi + ": " + e.getMessage());
|
||||
}
|
||||
curi.getNonFatalFailures().add(e);
|
||||
}
|
||||
}
|
||||
} catch (IOException e) {
|
||||
logger.log(Level.WARNING, "Failed reading content of " + curi, e);
|
||||
curi.getNonFatalFailures().add(e);
|
||||
return false;
|
||||
}
|
||||
// if it's not SVG, leave it to other extractors
|
||||
return isSvg;
|
||||
}
|
||||
|
||||
/**
|
||||
* Reads up to the root element. If it's an SVG {@code <svg>} element, extracts links from it
|
||||
* and from any {@code <?xml-stylesheet?>} processing instructions before it.
|
||||
*
|
||||
* @return false if the document isn't SVG
|
||||
*/
|
||||
protected boolean processRootElement(CrawlURI curi, XMLStreamReader reader)
|
||||
throws XMLStreamException {
|
||||
List<String> stylesheets = new ArrayList<>();
|
||||
while (reader.hasNext()) {
|
||||
int event = reader.next();
|
||||
if (event == XMLStreamConstants.PROCESSING_INSTRUCTION
|
||||
&& "xml-stylesheet".equals(reader.getPITarget())) {
|
||||
stylesheets.add(reader.getPIData());
|
||||
} else if (event == XMLStreamConstants.START_ELEMENT) {
|
||||
if (!SVG_NS.equals(reader.getNamespaceURI())
|
||||
|| !reader.getLocalName().equals("svg")) {
|
||||
return false;
|
||||
}
|
||||
for (String data : stylesheets) {
|
||||
processXmlStylesheet(curi, data);
|
||||
}
|
||||
processAttributes(curi, reader, "svg");
|
||||
return true;
|
||||
}
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
protected void extract(CrawlURI curi, XMLStreamReader reader) throws XMLStreamException {
|
||||
StringBuilder text = null; // content of the current <style> or <script>
|
||||
while (reader.hasNext()) {
|
||||
int event = reader.next();
|
||||
switch (event) {
|
||||
case XMLStreamConstants.START_ELEMENT:
|
||||
if (!SVG_NS.equals(reader.getNamespaceURI())) {
|
||||
break;
|
||||
}
|
||||
String element = reader.getLocalName();
|
||||
if (element.equals("foreignObject")) {
|
||||
processForeignObject(curi, reader);
|
||||
break;
|
||||
}
|
||||
processAttributes(curi, reader, element);
|
||||
if (element.equals("style") || element.equals("script")) {
|
||||
text = new StringBuilder();
|
||||
}
|
||||
break;
|
||||
case XMLStreamConstants.CHARACTERS:
|
||||
case XMLStreamConstants.CDATA:
|
||||
if (text != null) {
|
||||
text.append(reader.getText());
|
||||
}
|
||||
break;
|
||||
case XMLStreamConstants.END_ELEMENT:
|
||||
if (text != null && SVG_NS.equals(reader.getNamespaceURI())) {
|
||||
if (reader.getLocalName().equals("style")) {
|
||||
numberOfLinksExtracted.addAndGet(
|
||||
ExtractorCSS.processStyleCode(this, curi, text));
|
||||
} else if (reader.getLocalName().equals("script")) {
|
||||
processScriptCode(curi, text);
|
||||
}
|
||||
text = null;
|
||||
}
|
||||
break;
|
||||
default:
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
protected void processAttributes(CrawlURI curi, XMLStreamReader reader, String element) {
|
||||
String elementLC = element.toLowerCase(Locale.ROOT);
|
||||
boolean animatesHref = false;
|
||||
for (int i = 0; i < reader.getAttributeCount(); i++) {
|
||||
String namespace = reader.getAttributeNamespace(i);
|
||||
String name = reader.getAttributeLocalName(i);
|
||||
String value = reader.getAttributeValue(i);
|
||||
boolean noNamespace = namespace == null || namespace.isEmpty();
|
||||
if (name.equals("href") && (noNamespace || namespace.equals(XLINK_NS))) {
|
||||
String context = noNamespace ? "href" : "xlink:href";
|
||||
if (elementLC.equals("a")) {
|
||||
if (!obeyRelNofollow() || !isNofollow(reader.getAttributeValue(null, "rel"))) {
|
||||
addLink(curi, value, elementLC, context, Hop.NAVLINK);
|
||||
}
|
||||
} else if (EMBED_HREF_ELEMENTS.contains(elementLC)) {
|
||||
addLink(curi, value, elementLC, context, Hop.EMBED);
|
||||
}
|
||||
} else if (!noNamespace) {
|
||||
// other namespaced attributes aren't of interest
|
||||
} else if (name.equals("style") || URL_PRESENTATION_ATTRIBUTES.contains(name)) {
|
||||
numberOfLinksExtracted.addAndGet(ExtractorCSS.processStyleCode(this, curi, value));
|
||||
} else if (name.equals("attributeName")
|
||||
&& (value.trim().equals("href") || value.trim().equals("xlink:href"))) {
|
||||
animatesHref = true;
|
||||
}
|
||||
}
|
||||
|
||||
if (animatesHref && (elementLC.equals("animate") || elementLC.equals("set"))) {
|
||||
for (String attribute : new String[] {"from", "to", "values"}) {
|
||||
String value = reader.getAttributeValue(null, attribute);
|
||||
if (value == null) {
|
||||
continue;
|
||||
}
|
||||
for (String uri : value.split(";")) {
|
||||
addLink(curi, uri, elementLC, attribute, Hop.EMBED);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Follows the ExtractorHTML bean's obeyRelNofollow setting, if there is one.
|
||||
*/
|
||||
private boolean obeyRelNofollow() {
|
||||
ExtractorHTML html = getExtractorHTML();
|
||||
return html != null && html.getObeyRelNofollow();
|
||||
}
|
||||
|
||||
private static boolean isNofollow(String rel) {
|
||||
return rel != null && REL_NOFOLLOW.matcher(rel).find();
|
||||
}
|
||||
|
||||
protected void processXmlStylesheet(CrawlURI curi, String data) {
|
||||
if (data == null) {
|
||||
return;
|
||||
}
|
||||
Matcher matcher = PSEUDO_ATTRIBUTE_HREF.matcher(data);
|
||||
if (matcher.find()) {
|
||||
String href = matcher.group(1) != null ? matcher.group(1) : matcher.group(2);
|
||||
addLink(curi, StringEscapeUtils.unescapeXml(href), "?xml-stylesheet", "href",
|
||||
Hop.EMBED);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Serializes the content of a foreignObject element and extracts links from it as HTML.
|
||||
* Leaves the reader positioned at the element's end tag. Without an HTML extractor, the
|
||||
* content is left for the caller, which ignores elements outside the SVG namespace.
|
||||
*/
|
||||
protected void processForeignObject(CrawlURI curi, XMLStreamReader reader)
|
||||
throws XMLStreamException {
|
||||
ExtractorHTML html = getExtractorHTML();
|
||||
if (html == null) {
|
||||
return;
|
||||
}
|
||||
StringWriter buffer = new StringWriter();
|
||||
XMLStreamWriter writer = OUTPUT_FACTORY.createXMLStreamWriter(buffer);
|
||||
int depth = 1;
|
||||
while (depth > 0) {
|
||||
int event = reader.next();
|
||||
switch (event) {
|
||||
case XMLStreamConstants.START_ELEMENT:
|
||||
depth++;
|
||||
writer.writeStartElement(
|
||||
qualifiedName(reader.getPrefix(), reader.getLocalName()));
|
||||
for (int i = 0; i < reader.getAttributeCount(); i++) {
|
||||
writer.writeAttribute(
|
||||
qualifiedName(reader.getAttributePrefix(i),
|
||||
reader.getAttributeLocalName(i)),
|
||||
reader.getAttributeValue(i));
|
||||
}
|
||||
break;
|
||||
case XMLStreamConstants.END_ELEMENT:
|
||||
depth--;
|
||||
if (depth > 0) {
|
||||
writer.writeEndElement();
|
||||
}
|
||||
break;
|
||||
case XMLStreamConstants.CHARACTERS:
|
||||
case XMLStreamConstants.CDATA:
|
||||
case XMLStreamConstants.SPACE:
|
||||
writer.writeCharacters(reader.getText());
|
||||
break;
|
||||
default:
|
||||
break;
|
||||
}
|
||||
}
|
||||
writer.close();
|
||||
html.processSrcdoc(curi, buffer.toString());
|
||||
}
|
||||
|
||||
private static String qualifiedName(String prefix, String localName) {
|
||||
return prefix == null || prefix.isEmpty() ? localName : prefix + ":" + localName;
|
||||
}
|
||||
|
||||
protected void processScriptCode(CrawlURI curi, CharSequence code) {
|
||||
if (getExtractorJS() != null) {
|
||||
numberOfLinksExtracted.addAndGet(getExtractorJS().considerStrings(this, curi, code));
|
||||
}
|
||||
}
|
||||
|
||||
protected void addLink(CrawlURI curi, String uri, String element, String attribute, Hop hop) {
|
||||
uri = uri.trim();
|
||||
if (uri.isEmpty() || uri.startsWith("#")) {
|
||||
// a reference to an element in this same document
|
||||
return;
|
||||
}
|
||||
try {
|
||||
HTMLLinkContext context = HTMLLinkContext.get(
|
||||
ExtractorHTML.elementContext(element, attribute).toString());
|
||||
addRelativeToBase(curi, getExtractorParameters().getMaxOutlinks(), uri, context, hop);
|
||||
numberOfLinksExtracted.incrementAndGet();
|
||||
} catch (URIException e) {
|
||||
logUriError(e, curi.getUURI(), uri);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -131,8 +131,9 @@ public class JerichoExtractorHTML extends ExtractorHTML {
|
||||
|
||||
final boolean overlyEagerLinkDetection = getExtractValueAttributes();
|
||||
|
||||
// HREF
|
||||
if (((attr = attributes.get("href")) != null) &&
|
||||
// HREF (or XLINK:HREF in inline SVG)
|
||||
if ((((attr = attributes.get("href")) != null)
|
||||
|| ((attr = attributes.get("xlink:href")) != null)) &&
|
||||
((attrValue = attr.getValue()) != null)) {
|
||||
CharSequence context = elementContext(elementName, attr
|
||||
.getKey());
|
||||
@@ -149,6 +150,11 @@ public class JerichoExtractorHTML extends ExtractorHTML {
|
||||
} else {
|
||||
processLink(curi, attrValue, context);
|
||||
}
|
||||
} else if (ExtractorSVG.EMBED_HREF_ELEMENTS.contains(elementName)) {
|
||||
// inline SVG resources such as <image href> and <use href>
|
||||
if (!attrValue.startsWith("#")) {
|
||||
processEmbed(curi, attrValue, context);
|
||||
}
|
||||
} else {
|
||||
// other HREFs treated as links
|
||||
processLink(curi, attrValue, context);
|
||||
@@ -307,7 +313,16 @@ public class JerichoExtractorHTML extends ExtractorHTML {
|
||||
numberOfLinksExtracted.addAndGet(ExtractorCSS.processStyleCode(
|
||||
this, curi, attrValue));
|
||||
}
|
||||
|
||||
|
||||
// inline SVG presentation attributes, e.g. fill="url(...)"
|
||||
for (String name : ExtractorSVG.URL_PRESENTATION_ATTRIBUTES) {
|
||||
if (((attr = attributes.get(name)) != null) &&
|
||||
((attrValue = attr.getValue()) != null)) {
|
||||
numberOfLinksExtracted.addAndGet(ExtractorCSS.processStyleCode(
|
||||
this, curi, attrValue));
|
||||
}
|
||||
}
|
||||
|
||||
// FLASHVARS
|
||||
if (((attr = attributes.get("flashvars")) != null) &&
|
||||
((attrValue = attr.getValue()) != null)) {
|
||||
|
||||
@@ -22,6 +22,8 @@ package org.archive.modules.extractor;
|
||||
import static org.archive.modules.extractor.LinkContext.EMBED_MISC;
|
||||
import static org.archive.modules.extractor.LinkContext.NAVLINK_MISC;
|
||||
|
||||
import static org.junit.jupiter.api.Assertions.assertEquals;
|
||||
|
||||
import java.util.Collection;
|
||||
import java.util.Collections;
|
||||
|
||||
@@ -29,6 +31,7 @@ import org.archive.modules.CrawlURI;
|
||||
import org.archive.net.UURI;
|
||||
import org.archive.net.UURIFactory;
|
||||
import org.archive.util.Recorder;
|
||||
import org.junit.jupiter.api.Test;
|
||||
|
||||
|
||||
/**
|
||||
@@ -89,4 +92,14 @@ public class ExtractorCSSTest extends StringExtractorTestBase {
|
||||
return VALID_TEST_DATA;
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testSameDocumentReferenceIgnored() throws Exception {
|
||||
CrawlURI curi = new CrawlURI(UURIFactory.getInstance("http://www.archive.org/start/"));
|
||||
long found = ExtractorCSS.processStyleCode(makeExtractor(), curi,
|
||||
"rect { fill: url(#gradient) } p { background: url(bg.png) }");
|
||||
assertEquals(1, found);
|
||||
assertEquals("http://www.archive.org/start/bg.png",
|
||||
curi.getOutLinks().iterator().next().getURI());
|
||||
}
|
||||
|
||||
}
|
||||
@@ -974,6 +974,30 @@ public class ExtractorHTMLTest extends StringExtractorTestBase {
|
||||
assertTrue(!curi.containsDataKey(ExtractorHTML.A_META_ROBOTS), "meta robots");
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testInlineSvg() throws URIException {
|
||||
String html = "<svg><image href=/image.png /><use xlink:href='/icons.svg#a'/>" +
|
||||
"<use href='#local'/>" +
|
||||
"<filter><feImage href=\"/fe.png\"/></filter>" +
|
||||
"<rect fill='URL(/fill.svg#g)' stroke='url(#local)'/>" +
|
||||
"<a href=/page.html><text>x</text></a></svg>";
|
||||
assertEquals(List.of(
|
||||
"E https://www.example.org/fe.png feimage/@href",
|
||||
"E https://www.example.org/fill.svg " + LinkContext.EMBED_MISC,
|
||||
"E https://www.example.org/icons.svg use/@" + xlinkHrefAttributeContext(),
|
||||
"E https://www.example.org/image.png image/@href",
|
||||
"L https://www.example.org/page.html a/@href"),
|
||||
extractLinks(html));
|
||||
}
|
||||
|
||||
/**
|
||||
* The attribute name in the context of links from xlink:href attributes. ExtractorHTML's
|
||||
* regex matches only the 'href' part.
|
||||
*/
|
||||
protected String xlinkHrefAttributeContext() {
|
||||
return "href";
|
||||
}
|
||||
|
||||
/**
|
||||
* Runs the extractor over the given HTML and returns the outlinks
|
||||
* formatted as "hop uri context", sorted.
|
||||
|
||||
@@ -0,0 +1,340 @@
|
||||
/*
|
||||
* This file is part of the Heritrix web crawler (crawler.archive.org).
|
||||
*
|
||||
* Licensed to the Internet Archive (IA) by one or more individual
|
||||
* contributors.
|
||||
*
|
||||
* The IA licenses this file to You under the Apache License, Version 2.0
|
||||
* (the "License"); you may not use this file except in compliance with
|
||||
* the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
package org.archive.modules.extractor;
|
||||
|
||||
import static org.junit.jupiter.api.Assertions.assertEquals;
|
||||
import static org.junit.jupiter.api.Assertions.assertFalse;
|
||||
import static org.junit.jupiter.api.Assertions.assertTrue;
|
||||
|
||||
import java.nio.charset.StandardCharsets;
|
||||
import java.util.ArrayList;
|
||||
import java.util.Collections;
|
||||
import java.util.List;
|
||||
|
||||
import org.archive.modules.CrawlMetadata;
|
||||
import org.archive.modules.CrawlURI;
|
||||
import org.archive.net.UURIFactory;
|
||||
import org.junit.jupiter.api.Test;
|
||||
|
||||
public class ExtractorSVGTest extends ContentExtractorTestBase {
|
||||
|
||||
private static final String SVG_START = "<svg xmlns='http://www.w3.org/2000/svg' " +
|
||||
"xmlns:xlink='http://www.w3.org/1999/xlink'>";
|
||||
|
||||
@Override
|
||||
protected Extractor makeExtractor() {
|
||||
ExtractorSVG extractor = new ExtractorSVG();
|
||||
extractor.setLoggerModule(new UnitTestUriLoggerModule());
|
||||
extractor.setExtractorJS(new ExtractorJS());
|
||||
|
||||
ExtractorHTML extractorHTML = new ExtractorHTML();
|
||||
extractorHTML.setLoggerModule(new UnitTestUriLoggerModule());
|
||||
CrawlMetadata metadata = new CrawlMetadata();
|
||||
metadata.afterPropertiesSet();
|
||||
extractorHTML.setMetadata(metadata);
|
||||
extractorHTML.setExtractorJS(new ExtractorJS());
|
||||
extractorHTML.afterPropertiesSet();
|
||||
extractor.setExtractorHTML(extractorHTML);
|
||||
return extractor;
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testHrefs() throws Exception {
|
||||
assertEquals(List.of(
|
||||
"E https://example.org/a.png image/@href",
|
||||
"E https://example.org/b.svg use/@xlink:href",
|
||||
"E https://example.org/c.png feimage/@href",
|
||||
"E https://example.org/d.js script/@href",
|
||||
"E https://example.org/e.svg lineargradient/@xlink:href",
|
||||
"E https://example.org/f.svg pattern/@href",
|
||||
"E https://example.org/g.svg filter/@href",
|
||||
"E https://example.org/h.svg textpath/@href",
|
||||
"E https://example.org/i.svg mpath/@href",
|
||||
"E https://example.org/j.svg font-face-uri/@xlink:href",
|
||||
"E https://example.org/k.mp3 audio/@xlink:href",
|
||||
"E https://example.org/l.mp4 video/@xlink:href",
|
||||
"L https://example.org/page.html a/@href",
|
||||
"L https://example.org/page2.html a/@xlink:href"),
|
||||
extract(SVG_START +
|
||||
"<image href='a.png'/>" +
|
||||
"<use xlink:href='b.svg#icon'/>" +
|
||||
"<filter><feImage href='c.png'/></filter>" +
|
||||
"<script href='d.js'/>" +
|
||||
"<linearGradient xlink:href='e.svg#grad'/>" +
|
||||
"<pattern href='f.svg#p'/>" +
|
||||
"<filter href='g.svg#f'/>" +
|
||||
"<text><textPath href='h.svg#path'>x</textPath></text>" +
|
||||
"<animateMotion><mpath href='i.svg#path'/></animateMotion>" +
|
||||
"<font-face><font-face-src><font-face-uri xlink:href='j.svg#font'/>" +
|
||||
"</font-face-src></font-face>" +
|
||||
"<audio xlink:href='k.mp3'/><video xlink:href='l.mp4'/>" +
|
||||
"<a href='page.html'><text>x</text></a>" +
|
||||
"<a xlink:href='page2.html'><text>y</text></a>" +
|
||||
"</svg>"));
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testSameDocumentReferencesIgnored() throws Exception {
|
||||
assertEquals(List.of(),
|
||||
extract(SVG_START +
|
||||
"<linearGradient id='g'/>" +
|
||||
"<use href='#icon'/>" +
|
||||
"<rect fill='url(#g)' style='stroke: url(#g)'/>" +
|
||||
"<a href='#top'><text>x</text></a>" +
|
||||
"</svg>"));
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testObeyRelNofollow() throws Exception {
|
||||
String svg = SVG_START +
|
||||
"<a href='followed.html' rel='noopener'><text>x</text></a>" +
|
||||
"<a href='nofollow.html' rel='author NoFollow'><text>y</text></a>" +
|
||||
"<image rel='nofollow' href='image.png'/>" +
|
||||
"</svg>";
|
||||
assertEquals(List.of(
|
||||
"E https://example.org/image.png image/@href",
|
||||
"L https://example.org/followed.html a/@href",
|
||||
"L https://example.org/nofollow.html a/@href"),
|
||||
extract(svg));
|
||||
|
||||
((ExtractorSVG) extractor).getExtractorHTML().setObeyRelNofollow(true);
|
||||
assertEquals(List.of(
|
||||
"E https://example.org/image.png image/@href",
|
||||
"L https://example.org/followed.html a/@href"),
|
||||
extract(svg));
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testHrefOnOtherElementsIgnored() throws Exception {
|
||||
// href isn't an attribute of <rect>, and other namespaces aren't xlink
|
||||
assertEquals(List.of(),
|
||||
extract(SVG_START + "<rect href='a.png' xmlns:x='urn:x' x:href='b.png'/></svg>"));
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testXmlStylesheet() throws Exception {
|
||||
assertEquals(List.of("E https://example.org/style.css?a=1&b=2 ?xml-stylesheet/@href"),
|
||||
extract("<?xml version='1.0'?>" +
|
||||
"<?xml-stylesheet type='text/css' href='style.css?a=1&b=2'?>" +
|
||||
SVG_START + "</svg>"));
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testCss() throws Exception {
|
||||
assertEquals(List.of(
|
||||
"E https://example.org/clip.svg " + LinkContext.EMBED_MISC,
|
||||
"E https://example.org/fill.svg " + LinkContext.EMBED_MISC,
|
||||
"E https://example.org/font.woff " + LinkContext.EMBED_MISC,
|
||||
"E https://example.org/marker.svg " + LinkContext.EMBED_MISC,
|
||||
"E https://example.org/style.svg " + LinkContext.EMBED_MISC),
|
||||
extract(SVG_START +
|
||||
"<style>@font-face { src: url(font.woff) }</style>" +
|
||||
"<rect fill='url(fill.svg#g) red' clip-path='url(\"clip.svg#c\")' " +
|
||||
" marker-start='URL(marker.svg#m)' style='stroke: url(style.svg#s)'/>" +
|
||||
"<rect width='url(not-a-presentation-attribute.svg)'/>" +
|
||||
"</svg>"));
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testCdataStyleAndScript() throws Exception {
|
||||
assertEquals(List.of(
|
||||
"E https://example.org/bg.png " + LinkContext.EMBED_MISC,
|
||||
"X https://example.org/script.html " + LinkContext.JS_MISC),
|
||||
extract(SVG_START +
|
||||
"<style><![CDATA[ rect { fill: url(bg.png) } ]]></style>" +
|
||||
"<script><![CDATA[ if (a < b) location = '/script.html'; ]]></script>" +
|
||||
"</svg>"));
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testAnimateHref() throws Exception {
|
||||
assertEquals(List.of(
|
||||
"E https://example.org/1.png animate/@values",
|
||||
"E https://example.org/2.png animate/@values",
|
||||
"E https://example.org/3.png set/@to",
|
||||
"E https://example.org/4.png animate/@from",
|
||||
"E https://example.org/5.png animate/@to"),
|
||||
extract(SVG_START +
|
||||
"<image href='#none'>" +
|
||||
"<animate attributeName='href' values='1.png; 2.png;#x' dur='1s'/>" +
|
||||
"<set attributeName='xlink:href' to='3.png'/>" +
|
||||
"<animate attributeName='href' from='4.png' to='5.png'/>" +
|
||||
"<animate attributeName='x' from='6.png' to='7.png'/>" +
|
||||
"</image></svg>"));
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testForeignObject() throws Exception {
|
||||
assertEquals(List.of(
|
||||
"E https://example.org/after.png image/@href",
|
||||
"E https://example.org/inner.png img/@src",
|
||||
"E https://example.org/nested.png image/@href",
|
||||
"L https://example.org/inner.html a/@href"),
|
||||
extract(SVG_START +
|
||||
"<foreignObject width='100' height='100'>" +
|
||||
"<div xmlns='http://www.w3.org/1999/xhtml'>" +
|
||||
"<img src='inner.png' alt='a < b'/><a href='inner.html'>x</a>" +
|
||||
"<svg xmlns='http://www.w3.org/2000/svg'><image href='nested.png'/></svg>" +
|
||||
"</div></foreignObject>" +
|
||||
"<image href='after.png'/>" +
|
||||
"</svg>"));
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testForeignObjectWithoutExtractorHTML() throws Exception {
|
||||
((ExtractorSVG) extractor).setExtractorHTML(null);
|
||||
assertEquals(List.of(
|
||||
"E https://example.org/after.png image/@href",
|
||||
"E https://example.org/nested.png image/@href"),
|
||||
extract(SVG_START +
|
||||
"<foreignObject><div xmlns='http://www.w3.org/1999/xhtml'>" +
|
||||
"<img src='inner.png'/>" +
|
||||
"<svg xmlns='http://www.w3.org/2000/svg'><image href='nested.png'/></svg>" +
|
||||
"</div></foreignObject>" +
|
||||
"<image href='after.png'/>" +
|
||||
"</svg>"));
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testNonSvgNamespaceIgnored() throws Exception {
|
||||
assertEquals(List.of("E https://example.org/b.png image/@href"),
|
||||
extract(SVG_START + "<x:image xmlns:x='urn:x' href='a.png'/>" +
|
||||
"<image href='b.png'/></svg>"));
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testEntitiesFromInternalSubset() throws Exception {
|
||||
// as written by Adobe Illustrator
|
||||
assertEquals(List.of("E https://example.org/a.png image/@xlink:href"),
|
||||
extract("<?xml version='1.0' encoding='utf-8'?>" +
|
||||
"<!DOCTYPE svg PUBLIC '-//W3C//DTD SVG 1.1//EN' " +
|
||||
"'http://www.w3.org/Graphics/SVG/1.1/DTD/svg11.dtd' [" +
|
||||
"<!ENTITY ns_svg 'http://www.w3.org/2000/svg'>" +
|
||||
"<!ENTITY ns_xlink 'http://www.w3.org/1999/xlink'>" +
|
||||
"]><svg xmlns='&ns_svg;' xmlns:xlink='&ns_xlink;'>" +
|
||||
"<image xlink:href='a.png'/></svg>"));
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testExternalEntityNotLoaded() throws Exception {
|
||||
assertEquals(List.of("E https://example.org/a.png image/@href"),
|
||||
extract("<!DOCTYPE svg [<!ENTITY x SYSTEM 'file:///etc/passwd'>]>" +
|
||||
SVG_START + "<text>&x;</text><image href='a.png'/></svg>"));
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testMalformedKeepsEarlierLinks() throws Exception {
|
||||
CrawlURI curi = makeCrawlURI(SVG_START + "<image href='a.png'/><oops attr=unquoted/>" +
|
||||
"<image href='b.png'/></svg>", "image/svg+xml");
|
||||
extractor.process(curi);
|
||||
assertEquals(List.of("E https://example.org/a.png image/@href"), links(curi));
|
||||
assertTrue(curi.hasBeenLinkExtracted());
|
||||
assertEquals(1, curi.getNonFatalFailures().size());
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testMaxSizeToParse() throws Exception {
|
||||
String start = SVG_START + "<image href='a.png'/>";
|
||||
((ExtractorSVG) extractor).setMaxSizeToParse(start.length() + 10);
|
||||
CrawlURI curi = makeCrawlURI(start + "<image href='b.png'/></svg>", "image/svg+xml");
|
||||
extractor.process(curi);
|
||||
assertEquals(List.of("E https://example.org/a.png image/@href"), links(curi));
|
||||
assertTrue(curi.hasBeenLinkExtracted());
|
||||
assertTrue(curi.getNonFatalFailures().isEmpty(), curi.getNonFatalFailures().toString());
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testCharsetFromContentType() throws Exception {
|
||||
String svg = "<?xml version='1.0' encoding='iso-8859-1'?>" + SVG_START +
|
||||
"<image href='é.png'/></svg>";
|
||||
CrawlURI curi = makeCrawlURI(svg, "image/svg+xml; charset=UTF-8");
|
||||
extractor.process(curi);
|
||||
assertEquals(List.of("E https://example.org/%C3%A9.png image/@href"), links(curi));
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testShouldExtract() throws Exception {
|
||||
ExtractorSVG svg = (ExtractorSVG) extractor;
|
||||
String content = SVG_START + "</svg>";
|
||||
assertTrue(svg.shouldExtract(makeCrawlURI(content, "image/svg+xml")));
|
||||
assertTrue(svg.shouldExtract(makeCrawlURI(content, "IMAGE/SVG+XML; charset=utf-8")));
|
||||
assertTrue(svg.shouldExtract(makeCrawlURI(content, "text/xml")));
|
||||
assertTrue(svg.shouldExtract(makeCrawlURI(content, "application/xml")));
|
||||
assertFalse(svg.shouldExtract(makeCrawlURI(content, "text/html")));
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testGenericXmlWithSvgRoot() throws Exception {
|
||||
CrawlURI curi = makeCrawlURI("https://example.org/icon?id=3",
|
||||
SVG_START + "<image href='a.png'/></svg>", "text/xml");
|
||||
extractor.process(curi);
|
||||
assertEquals(List.of("E https://example.org/a.png image/@href"), links(curi));
|
||||
assertTrue(curi.hasBeenLinkExtracted());
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testNonSvgRootLeftToOtherExtractors() throws Exception {
|
||||
String[][] documents = {
|
||||
{"application/xml", "<?xml-stylesheet href='feed.css'?>" +
|
||||
"<rss><channel><link>https://example.org/post</link></channel></rss>"},
|
||||
{"text/xml", "<x:svg xmlns:x='urn:x'><image href='a.png'/></x:svg>"},
|
||||
{"image/svg+xml", "<svg><image href='a.png'/></svg>"}, // no namespace
|
||||
{"image/svg+xml", "not xml"},
|
||||
};
|
||||
for (String[] document : documents) {
|
||||
CrawlURI curi = makeCrawlURI(document[1], document[0]);
|
||||
extractor.process(curi);
|
||||
assertEquals(List.of(), links(curi), document[1]);
|
||||
assertFalse(curi.hasBeenLinkExtracted(), document[1]);
|
||||
assertTrue(curi.getNonFatalFailures().isEmpty(), document[1]);
|
||||
}
|
||||
}
|
||||
|
||||
private List<String> extract(String svg) throws Exception {
|
||||
CrawlURI curi = makeCrawlURI(svg, "image/svg+xml");
|
||||
extractor.process(curi);
|
||||
assertTrue(curi.getNonFatalFailures().isEmpty(), curi.getNonFatalFailures().toString());
|
||||
return links(curi);
|
||||
}
|
||||
|
||||
private CrawlURI makeCrawlURI(String content, String contentType) throws Exception {
|
||||
return makeCrawlURI("https://example.org/image.svg", content, contentType);
|
||||
}
|
||||
|
||||
private CrawlURI makeCrawlURI(String uri, String content, String contentType) throws Exception {
|
||||
CrawlURI curi = new CrawlURI(UURIFactory.getInstance(uri));
|
||||
curi.setContentType(contentType);
|
||||
curi.setRecorder(createRecorder(content, "UTF-8"));
|
||||
curi.setContentSize(content.getBytes(StandardCharsets.UTF_8).length);
|
||||
curi.setFetchStatus(200);
|
||||
return curi;
|
||||
}
|
||||
|
||||
/**
|
||||
* Outlinks formatted as "hop uri context", sorted.
|
||||
*/
|
||||
private static List<String> links(CrawlURI curi) {
|
||||
List<String> links = new ArrayList<>();
|
||||
for (CrawlURI link : curi.getOutLinks()) {
|
||||
links.add(link.getLastHop() + " " + link.getURI() + " " + link.getViaContext());
|
||||
}
|
||||
Collections.sort(links);
|
||||
return links;
|
||||
}
|
||||
}
|
||||
@@ -257,4 +257,9 @@ public class JerichoExtractorHTMLTest extends ExtractorHTMLTest {
|
||||
protected JerichoExtractorHTML getExtractor() {
|
||||
return (JerichoExtractorHTML) extractor;
|
||||
}
|
||||
|
||||
@Override
|
||||
protected String xlinkHrefAttributeContext() {
|
||||
return "xlink:href";
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user