mirror of
https://github.com/internetarchive/heritrix3.git
synced 2026-11-22 13:47:02 +00:00
ExtractorSVG: obey ExtractorHTML obeyRelNofollow setting
This commit is contained in:
1 parent
9d79ef7372
commit
5838dbf1f9
2 files changed
+41
-2
No files matched your search
@@ -86,6 +86,10 @@ public class ExtractorSVG extends ContentExtractor {
|
||||
private static final Pattern PSEUDO_ATTRIBUTE_HREF =
|
||||
Pattern.compile("(?:^|\\s)href\\s*=\\s*(?:\"([^\"]*)\"|'([^']*)')");
|
||||
|
||||
/** rel is a set of space-separated, case-insensitive tokens */
|
||||
private static final Pattern REL_NOFOLLOW =
|
||||
Pattern.compile("(?i)(?:^|\\s)nofollow(?:\\s|$)");
|
||||
|
||||
private static final XMLInputFactory INPUT_FACTORY = newInputFactory();
|
||||
private static final XMLOutputFactory OUTPUT_FACTORY = XMLOutputFactory.newFactory();
|
||||
|
||||
@@ -131,7 +135,8 @@ public class ExtractorSVG extends ContentExtractor {
|
||||
|
||||
/**
|
||||
* HTML extractor to use to process the content of foreignObject elements. Autowired if
|
||||
* available. If null, links will not be extracted from foreignObject elements.
|
||||
* available. If null, links will not be extracted from foreignObject elements. Its
|
||||
* obeyRelNofollow setting also applies to {@code <a>} links in the SVG.
|
||||
*/
|
||||
transient protected ExtractorHTML extractorHTML;
|
||||
public ExtractorHTML getExtractorHTML() {
|
||||
@@ -283,7 +288,9 @@ public class ExtractorSVG extends ContentExtractor {
|
||||
if (name.equals("href") && (noNamespace || namespace.equals(XLINK_NS))) {
|
||||
String context = noNamespace ? "href" : "xlink:href";
|
||||
if (elementLC.equals("a")) {
|
||||
addLink(curi, value, elementLC, context, Hop.NAVLINK);
|
||||
if (!obeyRelNofollow() || !isNofollow(reader.getAttributeValue(null, "rel"))) {
|
||||
addLink(curi, value, elementLC, context, Hop.NAVLINK);
|
||||
}
|
||||
} else if (EMBED_HREF_ELEMENTS.contains(elementLC)) {
|
||||
addLink(curi, value, elementLC, context, Hop.EMBED);
|
||||
}
|
||||
@@ -310,6 +317,18 @@ public class ExtractorSVG extends ContentExtractor {
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Follows the ExtractorHTML bean's obeyRelNofollow setting, if there is one.
|
||||
*/
|
||||
private boolean obeyRelNofollow() {
|
||||
ExtractorHTML html = getExtractorHTML();
|
||||
return html != null && html.getObeyRelNofollow();
|
||||
}
|
||||
|
||||
private static boolean isNofollow(String rel) {
|
||||
return rel != null && REL_NOFOLLOW.matcher(rel).find();
|
||||
}
|
||||
|
||||
protected void processXmlStylesheet(CrawlURI curi, String data) {
|
||||
if (data == null) {
|
||||
return;
|
||||
|
||||
@@ -100,6 +100,26 @@ public class ExtractorSVGTest extends ContentExtractorTestBase {
|
||||
"</svg>"));
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testObeyRelNofollow() throws Exception {
|
||||
String svg = SVG_START +
|
||||
"<a href='followed.html' rel='noopener'><text>x</text></a>" +
|
||||
"<a href='nofollow.html' rel='author NoFollow'><text>y</text></a>" +
|
||||
"<image rel='nofollow' href='image.png'/>" +
|
||||
"</svg>";
|
||||
assertEquals(List.of(
|
||||
"E https://example.org/image.png image/@href",
|
||||
"L https://example.org/followed.html a/@href",
|
||||
"L https://example.org/nofollow.html a/@href"),
|
||||
extract(svg));
|
||||
|
||||
((ExtractorSVG) extractor).getExtractorHTML().setObeyRelNofollow(true);
|
||||
assertEquals(List.of(
|
||||
"E https://example.org/image.png image/@href",
|
||||
"L https://example.org/followed.html a/@href"),
|
||||
extract(svg));
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testHrefOnOtherElementsIgnored() throws Exception {
|
||||
// href isn't an attribute of <rect>, and other namespaces aren't xlink
|
||||
|
||||
Reference in new issue
Block a user