From e33e1ff398f7b89dfce053506069ed2cbabfde8c Mon Sep 17 00:00:00 2001 From: Alex Osborne Date: Fri, 30 Jul 2021 14:41:02 +0900 Subject: [PATCH] Extractor: ignore data URIs when adding outlinks --- .../main/java/org/archive/util/UriUtils.java | 7 +++++ .../java/org/archive/util/UriUtilsTest.java | 11 ++++++++ .../archive/modules/extractor/Extractor.java | 26 ++++++++++++++++--- .../modules/extractor/ExtractorRobotsTxt.java | 4 +++ .../modules/extractor/ExtractorSitemap.java | 4 +++ .../modules/extractor/ExtractorHTMLTest.java | 8 +++++- 6 files changed, 56 insertions(+), 4 deletions(-) diff --git a/commons/src/main/java/org/archive/util/UriUtils.java b/commons/src/main/java/org/archive/util/UriUtils.java index d08e9128..a1ce231d 100644 --- a/commons/src/main/java/org/archive/util/UriUtils.java +++ b/commons/src/main/java/org/archive/util/UriUtils.java @@ -88,6 +88,13 @@ import org.archive.url.LaxURLCodec; public class UriUtils { private static final Logger LOGGER = Logger.getLogger(UriUtils.class.getName()); + /** + * Returns true when when given a CharSequence that looks like a data URI. + */ + public static boolean isDataUri(CharSequence candidate) { + return TextUtils.matches("(?i)\\s*data:.*", candidate); + } + // naive likely-uri test: // no '<' or '>' // at least one '.' or '/'; diff --git a/commons/src/test/java/org/archive/util/UriUtilsTest.java b/commons/src/test/java/org/archive/util/UriUtilsTest.java index 79db4fbd..db12553e 100644 --- a/commons/src/test/java/org/archive/util/UriUtilsTest.java +++ b/commons/src/test/java/org/archive/util/UriUtilsTest.java @@ -66,6 +66,17 @@ public class UriUtilsTest extends TestCase { "images/photo.jpg", "../../images/photo.jpg" }; + public void testIsDataUri() { + assertTrue(UriUtils.isDataUri("data:,hello")); + assertTrue(UriUtils.isDataUri("data:text/plain,hello")); + assertTrue(UriUtils.isDataUri(" data:,hello")); + assertTrue(UriUtils.isDataUri(" dAtA:,hello// ")); + assertFalse(UriUtils.isDataUri("")); + assertFalse(UriUtils.isDataUri(" http://example.org/")); + assertFalse(UriUtils.isDataUri("http://example.org/")); + assertFalse(UriUtils.isDataUri("\0\1\2\3garbage")); + } + /** check that plausible relative image URIs return true with legacy tests */ public void xestLegacySimpleImageRelatives() { legacyTryAll(urisRelativeImages, true); diff --git a/modules/src/main/java/org/archive/modules/extractor/Extractor.java b/modules/src/main/java/org/archive/modules/extractor/Extractor.java index b33df2cc..fcbd9cab 100644 --- a/modules/src/main/java/org/archive/modules/extractor/Extractor.java +++ b/modules/src/main/java/org/archive/modules/extractor/Extractor.java @@ -29,6 +29,7 @@ import org.archive.modules.CrawlURI; import org.archive.modules.Processor; import org.archive.net.UURI; import org.archive.net.UURIFactory; +import org.archive.util.UriUtils; import org.json.JSONException; import org.json.JSONObject; import org.springframework.beans.factory.annotation.Autowired; @@ -134,10 +135,13 @@ public abstract class Extractor extends Processor { /** * Create and add a 'Link' to the CrawlURI with given URI/context/hop-type - * @return the new outlink or null if it was not valid + * @return the new outlink or null if it was invalid or ignored */ protected CrawlURI addOutlink(CrawlURI curi, String uri, LinkContext context, Hop hop) { + if (UriUtils.isDataUri(uri)) { + return null; + } try { UURI dest = UURIFactory.getInstance(curi.getUURI(), uri); CrawlURI link = curi.createCrawlURI(dest, context, hop); @@ -151,6 +155,9 @@ public abstract class Extractor extends Processor { protected void addOutlink(CrawlURI curi, UURI uuri, LinkContext context, Hop hop) { + if ("data".equalsIgnoreCase(uuri.getScheme())) { + return; + } try { CrawlURI link = curi.createCrawlURI(uuri, context, hop); curi.getOutLinks().add(link); @@ -183,16 +190,29 @@ public abstract class Extractor extends Processor { ret.append(" " + numberOfLinksExtracted + " links from " + getURICount() +" CrawlURIs\n"); return ret.toString(); } - + + /** + * Adds an outlink to uri relative to uri.getBaseURI(). + * @return the new outlink or null if the outlink was ignored + */ public static CrawlURI addRelativeToBase(CrawlURI uri, int max, String newUri, LinkContext context, Hop hop) throws URIException { + if (UriUtils.isDataUri(newUri)) { + return null; + } UURI dest = UURIFactory.getInstance(uri.getBaseURI(), newUri); return add2(uri, max, dest, context, hop); } - + /** + * Adds an outlink to uri relative to uri.getVia(). + * @return the new outlink or null if the outlink was ignored + */ public static CrawlURI addRelativeToVia(CrawlURI uri, int max, String newUri, LinkContext context, Hop hop) throws URIException { + if (UriUtils.isDataUri(newUri)) { + return null; + } UURI relTo = uri.getVia(); if (relTo == null) { if (!uri.getAnnotations().contains("usedBaseForVia")) { diff --git a/modules/src/main/java/org/archive/modules/extractor/ExtractorRobotsTxt.java b/modules/src/main/java/org/archive/modules/extractor/ExtractorRobotsTxt.java index 27a9d0cd..0ad93ca8 100644 --- a/modules/src/main/java/org/archive/modules/extractor/ExtractorRobotsTxt.java +++ b/modules/src/main/java/org/archive/modules/extractor/ExtractorRobotsTxt.java @@ -75,6 +75,10 @@ public class ExtractorRobotsTxt extends ContentExtractor { CrawlURI newCuri = addRelativeToBase(curi, max, link, LinkContext.MANIFEST_MISC, Hop.MANIFEST); + if (newCuri == null) { + continue; + } + // Annotate as a Site Map: newCuri.getAnnotations().add( ExtractorRobotsTxt.ANNOTATION_IS_SITEMAP); diff --git a/modules/src/main/java/org/archive/modules/extractor/ExtractorSitemap.java b/modules/src/main/java/org/archive/modules/extractor/ExtractorSitemap.java index 7b95a887..a7a42d5f 100644 --- a/modules/src/main/java/org/archive/modules/extractor/ExtractorSitemap.java +++ b/modules/src/main/java/org/archive/modules/extractor/ExtractorSitemap.java @@ -163,6 +163,10 @@ public class ExtractorSitemap extends ContentExtractor { CrawlURI newCuri = addRelativeToBase(curi, max, newUri.toString(), LinkContext.MANIFEST_MISC, Hop.MANIFEST); + if (newCuri == null) { + return; + } + if (isSitemap) { // Annotate as a Site Map: newCuri.getAnnotations().add( diff --git a/modules/src/test/java/org/archive/modules/extractor/ExtractorHTMLTest.java b/modules/src/test/java/org/archive/modules/extractor/ExtractorHTMLTest.java index f2ba890a..53c0b34f 100644 --- a/modules/src/test/java/org/archive/modules/extractor/ExtractorHTMLTest.java +++ b/modules/src/test/java/org/archive/modules/extractor/ExtractorHTMLTest.java @@ -268,6 +268,13 @@ public class ExtractorHTMLTest extends StringExtractorTestBase { } })); } + + public void testDataUrisAreIgnored() throws URIException { + CrawlURI curi = new CrawlURI(UURIFactory.getInstance("http://www.example.com")); + CharSequence cs = ""; + getExtractor().extract(curi, cs); + assertEquals(0, curi.getOutLinks().size()); + } /** * Test that relative base href's are resolved correctly: @@ -521,7 +528,6 @@ public class ExtractorHTMLTest extends StringExtractorTestBase { Arrays.sort(links); String[] dest = { - "data:image/gif;base64,R0lGODlhAQABAIAAAAAAAP///yH5BAEAAAAALAAAAAABAAEAAAIBRAA7", "http://www.example.com/a,b,c", "http://www.example.com/images/foo.jpg", "http://www.example.com/images/foo1.jpg",