Extractor: ignore data URIs when adding outlinks

commons/src/main/java/org/archive/util/UriUtils.java

@@ -88,6 +88,13 @@ import org.archive.url.LaxURLCodec;
 public class UriUtils {
     private static final Logger LOGGER = Logger.getLogger(UriUtils.class.getName());
 
+    /**
+     * Returns true when when given a CharSequence that looks like a data URI.
+     */
+    public static boolean isDataUri(CharSequence candidate) {
+        return TextUtils.matches("(?i)\\s*data:.*", candidate);
+    }
+
     // naive likely-uri test: 
     //    no '<' or '>' 
     //    at least one '.' or '/';

commons/src/test/java/org/archive/util/UriUtilsTest.java

@@ -66,6 +66,17 @@ public class UriUtilsTest extends TestCase {
         "images/photo.jpg", 
         "../../images/photo.jpg" };
 
+    public void testIsDataUri() {
+        assertTrue(UriUtils.isDataUri("data:,hello"));
+        assertTrue(UriUtils.isDataUri("data:text/plain,hello"));
+        assertTrue(UriUtils.isDataUri("   data:,hello"));
+        assertTrue(UriUtils.isDataUri("   dAtA:,hello//  "));
+        assertFalse(UriUtils.isDataUri(""));
+        assertFalse(UriUtils.isDataUri(" http://example.org/"));
+        assertFalse(UriUtils.isDataUri("http://example.org/"));
+        assertFalse(UriUtils.isDataUri("\0\1\2\3garbage"));
+    }
+
     /** check that plausible relative image URIs return true with legacy tests */
     public void xestLegacySimpleImageRelatives() {
         legacyTryAll(urisRelativeImages, true);

modules/src/main/java/org/archive/modules/extractor/Extractor.java

@@ -29,6 +29,7 @@ import org.archive.modules.CrawlURI;
 import org.archive.modules.Processor;
 import org.archive.net.UURI;
 import org.archive.net.UURIFactory;
+import org.archive.util.UriUtils;
 import org.json.JSONException;
 import org.json.JSONObject;
 import org.springframework.beans.factory.annotation.Autowired;
@@ -134,10 +135,13 @@ public abstract class Extractor extends Processor {
 
     /**
      * Create and add a 'Link' to the CrawlURI with given URI/context/hop-type
-     * @return the new outlink or null if it was not valid
+     * @return the new outlink or null if it was invalid or ignored
      */
     protected CrawlURI addOutlink(CrawlURI curi, String uri, LinkContext context,
             Hop hop) {
+        if (UriUtils.isDataUri(uri)) {
+            return null;
+        }
         try {
             UURI dest = UURIFactory.getInstance(curi.getUURI(), uri);
             CrawlURI link = curi.createCrawlURI(dest, context, hop);
@@ -151,6 +155,9 @@ public abstract class Extractor extends Processor {
     
     protected void addOutlink(CrawlURI curi, UURI uuri, LinkContext context,
             Hop hop) {
+        if ("data".equalsIgnoreCase(uuri.getScheme())) {
+            return;
+        }
         try {
             CrawlURI link = curi.createCrawlURI(uuri, context, hop);
             curi.getOutLinks().add(link);
@@ -183,16 +190,29 @@ public abstract class Extractor extends Processor {
         ret.append("  " + numberOfLinksExtracted + " links from " + getURICount() +" CrawlURIs\n");
         return ret.toString();
     }
-    
+
+    /**
+     * Adds an outlink to uri relative to uri.getBaseURI().
+     * @return the new outlink or null if the outlink was ignored
+     */
     public static CrawlURI addRelativeToBase(CrawlURI uri, int max,
             String newUri, LinkContext context, Hop hop) throws URIException {
+        if (UriUtils.isDataUri(newUri)) {
+            return null;
+        }
         UURI dest = UURIFactory.getInstance(uri.getBaseURI(), newUri);
         return add2(uri, max, dest, context, hop);
     }
 
-    
+    /**
+     * Adds an outlink to uri relative to uri.getVia().
+     * @return the new outlink or null if the outlink was ignored
+     */
     public static CrawlURI addRelativeToVia(CrawlURI uri, int max, String newUri,
             LinkContext context, Hop hop) throws URIException {
+        if (UriUtils.isDataUri(newUri)) {
+            return null;
+        }
         UURI relTo = uri.getVia();
         if (relTo == null) {
             if (!uri.getAnnotations().contains("usedBaseForVia")) {

modules/src/main/java/org/archive/modules/extractor/ExtractorRobotsTxt.java

@@ -75,6 +75,10 @@ public class ExtractorRobotsTxt extends ContentExtractor {
                     CrawlURI newCuri = addRelativeToBase(curi, max, link,
                             LinkContext.MANIFEST_MISC, Hop.MANIFEST);
 
+                    if (newCuri == null) {
+                        continue;
+                    }
+
                     // Annotate as a Site Map:
                     newCuri.getAnnotations().add(
                             ExtractorRobotsTxt.ANNOTATION_IS_SITEMAP);

modules/src/main/java/org/archive/modules/extractor/ExtractorSitemap.java

@@ -163,6 +163,10 @@ public class ExtractorSitemap extends ContentExtractor {
             CrawlURI newCuri = addRelativeToBase(curi, max, newUri.toString(),
                     LinkContext.MANIFEST_MISC, Hop.MANIFEST);
 
+            if (newCuri == null) {
+                return;
+            }
+
             if (isSitemap) {
                 // Annotate as a Site Map:
                 newCuri.getAnnotations().add(

modules/src/test/java/org/archive/modules/extractor/ExtractorHTMLTest.java

@@ -268,6 +268,13 @@ public class ExtractorHTMLTest extends StringExtractorTestBase {
             }
         }));
     }
+
+    public void testDataUrisAreIgnored() throws URIException {
+        CrawlURI curi = new CrawlURI(UURIFactory.getInstance("http://www.example.com"));
+        CharSequence cs = "<img src='data:image/gif;base64,R0lGODlhAQABAAAAACH5BAEKAAEALAAAAAABAAEAAAICTAEAOw=='>";
+        getExtractor().extract(curi, cs);
+        assertEquals(0, curi.getOutLinks().size());
+    }
     
     /**
      * Test that relative base href's are resolved correctly:
@@ -521,7 +528,6 @@ public class ExtractorHTMLTest extends StringExtractorTestBase {
         Arrays.sort(links);
 
         String[] dest = {
-                "data:image/gif;base64,R0lGODlhAQABAIAAAAAAAP///yH5BAEAAAAALAAAAAABAAEAAAIBRAA7",
                 "http://www.example.com/a,b,c",
                 "http://www.example.com/images/foo.jpg",
                 "http://www.example.com/images/foo1.jpg",