Merge pull request #459 from internetarchive/parse-link-rel

modules/src/main/java/org/archive/modules/extractor/ExtractorHTML.java

@@ -25,10 +25,13 @@ import java.net.URLDecoder;
 import java.nio.charset.Charset;
 import java.util.ArrayList;
 import java.util.Iterator;
+import java.util.Locale;
 import java.util.logging.Level;
 import java.util.logging.Logger;
 import java.util.regex.Matcher;
+import java.util.regex.Pattern;
 
+import com.google.common.base.Ascii;
 import org.apache.commons.httpclient.URIException;
 import org.archive.io.ReplayCharSequence;
 import org.archive.modules.CoreAttributeConstants;
@@ -77,6 +80,9 @@ public class ExtractorHTML extends ContentExtractor implements InitializingBean
     public final static String A_META_ROBOTS = "meta-robots";
     
     public final static String A_FORM_OFFSETS = "form-offsets";
+
+    // As per https://infra.spec.whatwg.org/#ascii-whitespace
+    private final static Pattern ASCII_WHITESPACE = Pattern.compile("[\t\n\f\r ]+");
     
     {
         setMaxElementLength(64); 
@@ -385,7 +391,11 @@ public class ExtractorHTML extends ContentExtractor implements InitializingBean
         // Just in case it's a VALUE whose interpretation depends on accompanying NAME
         CharSequence valueVal = null; 
         CharSequence valueContext = null;
-        CharSequence nameVal = null; 
+        CharSequence nameVal = null;
+
+        // Just in case it's a LINK tag
+        CharSequence linkHref = null;
+        CharSequence linkRel = null;
         
         final boolean framesAsEmbeds = 
             getTreatFramesAsEmbedLinks();
@@ -417,8 +427,10 @@ public class ExtractorHTML extends ContentExtractor implements InitializingBean
                     context = elementContext(element, attr.group(2));
                 }
 
-                if ("a[data-remote='true']/@href".equals(context) || elementStr.equalsIgnoreCase(LINK)) {
-                    // <LINK> elements treated as embeds (css, ico, etc)
+                if (elementStr.equalsIgnoreCase(LINK)) {
+                    // delay handling LINK until the end as we need both HREF and REL
+                    linkHref = value;
+                } else if ("a[data-remote='true']/@href".equals(context)) {
                     processEmbed(curi, value, context);
                 } else {
                     // other HREFs treated as links
@@ -511,14 +523,16 @@ public class ExtractorHTML extends ContentExtractor implements InitializingBean
                 method = value;
                 // form processing finished at end (after ACTION also collected)
             } else if (attr.start(13) > -1) {
-                if("NAME".equalsIgnoreCase(attrName.toString())) {
+                if (Ascii.equalsIgnoreCase(attrName, "NAME")) {
                     // remember 'name' for end-analysis
                     nameVal = value; 
-                }
-                if("FLASHVARS".equalsIgnoreCase(attrName.toString())) {
+                } else if (Ascii.equalsIgnoreCase(attrName, "FLASHVARS")) {
                     // consider FLASHVARS attribute immediately
                     valueContext = elementContext(element,attr.group(13));
                     considerQueryStringValues(curi, value, valueContext,Hop.SPECULATIVE);
+                } else if (Ascii.equalsIgnoreCase(attrName, "REL")) {
+                    // remember 'rel' for end-analysis
+                    linkRel = value;
                 }
                 // any other attribute
                 // ignore for now
@@ -556,6 +570,11 @@ public class ExtractorHTML extends ContentExtractor implements InitializingBean
                     DevUtils.extraInfo(), e);
             }
         }
+
+        // finish handling LINK now both HREF and REL should be available
+        if (linkHref != null && linkRel != null) {
+            processLinkTagWithRel(curi, linkHref, linkRel);
+        }
            
         // finish handling form action, now method is available
         if(action != null) {
@@ -582,6 +601,38 @@ public class ExtractorHTML extends ContentExtractor implements InitializingBean
         }
     }
 
+    // see: https://html.spec.whatwg.org/multipage/links.html#linkTypes
+    protected void processLinkTagWithRel(CrawlURI curi, CharSequence href, CharSequence rel) {
+        boolean emitAsNavLink = false;
+        for (String keyword : ASCII_WHITESPACE.split(rel)) {
+            String linkType = keyword.toLowerCase(Locale.ROOT);
+            switch (linkType) {
+                case "icon":
+                case "stylesheet":
+                case "modulepreload":
+                case "prefetch":
+                case "prerender":
+                    // treat as an embedded resource
+                    processEmbed(curi, href, "link[rel='" + linkType + "']/@href");
+                    return;
+                case "pingback":
+                    // don't extract pingbacks
+                    return;
+                case "dns-prefetch":
+                case "preconnect":
+                case "":
+                    // ignore connection hints
+                    break;
+                default:
+                    // treat anything else as a navigation link
+                    emitAsNavLink = true;
+            }
+        }
+        if (emitAsNavLink) {
+            processLink(curi, href, "link/@href");
+        }
+    }
+
     /**
      * Consider a query-string-like collections of key=value[&amp;key=value]
      * pairs for URI-like strings in the values. Where URI-like strings are

modules/src/main/java/org/archive/modules/extractor/JerichoExtractorHTML.java

@@ -137,8 +137,11 @@ public class JerichoExtractorHTML extends ExtractorHTML {
             CharSequence context = elementContext(elementName, attr
                     .getKey());
             if ("link".equals(elementName)) {
-                // <LINK> elements treated as embeds (css, ico, etc)
-                processEmbed(curi, attrValue, context);
+                // <LINK> elements treated as embeds (css, ico) or links (next, author) depending on rel
+                String rel = attributes.getValue("rel");
+                if (rel != null) {
+                    processLinkTagWithRel(curi, attrValue, rel);
+                }
             } else {
                 // other HREFs treated as links
                 processLink(curi, attrValue, context);

modules/src/test/java/org/archive/modules/extractor/ExtractorHTMLTest.java

@@ -687,10 +687,40 @@ public class ExtractorHTMLTest extends StringExtractorTestBase {
 
         genericCrawl(curi, cs, dest);
         
-    }  
-    
-	      
-    
+    }
+
+    public void testLinkRel() throws URIException {
+        CrawlURI curi = new CrawlURI(UURIFactory.getInstance("https://www.example.org/"));
+
+        String html = "<link href='/pingback' rel='pingback'>" +
+                "<link href='/style.css' rel=stylesheet>" +
+                "<link rel='my stylesheet rocks' href=/style2.css>" +
+                "<link rel=icon href=/icon.ico>" +
+                "<link href='http://dns-prefetch.example.com/' rel=dns-prefetch>" +
+                "<link href=/without-rel>" +
+                "<link href=/empty-rel rel=>" +
+                "<link href=/just-spaces rel='   '>" +
+                "<link href=/canonical rel=canonical>" +
+                "<link href=/unknown rel=unknown>";
+
+        List<String> expectedLinks = Arrays.asList(
+                "E https://www.example.org/icon.ico",
+                "E https://www.example.org/style.css",
+                "E https://www.example.org/style2.css",
+                "L https://www.example.org/canonical",
+                "L https://www.example.org/unknown"
+        );
+
+        getExtractor().extract(curi, html);
+        List<String> actualLinks = new ArrayList<>();
+        for (CrawlURI link: curi.getOutLinks()) {
+            actualLinks.add(link.getLastHop() + " " + link.getURI());
+        }
+        Collections.sort(actualLinks);
+
+        assertEquals(expectedLinks, actualLinks);
+    }
+
     private void genericCrawl(CrawlURI curi, CharSequence cs,String[] dest){
         getExtractor().extract(curi, cs);