From dcf9b040077f906ab163a3f7d3c35577f1de44b7 Mon Sep 17 00:00:00 2001 From: Alex Osborne Date: Tue, 25 Jan 2022 17:57:32 +0900 Subject: [PATCH 1/2] ExtractorHTML: Determine LINK tag type by parsing REL attribute Only treat icons, stylesheets and (tentatively) resource preloads as embedded resources. Treat most other link types as navigation links. Previously we were treating all link tags as embedded resources including links like 'next', 'prev', 'canonical', 'author' which could pull in resources that should normally be considered out of scope. For example crawling a page containing an author link lke: Would end up transitively pulling 200+ MB of Twitter profile pages in multiple languages into a crawl for which twitter.com is not even supposed to be in scope at all. Fixes #263 --- .../modules/extractor/ExtractorHTML.java | 63 +++++++++++++++++-- .../modules/extractor/ExtractorHTMLTest.java | 38 +++++++++-- 2 files changed, 91 insertions(+), 10 deletions(-) diff --git a/modules/src/main/java/org/archive/modules/extractor/ExtractorHTML.java b/modules/src/main/java/org/archive/modules/extractor/ExtractorHTML.java index 671ac523..28e2e053 100644 --- a/modules/src/main/java/org/archive/modules/extractor/ExtractorHTML.java +++ b/modules/src/main/java/org/archive/modules/extractor/ExtractorHTML.java @@ -25,10 +25,13 @@ import java.net.URLDecoder; import java.nio.charset.Charset; import java.util.ArrayList; import java.util.Iterator; +import java.util.Locale; import java.util.logging.Level; import java.util.logging.Logger; import java.util.regex.Matcher; +import java.util.regex.Pattern; +import com.google.common.base.Ascii; import org.apache.commons.httpclient.URIException; import org.archive.io.ReplayCharSequence; import org.archive.modules.CoreAttributeConstants; @@ -77,6 +80,9 @@ public class ExtractorHTML extends ContentExtractor implements InitializingBean public final static String A_META_ROBOTS = "meta-robots"; public final static String A_FORM_OFFSETS = "form-offsets"; + + // As per https://infra.spec.whatwg.org/#ascii-whitespace + private final static Pattern ASCII_WHITESPACE = Pattern.compile("[\t\n\f\r ]+"); { setMaxElementLength(64); @@ -385,7 +391,11 @@ public class ExtractorHTML extends ContentExtractor implements InitializingBean // Just in case it's a VALUE whose interpretation depends on accompanying NAME CharSequence valueVal = null; CharSequence valueContext = null; - CharSequence nameVal = null; + CharSequence nameVal = null; + + // Just in case it's a LINK tag + CharSequence linkHref = null; + CharSequence linkRel = null; final boolean framesAsEmbeds = getTreatFramesAsEmbedLinks(); @@ -417,8 +427,10 @@ public class ExtractorHTML extends ContentExtractor implements InitializingBean context = elementContext(element, attr.group(2)); } - if ("a[data-remote='true']/@href".equals(context) || elementStr.equalsIgnoreCase(LINK)) { - // elements treated as embeds (css, ico, etc) + if (elementStr.equalsIgnoreCase(LINK)) { + // delay handling LINK until the end as we need both HREF and REL + linkHref = value; + } else if ("a[data-remote='true']/@href".equals(context)) { processEmbed(curi, value, context); } else { // other HREFs treated as links @@ -511,14 +523,16 @@ public class ExtractorHTML extends ContentExtractor implements InitializingBean method = value; // form processing finished at end (after ACTION also collected) } else if (attr.start(13) > -1) { - if("NAME".equalsIgnoreCase(attrName.toString())) { + if (Ascii.equalsIgnoreCase(attrName, "NAME")) { // remember 'name' for end-analysis nameVal = value; - } - if("FLASHVARS".equalsIgnoreCase(attrName.toString())) { + } else if (Ascii.equalsIgnoreCase(attrName, "FLASHVARS")) { // consider FLASHVARS attribute immediately valueContext = elementContext(element,attr.group(13)); considerQueryStringValues(curi, value, valueContext,Hop.SPECULATIVE); + } else if (Ascii.equalsIgnoreCase(attrName, "REL")) { + // remember 'rel' for end-analysis + linkRel = value; } // any other attribute // ignore for now @@ -556,6 +570,11 @@ public class ExtractorHTML extends ContentExtractor implements InitializingBean DevUtils.extraInfo(), e); } } + + // finish handling LINK now both HREF and REL should be available + if (linkHref != null && linkRel != null) { + processLinkTagWithRel(curi, linkHref, linkRel); + } // finish handling form action, now method is available if(action != null) { @@ -582,6 +601,38 @@ public class ExtractorHTML extends ContentExtractor implements InitializingBean } } + // see: https://html.spec.whatwg.org/multipage/links.html#linkTypes + private void processLinkTagWithRel(CrawlURI curi, CharSequence href, CharSequence rel) { + boolean emitAsNavLink = false; + for (String keyword : ASCII_WHITESPACE.split(rel)) { + String linkType = keyword.toLowerCase(Locale.ROOT); + switch (linkType) { + case "icon": + case "stylesheet": + case "modulepreload": + case "prefetch": + case "prerender": + // treat as an embedded resource + processEmbed(curi, href, "link[rel='" + linkType + "']/@href"); + return; + case "pingback": + // don't extract pingbacks + return; + case "dns-prefetch": + case "preconnect": + case "": + // ignore connection hints + break; + default: + // treat anything else as a navigation link + emitAsNavLink = true; + } + } + if (emitAsNavLink) { + processLink(curi, href, "link/@href"); + } + } + /** * Consider a query-string-like collections of key=value[&key=value] * pairs for URI-like strings in the values. Where URI-like strings are diff --git a/modules/src/test/java/org/archive/modules/extractor/ExtractorHTMLTest.java b/modules/src/test/java/org/archive/modules/extractor/ExtractorHTMLTest.java index 53c0b34f..d9fe10b5 100644 --- a/modules/src/test/java/org/archive/modules/extractor/ExtractorHTMLTest.java +++ b/modules/src/test/java/org/archive/modules/extractor/ExtractorHTMLTest.java @@ -687,10 +687,40 @@ public class ExtractorHTMLTest extends StringExtractorTestBase { genericCrawl(curi, cs, dest); - } - - - + } + + public void testLinkRel() throws URIException { + CrawlURI curi = new CrawlURI(UURIFactory.getInstance("https://www.example.org/")); + + String html = "" + + "" + + "" + + "" + + "" + + "" + + "" + + "" + + "" + + ""; + + List expectedLinks = Arrays.asList( + "E https://www.example.org/icon.ico", + "E https://www.example.org/style.css", + "E https://www.example.org/style2.css", + "L https://www.example.org/canonical", + "L https://www.example.org/unknown" + ); + + getExtractor().extract(curi, html); + List actualLinks = new ArrayList<>(); + for (CrawlURI link: curi.getOutLinks()) { + actualLinks.add(link.getLastHop() + " " + link.getURI()); + } + Collections.sort(actualLinks); + + assertEquals(expectedLinks, actualLinks); + } + private void genericCrawl(CrawlURI curi, CharSequence cs,String[] dest){ getExtractor().extract(curi, cs); From 1c4831d6e9bf145206225229ee4990accf9faf62 Mon Sep 17 00:00:00 2001 From: Alex Osborne Date: Tue, 25 Jan 2022 17:57:32 +0900 Subject: [PATCH 2/2] JerichoExtractorHTML: Determine LINK tag type by parsing REL attribute This commit extends ExtractorHTML's link[rel] parsing support added in dcf9b040 to also cover JerichoExtractorHTML. --- .../java/org/archive/modules/extractor/ExtractorHTML.java | 2 +- .../archive/modules/extractor/JerichoExtractorHTML.java | 7 +++++-- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/modules/src/main/java/org/archive/modules/extractor/ExtractorHTML.java b/modules/src/main/java/org/archive/modules/extractor/ExtractorHTML.java index 28e2e053..69b164dd 100644 --- a/modules/src/main/java/org/archive/modules/extractor/ExtractorHTML.java +++ b/modules/src/main/java/org/archive/modules/extractor/ExtractorHTML.java @@ -602,7 +602,7 @@ public class ExtractorHTML extends ContentExtractor implements InitializingBean } // see: https://html.spec.whatwg.org/multipage/links.html#linkTypes - private void processLinkTagWithRel(CrawlURI curi, CharSequence href, CharSequence rel) { + protected void processLinkTagWithRel(CrawlURI curi, CharSequence href, CharSequence rel) { boolean emitAsNavLink = false; for (String keyword : ASCII_WHITESPACE.split(rel)) { String linkType = keyword.toLowerCase(Locale.ROOT); diff --git a/modules/src/main/java/org/archive/modules/extractor/JerichoExtractorHTML.java b/modules/src/main/java/org/archive/modules/extractor/JerichoExtractorHTML.java index c12d91ed..17cad0e8 100644 --- a/modules/src/main/java/org/archive/modules/extractor/JerichoExtractorHTML.java +++ b/modules/src/main/java/org/archive/modules/extractor/JerichoExtractorHTML.java @@ -137,8 +137,11 @@ public class JerichoExtractorHTML extends ExtractorHTML { CharSequence context = elementContext(elementName, attr .getKey()); if ("link".equals(elementName)) { - // elements treated as embeds (css, ico, etc) - processEmbed(curi, attrValue, context); + // elements treated as embeds (css, ico) or links (next, author) depending on rel + String rel = attributes.getValue("rel"); + if (rel != null) { + processLinkTagWithRel(curi, attrValue, rel); + } } else { // other HREFs treated as links processLink(curi, attrValue, context);