From 4c0e52af6186ead09b7c2b7aad8d3b09f025a9b0 Mon Sep 17 00:00:00 2001 From: Neil Minton Date: Mon, 1 May 2017 17:08:15 -0700 Subject: [PATCH 1/2] Add parsing for srcset attributes. - Includes Unit Tests --- .../modules/extractor/ExtractorHTML.java | 24 +++++++-- .../modules/extractor/HTMLLinkContext.java | 7 +++ .../modules/extractor/ExtractorHTMLTest.java | 52 +++++++++++++++++++ 3 files changed, 79 insertions(+), 4 deletions(-) diff --git a/modules/src/main/java/org/archive/modules/extractor/ExtractorHTML.java b/modules/src/main/java/org/archive/modules/extractor/ExtractorHTML.java index dfb563b5..2795a666 100644 --- a/modules/src/main/java/org/archive/modules/extractor/ExtractorHTML.java +++ b/modules/src/main/java/org/archive/modules/extractor/ExtractorHTML.java @@ -187,8 +187,8 @@ public class ExtractorHTML extends ContentExtractor implements InitializingBean // sorts are matched specially static final String EACH_ATTRIBUTE_EXTRACTOR = "(?is)\\s?((href)|(action)|(on\\w*)" // 1, 2, 3, 4 - +"|((?:src)|(?:lowsrc)|(?:background)|(?:cite)|(?:longdesc)" // ... - +"|(?:usemap)|(?:profile)|(?:datasrc))" // 5 + +"|((?:src)|(?:srcset)|(?:lowsrc)|(?:background)|(?:cite)" // ... + +"|(?:longdesc)|(?:usemap)|(?:profile)|(?:datasrc))" // 5 +"|(codebase)|((?:classid)|(?:data))|(archive)|(code)" // 6, 7, 8, 9 +"|(value)|(style)|(method)" // 10, 11, 12 +"|([-\\w]{1,"+MAX_ATTR_NAME_REPLACE+"}))" // 13 @@ -201,7 +201,7 @@ public class ExtractorHTML extends ContentExtractor implements InitializingBean // 2: HREF - single URI relative to doc base, or occasionally javascript: // 3: ACTION - single URI relative to doc base, or occasionally javascript: // 4: ON[WHATEVER] - script handler - // 5: SRC,LOWSRC,BACKGROUND,CITE,LONGDESC,USEMAP,PROFILE, or DATASRC + // 5: SRC,SRCSET,LOWSRC,BACKGROUND,CITE,LONGDESC,USEMAP,PROFILE, or DATASRC // single URI relative to doc base // 6: CODEBASE - a single URI relative to doc base, affecting other // attributes @@ -662,7 +662,23 @@ public class ExtractorHTML extends ContentExtractor implements InitializingBean // ReplayCharSequence. HTMLLinkContext hc = HTMLLinkContext.get(context.toString()); int max = getExtractorParameters().getMaxOutlinks(); - addRelativeToBase(curi, max, uri.toString(), hc, hop); + if (hc.equals(HTMLLinkContext.IMG_SRCSET) || hc.equals(HTMLLinkContext.SOURCE_SRCSET)) { + + logger.fine("Found srcset listing: " + uri.toString()); + + Matcher srcSetUris = TextUtils.getMatcher("([\\w:/_.-]+)(?: [\\d.]+(?:w|x),?)*",uri); + String srcSetUri; + + while (srcSetUris.find()) { + srcSetUri = srcSetUris.group(1); + logger.finer("Found " + srcSetUri.toString() + "adding to outlinks."); + addRelativeToBase(curi, max, srcSetUri.toString(), hc, hop); + } + + TextUtils.recycleMatcher(srcSetUris); + } else { + addRelativeToBase(curi, max, uri.toString(), hc, hop); + } } catch (URIException e) { logUriError(e, curi.getUURI(), uri); } diff --git a/modules/src/main/java/org/archive/modules/extractor/HTMLLinkContext.java b/modules/src/main/java/org/archive/modules/extractor/HTMLLinkContext.java index f44cef74..f11aaa47 100644 --- a/modules/src/main/java/org/archive/modules/extractor/HTMLLinkContext.java +++ b/modules/src/main/java/org/archive/modules/extractor/HTMLLinkContext.java @@ -32,6 +32,8 @@ public class HTMLLinkContext extends LinkContext { final public static HTMLLinkContext META = new HTMLLinkContext("meta"); final public static HTMLLinkContext A_HREF = new HTMLLinkContext("a", "href"); final public static HTMLLinkContext IMG_SRC = new HTMLLinkContext("img", "src"); + final public static HTMLLinkContext IMG_SRCSET = new HTMLLinkContext("img", "srcset"); + final public static HTMLLinkContext SOURCE_SRCSET = new HTMLLinkContext("source", "srcset"); final public static HTMLLinkContext SCRIPT_SRC = new HTMLLinkContext("script", "src"); final public static HTMLLinkContext META_HREF = new HTMLLinkContext("meta", "href"); @@ -56,6 +58,9 @@ public class HTMLLinkContext extends LinkContext { } else if (attr.equals("src") || attr.equals("SRC")) { if (el.equals("img") || attr.equals("IMG")) return IMG_SRC; if (el.equals("script") || attr.equals("SCRIPT")) return SCRIPT_SRC; + } else if (attr.equals("srcset") || attr.equals("SRCSET")) { + if (el.equals("img") || attr.equals("IMG")) return IMG_SRCSET; + if (el.equals("source") || attr.equals("SOURCE")) return SOURCE_SRCSET; } return new HTMLLinkContext(el, attr); } @@ -72,6 +77,8 @@ public class HTMLLinkContext extends LinkContext { if (path.equalsIgnoreCase("a/@href")) return A_HREF; if (path.equalsIgnoreCase("meta/@href")) return META_HREF; if (path.equalsIgnoreCase("img/@src")) return IMG_SRC; + if (path.equalsIgnoreCase("img/@srcset")) return IMG_SRCSET; + if (path.equalsIgnoreCase("source/@srcset")) return SOURCE_SRCSET; if (path.equalsIgnoreCase("script/@src")) return SCRIPT_SRC; return new HTMLLinkContext(path); } diff --git a/modules/src/test/java/org/archive/modules/extractor/ExtractorHTMLTest.java b/modules/src/test/java/org/archive/modules/extractor/ExtractorHTMLTest.java index 9da31fc6..9c947067 100644 --- a/modules/src/test/java/org/archive/modules/extractor/ExtractorHTMLTest.java +++ b/modules/src/test/java/org/archive/modules/extractor/ExtractorHTMLTest.java @@ -434,5 +434,57 @@ public class ExtractorHTMLTest extends StringExtractorTestBase { links[1].getURI()); } + + public void testImgSrcSetAttribute() throws URIException { + CrawlURI curi = new CrawlURI(UURIFactory.getInstance("http://www.example.com/")); + + CharSequence cs = "\"\""; + + getExtractor().extract(curi, cs); + + CrawlURI[] links = curi.getOutLinks().toArray(new CrawlURI[0]); + Arrays.sort(links); + + String[] dest = { + "http://www.example.com/images/foo.jpg", + "http://www.example.com/images/foo1.jpg", + "http://www.example.com/images/foo2.jpg", + "http://www.example.com/images/foo3.jpg" }; + + for (int i = 0; i < links.length; i++) { + assertEquals("outlink from img", dest[i], links[i].getURI()); + } + + } + + public void testSourceSrcSetAttribute() throws URIException { + CrawlURI curi = new CrawlURI(UURIFactory.getInstance("http://www.example.com/")); + + CharSequence cs = "" + + " " + + " " + + " " + + "\"\" " + + ""; + + getExtractor().extract(curi, cs); + + CrawlURI[] links = curi.getOutLinks().toArray(new CrawlURI[0]); + Arrays.sort(links); + + String[] dest = { + "http://www.example.com/images/foo.jpg", + "http://www.example.com/images/foo1.jpg", + "http://www.example.com/images/foo2.jpg", + "http://www.example.com/images/foo3.jpg" }; + + for (int i = 0; i < links.length; i++) { + assertEquals("outlink from picture", dest[i], links[i].getURI()); + } + + } } From 10541184b5c47cf36524d58262177ba10a9e8fa2 Mon Sep 17 00:00:00 2001 From: Neil Minton Date: Thu, 18 May 2017 11:20:01 -0500 Subject: [PATCH 2/2] Clean up after code review. - Parse with string split instead of regex. - Move srcset logic up to processEmbed(CrawlURI, CharSequence, CharSequence, Hop). --- .../modules/extractor/ExtractorHTML.java | 42 +++++++++---------- 1 file changed, 20 insertions(+), 22 deletions(-) diff --git a/modules/src/main/java/org/archive/modules/extractor/ExtractorHTML.java b/modules/src/main/java/org/archive/modules/extractor/ExtractorHTML.java index 2795a666..6fa53825 100644 --- a/modules/src/main/java/org/archive/modules/extractor/ExtractorHTML.java +++ b/modules/src/main/java/org/archive/modules/extractor/ExtractorHTML.java @@ -662,23 +662,7 @@ public class ExtractorHTML extends ContentExtractor implements InitializingBean // ReplayCharSequence. HTMLLinkContext hc = HTMLLinkContext.get(context.toString()); int max = getExtractorParameters().getMaxOutlinks(); - if (hc.equals(HTMLLinkContext.IMG_SRCSET) || hc.equals(HTMLLinkContext.SOURCE_SRCSET)) { - - logger.fine("Found srcset listing: " + uri.toString()); - - Matcher srcSetUris = TextUtils.getMatcher("([\\w:/_.-]+)(?: [\\d.]+(?:w|x),?)*",uri); - String srcSetUri; - - while (srcSetUris.find()) { - srcSetUri = srcSetUris.group(1); - logger.finer("Found " + srcSetUri.toString() + "adding to outlinks."); - addRelativeToBase(curi, max, srcSetUri.toString(), hc, hop); - } - - TextUtils.recycleMatcher(srcSetUris); - } else { - addRelativeToBase(curi, max, uri.toString(), hc, hop); - } + addRelativeToBase(curi, max, uri.toString(), hc, hop); } catch (URIException e) { logUriError(e, curi.getUURI(), uri); } @@ -695,11 +679,25 @@ public class ExtractorHTML extends ContentExtractor implements InitializingBean logger.finest("embed (" + hop.getHopChar() + "): " + value.toString() + " from " + curi); } - addLinkFromString(curi, - (value instanceof String)? - (String)value: value.toString(), - context, hop); - numberOfLinksExtracted.incrementAndGet(); + + if (context.equals(HTMLLinkContext.IMG_SRCSET.toString()) || context.equals(HTMLLinkContext.SOURCE_SRCSET.toString())) { + + logger.fine("Found srcset listing: " + value.toString()); + + String[] links = value.toString().split(","); + for (int i=0; i < links.length; i++){ + String link = links[i].trim().split(" +")[0]; + logger.finer("Found " + link + " adding to outlinks."); + addLinkFromString(curi, link, context, hop); + numberOfLinksExtracted.incrementAndGet(); + } + } else { + addLinkFromString(curi, + (value instanceof String)? + (String)value: value.toString(), + context, hop); + numberOfLinksExtracted.incrementAndGet(); + } }