diff --git a/modules/src/main/java/org/archive/modules/extractor/ExtractorHTML.java b/modules/src/main/java/org/archive/modules/extractor/ExtractorHTML.java index dfb563b5..6fa53825 100644 --- a/modules/src/main/java/org/archive/modules/extractor/ExtractorHTML.java +++ b/modules/src/main/java/org/archive/modules/extractor/ExtractorHTML.java @@ -187,8 +187,8 @@ public class ExtractorHTML extends ContentExtractor implements InitializingBean // sorts are matched specially static final String EACH_ATTRIBUTE_EXTRACTOR = "(?is)\\s?((href)|(action)|(on\\w*)" // 1, 2, 3, 4 - +"|((?:src)|(?:lowsrc)|(?:background)|(?:cite)|(?:longdesc)" // ... - +"|(?:usemap)|(?:profile)|(?:datasrc))" // 5 + +"|((?:src)|(?:srcset)|(?:lowsrc)|(?:background)|(?:cite)" // ... + +"|(?:longdesc)|(?:usemap)|(?:profile)|(?:datasrc))" // 5 +"|(codebase)|((?:classid)|(?:data))|(archive)|(code)" // 6, 7, 8, 9 +"|(value)|(style)|(method)" // 10, 11, 12 +"|([-\\w]{1,"+MAX_ATTR_NAME_REPLACE+"}))" // 13 @@ -201,7 +201,7 @@ public class ExtractorHTML extends ContentExtractor implements InitializingBean // 2: HREF - single URI relative to doc base, or occasionally javascript: // 3: ACTION - single URI relative to doc base, or occasionally javascript: // 4: ON[WHATEVER] - script handler - // 5: SRC,LOWSRC,BACKGROUND,CITE,LONGDESC,USEMAP,PROFILE, or DATASRC + // 5: SRC,SRCSET,LOWSRC,BACKGROUND,CITE,LONGDESC,USEMAP,PROFILE, or DATASRC // single URI relative to doc base // 6: CODEBASE - a single URI relative to doc base, affecting other // attributes @@ -679,11 +679,25 @@ public class ExtractorHTML extends ContentExtractor implements InitializingBean logger.finest("embed (" + hop.getHopChar() + "): " + value.toString() + " from " + curi); } - addLinkFromString(curi, - (value instanceof String)? - (String)value: value.toString(), - context, hop); - numberOfLinksExtracted.incrementAndGet(); + + if (context.equals(HTMLLinkContext.IMG_SRCSET.toString()) || context.equals(HTMLLinkContext.SOURCE_SRCSET.toString())) { + + logger.fine("Found srcset listing: " + value.toString()); + + String[] links = value.toString().split(","); + for (int i=0; i < links.length; i++){ + String link = links[i].trim().split(" +")[0]; + logger.finer("Found " + link + " adding to outlinks."); + addLinkFromString(curi, link, context, hop); + numberOfLinksExtracted.incrementAndGet(); + } + } else { + addLinkFromString(curi, + (value instanceof String)? + (String)value: value.toString(), + context, hop); + numberOfLinksExtracted.incrementAndGet(); + } } diff --git a/modules/src/main/java/org/archive/modules/extractor/HTMLLinkContext.java b/modules/src/main/java/org/archive/modules/extractor/HTMLLinkContext.java index f44cef74..f11aaa47 100644 --- a/modules/src/main/java/org/archive/modules/extractor/HTMLLinkContext.java +++ b/modules/src/main/java/org/archive/modules/extractor/HTMLLinkContext.java @@ -32,6 +32,8 @@ public class HTMLLinkContext extends LinkContext { final public static HTMLLinkContext META = new HTMLLinkContext("meta"); final public static HTMLLinkContext A_HREF = new HTMLLinkContext("a", "href"); final public static HTMLLinkContext IMG_SRC = new HTMLLinkContext("img", "src"); + final public static HTMLLinkContext IMG_SRCSET = new HTMLLinkContext("img", "srcset"); + final public static HTMLLinkContext SOURCE_SRCSET = new HTMLLinkContext("source", "srcset"); final public static HTMLLinkContext SCRIPT_SRC = new HTMLLinkContext("script", "src"); final public static HTMLLinkContext META_HREF = new HTMLLinkContext("meta", "href"); @@ -56,6 +58,9 @@ public class HTMLLinkContext extends LinkContext { } else if (attr.equals("src") || attr.equals("SRC")) { if (el.equals("img") || attr.equals("IMG")) return IMG_SRC; if (el.equals("script") || attr.equals("SCRIPT")) return SCRIPT_SRC; + } else if (attr.equals("srcset") || attr.equals("SRCSET")) { + if (el.equals("img") || attr.equals("IMG")) return IMG_SRCSET; + if (el.equals("source") || attr.equals("SOURCE")) return SOURCE_SRCSET; } return new HTMLLinkContext(el, attr); } @@ -72,6 +77,8 @@ public class HTMLLinkContext extends LinkContext { if (path.equalsIgnoreCase("a/@href")) return A_HREF; if (path.equalsIgnoreCase("meta/@href")) return META_HREF; if (path.equalsIgnoreCase("img/@src")) return IMG_SRC; + if (path.equalsIgnoreCase("img/@srcset")) return IMG_SRCSET; + if (path.equalsIgnoreCase("source/@srcset")) return SOURCE_SRCSET; if (path.equalsIgnoreCase("script/@src")) return SCRIPT_SRC; return new HTMLLinkContext(path); } diff --git a/modules/src/test/java/org/archive/modules/extractor/ExtractorHTMLTest.java b/modules/src/test/java/org/archive/modules/extractor/ExtractorHTMLTest.java index 9da31fc6..9c947067 100644 --- a/modules/src/test/java/org/archive/modules/extractor/ExtractorHTMLTest.java +++ b/modules/src/test/java/org/archive/modules/extractor/ExtractorHTMLTest.java @@ -434,5 +434,57 @@ public class ExtractorHTMLTest extends StringExtractorTestBase { links[1].getURI()); } + + public void testImgSrcSetAttribute() throws URIException { + CrawlURI curi = new CrawlURI(UURIFactory.getInstance("http://www.example.com/")); + + CharSequence cs = "\"\""; + + getExtractor().extract(curi, cs); + + CrawlURI[] links = curi.getOutLinks().toArray(new CrawlURI[0]); + Arrays.sort(links); + + String[] dest = { + "http://www.example.com/images/foo.jpg", + "http://www.example.com/images/foo1.jpg", + "http://www.example.com/images/foo2.jpg", + "http://www.example.com/images/foo3.jpg" }; + + for (int i = 0; i < links.length; i++) { + assertEquals("outlink from img", dest[i], links[i].getURI()); + } + + } + + public void testSourceSrcSetAttribute() throws URIException { + CrawlURI curi = new CrawlURI(UURIFactory.getInstance("http://www.example.com/")); + + CharSequence cs = "" + + " " + + " " + + " " + + "\"\" " + + ""; + + getExtractor().extract(curi, cs); + + CrawlURI[] links = curi.getOutLinks().toArray(new CrawlURI[0]); + Arrays.sort(links); + + String[] dest = { + "http://www.example.com/images/foo.jpg", + "http://www.example.com/images/foo1.jpg", + "http://www.example.com/images/foo2.jpg", + "http://www.example.com/images/foo3.jpg" }; + + for (int i = 0; i < links.length; i++) { + assertEquals("outlink from picture", dest[i], links[i].getURI()); + } + + } }