lee/heritrix3
1
0
Fork 0
mirror of https://github.com/internetarchive/heritrix3.git synced 2026-07-22 15:31:47 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #179 from BitBaron/aitfive-1225

Browse Source 
Add parsing for srcset attributes
This commit is contained in:
Noah Levitt
2017-06-02 12:01:17 -07:00
committed by GitHub
parent 752ea028f6 10541184b5
commit a4bb99e828
3 changed files with 81 additions and 8 deletions
Download Patch File Download Diff File Expand all files Collapse all files
modules/src/main/java/org/archive/modules/extractor/ExtractorHTML.java
+22 -8
View File
@@ -187,8 +187,8 @@ public class ExtractorHTML extends ContentExtractor implements InitializingBean
// sorts are matched specially
static final String EACH_ATTRIBUTE_EXTRACTOR =
"(?is)\\s?((href)|(action)|(on\\w*)" // 1, 2, 3, 4
+"|((?:src)|(?:lowsrc)|(?:background)|(?:cite)|(?:longdesc)" // ...
+"|(?:usemap)|(?:profile)|(?:datasrc))" // 5
+"|((?:src)|(?:srcset)|(?:lowsrc)|(?:background)|(?:cite)" // ...
+"|(?:longdesc)|(?:usemap)|(?:profile)|(?:datasrc))" // 5
+"|(codebase)|((?:classid)|(?:data))|(archive)|(code)" // 6, 7, 8, 9
+"|(value)|(style)|(method)" // 10, 11, 12
+"|([-\\w]{1,"+MAX_ATTR_NAME_REPLACE+"}))" // 13
@@ -201,7 +201,7 @@ public class ExtractorHTML extends ContentExtractor implements InitializingBean
// 2: HREF - single URI relative to doc base, or occasionally javascript:
// 3: ACTION - single URI relative to doc base, or occasionally javascript:
// 4: ON[WHATEVER] - script handler
// 5: SRC,LOWSRC,BACKGROUND,CITE,LONGDESC,USEMAP,PROFILE, or DATASRC
// 5: SRC,SRCSET,LOWSRC,BACKGROUND,CITE,LONGDESC,USEMAP,PROFILE, or DATASRC
// single URI relative to doc base
// 6: CODEBASE - a single URI relative to doc base, affecting other
// attributes
@@ -679,11 +679,25 @@ public class ExtractorHTML extends ContentExtractor implements InitializingBean
logger.finest("embed (" + hop.getHopChar() + "): " + value.toString() +
" from " + curi);
}
addLinkFromString(curi,
(value instanceof String)?
(String)value: value.toString(),
context, hop);
numberOfLinksExtracted.incrementAndGet();
if (context.equals(HTMLLinkContext.IMG_SRCSET.toString()) || context.equals(HTMLLinkContext.SOURCE_SRCSET.toString())) {
logger.fine("Found srcset listing: " + value.toString());
String[] links = value.toString().split(",");
for (int i=0; i < links.length; i++){
String link = links[i].trim().split(" +")[0];
logger.finer("Found " + link + " adding to outlinks.");
addLinkFromString(curi, link, context, hop);
numberOfLinksExtracted.incrementAndGet();
}
} else {
addLinkFromString(curi,
(value instanceof String)?
(String)value: value.toString(),
context, hop);
numberOfLinksExtracted.incrementAndGet();
}
}
modules/src/main/java/org/archive/modules/extractor/HTMLLinkContext.java
+7
View File
@@ -32,6 +32,8 @@ public class HTMLLinkContext extends LinkContext {
final public static HTMLLinkContext META = new HTMLLinkContext("meta");
final public static HTMLLinkContext A_HREF = new HTMLLinkContext("a", "href");
final public static HTMLLinkContext IMG_SRC = new HTMLLinkContext("img", "src");
final public static HTMLLinkContext IMG_SRCSET = new HTMLLinkContext("img", "srcset");
final public static HTMLLinkContext SOURCE_SRCSET = new HTMLLinkContext("source", "srcset");
final public static HTMLLinkContext SCRIPT_SRC = new HTMLLinkContext("script", "src");
final public static HTMLLinkContext META_HREF = new HTMLLinkContext("meta", "href");
@@ -56,6 +58,9 @@ public class HTMLLinkContext extends LinkContext {
} else if (attr.equals("src") || attr.equals("SRC")) {
if (el.equals("img") || attr.equals("IMG")) return IMG_SRC;
if (el.equals("script") || attr.equals("SCRIPT")) return SCRIPT_SRC;
} else if (attr.equals("srcset") || attr.equals("SRCSET")) {
if (el.equals("img") || attr.equals("IMG")) return IMG_SRCSET;
if (el.equals("source") || attr.equals("SOURCE")) return SOURCE_SRCSET;
}
return new HTMLLinkContext(el, attr);
}
@@ -72,6 +77,8 @@ public class HTMLLinkContext extends LinkContext {
if (path.equalsIgnoreCase("a/@href")) return A_HREF;
if (path.equalsIgnoreCase("meta/@href")) return META_HREF;
if (path.equalsIgnoreCase("img/@src")) return IMG_SRC;
if (path.equalsIgnoreCase("img/@srcset")) return IMG_SRCSET;
if (path.equalsIgnoreCase("source/@srcset")) return SOURCE_SRCSET;
if (path.equalsIgnoreCase("script/@src")) return SCRIPT_SRC;
return new HTMLLinkContext(path);
}
modules/src/test/java/org/archive/modules/extractor/ExtractorHTMLTest.java
+52
View File
@@ -434,5 +434,57 @@ public class ExtractorHTMLTest extends StringExtractorTestBase {
links[1].getURI());
}
public void testImgSrcSetAttribute() throws URIException {
CrawlURI curi = new CrawlURI(UURIFactory.getInstance("http://www.example.com/"));
CharSequence cs = "<img width=\"800\" height=\"1200\" src=\"/images/foo.jpg\" "
+ "class=\"attachment-full size-full\" alt=\"\" "
+ "srcset=\"/images/foo1.jpg 800w, /images/foo2.jpg 480w, /images/foo3.jpg 96w\" "
+ "sizes=\"(max-width: 800px) 100vw, 800px\">";
getExtractor().extract(curi, cs);
CrawlURI[] links = curi.getOutLinks().toArray(new CrawlURI[0]);
Arrays.sort(links);
String[] dest = {
"http://www.example.com/images/foo.jpg",
"http://www.example.com/images/foo1.jpg",
"http://www.example.com/images/foo2.jpg",
"http://www.example.com/images/foo3.jpg" };
for (int i = 0; i < links.length; i++) {
assertEquals("outlink from img", dest[i], links[i].getURI());
}
}
public void testSourceSrcSetAttribute() throws URIException {
CrawlURI curi = new CrawlURI(UURIFactory.getInstance("http://www.example.com/"));
CharSequence cs = "<picture>"
+ "<source media=\"(min-width: 992px)\" srcset=\"images/foo1.jpg\"> "
+ "<source media=\"(min-width: 500px)\" srcset=\"images/foo2.jpg\"> "
+ "<source media=\"(min-width: 0px)\" srcset=\"images/foo3.jpg\"> "
+ "<img src=\"images/foo.jpg\" alt=\"\"> "
+ "</picture>";
getExtractor().extract(curi, cs);
CrawlURI[] links = curi.getOutLinks().toArray(new CrawlURI[0]);
Arrays.sort(links);
String[] dest = {
"http://www.example.com/images/foo.jpg",
"http://www.example.com/images/foo1.jpg",
"http://www.example.com/images/foo2.jpg",
"http://www.example.com/images/foo3.jpg" };
for (int i = 0; i < links.length; i++) {
assertEquals("outlink from picture", dest[i], links[i].getURI());
}
}
}