lee/heritrix3
1
0
Fork 0
mirror of https://github.com/internetarchive/heritrix3.git synced 2026-09-22 21:55:54 +00:00
Code Issues Packages Projects Releases Wiki Activity

Add parsing for HTML tags (data-src, data-srcset, data-original and data-original-set)

Browse Source
This commit is contained in:
Clara Wiatrowski
2020-04-10 15:59:50 +02:00
parent fc1de30668
commit 6710c8dba3
3 changed files with 1082 additions and 927 deletions
Download Patch File Download Diff File Expand all files Collapse all files
modules/src/main/java/org/archive/modules/extractor/ExtractorHTML.java
+927 -926
View File
File diff suppressed because it is too large Load Diff
modules/src/main/java/org/archive/modules/extractor/HTMLLinkContext.java
+19
View File
@@ -34,6 +34,11 @@ public class HTMLLinkContext extends LinkContext {
final public static HTMLLinkContext IMG_SRC = new HTMLLinkContext("img", "src");
final public static HTMLLinkContext IMG_SRCSET = new HTMLLinkContext("img", "srcset");
final public static HTMLLinkContext SOURCE_SRCSET = new HTMLLinkContext("source", "srcset");
final public static HTMLLinkContext IMG_DATA_SRC = new HTMLLinkContext("img", "data-src");
final public static HTMLLinkContext IMG_DATA_SRCSET = new HTMLLinkContext("img", "data-srcset");
final public static HTMLLinkContext IMG_DATA_ORIGINAL = new HTMLLinkContext("img", "data-original");
final public static HTMLLinkContext IMG_DATA_ORIGINAL_SET = new HTMLLinkContext("source", "data-original-set");
final public static HTMLLinkContext SOURCE_DATA_ORIGINAL_SET = new HTMLLinkContext("source", "data-original-set");
final public static HTMLLinkContext SCRIPT_SRC = new HTMLLinkContext("script", "src");
final public static HTMLLinkContext META_HREF = new HTMLLinkContext("meta", "href");
@@ -61,6 +66,15 @@ public class HTMLLinkContext extends LinkContext {
} else if (attr.equals("srcset") || attr.equals("SRCSET")) {
if (el.equals("img") || attr.equals("IMG")) return IMG_SRCSET;
if (el.equals("source") || attr.equals("SOURCE")) return SOURCE_SRCSET;
} else if (attr.equals("data-src") || attr.equals("DATA-SRC")) {
if (el.equals("img") || attr.equals("IMG")) return IMG_DATA_SRC;
} else if (attr.equals("data-srcset") || attr.equals("DATA-SRCSET")) {
if (el.equals("img") || attr.equals("IMG")) return IMG_DATA_SRCSET;
} else if (attr.equals("data-original") || attr.equals("DATA-ORIGINAL")) {
if (el.equals("img") || attr.equals("IMG")) return IMG_DATA_ORIGINAL;
} else if (attr.equals("data-original-set") || attr.equals("DATA-ORIGINAL-SET")) {
if (el.equals("img") || attr.equals("IMG")) return IMG_DATA_ORIGINAL_SET;
if (el.equals("source") || attr.equals("SOURCE")) return SOURCE_DATA_ORIGINAL_SET;
}
return new HTMLLinkContext(el, attr);
}
@@ -80,6 +94,11 @@ public class HTMLLinkContext extends LinkContext {
if (path.equalsIgnoreCase("img/@srcset")) return IMG_SRCSET;
if (path.equalsIgnoreCase("source/@srcset")) return SOURCE_SRCSET;
if (path.equalsIgnoreCase("script/@src")) return SCRIPT_SRC;
if (path.equalsIgnoreCase("img/@data-src")) return IMG_DATA_SRC;
if (path.equalsIgnoreCase("img/@data-srcset")) return IMG_DATA_SRCSET;
if (path.equalsIgnoreCase("img/@data-original")) return IMG_DATA_ORIGINAL;
if (path.equalsIgnoreCase("img/@data-original-set")) return IMG_DATA_ORIGINAL_SET;
if (path.equalsIgnoreCase("source/@data-original-set")) return SOURCE_DATA_ORIGINAL_SET;
return new HTMLLinkContext(path);
}
/**
modules/src/test/java/org/archive/modules/extractor/ExtractorHTMLTest.java
+136 -1
View File
@@ -67,7 +67,7 @@ public class ExtractorHTMLTest extends StringExtractorTestBase {
"<img src=\"foo.gif\"> IMG",
"http://www.archive.org/start/foo.gif",
};
@Override
@@ -560,5 +560,140 @@ public class ExtractorHTMLTest extends StringExtractorTestBase {
}
}
public void testDataAttributes20Minutes() throws URIException {
CrawlURI curi_src = new CrawlURI(UURIFactory.getInstance("https://www.20minutes.fr/"));
CharSequence cs_src = "<img class=\"b-lazy\" width=\"120\" height=\"78\""
+ "data-src=\"https://img.20mn.fr/shn9o66FT2-UHl5dl8D38Q/120x78_illustration-avocat.jpg\""
+ "sizes=\"7.5em\" alt=\"Illustration d&#039;un avocat.\"/>";
String[] dest_src = {
"https://img.20mn.fr/shn9o66FT2-UHl5dl8D38Q/120x78_illustration-avocat.jpg"};
genericCrawl(curi_src, cs_src, dest_src);
CrawlURI curi_srcset = new CrawlURI(UURIFactory.getInstance("https://www.20minutes.fr/"));
CharSequence cs_srcset = "<img class=\"b-lazy\" width=\"120\" height=\"78\""
+ "data-srcset=\"https://img.20mn.fr/shn9o66FT2-UHl5dl8D38Q/120x78_illustration-avocat.jpg 120w,https://img.20mn.fr/shn9o66FT2-UHl5dl8D38Q/240x156_illustration-avocat.jpg 240w\""
+ "sizes=\"7.5em\" alt=\"Illustration d&#039;un avocat.\"/>";
String[] dest_srcset = {
"https://img.20mn.fr/shn9o66FT2-UHl5dl8D38Q/120x78_illustration-avocat.jpg",
"https://img.20mn.fr/shn9o66FT2-UHl5dl8D38Q/240x156_illustration-avocat.jpg"};
genericCrawl(curi_srcset, cs_srcset, dest_srcset);
CrawlURI curi_srcset_one = new CrawlURI(UURIFactory.getInstance("https://www.20minutes.fr/"));
CharSequence cs_srcset_one = "<img class=\"b-lazy\" width=\"120\" height=\"78\""
+ "data-srcset=\"https://img.20mn.fr/shn9o66FT2-UHl5dl8D38Q/120x78_illustration-avocat.jpg 120w\"";
String[] dest_srcset_one = {
"https://img.20mn.fr/shn9o66FT2-UHl5dl8D38Q/120x78_illustration-avocat.jpg"};
genericCrawl(curi_srcset_one, cs_srcset_one, dest_srcset_one);
}
public void testDataAttributesTelerama() throws URIException {
CrawlURI curi = new CrawlURI(UURIFactory.getInstance("https://www.telerama.fr/"));
CharSequence cs = "<img itemprop=\"image\" src=\"https://www.telerama.fr/sites/tr_master/themes/tr/images/trans.gif\" " +
"data-original=\"https://www.telerama.fr/sites/tr_master/files/styles/m_640x314/public/standup.jpg?itok=w1aDSzBQsc=1012e84ed57e1b1e6ea74a47ec094242\"/>";
String[] dest = {
"https://www.telerama.fr/sites/tr_master/files/styles/m_640x314/public/standup.jpg?itok=w1aDSzBQsc=1012e84ed57e1b1e6ea74a47ec094242",
"https://www.telerama.fr/sites/tr_master/themes/tr/images/trans.gif"};
genericCrawl(curi, cs, dest);
}
public void testDataAttributesNouvelObs() throws URIException {
CrawlURI curi = new CrawlURI(UURIFactory.getInstance("https://www.telerama.fr/"));
CharSequence cs = "<source media=\"(min-width: 640px)\" data-original-set=\"http://focus.nouvelobs.com/2020/02/07/0/0/640/320/633/306/75/0/59de545_3vVuzAnVb95lp1Hm0OwQ_Jk2.jpeg\"" +
"srcset=\"http://focus.nouvelobs.com/2020/02/07/0/0/640/320/633/306/75/0/59de545_3vVuzAnVb95lp1Hm0OwQ_Jk2.jpeg\">";
String[] dest = {
"http://focus.nouvelobs.com/2020/02/07/0/0/640/320/633/306/75/0/59de545_3vVuzAnVb95lp1Hm0OwQ_Jk2.jpeg",
"http://focus.nouvelobs.com/2020/02/07/0/0/640/320/633/306/75/0/59de545_3vVuzAnVb95lp1Hm0OwQ_Jk2.jpeg"};
genericCrawl(curi, cs, dest);
}
public void testDataAttributesEuronews() throws URIException {
CrawlURI curi = new CrawlURI(UURIFactory.getInstance("https://www.euronews.com/"));
CharSequence cs = "<img class=\"m-img lazyload\" src=\"/images/vector/fallback.svg\""+
"data-src=\"https://static.euronews.com/articles/stories/04/54/38/12/400x225_cmsv2_081071de-f9f4-5341-9512-94f5f494f45f-4543812.jpg\""+
"data-srcset=\"https://static.euronews.com/articles/stories/04/54/38/12/100x56_cmsv2_081071de-f9f4-5341-9512-94f5f494f45f-4543812.jpg 100w, "+
"https://static.euronews.com/articles/stories/04/54/38/12/150x84_cmsv2_081071de-f9f4-5341-9512-94f5f494f45f-4543812.jpg 150w, "+
"https://static.euronews.com/articles/stories/04/54/38/12/300x169_cmsv2_081071de-f9f4-5341-9512-94f5f494f45f-4543812.jpg 300w, "+
"https://static.euronews.com/articles/stories/04/54/38/12/400x225_cmsv2_081071de-f9f4-5341-9512-94f5f494f45f-4543812.jpg 400w, "+
"https://static.euronews.com/articles/stories/04/54/38/12/600x338_cmsv2_081071de-f9f4-5341-9512-94f5f494f45f-4543812.jpg 600w, "+
"https://static.euronews.com/articles/stories/04/54/38/12/750x422_cmsv2_081071de-f9f4-5341-9512-94f5f494f45f-4543812.jpg 750w, "+
"https://static.euronews.com/articles/stories/04/54/38/12/1000x563_cmsv2_081071de-f9f4-5341-9512-94f5f494f45f-4543812.jpg 1000w, "+
"https://static.euronews.com/articles/stories/04/54/38/12/1200x675_cmsv2_081071de-f9f4-5341-9512-94f5f494f45f-4543812.jpg 1200w\""+
"data-sizes=\"(max-width: 768px) 33vw, (max-width: 1024px) 25vw, (max-width: 1280px) 17vw, 17vw\""+
"title=\"La lutte contre l&#039;épidémie de Covid-19 continue en Europe et dans le monde\"/>";
String[] dest = {
"https://static.euronews.com/articles/stories/04/54/38/12/1000x563_cmsv2_081071de-f9f4-5341-9512-94f5f494f45f-4543812.jpg",
"https://static.euronews.com/articles/stories/04/54/38/12/100x56_cmsv2_081071de-f9f4-5341-9512-94f5f494f45f-4543812.jpg",
"https://static.euronews.com/articles/stories/04/54/38/12/1200x675_cmsv2_081071de-f9f4-5341-9512-94f5f494f45f-4543812.jpg",
"https://static.euronews.com/articles/stories/04/54/38/12/150x84_cmsv2_081071de-f9f4-5341-9512-94f5f494f45f-4543812.jpg",
"https://static.euronews.com/articles/stories/04/54/38/12/300x169_cmsv2_081071de-f9f4-5341-9512-94f5f494f45f-4543812.jpg",
"https://static.euronews.com/articles/stories/04/54/38/12/400x225_cmsv2_081071de-f9f4-5341-9512-94f5f494f45f-4543812.jpg",
"https://static.euronews.com/articles/stories/04/54/38/12/400x225_cmsv2_081071de-f9f4-5341-9512-94f5f494f45f-4543812.jpg",
"https://static.euronews.com/articles/stories/04/54/38/12/600x338_cmsv2_081071de-f9f4-5341-9512-94f5f494f45f-4543812.jpg",
"https://static.euronews.com/articles/stories/04/54/38/12/750x422_cmsv2_081071de-f9f4-5341-9512-94f5f494f45f-4543812.jpg",
"https://www.euronews.com/images/vector/fallback.svg"
};
genericCrawl(curi, cs, dest);
}
public void testDataAttributesLeMonde() throws URIException {
CrawlURI curi = new CrawlURI(UURIFactory.getInstance("https://www.telerama.fr/"));
CharSequence cs = "<img data-srcset=\"http://img.lemde.fr/2020/02/29/0/0/4818/3212/384/0/60/0/c7dda5e_5282901-01-06.jpg 198w, "+
"http://img.lemde.fr/2020/02/29/0/0/4818/3212/114/0/95/0/c7dda5e_5282901-01-06.jpg 114w\" "+
"data-src=\"http://img.lemde.fr/2020/02/29/0/0/4818/3212/384/0/60/0/c7dda5e_5282901-01-06.jpg\" "+
"srcset=\"http://img.lemde.fr/2020/02/29/0/0/4818/3212/384/0/60/0/c7dda5e_5282901-01-06.jpg 198w, "+
"http://img.lemde.fr/2020/02/29/0/0/4818/3212/114/0/95/0/c7dda5e_5282901-01-06.jpg 114w\" "+
"src=\"http://img.lemde.fr/2020/02/29/0/0/4818/3212/384/0/60/0/c7dda5e_5282901-01-06.jpg\" >";
String[] dest = {
"http://img.lemde.fr/2020/02/29/0/0/4818/3212/114/0/95/0/c7dda5e_5282901-01-06.jpg",
"http://img.lemde.fr/2020/02/29/0/0/4818/3212/114/0/95/0/c7dda5e_5282901-01-06.jpg",
"http://img.lemde.fr/2020/02/29/0/0/4818/3212/384/0/60/0/c7dda5e_5282901-01-06.jpg",
"http://img.lemde.fr/2020/02/29/0/0/4818/3212/384/0/60/0/c7dda5e_5282901-01-06.jpg",
"http://img.lemde.fr/2020/02/29/0/0/4818/3212/384/0/60/0/c7dda5e_5282901-01-06.jpg",
"http://img.lemde.fr/2020/02/29/0/0/4818/3212/384/0/60/0/c7dda5e_5282901-01-06.jpg",
"http://img.lemde.fr/2020/02/29/0/0/4818/3212/384/0/60/0/c7dda5e_5282901-01-06.jpg"
};
genericCrawl(curi, cs, dest);
}
private void genericCrawl(CrawlURI curi, CharSequence cs,String[] dest){
getExtractor().extract(curi, cs);
CrawlURI[] links = curi.getOutLinks().toArray(new CrawlURI[0]);
Arrays.sort(links);
for (int i = 0; i < links.length; i++) {
assertEquals("outlink from picture", dest[i], links[i].getURI());
}
}
}