diff --git a/modules/src/main/java/org/archive/modules/extractor/ExtractorHTML.java b/modules/src/main/java/org/archive/modules/extractor/ExtractorHTML.java index 88366663..881b0c74 100644 --- a/modules/src/main/java/org/archive/modules/extractor/ExtractorHTML.java +++ b/modules/src/main/java/org/archive/modules/extractor/ExtractorHTML.java @@ -538,6 +538,25 @@ public class ExtractorHTML extends ContentExtractor implements InitializingBean // remember 'rel' for end-analysis linkRel = value; } + + // 2023 updates get img or source data attr + CharSequence context = elementContext(element, attr.group(13)); + if (TextUtils.matches( + "data-(src|src-small|src-medium|srcset|original|original-set|lazy|lazy-srcset|full-src)", // + attr.group(13).toLowerCase())) { + + // true, if we expect another HTML page instead of an image etc. + final Hop hop; + + if (!framesAsEmbeds + && (elementStr.equalsIgnoreCase(FRAME) || elementStr.equalsIgnoreCase(IFRAME))) { + hop = Hop.NAVLINK; + } else { + hop = Hop.EMBED; + } + processEmbed(curi, value, context, hop); + } + // any other attribute // ignore for now // could probe for path- or script-looking strings, but @@ -737,11 +756,18 @@ public class ExtractorHTML extends ContentExtractor implements InitializingBean " from " + curi); } - if (context.equals(HTMLLinkContext.IMG_SRCSET.toString()) + if (context.equals(HTMLLinkContext.IMG_SRCSET.toString()) + || context.equals(HTMLLinkContext.IMG_DATA_SRC.toString()) || context.equals(HTMLLinkContext.SOURCE_SRCSET.toString()) || context.equals(HTMLLinkContext.IMG_DATA_SRCSET.toString()) + || context.equals(HTMLLinkContext.SOURCE_DATA_SRCSET.toString()) + || context.equals(HTMLLinkContext.SOURCE_DATA_LAZY_SRCSET.toString()) + || context.equals(HTMLLinkContext.IMG_DATA_LAZY_SRCSET.toString()) + || context.equals(HTMLLinkContext.IMG_DATA_SRC_MEDIUM.toString()) + || context.equals(HTMLLinkContext.IMG_DATA_SRC_SMALL.toString()) || context.equals(HTMLLinkContext.IMG_DATA_ORIGINAL_SET.toString()) - || context.equals(HTMLLinkContext.SOURCE_DATA_ORIGINAL_SET.toString())) { + || context.equals(HTMLLinkContext.SOURCE_DATA_ORIGINAL_SET.toString()) + || context.equals(HTMLLinkContext.LINK_IMAGESRCSET.toString())) { logger.log(Level.FINE,"Found srcset listing: {0}", value); Matcher matcher = TextUtils.getMatcher("[\\s,]*(\\S*[^,\\s])(?:\\s(?:[^,(]+|\\([^)]*(?:\\)|$))*)?", value); diff --git a/modules/src/main/java/org/archive/modules/extractor/HTMLLinkContext.java b/modules/src/main/java/org/archive/modules/extractor/HTMLLinkContext.java index 4a541d5b..2186059e 100644 --- a/modules/src/main/java/org/archive/modules/extractor/HTMLLinkContext.java +++ b/modules/src/main/java/org/archive/modules/extractor/HTMLLinkContext.java @@ -35,12 +35,20 @@ public class HTMLLinkContext extends LinkContext { final public static HTMLLinkContext IMG_SRCSET = new HTMLLinkContext("img", "srcset"); final public static HTMLLinkContext SOURCE_SRCSET = new HTMLLinkContext("source", "srcset"); final public static HTMLLinkContext IMG_DATA_SRC = new HTMLLinkContext("img", "data-src"); + final public static HTMLLinkContext IMG_DATA_SRC_SMALL = new HTMLLinkContext("img", "data-src-small"); + final public static HTMLLinkContext IMG_DATA_SRC_MEDIUM = new HTMLLinkContext("img", "data-src-medium"); final public static HTMLLinkContext IMG_DATA_SRCSET = new HTMLLinkContext("img", "data-srcset"); + final public static HTMLLinkContext SOURCE_DATA_SRCSET = new HTMLLinkContext("source", "data-srcset"); final public static HTMLLinkContext IMG_DATA_ORIGINAL = new HTMLLinkContext("img", "data-original"); - final public static HTMLLinkContext IMG_DATA_ORIGINAL_SET = new HTMLLinkContext("source", "data-original-set"); - final public static HTMLLinkContext SOURCE_DATA_ORIGINAL_SET = new HTMLLinkContext("source", "data-original-set"); + final public static HTMLLinkContext IMG_DATA_ORIGINAL_SET = new HTMLLinkContext("img", "data-original-set"); + final public static HTMLLinkContext SOURCE_DATA_ORIGINAL_SET = new HTMLLinkContext("source", "data-original-set"); + final public static HTMLLinkContext IMG_DATA_LAZY = new HTMLLinkContext("img", "data-lazy"); + final public static HTMLLinkContext IMG_DATA_LAZY_SRCSET = new HTMLLinkContext("img", "data-lazy-srcset"); + final public static HTMLLinkContext SOURCE_DATA_LAZY_SRCSET = new HTMLLinkContext("source", "data-lazy-srcset"); + final public static HTMLLinkContext IMG_DATA_FULL_SRC = new HTMLLinkContext("img", "data-full-src"); final public static HTMLLinkContext SCRIPT_SRC = new HTMLLinkContext("script", "src"); final public static HTMLLinkContext META_HREF = new HTMLLinkContext("meta", "href"); + final public static HTMLLinkContext LINK_IMAGESRCSET = new HTMLLinkContext("link", "imagesrcset"); /** @@ -57,24 +65,54 @@ public class HTMLLinkContext extends LinkContext { * @return instance of HTMLLinkContext */ public static HTMLLinkContext get(CharSequence el, CharSequence attr) { - if (attr.equals("href") || attr.equals("HREF")) { - if (el.equals("a") || el.equals("A")) return A_HREF; - if (el.equals("meta") || el.equals("META")) return META_HREF; - } else if (attr.equals("src") || attr.equals("SRC")) { - if (el.equals("img") || attr.equals("IMG")) return IMG_SRC; - if (el.equals("script") || attr.equals("SCRIPT")) return SCRIPT_SRC; - } else if (attr.equals("srcset") || attr.equals("SRCSET")) { - if (el.equals("img") || attr.equals("IMG")) return IMG_SRCSET; - if (el.equals("source") || attr.equals("SOURCE")) return SOURCE_SRCSET; - } else if (attr.equals("data-src") || attr.equals("DATA-SRC")) { - if (el.equals("img") || attr.equals("IMG")) return IMG_DATA_SRC; - } else if (attr.equals("data-srcset") || attr.equals("DATA-SRCSET")) { - if (el.equals("img") || attr.equals("IMG")) return IMG_DATA_SRCSET; - } else if (attr.equals("data-original") || attr.equals("DATA-ORIGINAL")) { - if (el.equals("img") || attr.equals("IMG")) return IMG_DATA_ORIGINAL; - } else if (attr.equals("data-original-set") || attr.equals("DATA-ORIGINAL-SET")) { - if (el.equals("img") || attr.equals("IMG")) return IMG_DATA_ORIGINAL_SET; - if (el.equals("source") || attr.equals("SOURCE")) return SOURCE_DATA_ORIGINAL_SET; + switch (attr.toString().toLowerCase()) { + case "href": + if (el.toString().equalsIgnoreCase("a")) return A_HREF; + if (el.toString().equalsIgnoreCase("meta")) return META_HREF; + break; + case "src": + if (el.toString().equalsIgnoreCase("img")) return IMG_SRC; + if (el.toString().equalsIgnoreCase("script")) return SCRIPT_SRC; + break; + case "srcset": + if (el.toString().equalsIgnoreCase("img")) return IMG_SRCSET; + if (el.toString().equalsIgnoreCase("source")) return SOURCE_SRCSET; + break; + case "data-src": + if (el.toString().equalsIgnoreCase("img")) return IMG_DATA_SRC; + break; + case "data-srcset": + if (el.toString().equalsIgnoreCase("img")) return IMG_DATA_SRCSET; + if (el.toString().equalsIgnoreCase("source")) return SOURCE_DATA_SRCSET; + break; + case "data-original": + if (el.toString().equalsIgnoreCase("img")) return IMG_DATA_ORIGINAL; + break; + case "data-original-set": + if (el.toString().equalsIgnoreCase("img")) return IMG_DATA_ORIGINAL_SET; + if (el.toString().equalsIgnoreCase("source")) return SOURCE_DATA_ORIGINAL_SET; + break; + case "data-full-src": + if (el.toString().equalsIgnoreCase("img")) return IMG_DATA_FULL_SRC; + break; + case "data-lazy-srcset": + if (el.toString().equalsIgnoreCase("img")) return IMG_DATA_LAZY_SRCSET; + if (el.toString().equalsIgnoreCase("source")) return SOURCE_DATA_LAZY_SRCSET; + break; + case "data-lazy": + if (el.toString().equalsIgnoreCase("img")) return IMG_DATA_LAZY; + break; + case "data-src-small": + if (el.toString().equalsIgnoreCase("img")) return IMG_DATA_SRC_SMALL; + break; + case "data-src-medium": + if (el.toString().equalsIgnoreCase("img")) return IMG_DATA_SRC_MEDIUM; + break; + case "imagesrcset": + if (el.toString().equalsIgnoreCase("link")) return LINK_IMAGESRCSET; + break; + default: + return new HTMLLinkContext(el, attr); } return new HTMLLinkContext(el, attr); } @@ -95,10 +133,18 @@ public class HTMLLinkContext extends LinkContext { if (path.equalsIgnoreCase("source/@srcset")) return SOURCE_SRCSET; if (path.equalsIgnoreCase("script/@src")) return SCRIPT_SRC; if (path.equalsIgnoreCase("img/@data-src")) return IMG_DATA_SRC; + if (path.equalsIgnoreCase("img/@data-src-small")) return IMG_DATA_SRC_SMALL; + if (path.equalsIgnoreCase("img/@data-src-medium")) return IMG_DATA_SRC_MEDIUM; if (path.equalsIgnoreCase("img/@data-srcset")) return IMG_DATA_SRCSET; if (path.equalsIgnoreCase("img/@data-original")) return IMG_DATA_ORIGINAL; + if (path.equalsIgnoreCase("img/@data-full-src")) return IMG_DATA_FULL_SRC; if (path.equalsIgnoreCase("img/@data-original-set")) return IMG_DATA_ORIGINAL_SET; if (path.equalsIgnoreCase("source/@data-original-set")) return SOURCE_DATA_ORIGINAL_SET; + if (path.equalsIgnoreCase("img/@data-lazy-srcset")) return IMG_DATA_LAZY_SRCSET; + if (path.equalsIgnoreCase("source/@data-lazy-srcset")) return SOURCE_DATA_LAZY_SRCSET; + if (path.equalsIgnoreCase("img/@data-lazy")) return IMG_DATA_LAZY; + if (path.equalsIgnoreCase("source/@data-srcset")) return SOURCE_DATA_SRCSET; + if (path.equalsIgnoreCase("link/@imagesrcset")) return LINK_IMAGESRCSET; return new HTMLLinkContext(path); } /**