lee/heritrix3
1
0
Fork 0
mirror of https://github.com/internetarchive/heritrix3.git synced 2026-09-24 06:36:12 +00:00
Code Issues Packages Projects Releases Wiki Activity

Add new HTML tags and attributes for ExtractorHTML (#604)

Browse Source
This commit is contained in:
Leslie Bellony
2024-09-04 17:49:40 +02:00
parent 9405cb9af1
commit ca3508f55f
2 changed files with 94 additions and 22 deletions
Download Patch File Download Diff File Expand all files Collapse all files
modules/src/main/java/org/archive/modules/extractor/ExtractorHTML.java
+28 -2
View File
@@ -538,6 +538,25 @@ public class ExtractorHTML extends ContentExtractor implements InitializingBean
// remember 'rel' for end-analysis
linkRel = value;
}
// 2023 updates get img or source data attr
CharSequence context = elementContext(element, attr.group(13));
if (TextUtils.matches(
"data-(src|src-small|src-medium|srcset|original|original-set|lazy|lazy-srcset|full-src)", //
attr.group(13).toLowerCase())) {
// true, if we expect another HTML page instead of an image etc.
final Hop hop;
if (!framesAsEmbeds
&& (elementStr.equalsIgnoreCase(FRAME) || elementStr.equalsIgnoreCase(IFRAME))) {
hop = Hop.NAVLINK;
} else {
hop = Hop.EMBED;
}
processEmbed(curi, value, context, hop);
}
// any other attribute
// ignore for now
// could probe for path- or script-looking strings, but
@@ -737,11 +756,18 @@ public class ExtractorHTML extends ContentExtractor implements InitializingBean
" from " + curi);
}
if (context.equals(HTMLLinkContext.IMG_SRCSET.toString())
if (context.equals(HTMLLinkContext.IMG_SRCSET.toString())
|| context.equals(HTMLLinkContext.IMG_DATA_SRC.toString())
|| context.equals(HTMLLinkContext.SOURCE_SRCSET.toString())
|| context.equals(HTMLLinkContext.IMG_DATA_SRCSET.toString())
|| context.equals(HTMLLinkContext.SOURCE_DATA_SRCSET.toString())
|| context.equals(HTMLLinkContext.SOURCE_DATA_LAZY_SRCSET.toString())
|| context.equals(HTMLLinkContext.IMG_DATA_LAZY_SRCSET.toString())
|| context.equals(HTMLLinkContext.IMG_DATA_SRC_MEDIUM.toString())
|| context.equals(HTMLLinkContext.IMG_DATA_SRC_SMALL.toString())
|| context.equals(HTMLLinkContext.IMG_DATA_ORIGINAL_SET.toString())
|| context.equals(HTMLLinkContext.SOURCE_DATA_ORIGINAL_SET.toString())) {
|| context.equals(HTMLLinkContext.SOURCE_DATA_ORIGINAL_SET.toString())
|| context.equals(HTMLLinkContext.LINK_IMAGESRCSET.toString())) {
logger.log(Level.FINE,"Found srcset listing: {0}", value);
Matcher matcher = TextUtils.getMatcher("[\\s,]*(\\S*[^,\\s])(?:\\s(?:[^,(]+|\\([^)]*(?:\\)|$))*)?", value);
modules/src/main/java/org/archive/modules/extractor/HTMLLinkContext.java
+66 -20
View File
@@ -35,12 +35,20 @@ public class HTMLLinkContext extends LinkContext {
final public static HTMLLinkContext IMG_SRCSET = new HTMLLinkContext("img", "srcset");
final public static HTMLLinkContext SOURCE_SRCSET = new HTMLLinkContext("source", "srcset");
final public static HTMLLinkContext IMG_DATA_SRC = new HTMLLinkContext("img", "data-src");
final public static HTMLLinkContext IMG_DATA_SRC_SMALL = new HTMLLinkContext("img", "data-src-small");
final public static HTMLLinkContext IMG_DATA_SRC_MEDIUM = new HTMLLinkContext("img", "data-src-medium");
final public static HTMLLinkContext IMG_DATA_SRCSET = new HTMLLinkContext("img", "data-srcset");
final public static HTMLLinkContext SOURCE_DATA_SRCSET = new HTMLLinkContext("source", "data-srcset");
final public static HTMLLinkContext IMG_DATA_ORIGINAL = new HTMLLinkContext("img", "data-original");
final public static HTMLLinkContext IMG_DATA_ORIGINAL_SET = new HTMLLinkContext("source", "data-original-set");
final public static HTMLLinkContext SOURCE_DATA_ORIGINAL_SET = new HTMLLinkContext("source", "data-original-set");
final public static HTMLLinkContext IMG_DATA_ORIGINAL_SET = new HTMLLinkContext("img", "data-original-set");
final public static HTMLLinkContext SOURCE_DATA_ORIGINAL_SET = new HTMLLinkContext("source", "data-original-set");
final public static HTMLLinkContext IMG_DATA_LAZY = new HTMLLinkContext("img", "data-lazy");
final public static HTMLLinkContext IMG_DATA_LAZY_SRCSET = new HTMLLinkContext("img", "data-lazy-srcset");
final public static HTMLLinkContext SOURCE_DATA_LAZY_SRCSET = new HTMLLinkContext("source", "data-lazy-srcset");
final public static HTMLLinkContext IMG_DATA_FULL_SRC = new HTMLLinkContext("img", "data-full-src");
final public static HTMLLinkContext SCRIPT_SRC = new HTMLLinkContext("script", "src");
final public static HTMLLinkContext META_HREF = new HTMLLinkContext("meta", "href");
final public static HTMLLinkContext LINK_IMAGESRCSET = new HTMLLinkContext("link", "imagesrcset");
/**
@@ -57,24 +65,54 @@ public class HTMLLinkContext extends LinkContext {
* @return instance of HTMLLinkContext
*/
public static HTMLLinkContext get(CharSequence el, CharSequence attr) {
if (attr.equals("href") || attr.equals("HREF")) {
if (el.equals("a") || el.equals("A")) return A_HREF;
if (el.equals("meta") || el.equals("META")) return META_HREF;
} else if (attr.equals("src") || attr.equals("SRC")) {
if (el.equals("img") || attr.equals("IMG")) return IMG_SRC;
if (el.equals("script") || attr.equals("SCRIPT")) return SCRIPT_SRC;
} else if (attr.equals("srcset") || attr.equals("SRCSET")) {
if (el.equals("img") || attr.equals("IMG")) return IMG_SRCSET;
if (el.equals("source") || attr.equals("SOURCE")) return SOURCE_SRCSET;
} else if (attr.equals("data-src") || attr.equals("DATA-SRC")) {
if (el.equals("img") || attr.equals("IMG")) return IMG_DATA_SRC;
} else if (attr.equals("data-srcset") || attr.equals("DATA-SRCSET")) {
if (el.equals("img") || attr.equals("IMG")) return IMG_DATA_SRCSET;
} else if (attr.equals("data-original") || attr.equals("DATA-ORIGINAL")) {
if (el.equals("img") || attr.equals("IMG")) return IMG_DATA_ORIGINAL;
} else if (attr.equals("data-original-set") || attr.equals("DATA-ORIGINAL-SET")) {
if (el.equals("img") || attr.equals("IMG")) return IMG_DATA_ORIGINAL_SET;
if (el.equals("source") || attr.equals("SOURCE")) return SOURCE_DATA_ORIGINAL_SET;
switch (attr.toString().toLowerCase()) {
case "href":
if (el.toString().equalsIgnoreCase("a")) return A_HREF;
if (el.toString().equalsIgnoreCase("meta")) return META_HREF;
break;
case "src":
if (el.toString().equalsIgnoreCase("img")) return IMG_SRC;
if (el.toString().equalsIgnoreCase("script")) return SCRIPT_SRC;
break;
case "srcset":
if (el.toString().equalsIgnoreCase("img")) return IMG_SRCSET;
if (el.toString().equalsIgnoreCase("source")) return SOURCE_SRCSET;
break;
case "data-src":
if (el.toString().equalsIgnoreCase("img")) return IMG_DATA_SRC;
break;
case "data-srcset":
if (el.toString().equalsIgnoreCase("img")) return IMG_DATA_SRCSET;
if (el.toString().equalsIgnoreCase("source")) return SOURCE_DATA_SRCSET;
break;
case "data-original":
if (el.toString().equalsIgnoreCase("img")) return IMG_DATA_ORIGINAL;
break;
case "data-original-set":
if (el.toString().equalsIgnoreCase("img")) return IMG_DATA_ORIGINAL_SET;
if (el.toString().equalsIgnoreCase("source")) return SOURCE_DATA_ORIGINAL_SET;
break;
case "data-full-src":
if (el.toString().equalsIgnoreCase("img")) return IMG_DATA_FULL_SRC;
break;
case "data-lazy-srcset":
if (el.toString().equalsIgnoreCase("img")) return IMG_DATA_LAZY_SRCSET;
if (el.toString().equalsIgnoreCase("source")) return SOURCE_DATA_LAZY_SRCSET;
break;
case "data-lazy":
if (el.toString().equalsIgnoreCase("img")) return IMG_DATA_LAZY;
break;
case "data-src-small":
if (el.toString().equalsIgnoreCase("img")) return IMG_DATA_SRC_SMALL;
break;
case "data-src-medium":
if (el.toString().equalsIgnoreCase("img")) return IMG_DATA_SRC_MEDIUM;
break;
case "imagesrcset":
if (el.toString().equalsIgnoreCase("link")) return LINK_IMAGESRCSET;
break;
default:
return new HTMLLinkContext(el, attr);
}
return new HTMLLinkContext(el, attr);
}
@@ -95,10 +133,18 @@ public class HTMLLinkContext extends LinkContext {
if (path.equalsIgnoreCase("source/@srcset")) return SOURCE_SRCSET;
if (path.equalsIgnoreCase("script/@src")) return SCRIPT_SRC;
if (path.equalsIgnoreCase("img/@data-src")) return IMG_DATA_SRC;
if (path.equalsIgnoreCase("img/@data-src-small")) return IMG_DATA_SRC_SMALL;
if (path.equalsIgnoreCase("img/@data-src-medium")) return IMG_DATA_SRC_MEDIUM;
if (path.equalsIgnoreCase("img/@data-srcset")) return IMG_DATA_SRCSET;
if (path.equalsIgnoreCase("img/@data-original")) return IMG_DATA_ORIGINAL;
if (path.equalsIgnoreCase("img/@data-full-src")) return IMG_DATA_FULL_SRC;
if (path.equalsIgnoreCase("img/@data-original-set")) return IMG_DATA_ORIGINAL_SET;
if (path.equalsIgnoreCase("source/@data-original-set")) return SOURCE_DATA_ORIGINAL_SET;
if (path.equalsIgnoreCase("img/@data-lazy-srcset")) return IMG_DATA_LAZY_SRCSET;
if (path.equalsIgnoreCase("source/@data-lazy-srcset")) return SOURCE_DATA_LAZY_SRCSET;
if (path.equalsIgnoreCase("img/@data-lazy")) return IMG_DATA_LAZY;
if (path.equalsIgnoreCase("source/@data-srcset")) return SOURCE_DATA_SRCSET;
if (path.equalsIgnoreCase("link/@imagesrcset")) return LINK_IMAGESRCSET;
return new HTMLLinkContext(path);
}
/**