lee/heritrix3
1
0
Fork 0
mirror of https://github.com/internetarchive/heritrix3.git synced 2026-09-22 21:55:54 +00:00
Code Issues Packages Projects Releases Wiki Activity

Enable parsing of absolute URLs in meta HTML tags.

Browse Source 
- Refactors several utility methods in UriUtils for URI parsing.
- Tests URLs in meta tags for absolute URIs.

The current W3C spec indicates those links should be relative. Several
examples have been observed in the wild with absolute URLs.

Browsers do not throw a parsing error and do redirect approprately. This
change conforms to that behavior.

Reference:
https://www.w3.org/TR/html53/document-metadata.html#statedef-http-equiv-refresh
This commit is contained in:
Neil Minton
2020-08-06 19:23:50 -04:00
parent f879226c8e
commit 0dd65dfd0b
2 changed files with 53 additions and 23 deletions
Download Patch File Download Diff File Expand all files Collapse all files
commons/src/main/java/org/archive/util/UriUtils.java
+43 -15
View File
@@ -403,21 +403,15 @@ public class UriUtils {
public static boolean isVeryLikelyUri(CharSequence candidate) {
// must have a . or /
if (!TextUtils.matches(NAIVE_LIKELY_URI_PATTERN, candidate)) {
return false;
}
// absolute uri
if (TextUtils.matches("^(?i)https?://[^<>\\s/]+\\.[^<>\\s/]+(?:/[^<>\\s]*)?", candidate)) {
return true;
}
// "protocol-relative" uri
if (TextUtils.matches("^//[^<>\\s/]+\\.[^<>\\s/]+(?:/[^<>\\s]*)?", candidate)) {
return true;
}
if (isVeryLikelyAbsoluteUri(candidate) || isVeryLikelyRelativeUri(candidate)) {
return true;
}
if (!isCandidateUri(candidate)) {
return false;
}
// relative or server-relative uri
Matcher matcher = TextUtils.getMatcher(LIKELY_RELATIVE_URI_PATTERN, candidate);
if (!matcher.matches()) {
@@ -468,7 +462,41 @@ public class UriUtils {
return true;
}
protected static boolean isCandidateUri(CharSequence candidate) {
// must have a . or /
if (!TextUtils.matches(NAIVE_LIKELY_URI_PATTERN, candidate)) {
return false;
}
return true;
}
public static boolean isVeryLikelyAbsoluteUri(CharSequence candidate) {
if (!isCandidateUri(candidate)) {
return false;
}
// absolute uri
if (TextUtils.matches("^(?i)https?://[^<>\\s/]+\\.[^<>\\s/]+(?:/[^<>\\s]*)?", candidate)) {
return true;
}
return false;
}
public static boolean isVeryLikelyRelativeUri(CharSequence candidate) {
if (!isCandidateUri(candidate)) {
return false;
}
// "protocol-relative" uri
if (TextUtils.matches("^//[^<>\\s/]+\\.[^<>\\s/]+(?:/[^<>\\s]*)?", candidate)) {
return true;
}
return false;
}
//
// legacy likely-URI test from ExtractorJS
modules/src/main/java/org/archive/modules/extractor/ExtractorHTML.java
+10 -8
View File
@@ -982,14 +982,16 @@ public class ExtractorHTML extends ContentExtractor implements InitializingBean
int urlIndex = content.indexOf("=") + 1;
if(urlIndex>0) {
String refreshUri = content.substring(urlIndex);
try {
int max = getExtractorParameters().getMaxOutlinks();
addRelativeToBase(curi, max, refreshUri,
HTMLLinkContext.META, Hop.REFER);
} catch (URIException e) {
logUriError(e, curi.getUURI(), refreshUri);
}
}
try {
int max = getExtractorParameters().getMaxOutlinks();
if (UriUtils.isVeryLikelyAbsoluteUri(refreshUri)) {
add(curi, max, refreshUri, HTMLLinkContext.META, Hop.REFER);
}
addRelativeToBase(curi, max, refreshUri, HTMLLinkContext.META, Hop.REFER);
} catch (URIException e) {
logUriError(e, curi.getUURI(), refreshUri);
}
}
}
else if (content != null) {
//look for likely urls in 'content' attribute