lee/heritrix3
1
0
Fork 0
mirror of https://github.com/internetarchive/heritrix3.git synced 2026-09-24 06:36:12 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'absolute-meta-urls' into ait-qa

Browse Source
This commit is contained in:
Neil Minton
2020-08-14 10:59:17 -04:00
parent 6b1e88934a 941c6c50ad
commit 13a1028e2f
2 changed files with 25 additions and 56 deletions
Download Patch File Download Diff File Expand all files Collapse all files
commons/src/main/java/org/archive/util/UriUtils.java
+15 -43
View File
@@ -403,15 +403,21 @@ public class UriUtils {
public static boolean isVeryLikelyUri(CharSequence candidate) {
if (isVeryLikelyAbsoluteUri(candidate) || isVeryLikelyRelativeUri(candidate)) {
return true;
}
if (!isCandidateUri(candidate)) {
return false;
}
// must have a . or /
if (!TextUtils.matches(NAIVE_LIKELY_URI_PATTERN, candidate)) {
return false;
}
// absolute uri
if (TextUtils.matches("^(?i)https?://[^<>\\s/]+\\.[^<>\\s/]+(?:/[^<>\\s]*)?", candidate)) {
return true;
}
// "protocol-relative" uri
if (TextUtils.matches("^//[^<>\\s/]+\\.[^<>\\s/]+(?:/[^<>\\s]*)?", candidate)) {
return true;
}
// relative or server-relative uri
Matcher matcher = TextUtils.getMatcher(LIKELY_RELATIVE_URI_PATTERN, candidate);
if (!matcher.matches()) {
@@ -462,41 +468,7 @@ public class UriUtils {
return true;
}
protected static boolean isCandidateUri(CharSequence candidate) {
// must have a . or /
if (!TextUtils.matches(NAIVE_LIKELY_URI_PATTERN, candidate)) {
return false;
}
return true;
}
public static boolean isVeryLikelyAbsoluteUri(CharSequence candidate) {
if (!isCandidateUri(candidate)) {
return false;
}
// absolute uri
if (TextUtils.matches("^(?i)[\"']?https?://[^<>\\s/]+\\.[^<>\\s/]+(?:/[^<>\\s]*)?[\"']?", candidate)) {
return true;
}
return false;
}
public static boolean isVeryLikelyRelativeUri(CharSequence candidate) {
if (!isCandidateUri(candidate)) {
return false;
}
// "protocol-relative" uri
if (TextUtils.matches("^//[^<>\\s/]+\\.[^<>\\s/]+(?:/[^<>\\s]*)?", candidate)) {
return true;
}
return false;
}
//
// legacy likely-URI test from ExtractorJS
modules/src/main/java/org/archive/modules/extractor/ExtractorHTML.java
+10 -13
View File
@@ -981,19 +981,16 @@ public class ExtractorHTML extends ContentExtractor implements InitializingBean
} else if ("refresh".equalsIgnoreCase(httpEquiv) && content != null) {
int urlIndex = content.indexOf("=") + 1;
if(urlIndex>0) {
// strip any quotes ("') characters from the URL value.
String refreshUri = TextUtils.replaceAll("[\"']", content.substring(urlIndex), "");
try {
int max = getExtractorParameters().getMaxOutlinks();
if (UriUtils.isVeryLikelyAbsoluteUri(refreshUri)) {
add(curi, max, refreshUri, HTMLLinkContext.META, Hop.REFER);
} else {
addRelativeToBase(curi, max, refreshUri, HTMLLinkContext.META, Hop.REFER);
}
} catch (URIException e) {
logUriError(e, curi.getUURI(), refreshUri);
}
}
// strip any quotes ("') characters from the URL value.
String refreshUri = TextUtils.replaceAll("[\"']", content.substring(urlIndex), "");
try {
int max = getExtractorParameters().getMaxOutlinks();
addRelativeToBase(curi, max, refreshUri,
HTMLLinkContext.META, Hop.REFER);
} catch (URIException e) {
logUriError(e, curi.getUURI(), refreshUri);
}
}
}
else if (content != null) {
//look for likely urls in 'content' attribute