diff --git a/commons/src/main/java/org/archive/util/UriUtils.java b/commons/src/main/java/org/archive/util/UriUtils.java index d08e9128..e5f374ba 100644 --- a/commons/src/main/java/org/archive/util/UriUtils.java +++ b/commons/src/main/java/org/archive/util/UriUtils.java @@ -403,21 +403,15 @@ public class UriUtils { public static boolean isVeryLikelyUri(CharSequence candidate) { - // must have a . or / - if (!TextUtils.matches(NAIVE_LIKELY_URI_PATTERN, candidate)) { - return false; - } - - // absolute uri - if (TextUtils.matches("^(?i)https?://[^<>\\s/]+\\.[^<>\\s/]+(?:/[^<>\\s]*)?", candidate)) { - return true; - } - - // "protocol-relative" uri - if (TextUtils.matches("^//[^<>\\s/]+\\.[^<>\\s/]+(?:/[^<>\\s]*)?", candidate)) { - return true; - } - + + if (isVeryLikelyAbsoluteUri(candidate) || isVeryLikelyRelativeUri(candidate)) { + return true; + } + + if (!isCandidateUri(candidate)) { + return false; + } + // relative or server-relative uri Matcher matcher = TextUtils.getMatcher(LIKELY_RELATIVE_URI_PATTERN, candidate); if (!matcher.matches()) { @@ -468,7 +462,41 @@ public class UriUtils { return true; } + protected static boolean isCandidateUri(CharSequence candidate) { + // must have a . or / + if (!TextUtils.matches(NAIVE_LIKELY_URI_PATTERN, candidate)) { + return false; + } + return true; + } + + public static boolean isVeryLikelyAbsoluteUri(CharSequence candidate) { + + if (!isCandidateUri(candidate)) { + return false; + } + + // absolute uri + if (TextUtils.matches("^(?i)https?://[^<>\\s/]+\\.[^<>\\s/]+(?:/[^<>\\s]*)?", candidate)) { + return true; + } + + return false; + } + + public static boolean isVeryLikelyRelativeUri(CharSequence candidate) { + if (!isCandidateUri(candidate)) { + return false; + } + + // "protocol-relative" uri + if (TextUtils.matches("^//[^<>\\s/]+\\.[^<>\\s/]+(?:/[^<>\\s]*)?", candidate)) { + return true; + } + + return false; + } // // legacy likely-URI test from ExtractorJS diff --git a/modules/src/main/java/org/archive/modules/extractor/ExtractorHTML.java b/modules/src/main/java/org/archive/modules/extractor/ExtractorHTML.java index ebcaf47a..53218036 100644 --- a/modules/src/main/java/org/archive/modules/extractor/ExtractorHTML.java +++ b/modules/src/main/java/org/archive/modules/extractor/ExtractorHTML.java @@ -982,14 +982,16 @@ public class ExtractorHTML extends ContentExtractor implements InitializingBean int urlIndex = content.indexOf("=") + 1; if(urlIndex>0) { String refreshUri = content.substring(urlIndex); - try { - int max = getExtractorParameters().getMaxOutlinks(); - addRelativeToBase(curi, max, refreshUri, - HTMLLinkContext.META, Hop.REFER); - } catch (URIException e) { - logUriError(e, curi.getUURI(), refreshUri); - } - } + try { + int max = getExtractorParameters().getMaxOutlinks(); + if (UriUtils.isVeryLikelyAbsoluteUri(refreshUri)) { + add(curi, max, refreshUri, HTMLLinkContext.META, Hop.REFER); + } + addRelativeToBase(curi, max, refreshUri, HTMLLinkContext.META, Hop.REFER); + } catch (URIException e) { + logUriError(e, curi.getUURI(), refreshUri); + } + } } else if (content != null) { //look for likely urls in 'content' attribute