diff --git a/commons/src/main/java/org/archive/util/UriUtils.java b/commons/src/main/java/org/archive/util/UriUtils.java index 83f43589..d08e9128 100644 --- a/commons/src/main/java/org/archive/util/UriUtils.java +++ b/commons/src/main/java/org/archive/util/UriUtils.java @@ -403,15 +403,21 @@ public class UriUtils { public static boolean isVeryLikelyUri(CharSequence candidate) { - - if (isVeryLikelyAbsoluteUri(candidate) || isVeryLikelyRelativeUri(candidate)) { - return true; - } - - if (!isCandidateUri(candidate)) { - return false; - } - + // must have a . or / + if (!TextUtils.matches(NAIVE_LIKELY_URI_PATTERN, candidate)) { + return false; + } + + // absolute uri + if (TextUtils.matches("^(?i)https?://[^<>\\s/]+\\.[^<>\\s/]+(?:/[^<>\\s]*)?", candidate)) { + return true; + } + + // "protocol-relative" uri + if (TextUtils.matches("^//[^<>\\s/]+\\.[^<>\\s/]+(?:/[^<>\\s]*)?", candidate)) { + return true; + } + // relative or server-relative uri Matcher matcher = TextUtils.getMatcher(LIKELY_RELATIVE_URI_PATTERN, candidate); if (!matcher.matches()) { @@ -462,41 +468,7 @@ public class UriUtils { return true; } - protected static boolean isCandidateUri(CharSequence candidate) { - // must have a . or / - if (!TextUtils.matches(NAIVE_LIKELY_URI_PATTERN, candidate)) { - return false; - } - return true; - } - - public static boolean isVeryLikelyAbsoluteUri(CharSequence candidate) { - - if (!isCandidateUri(candidate)) { - return false; - } - - // absolute uri - if (TextUtils.matches("^(?i)[\"']?https?://[^<>\\s/]+\\.[^<>\\s/]+(?:/[^<>\\s]*)?[\"']?", candidate)) { - return true; - } - - return false; - } - - public static boolean isVeryLikelyRelativeUri(CharSequence candidate) { - if (!isCandidateUri(candidate)) { - return false; - } - - // "protocol-relative" uri - if (TextUtils.matches("^//[^<>\\s/]+\\.[^<>\\s/]+(?:/[^<>\\s]*)?", candidate)) { - return true; - } - - return false; - } // // legacy likely-URI test from ExtractorJS diff --git a/modules/src/main/java/org/archive/modules/extractor/ExtractorHTML.java b/modules/src/main/java/org/archive/modules/extractor/ExtractorHTML.java index eda7ba2b..b26eed3f 100644 --- a/modules/src/main/java/org/archive/modules/extractor/ExtractorHTML.java +++ b/modules/src/main/java/org/archive/modules/extractor/ExtractorHTML.java @@ -981,19 +981,16 @@ public class ExtractorHTML extends ContentExtractor implements InitializingBean } else if ("refresh".equalsIgnoreCase(httpEquiv) && content != null) { int urlIndex = content.indexOf("=") + 1; if(urlIndex>0) { - // strip any quotes ("') characters from the URL value. - String refreshUri = TextUtils.replaceAll("[\"']", content.substring(urlIndex), ""); - try { - int max = getExtractorParameters().getMaxOutlinks(); - if (UriUtils.isVeryLikelyAbsoluteUri(refreshUri)) { - add(curi, max, refreshUri, HTMLLinkContext.META, Hop.REFER); - } else { - addRelativeToBase(curi, max, refreshUri, HTMLLinkContext.META, Hop.REFER); - } - } catch (URIException e) { - logUriError(e, curi.getUURI(), refreshUri); - } - } + // strip any quotes ("') characters from the URL value. + String refreshUri = TextUtils.replaceAll("[\"']", content.substring(urlIndex), ""); + try { + int max = getExtractorParameters().getMaxOutlinks(); + addRelativeToBase(curi, max, refreshUri, + HTMLLinkContext.META, Hop.REFER); + } catch (URIException e) { + logUriError(e, curi.getUURI(), refreshUri); + } + } } else if (content != null) { //look for likely urls in 'content' attribute