From 0dd65dfd0b5b42537c27e48b17bb779efb82a826 Mon Sep 17 00:00:00 2001
From: Neil Minton <neil@archive.org>
Date: Thu, 6 Aug 2020 19:23:50 -0400
Subject: [PATCH] Enable parsing of absolute URLs in meta HTML tags.

- Refactors several utility methods in UriUtils for URI parsing.
- Tests URLs in meta tags for absolute URIs.

The current W3C spec indicates those links should be relative. Several
examples have been observed in the wild with absolute URLs.

Browsers do not throw a parsing error and do redirect approprately. This
change conforms to that behavior.

Reference:
https://www.w3.org/TR/html53/document-metadata.html#statedef-http-equiv-refresh
---
 .../main/java/org/archive/util/UriUtils.java  | 58 ++++++++++++++-----
 .../modules/extractor/ExtractorHTML.java      | 18 +++---
 2 files changed, 53 insertions(+), 23 deletions(-)

diff --git a/commons/src/main/java/org/archive/util/UriUtils.java b/commons/src/main/java/org/archive/util/UriUtils.java
index d08e9128..e5f374ba 100644
--- a/commons/src/main/java/org/archive/util/UriUtils.java
+++ b/commons/src/main/java/org/archive/util/UriUtils.java
@@ -403,21 +403,15 @@ public class UriUtils {
     
     
     public static boolean isVeryLikelyUri(CharSequence candidate) {
-        // must have a . or /
-        if (!TextUtils.matches(NAIVE_LIKELY_URI_PATTERN, candidate)) {
-            return false;
-        }
-        
-        // absolute uri
-        if (TextUtils.matches("^(?i)https?://[^<>\\s/]+\\.[^<>\\s/]+(?:/[^<>\\s]*)?", candidate)) {
-            return true;
-        }
-        
-        // "protocol-relative" uri
-        if (TextUtils.matches("^//[^<>\\s/]+\\.[^<>\\s/]+(?:/[^<>\\s]*)?", candidate)) {
-            return true;
-        }
-        
+
+	if (isVeryLikelyAbsoluteUri(candidate) || isVeryLikelyRelativeUri(candidate)) {
+	    return true;
+	}
+
+	if (!isCandidateUri(candidate)) {
+	    return false;
+	}
+
         // relative or server-relative uri
         Matcher matcher = TextUtils.getMatcher(LIKELY_RELATIVE_URI_PATTERN, candidate);
         if (!matcher.matches()) {
@@ -468,7 +462,41 @@ public class UriUtils {
         return true;
     }
 
+	protected static boolean isCandidateUri(CharSequence candidate) {
+		// must have a . or /
+		if (!TextUtils.matches(NAIVE_LIKELY_URI_PATTERN, candidate)) {
+			return false;
+		}
 
+		return true;
+	}
+
+	public static boolean isVeryLikelyAbsoluteUri(CharSequence candidate) {
+
+		if (!isCandidateUri(candidate)) {
+			return false;
+		}
+
+		// absolute uri
+		if (TextUtils.matches("^(?i)https?://[^<>\\s/]+\\.[^<>\\s/]+(?:/[^<>\\s]*)?", candidate)) {
+			return true;
+		}
+
+		return false;
+	}
+
+	public static boolean isVeryLikelyRelativeUri(CharSequence candidate) {
+		if (!isCandidateUri(candidate)) {
+			return false;
+		}
+
+		// "protocol-relative" uri
+		if (TextUtils.matches("^//[^<>\\s/]+\\.[^<>\\s/]+(?:/[^<>\\s]*)?", candidate)) {
+			return true;
+		}
+
+		return false;
+	}
     
 //
 // legacy likely-URI test from ExtractorJS
diff --git a/modules/src/main/java/org/archive/modules/extractor/ExtractorHTML.java b/modules/src/main/java/org/archive/modules/extractor/ExtractorHTML.java
index ebcaf47a..53218036 100644
--- a/modules/src/main/java/org/archive/modules/extractor/ExtractorHTML.java
+++ b/modules/src/main/java/org/archive/modules/extractor/ExtractorHTML.java
@@ -982,14 +982,16 @@ public class ExtractorHTML extends ContentExtractor implements InitializingBean
             int urlIndex = content.indexOf("=") + 1;
             if(urlIndex>0) {
                 String refreshUri = content.substring(urlIndex);
-                try {
-                    int max = getExtractorParameters().getMaxOutlinks();
-                    addRelativeToBase(curi, max, refreshUri, 
-                            HTMLLinkContext.META, Hop.REFER);
-                } catch (URIException e) {
-                    logUriError(e, curi.getUURI(), refreshUri);
-                }
-            }
+		try {
+		    int max = getExtractorParameters().getMaxOutlinks();
+		    if (UriUtils.isVeryLikelyAbsoluteUri(refreshUri)) {
+			add(curi, max, refreshUri, HTMLLinkContext.META, Hop.REFER);
+		    }
+		    addRelativeToBase(curi, max, refreshUri, HTMLLinkContext.META, Hop.REFER);
+		} catch (URIException e) {
+		    logUriError(e, curi.getUURI(), refreshUri);
+		}
+	    }
         }
         else if (content != null) {
             //look for likely urls in 'content' attribute