From 23885557bb385af0aabb12ea34bd6f20e72c12ea Mon Sep 17 00:00:00 2001
From: Hunter Stern <hunter@archive.org>
Date: Tue, 19 Jan 2016 18:07:05 -0800
Subject: [PATCH] Allow spaces in JavaScript urls, but only if they have a
 known good file extension

---
 .../main/java/org/archive/util/UriUtils.java  | 31 ++++++++++++++-----
 .../modules/extractor/ExtractorJS.java        |  2 +-
 2 files changed, 25 insertions(+), 8 deletions(-)

diff --git a/commons/src/main/java/org/archive/util/UriUtils.java b/commons/src/main/java/org/archive/util/UriUtils.java
index 5b78f4fb..f89fb9f9 100644
--- a/commons/src/main/java/org/archive/util/UriUtils.java
+++ b/commons/src/main/java/org/archive/util/UriUtils.java
@@ -381,6 +381,7 @@ public class UriUtils {
     }
     
     protected static final Set<String> KNOWN_GOOD_FILE_EXTENSIONS = new HashSet<String>();
+
     static {
         /*
          * Real known use cases for this are .min.js, .min.css, and we've seen
@@ -389,19 +390,20 @@ public class UriUtils {
          */
         KNOWN_GOOD_FILE_EXTENSIONS.addAll(Arrays.asList(".jpg", ".js", ".css",
                 ".png", ".gif", ".swf", ".flv", ".mp4", ".mp3", ".jpeg",
-                ".html"));
+                ".html", ".pdf"));
     }
 
     protected static final String QNV = "[a-zA-Z_]+=(?:[\\w-/.]|%[0-9a-fA-F]{2})*"; // name=value for query strings
     // group(1) filename
     // group(2) filename extension with leading '.'
     protected static final String LIKELY_RELATIVE_URI_PATTERN = 
-            "(?:\\.?/)?"                                         // may start with "/" or "./"
-            + "(?:(?:[\\w-]+|\\.\\.)/)*"                         // may have path/segments/
-            + "([\\w-]+(?:\\.[\\w-]+)?(\\.[a-zA-Z0-9]{2,5})?)?"  // may have a filename with or without an extension
-            + "(?:\\?(?:"+ QNV + ")(?:&(?:" + QNV + "))*)?"      // may have a ?query=string
-            + "(?:#[\\w-]+)?";                                   // may have a #fragment
-
+            "(?:\\.?/|\\\\u002f)?"                                          // may start with "/" or "./" or utf16 / which is (\u002f)
+            + "(?:(?:[\\s\\w-]+|\\.\\.)(?:/|\\\\u002f))*"                   // may have path/segments/segment2\u002fa\u002f
+            + "([\\s\\w-]+(?:\\.[\\w-]+)?(\\.[a-zA-Z0-9]{2,5})?)?"          // may have a filename with or without an extension
+            + "(?:\\?(?:"+ QNV + ")(?:&(?:" + QNV + "))*)?"                 // may have a ?query=string
+            + "(?:#[\\w-]+)?";                                              // may have a #fragment
+    
+    
     public static boolean isVeryLikelyUri(CharSequence candidate) {
         // must have a . or /
         if (!TextUtils.matches(NAIVE_LIKELY_URI_PATTERN, candidate)) {
@@ -423,6 +425,21 @@ public class UriUtils {
         if (!matcher.matches()) {
             return false;
         }
+
+        // if spaces in url, only allow file extensions that match known good extensions
+        if (TextUtils.matches(".*[\\s)]+.*", candidate)) {
+            String filename = matcher.group(1);
+            String extension = matcher.group(2);
+            if (filename != null && extension != null
+                    && KNOWN_GOOD_FILE_EXTENSIONS.contains(extension)) {
+                return true;
+            }
+        }
+        
+        // if spaces in url but doesn't match a known good file extension, discard
+        if (TextUtils.matches(".*[\\s)]+.*", candidate)) {
+            return false;
+        }
         
         /*
          * Remaining tests discard stuff that the
diff --git a/modules/src/main/java/org/archive/modules/extractor/ExtractorJS.java b/modules/src/main/java/org/archive/modules/extractor/ExtractorJS.java
index 10de06cf..72358815 100644
--- a/modules/src/main/java/org/archive/modules/extractor/ExtractorJS.java
+++ b/modules/src/main/java/org/archive/modules/extractor/ExtractorJS.java
@@ -67,7 +67,7 @@ public class ExtractorJS extends ContentExtractor {
     // (areas between paired ' or " characters, possibly backslash-quoted
     // on the ends, but not in the middle)
     protected static final String JAVASCRIPT_STRING_EXTRACTOR =
-    		"(\\\\{0,8}+(?:['\"]|u002[27]))([^\\s'\"]{1,"+UURI.MAX_URL_LENGTH+"})(?:\\1)";
+    		"(\\\\{0,8}+(?:['\"]|u002[27]))([^'\"]{0,"+UURI.MAX_URL_LENGTH+"})(?:\\1)";
     
     // GROUPS:
     // (G1) ' or " with optional leading backslashes