Merge pull request #120 from nlevitt/extractorjs-twodots

for ARI-4267 accept possible uris with two dots in the filename part if ...

commons/src/main/java/org/archive/util/UriUtils.java

@@ -379,14 +379,28 @@ public class UriUtils {
                         "tfoot", "th", "thead", "time", "title", "tr", "track",
                         "tt", "u", "ul", "var", "video", "wbr"));
     }
+    
+    protected static final Set<String> KNOWN_GOOD_FILE_EXTENSIONS = new HashSet<String>();
+    static {
+        /*
+         * Real known use cases for this are .min.js, .min.css, and we've seen
+         * .jpg files with an extra dot in them. Other extensions are included
+         * in the list somewhat arbitrarily.
+         */
+        KNOWN_GOOD_FILE_EXTENSIONS.addAll(Arrays.asList(".jpg", ".js", ".css",
+                ".png", ".gif", ".swf", ".flv", ".mp4", ".mp3", ".jpeg",
+                ".html"));
+    }
 
     protected static final String QNV = "[a-zA-Z_]+=(?:[\\w-/.]|%[0-9a-fA-F]{2})*"; // name=value for query strings
-    protected static final String VERY_LIKELY_RELATIVE_URI_PATTERN = 
-            "(?:\\.?/)?"                                    // may start with "/" or "./"
-            + "(?:(?:[\\w-]+|\\.\\.)/)*"                    // may have path/segments/
-            + "(?:[\\w-]+(?:\\.[a-zA-Z0-9]{2,5})?)?"        // may have a filename with or without an extension
-            + "(?:\\?(?:"+ QNV + ")(?:&(?:" + QNV + "))*)?" // may have a ?query=string
-            + "(?:#[\\w-]+)?";                              // may have a #fragment
+    // group(1) filename
+    // group(2) filename extension with leading '.'
+    protected static final String LIKELY_RELATIVE_URI_PATTERN = 
+            "(?:\\.?/)?"                                         // may start with "/" or "./"
+            + "(?:(?:[\\w-]+|\\.\\.)/)*"                         // may have path/segments/
+            + "([\\w-]+(?:\\.[\\w-]+)?(\\.[a-zA-Z0-9]{2,5})?)?"  // may have a filename with or without an extension
+            + "(?:\\?(?:"+ QNV + ")(?:&(?:" + QNV + "))*)?"      // may have a ?query=string
+            + "(?:#[\\w-]+)?";                                   // may have a #fragment
 
     public static boolean isVeryLikelyUri(CharSequence candidate) {
         // must have a . or /
@@ -405,15 +419,25 @@ public class UriUtils {
         }
         
         // relative or server-relative uri
-        if (!TextUtils.matches(VERY_LIKELY_RELATIVE_URI_PATTERN, candidate)) {
+        Matcher matcher = TextUtils.getMatcher(LIKELY_RELATIVE_URI_PATTERN, candidate);
+        if (!matcher.matches()) {
             return false;
         }
         
         /*
          * Remaining tests discard stuff that the
-         * VERY_LIKELY_RELATIVE_URI_PATTERN can't catch
+         * LIKELY_RELATIVE_URI_PATTERN can't catch
          */
-        
+
+        // if filename contains two dots, it must end with a known good extension
+        String filename = matcher.group(1);
+        String extension = matcher.group(2);
+        if (filename != null && extension != null
+                && filename.indexOf('.') != filename.lastIndexOf('.')
+                && !KNOWN_GOOD_FILE_EXTENSIONS.contains(extension)) {
+            return false;
+        }
+
         // text or application mimetype
         if (TextUtils.matches("(?:text|application)/[^/]+", candidate)) {
             return false;

modules/src/test/java/org/archive/modules/extractor/ExtractorJSTest.java

@@ -22,7 +22,6 @@ import java.util.ArrayList;
 import java.util.Collection;
 import java.util.List;
 
-import org.apache.commons.lang.StringEscapeUtils;
 import org.archive.modules.CrawlURI;
 import org.archive.net.UURI;
 import org.archive.net.UURIFactory;
@@ -114,12 +113,18 @@ public class ExtractorJSTest extends StringExtractorTestBase {
         
         "var blah='/good/query/value/with/url-escaping.html?foo=bar%20bar';",
         "http://www.archive.org/good/query/value/with/url-escaping.html?foo=bar%20bar",
-        
+
         "\\u0027project_detail.aspx?guid=unicodesinglequote\\u0027",
         "http://www.archive.org/foo/project_detail.aspx?guid=unicodesinglequote",
-        
+
         "\\u0022project_detail.aspx?guid=unicodedoublequote\\u0022",
-        "http://www.archive.org/foo/project_detail.aspx?guid=unicodedoublequote",        
+        "http://www.archive.org/foo/project_detail.aspx?guid=unicodedoublequote",
+
+        "{url: '/static/0000/2683/good_filename_with.two_dots.jpg',caption:'blah blah' }",
+        "http://www.archive.org/static/0000/2683/good_filename_with.two_dots.jpg",
+
+        "{nonUrl: 'non-filename.with_two.dots',etc:'foo foo' }",
+        null
     };
        
     @Override