diff --git a/commons/src/main/java/org/archive/util/UriUtils.java b/commons/src/main/java/org/archive/util/UriUtils.java index 5b78f4fb..e9a7bba5 100644 --- a/commons/src/main/java/org/archive/util/UriUtils.java +++ b/commons/src/main/java/org/archive/util/UriUtils.java @@ -89,9 +89,9 @@ public class UriUtils { private static final Logger LOGGER = Logger.getLogger(UriUtils.class.getName()); // naive likely-uri test: - // no whitespace or '<' or '>' + // no '<' or '>' // at least one '.' or '/'; - protected static final String NAIVE_LIKELY_URI_PATTERN = "[^<>\\s]*[\\./][^<>\\s]*"; + protected static final String NAIVE_LIKELY_URI_PATTERN = "[^<>]*[\\./][^<>]*"; public static boolean isPossibleUri(CharSequence candidate) { return TextUtils.matches(NAIVE_LIKELY_URI_PATTERN, candidate); @@ -381,6 +381,7 @@ public class UriUtils { } protected static final Set KNOWN_GOOD_FILE_EXTENSIONS = new HashSet(); + static { /* * Real known use cases for this are .min.js, .min.css, and we've seen @@ -389,19 +390,20 @@ public class UriUtils { */ KNOWN_GOOD_FILE_EXTENSIONS.addAll(Arrays.asList(".jpg", ".js", ".css", ".png", ".gif", ".swf", ".flv", ".mp4", ".mp3", ".jpeg", - ".html")); + ".html", ".pdf")); } protected static final String QNV = "[a-zA-Z_]+=(?:[\\w-/.]|%[0-9a-fA-F]{2})*"; // name=value for query strings // group(1) filename // group(2) filename extension with leading '.' protected static final String LIKELY_RELATIVE_URI_PATTERN = - "(?:\\.?/)?" // may start with "/" or "./" - + "(?:(?:[\\w-]+|\\.\\.)/)*" // may have path/segments/ - + "([\\w-]+(?:\\.[\\w-]+)?(\\.[a-zA-Z0-9]{2,5})?)?" // may have a filename with or without an extension - + "(?:\\?(?:"+ QNV + ")(?:&(?:" + QNV + "))*)?" // may have a ?query=string - + "(?:#[\\w-]+)?"; // may have a #fragment - + "(?:\\.?/)?" // may start with "/" or "./" + + "(?:(?:[\\s\\w-]+|\\.\\.)(?:/))*" // may have path/segments/segment2 + + "([\\s\\w-]+(?:\\.[\\w-]+)??(\\.[a-zA-Z0-9]{2,5})?)?" // may have a filename with or without an extension + + "(?:\\?(?:"+ QNV + ")(?:&(?:" + QNV + "))*)?" // may have a ?query=string + + "(?:#[\\w-]+)?"; // may have a #fragment + + public static boolean isVeryLikelyUri(CharSequence candidate) { // must have a . or / if (!TextUtils.matches(NAIVE_LIKELY_URI_PATTERN, candidate)) { @@ -423,7 +425,7 @@ public class UriUtils { if (!matcher.matches()) { return false; } - + /* * Remaining tests discard stuff that the * LIKELY_RELATIVE_URI_PATTERN can't catch @@ -438,6 +440,12 @@ public class UriUtils { return false; } + if (TextUtils.matches(".*\\s+.*", candidate) + && (extension == null + || !KNOWN_GOOD_FILE_EXTENSIONS.contains(extension))) { + return false; + } + // text or application mimetype if (TextUtils.matches("(?:text|application)/[^/]+", candidate)) { return false; diff --git a/modules/src/main/java/org/archive/modules/extractor/ExtractorJS.java b/modules/src/main/java/org/archive/modules/extractor/ExtractorJS.java index 10de06cf..8c89c40f 100644 --- a/modules/src/main/java/org/archive/modules/extractor/ExtractorJS.java +++ b/modules/src/main/java/org/archive/modules/extractor/ExtractorJS.java @@ -63,11 +63,11 @@ public class ExtractorJS extends ContentExtractor { private static Logger LOGGER = Logger.getLogger(ExtractorJS.class.getName()); - // finds whitespace- and quote-free strings in Javascript + // finds strings in Javascript // (areas between paired ' or " characters, possibly backslash-quoted // on the ends, but not in the middle) protected static final String JAVASCRIPT_STRING_EXTRACTOR = - "(\\\\{0,8}+(?:['\"]|u002[27]))([^\\s'\"]{1,"+UURI.MAX_URL_LENGTH+"})(?:\\1)"; + "(\\\\{0,8}+(?:['\"]|u002[27]))([^'\"]{0,"+UURI.MAX_URL_LENGTH+"})(?:\\1)"; // GROUPS: // (G1) ' or " with optional leading backslashes diff --git a/modules/src/test/java/org/archive/modules/extractor/ExtractorJSTest.java b/modules/src/test/java/org/archive/modules/extractor/ExtractorJSTest.java index 917f0efc..c684c231 100644 --- a/modules/src/test/java/org/archive/modules/extractor/ExtractorJSTest.java +++ b/modules/src/test/java/org/archive/modules/extractor/ExtractorJSTest.java @@ -124,9 +124,33 @@ public class ExtractorJSTest extends StringExtractorTestBase { "http://www.archive.org/static/0000/2683/good_filename_with.two_dots.jpg", "{nonUrl: 'non-filename.with_two.dots',etc:'foo foo' }", - null + null, + + "\"FileRef\": \"\\u002fsites\\u002fprb\\u002fPublic Comment Emails PDF\\u002fOpen Records Law_1r71vq4i.pdf\",\"", + "http://www.archive.org/sites/prb/Public Comment Emails PDF/Open Records Law_1r71vq4i.pdf", + + "\"FileRef\": \"\\u002fsites\\u002fprb\\u002fPublic Comment Emails PDF\\u002fOpen Records Law_1r71vq4i.bad\",\"", + null, + + "\"FileRef\": \"\\u002fsites\\u002fprb\\u002fPublic Comment Emails PDF\\u002fOpenRecordsLaw_1r71vq4i.bad\",\"", + null, + + "\"FileRef\": \"\\u002fsites\\u002fprb\\u002fPublic Comment Emails PDF\\u002fOpen Records Law_1r71vq4i\",\"", + null, + + "\"FileRef\": \"\\u002fsites\\u002fprb\\u002fPublic Comment Emails PDF\\u002fOpenRecordsLaw_1r71vq4i\",\"", + null, + + /* + * XXX this one fails currently because the string has no slashes or + * dots, until it is javascript-unescaped, which happens too late. + * Unescaping earlier involves converting many more CharSubSequence to + * String. Is it worth the performance hit? + */ + // "\"FileRef\": \"\\u002fsites\\u002fprb\\u002fPublicCommentEmailsPDF\\u002fOpenRecordsLaw_1r71vq4i\",\"", + // "http://www.archive.org/sites/prb/PublicCommentEmailsPDF/OpenRecordsLaw_1r71vq4i", }; - + @Override protected String[] getValidTestData() { return VALID_TEST_DATA; @@ -134,7 +158,6 @@ public class ExtractorJSTest extends StringExtractorTestBase { @Override protected Extractor makeExtractor() { - ExtractorJS result = new ExtractorJS(); UriErrorLoggerModule ulm = new UnitTestUriLoggerModule(); result.setLoggerModule(ulm);