lee/heritrix3
1
0
Fork 0
mirror of https://github.com/internetarchive/heritrix3.git synced 2026-07-19 22:17:13 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #145 from vonrosen/ARI-4713

Browse Source 
Make some urls with whitespace acceptable to JavaScript extractor.
This commit is contained in:
Hunter
2016-02-11 08:27:52 -08:00
parent d80cf8244b cbb5e3ab99
commit bfbd61edd7
3 changed files with 46 additions and 15 deletions
Download Patch File Download Diff File Expand all files Collapse all files
commons/src/main/java/org/archive/util/UriUtils.java
+18 -10
View File
@@ -89,9 +89,9 @@ public class UriUtils {
private static final Logger LOGGER = Logger.getLogger(UriUtils.class.getName());
// naive likely-uri test:
// no whitespace or '<' or '>'
// no '<' or '>'
// at least one '.' or '/';
protected static final String NAIVE_LIKELY_URI_PATTERN = "[^<>\\s]*[\\./][^<>\\s]*";
protected static final String NAIVE_LIKELY_URI_PATTERN = "[^<>]*[\\./][^<>]*";
public static boolean isPossibleUri(CharSequence candidate) {
return TextUtils.matches(NAIVE_LIKELY_URI_PATTERN, candidate);
@@ -381,6 +381,7 @@ public class UriUtils {
}
protected static final Set<String> KNOWN_GOOD_FILE_EXTENSIONS = new HashSet<String>();
static {
/*
* Real known use cases for this are .min.js, .min.css, and we've seen
@@ -389,19 +390,20 @@ public class UriUtils {
*/
KNOWN_GOOD_FILE_EXTENSIONS.addAll(Arrays.asList(".jpg", ".js", ".css",
".png", ".gif", ".swf", ".flv", ".mp4", ".mp3", ".jpeg",
".html"));
".html", ".pdf"));
}
protected static final String QNV = "[a-zA-Z_]+=(?:[\\w-/.]|%[0-9a-fA-F]{2})*"; // name=value for query strings
// group(1) filename
// group(2) filename extension with leading '.'
protected static final String LIKELY_RELATIVE_URI_PATTERN =
"(?:\\.?/)?" // may start with "/" or "./"
+ "(?:(?:[\\w-]+|\\.\\.)/)*" // may have path/segments/
+ "([\\w-]+(?:\\.[\\w-]+)?(\\.[a-zA-Z0-9]{2,5})?)?" // may have a filename with or without an extension
+ "(?:\\?(?:"+ QNV + ")(?:&(?:" + QNV + "))*)?" // may have a ?query=string
+ "(?:#[\\w-]+)?"; // may have a #fragment
"(?:\\.?/)?" // may start with "/" or "./"
+ "(?:(?:[\\s\\w-]+|\\.\\.)(?:/))*" // may have path/segments/segment2
+ "([\\s\\w-]+(?:\\.[\\w-]+)??(\\.[a-zA-Z0-9]{2,5})?)?" // may have a filename with or without an extension
+ "(?:\\?(?:"+ QNV + ")(?:&(?:" + QNV + "))*)?" // may have a ?query=string
+ "(?:#[\\w-]+)?"; // may have a #fragment
public static boolean isVeryLikelyUri(CharSequence candidate) {
// must have a . or /
if (!TextUtils.matches(NAIVE_LIKELY_URI_PATTERN, candidate)) {
@@ -423,7 +425,7 @@ public class UriUtils {
if (!matcher.matches()) {
return false;
}
/*
* Remaining tests discard stuff that the
* LIKELY_RELATIVE_URI_PATTERN can't catch
@@ -438,6 +440,12 @@ public class UriUtils {
return false;
}
if (TextUtils.matches(".*\\s+.*", candidate)
&& (extension == null
|| !KNOWN_GOOD_FILE_EXTENSIONS.contains(extension))) {
return false;
}
// text or application mimetype
if (TextUtils.matches("(?:text|application)/[^/]+", candidate)) {
return false;
modules/src/main/java/org/archive/modules/extractor/ExtractorJS.java
+2 -2
View File
@@ -63,11 +63,11 @@ public class ExtractorJS extends ContentExtractor {
private static Logger LOGGER =
Logger.getLogger(ExtractorJS.class.getName());
// finds whitespace- and quote-free strings in Javascript
// finds strings in Javascript
// (areas between paired ' or " characters, possibly backslash-quoted
// on the ends, but not in the middle)
protected static final String JAVASCRIPT_STRING_EXTRACTOR =
"(\\\\{0,8}+(?:['\"]|u002[27]))([^\\s'\"]{1,"+UURI.MAX_URL_LENGTH+"})(?:\\1)";
"(\\\\{0,8}+(?:['\"]|u002[27]))([^'\"]{0,"+UURI.MAX_URL_LENGTH+"})(?:\\1)";
// GROUPS:
// (G1) ' or " with optional leading backslashes
modules/src/test/java/org/archive/modules/extractor/ExtractorJSTest.java
+26 -3
View File
@@ -124,9 +124,33 @@ public class ExtractorJSTest extends StringExtractorTestBase {
"http://www.archive.org/static/0000/2683/good_filename_with.two_dots.jpg",
"{nonUrl: 'non-filename.with_two.dots',etc:'foo foo' }",
null
null,
"\"FileRef\": \"\\u002fsites\\u002fprb\\u002fPublic Comment Emails PDF\\u002fOpen Records Law_1r71vq4i.pdf\",\"",
"http://www.archive.org/sites/prb/Public Comment Emails PDF/Open Records Law_1r71vq4i.pdf",
"\"FileRef\": \"\\u002fsites\\u002fprb\\u002fPublic Comment Emails PDF\\u002fOpen Records Law_1r71vq4i.bad\",\"",
null,
"\"FileRef\": \"\\u002fsites\\u002fprb\\u002fPublic Comment Emails PDF\\u002fOpenRecordsLaw_1r71vq4i.bad\",\"",
null,
"\"FileRef\": \"\\u002fsites\\u002fprb\\u002fPublic Comment Emails PDF\\u002fOpen Records Law_1r71vq4i\",\"",
null,
"\"FileRef\": \"\\u002fsites\\u002fprb\\u002fPublic Comment Emails PDF\\u002fOpenRecordsLaw_1r71vq4i\",\"",
null,
/*
* XXX this one fails currently because the string has no slashes or
* dots, until it is javascript-unescaped, which happens too late.
* Unescaping earlier involves converting many more CharSubSequence to
* String. Is it worth the performance hit?
*/
// "\"FileRef\": \"\\u002fsites\\u002fprb\\u002fPublicCommentEmailsPDF\\u002fOpenRecordsLaw_1r71vq4i\",\"",
// "http://www.archive.org/sites/prb/PublicCommentEmailsPDF/OpenRecordsLaw_1r71vq4i",
};
@Override
protected String[] getValidTestData() {
return VALID_TEST_DATA;
@@ -134,7 +158,6 @@ public class ExtractorJSTest extends StringExtractorTestBase {
@Override
protected Extractor makeExtractor() {
ExtractorJS result = new ExtractorJS();
UriErrorLoggerModule ulm = new UnitTestUriLoggerModule();
result.setLoggerModule(ulm);