lee/heritrix3
1
0
Fork 0
mirror of https://github.com/internetarchive/heritrix3.git synced 2026-09-10 07:45:42 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #145 from vonrosen/ARI-4713

Browse Source 
Make some urls with whitespace acceptable to JavaScript extractor.
This commit is contained in:
Hunter
2016-02-11 08:27:52 -08:00
parent d80cf8244b cbb5e3ab99
commit bfbd61edd7
3 changed files with 46 additions and 15 deletions
Download Patch File Download Diff File Expand all files Collapse all files
modules/src/main/java/org/archive/modules/extractor/ExtractorJS.java
+2 -2
View File
@@ -63,11 +63,11 @@ public class ExtractorJS extends ContentExtractor {
private static Logger LOGGER =
Logger.getLogger(ExtractorJS.class.getName());
// finds whitespace- and quote-free strings in Javascript
// finds strings in Javascript
// (areas between paired ' or " characters, possibly backslash-quoted
// on the ends, but not in the middle)
protected static final String JAVASCRIPT_STRING_EXTRACTOR =
"(\\\\{0,8}+(?:['\"]|u002[27]))([^\\s'\"]{1,"+UURI.MAX_URL_LENGTH+"})(?:\\1)";
"(\\\\{0,8}+(?:['\"]|u002[27]))([^'\"]{0,"+UURI.MAX_URL_LENGTH+"})(?:\\1)";
// GROUPS:
// (G1) ' or " with optional leading backslashes
modules/src/test/java/org/archive/modules/extractor/ExtractorJSTest.java
+26 -3
View File
@@ -124,9 +124,33 @@ public class ExtractorJSTest extends StringExtractorTestBase {
"http://www.archive.org/static/0000/2683/good_filename_with.two_dots.jpg",
"{nonUrl: 'non-filename.with_two.dots',etc:'foo foo' }",
null
null,
"\"FileRef\": \"\\u002fsites\\u002fprb\\u002fPublic Comment Emails PDF\\u002fOpen Records Law_1r71vq4i.pdf\",\"",
"http://www.archive.org/sites/prb/Public Comment Emails PDF/Open Records Law_1r71vq4i.pdf",
"\"FileRef\": \"\\u002fsites\\u002fprb\\u002fPublic Comment Emails PDF\\u002fOpen Records Law_1r71vq4i.bad\",\"",
null,
"\"FileRef\": \"\\u002fsites\\u002fprb\\u002fPublic Comment Emails PDF\\u002fOpenRecordsLaw_1r71vq4i.bad\",\"",
null,
"\"FileRef\": \"\\u002fsites\\u002fprb\\u002fPublic Comment Emails PDF\\u002fOpen Records Law_1r71vq4i\",\"",
null,
"\"FileRef\": \"\\u002fsites\\u002fprb\\u002fPublic Comment Emails PDF\\u002fOpenRecordsLaw_1r71vq4i\",\"",
null,
/*
* XXX this one fails currently because the string has no slashes or
* dots, until it is javascript-unescaped, which happens too late.
* Unescaping earlier involves converting many more CharSubSequence to
* String. Is it worth the performance hit?
*/
// "\"FileRef\": \"\\u002fsites\\u002fprb\\u002fPublicCommentEmailsPDF\\u002fOpenRecordsLaw_1r71vq4i\",\"",
// "http://www.archive.org/sites/prb/PublicCommentEmailsPDF/OpenRecordsLaw_1r71vq4i",
};
@Override
protected String[] getValidTestData() {
return VALID_TEST_DATA;
@@ -134,7 +158,6 @@ public class ExtractorJSTest extends StringExtractorTestBase {
@Override
protected Extractor makeExtractor() {
ExtractorJS result = new ExtractorJS();
UriErrorLoggerModule ulm = new UnitTestUriLoggerModule();
result.setLoggerModule(ulm);