lee/heritrix3
1
0
Fork 0
mirror of https://github.com/internetarchive/heritrix3.git synced 2026-09-25 07:05:49 +00:00
Code Issues Packages Projects Releases Wiki Activity

Revise parsing of string with unicode quoted data

Browse Source
This commit is contained in:
Hunter Stern
2014-09-03 15:27:58 -07:00
parent ea235f4466
commit 2e4f513a4c
2 changed files with 14 additions and 3 deletions
Download Patch File Download Diff File Expand all files Collapse all files
modules/src/main/java/org/archive/modules/extractor/ExtractorJS.java
+6 -3
View File
@@ -67,7 +67,7 @@ public class ExtractorJS extends ContentExtractor {
// (areas between paired ' or " characters, possibly backslash-quoted
// on the ends, but not in the middle)
protected static final String JAVASCRIPT_STRING_EXTRACTOR =
"(\\\\{0,8}+['\"\\\\u{0027}\\\\u{0022}])([^\\s'\"]{1,"+UURI.MAX_URL_LENGTH+"})(?:\\1)";
"(\\\\{0,8}+['\"])([^\\s'\"]{1,"+UURI.MAX_URL_LENGTH+"})(?:\\1)";
// GROUPS:
// (G1) ' or " with optional leading backslashes
@@ -142,13 +142,16 @@ public class ExtractorJS extends ContentExtractor {
public long considerStrings(Extractor ext,
CrawlURI curi, CharSequence cs, boolean handlingJSFile) {
long foundLinks = 0;
String unescapedJavaScript = StringEscapeUtils.unescapeJavaScript(cs.toString());
Matcher strings =
TextUtils.getMatcher(JAVASCRIPT_STRING_EXTRACTOR, cs);
TextUtils.getMatcher(JAVASCRIPT_STRING_EXTRACTOR, unescapedJavaScript);
int startIndex = 0;
while (strings.find(startIndex)) {
CharSequence subsequence =
cs.subSequence(strings.start(2), strings.end(2));
unescapedJavaScript.subSequence(strings.start(2), strings.end(2));
if (UriUtils.isPossibleUri(subsequence)) {
if (considerString(ext, curi, handlingJSFile, subsequence.toString())) {
modules/src/test/java/org/archive/modules/extractor/ExtractorJSTest.java
+8
View File
@@ -22,6 +22,7 @@ import java.util.ArrayList;
import java.util.Collection;
import java.util.List;
import org.apache.commons.lang.StringEscapeUtils;
import org.archive.modules.CrawlURI;
import org.archive.net.UURI;
import org.archive.net.UURIFactory;
@@ -113,6 +114,12 @@ public class ExtractorJSTest extends StringExtractorTestBase {
"var blah='/good/query/value/with/url-escaping.html?foo=bar%20bar';",
"http://www.archive.org/good/query/value/with/url-escaping.html?foo=bar%20bar",
"\\u0027project_detail.aspx?guid=unicodesinglequote\\u0027",
"http://www.archive.org/foo/project_detail.aspx?guid=unicodesinglequote",
"\\u0022project_detail.aspx?guid=unicodedoublequote\\u0022",
"http://www.archive.org/foo/project_detail.aspx?guid=unicodedoublequote",
};
@Override
@@ -122,6 +129,7 @@ public class ExtractorJSTest extends StringExtractorTestBase {
@Override
protected Extractor makeExtractor() {
ExtractorJS result = new ExtractorJS();
UriErrorLoggerModule ulm = new UnitTestUriLoggerModule();
result.setLoggerModule(ulm);