lee/heritrix3
1
0
Fork 0
mirror of https://github.com/internetarchive/heritrix3.git synced 2026-09-25 07:05:49 +00:00
Code Issues Packages Projects Releases Wiki Activity

Revise regex instead of unescaping js.

Browse Source
This commit is contained in:
Hunter Stern
2014-09-08 16:37:03 -07:00
parent 2e4f513a4c
commit 2af19fa03b
1 changed files with 3 additions and 5 deletions
Download Patch File Download Diff File Expand all files Collapse all files
modules/src/main/java/org/archive/modules/extractor/ExtractorJS.java
+3 -5
View File
@@ -67,7 +67,7 @@ public class ExtractorJS extends ContentExtractor {
// (areas between paired ' or " characters, possibly backslash-quoted
// on the ends, but not in the middle)
protected static final String JAVASCRIPT_STRING_EXTRACTOR =
"(\\\\{0,8}+['\"])([^\\s'\"]{1,"+UURI.MAX_URL_LENGTH+"})(?:\\1)";
"(\\\\{0,8}+(?:['\"]|u002[27]))([^\\s'\"]{1,"+UURI.MAX_URL_LENGTH+"})(?:\\1)";
// GROUPS:
// (G1) ' or " with optional leading backslashes
@@ -143,15 +143,13 @@ public class ExtractorJS extends ContentExtractor {
CrawlURI curi, CharSequence cs, boolean handlingJSFile) {
long foundLinks = 0;
String unescapedJavaScript = StringEscapeUtils.unescapeJavaScript(cs.toString());
Matcher strings =
TextUtils.getMatcher(JAVASCRIPT_STRING_EXTRACTOR, unescapedJavaScript);
TextUtils.getMatcher(JAVASCRIPT_STRING_EXTRACTOR, cs);
int startIndex = 0;
while (strings.find(startIndex)) {
CharSequence subsequence =
unescapedJavaScript.subSequence(strings.start(2), strings.end(2));
cs.subSequence(strings.start(2), strings.end(2));
if (UriUtils.isPossibleUri(subsequence)) {
if (considerString(ext, curi, handlingJSFile, subsequence.toString())) {