diff --git a/modules/src/main/java/org/archive/modules/extractor/ExtractorJS.java b/modules/src/main/java/org/archive/modules/extractor/ExtractorJS.java index 815a280d..0757deb6 100644 --- a/modules/src/main/java/org/archive/modules/extractor/ExtractorJS.java +++ b/modules/src/main/java/org/archive/modules/extractor/ExtractorJS.java @@ -67,7 +67,7 @@ public class ExtractorJS extends ContentExtractor { // (areas between paired ' or " characters, possibly backslash-quoted // on the ends, but not in the middle) protected static final String JAVASCRIPT_STRING_EXTRACTOR = - "(\\\\{0,8}+['\"\\\\u{0027}\\\\u{0022}])([^\\s'\"]{1,"+UURI.MAX_URL_LENGTH+"})(?:\\1)"; + "(\\\\{0,8}+['\"])([^\\s'\"]{1,"+UURI.MAX_URL_LENGTH+"})(?:\\1)"; // GROUPS: // (G1) ' or " with optional leading backslashes @@ -142,13 +142,16 @@ public class ExtractorJS extends ContentExtractor { public long considerStrings(Extractor ext, CrawlURI curi, CharSequence cs, boolean handlingJSFile) { long foundLinks = 0; + + String unescapedJavaScript = StringEscapeUtils.unescapeJavaScript(cs.toString()); + Matcher strings = - TextUtils.getMatcher(JAVASCRIPT_STRING_EXTRACTOR, cs); + TextUtils.getMatcher(JAVASCRIPT_STRING_EXTRACTOR, unescapedJavaScript); int startIndex = 0; while (strings.find(startIndex)) { CharSequence subsequence = - cs.subSequence(strings.start(2), strings.end(2)); + unescapedJavaScript.subSequence(strings.start(2), strings.end(2)); if (UriUtils.isPossibleUri(subsequence)) { if (considerString(ext, curi, handlingJSFile, subsequence.toString())) { diff --git a/modules/src/test/java/org/archive/modules/extractor/ExtractorJSTest.java b/modules/src/test/java/org/archive/modules/extractor/ExtractorJSTest.java index cc095078..72900776 100644 --- a/modules/src/test/java/org/archive/modules/extractor/ExtractorJSTest.java +++ b/modules/src/test/java/org/archive/modules/extractor/ExtractorJSTest.java @@ -22,6 +22,7 @@ import java.util.ArrayList; import java.util.Collection; import java.util.List; +import org.apache.commons.lang.StringEscapeUtils; import org.archive.modules.CrawlURI; import org.archive.net.UURI; import org.archive.net.UURIFactory; @@ -113,6 +114,12 @@ public class ExtractorJSTest extends StringExtractorTestBase { "var blah='/good/query/value/with/url-escaping.html?foo=bar%20bar';", "http://www.archive.org/good/query/value/with/url-escaping.html?foo=bar%20bar", + + "\\u0027project_detail.aspx?guid=unicodesinglequote\\u0027", + "http://www.archive.org/foo/project_detail.aspx?guid=unicodesinglequote", + + "\\u0022project_detail.aspx?guid=unicodedoublequote\\u0022", + "http://www.archive.org/foo/project_detail.aspx?guid=unicodedoublequote", }; @Override @@ -122,6 +129,7 @@ public class ExtractorJSTest extends StringExtractorTestBase { @Override protected Extractor makeExtractor() { + ExtractorJS result = new ExtractorJS(); UriErrorLoggerModule ulm = new UnitTestUriLoggerModule(); result.setLoggerModule(ulm);