diff --git a/modules/src/main/java/org/archive/modules/extractor/ExtractorMultipleRegex.java b/modules/src/main/java/org/archive/modules/extractor/ExtractorMultipleRegex.java index a8a734a2..fd6a2fdb 100644 --- a/modules/src/main/java/org/archive/modules/extractor/ExtractorMultipleRegex.java +++ b/modules/src/main/java/org/archive/modules/extractor/ExtractorMultipleRegex.java @@ -20,12 +20,11 @@ package org.archive.modules.extractor; import java.io.IOException; import java.util.HashMap; +import java.util.LinkedHashMap; import java.util.LinkedList; import java.util.List; import java.util.Map; import java.util.Set; -import java.util.SortedMap; -import java.util.TreeMap; import java.util.logging.Level; import java.util.logging.Logger; import java.util.regex.Matcher; @@ -99,13 +98,13 @@ public class ExtractorMultipleRegex extends Extractor { if (!m.matches()) { return; } - String[] uriRegexGroups = new String[m.groupCount() +1]; - for(int i = 0; i < uriRegexGroups.length; i++) { - uriRegexGroups[i] = m.group(i); + List uriRegexGroups = new LinkedList(); + for(int i = 0; i <= m.groupCount(); i++) { + uriRegexGroups.add(m.group(i)); } // our data structure to prepopulate with matches for nested iteration - SortedMap> allMatches = new TreeMap>(); - LinkedList uriRegexMatchList = new LinkedList(); + LinkedHashMap>> allMatches = new LinkedHashMap>>(); + LinkedList> uriRegexMatchList = new LinkedList>(); uriRegexMatchList.add(uriRegexGroups); allMatches.put("uriRegex", uriRegexMatchList); @@ -124,12 +123,12 @@ public class ExtractorMultipleRegex extends Extractor { // the matcher for this patternName against the content Matcher namedMatcher = TextUtils.getMatcher(getContentRegexes().get(patternName), cs); // populate the list of finds for this patternName - List foundList = new LinkedList(); - while(namedMatcher.find()) { + List> foundList = new LinkedList>(); + while (namedMatcher.find()) { // +1 to include the full match in addition to the groups - String[] groups = new String[namedMatcher.groupCount() +1]; - for(int i = 0; i < groups.length; i++) { - groups[i] = namedMatcher.group(i); + LinkedList groups = new LinkedList(); + for (int i = 0; i <= namedMatcher.groupCount(); i++) { + groups.add(namedMatcher.group(i)); } foundList.add(groups); } @@ -141,15 +140,16 @@ public class ExtractorMultipleRegex extends Extractor { while (!done) { long tmp = i; SimpleBindings matches = new SimpleBindings(); - matches.put("index", i); String[] patternNames = allMatches.keySet().toArray(new String[0]); for (int j = 0; j < patternNames.length; j++) { - List matchList = allMatches.get(patternNames[j]); + List> matchList = allMatches.get(patternNames[j]); if (j == patternNames.length - 1 && tmp >= matchList.size()) { done = true; break; } - matches.put(patternNames[j], matchList.get((int) (tmp % matchList.size()))); + int index = (int) (tmp % matchList.size()); + matches.put(patternNames[j], matchList.get(index)); + matches.put(patternNames[j] + "Index", index); tmp = tmp / matchList.size(); } @@ -165,14 +165,14 @@ public class ExtractorMultipleRegex extends Extractor { GroovyScriptEngineImpl gse = new GroovyScriptEngineImpl(); String stringUri = null; try { - stringUri = (String) gse.eval("\""+ StringEscapeUtils.escapeJava(getTemplate()) +"\"", matches); + stringUri = gse.eval("\""+ StringEscapeUtils.escapeJava(getTemplate()) +"\"", matches).toString(); } catch (ScriptException e) { logUriError(new URIException(e.toString()), curi.getUURI(), stringUri); return; } try { - int max = getExtractorParameters().getMaxOutlinks(); - Link.addRelativeToBase(curi, max, stringUri, + Link.addRelativeToBase(curi, + getExtractorParameters().getMaxOutlinks(), stringUri, HTMLLinkContext.INFERRED_MISC, Hop.INFERRED); } catch (URIException e) { logUriError(e, curi.getUURI(), stringUri); diff --git a/modules/src/test/java/org/archive/modules/extractor/ExtractorMultipleRegexTest.java b/modules/src/test/java/org/archive/modules/extractor/ExtractorMultipleRegexTest.java index cdae5bfc..9ef3d91e 100644 --- a/modules/src/test/java/org/archive/modules/extractor/ExtractorMultipleRegexTest.java +++ b/modules/src/test/java/org/archive/modules/extractor/ExtractorMultipleRegexTest.java @@ -20,6 +20,7 @@ package org.archive.modules.extractor; import java.util.ArrayList; import java.util.Collection; +import java.util.LinkedHashMap; import java.util.List; import org.archive.modules.CrawlURI; @@ -29,12 +30,6 @@ import org.archive.util.Recorder; public class ExtractorMultipleRegexTest extends StringExtractorTestBase { final public static String[] VALID_TEST_DATA = new String[] { -// "yellow journalism A", -// "http://www.slashdot.org", -// -// " IMG", -// "http://www.archive.org/start/foo.gif", - // https://www.facebook.com/NorthCarolinaStateParks some time in the past "{\"profile_id\":143412869029,\"start\":1351753200,\"end" + "\":1354348799,\"query_type\":31,\"section_pagelet_id\":\"" + @@ -81,18 +76,49 @@ public class ExtractorMultipleRegexTest extends StringExtractorTestBase { "http://nourl.com/dne", }; - @Override protected String[] getValidTestData() { return VALID_TEST_DATA; } + /* + * Settings for extracting scroll-down ajax urls from facebook, as they are + * constructed as of Nov 14 2012. + * + * + * + * + * + * + * + * + * + * + * + * /ajax/pagelet/generic.php/ProfileTimelineSectionPagelet?ajaxpipe=1&ajaxpipe_token=${ajaxpipeToken[1]}&no_script_path=1&data=${java.net.URLEncoder.encode('{' + jsonBlob[1] + ',"time_cutoff":' + timeCutoff[1] + ',"force_no_friend_activity":false}', 'UTF-8')}&__user=0&__a=1&__adt=${jsonBlobIndex+1} + * + * + */ @Override protected Extractor makeExtractor() { - ExtractorMultipleRegex result = new ExtractorMultipleRegex(); + ExtractorMultipleRegex extractor = new ExtractorMultipleRegex(); UriErrorLoggerModule ulm = new UnitTestUriLoggerModule(); - result.setLoggerModule(ulm); - return result; + extractor.setLoggerModule(ulm); + + extractor.setUriRegex("^https?://(?:www\\.)?facebook\\.com/[^/?]+$"); + + LinkedHashMap contentRegexes = new LinkedHashMap(); + contentRegexes.put("jsonBlob", "\\{(\"profile_id\":\\d+,[^}]+)\\}"); + contentRegexes.put("ajaxpipeToken", "\"ajaxpipe_token\":\"([^\"]+)\""); + contentRegexes.put("timeCutoff", "\"setTimeCutoff\",[^,]*,\\[(\\d+)\\]\\]"); + extractor.setContentRegexes(contentRegexes); + + extractor.setTemplate("/ajax/pagelet/generic.php/ProfileTimelineSectionPagelet" + + "?ajaxpipe=1&ajaxpipe_token=${ajaxpipeToken[1]}&no_script_path=1" + + "&data=${java.net.URLEncoder.encode('{' + jsonBlob[1] + ',\"time_cutoff\":' + timeCutoff[1] + ',\"force_no_friend_activity\":false}', 'UTF-8')}" + + "&__user=0&__a=1&__adt=${jsonBlobIndex+1}"); + + return extractor; } @Override @@ -121,5 +147,4 @@ public class ExtractorMultipleRegexTest extends StringExtractorTestBase { return result; } - }