diff --git a/modules/src/main/java/org/archive/modules/extractor/ExtractorMultipleRegex.java b/modules/src/main/java/org/archive/modules/extractor/ExtractorMultipleRegex.java index fd6a2fdb..79c81d6f 100644 --- a/modules/src/main/java/org/archive/modules/extractor/ExtractorMultipleRegex.java +++ b/modules/src/main/java/org/archive/modules/extractor/ExtractorMultipleRegex.java @@ -98,13 +98,15 @@ public class ExtractorMultipleRegex extends Extractor { if (!m.matches()) { return; } + + // our data structure to prepopulate with matches for nested iteration + Map>> allMatches = new LinkedHashMap>>(); + List uriRegexGroups = new LinkedList(); for(int i = 0; i <= m.groupCount(); i++) { uriRegexGroups.add(m.group(i)); } - // our data structure to prepopulate with matches for nested iteration - LinkedHashMap>> allMatches = new LinkedHashMap>>(); - LinkedList> uriRegexMatchList = new LinkedList>(); + List> uriRegexMatchList = new LinkedList>(); uriRegexMatchList.add(uriRegexGroups); allMatches.put("uriRegex", uriRegexMatchList); @@ -117,16 +119,17 @@ public class ExtractorMultipleRegex extends Extractor { Thread.currentThread().getName(), e); return; } + // the names for regexes given in the config Set names = getContentRegexes().keySet(); - for (String patternName : names) { + for (String patternName: names) { // the matcher for this patternName against the content Matcher namedMatcher = TextUtils.getMatcher(getContentRegexes().get(patternName), cs); // populate the list of finds for this patternName List> foundList = new LinkedList>(); while (namedMatcher.find()) { - // +1 to include the full match in addition to the groups LinkedList groups = new LinkedList(); + // include group 0, the full pattern match for (int i = 0; i <= namedMatcher.groupCount(); i++) { groups.add(namedMatcher.group(i)); } @@ -135,26 +138,35 @@ public class ExtractorMultipleRegex extends Extractor { allMatches.put(patternName, foundList); } + long i = 0; boolean done = false; while (!done) { long tmp = i; - SimpleBindings matches = new SimpleBindings(); + + // bindings are the variables available to populate the template + // { String patternName => List groups } + Bindings bindings = new SimpleBindings(); String[] patternNames = allMatches.keySet().toArray(new String[0]); for (int j = 0; j < patternNames.length; j++) { List> matchList = allMatches.get(patternNames[j]); + if (j == patternNames.length - 1 && tmp >= matchList.size()) { done = true; break; } + int index = (int) (tmp % matchList.size()); - matches.put(patternNames[j], matchList.get(index)); - matches.put(patternNames[j] + "Index", index); + bindings.put(patternNames[j], matchList.get(index)); + + // make the index of this match available to the template as well + bindings.put(patternNames[j] + "Index", index); + tmp = tmp / matchList.size(); } if (!done) { - addOutlink(curi, matches); + addOutlink(curi, bindings); } i++; diff --git a/modules/src/test/java/org/archive/modules/extractor/ExtractorMultipleRegexTest.java b/modules/src/test/java/org/archive/modules/extractor/ExtractorMultipleRegexTest.java index 9ef3d91e..a06938ae 100644 --- a/modules/src/test/java/org/archive/modules/extractor/ExtractorMultipleRegexTest.java +++ b/modules/src/test/java/org/archive/modules/extractor/ExtractorMultipleRegexTest.java @@ -18,68 +18,115 @@ */ package org.archive.modules.extractor; -import java.util.ArrayList; -import java.util.Collection; +import java.io.BufferedReader; +import java.io.FileInputStream; +import java.io.IOException; +import java.io.InputStreamReader; import java.util.LinkedHashMap; -import java.util.List; +import org.apache.commons.lang.StringEscapeUtils; import org.archive.modules.CrawlURI; import org.archive.net.UURI; import org.archive.net.UURIFactory; import org.archive.util.Recorder; -public class ExtractorMultipleRegexTest extends StringExtractorTestBase { - final public static String[] VALID_TEST_DATA = new String[] { - // https://www.facebook.com/NorthCarolinaStateParks some time in the past - "{\"profile_id\":143412869029,\"start\":1351753200,\"end" + - "\":1354348799,\"query_type\":31,\"section_pagelet_id\":\"" + - "pagelet_timeline_earlier_this_month_all\",\"load_immediately\"" + - ":false},false,null,1,-1]],[\"TimelineContentLoader\",\"load" + - "SectionOnClick\",[\"m959614_181\"],[{\"__m\":\"m959614_181\"}" + - ",\"month_2012_10\"]],[\"TimelineContentLoader\",\"registerTim" + - "ePeriod\",[\"m959614_182\"],[{\"__m\":\"m959614_182\"},\"month_20" + - "12_10\",{\"profile_id\":143412869029,\"start\":1349074800,\"end\":13517" + - "53199,\"query_type\":25,\"section_pagelet_id\":\"pagelet_timeline_month_al" + - "l_last\",\"load_immediately\":false},false,null,2,-1]],[\"TimelineContent" + - "Loader\",\"loadSectionOnClick\",[\"m959614_183\"],[{\"__m\":\"m959614" + - "_183\"},\"year_2012\"]],[\"TimelineContentLoader\",\"registerTimePeri" + - "od\",[\"m959614_184\"],[{\"__m\":\"m959614_184\"},\"year_2012\",{\"profile_" + - "id\":143412869029,\"start\":1325404800,\"end\":1357027199,\"query_type\":8" + - ",\"filter_after_timestamp\":1349074799,\"section_pagelet_id\":\"pagelet_" + - "timeline_year_current\",\"load_immediately\":false},false,null,3,\"101.67" + - "277588916\"]],\n [\"TimelineContentLoader\",\"setExpandLoadDataForSection\",[],[\"y" + - "ear_2012\",{\"profile_id\":143412869029,\"start\":1325404800,\"end\":135" + - "7027199,\"query_type\":9}]],[\"TimelineContentLoader\",\"loadSectionOnClick" + - "\",[\"m959614_185\"],[{\"__m\":\"m959614_185\"},\"year_2011\"]],[\"TimelineCo" + - "ntentLoader\",\"registerTimePeriod\",[\"m959614_186\"],[{\"__m\":\"m959614_186\"}" + - ",\"year_2011\",{\"profile_id\":143412869029,\"start\":1293868800,\"end\":13254" + - "04799,\"query_type\":8,\"section_pagelet_id\":\"pagelet_timeline_year_last\",\"l" + - "oad_immediately\":false},false,null,4,\"86.97249252724\"]],[\"TimelineContentLoad" + - "er\",\"setExpandLoadDataForSection\",[],[\"year_2011\",{\"profile_id\":143412869029" + - ",\"start\":1293868800,\"end\":1325404799,\"query_type\":9}]],[\"TimelineContentLoad" + - "er\",\"loadSectionOnClick\",[\"m959614_187\"],[{\"__m\":\"m959614_187\"},\"year_20" + - "10\"]],[\"TimelineContentLoader\",\"registerTimePeriod\",[\"m959614_188\"],[{\"_" + - "_m\":\"m959614_188\"},\"year_2010\",{\"profile_id\":143412869029,\"start\":126233" + - "2800,\"end\":1293868799,\"query_type\":8,\"section_pagelet_id\":\"pagelet\n _timel" + - "ine_year_2010\",\"load_immediately\":false},false,null,5,\"88.440195233432\"]],[\"Ti" + - "melineContentLoader\",\"setExpandLoadDataForSection\",[],[\"year_2010\",{\"profile_i" + - "d\":143412869029,\"start\":1262332800,\"end\":1293868799,\"query_type\":9}]],[\"Time" + - "lineContentLoader\",\"loadSectionOnClick\",[\"m959614_189\"],[{\"__m\":\"m959614_189" + - "\"},\"year_2009\"]],[\"TimelineContentLoader\",\"registerTimePeriod\",[\"m959614_190" + - "\"],[{\"__m\":\"m959614_190\"},\"year_2009\",{\"profile_id\":143412869029,\"start\":" + - "1230796800,\"end\":1262332799,\"query_type\":8,\"section_pagelet_id\":\"pagelet_time" + - "line_year_2009\",\"load_immediately\":false},false,null,6,\"41.800676041109\"]],[\"T" + - "imelineContentLoader\",\"setExpandLoadDataForSection\",[],[\"year_2009\",{\"profile_" + - "id\":143412869029,\"start\":1230796800,\"end\":1262332799,\"query_type\":9}]]]},\"cs" + - "s\":[\"M501h\",\"o8oNt\"],\"js\":[\"dERRF\",\"yEdv1\",\"JWWMg\"],\"id\":\"timeline_s" + - "ection_placeholders\",\"phase\":3})\n", - "http://nourl.com/dne", - }; +public class ExtractorMultipleRegexTest extends ContentExtractorTestBase { + + /* + * Test data is actual content of + * https://www.facebook.com/NorthCarolinaStateParks as retrieved in browser + * on Nov 15 2012, and expected outlinks are actual urls retrieved as a + * result of scrolling down. + */ + protected static final String TEST_URI = "https://www.facebook.com/NorthCarolinaStateParks"; - @Override - protected String[] getValidTestData() { - return VALID_TEST_DATA; - } + protected static final String[] EXPECTED_OUTLINKS = { + "https://www.facebook.com/ajax/pagelet/generic.php/ProfileTimelineSectionPagelet?ajaxpipe=1&ajaxpipe_token=AXiCHz3XOZzlpLVQ&no_script_path=1&data=%7B%22profile_id%22%3A143412869029%2C%22start%22%3A1349074800%2C%22end%22%3A1351753199%2C%22query_type%22%3A25%2C%22section_pagelet_id%22%3A%22pagelet_timeline_month_all_last%22%2C%22load_immediately%22%3Afalse%2C%22time_cutoff%22%3A1351254247%2C%22force_no_friend_activity%22%3Afalse%7D&__user=0&__a=1&__adt=2", + "https://www.facebook.com/ajax/pagelet/generic.php/ProfileTimelineSectionPagelet?ajaxpipe=1&ajaxpipe_token=AXiCHz3XOZzlpLVQ&no_script_path=1&data=%7B%22profile_id%22%3A143412869029%2C%22start%22%3A1325404800%2C%22end%22%3A1357027199%2C%22query_type%22%3A8%2C%22filter_after_timestamp%22%3A1349074799%2C%22section_pagelet_id%22%3A%22pagelet_timeline_year_current%22%2C%22load_immediately%22%3Afalse%2C%22time_cutoff%22%3A1351254247%2C%22force_no_friend_activity%22%3Afalse%7D&__user=0&__a=1&__adt=3", + "https://www.facebook.com/ajax/pagelet/generic.php/ProfileTimelineSectionPagelet?ajaxpipe=1&ajaxpipe_token=AXiCHz3XOZzlpLVQ&no_script_path=1&data=%7B%22profile_id%22%3A143412869029%2C%22start%22%3A1293868800%2C%22end%22%3A1325404799%2C%22query_type%22%3A8%2C%22section_pagelet_id%22%3A%22pagelet_timeline_year_last%22%2C%22load_immediately%22%3Afalse%2C%22time_cutoff%22%3A1351254247%2C%22force_no_friend_activity%22%3Afalse%7D&__user=0&__a=1&__adt=4", + "https://www.facebook.com/ajax/pagelet/generic.php/ProfileTimelineSectionPagelet?ajaxpipe=1&ajaxpipe_token=AXiCHz3XOZzlpLVQ&no_script_path=1&data=%7B%22profile_id%22%3A143412869029%2C%22start%22%3A1262332800%2C%22end%22%3A1293868799%2C%22query_type%22%3A8%2C%22section_pagelet_id%22%3A%22pagelet_timeline_year_2010%22%2C%22load_immediately%22%3Afalse%2C%22time_cutoff%22%3A1351254247%2C%22force_no_friend_activity%22%3Afalse%7D&__user=0&__a=1&__adt=5", + "https://www.facebook.com/ajax/pagelet/generic.php/ProfileTimelineSectionPagelet?ajaxpipe=1&ajaxpipe_token=AXiCHz3XOZzlpLVQ&no_script_path=1&data=%7B%22profile_id%22%3A143412869029%2C%22start%22%3A1230796800%2C%22end%22%3A1262332799%2C%22query_type%22%3A8%2C%22section_pagelet_id%22%3A%22pagelet_timeline_year_2009%22%2C%22load_immediately%22%3Afalse%2C%22time_cutoff%22%3A1351254247%2C%22force_no_friend_activity%22%3Afalse%7D&__user=0&__a=1&__adt=6", + "https://www.facebook.com/ajax/pagelet/generic.php/ProfileTimelineSectionPagelet?ajaxpipe=1&ajaxpipe_token=AXiCHz3XOZzlpLVQ&no_script_path=1&data=%7B%22profile_id%22%3A143412869029%2C%22start%22%3A1291190400%2C%22end%22%3A1293868799%2C%22query_type%22%3A25%2C%22section_pagelet_id%22%3A%22pagelet_timeline_month_all_2010_12%22%2C%22load_immediately%22%3Afalse%2C%22time_cutoff%22%3A1351254247%2C%22parent_key%22%3A%22year_2010%22%2C%22force_no_friend_activity%22%3Afalse%7D&__user=0&__a=1&__adt=7", + "https://www.facebook.com/ajax/pagelet/generic.php/ProfileTimelineSectionPagelet?ajaxpipe=1&ajaxpipe_token=AXiCHz3XOZzlpLVQ&no_script_path=1&data=%7B%22profile_id%22%3A143412869029%2C%22start%22%3A1288594800%2C%22end%22%3A1291190399%2C%22query_type%22%3A25%2C%22section_pagelet_id%22%3A%22pagelet_timeline_month_all_2010_11%22%2C%22load_immediately%22%3Afalse%2C%22time_cutoff%22%3A1351254247%2C%22parent_key%22%3A%22year_2010%22%2C%22force_no_friend_activity%22%3Afalse%7D&__user=0&__a=1&__adt=8" + }; + + /* + * Actual content of https://www.facebook.com/NorthCarolinaStateParks as + * retrieved in browser in Nov 15 2012. Java doesn't allow constants longer + * than 65535 bytes, so we concatenate these chunks in code. + */ + protected static final String[] TEST_CONTENT_CHUNKS = { + "\n", + "\n", + "North Carolina State Parks and Recreation | Facebook\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "
Facebook \u00A9 2012 \u00B7 English (US)
Mobile \u00B7 Find Friends \u00B7 Badges \u00B7 People \u00B7 Pages \u00B7 Apps \u00B7 Games \u00B7 Music \u00B7 About \u00B7 Create an Ad \u00B7 Create a Page \u00B7 Developers \u00B7 Careers \u00B7 Privacy \u00B7 Cookies \u00B7 Terms \u00B7 Help
\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + " \n", + }; /* * Settings for extracting scroll-down ajax urls from facebook, as they are @@ -116,35 +163,44 @@ public class ExtractorMultipleRegexTest extends StringExtractorTestBase { extractor.setTemplate("/ajax/pagelet/generic.php/ProfileTimelineSectionPagelet" + "?ajaxpipe=1&ajaxpipe_token=${ajaxpipeToken[1]}&no_script_path=1" + "&data=${java.net.URLEncoder.encode('{' + jsonBlob[1] + ',\"time_cutoff\":' + timeCutoff[1] + ',\"force_no_friend_activity\":false}', 'UTF-8')}" - + "&__user=0&__a=1&__adt=${jsonBlobIndex+1}"); + + "&__user=0&__a=1&__adt=${jsonBlobIndex+2}"); return extractor; } - @Override - protected Collection makeData(String content, String destURI) - throws Exception { - List result = new ArrayList(); - UURI src = UURIFactory.getInstance("https://www.facebook.com/NorthCarolinaStateParks"); - CrawlURI euri = new CrawlURI(src, null, null, - LinkContext.NAVLINK_MISC); - Recorder recorder = createRecorder(content); - euri.setContentType("text/html"); - euri.setRecorder(recorder); - euri.setContentSize(content.length()); - - UURI dest = UURIFactory.getInstance(destURI); - Link link = new Link(src, dest, HTMLLinkContext.INFERRED_MISC, Hop.INFERRED); - result.add(new TestData(euri, link)); + public void testFacebookScrollExample() throws Exception { + UURI testUuri = UURIFactory.getInstance(TEST_URI); + CrawlURI testUri = new CrawlURI(testUuri, null, null, LinkContext.NAVLINK_MISC); + StringBuilder content = new StringBuilder(); + for (String chunk: TEST_CONTENT_CHUNKS) { + content.append(chunk); + } + Recorder recorder = createRecorder(content.toString(), "UTF-8"); + testUri.setContentType("text/html"); + testUri.setRecorder(recorder); + testUri.setContentSize(content.length()); - euri = new CrawlURI(src, null, null, LinkContext.NAVLINK_MISC); - recorder = createRecorder(content); - euri.setContentType("application/xhtml"); - euri.setRecorder(recorder); - euri.setContentSize(content.length()); - result.add(new TestData(euri, link)); + extractor.process(testUri); - return result; + for (Link outlink: testUri.getOutLinks()) { + System.out.println("extracted: " + outlink); + } + + for (String expectedLinkString: EXPECTED_OUTLINKS) { + Link expectedLink = new Link(TEST_URI, expectedLinkString, + HTMLLinkContext.INFERRED_MISC, Hop.INFERRED); + assertTrue(testUri.getOutLinks().contains(expectedLink)); + } + } + + public static void main(String[] args) throws IOException { + BufferedReader reader = new BufferedReader(new InputStreamReader(new FileInputStream("/tmp/input.txt"), "UTF-8")); + for (String line = reader.readLine(); line != null; line = reader.readLine()) { + String escapedLine = StringEscapeUtils.escapeJava(line.trim()); + System.out.print("\""); + System.out.print(escapedLine); + System.out.println("\\n\", "); + } } }