lee/heritrix3
1
0
Fork 0
mirror of https://github.com/internetarchive/heritrix3.git synced 2026-09-24 14:45:56 +00:00
Code Issues Packages Projects Releases Wiki Activity

more progress on ExtractorMultipleRegex

Browse Source
This commit is contained in:
Noah Levitt
2012-11-14 19:58:43 -08:00
parent 1998337645
commit 1d994493e2
2 changed files with 54 additions and 29 deletions
Download Patch File Download Diff File Expand all files Collapse all files
modules/src/main/java/org/archive/modules/extractor/ExtractorMultipleRegex.java
+18 -18
View File
@@ -20,12 +20,11 @@ package org.archive.modules.extractor;
import java.io.IOException;
import java.util.HashMap;
import java.util.LinkedHashMap;
import java.util.LinkedList;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.SortedMap;
import java.util.TreeMap;
import java.util.logging.Level;
import java.util.logging.Logger;
import java.util.regex.Matcher;
@@ -99,13 +98,13 @@ public class ExtractorMultipleRegex extends Extractor {
if (!m.matches()) {
return;
}
String[] uriRegexGroups = new String[m.groupCount() +1];
for(int i = 0; i < uriRegexGroups.length; i++) {
uriRegexGroups[i] = m.group(i);
List<String> uriRegexGroups = new LinkedList<String>();
for(int i = 0; i <= m.groupCount(); i++) {
uriRegexGroups.add(m.group(i));
}
// our data structure to prepopulate with matches for nested iteration
SortedMap<String,List<String[]>> allMatches = new TreeMap<String, List<String[]>>();
LinkedList<String[]> uriRegexMatchList = new LinkedList<String[]>();
LinkedHashMap<String,List<List<String>>> allMatches = new LinkedHashMap<String, List<List<String>>>();
LinkedList<List<String>> uriRegexMatchList = new LinkedList<List<String>>();
uriRegexMatchList.add(uriRegexGroups);
allMatches.put("uriRegex", uriRegexMatchList);
@@ -124,12 +123,12 @@ public class ExtractorMultipleRegex extends Extractor {
// the matcher for this patternName against the content
Matcher namedMatcher = TextUtils.getMatcher(getContentRegexes().get(patternName), cs);
// populate the list of finds for this patternName
List<String[]> foundList = new LinkedList<String[]>();
while(namedMatcher.find()) {
List<List<String>> foundList = new LinkedList<List<String>>();
while (namedMatcher.find()) {
// +1 to include the full match in addition to the groups
String[] groups = new String[namedMatcher.groupCount() +1];
for(int i = 0; i < groups.length; i++) {
groups[i] = namedMatcher.group(i);
LinkedList<String> groups = new LinkedList<String>();
for (int i = 0; i <= namedMatcher.groupCount(); i++) {
groups.add(namedMatcher.group(i));
}
foundList.add(groups);
}
@@ -141,15 +140,16 @@ public class ExtractorMultipleRegex extends Extractor {
while (!done) {
long tmp = i;
SimpleBindings matches = new SimpleBindings();
matches.put("index", i);
String[] patternNames = allMatches.keySet().toArray(new String[0]);
for (int j = 0; j < patternNames.length; j++) {
List<String[]> matchList = allMatches.get(patternNames[j]);
List<List<String>> matchList = allMatches.get(patternNames[j]);
if (j == patternNames.length - 1 && tmp >= matchList.size()) {
done = true;
break;
}
matches.put(patternNames[j], matchList.get((int) (tmp % matchList.size())));
int index = (int) (tmp % matchList.size());
matches.put(patternNames[j], matchList.get(index));
matches.put(patternNames[j] + "Index", index);
tmp = tmp / matchList.size();
}
@@ -165,14 +165,14 @@ public class ExtractorMultipleRegex extends Extractor {
GroovyScriptEngineImpl gse = new GroovyScriptEngineImpl();
String stringUri = null;
try {
stringUri = (String) gse.eval("\""+ StringEscapeUtils.escapeJava(getTemplate()) +"\"", matches);
stringUri = gse.eval("\""+ StringEscapeUtils.escapeJava(getTemplate()) +"\"", matches).toString();
} catch (ScriptException e) {
logUriError(new URIException(e.toString()), curi.getUURI(), stringUri);
return;
}
try {
int max = getExtractorParameters().getMaxOutlinks();
Link.addRelativeToBase(curi, max, stringUri,
Link.addRelativeToBase(curi,
getExtractorParameters().getMaxOutlinks(), stringUri,
HTMLLinkContext.INFERRED_MISC, Hop.INFERRED);
} catch (URIException e) {
logUriError(e, curi.getUURI(), stringUri);
modules/src/test/java/org/archive/modules/extractor/ExtractorMultipleRegexTest.java
+36 -11
View File
@@ -20,6 +20,7 @@ package org.archive.modules.extractor;
import java.util.ArrayList;
import java.util.Collection;
import java.util.LinkedHashMap;
import java.util.List;
import org.archive.modules.CrawlURI;
@@ -29,12 +30,6 @@ import org.archive.util.Recorder;
public class ExtractorMultipleRegexTest extends StringExtractorTestBase {
final public static String[] VALID_TEST_DATA = new String[] {
// "<a href=\"http://www.slashdot.org\">yellow journalism</a> A",
// "http://www.slashdot.org",
//
// "<img src=\"foo.gif\"> IMG",
// "http://www.archive.org/start/foo.gif",
// https://www.facebook.com/NorthCarolinaStateParks some time in the past
"{\"profile_id\":143412869029,\"start\":1351753200,\"end" +
"\":1354348799,\"query_type\":31,\"section_pagelet_id\":\"" +
@@ -81,18 +76,49 @@ public class ExtractorMultipleRegexTest extends StringExtractorTestBase {
"http://nourl.com/dne",
};
@Override
protected String[] getValidTestData() {
return VALID_TEST_DATA;
}
/*
* Settings for extracting scroll-down ajax urls from facebook, as they are
* constructed as of Nov 14 2012.
*
* <bean class="org.archive.modules.extractor.ExtractorMultipleRegex">
* <property name="uriRegex" value="^https?://(?:www\.)?facebook\.com/[^/?]+$" />
* <property name="contentRegexes">
* <map>
* <entry key="jsonBlob" value='\{("profile_id":\d+,[^}]+)\}' />
* <entry key="ajaxpipeToken" value='"ajaxpipe_token":"([^"]+)"' />
* <entry key="timeCutoff" value='"setTimeCutoff",[^,]*,\[(\d+)\]\]' />
* </map>
* </property>
* <property name="template">
* <value>/ajax/pagelet/generic.php/ProfileTimelineSectionPagelet?ajaxpipe=1&amp;ajaxpipe_token=${ajaxpipeToken[1]}&amp;no_script_path=1&amp;data=${java.net.URLEncoder.encode('{' + jsonBlob[1] + ',"time_cutoff":' + timeCutoff[1] + ',"force_no_friend_activity":false}', 'UTF-8')}&amp;__user=0&amp;__a=1&amp;__adt=${jsonBlobIndex+1}</value>
* </property>
* </bean>
*/
@Override
protected Extractor makeExtractor() {
ExtractorMultipleRegex result = new ExtractorMultipleRegex();
ExtractorMultipleRegex extractor = new ExtractorMultipleRegex();
UriErrorLoggerModule ulm = new UnitTestUriLoggerModule();
result.setLoggerModule(ulm);
return result;
extractor.setLoggerModule(ulm);
extractor.setUriRegex("^https?://(?:www\\.)?facebook\\.com/[^/?]+$");
LinkedHashMap<String, String> contentRegexes = new LinkedHashMap<String,String>();
contentRegexes.put("jsonBlob", "\\{(\"profile_id\":\\d+,[^}]+)\\}");
contentRegexes.put("ajaxpipeToken", "\"ajaxpipe_token\":\"([^\"]+)\"");
contentRegexes.put("timeCutoff", "\"setTimeCutoff\",[^,]*,\\[(\\d+)\\]\\]");
extractor.setContentRegexes(contentRegexes);
extractor.setTemplate("/ajax/pagelet/generic.php/ProfileTimelineSectionPagelet"
+ "?ajaxpipe=1&ajaxpipe_token=${ajaxpipeToken[1]}&no_script_path=1"
+ "&data=${java.net.URLEncoder.encode('{' + jsonBlob[1] + ',\"time_cutoff\":' + timeCutoff[1] + ',\"force_no_friend_activity\":false}', 'UTF-8')}"
+ "&__user=0&__a=1&__adt=${jsonBlobIndex+1}");
return extractor;
}
@Override
@@ -121,5 +147,4 @@ public class ExtractorMultipleRegexTest extends StringExtractorTestBase {
return result;
}
}