mirror of
https://github.com/internetarchive/heritrix3.git
synced 2026-09-24 06:36:12 +00:00
tmp commit
This commit is contained in:
Travis Wellman
+1
-1
@@ -38,7 +38,7 @@
|
||||
<classpathentry kind="var" path="M2_REPO/org/restlet/org.restlet/1.1.10/org.restlet-1.1.10.jar" sourcepath="/M2_REPO/org/restlet/org.restlet/1.1.10/org.restlet-1.1.10-sources.jar"/>
|
||||
<classpathentry kind="var" path="M2_REPO/com/noelios/restlet/com.noelios.restlet/1.1.10/com.noelios.restlet-1.1.10.jar" sourcepath="/M2_REPO/com/noelios/restlet/com.noelios.restlet/1.1.10/com.noelios.restlet-1.1.10-sources.jar"/>
|
||||
<classpathentry kind="var" path="M2_REPO/com/noelios/restlet/com.noelios.restlet.ext.jetty/1.1.10/com.noelios.restlet.ext.jetty-1.1.10.jar"/>
|
||||
<classpathentry kind="var" path="M2_REPO/org/codehaus/groovy/groovy-all/1.6.3/groovy-all-1.6.3.jar"/>
|
||||
<classpathentry kind="var" path="M2_REPO/org/codehaus/groovy/groovy-all/1.6.3/groovy-all-1.6.3.jar" sourcepath="/M2_REPO/org/codehaus/groovy/groovy-all/1.6.3/groovy-all-1.6.3-sources.jar"/>
|
||||
<classpathentry kind="lib" path="engine/src/main/resources"/>
|
||||
<classpathentry kind="lib" path="modules/src/main/resources"/>
|
||||
<classpathentry kind="lib" path="commons/src/main/resources"/>
|
||||
|
||||
@@ -0,0 +1,182 @@
|
||||
/*
|
||||
* This file is part of the Heritrix web crawler (crawler.archive.org).
|
||||
*
|
||||
* Licensed to the Internet Archive (IA) by one or more individual
|
||||
* contributors.
|
||||
*
|
||||
* The IA licenses this file to You under the Apache License, Version 2.0
|
||||
* (the "License"); you may not use this file except in compliance with
|
||||
* the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
package org.archive.modules.extractor;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.util.HashMap;
|
||||
import java.util.LinkedList;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.Set;
|
||||
import java.util.SortedMap;
|
||||
import java.util.TreeMap;
|
||||
import java.util.logging.Level;
|
||||
import java.util.logging.Logger;
|
||||
import java.util.regex.Matcher;
|
||||
|
||||
import javax.script.Bindings;
|
||||
import javax.script.ScriptException;
|
||||
import javax.script.SimpleBindings;
|
||||
|
||||
import org.apache.commons.httpclient.URIException;
|
||||
import org.apache.commons.lang.StringEscapeUtils;
|
||||
import org.archive.io.ReplayCharSequence;
|
||||
import org.archive.modules.CrawlURI;
|
||||
import org.archive.modules.fetcher.FetchStatusCodes;
|
||||
import org.archive.util.TextUtils;
|
||||
import org.codehaus.groovy.jsr223.GroovyScriptEngineImpl;
|
||||
|
||||
public class ExtractorMultipleRegex extends Extractor {
|
||||
|
||||
private static Logger LOGGER =
|
||||
Logger.getLogger(ExtractorMultipleRegex.class.getName());
|
||||
|
||||
{
|
||||
setContentRegexes(new HashMap<String,String>());
|
||||
}
|
||||
public void setContentRegexes(Map<String, String> regexes) {
|
||||
kp.put("contentRegexes", regexes);
|
||||
}
|
||||
@SuppressWarnings("unchecked")
|
||||
public Map<String, String> getContentRegexes() {
|
||||
return (Map<String, String>) kp.get("contentRegexes");
|
||||
}
|
||||
|
||||
{
|
||||
setTemplate("");
|
||||
}
|
||||
public String getTemplate() {
|
||||
return (String) kp.get("template");
|
||||
}
|
||||
public void setTemplate(String templ) {
|
||||
kp.put("template", templ);
|
||||
}
|
||||
|
||||
|
||||
{
|
||||
setUriRegex("");
|
||||
}
|
||||
public void setUriRegex(String reg) {
|
||||
kp.put("uriRegex", reg);
|
||||
}
|
||||
public String getUriRegex() {
|
||||
return (String) kp.get("uriRegex");
|
||||
}
|
||||
|
||||
|
||||
@Override
|
||||
protected boolean shouldProcess(CrawlURI uri) {
|
||||
if (uri.getContentLength() <= 0) {
|
||||
return false;
|
||||
}
|
||||
if (!getExtractorParameters().getExtract404s()
|
||||
&& uri.getFetchStatus()==FetchStatusCodes.S_NOT_FOUND) {
|
||||
return false;
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
@Override
|
||||
public void extract(CrawlURI curi) {
|
||||
|
||||
Matcher m = TextUtils.getMatcher(getUriRegex(), curi.getURI());
|
||||
if (!m.matches()) {
|
||||
return;
|
||||
}
|
||||
String[] uriRegexGroups = new String[m.groupCount() +1];
|
||||
for(int i = 0; i < uriRegexGroups.length; i++) {
|
||||
uriRegexGroups[i] = m.group(i);
|
||||
}
|
||||
// our data structure to prepopulate with matches for nested iteration
|
||||
SortedMap<String,List<String[]>> allMatches = new TreeMap<String, List<String[]>>();
|
||||
LinkedList<String[]> uriRegexMatchList = new LinkedList<String[]>();
|
||||
uriRegexMatchList.add(uriRegexGroups);
|
||||
allMatches.put("uriRegex", uriRegexMatchList);
|
||||
|
||||
ReplayCharSequence cs;
|
||||
try {
|
||||
cs = curi.getRecorder().getContentReplayCharSequence();
|
||||
} catch (IOException e) {
|
||||
curi.getNonFatalFailures().add(e);
|
||||
LOGGER.log(Level.WARNING,"Failed get of replay char sequence in " +
|
||||
Thread.currentThread().getName(), e);
|
||||
return;
|
||||
}
|
||||
// the names for regexes given in the config
|
||||
Set<String> names = getContentRegexes().keySet();
|
||||
for (String patternName : names) {
|
||||
// the matcher for this patternName against the content
|
||||
Matcher namedMatcher = TextUtils.getMatcher(getContentRegexes().get(patternName), cs);
|
||||
// populate the list of finds for this patternName
|
||||
List<String[]> foundList = new LinkedList<String[]>();
|
||||
while(namedMatcher.find()) {
|
||||
// +1 to include the full match in addition to the groups
|
||||
String[] groups = new String[namedMatcher.groupCount() +1];
|
||||
for(int i = 0; i < groups.length; i++) {
|
||||
groups[i] = namedMatcher.group(i);
|
||||
}
|
||||
foundList.add(groups);
|
||||
}
|
||||
allMatches.put(patternName, foundList);
|
||||
}
|
||||
|
||||
long i = 0;
|
||||
boolean done = false;
|
||||
while (!done) {
|
||||
long tmp = i;
|
||||
SimpleBindings matches = new SimpleBindings();
|
||||
matches.put("index", i);
|
||||
String[] patternNames = allMatches.keySet().toArray(new String[0]);
|
||||
for (int j = 0; j < patternNames.length; j++) {
|
||||
List<String[]> matchList = allMatches.get(patternNames[j]);
|
||||
if (j == patternNames.length - 1 && tmp >= matchList.size()) {
|
||||
done = true;
|
||||
break;
|
||||
}
|
||||
matches.put(patternNames[j], matchList.get((int) (tmp % matchList.size())));
|
||||
tmp = tmp / matchList.size();
|
||||
}
|
||||
|
||||
if (!done) {
|
||||
addOutlink(curi, matches);
|
||||
}
|
||||
|
||||
i++;
|
||||
}
|
||||
}
|
||||
|
||||
protected void addOutlink(CrawlURI curi, Bindings matches) {
|
||||
GroovyScriptEngineImpl gse = new GroovyScriptEngineImpl();
|
||||
String stringUri = null;
|
||||
try {
|
||||
stringUri = (String) gse.eval("\""+ StringEscapeUtils.escapeJava(getTemplate()) +"\"", matches);
|
||||
} catch (ScriptException e) {
|
||||
logUriError(new URIException(e.toString()), curi.getUURI(), stringUri);
|
||||
return;
|
||||
}
|
||||
try {
|
||||
int max = getExtractorParameters().getMaxOutlinks();
|
||||
Link.addRelativeToBase(curi, max, stringUri,
|
||||
HTMLLinkContext.INFERRED_MISC, Hop.INFERRED);
|
||||
} catch (URIException e) {
|
||||
logUriError(e, curi.getUURI(), stringUri);
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
@@ -0,0 +1,125 @@
|
||||
/*
|
||||
* This file is part of the Heritrix web crawler (crawler.archive.org).
|
||||
*
|
||||
* Licensed to the Internet Archive (IA) by one or more individual
|
||||
* contributors.
|
||||
*
|
||||
* The IA licenses this file to You under the Apache License, Version 2.0
|
||||
* (the "License"); you may not use this file except in compliance with
|
||||
* the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
package org.archive.modules.extractor;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.Collection;
|
||||
import java.util.List;
|
||||
|
||||
import org.archive.modules.CrawlURI;
|
||||
import org.archive.net.UURI;
|
||||
import org.archive.net.UURIFactory;
|
||||
import org.archive.util.Recorder;
|
||||
|
||||
public class ExtractorMultipleRegexTest extends StringExtractorTestBase {
|
||||
final public static String[] VALID_TEST_DATA = new String[] {
|
||||
// "<a href=\"http://www.slashdot.org\">yellow journalism</a> A",
|
||||
// "http://www.slashdot.org",
|
||||
//
|
||||
// "<img src=\"foo.gif\"> IMG",
|
||||
// "http://www.archive.org/start/foo.gif",
|
||||
|
||||
// https://www.facebook.com/NorthCarolinaStateParks some time in the past
|
||||
"{\"profile_id\":143412869029,\"start\":1351753200,\"end" +
|
||||
"\":1354348799,\"query_type\":31,\"section_pagelet_id\":\"" +
|
||||
"pagelet_timeline_earlier_this_month_all\",\"load_immediately\"" +
|
||||
":false},false,null,1,-1]],[\"TimelineContentLoader\",\"load" +
|
||||
"SectionOnClick\",[\"m959614_181\"],[{\"__m\":\"m959614_181\"}" +
|
||||
",\"month_2012_10\"]],[\"TimelineContentLoader\",\"registerTim" +
|
||||
"ePeriod\",[\"m959614_182\"],[{\"__m\":\"m959614_182\"},\"month_20" +
|
||||
"12_10\",{\"profile_id\":143412869029,\"start\":1349074800,\"end\":13517" +
|
||||
"53199,\"query_type\":25,\"section_pagelet_id\":\"pagelet_timeline_month_al" +
|
||||
"l_last\",\"load_immediately\":false},false,null,2,-1]],[\"TimelineContent" +
|
||||
"Loader\",\"loadSectionOnClick\",[\"m959614_183\"],[{\"__m\":\"m959614" +
|
||||
"_183\"},\"year_2012\"]],[\"TimelineContentLoader\",\"registerTimePeri" +
|
||||
"od\",[\"m959614_184\"],[{\"__m\":\"m959614_184\"},\"year_2012\",{\"profile_" +
|
||||
"id\":143412869029,\"start\":1325404800,\"end\":1357027199,\"query_type\":8" +
|
||||
",\"filter_after_timestamp\":1349074799,\"section_pagelet_id\":\"pagelet_" +
|
||||
"timeline_year_current\",\"load_immediately\":false},false,null,3,\"101.67" +
|
||||
"277588916\"]],\n [\"TimelineContentLoader\",\"setExpandLoadDataForSection\",[],[\"y" +
|
||||
"ear_2012\",{\"profile_id\":143412869029,\"start\":1325404800,\"end\":135" +
|
||||
"7027199,\"query_type\":9}]],[\"TimelineContentLoader\",\"loadSectionOnClick" +
|
||||
"\",[\"m959614_185\"],[{\"__m\":\"m959614_185\"},\"year_2011\"]],[\"TimelineCo" +
|
||||
"ntentLoader\",\"registerTimePeriod\",[\"m959614_186\"],[{\"__m\":\"m959614_186\"}" +
|
||||
",\"year_2011\",{\"profile_id\":143412869029,\"start\":1293868800,\"end\":13254" +
|
||||
"04799,\"query_type\":8,\"section_pagelet_id\":\"pagelet_timeline_year_last\",\"l" +
|
||||
"oad_immediately\":false},false,null,4,\"86.97249252724\"]],[\"TimelineContentLoad" +
|
||||
"er\",\"setExpandLoadDataForSection\",[],[\"year_2011\",{\"profile_id\":143412869029" +
|
||||
",\"start\":1293868800,\"end\":1325404799,\"query_type\":9}]],[\"TimelineContentLoad" +
|
||||
"er\",\"loadSectionOnClick\",[\"m959614_187\"],[{\"__m\":\"m959614_187\"},\"year_20" +
|
||||
"10\"]],[\"TimelineContentLoader\",\"registerTimePeriod\",[\"m959614_188\"],[{\"_" +
|
||||
"_m\":\"m959614_188\"},\"year_2010\",{\"profile_id\":143412869029,\"start\":126233" +
|
||||
"2800,\"end\":1293868799,\"query_type\":8,\"section_pagelet_id\":\"pagelet\n _timel" +
|
||||
"ine_year_2010\",\"load_immediately\":false},false,null,5,\"88.440195233432\"]],[\"Ti" +
|
||||
"melineContentLoader\",\"setExpandLoadDataForSection\",[],[\"year_2010\",{\"profile_i" +
|
||||
"d\":143412869029,\"start\":1262332800,\"end\":1293868799,\"query_type\":9}]],[\"Time" +
|
||||
"lineContentLoader\",\"loadSectionOnClick\",[\"m959614_189\"],[{\"__m\":\"m959614_189" +
|
||||
"\"},\"year_2009\"]],[\"TimelineContentLoader\",\"registerTimePeriod\",[\"m959614_190" +
|
||||
"\"],[{\"__m\":\"m959614_190\"},\"year_2009\",{\"profile_id\":143412869029,\"start\":" +
|
||||
"1230796800,\"end\":1262332799,\"query_type\":8,\"section_pagelet_id\":\"pagelet_time" +
|
||||
"line_year_2009\",\"load_immediately\":false},false,null,6,\"41.800676041109\"]],[\"T" +
|
||||
"imelineContentLoader\",\"setExpandLoadDataForSection\",[],[\"year_2009\",{\"profile_" +
|
||||
"id\":143412869029,\"start\":1230796800,\"end\":1262332799,\"query_type\":9}]]]},\"cs" +
|
||||
"s\":[\"M501h\",\"o8oNt\"],\"js\":[\"dERRF\",\"yEdv1\",\"JWWMg\"],\"id\":\"timeline_s" +
|
||||
"ection_placeholders\",\"phase\":3})</script>\n",
|
||||
"http://nourl.com/dne",
|
||||
};
|
||||
|
||||
|
||||
@Override
|
||||
protected String[] getValidTestData() {
|
||||
return VALID_TEST_DATA;
|
||||
}
|
||||
|
||||
@Override
|
||||
protected Extractor makeExtractor() {
|
||||
ExtractorMultipleRegex result = new ExtractorMultipleRegex();
|
||||
UriErrorLoggerModule ulm = new UnitTestUriLoggerModule();
|
||||
result.setLoggerModule(ulm);
|
||||
return result;
|
||||
}
|
||||
|
||||
@Override
|
||||
protected Collection<TestData> makeData(String content, String destURI)
|
||||
throws Exception {
|
||||
List<TestData> result = new ArrayList<TestData>();
|
||||
UURI src = UURIFactory.getInstance("https://www.facebook.com/NorthCarolinaStateParks");
|
||||
CrawlURI euri = new CrawlURI(src, null, null,
|
||||
LinkContext.NAVLINK_MISC);
|
||||
Recorder recorder = createRecorder(content);
|
||||
euri.setContentType("text/html");
|
||||
euri.setRecorder(recorder);
|
||||
euri.setContentSize(content.length());
|
||||
|
||||
UURI dest = UURIFactory.getInstance(destURI);
|
||||
Link link = new Link(src, dest, HTMLLinkContext.INFERRED_MISC, Hop.INFERRED);
|
||||
result.add(new TestData(euri, link));
|
||||
|
||||
euri = new CrawlURI(src, null, null, LinkContext.NAVLINK_MISC);
|
||||
recorder = createRecorder(content);
|
||||
euri.setContentType("application/xhtml");
|
||||
euri.setRecorder(recorder);
|
||||
euri.setContentSize(content.length());
|
||||
result.add(new TestData(euri, link));
|
||||
|
||||
return result;
|
||||
}
|
||||
|
||||
|
||||
}
|
||||
Reference in New Issue
Block a user