diff --git a/.classpath b/.classpath index 2bab47d4..920788cd 100644 --- a/.classpath +++ b/.classpath @@ -38,7 +38,7 @@ - + diff --git a/modules/src/main/java/org/archive/modules/extractor/ExtractorMultipleRegex.java b/modules/src/main/java/org/archive/modules/extractor/ExtractorMultipleRegex.java new file mode 100644 index 00000000..a8a734a2 --- /dev/null +++ b/modules/src/main/java/org/archive/modules/extractor/ExtractorMultipleRegex.java @@ -0,0 +1,182 @@ +/* + * This file is part of the Heritrix web crawler (crawler.archive.org). + * + * Licensed to the Internet Archive (IA) by one or more individual + * contributors. + * + * The IA licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.archive.modules.extractor; + +import java.io.IOException; +import java.util.HashMap; +import java.util.LinkedList; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.SortedMap; +import java.util.TreeMap; +import java.util.logging.Level; +import java.util.logging.Logger; +import java.util.regex.Matcher; + +import javax.script.Bindings; +import javax.script.ScriptException; +import javax.script.SimpleBindings; + +import org.apache.commons.httpclient.URIException; +import org.apache.commons.lang.StringEscapeUtils; +import org.archive.io.ReplayCharSequence; +import org.archive.modules.CrawlURI; +import org.archive.modules.fetcher.FetchStatusCodes; +import org.archive.util.TextUtils; +import org.codehaus.groovy.jsr223.GroovyScriptEngineImpl; + +public class ExtractorMultipleRegex extends Extractor { + + private static Logger LOGGER = + Logger.getLogger(ExtractorMultipleRegex.class.getName()); + + { + setContentRegexes(new HashMap()); + } + public void setContentRegexes(Map regexes) { + kp.put("contentRegexes", regexes); + } + @SuppressWarnings("unchecked") + public Map getContentRegexes() { + return (Map) kp.get("contentRegexes"); + } + + { + setTemplate(""); + } + public String getTemplate() { + return (String) kp.get("template"); + } + public void setTemplate(String templ) { + kp.put("template", templ); + } + + + { + setUriRegex(""); + } + public void setUriRegex(String reg) { + kp.put("uriRegex", reg); + } + public String getUriRegex() { + return (String) kp.get("uriRegex"); + } + + + @Override + protected boolean shouldProcess(CrawlURI uri) { + if (uri.getContentLength() <= 0) { + return false; + } + if (!getExtractorParameters().getExtract404s() + && uri.getFetchStatus()==FetchStatusCodes.S_NOT_FOUND) { + return false; + } + return true; + } + + @Override + public void extract(CrawlURI curi) { + + Matcher m = TextUtils.getMatcher(getUriRegex(), curi.getURI()); + if (!m.matches()) { + return; + } + String[] uriRegexGroups = new String[m.groupCount() +1]; + for(int i = 0; i < uriRegexGroups.length; i++) { + uriRegexGroups[i] = m.group(i); + } + // our data structure to prepopulate with matches for nested iteration + SortedMap> allMatches = new TreeMap>(); + LinkedList uriRegexMatchList = new LinkedList(); + uriRegexMatchList.add(uriRegexGroups); + allMatches.put("uriRegex", uriRegexMatchList); + + ReplayCharSequence cs; + try { + cs = curi.getRecorder().getContentReplayCharSequence(); + } catch (IOException e) { + curi.getNonFatalFailures().add(e); + LOGGER.log(Level.WARNING,"Failed get of replay char sequence in " + + Thread.currentThread().getName(), e); + return; + } + // the names for regexes given in the config + Set names = getContentRegexes().keySet(); + for (String patternName : names) { + // the matcher for this patternName against the content + Matcher namedMatcher = TextUtils.getMatcher(getContentRegexes().get(patternName), cs); + // populate the list of finds for this patternName + List foundList = new LinkedList(); + while(namedMatcher.find()) { + // +1 to include the full match in addition to the groups + String[] groups = new String[namedMatcher.groupCount() +1]; + for(int i = 0; i < groups.length; i++) { + groups[i] = namedMatcher.group(i); + } + foundList.add(groups); + } + allMatches.put(patternName, foundList); + } + + long i = 0; + boolean done = false; + while (!done) { + long tmp = i; + SimpleBindings matches = new SimpleBindings(); + matches.put("index", i); + String[] patternNames = allMatches.keySet().toArray(new String[0]); + for (int j = 0; j < patternNames.length; j++) { + List matchList = allMatches.get(patternNames[j]); + if (j == patternNames.length - 1 && tmp >= matchList.size()) { + done = true; + break; + } + matches.put(patternNames[j], matchList.get((int) (tmp % matchList.size()))); + tmp = tmp / matchList.size(); + } + + if (!done) { + addOutlink(curi, matches); + } + + i++; + } + } + + protected void addOutlink(CrawlURI curi, Bindings matches) { + GroovyScriptEngineImpl gse = new GroovyScriptEngineImpl(); + String stringUri = null; + try { + stringUri = (String) gse.eval("\""+ StringEscapeUtils.escapeJava(getTemplate()) +"\"", matches); + } catch (ScriptException e) { + logUriError(new URIException(e.toString()), curi.getUURI(), stringUri); + return; + } + try { + int max = getExtractorParameters().getMaxOutlinks(); + Link.addRelativeToBase(curi, max, stringUri, + HTMLLinkContext.INFERRED_MISC, Hop.INFERRED); + } catch (URIException e) { + logUriError(e, curi.getUURI(), stringUri); + } + } + +} diff --git a/modules/src/test/java/org/archive/modules/extractor/ExtractorMultipleRegexTest.java b/modules/src/test/java/org/archive/modules/extractor/ExtractorMultipleRegexTest.java new file mode 100644 index 00000000..cdae5bfc --- /dev/null +++ b/modules/src/test/java/org/archive/modules/extractor/ExtractorMultipleRegexTest.java @@ -0,0 +1,125 @@ +/* + * This file is part of the Heritrix web crawler (crawler.archive.org). + * + * Licensed to the Internet Archive (IA) by one or more individual + * contributors. + * + * The IA licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.archive.modules.extractor; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.List; + +import org.archive.modules.CrawlURI; +import org.archive.net.UURI; +import org.archive.net.UURIFactory; +import org.archive.util.Recorder; + +public class ExtractorMultipleRegexTest extends StringExtractorTestBase { + final public static String[] VALID_TEST_DATA = new String[] { +// "yellow journalism A", +// "http://www.slashdot.org", +// +// " IMG", +// "http://www.archive.org/start/foo.gif", + + // https://www.facebook.com/NorthCarolinaStateParks some time in the past + "{\"profile_id\":143412869029,\"start\":1351753200,\"end" + + "\":1354348799,\"query_type\":31,\"section_pagelet_id\":\"" + + "pagelet_timeline_earlier_this_month_all\",\"load_immediately\"" + + ":false},false,null,1,-1]],[\"TimelineContentLoader\",\"load" + + "SectionOnClick\",[\"m959614_181\"],[{\"__m\":\"m959614_181\"}" + + ",\"month_2012_10\"]],[\"TimelineContentLoader\",\"registerTim" + + "ePeriod\",[\"m959614_182\"],[{\"__m\":\"m959614_182\"},\"month_20" + + "12_10\",{\"profile_id\":143412869029,\"start\":1349074800,\"end\":13517" + + "53199,\"query_type\":25,\"section_pagelet_id\":\"pagelet_timeline_month_al" + + "l_last\",\"load_immediately\":false},false,null,2,-1]],[\"TimelineContent" + + "Loader\",\"loadSectionOnClick\",[\"m959614_183\"],[{\"__m\":\"m959614" + + "_183\"},\"year_2012\"]],[\"TimelineContentLoader\",\"registerTimePeri" + + "od\",[\"m959614_184\"],[{\"__m\":\"m959614_184\"},\"year_2012\",{\"profile_" + + "id\":143412869029,\"start\":1325404800,\"end\":1357027199,\"query_type\":8" + + ",\"filter_after_timestamp\":1349074799,\"section_pagelet_id\":\"pagelet_" + + "timeline_year_current\",\"load_immediately\":false},false,null,3,\"101.67" + + "277588916\"]],\n [\"TimelineContentLoader\",\"setExpandLoadDataForSection\",[],[\"y" + + "ear_2012\",{\"profile_id\":143412869029,\"start\":1325404800,\"end\":135" + + "7027199,\"query_type\":9}]],[\"TimelineContentLoader\",\"loadSectionOnClick" + + "\",[\"m959614_185\"],[{\"__m\":\"m959614_185\"},\"year_2011\"]],[\"TimelineCo" + + "ntentLoader\",\"registerTimePeriod\",[\"m959614_186\"],[{\"__m\":\"m959614_186\"}" + + ",\"year_2011\",{\"profile_id\":143412869029,\"start\":1293868800,\"end\":13254" + + "04799,\"query_type\":8,\"section_pagelet_id\":\"pagelet_timeline_year_last\",\"l" + + "oad_immediately\":false},false,null,4,\"86.97249252724\"]],[\"TimelineContentLoad" + + "er\",\"setExpandLoadDataForSection\",[],[\"year_2011\",{\"profile_id\":143412869029" + + ",\"start\":1293868800,\"end\":1325404799,\"query_type\":9}]],[\"TimelineContentLoad" + + "er\",\"loadSectionOnClick\",[\"m959614_187\"],[{\"__m\":\"m959614_187\"},\"year_20" + + "10\"]],[\"TimelineContentLoader\",\"registerTimePeriod\",[\"m959614_188\"],[{\"_" + + "_m\":\"m959614_188\"},\"year_2010\",{\"profile_id\":143412869029,\"start\":126233" + + "2800,\"end\":1293868799,\"query_type\":8,\"section_pagelet_id\":\"pagelet\n _timel" + + "ine_year_2010\",\"load_immediately\":false},false,null,5,\"88.440195233432\"]],[\"Ti" + + "melineContentLoader\",\"setExpandLoadDataForSection\",[],[\"year_2010\",{\"profile_i" + + "d\":143412869029,\"start\":1262332800,\"end\":1293868799,\"query_type\":9}]],[\"Time" + + "lineContentLoader\",\"loadSectionOnClick\",[\"m959614_189\"],[{\"__m\":\"m959614_189" + + "\"},\"year_2009\"]],[\"TimelineContentLoader\",\"registerTimePeriod\",[\"m959614_190" + + "\"],[{\"__m\":\"m959614_190\"},\"year_2009\",{\"profile_id\":143412869029,\"start\":" + + "1230796800,\"end\":1262332799,\"query_type\":8,\"section_pagelet_id\":\"pagelet_time" + + "line_year_2009\",\"load_immediately\":false},false,null,6,\"41.800676041109\"]],[\"T" + + "imelineContentLoader\",\"setExpandLoadDataForSection\",[],[\"year_2009\",{\"profile_" + + "id\":143412869029,\"start\":1230796800,\"end\":1262332799,\"query_type\":9}]]]},\"cs" + + "s\":[\"M501h\",\"o8oNt\"],\"js\":[\"dERRF\",\"yEdv1\",\"JWWMg\"],\"id\":\"timeline_s" + + "ection_placeholders\",\"phase\":3})\n", + "http://nourl.com/dne", + }; + + + @Override + protected String[] getValidTestData() { + return VALID_TEST_DATA; + } + + @Override + protected Extractor makeExtractor() { + ExtractorMultipleRegex result = new ExtractorMultipleRegex(); + UriErrorLoggerModule ulm = new UnitTestUriLoggerModule(); + result.setLoggerModule(ulm); + return result; + } + + @Override + protected Collection makeData(String content, String destURI) + throws Exception { + List result = new ArrayList(); + UURI src = UURIFactory.getInstance("https://www.facebook.com/NorthCarolinaStateParks"); + CrawlURI euri = new CrawlURI(src, null, null, + LinkContext.NAVLINK_MISC); + Recorder recorder = createRecorder(content); + euri.setContentType("text/html"); + euri.setRecorder(recorder); + euri.setContentSize(content.length()); + + UURI dest = UURIFactory.getInstance(destURI); + Link link = new Link(src, dest, HTMLLinkContext.INFERRED_MISC, Hop.INFERRED); + result.add(new TestData(euri, link)); + + euri = new CrawlURI(src, null, null, LinkContext.NAVLINK_MISC); + recorder = createRecorder(content); + euri.setContentType("application/xhtml"); + euri.setRecorder(recorder); + euri.setContentSize(content.length()); + result.add(new TestData(euri, link)); + + return result; + } + + +}