mirror of
https://github.com/internetarchive/heritrix3.git
synced 2026-09-26 07:35:39 +00:00
Added a timeout to crawlertrap regex matching
This commit is contained in:
Colin Rosenthal
Vendored
+85
@@ -0,0 +1,85 @@
|
||||
<?xml version="1.0" encoding="UTF-8"?>
|
||||
<module org.jetbrains.idea.maven.project.MavenProjectsManager.isMavenModule="true" version="4">
|
||||
<component name="NewModuleRootManager" LANGUAGE_LEVEL="JDK_1_8">
|
||||
<output url="file://$MODULE_DIR$/target/classes" />
|
||||
<output-test url="file://$MODULE_DIR$/target/test-classes" />
|
||||
<content url="file://$MODULE_DIR$">
|
||||
<sourceFolder url="file://$MODULE_DIR$/src/test/java" isTestSource="true" />
|
||||
<sourceFolder url="file://$MODULE_DIR$/src/test/resources" type="java-test-resource" />
|
||||
<excludeFolder url="file://$MODULE_DIR$/target" />
|
||||
</content>
|
||||
<orderEntry type="inheritedJdk" />
|
||||
<orderEntry type="sourceFolder" forTests="false" />
|
||||
<orderEntry type="module" module-name="heritrix-engine" />
|
||||
<orderEntry type="module" module-name="heritrix-modules" />
|
||||
<orderEntry type="module" module-name="heritrix-commons" />
|
||||
<orderEntry type="library" name="Maven: org.apache.httpcomponents:httpclient:4.3.6" level="project" />
|
||||
<orderEntry type="library" name="Maven: org.apache.httpcomponents:httpcore:4.3.3" level="project" />
|
||||
<orderEntry type="library" name="Maven: org.apache.httpcomponents:httpmime:4.3.6" level="project" />
|
||||
<orderEntry type="library" name="Maven: com.sleepycat:je:4.1.6" level="project" />
|
||||
<orderEntry type="library" name="Maven: commons-lang:commons-lang:2.6" level="project" />
|
||||
<orderEntry type="library" name="Maven: commons-logging:commons-logging:1.0.4" level="project" />
|
||||
<orderEntry type="library" name="Maven: commons-net:commons-net:2.0" level="project" />
|
||||
<orderEntry type="library" name="Maven: commons-io:commons-io:1.4" level="project" />
|
||||
<orderEntry type="library" name="Maven: commons-codec:commons-codec:1.6" level="project" />
|
||||
<orderEntry type="library" name="Maven: commons-collections:commons-collections:3.1" level="project" />
|
||||
<orderEntry type="library" name="Maven: commons-cli:commons-cli:1.1" level="project" />
|
||||
<orderEntry type="library" name="Maven: net.htmlparser.jericho:jericho-html:2.6.1" level="project" />
|
||||
<orderEntry type="library" name="Maven: org.dnsjava:dnsjava:2.0.3" level="project" />
|
||||
<orderEntry type="library" name="Maven: poi:poi:2.5.1-final-20040804" level="project" />
|
||||
<orderEntry type="library" name="Maven: poi:poi-scratchpad:2.5.1-final-20040804" level="project" />
|
||||
<orderEntry type="library" name="Maven: com.lowagie:itext:1.3" level="project" />
|
||||
<orderEntry type="library" name="Maven: junit:junit:3.8.2" level="project" />
|
||||
<orderEntry type="library" name="Maven: it.unimi.dsi:mg4j:1.0.1" level="project" />
|
||||
<orderEntry type="library" name="Maven: com.anotherbigidea:javaswf:CVS-SNAPSHOT-1" level="project" />
|
||||
<orderEntry type="library" name="Maven: org.springframework:spring-core:3.0.5.RELEASE" level="project" />
|
||||
<orderEntry type="library" name="Maven: org.springframework:spring-beans:3.0.5.RELEASE" level="project" />
|
||||
<orderEntry type="library" name="Maven: org.springframework:spring-context:3.0.5.RELEASE" level="project" />
|
||||
<orderEntry type="library" name="Maven: org.springframework:spring-aop:3.0.5.RELEASE" level="project" />
|
||||
<orderEntry type="library" name="Maven: aopalliance:aopalliance:1.0" level="project" />
|
||||
<orderEntry type="library" name="Maven: org.springframework:spring-asm:3.0.5.RELEASE" level="project" />
|
||||
<orderEntry type="library" name="Maven: org.springframework:spring-expression:3.0.5.RELEASE" level="project" />
|
||||
<orderEntry type="library" name="Maven: com.esotericsoftware:kryo:1.01" level="project" />
|
||||
<orderEntry type="library" scope="RUNTIME" name="Maven: com.esotericsoftware:reflectasm:0.8" level="project" />
|
||||
<orderEntry type="library" scope="RUNTIME" name="Maven: com.esotericsoftware:minlog:1.2" level="project" />
|
||||
<orderEntry type="library" name="Maven: net.java.dev.jna:jna:3.2.3" level="project" />
|
||||
<orderEntry type="library" name="Maven: org.netpreserve.commons:webarchive-commons:1.1.8" level="project" />
|
||||
<orderEntry type="library" name="Maven: com.google.guava:guava:17.0" level="project" />
|
||||
<orderEntry type="library" name="Maven: org.json:json:20131018" level="project" />
|
||||
<orderEntry type="library" name="Maven: org.htmlparser:htmlparser:1.6" level="project" />
|
||||
<orderEntry type="library" name="Maven: com.googlecode.juniversalchardet:juniversalchardet:1.0.3" level="project" />
|
||||
<orderEntry type="library" name="Maven: commons-httpclient:commons-httpclient:3.1" level="project" />
|
||||
<orderEntry type="library" name="Maven: org.gnu.inet:libidn:1.15" level="project" />
|
||||
<orderEntry type="library" name="Maven: it.unimi.dsi:dsiutils:2.0.12" level="project" />
|
||||
<orderEntry type="library" name="Maven: it.unimi.dsi:fastutil:6.5.2" level="project" />
|
||||
<orderEntry type="library" name="Maven: com.martiansoftware:jsap:2.1" level="project" />
|
||||
<orderEntry type="library" name="Maven: org.slf4j:slf4j-api:1.7.2" level="project" />
|
||||
<orderEntry type="library" name="Maven: log4j:log4j:1.2.17" level="project" />
|
||||
<orderEntry type="library" name="Maven: commons-configuration:commons-configuration:1.8" level="project" />
|
||||
<orderEntry type="library" name="Maven: org.apache.commons:commons-math3:3.1.1" level="project" />
|
||||
<orderEntry type="library" name="Maven: org.beanshell:bsh:2.0b5" level="project" />
|
||||
<orderEntry type="library" name="Maven: org.codehaus.groovy:groovy-all:1.6.3" level="project" />
|
||||
<orderEntry type="library" name="Maven: org.apache.ant:ant:1.7.1" level="project" />
|
||||
<orderEntry type="library" name="Maven: org.apache.ant:ant-launcher:1.7.1" level="project" />
|
||||
<orderEntry type="library" name="Maven: jline:jline:0.9.94" level="project" />
|
||||
<orderEntry type="library" name="Maven: org.eclipse.jetty:jetty-security:9.4.19.v20190610" level="project" />
|
||||
<orderEntry type="library" name="Maven: org.eclipse.jetty:jetty-server:9.4.19.v20190610" level="project" />
|
||||
<orderEntry type="library" name="Maven: javax.servlet:javax.servlet-api:3.1.0" level="project" />
|
||||
<orderEntry type="library" name="Maven: org.eclipse.jetty:jetty-http:9.4.19.v20190610" level="project" />
|
||||
<orderEntry type="library" name="Maven: org.eclipse.jetty:jetty-io:9.4.19.v20190610" level="project" />
|
||||
<orderEntry type="library" name="Maven: org.eclipse.jetty:jetty-util:9.4.19.v20190610" level="project" />
|
||||
<orderEntry type="library" name="Maven: org.restlet.jse:org.restlet:2.4.0" level="project" />
|
||||
<orderEntry type="library" name="Maven: org.restlet.jse:org.restlet.ext.jetty:2.4.0" level="project" />
|
||||
<orderEntry type="library" name="Maven: org.restlet.jse:org.restlet.ext.xml:2.4.0" level="project" />
|
||||
<orderEntry type="library" name="Maven: org.restlet.jse:org.restlet.ext.crypto:2.4.0" level="project" />
|
||||
<orderEntry type="library" name="Maven: joda-time:joda-time:1.6" level="project" />
|
||||
<orderEntry type="library" name="Maven: javax.xml.bind:jaxb-api:2.3.1" level="project" />
|
||||
<orderEntry type="library" name="Maven: javax.activation:javax.activation-api:1.2.0" level="project" />
|
||||
<orderEntry type="library" scope="RUNTIME" name="Maven: org.glassfish.jaxb:jaxb-runtime:2.3.1" level="project" />
|
||||
<orderEntry type="library" scope="RUNTIME" name="Maven: org.glassfish.jaxb:txw2:2.3.1" level="project" />
|
||||
<orderEntry type="library" scope="RUNTIME" name="Maven: com.sun.istack:istack-commons-runtime:3.0.7" level="project" />
|
||||
<orderEntry type="library" scope="RUNTIME" name="Maven: org.jvnet.staxex:stax-ex:1.8" level="project" />
|
||||
<orderEntry type="library" scope="RUNTIME" name="Maven: com.sun.xml.fastinfoset:FastInfoset:1.2.15" level="project" />
|
||||
<orderEntry type="library" name="Maven: org.freemarker:freemarker:2.3.19" level="project" />
|
||||
</component>
|
||||
</module>
|
||||
+28
-2
@@ -20,6 +20,10 @@ package org.archive.modules.deciderules;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
import java.util.concurrent.CompletableFuture;
|
||||
import java.util.concurrent.ExecutionException;
|
||||
import java.util.concurrent.TimeUnit;
|
||||
import java.util.concurrent.TimeoutException;
|
||||
import java.util.logging.Level;
|
||||
import java.util.logging.Logger;
|
||||
import java.util.regex.Pattern;
|
||||
@@ -39,7 +43,19 @@ import org.archive.modules.CrawlURI;
|
||||
public class MatchesListRegexDecideRule extends PredicatedDecideRule {
|
||||
private static final long serialVersionUID = 3L;
|
||||
private static final Logger logger =
|
||||
Logger.getLogger(MatchesListRegexDecideRule.class.getName());
|
||||
Logger.getLogger(MatchesListRegexDecideRule.class.getName());
|
||||
|
||||
/**
|
||||
* The timeout for regular expression matching, in seconds. If set to 0 or negative then no timeout is specified and
|
||||
* there is no upper limit to how long the matching may take. See the corresponding test class MatchesListRegexDecideRuleTest
|
||||
* for a pathological example.
|
||||
*/
|
||||
{
|
||||
setTimeoutPerRegexSeconds(0L);
|
||||
}
|
||||
public long getTimeoutPerRegexSeconds() { return (Long) kp.get("timeout");}
|
||||
public void setTimeoutPerRegexSeconds(long timeoutPerRegexSeconds) { kp.put("timeout", timeoutPerRegexSeconds);}
|
||||
|
||||
|
||||
/**
|
||||
* The list of regular expressions to evalute against the URI.
|
||||
@@ -91,7 +107,17 @@ public class MatchesListRegexDecideRule extends PredicatedDecideRule {
|
||||
boolean listLogicOR = getListLogicalOr();
|
||||
|
||||
for (Pattern p: regexes) {
|
||||
boolean matches = p.matcher(str).matches();
|
||||
boolean matches = false;
|
||||
if (getTimeoutPerRegexSeconds() <= 0) {
|
||||
matches = p.matcher(str).matches();
|
||||
} else {
|
||||
CompletableFuture<Boolean> matchesFuture = CompletableFuture.supplyAsync(() -> p.matcher(str).matches());
|
||||
try {
|
||||
matches = matchesFuture.get(getTimeoutPerRegexSeconds(), TimeUnit.SECONDS);
|
||||
} catch (Exception e) {
|
||||
logger.info("Exception while matching regex '" + p + "' to url '" + str + "' so assuming no match. " + e.getClass().getName());
|
||||
}
|
||||
}
|
||||
|
||||
if (logger.isLoggable(Level.FINER)) {
|
||||
logger.finer("Tested '" + str + "' match with regex '" +
|
||||
|
||||
+34
@@ -0,0 +1,34 @@
|
||||
package org.archive.modules.deciderules;
|
||||
|
||||
import junit.framework.TestCase;
|
||||
import org.apache.commons.httpclient.URIException;
|
||||
import org.archive.modules.CrawlURI;
|
||||
import org.archive.net.UURIFactory;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
import java.util.regex.Pattern;
|
||||
|
||||
public class MatchesListRegexDecideRuleTest extends TestCase {
|
||||
|
||||
/**
|
||||
* Not easy to test this code in older versions of junit. Basically with the timeout set to "0", this method
|
||||
* will never return.
|
||||
* @throws URIException
|
||||
*/
|
||||
public void xtestEvaluate() throws URIException {
|
||||
final String regex = "http://www\\.netarkivet\\.dk/((x+x+)+)y";
|
||||
String seed = "http://www.netarkivet.dk/xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx";
|
||||
MatchesListRegexDecideRule rule = new MatchesListRegexDecideRule();
|
||||
List<Pattern> patternList = new ArrayList<>();
|
||||
patternList.add(Pattern.compile(regex));
|
||||
rule.setRegexList(patternList);
|
||||
rule.setEnabled(true);
|
||||
rule.setListLogicalOr(true);
|
||||
rule.setDecision(DecideResult.REJECT);
|
||||
rule.setTimeoutPerRegexSeconds(2);
|
||||
final CrawlURI curi = new CrawlURI(UURIFactory.getInstance(seed));
|
||||
final DecideResult decideResult = rule.decisionFor(curi);
|
||||
assertEquals("Expected NONE not " + decideResult , DecideResult.NONE, decideResult);
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user