diff --git a/dist/heritrix.iml b/dist/heritrix.iml
new file mode 100644
index 00000000..f1b8c386
--- /dev/null
+++ b/dist/heritrix.iml
@@ -0,0 +1,85 @@
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
\ No newline at end of file
diff --git a/modules/src/main/java/org/archive/modules/deciderules/MatchesListRegexDecideRule.java b/modules/src/main/java/org/archive/modules/deciderules/MatchesListRegexDecideRule.java
index 1344b30b..2aedcc23 100644
--- a/modules/src/main/java/org/archive/modules/deciderules/MatchesListRegexDecideRule.java
+++ b/modules/src/main/java/org/archive/modules/deciderules/MatchesListRegexDecideRule.java
@@ -20,6 +20,10 @@ package org.archive.modules.deciderules;
import java.util.ArrayList;
import java.util.List;
+import java.util.concurrent.CompletableFuture;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.TimeoutException;
import java.util.logging.Level;
import java.util.logging.Logger;
import java.util.regex.Pattern;
@@ -39,7 +43,19 @@ import org.archive.modules.CrawlURI;
public class MatchesListRegexDecideRule extends PredicatedDecideRule {
private static final long serialVersionUID = 3L;
private static final Logger logger =
- Logger.getLogger(MatchesListRegexDecideRule.class.getName());
+ Logger.getLogger(MatchesListRegexDecideRule.class.getName());
+
+ /**
+ * The timeout for regular expression matching, in seconds. If set to 0 or negative then no timeout is specified and
+ * there is no upper limit to how long the matching may take. See the corresponding test class MatchesListRegexDecideRuleTest
+ * for a pathological example.
+ */
+ {
+ setTimeoutPerRegexSeconds(0L);
+ }
+ public long getTimeoutPerRegexSeconds() { return (Long) kp.get("timeout");}
+ public void setTimeoutPerRegexSeconds(long timeoutPerRegexSeconds) { kp.put("timeout", timeoutPerRegexSeconds);}
+
/**
* The list of regular expressions to evalute against the URI.
@@ -91,7 +107,17 @@ public class MatchesListRegexDecideRule extends PredicatedDecideRule {
boolean listLogicOR = getListLogicalOr();
for (Pattern p: regexes) {
- boolean matches = p.matcher(str).matches();
+ boolean matches = false;
+ if (getTimeoutPerRegexSeconds() <= 0) {
+ matches = p.matcher(str).matches();
+ } else {
+ CompletableFuture matchesFuture = CompletableFuture.supplyAsync(() -> p.matcher(str).matches());
+ try {
+ matches = matchesFuture.get(getTimeoutPerRegexSeconds(), TimeUnit.SECONDS);
+ } catch (Exception e) {
+ logger.info("Exception while matching regex '" + p + "' to url '" + str + "' so assuming no match. " + e.getClass().getName());
+ }
+ }
if (logger.isLoggable(Level.FINER)) {
logger.finer("Tested '" + str + "' match with regex '" +
diff --git a/modules/src/test/java/org/archive/modules/deciderules/MatchesListRegexDecideRuleTest.java b/modules/src/test/java/org/archive/modules/deciderules/MatchesListRegexDecideRuleTest.java
new file mode 100644
index 00000000..e7019500
--- /dev/null
+++ b/modules/src/test/java/org/archive/modules/deciderules/MatchesListRegexDecideRuleTest.java
@@ -0,0 +1,34 @@
+package org.archive.modules.deciderules;
+
+import junit.framework.TestCase;
+import org.apache.commons.httpclient.URIException;
+import org.archive.modules.CrawlURI;
+import org.archive.net.UURIFactory;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.regex.Pattern;
+
+public class MatchesListRegexDecideRuleTest extends TestCase {
+
+ /**
+ * Not easy to test this code in older versions of junit. Basically with the timeout set to "0", this method
+ * will never return.
+ * @throws URIException
+ */
+ public void xtestEvaluate() throws URIException {
+ final String regex = "http://www\\.netarkivet\\.dk/((x+x+)+)y";
+ String seed = "http://www.netarkivet.dk/xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx";
+ MatchesListRegexDecideRule rule = new MatchesListRegexDecideRule();
+ List patternList = new ArrayList<>();
+ patternList.add(Pattern.compile(regex));
+ rule.setRegexList(patternList);
+ rule.setEnabled(true);
+ rule.setListLogicalOr(true);
+ rule.setDecision(DecideResult.REJECT);
+ rule.setTimeoutPerRegexSeconds(2);
+ final CrawlURI curi = new CrawlURI(UURIFactory.getInstance(seed));
+ final DecideResult decideResult = rule.decisionFor(curi);
+ assertEquals("Expected NONE not " + decideResult , DecideResult.NONE, decideResult);
+ }
+}
\ No newline at end of file