diff --git a/modules/pom.xml b/modules/pom.xml index eccad158..b4747397 100644 --- a/modules/pom.xml +++ b/modules/pom.xml @@ -1,5 +1,6 @@ - + org.archive heritrix @@ -46,12 +47,12 @@ 6.1.26 test - - org.mortbay.jetty - jetty - 6.1.26 - test - + + org.mortbay.jetty + jetty + 6.1.26 + test + org.littleshoot littleproxy diff --git a/modules/src/main/java/org/archive/modules/writer/WriterPoolProcessor.java b/modules/src/main/java/org/archive/modules/writer/WriterPoolProcessor.java index e5f704e2..8fa1ea55 100644 --- a/modules/src/main/java/org/archive/modules/writer/WriterPoolProcessor.java +++ b/modules/src/main/java/org/archive/modules/writer/WriterPoolProcessor.java @@ -42,6 +42,7 @@ import org.archive.modules.CrawlMetadata; import org.archive.modules.CrawlURI; import org.archive.modules.ProcessResult; import org.archive.modules.Processor; +import org.archive.modules.deciderules.DecideResult; import org.archive.modules.deciderules.recrawl.IdenticalDigestDecideRule; import org.archive.modules.net.CrawlHost; import org.archive.modules.net.ServerCache; @@ -367,6 +368,12 @@ implements Lifecycle, Checkpointable, WriterPoolSettings { return false; } + if (getShouldProcessRule().decisionFor(curi) == DecideResult.REJECT) { + curi.getAnnotations().add(ANNOTATION_UNWRITTEN + ":rejected(" + + getShouldProcessRule().getClass() + ")"); + return false; + } + return true; }