mirror of
https://github.com/internetarchive/heritrix3.git
synced 2026-11-16 10:47:42 +00:00
Merge branch 'bitbaron/master' into ait-qa
This commit is contained in:
commit
4d6969554c
2 files changed
+15
-7
No files matched your search
+8
-7
@@ -1,5 +1,6 @@
|
||||
<?xml version="1.0"?>
|
||||
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
|
||||
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
|
||||
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
|
||||
<parent>
|
||||
<groupId>org.archive</groupId>
|
||||
<artifactId>heritrix</artifactId>
|
||||
@@ -46,12 +47,12 @@
|
||||
<version>6.1.26</version>
|
||||
<scope>test</scope>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.mortbay.jetty</groupId>
|
||||
<artifactId>jetty</artifactId>
|
||||
<version>6.1.26</version>
|
||||
<scope>test</scope>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.mortbay.jetty</groupId>
|
||||
<artifactId>jetty</artifactId>
|
||||
<version>6.1.26</version>
|
||||
<scope>test</scope>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.littleshoot</groupId>
|
||||
<artifactId>littleproxy</artifactId>
|
||||
|
||||
@@ -42,6 +42,7 @@ import org.archive.modules.CrawlMetadata;
|
||||
import org.archive.modules.CrawlURI;
|
||||
import org.archive.modules.ProcessResult;
|
||||
import org.archive.modules.Processor;
|
||||
import org.archive.modules.deciderules.DecideResult;
|
||||
import org.archive.modules.deciderules.recrawl.IdenticalDigestDecideRule;
|
||||
import org.archive.modules.net.CrawlHost;
|
||||
import org.archive.modules.net.ServerCache;
|
||||
@@ -367,6 +368,12 @@ implements Lifecycle, Checkpointable, WriterPoolSettings {
|
||||
return false;
|
||||
}
|
||||
|
||||
if (getShouldProcessRule().decisionFor(curi) == DecideResult.REJECT) {
|
||||
curi.getAnnotations().add(ANNOTATION_UNWRITTEN + ":rejected("
|
||||
+ getShouldProcessRule().getClass() + ")");
|
||||
return false;
|
||||
}
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
|
||||
Reference in new issue
Block a user