lee/heritrix3
1
0
Fork 0
mirror of https://github.com/internetarchive/heritrix3.git synced 2026-09-23 22:26:01 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'master' into fix-ci-and-add-java-8

Browse Source
This commit is contained in:
Andrew Jackson
2017-10-12 09:31:47 +01:00
parent 09ee95f37b ae8a0ef95f
commit f7a5376fbe
4 changed files with 35 additions and 12 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.travis.yml
+1 -1
View File
@@ -3,9 +3,9 @@ sudo: false
language: java
jdk:
- oraclejdk8
- openjdk7
- openjdk8
- oraclejdk8
before_install:
- "export JAVA_OPTS=-Xmx1500m"
commons/pom.xml
+1 -1
View File
@@ -188,7 +188,7 @@
<dependency>
<groupId>org.netpreserve.commons</groupId>
<artifactId>webarchive-commons</artifactId>
<version>1.1.5</version>
<version>1.1.8</version>
<exclusions>
<exclusion>
<groupId>org.apache.hadoop</groupId>
contrib/src/main/java/org/archive/modules/postprocessor/WARCLimitEnforcer.java
+32 -8
View File
@@ -22,7 +22,8 @@ import java.util.HashMap;
import java.util.Map;
import java.util.concurrent.atomic.AtomicLong;
import java.util.logging.Logger;
import java.util.ArrayList;
import java.util.List;
import org.archive.crawler.framework.CrawlController;
import org.archive.crawler.framework.CrawlStatus;
import org.archive.modules.CrawlURI;
@@ -56,6 +57,17 @@ public class WARCLimitEnforcer extends Processor {
return warcWriter;
}
{
setWarcWriters(new ArrayList<WARCWriterProcessor>());
}
@SuppressWarnings("unchecked")
public List<WARCWriterProcessor> getWarcWriters() {
return (List<WARCWriterProcessor>) kp.get("warcWriters");
}
public void setWarcWriters(List<WARCWriterProcessor> warcWriters) {
kp.put("warcWriters", warcWriters);
}
protected CrawlController controller;
public CrawlController getCrawlController() {
return this.controller;
@@ -76,15 +88,27 @@ public class WARCLimitEnforcer extends Processor {
for (String k: limits.get(j).keySet()) {
Long limit = limits.get(j).get(k);
Map<String, AtomicLong> valueBucket = warcWriter.getStats().get(j);
if (valueBucket != null) {
AtomicLong value = valueBucket.get(k);
if (value != null
&& value.get() >= limit) {
log.info("stopping crawl because warcwriter stats['" + j + "']['" + k + "']=" + value.get() + " exceeds limit " + limit);
controller.requestCrawlStop(CrawlStatus.FINISHED_WRITE_LIMIT);
AtomicLong value = null;
if(getWarcWriters() !=null && getWarcWriters().size()>0) {
value = new AtomicLong(0);
for (WARCWriterProcessor w: getWarcWriters()) {
Map<String, AtomicLong> valueBucket = w.getStats().get(j);
if(valueBucket != null) {
value.set(value.addAndGet(valueBucket.get(k).get()));
}
}
}
else {
Map<String, AtomicLong> valueBucket = warcWriter.getStats().get(j);
if(valueBucket != null) {
value = valueBucket.get(k);
}
}
if (value != null
&& value.get() >= limit) {
log.info("stopping crawl because warcwriter stats['" + j + "']['" + k + "']=" + value.get() + " exceeds limit " + limit);
controller.requestCrawlStop(CrawlStatus.FINISHED_WRITE_LIMIT);
}
}
}
}
modules/src/test/java/org/archive/modules/writer/WARCWriterProcessorTest.java
+1 -2
View File
@@ -185,8 +185,7 @@ public class WARCWriterProcessorTest extends ProcessorTestBase {
assertEquals(4, wwp.getStats().get("totals").get("numRecords").get());
assertEquals(responseBytes.length, wwp.getStats().get("response").get("contentBytes").get());
// XXX fails currently, needs https://github.com/iipc/webarchive-commons/pull/51
// assertEquals(warc.length(), wwp.getStats().get("totals").get("sizeOnDisk").get());
assertEquals(warc.length(), wwp.getStats().get("totals").get("sizeOnDisk").get());
}
/**