diff --git a/.travis.yml b/.travis.yml index 08e91b4b..9f2b022c 100644 --- a/.travis.yml +++ b/.travis.yml @@ -3,9 +3,9 @@ sudo: false language: java jdk: + - oraclejdk8 - openjdk7 - openjdk8 - - oraclejdk8 before_install: - "export JAVA_OPTS=-Xmx1500m" diff --git a/commons/pom.xml b/commons/pom.xml index 75c31503..f3d5e564 100644 --- a/commons/pom.xml +++ b/commons/pom.xml @@ -188,7 +188,7 @@ org.netpreserve.commons webarchive-commons - 1.1.5 + 1.1.8 org.apache.hadoop diff --git a/contrib/src/main/java/org/archive/modules/postprocessor/WARCLimitEnforcer.java b/contrib/src/main/java/org/archive/modules/postprocessor/WARCLimitEnforcer.java index 29f82f0b..bd29362f 100644 --- a/contrib/src/main/java/org/archive/modules/postprocessor/WARCLimitEnforcer.java +++ b/contrib/src/main/java/org/archive/modules/postprocessor/WARCLimitEnforcer.java @@ -22,7 +22,8 @@ import java.util.HashMap; import java.util.Map; import java.util.concurrent.atomic.AtomicLong; import java.util.logging.Logger; - +import java.util.ArrayList; +import java.util.List; import org.archive.crawler.framework.CrawlController; import org.archive.crawler.framework.CrawlStatus; import org.archive.modules.CrawlURI; @@ -56,6 +57,17 @@ public class WARCLimitEnforcer extends Processor { return warcWriter; } + { + setWarcWriters(new ArrayList()); + } + @SuppressWarnings("unchecked") + public List getWarcWriters() { + return (List) kp.get("warcWriters"); + } + public void setWarcWriters(List warcWriters) { + kp.put("warcWriters", warcWriters); + } + protected CrawlController controller; public CrawlController getCrawlController() { return this.controller; @@ -76,15 +88,27 @@ public class WARCLimitEnforcer extends Processor { for (String k: limits.get(j).keySet()) { Long limit = limits.get(j).get(k); - Map valueBucket = warcWriter.getStats().get(j); - if (valueBucket != null) { - AtomicLong value = valueBucket.get(k); - if (value != null - && value.get() >= limit) { - log.info("stopping crawl because warcwriter stats['" + j + "']['" + k + "']=" + value.get() + " exceeds limit " + limit); - controller.requestCrawlStop(CrawlStatus.FINISHED_WRITE_LIMIT); + AtomicLong value = null; + if(getWarcWriters() !=null && getWarcWriters().size()>0) { + value = new AtomicLong(0); + for (WARCWriterProcessor w: getWarcWriters()) { + Map valueBucket = w.getStats().get(j); + if(valueBucket != null) { + value.set(value.addAndGet(valueBucket.get(k).get())); + } } } + else { + Map valueBucket = warcWriter.getStats().get(j); + if(valueBucket != null) { + value = valueBucket.get(k); + } + } + if (value != null + && value.get() >= limit) { + log.info("stopping crawl because warcwriter stats['" + j + "']['" + k + "']=" + value.get() + " exceeds limit " + limit); + controller.requestCrawlStop(CrawlStatus.FINISHED_WRITE_LIMIT); + } } } } diff --git a/modules/src/test/java/org/archive/modules/writer/WARCWriterProcessorTest.java b/modules/src/test/java/org/archive/modules/writer/WARCWriterProcessorTest.java index 22ef5811..582705b0 100644 --- a/modules/src/test/java/org/archive/modules/writer/WARCWriterProcessorTest.java +++ b/modules/src/test/java/org/archive/modules/writer/WARCWriterProcessorTest.java @@ -185,8 +185,7 @@ public class WARCWriterProcessorTest extends ProcessorTestBase { assertEquals(4, wwp.getStats().get("totals").get("numRecords").get()); assertEquals(responseBytes.length, wwp.getStats().get("response").get("contentBytes").get()); - // XXX fails currently, needs https://github.com/iipc/webarchive-commons/pull/51 - // assertEquals(warc.length(), wwp.getStats().get("totals").get("sizeOnDisk").get()); + assertEquals(warc.length(), wwp.getStats().get("totals").get("sizeOnDisk").get()); } /**