lee/heritrix3
1
0
Fork 0
mirror of https://github.com/internetarchive/heritrix3.git synced 2026-09-25 07:05:49 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #189 from adam-miller/master

Browse Source 
WARCLimitEnforcer.java - Add support for multiple warc writers.
This commit is contained in:
Noah Levitt
2017-10-11 14:46:58 -07:00
committed by GitHub
parent a0a783c750 b702bf3d86
commit 4e96d0c94f
1 changed files with 32 additions and 8 deletions
Download Patch File Download Diff File Expand all files Collapse all files
contrib/src/main/java/org/archive/modules/postprocessor/WARCLimitEnforcer.java
+32 -8
View File
@@ -22,7 +22,8 @@ import java.util.HashMap;
import java.util.Map;
import java.util.concurrent.atomic.AtomicLong;
import java.util.logging.Logger;
import java.util.ArrayList;
import java.util.List;
import org.archive.crawler.framework.CrawlController;
import org.archive.crawler.framework.CrawlStatus;
import org.archive.modules.CrawlURI;
@@ -56,6 +57,17 @@ public class WARCLimitEnforcer extends Processor {
return warcWriter;
}
{
setWarcWriters(new ArrayList<WARCWriterProcessor>());
}
@SuppressWarnings("unchecked")
public List<WARCWriterProcessor> getWarcWriters() {
return (List<WARCWriterProcessor>) kp.get("warcWriters");
}
public void setWarcWriters(List<WARCWriterProcessor> warcWriters) {
kp.put("warcWriters", warcWriters);
}
protected CrawlController controller;
public CrawlController getCrawlController() {
return this.controller;
@@ -76,15 +88,27 @@ public class WARCLimitEnforcer extends Processor {
for (String k: limits.get(j).keySet()) {
Long limit = limits.get(j).get(k);
Map<String, AtomicLong> valueBucket = warcWriter.getStats().get(j);
if (valueBucket != null) {
AtomicLong value = valueBucket.get(k);
if (value != null
&& value.get() >= limit) {
log.info("stopping crawl because warcwriter stats['" + j + "']['" + k + "']=" + value.get() + " exceeds limit " + limit);
controller.requestCrawlStop(CrawlStatus.FINISHED_WRITE_LIMIT);
AtomicLong value = null;
if(getWarcWriters() !=null && getWarcWriters().size()>0) {
value = new AtomicLong(0);
for (WARCWriterProcessor w: getWarcWriters()) {
Map<String, AtomicLong> valueBucket = w.getStats().get(j);
if(valueBucket != null) {
value.set(value.addAndGet(valueBucket.get(k).get()));
}
}
}
else {
Map<String, AtomicLong> valueBucket = warcWriter.getStats().get(j);
if(valueBucket != null) {
value = valueBucket.get(k);
}
}
if (value != null
&& value.get() >= limit) {
log.info("stopping crawl because warcwriter stats['" + j + "']['" + k + "']=" + value.get() + " exceeds limit " + limit);
controller.requestCrawlStop(CrawlStatus.FINISHED_WRITE_LIMIT);
}
}
}
}