diff --git a/engine/src/main/java/org/archive/crawler/framework/CrawlJob.java b/engine/src/main/java/org/archive/crawler/framework/CrawlJob.java index 1a19f3f2..cb63ee68 100644 --- a/engine/src/main/java/org/archive/crawler/framework/CrawlJob.java +++ b/engine/src/main/java/org/archive/crawler/framework/CrawlJob.java @@ -31,6 +31,7 @@ import java.util.LinkedHashMap; import java.util.LinkedList; import java.util.List; import java.util.Map; +import java.util.TreeMap; import java.util.logging.FileHandler; import java.util.logging.Formatter; import java.util.logging.Handler; @@ -797,7 +798,12 @@ public class CrawlJob implements Comparable, ApplicationListener { if(stats==null) { return null; } - return stats.getCrawledBytes(); + + // stats.crawledBytesSummary() also includes totals, so add those in here + TreeMap map = new TreeMap(stats.getCrawledBytes()); + map.put("total", stats.getCrawledBytes().getTotalBytes()); + map.put("total-count", stats.getCrawledBytes().getTotalUrls()); + return map; } public String sizeTotalsReport() { diff --git a/engine/src/main/java/org/archive/crawler/util/CrawledBytesHistotable.java b/engine/src/main/java/org/archive/crawler/util/CrawledBytesHistotable.java index 0bb0fddb..6797bd4b 100644 --- a/engine/src/main/java/org/archive/crawler/util/CrawledBytesHistotable.java +++ b/engine/src/main/java/org/archive/crawler/util/CrawledBytesHistotable.java @@ -79,4 +79,8 @@ implements CoreAttributeConstants { public long getTotalBytes() { return get(NOVEL) + get(DUPLICATE) + get(NOTMODIFIED); } + + public long getTotalUrls() { + return get(NOVELCOUNT) + get(DUPLICATECOUNT) + get(NOTMODIFIEDCOUNT); + } }