From ce4a865764ff8202ee1772df8f49d49713d1be34 Mon Sep 17 00:00:00 2001 From: Noah Levitt Date: Thu, 3 Dec 2015 16:13:34 -0800 Subject: [PATCH] add support to StatisticsTracker to keep a CrawledBytesHistotable per source tag when trackSources is enabled; integration test for this functionality --- .../crawler/reporting/StatisticsTracker.java | 37 +++++++++++++++-- .../crawler/selftest/StatisticsSelfTest.java | 40 +++++++++++++++++++ .../selftest/StatisticsSelfTest/htdocs/a.html | 1 + .../selftest/StatisticsSelfTest/htdocs/b.html | 4 ++ .../StatisticsSelfTest/htdocs/b1.html | 7 ++++ .../StatisticsSelfTest/htdocs/b2.html | 7 ++++ .../StatisticsSelfTest/htdocs/b3.html | 7 ++++ .../selftest/conf/selftest-crawler-beans.cxml | 1 + 8 files changed, 101 insertions(+), 3 deletions(-) create mode 100644 engine/src/test/java/org/archive/crawler/selftest/StatisticsSelfTest.java create mode 100644 engine/testdata/selftest/StatisticsSelfTest/htdocs/a.html create mode 100644 engine/testdata/selftest/StatisticsSelfTest/htdocs/b.html create mode 100644 engine/testdata/selftest/StatisticsSelfTest/htdocs/b1.html create mode 100644 engine/testdata/selftest/StatisticsSelfTest/htdocs/b2.html create mode 100644 engine/testdata/selftest/StatisticsSelfTest/htdocs/b3.html diff --git a/engine/src/main/java/org/archive/crawler/reporting/StatisticsTracker.java b/engine/src/main/java/org/archive/crawler/reporting/StatisticsTracker.java index e005efbf..bfe27e16 100644 --- a/engine/src/main/java/org/archive/crawler/reporting/StatisticsTracker.java +++ b/engine/src/main/java/org/archive/crawler/reporting/StatisticsTracker.java @@ -288,6 +288,12 @@ public class StatisticsTracker // TODO: restore spill-to-disk, like with processedSeedsRecords protected ConcurrentHashMap> sourceHostDistribution = new ConcurrentHashMap>(); + + /** Keep track of crawled bytes stats per seed */ + // TODO: spill-to-disk (requires bdb replacement for Histotable, or some + // other refactoring) + protected ConcurrentHashMap statsBySource = + new ConcurrentHashMap(); /* Keep track of 'top' hosts for live reports */ protected TopNSet hostsDistributionTop; @@ -396,6 +402,15 @@ public class StatisticsTracker sourceHostDistribution.put(source, hostUriCount); } + JSONObject ss = json.getJSONObject("sourceStats"); + keyIter = ss.keys(); + for(; keyIter.hasNext();) { + String source = keyIter.next(); + CrawledBytesHistotable cb = new CrawledBytesHistotable(); + JSONUtils.putAllLongs(cb, ss.getJSONObject(source)); + statsBySource.put(source, cb); + } + JSONUtils.putAllLongs( crawledBytes, json.getJSONObject("crawledBytes")); @@ -768,9 +783,9 @@ public class StatisticsTracker curi.getContentSize()); if (getTrackSources() && curi.getData().containsKey(A_SOURCE_TAG)) { - saveSourceStats((String)curi.getData().get(A_SOURCE_TAG), - sc.getHostFor(curi.getUURI()). - getHostName()); + saveSourceStats(curi.getSourceTag(), + sc.getHostFor(curi.getUURI()).getHostName()); + tallySourceStats(curi); } } @@ -784,7 +799,16 @@ public class StatisticsTracker } } incrementMapCount(hostUriCount, hostname); + } + protected void tallySourceStats(CrawlURI curi) { + String source = curi.getSourceTag(); + CrawledBytesHistotable sourceStats = statsBySource.get(source); + if (sourceStats == null) { + sourceStats = new CrawledBytesHistotable(); + statsBySource.put(source, sourceStats); + } + sourceStats.accumulate(curi); } /** @@ -1087,6 +1111,7 @@ public class StatisticsTracker json.put("statusCodeDistribution", statusCodeDistribution); json.put("sourceHostDistribution", sourceHostDistribution); + json.put("sourceStats", statsBySource); json.put("crawledBytes", crawledBytes); @@ -1103,4 +1128,10 @@ public class StatisticsTracker this.recoveryCheckpoint = recoveryCheckpoint; } + public CrawledBytesHistotable getSourceStats(String source) { + return statsBySource.get(source); + } + + + } diff --git a/engine/src/test/java/org/archive/crawler/selftest/StatisticsSelfTest.java b/engine/src/test/java/org/archive/crawler/selftest/StatisticsSelfTest.java new file mode 100644 index 00000000..3eb97acc --- /dev/null +++ b/engine/src/test/java/org/archive/crawler/selftest/StatisticsSelfTest.java @@ -0,0 +1,40 @@ +package org.archive.crawler.selftest; + +import org.archive.crawler.reporting.StatisticsTracker; +import org.archive.crawler.util.CrawledBytesHistotable; + +public class StatisticsSelfTest extends SelfTestBase { + + @Override + protected String getSeedsString() { + // we have these coming from different hosts, so that robots.txt is + // fetched for both; otherwise what is crawled would be + // non-deterministic + return "http://127.0.0.1:7777/a.html\\nhttp://localhost:7777/b.html"; + } + + @Override + protected void verify() throws Exception { + verifySourceStats(); + } + + protected void verifySourceStats() throws Exception { + StatisticsTracker stats = heritrix.getEngine().getJob("selftest-job").getCrawlController().getStatisticsTracker(); + assertNotNull(stats); + assertTrue(stats.getTrackSources()); + CrawledBytesHistotable sourceStats = stats.getSourceStats("http://127.0.0.1:7777/index.html"); + assertNull(sourceStats); + sourceStats = stats.getSourceStats("http://127.0.0.1:7777/a.html"); + assertNotNull(sourceStats); + assertEquals(2, sourceStats.keySet().size()); + assertEquals(2942l, (long) sourceStats.get("novel")); + assertEquals(3l, (long) sourceStats.get("novelCount")); + + sourceStats = stats.getSourceStats("http://localhost:7777/b.html"); + assertNotNull(sourceStats); + assertEquals(2, sourceStats.keySet().size()); + assertEquals(9776l, (long) sourceStats.get("novel")); + assertEquals(11l, (long) sourceStats.get("novelCount")); + } + +} diff --git a/engine/testdata/selftest/StatisticsSelfTest/htdocs/a.html b/engine/testdata/selftest/StatisticsSelfTest/htdocs/a.html new file mode 100644 index 00000000..e6643e03 --- /dev/null +++ b/engine/testdata/selftest/StatisticsSelfTest/htdocs/a.html @@ -0,0 +1 @@ +

nothing here

diff --git a/engine/testdata/selftest/StatisticsSelfTest/htdocs/b.html b/engine/testdata/selftest/StatisticsSelfTest/htdocs/b.html new file mode 100644 index 00000000..8748ff55 --- /dev/null +++ b/engine/testdata/selftest/StatisticsSelfTest/htdocs/b.html @@ -0,0 +1,4 @@ +b +b1 +b2 +b3 diff --git a/engine/testdata/selftest/StatisticsSelfTest/htdocs/b1.html b/engine/testdata/selftest/StatisticsSelfTest/htdocs/b1.html new file mode 100644 index 00000000..db7fabf2 --- /dev/null +++ b/engine/testdata/selftest/StatisticsSelfTest/htdocs/b1.html @@ -0,0 +1,7 @@ +index + +Link 1 + + Link 2 + + Link 3 diff --git a/engine/testdata/selftest/StatisticsSelfTest/htdocs/b2.html b/engine/testdata/selftest/StatisticsSelfTest/htdocs/b2.html new file mode 100644 index 00000000..db7fabf2 --- /dev/null +++ b/engine/testdata/selftest/StatisticsSelfTest/htdocs/b2.html @@ -0,0 +1,7 @@ +index + +Link 1 + + Link 2 + + Link 3 diff --git a/engine/testdata/selftest/StatisticsSelfTest/htdocs/b3.html b/engine/testdata/selftest/StatisticsSelfTest/htdocs/b3.html new file mode 100644 index 00000000..db7fabf2 --- /dev/null +++ b/engine/testdata/selftest/StatisticsSelfTest/htdocs/b3.html @@ -0,0 +1,7 @@ +index + +Link 1 + + Link 2 + + Link 3 diff --git a/engine/testdata/selftest/conf/selftest-crawler-beans.cxml b/engine/testdata/selftest/conf/selftest-crawler-beans.cxml index 55a20702..2332bfb2 100644 --- a/engine/testdata/selftest/conf/selftest-crawler-beans.cxml +++ b/engine/testdata/selftest/conf/selftest-crawler-beans.cxml @@ -43,6 +43,7 @@ crawlController.pauseAtStart=false +