diff --git a/engine/src/test/java/org/archive/crawler/selftest/StatisticsSelfTest.java b/engine/src/test/java/org/archive/crawler/selftest/StatisticsSelfTest.java index d7d5d10e..4adc8586 100644 --- a/engine/src/test/java/org/archive/crawler/selftest/StatisticsSelfTest.java +++ b/engine/src/test/java/org/archive/crawler/selftest/StatisticsSelfTest.java @@ -47,14 +47,56 @@ public class StatisticsSelfTest extends SelfTestBase { protected void verifyWarcStats() { StatisticsTracker stats = heritrix.getEngine().getJob("selftest-job").getCrawlController().getStatisticsTracker(); assertNotNull(stats); - assertEquals(14, (long) stats.getCrawledBytes().get(CrawledBytesHistotable.WARC_NOVEL_URLS)); - assertEquals(12669, (long) stats.getCrawledBytes().get(CrawledBytesHistotable.WARC_NOVEL_CONTENT_BYTES) - stats.getBytesPerHost("dns:")); - assertEquals(3, (long) stats.getServerCache().getHostFor("127.0.0.1").getSubstats().get(CrawledBytesHistotable.WARC_NOVEL_URLS)); - assertEquals(2942, (long) stats.getServerCache().getHostFor("127.0.0.1").getSubstats().get(CrawledBytesHistotable.WARC_NOVEL_CONTENT_BYTES)); - assertEquals(10, (long) stats.getServerCache().getHostFor("localhost").getSubstats().get(CrawledBytesHistotable.WARC_NOVEL_URLS)); - assertEquals(9727, (long) stats.getServerCache().getHostFor("localhost").getSubstats().get(CrawledBytesHistotable.WARC_NOVEL_CONTENT_BYTES)); - assertEquals(1, (long) stats.getServerCache().getHostFor("dns:").getSubstats().get(CrawledBytesHistotable.WARC_NOVEL_URLS)); + long dnsCount = stats.getBytesPerHost("dns:"); + // Cope when DNS lookup may or may not be used for localhost: + if (dnsCount == -1) { + // No DNS record: + assertEquals(13, (long) stats.getCrawledBytes() + .get(CrawledBytesHistotable.WARC_NOVEL_URLS)); + assertEquals(12669, (long) stats.getCrawledBytes() + .get(CrawledBytesHistotable.WARC_NOVEL_CONTENT_BYTES)); + + assertEquals(3, + (long) stats.getServerCache().getHostFor("127.0.0.1") + .getSubstats() + .get(CrawledBytesHistotable.WARC_NOVEL_URLS)); + assertEquals(2942, (long) stats.getServerCache() + .getHostFor("127.0.0.1").getSubstats() + .get(CrawledBytesHistotable.WARC_NOVEL_CONTENT_BYTES)); + assertEquals(10, + (long) stats.getServerCache().getHostFor("localhost") + .getSubstats() + .get(CrawledBytesHistotable.WARC_NOVEL_URLS)); + assertEquals(9727, (long) stats.getServerCache() + .getHostFor("localhost").getSubstats() + .get(CrawledBytesHistotable.WARC_NOVEL_CONTENT_BYTES)); + } else { + // DNS record exists: + assertEquals(14, (long) stats.getCrawledBytes() + .get(CrawledBytesHistotable.WARC_NOVEL_URLS)); + assertEquals(12669, + (long) stats.getCrawledBytes() + .get(CrawledBytesHistotable.WARC_NOVEL_CONTENT_BYTES) + - stats.getBytesPerHost("dns:")); + + assertEquals(3, + (long) stats.getServerCache().getHostFor("127.0.0.1") + .getSubstats() + .get(CrawledBytesHistotable.WARC_NOVEL_URLS)); + assertEquals(2942, (long) stats.getServerCache() + .getHostFor("127.0.0.1").getSubstats() + .get(CrawledBytesHistotable.WARC_NOVEL_CONTENT_BYTES)); + assertEquals(10, + (long) stats.getServerCache().getHostFor("localhost") + .getSubstats() + .get(CrawledBytesHistotable.WARC_NOVEL_URLS)); + assertEquals(9727, (long) stats.getServerCache() + .getHostFor("localhost").getSubstats() + .get(CrawledBytesHistotable.WARC_NOVEL_CONTENT_BYTES)); + assertEquals(1, (long) stats.getServerCache().getHostFor("dns:") + .getSubstats().get(CrawledBytesHistotable.WARC_NOVEL_URLS)); + } } protected void verifySourceStats() throws Exception { @@ -76,8 +118,19 @@ public class StatisticsSelfTest extends SelfTestBase { assertEquals(4, sourceStats.keySet().size()); assertEquals(9727l, (long) sourceStats.get("novel") - stats.getBytesPerHost("dns:")); assertEquals(11l, (long) sourceStats.get("novelCount")); - assertEquals(9727l, (long) sourceStats.get("warcNovelContentBytes") - stats.getBytesPerHost("dns:")); - assertEquals(11l, (long) sourceStats.get("warcNovelUrls")); + long dnsCount = stats.getBytesPerHost("dns:"); + // Cope when DNS lookup may or may not be used for localhost: + if (dnsCount == -1) { + // No DNS record: + assertEquals(9727l, + (long) sourceStats.get("warcNovelContentBytes")); + assertEquals(10l, (long) sourceStats.get("warcNovelUrls")); + } else { + // DNS record exists: + assertEquals(9727l, + (long) sourceStats.get("warcNovelContentBytes") - dnsCount); + assertEquals(11l, (long) sourceStats.get("warcNovelUrls")); + } } }