lee/heritrix3
1
0
Fork 0
mirror of https://github.com/internetarchive/heritrix3.git synced 2026-09-23 14:15:47 +00:00
Code Issues Packages Projects Releases Wiki Activity

Adapt test to cope with variation in how localhost gets resolved.

Browse Source
This commit is contained in:
Andrew Jackson
2017-10-09 21:50:05 +01:00
parent 1de6f7d2a0
commit 401e27fc64
1 changed files with 62 additions and 9 deletions
Download Patch File Download Diff File Expand all files Collapse all files
engine/src/test/java/org/archive/crawler/selftest/StatisticsSelfTest.java
+62 -9
View File
@@ -47,14 +47,56 @@ public class StatisticsSelfTest extends SelfTestBase {
protected void verifyWarcStats() {
StatisticsTracker stats = heritrix.getEngine().getJob("selftest-job").getCrawlController().getStatisticsTracker();
assertNotNull(stats);
assertEquals(14, (long) stats.getCrawledBytes().get(CrawledBytesHistotable.WARC_NOVEL_URLS));
assertEquals(12669, (long) stats.getCrawledBytes().get(CrawledBytesHistotable.WARC_NOVEL_CONTENT_BYTES) - stats.getBytesPerHost("dns:"));
assertEquals(3, (long) stats.getServerCache().getHostFor("127.0.0.1").getSubstats().get(CrawledBytesHistotable.WARC_NOVEL_URLS));
assertEquals(2942, (long) stats.getServerCache().getHostFor("127.0.0.1").getSubstats().get(CrawledBytesHistotable.WARC_NOVEL_CONTENT_BYTES));
assertEquals(10, (long) stats.getServerCache().getHostFor("localhost").getSubstats().get(CrawledBytesHistotable.WARC_NOVEL_URLS));
assertEquals(9727, (long) stats.getServerCache().getHostFor("localhost").getSubstats().get(CrawledBytesHistotable.WARC_NOVEL_CONTENT_BYTES));
assertEquals(1, (long) stats.getServerCache().getHostFor("dns:").getSubstats().get(CrawledBytesHistotable.WARC_NOVEL_URLS));
long dnsCount = stats.getBytesPerHost("dns:");
// Cope when DNS lookup may or may not be used for localhost:
if (dnsCount == -1) {
// No DNS record:
assertEquals(13, (long) stats.getCrawledBytes()
.get(CrawledBytesHistotable.WARC_NOVEL_URLS));
assertEquals(12669, (long) stats.getCrawledBytes()
.get(CrawledBytesHistotable.WARC_NOVEL_CONTENT_BYTES));
assertEquals(3,
(long) stats.getServerCache().getHostFor("127.0.0.1")
.getSubstats()
.get(CrawledBytesHistotable.WARC_NOVEL_URLS));
assertEquals(2942, (long) stats.getServerCache()
.getHostFor("127.0.0.1").getSubstats()
.get(CrawledBytesHistotable.WARC_NOVEL_CONTENT_BYTES));
assertEquals(10,
(long) stats.getServerCache().getHostFor("localhost")
.getSubstats()
.get(CrawledBytesHistotable.WARC_NOVEL_URLS));
assertEquals(9727, (long) stats.getServerCache()
.getHostFor("localhost").getSubstats()
.get(CrawledBytesHistotable.WARC_NOVEL_CONTENT_BYTES));
} else {
// DNS record exists:
assertEquals(14, (long) stats.getCrawledBytes()
.get(CrawledBytesHistotable.WARC_NOVEL_URLS));
assertEquals(12669,
(long) stats.getCrawledBytes()
.get(CrawledBytesHistotable.WARC_NOVEL_CONTENT_BYTES)
- stats.getBytesPerHost("dns:"));
assertEquals(3,
(long) stats.getServerCache().getHostFor("127.0.0.1")
.getSubstats()
.get(CrawledBytesHistotable.WARC_NOVEL_URLS));
assertEquals(2942, (long) stats.getServerCache()
.getHostFor("127.0.0.1").getSubstats()
.get(CrawledBytesHistotable.WARC_NOVEL_CONTENT_BYTES));
assertEquals(10,
(long) stats.getServerCache().getHostFor("localhost")
.getSubstats()
.get(CrawledBytesHistotable.WARC_NOVEL_URLS));
assertEquals(9727, (long) stats.getServerCache()
.getHostFor("localhost").getSubstats()
.get(CrawledBytesHistotable.WARC_NOVEL_CONTENT_BYTES));
assertEquals(1, (long) stats.getServerCache().getHostFor("dns:")
.getSubstats().get(CrawledBytesHistotable.WARC_NOVEL_URLS));
}
}
protected void verifySourceStats() throws Exception {
@@ -76,8 +118,19 @@ public class StatisticsSelfTest extends SelfTestBase {
assertEquals(4, sourceStats.keySet().size());
assertEquals(9727l, (long) sourceStats.get("novel") - stats.getBytesPerHost("dns:"));
assertEquals(11l, (long) sourceStats.get("novelCount"));
assertEquals(9727l, (long) sourceStats.get("warcNovelContentBytes") - stats.getBytesPerHost("dns:"));
assertEquals(11l, (long) sourceStats.get("warcNovelUrls"));
long dnsCount = stats.getBytesPerHost("dns:");
// Cope when DNS lookup may or may not be used for localhost:
if (dnsCount == -1) {
// No DNS record:
assertEquals(9727l,
(long) sourceStats.get("warcNovelContentBytes"));
assertEquals(10l, (long) sourceStats.get("warcNovelUrls"));
} else {
// DNS record exists:
assertEquals(9727l,
(long) sourceStats.get("warcNovelContentBytes") - dnsCount);
assertEquals(11l, (long) sourceStats.get("warcNovelUrls"));
}
}
}