diff --git a/engine/src/main/java/org/archive/crawler/reporting/HostsReport.java b/engine/src/main/java/org/archive/crawler/reporting/HostsReport.java index 8206e3f3..6a3175dd 100644 --- a/engine/src/main/java/org/archive/crawler/reporting/HostsReport.java +++ b/engine/src/main/java/org/archive/crawler/reporting/HostsReport.java @@ -47,15 +47,34 @@ public class HostsReport extends Report { * A value of zero means never sort. Default -1, always sort. This matches the behavior before this * parameter was introduced. * - * This value can not be overridden by a sheet. + * This value can not be overridden by a sheet. It may be safely edited at runtime. * * @param maxSortSize */ public void setMaxSortSize(int maxSortSize) { this.maxSortSize = maxSortSize; } - - @Override + + boolean suppressEmptyHosts = false; + public boolean isSuppressEmptyHosts() { + return suppressEmptyHosts; + } + /** + * If true, hosts for whom no URLs have been fetched will be suppressed in this report. + * Such hosts are recorded when the crawler encounters an URL for a host but has not yet (and may never) + * processed any URL for the host. This can happen for many reason's, related to scoping and queue budgeting + * among others. + * Default behavior is to include these non-crawled hosts. + * + * This value can no t be overridden by a sheet. It may be safely edited at runtime. + * + * @param suppressEmptyHosts + */ + public void setSuppressEmptyHosts(boolean suppressEmptyHosts) { + this.suppressEmptyHosts = suppressEmptyHosts; + } + + @Override public void write(final PrintWriter writer, StatisticsTracker stats) { Collection keys = null; DisposableStoredSortedMap hd = null; @@ -70,18 +89,21 @@ public class HostsReport extends Report { // key is -count, value is hostname try { CrawlHost host = stats.serverCache.getHostFor(key); - writeReportLine(writer, - host.getSubstats().getFetchSuccesses(), - host.getSubstats().getTotalBytes(), - host.fixUpName(), - host.getSubstats().getRobotsDenials(), - host.getSubstats().getRemaining(), - host.getSubstats().getNovelUrls(), - host.getSubstats().getNovelBytes(), - host.getSubstats().getDupByHashUrls(), - host.getSubstats().getDupByHashBytes(), - host.getSubstats().getNotModifiedUrls(), - host.getSubstats().getNotModifiedBytes()); + long fetchSuccesses = host.getSubstats().getFetchSuccesses(); + if (!suppressEmptyHosts || fetchSuccesses>0) { + writeReportLine(writer, + fetchSuccesses, + host.getSubstats().getTotalBytes(), + host.fixUpName(), + host.getSubstats().getRobotsDenials(), + host.getSubstats().getRemaining(), + host.getSubstats().getNovelUrls(), + host.getSubstats().getNovelBytes(), + host.getSubstats().getDupByHashUrls(), + host.getSubstats().getDupByHashBytes(), + host.getSubstats().getNotModifiedUrls(), + host.getSubstats().getNotModifiedBytes()); + } } catch (Exception e) { logger.log(Level.WARNING, "unable to tally host stats for " + key, e); } diff --git a/engine/src/main/resources/org/archive/crawler/restlet/profile-crawler-beans.cxml b/engine/src/main/resources/org/archive/crawler/restlet/profile-crawler-beans.cxml index f3e246ce..589646e4 100644 --- a/engine/src/main/resources/org/archive/crawler/restlet/profile-crawler-beans.cxml +++ b/engine/src/main/resources/org/archive/crawler/restlet/profile-crawler-beans.cxml @@ -630,6 +630,7 @@ http://example.example/example +