Enable suppression of empty hosts in report.

Until now hosts that have never been crawled are included in the hosts
report. This can be a very large number of hosts and is infrequently of
interest. This makes it possible to exclude the empty hosts from the
report. Default behavior remains to include them.
The option is highlighted by including it in the relevant, commented-out
section of the default profile.

engine/src/main/java/org/archive/crawler/reporting/HostsReport.java

@@ -47,15 +47,34 @@ public class HostsReport extends Report {
      * A value of zero means never sort. Default -1, always sort. This matches the behavior before this 
      * parameter was introduced.
      * 
-     * This value can not be overridden by a sheet.
+     * This value can not be overridden by a sheet. It may be safely edited at runtime.
      * 
      * @param maxSortSize
      */
     public void setMaxSortSize(int maxSortSize) {
     	this.maxSortSize = maxSortSize;
     }
-
-    @Override
+    
+    boolean suppressEmptyHosts = false;
+    public boolean isSuppressEmptyHosts() {
+		return suppressEmptyHosts;
+	}
+    /**
+     * If true, hosts for whom no URLs have been fetched will be suppressed in this report.
+     * Such hosts are recorded when the crawler encounters an URL for a host but has not yet (and may never) 
+     * processed any URL for the host. This can happen for many reason's, related to scoping and queue budgeting
+     * among others.
+     * Default behavior is to include these non-crawled hosts.
+     * 
+     * This value can no t be overridden by a sheet. It may be safely edited at runtime.
+     *  
+     * @param suppressEmptyHosts 
+     */
+	public void setSuppressEmptyHosts(boolean suppressEmptyHosts) {
+		this.suppressEmptyHosts = suppressEmptyHosts;
+	}
+	
+	@Override
     public void write(final PrintWriter writer, StatisticsTracker stats) {
     	Collection<String> keys = null;
     	DisposableStoredSortedMap<Long, String> hd = null;
@@ -70,18 +89,21 @@ public class HostsReport extends Report {
             // key is -count, value is hostname
             try {
                 CrawlHost host = stats.serverCache.getHostFor(key);
-                writeReportLine(writer,
-                        host.getSubstats().getFetchSuccesses(),
-                        host.getSubstats().getTotalBytes(),
-                        host.fixUpName(),
-                        host.getSubstats().getRobotsDenials(),
-                        host.getSubstats().getRemaining(), 
-                        host.getSubstats().getNovelUrls(),
-                        host.getSubstats().getNovelBytes(),
-                        host.getSubstats().getDupByHashUrls(),
-                        host.getSubstats().getDupByHashBytes(),
-                        host.getSubstats().getNotModifiedUrls(),
-                        host.getSubstats().getNotModifiedBytes());
+                long fetchSuccesses = host.getSubstats().getFetchSuccesses();
+                if (!suppressEmptyHosts || fetchSuccesses>0) {
+	                writeReportLine(writer,
+	                        fetchSuccesses,
+	                        host.getSubstats().getTotalBytes(),
+	                        host.fixUpName(),
+	                        host.getSubstats().getRobotsDenials(),
+	                        host.getSubstats().getRemaining(), 
+	                        host.getSubstats().getNovelUrls(),
+	                        host.getSubstats().getNovelBytes(),
+	                        host.getSubstats().getDupByHashUrls(),
+	                        host.getSubstats().getDupByHashBytes(),
+	                        host.getSubstats().getNotModifiedUrls(),
+	                        host.getSubstats().getNotModifiedBytes());
+                }
             } catch (Exception e) {
                 logger.log(Level.WARNING, "unable to tally host stats for " + key, e);
             }

engine/src/main/resources/org/archive/crawler/restlet/profile-crawler-beans.cxml

@@ -630,6 +630,7 @@ http://example.example/example
          <bean id="seedsReport" class="org.archive.crawler.reporting.SeedsReport" />
          <bean id="hostsReport" class="org.archive.crawler.reporting.HostsReport">
      		<property name="maxSortSize" value="-1" />
+     		<property name="suppressEmptyHosts" value="false" />
          </bean>
          <bean id="sourceTagsReport" class="org.archive.crawler.reporting.SourceTagsReport" />
          <bean id="mimetypesReport" class="org.archive.crawler.reporting.MimetypesReport" />