Merge pull request #123 from kris-sigur/HostsReport

Hosts report

engine/src/main/java/org/archive/crawler/reporting/HostsReport.java

@@ -20,7 +20,7 @@
 package org.archive.crawler.reporting;
 
 import java.io.PrintWriter;
-import java.util.Map;
+import java.util.Collection;
 import java.util.logging.Level;
 import java.util.logging.Logger;
 
@@ -36,33 +36,81 @@ public class HostsReport extends Report {
     
     private final static Logger logger =
             Logger.getLogger(HostsReport.class.getName());
-
-    @Override
+    
+    int maxSortSize = -1;
+    public int getMaxSortSize() {
+    	return maxSortSize;
+    }
+    /**
+     * The maximum number of hosts allowed in a report while still sorting it. If the number of hosts exceeds
+     * this value, the generated report will not be sorted. A negative signifies no limit (always sort). 
+     * A value of zero means never sort. Default -1, always sort. This matches the behavior before this 
+     * parameter was introduced.
+     * 
+     * This value can not be overridden by a sheet. It may be safely edited at runtime.
+     * 
+     * @param maxSortSize
+     */
+    public void setMaxSortSize(int maxSortSize) {
+    	this.maxSortSize = maxSortSize;
+    }
+    
+    boolean suppressEmptyHosts = false;
+    public boolean isSuppressEmptyHosts() {
+		return suppressEmptyHosts;
+	}
+    /**
+     * If true, hosts for whom no URLs have been fetched will be suppressed in this report.
+     * Such hosts are recorded when the crawler encounters an URL for a host but has not yet (and may never) 
+     * processed any URL for the host. This can happen for many reason's, related to scoping and queue budgeting
+     * among others.
+     * Default behavior is to include these non-crawled hosts.
+     * 
+     * This value can not be overridden by a sheet. It may be safely edited at runtime.
+     *  
+     * @param suppressEmptyHosts 
+     */
+	public void setSuppressEmptyHosts(boolean suppressEmptyHosts) {
+		this.suppressEmptyHosts = suppressEmptyHosts;
+	}
+	
+	@Override
     public void write(final PrintWriter writer, StatisticsTracker stats) {
-        // TODO: only perform sorting on manageable number of hosts
-        DisposableStoredSortedMap<Long,String> hd = stats.calcReverseSortedHostsDistribution();
+    	Collection<String> keys = null;
+    	DisposableStoredSortedMap<Long, String> hd = null;
+    	if (maxSortSize<0 || maxSortSize>stats.serverCache.hostKeys().size()) {
+    		hd = stats.calcReverseSortedHostsDistribution();
+        	keys = hd.values();
+        } else {
+        	keys = stats.serverCache.hostKeys();
+        }
         writer.print("[#urls] [#bytes] [host] [#robots] [#remaining] [#novel-urls] [#novel-bytes] [#dup-by-hash-urls] [#dup-by-hash-bytes] [#not-modified-urls] [#not-modified-bytes]\n"); 
-        for (Map.Entry<Long,String> entry : hd.entrySet()) {
+        for (String key : keys) {
             // key is -count, value is hostname
             try {
-                CrawlHost host = stats.serverCache.getHostFor(entry.getValue());
-                writeReportLine(writer,
-                        host.getSubstats().getFetchSuccesses(),
-                        host.getSubstats().getTotalBytes(),
-                        host.fixUpName(),
-                        host.getSubstats().getRobotsDenials(),
-                        host.getSubstats().getRemaining(), 
-                        host.getSubstats().getNovelUrls(),
-                        host.getSubstats().getNovelBytes(),
-                        host.getSubstats().getDupByHashUrls(),
-                        host.getSubstats().getDupByHashBytes(),
-                        host.getSubstats().getNotModifiedUrls(),
-                        host.getSubstats().getNotModifiedBytes());
+                CrawlHost host = stats.serverCache.getHostFor(key);
+                long fetchSuccesses = host.getSubstats().getFetchSuccesses();
+                if (!suppressEmptyHosts || fetchSuccesses>0) {
+	                writeReportLine(writer,
+	                        fetchSuccesses,
+	                        host.getSubstats().getTotalBytes(),
+	                        host.fixUpName(),
+	                        host.getSubstats().getRobotsDenials(),
+	                        host.getSubstats().getRemaining(), 
+	                        host.getSubstats().getNovelUrls(),
+	                        host.getSubstats().getNovelBytes(),
+	                        host.getSubstats().getDupByHashUrls(),
+	                        host.getSubstats().getDupByHashBytes(),
+	                        host.getSubstats().getNotModifiedUrls(),
+	                        host.getSubstats().getNotModifiedBytes());
+                }
             } catch (Exception e) {
-                logger.log(Level.WARNING, "unable to tally host stats for " + entry.getValue(), e);
+                logger.log(Level.WARNING, "unable to tally host stats for " + key, e);
             }
         }
-        hd.dispose();
+        if (hd!=null) {
+        	hd.dispose();
+        }
     }
 
     protected void writeReportLine(PrintWriter writer, Object  ... fields) {

engine/src/main/java/org/archive/crawler/reporting/StatisticsTracker.java

@@ -50,7 +50,6 @@ import org.archive.crawler.event.StatSnapshotEvent;
 import org.archive.crawler.framework.CrawlController;
 import org.archive.crawler.framework.Engine;
 import org.archive.crawler.util.CrawledBytesHistotable;
-import org.archive.crawler.util.TopNSet;
 import org.archive.modules.CrawlURI;
 import org.archive.modules.net.CrawlHost;
 import org.archive.modules.net.ServerCache;
@@ -139,7 +138,7 @@ public class StatisticsTracker
         Checkpointable,
         BeanNameAware {
     @SuppressWarnings("unused")
-    private static final long serialVersionUID = 5L;
+    private static final long serialVersionUID = 6L;
 
     protected SeedModule seeds;
     public SeedModule getSeeds() {
@@ -295,11 +294,6 @@ public class StatisticsTracker
     protected ConcurrentHashMap<String, CrawledBytesHistotable> statsBySource =
             new ConcurrentHashMap<String, CrawledBytesHistotable>();
 
-    /* Keep track of 'top' hosts for live reports */
-    protected TopNSet hostsDistributionTop;
-    protected TopNSet hostsBytesTop;
-    protected TopNSet hostsLastFinishedTop;
-    
     /**
      * Record of seeds and latest results
      */
@@ -356,10 +350,6 @@ public class StatisticsTracker
             this.processedSeedsRecords = bdb.getObjectCache("processedSeedsRecords",
                     isRecover, SeedRecord.class);
             
-            this.hostsDistributionTop = new TopNSet(getLiveHostReportSize());
-            this.hostsBytesTop = new TopNSet(getLiveHostReportSize());
-            this.hostsLastFinishedTop = new TopNSet(getLiveHostReportSize());
-            
             if(isRecover) {
                 JSONObject json = recoveryCheckpoint.loadJson(beanName);
                 
@@ -369,19 +359,6 @@ public class StatisticsTracker
                 crawlPauseStarted = json.getLong("crawlPauseStarted");
                 tallyCurrentPause();
                 
-                JSONUtils.putAllLongs(
-                        hostsDistributionTop.getTopSet(),
-                        json.getJSONObject("hostsDistributionTop"));
-                hostsDistributionTop.updateBounds();
-                JSONUtils.putAllLongs(
-                        hostsBytesTop.getTopSet(),
-                        json.getJSONObject("hostsBytesTop"));
-                hostsBytesTop.updateBounds();
-                JSONUtils.putAllLongs(
-                        hostsLastFinishedTop.getTopSet(),
-                        json.getJSONObject("hostsLastFinishedTop"));
-                hostsLastFinishedTop.updateBounds();
-                
                 JSONUtils.putAllAtomicLongs(
                     mimeTypeDistribution,
                     json.getJSONObject("mimeTypeDistribution"));
@@ -780,11 +757,7 @@ public class StatisticsTracker
         incrementMapCount(mimeTypeDistribution, mime);
         incrementMapCount(mimeTypeBytes, mime, curi.getContentSize());
 
-        // Save hosts stats.
         ServerCache sc = serverCache;
-        saveHostStats(sc.getHostFor(curi.getUURI()).getHostName(),
-                curi.getContentSize());
-        
         if (getTrackSources() && curi.getData().containsKey(A_SOURCE_TAG)) {
         	saveSourceStats(curi.getSourceTag(), 
         	        sc.getHostFor(curi.getUURI()).getHostName());
@@ -814,22 +787,6 @@ public class StatisticsTracker
         sourceStats.accumulate(curi);
     }
     
-    /**
-     * Update some running-stats based on a URI success
-     * 
-     * @param hostname
-     * @param size
-     */
-    protected void saveHostStats(String hostname, long size) {
-        // TODO: consider moving 'top' accounting elsewhere, such 
-        // as the frontier or ServerCache itself
-        
-        CrawlHost host = serverCache.getHostFor(hostname); 
-        hostsDistributionTop.update(hostname, host.getSubstats().getFetchSuccesses()); 
-        hostsBytesTop.update(hostname, host.getSubstats().getSuccessBytes());
-        hostsLastFinishedTop.update(hostname, host.getSubstats().getLastSuccessTime());
-    }
-
     public void crawledURINeedRetry(CrawlURI curi) {
         handleSeed(curi,"Failed to crawl seed, will retry");
     }
@@ -1105,10 +1062,6 @@ public class StatisticsTracker
             json.put("crawlPauseStarted",virtualCrawlPauseStarted);
             json.put("crawlTotalPausedTime",crawlTotalPausedTime);
             
-            json.put("hostsDistributionTop", hostsDistributionTop.getTopSet());
-            json.put("hostsBytesTop", hostsBytesTop.getTopSet());
-            json.put("hostsLastFinishedTop", hostsLastFinishedTop.getTopSet());
-
             json.put("mimeTypeDistribution", mimeTypeDistribution);
             json.put("mimeTypeBytes", mimeTypeBytes);
             json.put("statusCodeDistribution", statusCodeDistribution);

engine/src/main/resources/org/archive/crawler/restlet/profile-crawler-beans.cxml

@@ -628,7 +628,10 @@ http://example.example/example
         <list>
          <bean id="crawlSummaryReport" class="org.archive.crawler.reporting.CrawlSummaryReport" />
          <bean id="seedsReport" class="org.archive.crawler.reporting.SeedsReport" />
-         <bean id="hostsReport" class="org.archive.crawler.reporting.HostsReport" />
+         <bean id="hostsReport" class="org.archive.crawler.reporting.HostsReport">
+     		<property name="maxSortSize" value="-1" />
+     		<property name="suppressEmptyHosts" value="false" />
+         </bean>
          <bean id="sourceTagsReport" class="org.archive.crawler.reporting.SourceTagsReport" />
          <bean id="mimetypesReport" class="org.archive.crawler.reporting.MimetypesReport" />
          <bean id="responseCodeReport" class="org.archive.crawler.reporting.ResponseCodeReport" />