From a490737b59ff17bcd0815da2f805a47fc92838bc Mon Sep 17 00:00:00 2001
From: gojomo <gojomo@daa5b2f2-a927-0410-8b2d-f5f262fa301a>
Date: Wed, 20 Jul 2011 23:36:19 +0000
Subject: [PATCH] * StatisticsTracker     add 'trackSources' to make
 hosts-by-source-tag report optional in large crawls, where it is very
 expensive (and an OOME risk until other changes are also made)

---
 .../crawler/reporting/StatisticsTracker.java  | 19 ++++++++++++++++---
 1 file changed, 16 insertions(+), 3 deletions(-)

diff --git a/engine/src/main/java/org/archive/crawler/reporting/StatisticsTracker.java b/engine/src/main/java/org/archive/crawler/reporting/StatisticsTracker.java
index 5f4b6332..6d56a825 100644
--- a/engine/src/main/java/org/archive/crawler/reporting/StatisticsTracker.java
+++ b/engine/src/main/java/org/archive/crawler/reporting/StatisticsTracker.java
@@ -201,6 +201,19 @@ public class StatisticsTracker
     public void setTrackSeeds(boolean trackSeeds) {
         this.trackSeeds = trackSeeds;
     }
+    
+    /**
+     * Whether to maintain hosts-per-source-tag records for; very expensive in 
+     * crawls with large numbers of source-tags (seeds) or large crawls 
+     * over many hosts
+     */
+    boolean trackSources = true;
+    public boolean getTrackSources() {
+        return this.trackSources;
+    }
+    public void setTrackSources(boolean trackSources) {
+        this.trackSources = trackSources;
+    }
             
     /**
      * The interval between writing progress information to log.
@@ -270,6 +283,7 @@ public class StatisticsTracker
     = new ConcurrentHashMap<String, AtomicLong>();
     
     /** Keep track of URL counts per host per seed */
+    // TODO: restore spill-to-disk, like with processedSeedsRecords
     protected ConcurrentHashMap<String, ConcurrentMap<String, AtomicLong>> sourceHostDistribution = 
         new ConcurrentHashMap<String, ConcurrentMap<String,AtomicLong>>(); 
 
@@ -290,8 +304,6 @@ public class StatisticsTracker
         
     }
     
-    boolean isRunning = false;
-
     protected List<Report> reports;
     
     public List<Report> getReports() {
@@ -317,6 +329,7 @@ public class StatisticsTracker
         this.reports = reports;
     }
 
+    boolean isRunning = false;
     public boolean isRunning() {
         return isRunning;
     }
@@ -748,7 +761,7 @@ public class StatisticsTracker
         saveHostStats(sc.getHostFor(curi.getUURI()).getHostName(),
                 curi.getContentSize());
         
-        if (curi.getData().containsKey(A_SOURCE_TAG)) {
+        if (getTrackSources() && curi.getData().containsKey(A_SOURCE_TAG)) {
         	saveSourceStats((String)curi.getData().get(A_SOURCE_TAG),
                         sc.getHostFor(curi.getUURI()).
                     getHostName());