diff --git a/engine/src/main/java/org/archive/crawler/reporting/StatisticsTracker.java b/engine/src/main/java/org/archive/crawler/reporting/StatisticsTracker.java index 5f4b6332..6d56a825 100644 --- a/engine/src/main/java/org/archive/crawler/reporting/StatisticsTracker.java +++ b/engine/src/main/java/org/archive/crawler/reporting/StatisticsTracker.java @@ -201,6 +201,19 @@ public class StatisticsTracker public void setTrackSeeds(boolean trackSeeds) { this.trackSeeds = trackSeeds; } + + /** + * Whether to maintain hosts-per-source-tag records for; very expensive in + * crawls with large numbers of source-tags (seeds) or large crawls + * over many hosts + */ + boolean trackSources = true; + public boolean getTrackSources() { + return this.trackSources; + } + public void setTrackSources(boolean trackSources) { + this.trackSources = trackSources; + } /** * The interval between writing progress information to log. @@ -270,6 +283,7 @@ public class StatisticsTracker = new ConcurrentHashMap(); /** Keep track of URL counts per host per seed */ + // TODO: restore spill-to-disk, like with processedSeedsRecords protected ConcurrentHashMap> sourceHostDistribution = new ConcurrentHashMap>(); @@ -290,8 +304,6 @@ public class StatisticsTracker } - boolean isRunning = false; - protected List reports; public List getReports() { @@ -317,6 +329,7 @@ public class StatisticsTracker this.reports = reports; } + boolean isRunning = false; public boolean isRunning() { return isRunning; } @@ -748,7 +761,7 @@ public class StatisticsTracker saveHostStats(sc.getHostFor(curi.getUURI()).getHostName(), curi.getContentSize()); - if (curi.getData().containsKey(A_SOURCE_TAG)) { + if (getTrackSources() && curi.getData().containsKey(A_SOURCE_TAG)) { saveSourceStats((String)curi.getData().get(A_SOURCE_TAG), sc.getHostFor(curi.getUURI()). getHostName());