From 71bffa42e61e2b325a1d2cd70a706dd39faa200e Mon Sep 17 00:00:00 2001 From: Leslie Bellony Date: Mon, 3 Aug 2026 14:25:53 +0200 Subject: [PATCH] Issue #748 extend SourceTags report with status code --- .../crawler/reporting/SourceTagsReport.java | 71 +++++++++++---- .../crawler/reporting/StatisticsTracker.java | 86 ++++++++++++++++--- 2 files changed, 126 insertions(+), 31 deletions(-) diff --git a/engine/src/main/java/org/archive/crawler/reporting/SourceTagsReport.java b/engine/src/main/java/org/archive/crawler/reporting/SourceTagsReport.java index 9547cd52..c5c1e692 100644 --- a/engine/src/main/java/org/archive/crawler/reporting/SourceTagsReport.java +++ b/engine/src/main/java/org/archive/crawler/reporting/SourceTagsReport.java @@ -21,6 +21,7 @@ package org.archive.crawler.reporting; import java.io.PrintWriter; import java.util.Map; import java.util.Set; +import java.util.concurrent.ConcurrentMap; import java.util.concurrent.atomic.AtomicLong; import org.archive.bdb.DisposableStoredSortedMap; @@ -32,6 +33,16 @@ import org.archive.bdb.DisposableStoredSortedMap; */ public class SourceTagsReport extends Report { + private boolean includeResCode = false; + + public boolean isIncludeResCode() { + return includeResCode; + } + + public void setIncludeResCode(boolean includeResCode) { + this.includeResCode = includeResCode; + } + @Override public void write(PrintWriter writer, StatisticsTracker stats) { @@ -41,25 +52,49 @@ public class SourceTagsReport extends Report { writer.println("No source tag information. (Is 'sourceTagSeeds' enabled?)"); return; } - - writer.print("[source] [host] [#urls]\n"); - // for each source - for (String sourceKey : sourceTags) { - Map hostCounts = - (Map)stats.sourceHostDistribution.get(sourceKey); - // sort hosts by #urls - DisposableStoredSortedMap sortedHostCounts = - stats.getReverseSortedHostCounts(hostCounts); - // for each host - for (Map.Entry entry : sortedHostCounts.entrySet()) { - writer.print(sourceKey.toString()); - writer.print(" "); - writer.print(entry.getValue()); - writer.print(" "); - writer.print(Math.abs(entry.getKey())); - writer.print("\n"); + + if (isIncludeResCode()) { + writer.print("[source] [host] [rescode] [#urls]\n"); + // for each source + for (String sourceKey : sourceTags) { + Map> hostCounts = stats.sourceHostStatusDistribution.get(sourceKey); + if (hostCounts != null) { + // For each sorted host + for (Map.Entry> entry : hostCounts.entrySet().stream().sorted(Map.Entry.comparingByKey()).toList()) { + Map map = entry.getValue(); + // Sort status code by #urls + DisposableStoredSortedMap sortedHostCounts = stats.getReverseSortedHostCounts(map); + for (Map.Entry entry2 : sortedHostCounts.entrySet()) { + writer.print(sourceKey.toString()); + writer.print(" "); + writer.print(entry.getKey()); + writer.print(" "); + writer.print(entry2.getValue()); + writer.print(" "); + writer.print(Math.abs(entry2.getKey())); + writer.print("\n"); + } + } + } + } + } else { + writer.print("[source] [host] [#urls]\n"); + // for each source + for (String sourceKey : sourceTags) { + Map hostCounts = (Map) stats.sourceHostDistribution.get(sourceKey); + // sort hosts by #urls + DisposableStoredSortedMap sortedHostCounts = stats.getReverseSortedHostCounts(hostCounts); + // for each host + for (Map.Entry entry : sortedHostCounts.entrySet()) { + writer.print(sourceKey.toString()); + writer.print(" "); + writer.print(entry.getValue()); + writer.print(" "); + writer.print(Math.abs(entry.getKey())); + writer.print("\n"); + } + sortedHostCounts.dispose(); } - sortedHostCounts.dispose(); } } diff --git a/engine/src/main/java/org/archive/crawler/reporting/StatisticsTracker.java b/engine/src/main/java/org/archive/crawler/reporting/StatisticsTracker.java index 2b47be10..a4e1b6dd 100644 --- a/engine/src/main/java/org/archive/crawler/reporting/StatisticsTracker.java +++ b/engine/src/main/java/org/archive/crawler/reporting/StatisticsTracker.java @@ -188,7 +188,7 @@ public class StatisticsTracker /** * Messages from the StatisticsTracker. */ - private final static Logger logger = + private static final Logger logger = Logger.getLogger(StatisticsTracker.class.getName()); /** @@ -288,6 +288,11 @@ public class StatisticsTracker protected ConcurrentHashMap> sourceHostDistribution = new ConcurrentHashMap>(); + /** Keep track of URL counts per status code per host per seed */ + // TODO: restore spill-to-disk, like with processedSeedsRecords + protected ConcurrentHashMap>> sourceHostStatusDistribution = + new ConcurrentHashMap<>(); + /** Keep track of crawled bytes stats per seed */ // TODO: spill-to-disk (requires bdb replacement for Histotable, or some // other refactoring) @@ -369,14 +374,33 @@ public class StatisticsTracker statusCodeDistribution, json.getJSONObject("statusCodeDistribution")); - - JSONObject shd = json.getJSONObject("sourceHostDistribution"); - Iterator keyIter = shd.keys(); - for(; keyIter.hasNext();) { - String source = keyIter.next(); - ConcurrentHashMap hostUriCount = new ConcurrentHashMap(); - JSONUtils.putAllAtomicLongs(hostUriCount,shd.getJSONObject(source)); - sourceHostDistribution.put(source, hostUriCount); + SourceTagsReport str = getReport(SourceTagsReport.class); + Iterator keyIter; + if (str.isIncludeResCode()) { + JSONObject shsd = json.getJSONObject("sourceHostStatusDistribution"); + keyIter = shsd.keys(); + for (; keyIter.hasNext();) { + String source = keyIter.next(); + ConcurrentHashMap> hostStatusUriCount = new ConcurrentHashMap<>(); + JSONObject hostMap = shsd.getJSONObject(source); + Iterator hostKeys = hostMap.keys(); + for (; hostKeys.hasNext();) { + String host = hostKeys.next(); + ConcurrentHashMap statusUriCount = new ConcurrentHashMap<>(); + JSONUtils.putAllAtomicLongs(statusUriCount, hostMap.getJSONObject(host)); + hostStatusUriCount.put(host, statusUriCount); + } + sourceHostStatusDistribution.put(source, hostStatusUriCount); + } + } else { + JSONObject shd = json.getJSONObject("sourceHostDistribution"); + keyIter = shd.keys(); + for (; keyIter.hasNext();) { + String source = keyIter.next(); + ConcurrentHashMap hostUriCount = new ConcurrentHashMap(); + JSONUtils.putAllAtomicLongs(hostUriCount, shd.getJSONObject(source)); + sourceHostDistribution.put(source, hostUriCount); + } } // optional so we can still recover checkpoints from earlier versions of heritrix @@ -740,6 +764,15 @@ public class StatisticsTracker } } // else ignore } + + public T getReport(Class reportClass) { + for (Report r : getReports()) { + if (reportClass.isInstance(r)) { + return reportClass.cast(r); + } + } + return null; + } public void crawledURISuccessful(CrawlURI curi) { handleSeed(curi,"Seed successfully crawled"); @@ -757,12 +790,33 @@ public class StatisticsTracker ServerCache sc = serverCache; if (getTrackSources() && curi.getData().containsKey(A_SOURCE_TAG)) { - saveSourceStats(curi.getSourceTag(), - sc.getHostFor(curi.getUURI()).getHostName()); - tallySourceStats(curi); + SourceTagsReport str = getReport(SourceTagsReport.class); + if (str.isIncludeResCode()) { + saveSourceStats(curi.getSourceTag(), sc.getHostFor(curi.getUURI()).getHostName(), String.valueOf(curi.getFetchStatus())); + } + saveSourceStats(curi.getSourceTag(), + sc.getHostFor(curi.getUURI()).getHostName()); + tallySourceStats(curi); } } + protected void saveSourceStats(String source, String hostname, String statuscode) { + ConcurrentMap> hostStatusUriCount = sourceHostStatusDistribution.get(source); + if (hostStatusUriCount == null) { + hostStatusUriCount = new ConcurrentHashMap<>(); + } + ConcurrentMap statusUriCount = hostStatusUriCount.get(hostname); + if (statusUriCount == null) { + statusUriCount = new ConcurrentHashMap<>(); + ConcurrentMap prevVal = hostStatusUriCount.putIfAbsent(hostname, statusUriCount); + if (prevVal != null) { + statusUriCount = prevVal; + } + } + incrementMapCount(statusUriCount, statuscode); + sourceHostStatusDistribution.putIfAbsent(source, hostStatusUriCount); + } + protected void saveSourceStats(String source, String hostname) { ConcurrentMap hostUriCount = sourceHostDistribution.get(source); if(hostUriCount == null) { @@ -1072,7 +1126,13 @@ public class StatisticsTracker json.put("mimeTypeBytes", mimeTypeBytes); json.put("statusCodeDistribution", statusCodeDistribution); - json.put("sourceHostDistribution", sourceHostDistribution); + SourceTagsReport str = getReport(SourceTagsReport.class); + if (str.isIncludeResCode()) { + json.put("sourceHostStatusDistribution", sourceHostStatusDistribution); + } else { + json.put("sourceHostDistribution", sourceHostDistribution); + } + json.put("statsBySource", statsBySource); json.put("crawledBytes", crawledBytes);