From d5693563f8cb3d8d90077610cbbf5a24b2f84175 Mon Sep 17 00:00:00 2001
From: Noah Levitt <nlevitt@archive.org>
Date: Mon, 29 Apr 2013 18:11:24 -0700
Subject: [PATCH] fix bug pointed out in
 https://webarchive.jira.com/browse/ARI-3176?focusedCommentId=35314 -
 url-agnostic dedupe not reported in host report (and other reports)

---
 .../main/java/org/archive/modules/fetcher/FetchStats.java | 8 +++++++-
 1 file changed, 7 insertions(+), 1 deletion(-)

diff --git a/modules/src/main/java/org/archive/modules/fetcher/FetchStats.java b/modules/src/main/java/org/archive/modules/fetcher/FetchStats.java
index a9b53534..729aea43 100644
--- a/modules/src/main/java/org/archive/modules/fetcher/FetchStats.java
+++ b/modules/src/main/java/org/archive/modules/fetcher/FetchStats.java
@@ -72,6 +72,9 @@ public class FetchStats implements Serializable, FetchStatusCodes, Reporter {
     
     protected long lastSuccessTime; 
     
+    /*
+     * XXX redundancy with StatisticsTracker.onApplicationEvent() ... CrawledBytesHistotable.accumulate() code path
+     */
     public synchronized void tally(CrawlURI curi, Stage stage) {
         switch(stage) {
             case SCHEDULED:
@@ -91,7 +94,10 @@ public class FetchStats implements Serializable, FetchStatusCodes, Reporter {
                 if (curi.getFetchStatus() == HttpStatus.SC_NOT_MODIFIED) {
                     notModifiedBytes += curi.getContentSize();
                     notModifiedUrls++;
-                } else if (IdenticalDigestDecideRule.hasIdenticalDigest(curi)){
+                } else if (IdenticalDigestDecideRule.hasIdenticalDigest(curi)) {
+                    dupByHashBytes += curi.getContentSize();
+                    dupByHashUrls++;
+                } else if (curi.getAnnotations().contains("warcRevisit:uriAgnosticDigest")) {
                     dupByHashBytes += curi.getContentSize();
                     dupByHashUrls++;
                 } else {