diff --git a/engine/src/main/java/org/archive/crawler/util/CrawledBytesHistotable.java b/engine/src/main/java/org/archive/crawler/util/CrawledBytesHistotable.java index 36dd7579..dcc800e7 100644 --- a/engine/src/main/java/org/archive/crawler/util/CrawledBytesHistotable.java +++ b/engine/src/main/java/org/archive/crawler/util/CrawledBytesHistotable.java @@ -48,6 +48,9 @@ implements CoreAttributeConstants { } else if (IdenticalDigestDecideRule.hasIdenticalDigest(curi)) { tally(DUPLICATE,curi.getContentSize()); tally(DUPLICATECOUNT,1); + } else if (curi.getAnnotations().contains("warcRevisit:uriAgnosticDigest")) { + tally(DUPLICATE,curi.getContentSize()); + tally(DUPLICATECOUNT,1); } else { tally(NOVEL,curi.getContentSize()); tally(NOVELCOUNT,1); diff --git a/modules/src/test/java/org/archive/modules/recrawl/ContentDigestHistoryTest.java b/modules/src/test/java/org/archive/modules/recrawl/ContentDigestHistoryTest.java index b543e9a0..0c01aa0f 100644 --- a/modules/src/test/java/org/archive/modules/recrawl/ContentDigestHistoryTest.java +++ b/modules/src/test/java/org/archive/modules/recrawl/ContentDigestHistoryTest.java @@ -249,6 +249,7 @@ public class ContentDigestHistoryTest extends TmpDirTestCase { assertEquals(1, curi2.getContentDigestHistory().get(A_CONTENT_DIGEST_COUNT)); warcWriter.process(curi2); + assertTrue(curi2.getAnnotations().contains("warcRevisit:uriAgnosticDigest")); assertEquals(curi1.getUURI().toString(), curi2.getContentDigestHistory().get(A_ORIGINAL_URL)); assertNotSame(curi2.getUURI().toString(), curi2.getContentDigestHistory().get(A_ORIGINAL_URL)); assertEquals(2, curi2.getContentDigestHistory().get(A_CONTENT_DIGEST_COUNT));