lee/heritrix3
1
0
Fork 0
mirror of https://github.com/internetarchive/heritrix3.git synced 2026-08-13 10:11:10 +00:00
Code Issues Packages Projects Releases Wiki Activity

Count url-agnostic content digest duplicates as duplicates in statistics

Browse Source 
* CrawledBytesHistotable.java
    accumulate() - count urls with warcRevisit:uriAgnosticDigest annotation as duplicates
* ContentDigestHistoryTest.java
    testWarcDedupe() - check for presence of warcRevisit:uriAgnosticDigest annotation
This commit is contained in:
Noah Levitt
2012-09-27 16:56:14 -07:00
parent a1d135edbc
commit 4470ba512e
2 changed files with 4 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files
modules/src/test/java/org/archive/modules/recrawl/ContentDigestHistoryTest.java
+1
View File
@@ -249,6 +249,7 @@ public class ContentDigestHistoryTest extends TmpDirTestCase {
assertEquals(1, curi2.getContentDigestHistory().get(A_CONTENT_DIGEST_COUNT));
warcWriter.process(curi2);
assertTrue(curi2.getAnnotations().contains("warcRevisit:uriAgnosticDigest"));
assertEquals(curi1.getUURI().toString(), curi2.getContentDigestHistory().get(A_ORIGINAL_URL));
assertNotSame(curi2.getUURI().toString(), curi2.getContentDigestHistory().get(A_ORIGINAL_URL));
assertEquals(2, curi2.getContentDigestHistory().get(A_CONTENT_DIGEST_COUNT));