lee/heritrix3
1
0
Fork 0
mirror of https://github.com/internetarchive/heritrix3.git synced 2026-07-19 22:17:13 +00:00
Code Issues Packages Projects Releases Wiki Activity

Count url-agnostic content digest duplicates as duplicates in statistics

Browse Source 
* CrawledBytesHistotable.java
    accumulate() - count urls with warcRevisit:uriAgnosticDigest annotation as duplicates
* ContentDigestHistoryTest.java
    testWarcDedupe() - check for presence of warcRevisit:uriAgnosticDigest annotation
This commit is contained in:
Noah Levitt
2012-09-27 16:56:14 -07:00
parent a1d135edbc
commit 4470ba512e
2 changed files with 4 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files
engine/src/main/java/org/archive/crawler/util/CrawledBytesHistotable.java
+3
View File
@@ -48,6 +48,9 @@ implements CoreAttributeConstants {
} else if (IdenticalDigestDecideRule.hasIdenticalDigest(curi)) {
tally(DUPLICATE,curi.getContentSize());
tally(DUPLICATECOUNT,1);
} else if (curi.getAnnotations().contains("warcRevisit:uriAgnosticDigest")) {
tally(DUPLICATE,curi.getContentSize());
tally(DUPLICATECOUNT,1);
} else {
tally(NOVEL,curi.getContentSize());
tally(NOVELCOUNT,1);