lee/heritrix3
1
0 Fork 0
mirror of https://github.com/internetarchive/heritrix3.git synced 2026-11-04 04:53:39 +00:00
Code Issues Packages Projects Releases Wiki Activity

Count url-agnostic content digest duplicates as duplicates in statistics

Browse Source 
* CrawledBytesHistotable.java
    accumulate() - count urls with warcRevisit:uriAgnosticDigest annotation as duplicates
* ContentDigestHistoryTest.java
    testWarcDedupe() - check for presence of warcRevisit:uriAgnosticDigest annotation
This commit is contained in:
Noah Levitt committed 2012-09-27 16:56:14 -07:00
1 parent a1d135edbc
commit 4470ba512e
2 files changed
+4
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

engine/src/main/java/org/archive/crawler/util/CrawledBytesHistotable.java
+3
View File
@@ -48,6 +48,9 @@ implements CoreAttributeConstants {
} else if (IdenticalDigestDecideRule.hasIdenticalDigest(curi)) {
tally(DUPLICATE,curi.getContentSize());
tally(DUPLICATECOUNT,1);
} else if (curi.getAnnotations().contains("warcRevisit:uriAgnosticDigest")) {
tally(DUPLICATE,curi.getContentSize());
tally(DUPLICATECOUNT,1);
} else {
tally(NOVEL,curi.getContentSize());
tally(NOVELCOUNT,1);