From fe1df5883fedd77ae807ed90c46cdc366d057753 Mon Sep 17 00:00:00 2001 From: Noah Levitt Date: Tue, 25 Sep 2012 14:35:26 -0700 Subject: [PATCH] Some tweaks on maintaining content digest history * CrawlURI.java utility method hasContentDigestHistory() * BdbContentDigestHistory.java load() - make sure to call curi.getContentDigestHistory() in all cases so the value is initialized and WARCWriterProcessor knows it should put the info in there * WARCWriterProcessor.java updateMetadataAfterWrite() - update curi.getContentDigestHistory() only if curi.hasContentDigestHistory() for efficiency, like old uri-based fetch history; update the count after writing a revisit record --- .../src/main/java/org/archive/modules/CrawlURI.java | 4 ++++ .../modules/recrawl/BdbContentDigestHistory.java | 7 ++++++- .../archive/modules/writer/WARCWriterProcessor.java | 12 +++++++++--- 3 files changed, 19 insertions(+), 4 deletions(-) diff --git a/modules/src/main/java/org/archive/modules/CrawlURI.java b/modules/src/main/java/org/archive/modules/CrawlURI.java index 4525a978..93e591a6 100644 --- a/modules/src/main/java/org/archive/modules/CrawlURI.java +++ b/modules/src/main/java/org/archive/modules/CrawlURI.java @@ -1898,4 +1898,8 @@ implements Reporter, Serializable, OverlayContext { return contentDigestHistory; } + public boolean hasContentDigestHistory() { + return getData().get(A_CONTENT_DIGEST_HISTORY) != null; + } + } diff --git a/modules/src/main/java/org/archive/modules/recrawl/BdbContentDigestHistory.java b/modules/src/main/java/org/archive/modules/recrawl/BdbContentDigestHistory.java index 81fc9f35..c8de30e0 100644 --- a/modules/src/main/java/org/archive/modules/recrawl/BdbContentDigestHistory.java +++ b/modules/src/main/java/org/archive/modules/recrawl/BdbContentDigestHistory.java @@ -18,6 +18,7 @@ */ package org.archive.modules.recrawl; +import java.util.HashMap; import java.util.Map; import java.util.logging.Level; import java.util.logging.Logger; @@ -106,6 +107,10 @@ public class BdbContentDigestHistory extends AbstractContentDigestHistory implem } public void load(CrawlURI curi) { + // make this call in all cases so that the value is initialized and + // WARCWriterProcessor knows it should put the info in there + HashMap contentDigestHistory = curi.getContentDigestHistory(); + @SuppressWarnings("unchecked") Map loadedHistory = store.get(persistKeyFor(curi)); if (loadedHistory != null) { @@ -113,7 +118,7 @@ public class BdbContentDigestHistory extends AbstractContentDigestHistory implem logger.finer("loaded history by digest " + persistKeyFor(curi) + " for uri " + curi + " - " + loadedHistory); } - curi.getContentDigestHistory().putAll(loadedHistory); + contentDigestHistory.putAll(loadedHistory); } } diff --git a/modules/src/main/java/org/archive/modules/writer/WARCWriterProcessor.java b/modules/src/main/java/org/archive/modules/writer/WARCWriterProcessor.java index acd229fe..d432609f 100644 --- a/modules/src/main/java/org/archive/modules/writer/WARCWriterProcessor.java +++ b/modules/src/main/java/org/archive/modules/writer/WARCWriterProcessor.java @@ -315,7 +315,7 @@ public class WARCWriterProcessor extends WriterPoolProcessor implements WARCWrit } // history for uri-agnostic, content digest based dedupe - if (curi.getContentDigest() != null) { + if (curi.getContentDigest() != null && curi.hasContentDigestHistory()) { for (WARCRecordInfo warcRecord: writer.getTmpRecordLog()) { if ((warcRecord.getType() == WARCRecordType.RESPONSE || warcRecord.getType() == WARCRecordType.RESOURCE) @@ -327,8 +327,14 @@ public class WARCWriterProcessor extends WriterPoolProcessor implements WARCWrit curi.getContentDigestHistory().put(A_WARC_FILE_OFFSET, warcRecord.getWARCFileOffset()); curi.getContentDigestHistory().put(A_ORIGINAL_DATE, warcRecord.getCreate14DigitDate()); curi.getContentDigestHistory().put(A_CONTENT_DIGEST_COUNT, 1); - // } else if (warcRecord.getType() == WARCRecordType.REVISIT) { - // XXX add to content-digest-count IF it's a content digest based revisit record + } else if (warcRecord.getType() == WARCRecordType.REVISIT + && curi.getAnnotations().contains("warcRevisit:uriAgnosticDigest")) { + Integer oldCount = (Integer) curi.getContentDigestHistory().get(A_CONTENT_DIGEST_COUNT); + if (oldCount == null) { + // shouldn't happen, log a warning? + oldCount = 1; + } + curi.getContentDigestHistory().put(A_CONTENT_DIGEST_COUNT, oldCount + 1); } } }