lee/heritrix3
1
0
Fork 0
mirror of https://github.com/internetarchive/heritrix3.git synced 2026-08-22 06:26:32 +00:00
Code Issues Packages Projects Releases Wiki Activity

Some tweaks on maintaining content digest history

Browse Source 
* CrawlURI.java
    utility method hasContentDigestHistory()
* BdbContentDigestHistory.java
    load() - make sure to call curi.getContentDigestHistory() in all cases so the value is initialized and WARCWriterProcessor knows it should put the info in there
* WARCWriterProcessor.java
    updateMetadataAfterWrite() - update curi.getContentDigestHistory() only if curi.hasContentDigestHistory() for efficiency, like old uri-based fetch history; update the count after writing a revisit record
This commit is contained in:
Noah Levitt
2012-09-25 14:35:26 -07:00
parent 25a9155298
commit fe1df5883f
3 changed files with 19 additions and 4 deletions
Download Patch File Download Diff File Expand all files Collapse all files
modules/src/main/java/org/archive/modules/CrawlURI.java
+4
View File
@@ -1898,4 +1898,8 @@ implements Reporter, Serializable, OverlayContext {
return contentDigestHistory;
}
public boolean hasContentDigestHistory() {
return getData().get(A_CONTENT_DIGEST_HISTORY) != null;
}
}
modules/src/main/java/org/archive/modules/recrawl/BdbContentDigestHistory.java
+6 -1
View File
@@ -18,6 +18,7 @@
*/
package org.archive.modules.recrawl;
import java.util.HashMap;
import java.util.Map;
import java.util.logging.Level;
import java.util.logging.Logger;
@@ -106,6 +107,10 @@ public class BdbContentDigestHistory extends AbstractContentDigestHistory implem
}
public void load(CrawlURI curi) {
// make this call in all cases so that the value is initialized and
// WARCWriterProcessor knows it should put the info in there
HashMap<String, Object> contentDigestHistory = curi.getContentDigestHistory();
@SuppressWarnings("unchecked")
Map<String, Object> loadedHistory = store.get(persistKeyFor(curi));
if (loadedHistory != null) {
@@ -113,7 +118,7 @@ public class BdbContentDigestHistory extends AbstractContentDigestHistory implem
logger.finer("loaded history by digest " + persistKeyFor(curi)
+ " for uri " + curi + " - " + loadedHistory);
}
curi.getContentDigestHistory().putAll(loadedHistory);
contentDigestHistory.putAll(loadedHistory);
}
}
modules/src/main/java/org/archive/modules/writer/WARCWriterProcessor.java
+9 -3
View File
@@ -315,7 +315,7 @@ public class WARCWriterProcessor extends WriterPoolProcessor implements WARCWrit
}
// history for uri-agnostic, content digest based dedupe
if (curi.getContentDigest() != null) {
if (curi.getContentDigest() != null && curi.hasContentDigestHistory()) {
for (WARCRecordInfo warcRecord: writer.getTmpRecordLog()) {
if ((warcRecord.getType() == WARCRecordType.RESPONSE
|| warcRecord.getType() == WARCRecordType.RESOURCE)
@@ -327,8 +327,14 @@ public class WARCWriterProcessor extends WriterPoolProcessor implements WARCWrit
curi.getContentDigestHistory().put(A_WARC_FILE_OFFSET, warcRecord.getWARCFileOffset());
curi.getContentDigestHistory().put(A_ORIGINAL_DATE, warcRecord.getCreate14DigitDate());
curi.getContentDigestHistory().put(A_CONTENT_DIGEST_COUNT, 1);
// } else if (warcRecord.getType() == WARCRecordType.REVISIT) {
// XXX add to content-digest-count IF it's a content digest based revisit record
} else if (warcRecord.getType() == WARCRecordType.REVISIT
&& curi.getAnnotations().contains("warcRevisit:uriAgnosticDigest")) {
Integer oldCount = (Integer) curi.getContentDigestHistory().get(A_CONTENT_DIGEST_COUNT);
if (oldCount == null) {
// shouldn't happen, log a warning?
oldCount = 1;
}
curi.getContentDigestHistory().put(A_CONTENT_DIGEST_COUNT, oldCount + 1);
}
}
}