From 0eac126f7dc4b2cff1e84f93c16f395c669fc3a2 Mon Sep 17 00:00:00 2001 From: Noah Levitt Date: Thu, 13 Dec 2012 14:11:14 -0800 Subject: [PATCH] Special WARC-Profile for uri-agnostic revisit records. See HER-2022, http://tech.groups.yahoo.com/group/archive-crawler/message/7806 --- commons/src/main/java/org/archive/io/warc/WARCConstants.java | 2 ++ .../java/org/archive/modules/writer/WARCWriterProcessor.java | 3 ++- .../org/archive/modules/recrawl/ContentDigestHistoryTest.java | 4 ++-- 3 files changed, 6 insertions(+), 3 deletions(-) diff --git a/commons/src/main/java/org/archive/io/warc/WARCConstants.java b/commons/src/main/java/org/archive/io/warc/WARCConstants.java index 862f8a16..cd31ecf1 100644 --- a/commons/src/main/java/org/archive/io/warc/WARCConstants.java +++ b/commons/src/main/java/org/archive/io/warc/WARCConstants.java @@ -206,6 +206,8 @@ public interface WARCConstants extends ArchiveFileConstants { public static final String HEADER_KEY_REFERS_TO_FILENAME = "WARC-Refers-To-Filename"; public static final String HEADER_KEY_REFERS_TO_FILE_OFFSET = "WARC-Refers-To-File-Offset"; + public static final String PROFILE_REVISIT_URI_AGNOSTIC_IDENTICAL_DIGEST = + "http://netpreserve.org/warc/1.0/revisit/uri-agnostic-identical-payload-digest"; public static final String PROFILE_REVISIT_IDENTICAL_DIGEST = "http://netpreserve.org/warc/1.0/revisit/identical-payload-digest"; public static final String PROFILE_REVISIT_NOT_MODIFIED = diff --git a/modules/src/main/java/org/archive/modules/writer/WARCWriterProcessor.java b/modules/src/main/java/org/archive/modules/writer/WARCWriterProcessor.java index ed54c7e9..a730947d 100644 --- a/modules/src/main/java/org/archive/modules/writer/WARCWriterProcessor.java +++ b/modules/src/main/java/org/archive/modules/writer/WARCWriterProcessor.java @@ -39,6 +39,7 @@ import static org.archive.io.warc.WARCConstants.NAMED_FIELD_TRUNCATED_VALUE_LENG import static org.archive.io.warc.WARCConstants.NAMED_FIELD_TRUNCATED_VALUE_TIME; import static org.archive.io.warc.WARCConstants.PROFILE_REVISIT_IDENTICAL_DIGEST; import static org.archive.io.warc.WARCConstants.PROFILE_REVISIT_NOT_MODIFIED; +import static org.archive.io.warc.WARCConstants.PROFILE_REVISIT_URI_AGNOSTIC_IDENTICAL_DIGEST; import static org.archive.io.warc.WARCConstants.TYPE; import static org.archive.modules.CoreAttributeConstants.A_DNS_SERVER_IP_LABEL; import static org.archive.modules.CoreAttributeConstants.A_FTP_CONTROL_CONVERSATION; @@ -697,7 +698,7 @@ public class WARCWriterProcessor extends WriterPoolProcessor implements WARCWrit recordInfo.setContentLength(revisedLength); headers.addLabelValue( - HEADER_KEY_PROFILE, PROFILE_REVISIT_IDENTICAL_DIGEST); + HEADER_KEY_PROFILE, PROFILE_REVISIT_URI_AGNOSTIC_IDENTICAL_DIGEST); headers.addLabelValue( HEADER_KEY_TRUNCATED, NAMED_FIELD_TRUNCATED_VALUE_LENGTH); diff --git a/modules/src/test/java/org/archive/modules/recrawl/ContentDigestHistoryTest.java b/modules/src/test/java/org/archive/modules/recrawl/ContentDigestHistoryTest.java index a6dd1ac0..9fde90e0 100644 --- a/modules/src/test/java/org/archive/modules/recrawl/ContentDigestHistoryTest.java +++ b/modules/src/test/java/org/archive/modules/recrawl/ContentDigestHistoryTest.java @@ -34,7 +34,7 @@ import static org.archive.io.warc.WARCConstants.HEADER_KEY_TYPE; import static org.archive.io.warc.WARCConstants.HEADER_KEY_URI; import static org.archive.io.warc.WARCConstants.HTTP_RESPONSE_MIMETYPE; import static org.archive.io.warc.WARCConstants.NAMED_FIELD_TRUNCATED_VALUE_LENGTH; -import static org.archive.io.warc.WARCConstants.PROFILE_REVISIT_IDENTICAL_DIGEST; +import static org.archive.io.warc.WARCConstants.PROFILE_REVISIT_URI_AGNOSTIC_IDENTICAL_DIGEST; import static org.archive.modules.recrawl.RecrawlAttributeConstants.A_CONTENT_DIGEST_COUNT; import static org.archive.modules.recrawl.RecrawlAttributeConstants.A_ORIGINAL_DATE; import static org.archive.modules.recrawl.RecrawlAttributeConstants.A_ORIGINAL_URL; @@ -309,7 +309,7 @@ public class ContentDigestHistoryTest extends TmpDirTestCase { assertEquals(NAMED_FIELD_TRUNCATED_VALUE_LENGTH, record.getHeader().getHeaderValue(HEADER_KEY_TRUNCATED)); assertEquals(HTTP_RESPONSE_MIMETYPE, record.getHeader().getHeaderValue(CONTENT_TYPE)); assertEquals(expectedDigest, record.getHeader().getHeaderValue(HEADER_KEY_PAYLOAD_DIGEST)); - assertEquals(PROFILE_REVISIT_IDENTICAL_DIGEST, + assertEquals(PROFILE_REVISIT_URI_AGNOSTIC_IDENTICAL_DIGEST, record.getHeader().getHeaderValue(HEADER_KEY_PROFILE)); assertEquals(curi1.getUURI().toString(), record.getHeader().getHeaderValue(HEADER_KEY_REFERS_TO_TARGET_URI)); assertEquals(historyStore().store.get(expectedDigest).get(A_ORIGINAL_DATE),