lee/heritrix3
1
0
Fork 0
mirror of https://github.com/internetarchive/heritrix3.git synced 2026-09-26 07:35:39 +00:00
Code Issues Packages Projects Releases Wiki Activity

Special WARC-Profile for uri-agnostic revisit records. See HER-2022, http://tech.groups.yahoo.com/group/archive-crawler/message/7806

Browse Source
This commit is contained in:
Noah Levitt
2012-12-13 14:11:14 -08:00
parent 5586d5452d
commit 0eac126f7d
3 changed files with 6 additions and 3 deletions
Download Patch File Download Diff File Expand all files Collapse all files
commons/src/main/java/org/archive/io/warc/WARCConstants.java
+2
View File
@@ -206,6 +206,8 @@ public interface WARCConstants extends ArchiveFileConstants {
public static final String HEADER_KEY_REFERS_TO_FILENAME = "WARC-Refers-To-Filename";
public static final String HEADER_KEY_REFERS_TO_FILE_OFFSET = "WARC-Refers-To-File-Offset";
public static final String PROFILE_REVISIT_URI_AGNOSTIC_IDENTICAL_DIGEST =
"http://netpreserve.org/warc/1.0/revisit/uri-agnostic-identical-payload-digest";
public static final String PROFILE_REVISIT_IDENTICAL_DIGEST =
"http://netpreserve.org/warc/1.0/revisit/identical-payload-digest";
public static final String PROFILE_REVISIT_NOT_MODIFIED =
modules/src/main/java/org/archive/modules/writer/WARCWriterProcessor.java
+2 -1
View File
@@ -39,6 +39,7 @@ import static org.archive.io.warc.WARCConstants.NAMED_FIELD_TRUNCATED_VALUE_LENG
import static org.archive.io.warc.WARCConstants.NAMED_FIELD_TRUNCATED_VALUE_TIME;
import static org.archive.io.warc.WARCConstants.PROFILE_REVISIT_IDENTICAL_DIGEST;
import static org.archive.io.warc.WARCConstants.PROFILE_REVISIT_NOT_MODIFIED;
import static org.archive.io.warc.WARCConstants.PROFILE_REVISIT_URI_AGNOSTIC_IDENTICAL_DIGEST;
import static org.archive.io.warc.WARCConstants.TYPE;
import static org.archive.modules.CoreAttributeConstants.A_DNS_SERVER_IP_LABEL;
import static org.archive.modules.CoreAttributeConstants.A_FTP_CONTROL_CONVERSATION;
@@ -697,7 +698,7 @@ public class WARCWriterProcessor extends WriterPoolProcessor implements WARCWrit
recordInfo.setContentLength(revisedLength);
headers.addLabelValue(
HEADER_KEY_PROFILE, PROFILE_REVISIT_IDENTICAL_DIGEST);
HEADER_KEY_PROFILE, PROFILE_REVISIT_URI_AGNOSTIC_IDENTICAL_DIGEST);
headers.addLabelValue(
HEADER_KEY_TRUNCATED, NAMED_FIELD_TRUNCATED_VALUE_LENGTH);
modules/src/test/java/org/archive/modules/recrawl/ContentDigestHistoryTest.java
+2 -2
View File
@@ -34,7 +34,7 @@ import static org.archive.io.warc.WARCConstants.HEADER_KEY_TYPE;
import static org.archive.io.warc.WARCConstants.HEADER_KEY_URI;
import static org.archive.io.warc.WARCConstants.HTTP_RESPONSE_MIMETYPE;
import static org.archive.io.warc.WARCConstants.NAMED_FIELD_TRUNCATED_VALUE_LENGTH;
import static org.archive.io.warc.WARCConstants.PROFILE_REVISIT_IDENTICAL_DIGEST;
import static org.archive.io.warc.WARCConstants.PROFILE_REVISIT_URI_AGNOSTIC_IDENTICAL_DIGEST;
import static org.archive.modules.recrawl.RecrawlAttributeConstants.A_CONTENT_DIGEST_COUNT;
import static org.archive.modules.recrawl.RecrawlAttributeConstants.A_ORIGINAL_DATE;
import static org.archive.modules.recrawl.RecrawlAttributeConstants.A_ORIGINAL_URL;
@@ -309,7 +309,7 @@ public class ContentDigestHistoryTest extends TmpDirTestCase {
assertEquals(NAMED_FIELD_TRUNCATED_VALUE_LENGTH, record.getHeader().getHeaderValue(HEADER_KEY_TRUNCATED));
assertEquals(HTTP_RESPONSE_MIMETYPE, record.getHeader().getHeaderValue(CONTENT_TYPE));
assertEquals(expectedDigest, record.getHeader().getHeaderValue(HEADER_KEY_PAYLOAD_DIGEST));
assertEquals(PROFILE_REVISIT_IDENTICAL_DIGEST,
assertEquals(PROFILE_REVISIT_URI_AGNOSTIC_IDENTICAL_DIGEST,
record.getHeader().getHeaderValue(HEADER_KEY_PROFILE));
assertEquals(curi1.getUURI().toString(), record.getHeader().getHeaderValue(HEADER_KEY_REFERS_TO_TARGET_URI));
assertEquals(historyStore().store.get(expectedDigest).get(A_ORIGINAL_DATE),