lee/heritrix3
1
0
Fork 0
mirror of https://github.com/internetarchive/heritrix3.git synced 2026-09-25 23:25:44 +00:00
Code Issues Packages Projects Releases Wiki Activity

* WARCConstants.java

Browse Source 
remove rfc6648-deprecated "X-" prefix to "WARC-Reference-Location", and document the field
* WARCWriterProcessor.java
    writeRevisitUriAgnosticDigest() - add some comments
This commit is contained in:
Noah Levitt
2012-09-11 18:21:20 -07:00
parent 1048536a25
commit a8aae91167
2 changed files with 22 additions and 1 deletions
Download Patch File Download Diff File Expand all files Collapse all files
commons/src/main/java/org/archive/io/warc/WARCConstants.java
+15 -1
View File
@@ -20,6 +20,7 @@
package org.archive.io.warc;
import org.archive.io.ArchiveFileConstants;
import org.archive.modules.writer.WARCWriterProcessor;
/**
* WARC Constants used by WARC readers and writers.
@@ -194,7 +195,20 @@ public interface WARCConstants extends ArchiveFileConstants {
public static final String HEADER_KEY_ETAG = "WARC-Etag";
public static final String HEADER_KEY_LAST_MODIFIED = "WARC-Last-Modified";
public static final String HEADER_KEY_REFERS_TO = "WARC-Refers-To";
public static final String HEADER_KEY_REFERENCE_LOCATION = "X-WARC-Reference-Location";
/**
* Information that helps to find the record that
* {@value #HEADER_KEY_REFERS_TO} refers to. This header is a heritrix
* extension not defined in the WARC spec version 1.0. Current
* implementation writes a JSON string that looks something like this:
* {"original-url":"http://archive.org/robots.txt"
* ,"warc-file-offset":1976,"warc-filename":
* "WEB-20120912001108855-00000-6882~desktop-nlevitt.sf.archive.org~6440.warc.gz"
* ,"content-written-date":"2012-09-12T00:11:10Z"}.
*
* @see WARCWriterProcessor
*/
public static final String HEADER_KEY_REFERENCE_LOCATION = "WARC-Reference-Location";
public static final String PROFILE_REVISIT_IDENTICAL_DIGEST =
"http://netpreserve.org/warc/1.0/revisit/identical-payload-digest";
modules/src/main/java/org/archive/modules/writer/WARCWriterProcessor.java
+7
View File
@@ -692,9 +692,16 @@ public class WARCWriterProcessor extends WriterPoolProcessor implements WARCWrit
headers.addLabelValue(
HEADER_KEY_TRUNCATED, NAMED_FIELD_TRUNCATED_VALUE_LENGTH);
/*
* ISO 28500 WARC ISO standard draft says: "The WARC-Refers-To field may
* also be used to associate a record of type 'revisit' or 'conversion'
* with the preceding record which helped determine the present record
* content."
*/
headers.addLabelValue(HEADER_KEY_REFERS_TO,
curi.getContentDigestHistory().get(A_WARC_RECORD_ID).toString());
// {"original-url":"http://archive.org/robots.txt","warc-file-offset":1976,"warc-filename":"WEB-20120912001108855-00000-6882~desktop-nlevitt.sf.archive.org~6440.warc.gz","content-written-date":"2012-09-12T00:11:10Z"}
@SuppressWarnings("unchecked")
JSONObject refLoc = new JSONObject((HashMap<String,Object>) curi.getContentDigestHistory().clone());
refLoc.remove(A_CONTENT_DIGEST_COUNT);