lee/heritrix3
1
0
Fork 0
mirror of https://github.com/internetarchive/heritrix3.git synced 2026-09-16 18:55:35 +00:00
Code Issues Packages Projects Releases Wiki Activity

Stop automatically writing WARC-Payload-Digest for revisit records.

Browse Source 
Refer that functionality to the relevant revisit profile classes.
This commit is contained in:
Kristinn Sigurðsson
2015-03-11 13:52:31 +00:00
parent b5da93d04b
commit 3ffe5b8b4f
3 changed files with 24 additions and 6 deletions
Download Patch File Download Diff File Expand all files Collapse all files
modules/src/main/java/org/archive/modules/revisit/IdenticalPayloadDigestRevisit.java
+2 -2
View File
@@ -1,5 +1,6 @@
package org.archive.modules.revisit;
import static org.archive.format.warc.WARCConstants.HEADER_KEY_PAYLOAD_DIGEST;
import static org.archive.format.warc.WARCConstants.HEADER_KEY_REFERS_TO_TARGET_URI;
import java.util.Map;
@@ -32,8 +33,7 @@ public class IdenticalPayloadDigestRevisit extends AbstractProfile {
public Map<String, String> getWarcHeaders() {
Map<String, String> headers = super.getWarcHeaders();
// Written automatically by WarcWriterProcessor for all HTTP responses
// headers.put(HEADER_KEY_PAYLOAD_DIGEST, payloadDigest);
headers.put(HEADER_KEY_PAYLOAD_DIGEST, payloadDigest);
if (refersToTargetURI!=null) {
headers.put(HEADER_KEY_REFERS_TO_TARGET_URI, refersToTargetURI);
modules/src/main/java/org/archive/modules/revisit/ServerNotModifiedRevisit.java
+18
View File
@@ -2,6 +2,7 @@ package org.archive.modules.revisit;
import static org.archive.format.warc.WARCConstants.HEADER_KEY_ETAG;
import static org.archive.format.warc.WARCConstants.HEADER_KEY_LAST_MODIFIED;
import static org.archive.format.warc.WARCConstants.HEADER_KEY_PAYLOAD_DIGEST;
import java.util.Map;
@@ -13,6 +14,9 @@ public class ServerNotModifiedRevisit extends AbstractProfile {
protected String eTag;
protected String lastModified;
// Optional. Digest of original capture
protected String payloadDigest;
/**
* Minimal constructor.
*/
@@ -29,6 +33,10 @@ public class ServerNotModifiedRevisit extends AbstractProfile {
public Map<String, String> getWarcHeaders() {
Map<String, String> headers = super.getWarcHeaders();
if (payloadDigest!=null) {
headers.put(HEADER_KEY_PAYLOAD_DIGEST, payloadDigest);
}
if (eTag!=null) {
headers.put(HEADER_KEY_ETAG, eTag);
}
@@ -59,5 +67,15 @@ public class ServerNotModifiedRevisit extends AbstractProfile {
this.lastModified = lastModified;
}
public String getPayloadDigest() {
return payloadDigest;
}
public void setPayloadDigest(String payloadDigest) {
this.payloadDigest = payloadDigest;
}
}
modules/src/main/java/org/archive/modules/writer/WARCWriterProcessor.java
+4 -4
View File
@@ -392,10 +392,6 @@ public class WARCWriterProcessor extends WriterPoolProcessor implements WARCWrit
// TODO: Use other than ANVL (or rename ANVL as NameValue or
// use RFC822 (commons-httpclient?).
ANVLRecord headers = new ANVLRecord();
if (curi.getContentDigest() != null) {
headers.addLabelValue(HEADER_KEY_PAYLOAD_DIGEST,
curi.getContentDigestSchemeString());
}
headers.addLabelValue(HEADER_KEY_IP, getHostAddress(curi));
URI rid;
@@ -403,6 +399,10 @@ public class WARCWriterProcessor extends WriterPoolProcessor implements WARCWrit
if (curi.isRevisit()) {
rid = writeRevisit(w, timestamp, HTTP_RESPONSE_MIMETYPE, baseid, curi, headers);
} else {
if (curi.getContentDigest() != null) {
headers.addLabelValue(HEADER_KEY_PAYLOAD_DIGEST,
curi.getContentDigestSchemeString());
}
// Check for truncated annotation
String value = null;
Collection<String> anno = curi.getAnnotations();