From 3f8590fe44d0d033cb71bfa8a4d21bde08ac8b8c Mon Sep 17 00:00:00 2001 From: gojomo Date: Mon, 11 Mar 2013 20:57:41 -0700 Subject: [PATCH] when found in CrawlURI, add WARC-Simple-Form-Province-Status header --- .../archive/modules/writer/WARCWriterProcessor.java | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/modules/src/main/java/org/archive/modules/writer/WARCWriterProcessor.java b/modules/src/main/java/org/archive/modules/writer/WARCWriterProcessor.java index ed54c7e9..14a5406e 100644 --- a/modules/src/main/java/org/archive/modules/writer/WARCWriterProcessor.java +++ b/modules/src/main/java/org/archive/modules/writer/WARCWriterProcessor.java @@ -47,6 +47,7 @@ import static org.archive.modules.CoreAttributeConstants.A_SOURCE_TAG; import static org.archive.modules.CoreAttributeConstants.HEADER_TRUNC; import static org.archive.modules.CoreAttributeConstants.LENGTH_TRUNC; import static org.archive.modules.CoreAttributeConstants.TIMER_TRUNC; +import static org.archive.modules.CoreAttributeConstants.A_WARC_RESPONSE_HEADERS; import static org.archive.modules.recrawl.RecrawlAttributeConstants.A_CONTENT_DIGEST_COUNT; import static org.archive.modules.recrawl.RecrawlAttributeConstants.A_ETAG_HEADER; import static org.archive.modules.recrawl.RecrawlAttributeConstants.A_FETCH_HISTORY; @@ -430,6 +431,7 @@ public class WARCWriterProcessor extends WriterPoolProcessor implements WARCWrit curi.getContentDigestSchemeString()); } headers.addLabelValue(HEADER_KEY_IP, getHostAddress(curi)); + URI rid; if (getWriteRevisitForIdenticalDigests() @@ -583,8 +585,17 @@ public class WARCWriterProcessor extends WriterPoolProcessor implements WARCWrit protected URI writeResponse(final WARCWriter w, final String timestamp, final String mimetype, final URI baseid, final CrawlURI curi, - final ANVLRecord namedFields) + final ANVLRecord suppliedFields) throws IOException { + ANVLRecord namedFields = suppliedFields; + if(curi.getData().containsKey(A_WARC_RESPONSE_HEADERS)) { + namedFields = namedFields.clone(); + for (Object headerObj : curi.getDataList(A_WARC_RESPONSE_HEADERS)) { + String[] kv = StringUtils.split(((String)headerObj),":",2); + namedFields.addLabelValue(kv[0].trim(), kv[1].trim()); + } + } + WARCRecordInfo recordInfo = new WARCRecordInfo(); recordInfo.setType(WARCRecordType.RESPONSE); recordInfo.setUrl(curi.toString());