lee/heritrix3
1
0
Fork 0
mirror of https://github.com/internetarchive/heritrix3.git synced 2026-09-27 16:15:52 +00:00
Code Issues Packages Projects Releases Wiki Activity

when found in CrawlURI, add WARC-Simple-Form-Province-Status header

Browse Source
This commit is contained in:
gojomo
2013-03-11 20:57:41 -07:00
parent bf9cab8eb2
commit 3f8590fe44
1 changed files with 12 additions and 1 deletions
Download Patch File Download Diff File Expand all files Collapse all files
modules/src/main/java/org/archive/modules/writer/WARCWriterProcessor.java
+12 -1
View File
@@ -47,6 +47,7 @@ import static org.archive.modules.CoreAttributeConstants.A_SOURCE_TAG;
import static org.archive.modules.CoreAttributeConstants.HEADER_TRUNC;
import static org.archive.modules.CoreAttributeConstants.LENGTH_TRUNC;
import static org.archive.modules.CoreAttributeConstants.TIMER_TRUNC;
import static org.archive.modules.CoreAttributeConstants.A_WARC_RESPONSE_HEADERS;
import static org.archive.modules.recrawl.RecrawlAttributeConstants.A_CONTENT_DIGEST_COUNT;
import static org.archive.modules.recrawl.RecrawlAttributeConstants.A_ETAG_HEADER;
import static org.archive.modules.recrawl.RecrawlAttributeConstants.A_FETCH_HISTORY;
@@ -430,6 +431,7 @@ public class WARCWriterProcessor extends WriterPoolProcessor implements WARCWrit
curi.getContentDigestSchemeString());
}
headers.addLabelValue(HEADER_KEY_IP, getHostAddress(curi));
URI rid;
if (getWriteRevisitForIdenticalDigests()
@@ -583,8 +585,17 @@ public class WARCWriterProcessor extends WriterPoolProcessor implements WARCWrit
protected URI writeResponse(final WARCWriter w,
final String timestamp, final String mimetype,
final URI baseid, final CrawlURI curi,
final ANVLRecord namedFields)
final ANVLRecord suppliedFields)
throws IOException {
ANVLRecord namedFields = suppliedFields;
if(curi.getData().containsKey(A_WARC_RESPONSE_HEADERS)) {
namedFields = namedFields.clone();
for (Object headerObj : curi.getDataList(A_WARC_RESPONSE_HEADERS)) {
String[] kv = StringUtils.split(((String)headerObj),":",2);
namedFields.addLabelValue(kv[0].trim(), kv[1].trim());
}
}
WARCRecordInfo recordInfo = new WARCRecordInfo();
recordInfo.setType(WARCRecordType.RESPONSE);
recordInfo.setUrl(curi.toString());