From 65d552ce5c8adaab047c82d51cfdbb73b25a3a1c Mon Sep 17 00:00:00 2001 From: szznax Date: Mon, 8 Mar 2010 17:43:42 +0000 Subject: [PATCH] fix for HER-1748 Arc2Warc should generally write WARC "response" records for ARC HTTP responses * Arc2Warc.java when HTTP response, write response record rather than resource enable reconstruction of ARC from WARC via namedFields * ARCRecord.java added headerString and accessor to save verbatim ARC header string --- .../main/java/org/archive/io/Arc2Warc.java | 48 +++++++++++++------ .../java/org/archive/io/arc/ARCRecord.java | 12 +++++ 2 files changed, 45 insertions(+), 15 deletions(-) diff --git a/commons/src/main/java/org/archive/io/Arc2Warc.java b/commons/src/main/java/org/archive/io/Arc2Warc.java index 2389cbd1..aeb8e030 100644 --- a/commons/src/main/java/org/archive/io/Arc2Warc.java +++ b/commons/src/main/java/org/archive/io/Arc2Warc.java @@ -23,6 +23,7 @@ import java.io.ByteArrayOutputStream; import java.io.File; import java.io.FileOutputStream; import java.io.IOException; +import java.net.URISyntaxException; import java.util.ArrayList; import java.util.Iterator; import java.util.List; @@ -66,7 +67,7 @@ public class Arc2Warc { } public void transform(final File arc, final File warc, final boolean force) - throws IOException { + throws IOException, URISyntaxException { FileUtils.assertReadable(arc); if (warc.exists() && !force) { throw new IOException("Target WARC already exists. " + @@ -78,7 +79,7 @@ public class Arc2Warc { } protected void transform(final ARCReader reader, final File warc) - throws IOException { + throws IOException, URISyntaxException { WARCWriter writer = null; // No point digesting. Digest is available after reading of ARC which // is too late for inclusion in WARC. @@ -133,8 +134,8 @@ public class Arc2Warc { } } - protected void write(final WARCWriter writer, final ARCRecord r) - throws IOException { +protected void write(final WARCWriter writer, final ARCRecord r) + throws IOException, URISyntaxException { // convert ARC date to WARC-Date format String arcDateString = r.getHeader().getDate(); @@ -143,21 +144,37 @@ public class Arc2Warc { .parseDateTime(arcDateString) .toString(ISODateTimeFormat.dateTimeNoMillis()); - // If contentBody > 0, assume http headers. Make the mimetype - // be application/http. Otherwise, give it ARC mimetype. - String warcMimeTypeString = (r.getHeader().getContentBegin() > 0) ? - WARCConstants.HTTP_RESPONSE_MIMETYPE : - r.getHeader().getMimetype(); - ANVLRecord ar = new ANVLRecord(); - String ip = (String)r.getHeader(). - getHeaderValue((ARCConstants.IP_HEADER_FIELD_KEY)); + String ip = (String)r.getHeader() + .getHeaderValue((ARCConstants.IP_HEADER_FIELD_KEY)); if (ip != null && ip.length() > 0) { ar.addLabelValue(WARCConstants.NAMED_FIELD_IP_LABEL, ip); + r.getMetaData(); + // enable reconstruction of ARC from WARC + ar.addLabelValue("ARC-Header-Line", + r.getHeaderString()); + ar.addLabelValue("ARC-File", + r.getMetaData().getArc()); + ar.addLabelValue("ARC-Offset", + String.valueOf(r.getHeader().getOffset())); + ar.addLabelValue("ARC-Length", + String.valueOf(r.getHeader().getLength())); + } + + // If contentBody > 0, assume http headers. Make the mimetype + // be application/http. Otherwise, give it ARC mimetype. + String warcMimeTypeString; + if (r.getHeader().getContentBegin() > 0) { + warcMimeTypeString = WARCConstants.HTTP_RESPONSE_MIMETYPE; + writer.writeResponseRecord(r.getHeader().getUrl(), warcDateString, + warcMimeTypeString, WARCWriter.getRecordID(), ar, r, + r.getHeader().getLength()); + } else { + warcMimeTypeString = r.getHeader().getMimetype(); + writer.writeResourceRecord(r.getHeader().getUrl(), warcDateString, + warcMimeTypeString, ar, r, r.getHeader().getLength()); } - writer.writeResourceRecord(r.getHeader().getUrl(), warcDateString, - warcMimeTypeString, ar, r, r.getHeader().getLength()); } /** @@ -167,10 +184,11 @@ public class Arc2Warc { * @throws ParseException Failed parse of the command line. * @throws IOException * @throws java.text.ParseException + * @throws URISyntaxException */ @SuppressWarnings("unchecked") public static void main(String [] args) - throws ParseException, IOException, java.text.ParseException { + throws ParseException, IOException, java.text.ParseException, URISyntaxException { Options options = new Options(); options.addOption(new Option("h","help", false, "Prints this message and exits.")); diff --git a/commons/src/main/java/org/archive/io/arc/ARCRecord.java b/commons/src/main/java/org/archive/io/arc/ARCRecord.java index c2e042c9..812e01f4 100644 --- a/commons/src/main/java/org/archive/io/arc/ARCRecord.java +++ b/commons/src/main/java/org/archive/io/arc/ARCRecord.java @@ -36,6 +36,7 @@ import org.apache.commons.httpclient.Header; import org.apache.commons.httpclient.HttpParser; import org.apache.commons.httpclient.StatusLine; import org.apache.commons.httpclient.util.EncodingUtil; +import org.apache.commons.lang.StringUtils; import org.archive.io.ArchiveRecord; import org.archive.io.ArchiveRecordHeader; import org.archive.io.RecoverableIOException; @@ -128,6 +129,14 @@ public class ARCRecord extends ArchiveRecord implements ARCConstants { * errors encountered reading record */ public List errors = new ArrayList(); + + /** + * verbatim ARC record header string + */ + private String headerString; + public String getHeaderString() { + return this.headerString; + } /** * Constructor. @@ -332,6 +341,9 @@ public class ARCRecord extends ArchiveRecord implements ARCConstants { throw new IOException("Unparseable header line: " + list); } + // save verbatim header String + this.headerString = StringUtils.join(list," "); + return read; }