From a414f522f60e8002a1bf1b72b468ff70b907a00e Mon Sep 17 00:00:00 2001 From: Noah Levitt Date: Mon, 10 Sep 2012 18:03:25 -0700 Subject: [PATCH] Continuing to work on cleaning up WARC writer code: simply creation of WARC record headers. * ANVLRecord.java extend LinkedList instead of ArrayList (it was inane to guess the number of records everywhere, when it was never much more than ~10) * ANVLRecords.java remove (seemed to be an incomplete duplicate of ANVLRecord) * WARCRecordInfo.java addExtraHeader() - convenience method * Arc2Warc.java, WARCWriter.java, WARCWriter.java, WARCWriterProcessor.java use WARCRecordInfo.addExtraHeader() where appropriate, and use no-argument ANVLRecord constructor instead of deprecated ANVLRecord(int) --- .../main/java/org/archive/io/Arc2Warc.java | 2 +- .../org/archive/io/warc/WARCRecordInfo.java | 7 ++ .../java/org/archive/io/warc/WARCWriter.java | 8 +-- .../org/archive/util/anvl/ANVLRecord.java | 7 +- .../org/archive/util/anvl/ANVLRecords.java | 61 ------------------ .../org/archive/io/warc/WARCWriterTest.java | 4 +- .../modules/writer/WARCWriterProcessor.java | 64 +++++++++++++++---- 7 files changed, 65 insertions(+), 88 deletions(-) delete mode 100644 commons/src/main/java/org/archive/util/anvl/ANVLRecords.java diff --git a/commons/src/main/java/org/archive/io/Arc2Warc.java b/commons/src/main/java/org/archive/io/Arc2Warc.java index 96f1029b..72ddf192 100644 --- a/commons/src/main/java/org/archive/io/Arc2Warc.java +++ b/commons/src/main/java/org/archive/io/Arc2Warc.java @@ -104,7 +104,7 @@ public class Arc2Warc { getLength()); firstRecord.dump(baos); // Add ARC first record content as an ANVLRecord. - ANVLRecord ar = new ANVLRecord(1); + ANVLRecord ar = new ANVLRecord(); ar.addLabelValue("Filedesc", baos.toString()); List metadata = new ArrayList(1); metadata.add(ar.toString()); diff --git a/commons/src/main/java/org/archive/io/warc/WARCRecordInfo.java b/commons/src/main/java/org/archive/io/warc/WARCRecordInfo.java index 432323b4..be7cff5b 100644 --- a/commons/src/main/java/org/archive/io/warc/WARCRecordInfo.java +++ b/commons/src/main/java/org/archive/io/warc/WARCRecordInfo.java @@ -111,4 +111,11 @@ public class WARCRecordInfo { public String getUrl() { return url; } + + public void addExtraHeader(String label, String value) { + if (extraHeaders == null) { + extraHeaders = new ANVLRecord(); + } + extraHeaders.addLabelValue(label, value); + } } diff --git a/commons/src/main/java/org/archive/io/warc/WARCWriter.java b/commons/src/main/java/org/archive/io/warc/WARCWriter.java index 675f22d6..40abe4c5 100644 --- a/commons/src/main/java/org/archive/io/warc/WARCWriter.java +++ b/commons/src/main/java/org/archive/io/warc/WARCWriter.java @@ -23,7 +23,6 @@ import java.io.ByteArrayInputStream; import java.io.ByteArrayOutputStream; import java.io.File; import java.io.IOException; -import java.io.InputStream; import java.io.OutputStream; import java.net.URI; import java.util.HashMap; @@ -41,7 +40,6 @@ import org.archive.io.UTF8Bytes; import org.archive.io.WriterPoolMember; import org.archive.modules.writer.WARCWriterProcessor; import org.archive.util.ArchiveUtils; -import org.archive.util.anvl.ANVLRecord; import org.archive.util.anvl.Element; @@ -345,12 +343,10 @@ implements WARCConstants { filename = filename.substring(0, filename.length() - WriterPoolMember.OCCUPIED_SUFFIX.length()); } - ANVLRecord extraHeaders = new ANVLRecord(2); - extraHeaders.addLabelValue(HEADER_KEY_FILENAME, filename); + recordInfo.addExtraHeader(HEADER_KEY_FILENAME, filename); if (description != null && description.length() > 0) { - extraHeaders.addLabelValue(CONTENT_DESCRIPTION, description); + recordInfo.addExtraHeader(CONTENT_DESCRIPTION, description); } - recordInfo.setExtraHeaders(extraHeaders); // Add warcinfo body. byte [] warcinfoBody = null; diff --git a/commons/src/main/java/org/archive/util/anvl/ANVLRecord.java b/commons/src/main/java/org/archive/util/anvl/ANVLRecord.java index cfabb1dc..ad368fee 100644 --- a/commons/src/main/java/org/archive/util/anvl/ANVLRecord.java +++ b/commons/src/main/java/org/archive/util/anvl/ANVLRecord.java @@ -22,10 +22,10 @@ import java.io.ByteArrayOutputStream; import java.io.IOException; import java.io.InputStream; import java.io.UnsupportedEncodingException; -import java.util.ArrayList; import java.util.Collection; import java.util.HashMap; import java.util.Iterator; +import java.util.LinkedList; import java.util.List; import java.util.Map; import java.util.logging.Level; @@ -42,7 +42,7 @@ import org.archive.io.UTF8Bytes; * Language (ANVL) * @author stack */ -public class ANVLRecord extends ArrayList implements UTF8Bytes { +public class ANVLRecord extends LinkedList implements UTF8Bytes { private static final long serialVersionUID = -4610638888453052958L; private static final Logger logger = Logger.getLogger(ANVLRecord.class.getName()); @@ -73,8 +73,9 @@ public class ANVLRecord extends ArrayList implements UTF8Bytes { super(c); } + /** @deprecated */ public ANVLRecord(int initialCapacity) { - super(initialCapacity); + super(); } public boolean addLabel(final String l) { diff --git a/commons/src/main/java/org/archive/util/anvl/ANVLRecords.java b/commons/src/main/java/org/archive/util/anvl/ANVLRecords.java deleted file mode 100644 index 8ed5f43c..00000000 --- a/commons/src/main/java/org/archive/util/anvl/ANVLRecords.java +++ /dev/null @@ -1,61 +0,0 @@ -/* - * This file is part of the Heritrix web crawler (crawler.archive.org). - * - * Licensed to the Internet Archive (IA) by one or more individual - * contributors. - * - * The IA licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.archive.util.anvl; - -import java.io.UnsupportedEncodingException; -import java.util.ArrayList; -import java.util.Collection; -import java.util.Iterator; - -import org.archive.io.UTF8Bytes; - -/** - * List of {@link ANVLRecord}s. - * @author stack - * @version $Date$ $Version$ - */ -public class ANVLRecords extends ArrayList implements UTF8Bytes { - private static final long serialVersionUID = 5361551920550106113L; - - public ANVLRecords() { - super(); - } - - public ANVLRecords(int initialCapacity) { - super(initialCapacity); - } - - public ANVLRecords(Collection c) { - super(c); - } - - public byte[] getUTF8Bytes() throws UnsupportedEncodingException { - return toString().getBytes(UTF8); - } - - @Override - public String toString() { - StringBuilder sb = new StringBuilder(); - for (final Iterator i = iterator(); i.hasNext();) { - sb.append(i.next().toString()); - } - return super.toString(); - } -} \ No newline at end of file diff --git a/commons/src/test/java/org/archive/io/warc/WARCWriterTest.java b/commons/src/test/java/org/archive/io/warc/WARCWriterTest.java index b366d33f..fa19a5e2 100644 --- a/commons/src/test/java/org/archive/io/warc/WARCWriterTest.java +++ b/commons/src/test/java/org/archive/io/warc/WARCWriterTest.java @@ -227,9 +227,7 @@ extends TmpDirTestCase implements WARCConstants { // Add named fields for ip, checksum, and relate the metadata // and request to the resource field. - ANVLRecord r = new ANVLRecord(1); - r.addLabelValue(NAMED_FIELD_IP_LABEL, "127.0.0.1"); - recordInfo.setExtraHeaders(r); + recordInfo.addExtraHeader(NAMED_FIELD_IP_LABEL, "127.0.0.1"); w.writeRecord(recordInfo); return record.length; diff --git a/modules/src/main/java/org/archive/modules/writer/WARCWriterProcessor.java b/modules/src/main/java/org/archive/modules/writer/WARCWriterProcessor.java index 2dc6a109..c66bf3f2 100644 --- a/modules/src/main/java/org/archive/modules/writer/WARCWriterProcessor.java +++ b/modules/src/main/java/org/archive/modules/writer/WARCWriterProcessor.java @@ -320,24 +320,60 @@ public class WARCWriterProcessor extends WriterPoolProcessor implements WARCWrit private void writeDnsRecords(final CrawlURI curi, WARCWriter w, final URI baseid, final String timestamp) throws IOException { - ANVLRecord headers = null; + WARCRecordInfo recordInfo = new WARCRecordInfo(); + recordInfo.setType(WARCRecordType.RESPONSE); + recordInfo.setUrl(curi.toString()); + recordInfo.setCreate14DigitDate(timestamp); + recordInfo.setMimetype(curi.getContentType()); + recordInfo.setRecordId(baseid); + + recordInfo.setContentLength(curi.getRecorder().getRecordedInput().getSize()); + recordInfo.setEnforceLength(true); + String ip = (String)curi.getData().get(A_DNS_SERVER_IP_LABEL); if (ip != null && ip.length() > 0) { - headers = new ANVLRecord(1); - headers.addLabelValue(HEADER_KEY_IP, ip); + recordInfo.addExtraHeader(HEADER_KEY_IP, ip); } - writeResponse(w, timestamp, curi.getContentType(), baseid, - curi, headers); + + ReplayInputStream ris = + curi.getRecorder().getRecordedInput().getReplayInputStream(); + recordInfo.setContentStream(ris); + + try { + w.writeRecord(recordInfo); + } finally { + IOUtils.closeQuietly(ris); + } + + recordInfo.getRecordId(); } private void writeWhoisRecords(WARCWriter w, CrawlURI curi, URI baseid, String timestamp) throws IOException { - ANVLRecord headers = new ANVLRecord(1); + WARCRecordInfo recordInfo = new WARCRecordInfo(); + recordInfo.setType(WARCRecordType.RESPONSE); + recordInfo.setUrl(curi.toString()); + recordInfo.setCreate14DigitDate(timestamp); + recordInfo.setMimetype(curi.getContentType()); + recordInfo.setRecordId(baseid); + recordInfo.setContentLength(curi.getRecorder().getRecordedInput().getSize()); + recordInfo.setEnforceLength(true); + Object whoisServerIP = curi.getData().get(CoreAttributeConstants.A_WHOIS_SERVER_IP); if (whoisServerIP != null) { - headers.addLabelValue(HEADER_KEY_IP, whoisServerIP.toString()); + recordInfo.addExtraHeader(HEADER_KEY_IP, whoisServerIP.toString()); } - writeResponse(w, timestamp, curi.getContentType(), baseid, curi, headers); + + ReplayInputStream ris = + curi.getRecorder().getRecordedInput().getReplayInputStream(); + recordInfo.setContentStream(ris); + + try { + w.writeRecord(recordInfo); + } finally { + IOUtils.closeQuietly(ris); + } + recordInfo.getRecordId(); } private void writeHttpRecords(final CrawlURI curi, WARCWriter w, @@ -346,7 +382,7 @@ public class WARCWriterProcessor extends WriterPoolProcessor implements WARCWrit // and request to the resource field. // TODO: Use other than ANVL (or rename ANVL as NameValue or // use RFC822 (commons-httpclient?). - ANVLRecord headers = new ANVLRecord(5); + ANVLRecord headers = new ANVLRecord(); if (curi.getContentDigest() != null) { headers.addLabelValue(HEADER_KEY_PAYLOAD_DIGEST, curi.getContentDigestSchemeString()); @@ -381,7 +417,7 @@ public class WARCWriterProcessor extends WriterPoolProcessor implements WARCWrit baseid, curi, headers); } - headers = new ANVLRecord(1); + headers = new ANVLRecord(); headers.addLabelValue(HEADER_KEY_CONCURRENT_TO, '<' + rid.toString() + '>'); @@ -396,7 +432,7 @@ public class WARCWriterProcessor extends WriterPoolProcessor implements WARCWrit private void writeFtpRecords(WARCWriter w, final CrawlURI curi, final URI baseid, final String timestamp) throws IOException { - ANVLRecord headers = new ANVLRecord(3); + ANVLRecord headers = new ANVLRecord(); headers.addLabelValue(HEADER_KEY_IP, getHostAddress(curi)); String controlConversation = curi.getData().get(A_FTP_CONTROL_CONVERSATION).toString(); URI rid = writeFtpControlConversation(w, timestamp, baseid, curi, headers, controlConversation); @@ -412,7 +448,7 @@ public class WARCWriterProcessor extends WriterPoolProcessor implements WARCWrit rid = writeRevisitDigest(w, timestamp, null, baseid, curi, headers, 0); } else { - headers = new ANVLRecord(3); + headers = new ANVLRecord(); // Check for truncated annotation String value = null; Collection anno = curi.getAnnotations(); @@ -437,7 +473,7 @@ public class WARCWriterProcessor extends WriterPoolProcessor implements WARCWrit } } if (getWriteMetadata()) { - headers = new ANVLRecord(1); + headers = new ANVLRecord(); headers.addLabelValue(HEADER_KEY_CONCURRENT_TO, '<' + rid.toString() + '>'); writeMetadata(w, timestamp, baseid, curi, headers); } @@ -754,7 +790,7 @@ public class WARCWriterProcessor extends WriterPoolProcessor implements WARCWrit if (cachedMetadata != null) { return cachedMetadata; } - ANVLRecord record = new ANVLRecord(7); + ANVLRecord record = new ANVLRecord(); record.addLabelValue("software", "Heritrix/" + ArchiveUtils.VERSION + " http://crawler.archive.org"); try {