lee/heritrix3
1
0
Fork 0
mirror of https://github.com/internetarchive/heritrix3.git synced 2026-09-27 08:05:46 +00:00
Code Issues Packages Projects Releases Wiki Activity

Continuing to work on cleaning up WARC writer code: simply creation of WARC record headers.

Browse Source 
* ANVLRecord.java
    extend LinkedList instead of ArrayList (it was inane to guess the number of records everywhere, when it was never much more than ~10)
* ANVLRecords.java
    remove (seemed to be an incomplete duplicate of ANVLRecord)
* WARCRecordInfo.java
    addExtraHeader() - convenience method
* Arc2Warc.java, WARCWriter.java, WARCWriter.java, WARCWriterProcessor.java
    use WARCRecordInfo.addExtraHeader() where appropriate, and use no-argument ANVLRecord constructor instead of deprecated ANVLRecord(int)
This commit is contained in:
Noah Levitt
2012-09-10 18:03:25 -07:00
parent 360ac00000
commit a414f522f6
7 changed files with 65 additions and 88 deletions
Download Patch File Download Diff File Expand all files Collapse all files
commons/src/main/java/org/archive/io/Arc2Warc.java
+1 -1
View File
@@ -104,7 +104,7 @@ public class Arc2Warc {
getLength());
firstRecord.dump(baos);
// Add ARC first record content as an ANVLRecord.
ANVLRecord ar = new ANVLRecord(1);
ANVLRecord ar = new ANVLRecord();
ar.addLabelValue("Filedesc", baos.toString());
List<String> metadata = new ArrayList<String>(1);
metadata.add(ar.toString());
commons/src/main/java/org/archive/io/warc/WARCRecordInfo.java
+7
View File
@@ -111,4 +111,11 @@ public class WARCRecordInfo {
public String getUrl() {
return url;
}
public void addExtraHeader(String label, String value) {
if (extraHeaders == null) {
extraHeaders = new ANVLRecord();
}
extraHeaders.addLabelValue(label, value);
}
}
commons/src/main/java/org/archive/io/warc/WARCWriter.java
+2 -6
View File
@@ -23,7 +23,6 @@ import java.io.ByteArrayInputStream;
import java.io.ByteArrayOutputStream;
import java.io.File;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
import java.net.URI;
import java.util.HashMap;
@@ -41,7 +40,6 @@ import org.archive.io.UTF8Bytes;
import org.archive.io.WriterPoolMember;
import org.archive.modules.writer.WARCWriterProcessor;
import org.archive.util.ArchiveUtils;
import org.archive.util.anvl.ANVLRecord;
import org.archive.util.anvl.Element;
@@ -345,12 +343,10 @@ implements WARCConstants {
filename = filename.substring(0,
filename.length() - WriterPoolMember.OCCUPIED_SUFFIX.length());
}
ANVLRecord extraHeaders = new ANVLRecord(2);
extraHeaders.addLabelValue(HEADER_KEY_FILENAME, filename);
recordInfo.addExtraHeader(HEADER_KEY_FILENAME, filename);
if (description != null && description.length() > 0) {
extraHeaders.addLabelValue(CONTENT_DESCRIPTION, description);
recordInfo.addExtraHeader(CONTENT_DESCRIPTION, description);
}
recordInfo.setExtraHeaders(extraHeaders);
// Add warcinfo body.
byte [] warcinfoBody = null;
commons/src/main/java/org/archive/util/anvl/ANVLRecord.java
+4 -3
View File
@@ -22,10 +22,10 @@ import java.io.ByteArrayOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.UnsupportedEncodingException;
import java.util.ArrayList;
import java.util.Collection;
import java.util.HashMap;
import java.util.Iterator;
import java.util.LinkedList;
import java.util.List;
import java.util.Map;
import java.util.logging.Level;
@@ -42,7 +42,7 @@ import org.archive.io.UTF8Bytes;
* Language (ANVL)</a>
* @author stack
*/
public class ANVLRecord extends ArrayList<Element> implements UTF8Bytes {
public class ANVLRecord extends LinkedList<Element> implements UTF8Bytes {
private static final long serialVersionUID = -4610638888453052958L;
private static final Logger logger =
Logger.getLogger(ANVLRecord.class.getName());
@@ -73,8 +73,9 @@ public class ANVLRecord extends ArrayList<Element> implements UTF8Bytes {
super(c);
}
/** @deprecated */
public ANVLRecord(int initialCapacity) {
super(initialCapacity);
super();
}
public boolean addLabel(final String l) {
commons/src/main/java/org/archive/util/anvl/ANVLRecords.java
-61
View File
@@ -1,61 +0,0 @@
/*
* This file is part of the Heritrix web crawler (crawler.archive.org).
*
* Licensed to the Internet Archive (IA) by one or more individual
* contributors.
*
* The IA licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.archive.util.anvl;
import java.io.UnsupportedEncodingException;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Iterator;
import org.archive.io.UTF8Bytes;
/**
* List of {@link ANVLRecord}s.
* @author stack
* @version $Date$ $Version$
*/
public class ANVLRecords extends ArrayList<ANVLRecord> implements UTF8Bytes {
private static final long serialVersionUID = 5361551920550106113L;
public ANVLRecords() {
super();
}
public ANVLRecords(int initialCapacity) {
super(initialCapacity);
}
public ANVLRecords(Collection<ANVLRecord> c) {
super(c);
}
public byte[] getUTF8Bytes() throws UnsupportedEncodingException {
return toString().getBytes(UTF8);
}
@Override
public String toString() {
StringBuilder sb = new StringBuilder();
for (final Iterator<ANVLRecord> i = iterator(); i.hasNext();) {
sb.append(i.next().toString());
}
return super.toString();
}
}
commons/src/test/java/org/archive/io/warc/WARCWriterTest.java
+1 -3
View File
@@ -227,9 +227,7 @@ extends TmpDirTestCase implements WARCConstants {
// Add named fields for ip, checksum, and relate the metadata
// and request to the resource field.
ANVLRecord r = new ANVLRecord(1);
r.addLabelValue(NAMED_FIELD_IP_LABEL, "127.0.0.1");
recordInfo.setExtraHeaders(r);
recordInfo.addExtraHeader(NAMED_FIELD_IP_LABEL, "127.0.0.1");
w.writeRecord(recordInfo);
return record.length;
modules/src/main/java/org/archive/modules/writer/WARCWriterProcessor.java
+50 -14
View File
@@ -320,24 +320,60 @@ public class WARCWriterProcessor extends WriterPoolProcessor implements WARCWrit
private void writeDnsRecords(final CrawlURI curi, WARCWriter w,
final URI baseid, final String timestamp) throws IOException {
ANVLRecord headers = null;
WARCRecordInfo recordInfo = new WARCRecordInfo();
recordInfo.setType(WARCRecordType.RESPONSE);
recordInfo.setUrl(curi.toString());
recordInfo.setCreate14DigitDate(timestamp);
recordInfo.setMimetype(curi.getContentType());
recordInfo.setRecordId(baseid);
recordInfo.setContentLength(curi.getRecorder().getRecordedInput().getSize());
recordInfo.setEnforceLength(true);
String ip = (String)curi.getData().get(A_DNS_SERVER_IP_LABEL);
if (ip != null && ip.length() > 0) {
headers = new ANVLRecord(1);
headers.addLabelValue(HEADER_KEY_IP, ip);
recordInfo.addExtraHeader(HEADER_KEY_IP, ip);
}
writeResponse(w, timestamp, curi.getContentType(), baseid,
curi, headers);
ReplayInputStream ris =
curi.getRecorder().getRecordedInput().getReplayInputStream();
recordInfo.setContentStream(ris);
try {
w.writeRecord(recordInfo);
} finally {
IOUtils.closeQuietly(ris);
}
recordInfo.getRecordId();
}
private void writeWhoisRecords(WARCWriter w, CrawlURI curi, URI baseid,
String timestamp) throws IOException {
ANVLRecord headers = new ANVLRecord(1);
WARCRecordInfo recordInfo = new WARCRecordInfo();
recordInfo.setType(WARCRecordType.RESPONSE);
recordInfo.setUrl(curi.toString());
recordInfo.setCreate14DigitDate(timestamp);
recordInfo.setMimetype(curi.getContentType());
recordInfo.setRecordId(baseid);
recordInfo.setContentLength(curi.getRecorder().getRecordedInput().getSize());
recordInfo.setEnforceLength(true);
Object whoisServerIP = curi.getData().get(CoreAttributeConstants.A_WHOIS_SERVER_IP);
if (whoisServerIP != null) {
headers.addLabelValue(HEADER_KEY_IP, whoisServerIP.toString());
recordInfo.addExtraHeader(HEADER_KEY_IP, whoisServerIP.toString());
}
writeResponse(w, timestamp, curi.getContentType(), baseid, curi, headers);
ReplayInputStream ris =
curi.getRecorder().getRecordedInput().getReplayInputStream();
recordInfo.setContentStream(ris);
try {
w.writeRecord(recordInfo);
} finally {
IOUtils.closeQuietly(ris);
}
recordInfo.getRecordId();
}
private void writeHttpRecords(final CrawlURI curi, WARCWriter w,
@@ -346,7 +382,7 @@ public class WARCWriterProcessor extends WriterPoolProcessor implements WARCWrit
// and request to the resource field.
// TODO: Use other than ANVL (or rename ANVL as NameValue or
// use RFC822 (commons-httpclient?).
ANVLRecord headers = new ANVLRecord(5);
ANVLRecord headers = new ANVLRecord();
if (curi.getContentDigest() != null) {
headers.addLabelValue(HEADER_KEY_PAYLOAD_DIGEST,
curi.getContentDigestSchemeString());
@@ -381,7 +417,7 @@ public class WARCWriterProcessor extends WriterPoolProcessor implements WARCWrit
baseid, curi, headers);
}
headers = new ANVLRecord(1);
headers = new ANVLRecord();
headers.addLabelValue(HEADER_KEY_CONCURRENT_TO,
'<' + rid.toString() + '>');
@@ -396,7 +432,7 @@ public class WARCWriterProcessor extends WriterPoolProcessor implements WARCWrit
private void writeFtpRecords(WARCWriter w, final CrawlURI curi, final URI baseid,
final String timestamp) throws IOException {
ANVLRecord headers = new ANVLRecord(3);
ANVLRecord headers = new ANVLRecord();
headers.addLabelValue(HEADER_KEY_IP, getHostAddress(curi));
String controlConversation = curi.getData().get(A_FTP_CONTROL_CONVERSATION).toString();
URI rid = writeFtpControlConversation(w, timestamp, baseid, curi, headers, controlConversation);
@@ -412,7 +448,7 @@ public class WARCWriterProcessor extends WriterPoolProcessor implements WARCWrit
rid = writeRevisitDigest(w, timestamp, null,
baseid, curi, headers, 0);
} else {
headers = new ANVLRecord(3);
headers = new ANVLRecord();
// Check for truncated annotation
String value = null;
Collection<String> anno = curi.getAnnotations();
@@ -437,7 +473,7 @@ public class WARCWriterProcessor extends WriterPoolProcessor implements WARCWrit
}
}
if (getWriteMetadata()) {
headers = new ANVLRecord(1);
headers = new ANVLRecord();
headers.addLabelValue(HEADER_KEY_CONCURRENT_TO, '<' + rid.toString() + '>');
writeMetadata(w, timestamp, baseid, curi, headers);
}
@@ -754,7 +790,7 @@ public class WARCWriterProcessor extends WriterPoolProcessor implements WARCWrit
if (cachedMetadata != null) {
return cachedMetadata;
}
ANVLRecord record = new ANVLRecord(7);
ANVLRecord record = new ANVLRecord();
record.addLabelValue("software", "Heritrix/" +
ArchiveUtils.VERSION + " http://crawler.archive.org");
try {