From dd5e51fd9d5d0e7db33eaa24148bc331031a440c Mon Sep 17 00:00:00 2001 From: Noah Levitt Date: Mon, 10 Sep 2012 15:43:37 -0700 Subject: [PATCH 01/19] Working on cleaning up WARC writer code. * WARCRecordInfo.java new class to hold warc record info to write, so we don't have to pass around all these long lists of variables * WARCConstants.java new enum WARCRecordType replaces bunch of string constants * WARCWriter.java, WARCWriterProcessor.java, WARCWriterTest.java use WARCRecordInfo, WARCRecordType --- .../org/archive/io/warc/WARCConstants.java | 48 ++++---- .../org/archive/io/warc/WARCRecordInfo.java | 111 ++++++++++++++++++ .../java/org/archive/io/warc/WARCWriter.java | 108 +++++++++-------- .../org/archive/io/warc/WARCWriterTest.java | 4 +- .../modules/writer/WARCWriterProcessor.java | 26 ++-- .../writer/WARCWriterProcessorTest.java | 2 +- 6 files changed, 204 insertions(+), 95 deletions(-) create mode 100644 commons/src/main/java/org/archive/io/warc/WARCRecordInfo.java diff --git a/commons/src/main/java/org/archive/io/warc/WARCConstants.java b/commons/src/main/java/org/archive/io/warc/WARCConstants.java index 8453238e..685c9364 100644 --- a/commons/src/main/java/org/archive/io/warc/WARCConstants.java +++ b/commons/src/main/java/org/archive/io/warc/WARCConstants.java @@ -19,9 +19,6 @@ package org.archive.io.warc; -import java.util.Arrays; -import java.util.List; - import org.archive.io.ArchiveFileConstants; /** @@ -119,34 +116,29 @@ public interface WARCConstants extends ArchiveFileConstants { /** * WARC Record Types. */ - public static final String WARCINFO = "warcinfo"; - public static final String RESPONSE = "response"; - public static final String RESOURCE = "resource"; - public static final String REQUEST = "request"; - public static final String METADATA = "metadata"; - public static final String REVISIT = "revisit"; - public static final String CONVERSION = "conversion"; - public static final String CONTINUATION = "continuation"; + enum WARCRecordType { + WARCINFO("warcinfo"), + RESPONSE("response"), + RESOURCE("resource"), + REQUEST("request"), + METADATA("metadata"), + REVISIT("revisit"), + CONVERSION("conversion"), + CONTINUATION("continuation"); + + private String value; + private WARCRecordType(String value) { + this.value = value; + } + + @Override + public String toString() { + return value; + } + } public static final String TYPE = "type"; - // List of all WARC Record TYPES - public static final String [] TYPES = {WARCINFO, RESPONSE, RESOURCE, - REQUEST, METADATA, REVISIT, CONVERSION, CONTINUATION}; - - // Indices into TYPES array. - public static final int WARCINFO_INDEX = 0; - public static final int RESPONSE_INDEX = 1; - public static final int RESOURCE_INDEX = 2; - public static final int REQUEST_INDEX = 3; - public static final int METADATA_INDEX = 4; - public static final int REVISIT_INDEX = 5; - public static final int CONVERSION_INDEX = 6; - public static final int CONTINUATION_INDEX = 7; - - // TYPES as List. - public static final List TYPES_LIST = Arrays.asList(TYPES); - /** * WARC-ID */ diff --git a/commons/src/main/java/org/archive/io/warc/WARCRecordInfo.java b/commons/src/main/java/org/archive/io/warc/WARCRecordInfo.java new file mode 100644 index 00000000..a4422508 --- /dev/null +++ b/commons/src/main/java/org/archive/io/warc/WARCRecordInfo.java @@ -0,0 +1,111 @@ +/* + * This file is part of the Heritrix web crawler (crawler.archive.org). + * + * Licensed to the Internet Archive (IA) by one or more individual + * contributors. + * + * The IA licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.archive.io.warc; + +import java.io.InputStream; +import java.net.URI; + +import org.archive.io.warc.WARCConstants.WARCRecordType; +import org.archive.util.anvl.ANVLRecord; + +public class WARCRecordInfo { + + protected WARCRecordType type; + protected String url; + protected String create14DigitDate; + protected String mimetype; + protected URI recordId; + protected ANVLRecord extraHeaders; + protected InputStream contentStream; + protected long contentLength; + protected boolean enforceLength; + + public WARCRecordInfo(WARCRecordType type, String url) { + this.type = type; + this.url = url; + } + + public String getCreate14DigitDate() { + return create14DigitDate; + } + + public void setCreate14DigitDate(String create14DigitDate) { + this.create14DigitDate = create14DigitDate; + } + + public String getMimetype() { + return mimetype; + } + + public void setMimetype(String mimetype) { + this.mimetype = mimetype; + } + + public URI getRecordId() { + return recordId; + } + + public void setRecordId(URI recordId) { + this.recordId = recordId; + } + + public ANVLRecord getExtraHeaders() { + return extraHeaders; + } + + public void setExtraHeaders(ANVLRecord extraHeaders) { + this.extraHeaders = extraHeaders; + } + + public InputStream getContentStream() { + return contentStream; + } + + public void setContentStream(InputStream contentStream) { + this.contentStream = contentStream; + } + + public long getContentLength() { + return contentLength; + } + + public void setContentLength(long contentLength) { + this.contentLength = contentLength; + } + + public boolean isEnforceLength() { + return enforceLength; + } + + public boolean getEnforceLength() { + return enforceLength; + } + + public void setEnforceLength(boolean enforceLength) { + this.enforceLength = enforceLength; + } + + public WARCRecordType getType() { + return type; + } + + public String getUrl() { + return url; + } +} diff --git a/commons/src/main/java/org/archive/io/warc/WARCWriter.java b/commons/src/main/java/org/archive/io/warc/WARCWriter.java index 8cfccdd1..c0127bf3 100644 --- a/commons/src/main/java/org/archive/io/warc/WARCWriter.java +++ b/commons/src/main/java/org/archive/io/warc/WARCWriter.java @@ -36,8 +36,10 @@ import java.util.concurrent.atomic.AtomicLong; import java.util.logging.Level; import java.util.logging.Logger; +import org.apache.commons.lang.StringUtils; import org.archive.io.UTF8Bytes; import org.archive.io.WriterPoolMember; +import org.archive.modules.writer.WARCWriterProcessor; import org.archive.util.ArchiveUtils; import org.archive.util.anvl.ANVLRecord; import org.archive.util.anvl.Element; @@ -84,7 +86,7 @@ implements WARCConstants { * {@link #resetTmpStats()}, write some records, then add * {@link #getTmpStats()} into its long-term running totals. */ - private Map> tmpStats; + private Map> tmpStats; /** * Constructor. @@ -173,37 +175,38 @@ implements WARCConstants { return sb.toString(); } - protected String createRecordHeader(final String type, - final String url, final String create14DigitDate, - final String mimetype, final URI recordId, - final ANVLRecord xtraHeaders, final long contentLength) +// protected String createRecordHeader(final String type, +// final String url, final String create14DigitDate, +// final String mimetype, final URI recordId, +// final ANVLRecord xtraHeaders, final long contentLength) + protected String createRecordHeader(WARCRecordInfo metaRecord) throws IllegalArgumentException { final StringBuilder sb = new StringBuilder(2048/*A SWAG: TODO: Do analysis.*/); sb.append(WARC_ID).append(CRLF); - sb.append(HEADER_KEY_TYPE).append(COLON_SPACE).append(type). + sb.append(HEADER_KEY_TYPE).append(COLON_SPACE).append(metaRecord.getType()). append(CRLF); // Do not write a subject-uri if not one present. - if (url != null && url.length() > 0) { + if (!StringUtils.isEmpty(metaRecord.getUrl())) { sb.append(HEADER_KEY_URI).append(COLON_SPACE). - append(checkHeaderValue(url)).append(CRLF); + append(checkHeaderValue(metaRecord.getUrl())).append(CRLF); } sb.append(HEADER_KEY_DATE).append(COLON_SPACE). - append(create14DigitDate).append(CRLF); - if (xtraHeaders != null) { - for (final Iterator i = xtraHeaders.iterator(); i.hasNext();) { + append(metaRecord.getCreate14DigitDate()).append(CRLF); + if (metaRecord.getExtraHeaders() != null) { + for (final Iterator i = metaRecord.getExtraHeaders().iterator(); i.hasNext();) { sb.append(i.next()).append(CRLF); } } sb.append(HEADER_KEY_ID).append(COLON_SPACE).append('<'). - append(recordId.toString()).append('>').append(CRLF); - if (contentLength > 0) { + append(metaRecord.getRecordId().toString()).append('>').append(CRLF); + if (metaRecord.getContentLength() > 0) { sb.append(CONTENT_TYPE).append(COLON_SPACE).append( - checkHeaderLineMimetypeParameter(mimetype)).append(CRLF); + checkHeaderLineMimetypeParameter(metaRecord.getMimetype())).append(CRLF); } sb.append(CONTENT_LENGTH).append(COLON_SPACE). - append(Long.toString(contentLength)).append(CRLF); + append(Long.toString(metaRecord.getContentLength())).append(CRLF); return sb.toString(); } @@ -211,7 +214,7 @@ implements WARCConstants { /** * @deprecated Use {@link #writeRecord(String,String,String,String,URI,ANVLRecord,InputStream,long,boolean)} instead */ - protected void writeRecord(final String type, final String url, + protected void writeRecord(final WARCRecordType type, final String url, final String create14DigitDate, final String mimetype, final URI recordId, ANVLRecord xtraHeaders, final InputStream contentStream, final long contentLength) @@ -219,30 +222,42 @@ implements WARCConstants { writeRecord(type, url, create14DigitDate, mimetype, recordId, xtraHeaders, contentStream, contentLength, true); } - protected void writeRecord(final String type, final String url, + /** @deprecated */ + public void writeRecord(final WARCRecordType type, final String url, final String create14DigitDate, final String mimetype, final URI recordId, ANVLRecord xtraHeaders, final InputStream contentStream, final long contentLength, boolean enforceLength) throws IOException { - if (!TYPES_LIST.contains(type)) { - throw new IllegalArgumentException("Unknown record type: " + type); - } - if (contentLength == 0 && - (xtraHeaders == null || xtraHeaders.size() <= 0)) { + + WARCRecordInfo metaRecord = new WARCRecordInfo(type, url); + metaRecord.setCreate14DigitDate(create14DigitDate); + metaRecord.setMimetype(mimetype); + metaRecord.setRecordId(recordId); + metaRecord.setExtraHeaders(xtraHeaders); + metaRecord.setContentStream(contentStream); + metaRecord.setContentLength(contentLength); + metaRecord.setEnforceLength(enforceLength); + + writeRecord(metaRecord); + } + + protected void writeRecord(WARCRecordInfo metaRecord) + throws IOException { + + if (metaRecord.getContentLength() == 0 && + (metaRecord.getExtraHeaders() == null || metaRecord.getExtraHeaders().size() <= 0)) { throw new IllegalArgumentException("Cannot write record " + "of content-length zero and base headers only."); } String header; try { - header = createRecordHeader(type, url, - create14DigitDate, mimetype, recordId, xtraHeaders, - contentLength); + header = createRecordHeader(metaRecord); } catch (IllegalArgumentException e) { - logger.log(Level.SEVERE,"could not write record type: " + type - + "for URL: " + url, e); + logger.log(Level.SEVERE,"could not write record type: " + metaRecord.getType() + + "for URL: " + metaRecord.getUrl(), e); return; } @@ -260,11 +275,13 @@ implements WARCConstants { totalBytes += bytes.length; - if (contentStream != null && contentLength > 0) { + if (metaRecord.getContentStream() != null && metaRecord.getContentLength() > 0) { // Write out the header/body separator. write(CRLF_BYTES); // TODO: should this be written even for zero-length? totalBytes += CRLF_BYTES.length; - contentBytes += copyFrom(contentStream, contentLength, enforceLength); + contentBytes += copyFrom(metaRecord.getContentStream(), + metaRecord.getContentLength(), + metaRecord.getEnforceLength()); totalBytes += contentBytes; } @@ -277,21 +294,21 @@ implements WARCConstants { } // TODO: should this be in the finally block? - tally(type, contentBytes, totalBytes, getPosition() - startPosition); + tally(metaRecord.getType(), contentBytes, totalBytes, getPosition() - startPosition); } // if compression is enabled, sizeOnDisk means compressed bytes; if not, it // should be the same as totalBytes (right?) - protected void tally(String recordType, long contentBytes, long totalBytes, long sizeOnDisk) { + protected void tally(WARCRecordType warcRecordType, long contentBytes, long totalBytes, long sizeOnDisk) { if (tmpStats == null) { tmpStats = new HashMap>(); } // add to stats for this record type - Map substats = tmpStats.get(recordType); + Map substats = tmpStats.get(warcRecordType.toString()); if (substats == null) { substats = new HashMap(); - tmpStats.put(recordType, substats); + tmpStats.put(warcRecordType.toString(), substats); } subtally(substats, contentBytes, totalBytes, sizeOnDisk); @@ -395,12 +412,12 @@ implements WARCConstants { final ANVLRecord namedFields, final InputStream fileMetadata, final long fileMetadataLength) throws IOException { - final URI recordid = generateRecordId(TYPE, WARCINFO); + final URI recordid = generateRecordId(TYPE, WARCRecordType.WARCINFO.toString()); writeWarcinfoRecord(ArchiveUtils.getLog14Date(), mimetype, recordid, namedFields, fileMetadata, fileMetadataLength); return recordid; } - + /** * Write a warcinfo to current file. * The warcinfo type uses its recordId as its URL. @@ -416,7 +433,7 @@ implements WARCConstants { final String mimetype, final URI recordId, final ANVLRecord namedFields, final InputStream fileMetadata, final long fileMetadataLength) throws IOException { - writeRecord(WARCINFO, null, create14DigitDate, mimetype, + writeRecord(WARCRecordType.WARCINFO, null, create14DigitDate, mimetype, recordId, namedFields, fileMetadata, fileMetadataLength, true); } @@ -426,7 +443,7 @@ implements WARCConstants { final ANVLRecord namedFields, final InputStream request, final long requestLength) throws IOException { - writeRecord(REQUEST, url, create14DigitDate, + writeRecord(WARCRecordType.REQUEST, url, create14DigitDate, mimetype, recordId, namedFields, request, requestLength, true); } @@ -447,7 +464,7 @@ implements WARCConstants { final ANVLRecord namedFields, final InputStream response, final long responseLength) throws IOException { - writeRecord(RESOURCE, url, create14DigitDate, + writeRecord(WARCRecordType.RESOURCE, url, create14DigitDate, mimetype, recordId, namedFields, response, responseLength, true); } @@ -458,7 +475,7 @@ implements WARCConstants { final ANVLRecord namedFields, final InputStream response, final long responseLength) throws IOException { - writeRecord(RESPONSE, url, create14DigitDate, + writeRecord(WARCRecordType.RESPONSE, url, create14DigitDate, mimetype, recordId, namedFields, response, responseLength, true); } @@ -469,22 +486,11 @@ implements WARCConstants { final ANVLRecord namedFields, final InputStream response, final long responseLength) throws IOException { - writeRecord(REVISIT, url, create14DigitDate, + writeRecord(WARCRecordType.REVISIT, url, create14DigitDate, mimetype, recordId, namedFields, response, responseLength, false); } - public void writeMetadataRecord(final String url, - final String create14DigitDate, final String mimetype, - final URI recordId, - final ANVLRecord namedFields, final InputStream metadata, - final long metadataLength) - throws IOException { - writeRecord(METADATA, url, create14DigitDate, - mimetype, recordId, namedFields, metadata, - metadataLength, true); - } - /** * @see WARCWriter#tmpStats for usage model */ diff --git a/commons/src/test/java/org/archive/io/warc/WARCWriterTest.java b/commons/src/test/java/org/archive/io/warc/WARCWriterTest.java index fad081fe..bd4f3b3e 100644 --- a/commons/src/test/java/org/archive/io/warc/WARCWriterTest.java +++ b/commons/src/test/java/org/archive/io/warc/WARCWriterTest.java @@ -144,12 +144,12 @@ extends TmpDirTestCase implements WARCConstants { headerFields.addLabelValue("x", "y"); headerFields.addLabelValue("a", "b"); - URI rid = (new UUIDGenerator()).getQualifiedRecordID(TYPE, METADATA); + URI rid = (new UUIDGenerator()).getQualifiedRecordID(TYPE, WARCRecordType.METADATA.toString()); final String content = "Any old content."; for (int i = 0; i < 10; i++) { String body = i + ". " + content; byte [] bodyBytes = body.getBytes(UTF8Bytes.UTF8); - writer.writeRecord(METADATA, "http://www.archive.org/", + writer.writeRecord(WARCRecordType.METADATA, "http://www.archive.org/", ArchiveUtils.get14DigitDate(), "no/type", rid, headerFields, new ByteArrayInputStream(bodyBytes), (long)bodyBytes.length, true); diff --git a/modules/src/main/java/org/archive/modules/writer/WARCWriterProcessor.java b/modules/src/main/java/org/archive/modules/writer/WARCWriterProcessor.java index 7aa407f3..1ce328d2 100644 --- a/modules/src/main/java/org/archive/modules/writer/WARCWriterProcessor.java +++ b/modules/src/main/java/org/archive/modules/writer/WARCWriterProcessor.java @@ -29,13 +29,11 @@ import static org.archive.io.warc.WARCConstants.HEADER_KEY_PROFILE; import static org.archive.io.warc.WARCConstants.HEADER_KEY_TRUNCATED; import static org.archive.io.warc.WARCConstants.HTTP_REQUEST_MIMETYPE; import static org.archive.io.warc.WARCConstants.HTTP_RESPONSE_MIMETYPE; -import static org.archive.io.warc.WARCConstants.METADATA; import static org.archive.io.warc.WARCConstants.NAMED_FIELD_TRUNCATED_VALUE_HEAD; import static org.archive.io.warc.WARCConstants.NAMED_FIELD_TRUNCATED_VALUE_LENGTH; import static org.archive.io.warc.WARCConstants.NAMED_FIELD_TRUNCATED_VALUE_TIME; import static org.archive.io.warc.WARCConstants.PROFILE_REVISIT_IDENTICAL_DIGEST; import static org.archive.io.warc.WARCConstants.PROFILE_REVISIT_NOT_MODIFIED; -import static org.archive.io.warc.WARCConstants.REQUEST; import static org.archive.io.warc.WARCConstants.TYPE; import static org.archive.modules.CoreAttributeConstants.A_DNS_SERVER_IP_LABEL; import static org.archive.modules.CoreAttributeConstants.A_FTP_CONTROL_CONVERSATION; @@ -74,6 +72,7 @@ import org.apache.commons.io.IOUtils; import org.apache.commons.lang.StringUtils; import org.archive.io.ArchiveFileConstants; import org.archive.io.ReplayInputStream; +import org.archive.io.warc.WARCConstants.WARCRecordType; import org.archive.io.warc.WARCWriter; import org.archive.io.warc.WARCWriterPool; import org.archive.io.warc.WARCWriterPoolSettings; @@ -446,11 +445,11 @@ public class WARCWriterProcessor extends WriterPoolProcessor implements WARCWrit protected URI writeFtpControlConversation(WARCWriter w, String timestamp, URI baseid, CrawlURI curi, ANVLRecord headers, String controlConversation) throws IOException { - final URI uid = qualifyRecordID(baseid, TYPE, METADATA); + final URI uid = qualifyRecordID(baseid, TYPE, WARCRecordType.METADATA.toString()); byte[] b = controlConversation.getBytes("UTF-8"); - w.writeMetadataRecord(curi.toString(), timestamp, - FTP_CONTROL_CONVERSATION_MIMETYPE, uid, headers, - new ByteArrayInputStream(b), b.length); + w.writeRecord(WARCRecordType.METADATA, curi.toString(), timestamp, + FTP_CONTROL_CONVERSATION_MIMETYPE, uid, headers, new ByteArrayInputStream(b), + (long) b.length, true); return uid; } @@ -459,7 +458,7 @@ public class WARCWriterProcessor extends WriterPoolProcessor implements WARCWrit final URI baseid, final CrawlURI curi, final ANVLRecord namedFields) throws IOException { - final URI uid = qualifyRecordID(baseid, TYPE, REQUEST); + final URI uid = qualifyRecordID(baseid, TYPE, WARCRecordType.REQUEST.toString()); ReplayInputStream ris = curi.getRecorder().getRecordedOutput().getReplayInputStream(); try { @@ -588,7 +587,7 @@ public class WARCWriterProcessor extends WriterPoolProcessor implements WARCWrit final URI baseid, final CrawlURI curi, final ANVLRecord namedFields) throws IOException { - final URI uid = qualifyRecordID(baseid, TYPE, METADATA); + final URI uid = qualifyRecordID(baseid, TYPE, WARCRecordType.METADATA.toString()); // Get some metadata from the curi. // TODO: Get all curi metadata. // TODO: Use other than ANVL (or rename ANVL as NameValue or use @@ -649,8 +648,9 @@ public class WARCWriterProcessor extends WriterPoolProcessor implements WARCWrit // Annotations. byte [] b = r.getUTF8Bytes(); - w.writeMetadataRecord(curi.toString(), timestamp, ANVLRecord.MIMETYPE, - uid, namedFields, new ByteArrayInputStream(b), b.length); + w.writeRecord(WARCRecordType.METADATA, curi.toString(), timestamp, + ANVLRecord.MIMETYPE, uid, namedFields, new ByteArrayInputStream(b), + (long) b.length, true); return uid; } @@ -735,10 +735,10 @@ public class WARCWriterProcessor extends WriterPoolProcessor implements WARCWrit buf.append("Processor: " + getClass().getName() + "\n"); buf.append(" Function: Writes WARCs\n"); buf.append(" Total CrawlURIs: " + urlsWritten + "\n"); - buf.append(" Revisit records: " + WARCWriter.getStat(stats, WARCWriter.REVISIT, WARCWriter.NUM_RECORDS) + "\n"); + buf.append(" Revisit records: " + WARCWriter.getStat(stats, WARCRecordType.REVISIT.toString(), WARCWriter.NUM_RECORDS) + "\n"); - long bytes = WARCWriter.getStat(stats, WARCWriter.RESPONSE, WARCWriter.CONTENT_BYTES) - + WARCWriter.getStat(stats, WARCWriter.RESOURCE, WARCWriter.CONTENT_BYTES); + long bytes = WARCWriter.getStat(stats, WARCRecordType.RESPONSE.toString(), WARCWriter.CONTENT_BYTES) + + WARCWriter.getStat(stats, WARCRecordType.RESOURCE.toString(), WARCWriter.CONTENT_BYTES); buf.append(" Crawled content bytes (including http headers): " + bytes + " (" + ArchiveUtils.formatBytesForDisplay(bytes) + ")\n"); diff --git a/modules/src/test/java/org/archive/modules/writer/WARCWriterProcessorTest.java b/modules/src/test/java/org/archive/modules/writer/WARCWriterProcessorTest.java index 8ea8b660..8dad6ec5 100644 --- a/modules/src/test/java/org/archive/modules/writer/WARCWriterProcessorTest.java +++ b/modules/src/test/java/org/archive/modules/writer/WARCWriterProcessorTest.java @@ -141,7 +141,7 @@ public class WARCWriterProcessorTest extends ProcessorTestBase { super(serial, settings); } @Override - protected void writeRecord(String type, String url, + public void writeRecord(WARCRecordType type, String url, String create14DigitDate, String mimetype, URI recordId, ANVLRecord xtraHeaders, InputStream contentStream, long contentLength, boolean enforceLength) throws IOException { From 360ac000003f14a723a03c612e883a76c91b9ab7 Mon Sep 17 00:00:00 2001 From: Noah Levitt Date: Mon, 10 Sep 2012 17:41:43 -0700 Subject: [PATCH 02/19] Continuing to work on cleaning up WARC writer code. * WARCRecordInfo.java get rid of constructor with arguments, add setUrl() and setType() * WARCWriter.java get rid of writeRecord() taking a million arguments * Arc2Warc.java, WARCWriter.java, WARCWriterTest.java, WARCWriterProcessor.java, WARCWriterProcessorTest.java construct WARCRecordInfo object and call WARCWriter.writeRecord(recordInfo) instead of million argument method --- .../main/java/org/archive/io/Arc2Warc.java | 25 ++- .../org/archive/io/warc/WARCRecordInfo.java | 5 +- .../java/org/archive/io/warc/WARCWriter.java | 173 +++--------------- .../org/archive/io/warc/WARCWriterTest.java | 80 +++++--- .../modules/writer/WARCWriterProcessor.java | 142 +++++++++++--- .../writer/WARCWriterProcessorTest.java | 10 +- 6 files changed, 225 insertions(+), 210 deletions(-) diff --git a/commons/src/main/java/org/archive/io/Arc2Warc.java b/commons/src/main/java/org/archive/io/Arc2Warc.java index eedd848f..96f1029b 100644 --- a/commons/src/main/java/org/archive/io/Arc2Warc.java +++ b/commons/src/main/java/org/archive/io/Arc2Warc.java @@ -41,7 +41,10 @@ import org.archive.io.arc.ARCReader; import org.archive.io.arc.ARCReaderFactory; import org.archive.io.arc.ARCRecord; import org.archive.io.warc.WARCConstants; +import org.archive.io.warc.WARCConstants.WARCRecordType; +import org.archive.io.warc.WARCRecordInfo; import org.archive.io.warc.WARCWriter; +import org.archive.io.warc.WARCWriterPoolSettings; import org.archive.io.warc.WARCWriterPoolSettingsData; import org.archive.uid.RecordIDGenerator; import org.archive.uid.UUIDGenerator; @@ -145,6 +148,11 @@ public class Arc2Warc { protected void write(final WARCWriter writer, final ARCRecord r) throws IOException { + WARCRecordInfo recordInfo = new WARCRecordInfo(); + recordInfo.setUrl(r.getHeader().getUrl()); + recordInfo.setContentStream(r); + recordInfo.setContentLength(r.getHeader().getLength()); + recordInfo.setEnforceLength(true); // convert ARC date to WARC-Date format String arcDateString = r.getHeader().getDate(); @@ -152,6 +160,7 @@ public class Arc2Warc { .withZone(DateTimeZone.UTC) .parseDateTime(arcDateString) .toString(ISODateTimeFormat.dateTimeNoMillis()); + recordInfo.setCreate14DigitDate(warcDateString); ANVLRecord ar = new ANVLRecord(); String ip = (String)r.getHeader() @@ -160,6 +169,7 @@ public class Arc2Warc { ar.addLabelValue(WARCConstants.NAMED_FIELD_IP_LABEL, ip); r.getMetaData(); } + recordInfo.setExtraHeaders(ar); // enable reconstruction of ARC from transformed WARC // TODO: deferred for further analysis (see HER-1750) @@ -167,18 +177,17 @@ public class Arc2Warc { // If contentBody > 0, assume http headers. Make the mimetype // be application/http. Otherwise, give it ARC mimetype. - String warcMimeTypeString; if (r.getHeader().getContentBegin() > 0) { - warcMimeTypeString = WARCConstants.HTTP_RESPONSE_MIMETYPE; - writer.writeResponseRecord(r.getHeader().getUrl(), warcDateString, - warcMimeTypeString, generator.getRecordID(), ar, r, - r.getHeader().getLength()); + recordInfo.setType(WARCRecordType.RESPONSE); + recordInfo.setMimetype(WARCConstants.HTTP_RESPONSE_MIMETYPE); + recordInfo.setRecordId(generator.getRecordID()); } else { - warcMimeTypeString = r.getHeader().getMimetype(); - writer.writeResourceRecord(r.getHeader().getUrl(), warcDateString, - warcMimeTypeString, ar, r, r.getHeader().getLength()); + recordInfo.setType(WARCRecordType.RESOURCE); + recordInfo.setMimetype(r.getHeader().getMimetype()); + recordInfo.setRecordId(((WARCWriterPoolSettings)writer.settings).getRecordIDGenerator().getRecordID()); } + writer.writeRecord(recordInfo); } /** diff --git a/commons/src/main/java/org/archive/io/warc/WARCRecordInfo.java b/commons/src/main/java/org/archive/io/warc/WARCRecordInfo.java index a4422508..432323b4 100644 --- a/commons/src/main/java/org/archive/io/warc/WARCRecordInfo.java +++ b/commons/src/main/java/org/archive/io/warc/WARCRecordInfo.java @@ -36,8 +36,11 @@ public class WARCRecordInfo { protected long contentLength; protected boolean enforceLength; - public WARCRecordInfo(WARCRecordType type, String url) { + public void setType(WARCRecordType type) { this.type = type; + } + + public void setUrl(String url) { this.url = url; } diff --git a/commons/src/main/java/org/archive/io/warc/WARCWriter.java b/commons/src/main/java/org/archive/io/warc/WARCWriter.java index c0127bf3..675f22d6 100644 --- a/commons/src/main/java/org/archive/io/warc/WARCWriter.java +++ b/commons/src/main/java/org/archive/io/warc/WARCWriter.java @@ -211,53 +211,22 @@ implements WARCConstants { return sb.toString(); } - /** - * @deprecated Use {@link #writeRecord(String,String,String,String,URI,ANVLRecord,InputStream,long,boolean)} instead - */ - protected void writeRecord(final WARCRecordType type, final String url, - final String create14DigitDate, final String mimetype, - final URI recordId, ANVLRecord xtraHeaders, - final InputStream contentStream, final long contentLength) + public void writeRecord(WARCRecordInfo recordInfo) throws IOException { - writeRecord(type, url, create14DigitDate, mimetype, recordId, xtraHeaders, contentStream, contentLength, true); - } - /** @deprecated */ - public void writeRecord(final WARCRecordType type, final String url, - final String create14DigitDate, final String mimetype, - final URI recordId, ANVLRecord xtraHeaders, - final InputStream contentStream, final long contentLength, - boolean enforceLength) - throws IOException { - - WARCRecordInfo metaRecord = new WARCRecordInfo(type, url); - metaRecord.setCreate14DigitDate(create14DigitDate); - metaRecord.setMimetype(mimetype); - metaRecord.setRecordId(recordId); - metaRecord.setExtraHeaders(xtraHeaders); - metaRecord.setContentStream(contentStream); - metaRecord.setContentLength(contentLength); - metaRecord.setEnforceLength(enforceLength); - - writeRecord(metaRecord); - } - - protected void writeRecord(WARCRecordInfo metaRecord) - throws IOException { - - if (metaRecord.getContentLength() == 0 && - (metaRecord.getExtraHeaders() == null || metaRecord.getExtraHeaders().size() <= 0)) { + if (recordInfo.getContentLength() == 0 && + (recordInfo.getExtraHeaders() == null || recordInfo.getExtraHeaders().size() <= 0)) { throw new IllegalArgumentException("Cannot write record " + "of content-length zero and base headers only."); } String header; try { - header = createRecordHeader(metaRecord); + header = createRecordHeader(recordInfo); } catch (IllegalArgumentException e) { - logger.log(Level.SEVERE,"could not write record type: " + metaRecord.getType() - + "for URL: " + metaRecord.getUrl(), e); + logger.log(Level.SEVERE,"could not write record type: " + recordInfo.getType() + + "for URL: " + recordInfo.getUrl(), e); return; } @@ -275,13 +244,13 @@ implements WARCConstants { totalBytes += bytes.length; - if (metaRecord.getContentStream() != null && metaRecord.getContentLength() > 0) { + if (recordInfo.getContentStream() != null && recordInfo.getContentLength() > 0) { // Write out the header/body separator. write(CRLF_BYTES); // TODO: should this be written even for zero-length? totalBytes += CRLF_BYTES.length; - contentBytes += copyFrom(metaRecord.getContentStream(), - metaRecord.getContentLength(), - metaRecord.getEnforceLength()); + contentBytes += copyFrom(recordInfo.getContentStream(), + recordInfo.getContentLength(), + recordInfo.getEnforceLength()); totalBytes += contentBytes; } @@ -294,7 +263,7 @@ implements WARCConstants { } // TODO: should this be in the finally block? - tally(metaRecord.getType(), contentBytes, totalBytes, getPosition() - startPosition); + tally(recordInfo.getType(), contentBytes, totalBytes, getPosition() - startPosition); } // if compression is enabled, sizeOnDisk means compressed bytes; if not, it @@ -366,16 +335,23 @@ implements WARCConstants { public URI writeWarcinfoRecord(String filename, final String description) throws IOException { + WARCRecordInfo recordInfo = new WARCRecordInfo(); + recordInfo.setType(WARCRecordType.WARCINFO); + recordInfo.setCreate14DigitDate(ArchiveUtils.getLog14Date()); + recordInfo.setMimetype("application/warc-fields"); + // Strip .open suffix if present. if (filename.endsWith(WriterPoolMember.OCCUPIED_SUFFIX)) { filename = filename.substring(0, filename.length() - WriterPoolMember.OCCUPIED_SUFFIX.length()); } - ANVLRecord record = new ANVLRecord(2); - record.addLabelValue(HEADER_KEY_FILENAME, filename); + ANVLRecord extraHeaders = new ANVLRecord(2); + extraHeaders.addLabelValue(HEADER_KEY_FILENAME, filename); if (description != null && description.length() > 0) { - record.addLabelValue(CONTENT_DESCRIPTION, description); + extraHeaders.addLabelValue(CONTENT_DESCRIPTION, description); } + recordInfo.setExtraHeaders(extraHeaders); + // Add warcinfo body. byte [] warcinfoBody = null; if (settings.getMetadata() == null) { @@ -389,106 +365,17 @@ implements WARCConstants { } warcinfoBody = baos.toByteArray(); } - URI uri = writeWarcinfoRecord("application/warc-fields", record, - new ByteArrayInputStream(warcinfoBody), warcinfoBody.length); + recordInfo.setContentStream(new ByteArrayInputStream(warcinfoBody)); + recordInfo.setContentLength((long) warcinfoBody.length); + recordInfo.setEnforceLength(true); + + recordInfo.setRecordId(generateRecordId(TYPE, WARCRecordType.WARCINFO.toString())); + + writeRecord(recordInfo); + // TODO: If at start of file, and we're writing compressed, // write out our distinctive GZIP extensions. - return uri; - } - - /** - * Write a warcinfo to current file. - * TODO: Write crawl metadata or pointers to crawl description. - * @param mimetype Mimetype of the fileMetadata block. - * @param namedFields Named fields. Pass null if none. - * @param fileMetadata Metadata about this WARC as RDF, ANVL, etc. - * @param fileMetadataLength Length of fileMetadata. - * @throws IOException - * @return Generated record-id made with - * data: scheme and - * the current filename. - */ - public URI writeWarcinfoRecord(final String mimetype, - final ANVLRecord namedFields, final InputStream fileMetadata, - final long fileMetadataLength) - throws IOException { - final URI recordid = generateRecordId(TYPE, WARCRecordType.WARCINFO.toString()); - writeWarcinfoRecord(ArchiveUtils.getLog14Date(), mimetype, recordid, - namedFields, fileMetadata, fileMetadataLength); - return recordid; - } - - /** - * Write a warcinfo to current file. - * The warcinfo type uses its recordId as its URL. - * @param recordId URI to use for this warcinfo. - * @param create14DigitDate Record creation date as 14 digit date. - * @param mimetype Mimetype of the fileMetadata. - * @param namedFields Named fields. - * @param fileMetadata Metadata about this WARC as RDF, ANVL, etc. - * @param fileMetadataLength Length of fileMetadata. - * @throws IOException - */ - public void writeWarcinfoRecord(final String create14DigitDate, - final String mimetype, final URI recordId, final ANVLRecord namedFields, - final InputStream fileMetadata, final long fileMetadataLength) - throws IOException { - writeRecord(WARCRecordType.WARCINFO, null, create14DigitDate, mimetype, - recordId, namedFields, fileMetadata, fileMetadataLength, true); - } - - public void writeRequestRecord(final String url, - final String create14DigitDate, final String mimetype, - final URI recordId, - final ANVLRecord namedFields, final InputStream request, - final long requestLength) - throws IOException { - writeRecord(WARCRecordType.REQUEST, url, create14DigitDate, - mimetype, recordId, namedFields, request, - requestLength, true); - } - - public void writeResourceRecord(final String url, - final String create14DigitDate, final String mimetype, - final ANVLRecord namedFields, final InputStream response, - final long responseLength) - throws IOException { - writeResourceRecord(url, create14DigitDate, mimetype, - ((WARCWriterPoolSettings)settings).getRecordIDGenerator().getRecordID(), - namedFields, response, responseLength); - } - - public void writeResourceRecord(final String url, - final String create14DigitDate, final String mimetype, - final URI recordId, - final ANVLRecord namedFields, final InputStream response, - final long responseLength) - throws IOException { - writeRecord(WARCRecordType.RESOURCE, url, create14DigitDate, - mimetype, recordId, namedFields, response, - responseLength, true); - } - - public void writeResponseRecord(final String url, - final String create14DigitDate, final String mimetype, - final URI recordId, - final ANVLRecord namedFields, final InputStream response, - final long responseLength) - throws IOException { - writeRecord(WARCRecordType.RESPONSE, url, create14DigitDate, - mimetype, recordId, namedFields, response, - responseLength, true); - } - - public void writeRevisitRecord(final String url, - final String create14DigitDate, final String mimetype, - final URI recordId, - final ANVLRecord namedFields, final InputStream response, - final long responseLength) - throws IOException { - writeRecord(WARCRecordType.REVISIT, url, create14DigitDate, - mimetype, recordId, namedFields, response, - responseLength, false); + return recordInfo.getRecordId(); } /** diff --git a/commons/src/test/java/org/archive/io/warc/WARCWriterTest.java b/commons/src/test/java/org/archive/io/warc/WARCWriterTest.java index bd4f3b3e..b366d33f 100644 --- a/commons/src/test/java/org/archive/io/warc/WARCWriterTest.java +++ b/commons/src/test/java/org/archive/io/warc/WARCWriterTest.java @@ -130,29 +130,51 @@ extends TmpDirTestCase implements WARCConstants { private void writeWarcinfoRecord(WARCWriter writer) throws IOException { + WARCRecordInfo recordInfo = new WARCRecordInfo(); + recordInfo.setType(WARCRecordType.WARCINFO); + recordInfo.setUrl(null); + recordInfo.setCreate14DigitDate(ArchiveUtils.getLog14Date()); + recordInfo.setMimetype(ANVLRecord.MIMETYPE); + recordInfo.setExtraHeaders(null); + recordInfo.setEnforceLength(true); + ANVLRecord meta = new ANVLRecord(); meta.addLabelValue("size", "1G"); meta.addLabelValue("operator", "igor"); byte [] bytes = meta.getUTF8Bytes(); - writer.writeWarcinfoRecord(ANVLRecord.MIMETYPE, null, - new ByteArrayInputStream(bytes), bytes.length); + recordInfo.setContentStream(new ByteArrayInputStream(bytes)); + recordInfo.setContentLength((long) bytes.length); + + final URI recordid = writer.generateRecordId(WARCWriter.TYPE, WARCRecordType.WARCINFO.toString()); + recordInfo.setRecordId(recordid); + + writer.writeRecord(recordInfo); } protected void writeBasicRecords(final WARCWriter writer) throws IOException { + WARCRecordInfo recordInfo = new WARCRecordInfo(); + recordInfo.setType(WARCRecordType.METADATA); + recordInfo.setUrl("http://www.archive.org/"); + recordInfo.setCreate14DigitDate(ArchiveUtils.get14DigitDate()); + recordInfo.setMimetype("no/type"); + recordInfo.setEnforceLength(true); + ANVLRecord headerFields = new ANVLRecord(); headerFields.addLabelValue("x", "y"); headerFields.addLabelValue("a", "b"); + recordInfo.setExtraHeaders(headerFields); URI rid = (new UUIDGenerator()).getQualifiedRecordID(TYPE, WARCRecordType.METADATA.toString()); + recordInfo.setRecordId(rid); + final String content = "Any old content."; for (int i = 0; i < 10; i++) { String body = i + ". " + content; byte [] bodyBytes = body.getBytes(UTF8Bytes.UTF8); - writer.writeRecord(WARCRecordType.METADATA, "http://www.archive.org/", - ArchiveUtils.get14DigitDate(), "no/type", - rid, headerFields, new ByteArrayInputStream(bodyBytes), - (long)bodyBytes.length, true); + recordInfo.setContentStream(new ByteArrayInputStream(bodyBytes)); + recordInfo.setContentLength((long)bodyBytes.length); + writer.writeRecord(recordInfo); } } @@ -186,23 +208,31 @@ extends TmpDirTestCase implements WARCConstants { */ protected int writeRandomHTTPRecord(WARCWriter w, int index) throws IOException { - ByteArrayOutputStream baos = new ByteArrayOutputStream(); + WARCRecordInfo recordInfo = new WARCRecordInfo(); + recordInfo.setType(WARCRecordType.RESOURCE); + recordInfo.setCreate14DigitDate(ArchiveUtils.get14DigitDate()); + recordInfo.setMimetype("text/html; charset=UTF-8"); + recordInfo.setRecordId(w.generateRecordId(null)); + recordInfo.setEnforceLength(true); + String indexStr = Integer.toString(index); + recordInfo.setUrl("http://www.one.net/id=" + indexStr); + + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + recordInfo.setContentStream(new ByteArrayInputStream(baos.toByteArray())); + byte[] record = (getContent(indexStr)).getBytes(); - int recordLength = record.length; + recordInfo.setContentLength((long) record.length); baos.write(record); + // Add named fields for ip, checksum, and relate the metadata // and request to the resource field. ANVLRecord r = new ANVLRecord(1); r.addLabelValue(NAMED_FIELD_IP_LABEL, "127.0.0.1"); - w.writeResourceRecord( - "http://www.one.net/id=" + indexStr, - ArchiveUtils.get14DigitDate(), - "text/html; charset=UTF-8", - r, - new ByteArrayInputStream(baos.toByteArray()), - recordLength); - return recordLength; + recordInfo.setExtraHeaders(r); + + w.writeRecord(recordInfo); + return record.length; } /** @@ -352,12 +382,18 @@ extends TmpDirTestCase implements WARCConstants { protected static void writeRecord(WARCWriter w, String url, String mimetype, int len, ByteArrayOutputStream baos) throws IOException { - w.writeResourceRecord(url, - ArchiveUtils.get14DigitDate(), - mimetype, - null, - new ByteArrayInputStream(baos.toByteArray()), - len); + WARCRecordInfo recordInfo = new WARCRecordInfo(); + recordInfo.setType(WARCRecordType.RESOURCE); + recordInfo.setUrl(url); + recordInfo.setCreate14DigitDate(ArchiveUtils.get14DigitDate()); + recordInfo.setMimetype(mimetype); + recordInfo.setRecordId(w.generateRecordId(null)); + recordInfo.setExtraHeaders(null); + recordInfo.setContentStream(new ByteArrayInputStream(baos.toByteArray())); + recordInfo.setContentLength((long) len); + recordInfo.setEnforceLength(true); + + w.writeRecord(recordInfo); } protected int iterateRecords(WARCReader r) diff --git a/modules/src/main/java/org/archive/modules/writer/WARCWriterProcessor.java b/modules/src/main/java/org/archive/modules/writer/WARCWriterProcessor.java index 1ce328d2..2dc6a109 100644 --- a/modules/src/main/java/org/archive/modules/writer/WARCWriterProcessor.java +++ b/modules/src/main/java/org/archive/modules/writer/WARCWriterProcessor.java @@ -73,6 +73,7 @@ import org.apache.commons.lang.StringUtils; import org.archive.io.ArchiveFileConstants; import org.archive.io.ReplayInputStream; import org.archive.io.warc.WARCConstants.WARCRecordType; +import org.archive.io.warc.WARCRecordInfo; import org.archive.io.warc.WARCWriter; import org.archive.io.warc.WARCWriterPool; import org.archive.io.warc.WARCWriterPoolSettings; @@ -445,12 +446,25 @@ public class WARCWriterProcessor extends WriterPoolProcessor implements WARCWrit protected URI writeFtpControlConversation(WARCWriter w, String timestamp, URI baseid, CrawlURI curi, ANVLRecord headers, String controlConversation) throws IOException { - final URI uid = qualifyRecordID(baseid, TYPE, WARCRecordType.METADATA.toString()); + + WARCRecordInfo recordInfo = new WARCRecordInfo(); + recordInfo.setCreate14DigitDate(timestamp); + recordInfo.setUrl(curi.toString()); + recordInfo.setMimetype(FTP_CONTROL_CONVERSATION_MIMETYPE); + recordInfo.setExtraHeaders(headers); + recordInfo.setEnforceLength(true); + recordInfo.setType(WARCRecordType.METADATA); + + recordInfo.setRecordId(qualifyRecordID(baseid, TYPE, WARCRecordType.METADATA.toString())); + byte[] b = controlConversation.getBytes("UTF-8"); - w.writeRecord(WARCRecordType.METADATA, curi.toString(), timestamp, - FTP_CONTROL_CONVERSATION_MIMETYPE, uid, headers, new ByteArrayInputStream(b), - (long) b.length, true); - return uid; + + recordInfo.setContentStream(new ByteArrayInputStream(b)); + recordInfo.setContentLength((long) b.length); + + w.writeRecord(recordInfo); + + return recordInfo.getRecordId(); } protected URI writeRequest(final WARCWriter w, @@ -458,17 +472,29 @@ public class WARCWriterProcessor extends WriterPoolProcessor implements WARCWrit final URI baseid, final CrawlURI curi, final ANVLRecord namedFields) throws IOException { + WARCRecordInfo recordInfo = new WARCRecordInfo(); + recordInfo.setType(WARCRecordType.REQUEST); + recordInfo.setUrl(curi.toString()); + recordInfo.setCreate14DigitDate(timestamp); + recordInfo.setMimetype(mimetype); + recordInfo.setExtraHeaders(namedFields); + recordInfo.setContentLength(curi.getRecorder().getRecordedOutput().getSize()); + recordInfo.setEnforceLength(true); + final URI uid = qualifyRecordID(baseid, TYPE, WARCRecordType.REQUEST.toString()); + recordInfo.setRecordId(uid); + ReplayInputStream ris = curi.getRecorder().getRecordedOutput().getReplayInputStream(); + recordInfo.setContentStream(ris); + try { - w.writeRequestRecord(curi.toString(), timestamp, mimetype, uid, - namedFields, ris, - curi.getRecorder().getRecordedOutput().getSize()); + w.writeRecord(recordInfo); } finally { IOUtils.closeQuietly(ris); } - return uid; + + return recordInfo.getRecordId(); } protected URI writeResponse(final WARCWriter w, @@ -476,16 +502,27 @@ public class WARCWriterProcessor extends WriterPoolProcessor implements WARCWrit final URI baseid, final CrawlURI curi, final ANVLRecord namedFields) throws IOException { + WARCRecordInfo recordInfo = new WARCRecordInfo(); + recordInfo.setType(WARCRecordType.RESPONSE); + recordInfo.setUrl(curi.toString()); + recordInfo.setCreate14DigitDate(timestamp); + recordInfo.setMimetype(mimetype); + recordInfo.setRecordId(baseid); + recordInfo.setExtraHeaders(namedFields); + recordInfo.setContentLength(curi.getRecorder().getRecordedInput().getSize()); + recordInfo.setEnforceLength(true); + ReplayInputStream ris = curi.getRecorder().getRecordedInput().getReplayInputStream(); + recordInfo.setContentStream(ris); + try { - w.writeResponseRecord(curi.toString(), timestamp, mimetype, baseid, - namedFields, ris, - curi.getRecorder().getRecordedInput().getSize()); + w.writeRecord(recordInfo); } finally { IOUtils.closeQuietly(ris); } - return baseid; + + return recordInfo.getRecordId(); } protected URI writeResource(final WARCWriter w, @@ -493,15 +530,25 @@ public class WARCWriterProcessor extends WriterPoolProcessor implements WARCWrit final URI baseid, final CrawlURI curi, final ANVLRecord namedFields) throws IOException { + WARCRecordInfo recordInfo = new WARCRecordInfo(); + recordInfo.setType(WARCRecordType.RESOURCE); + recordInfo.setUrl(curi.toString()); + recordInfo.setCreate14DigitDate(timestamp); + recordInfo.setMimetype(mimetype); + recordInfo.setRecordId(baseid); + recordInfo.setExtraHeaders(namedFields); + recordInfo.setContentLength(curi.getRecorder().getRecordedInput().getSize()); + recordInfo.setEnforceLength(true); + ReplayInputStream ris = curi.getRecorder().getRecordedInput().getReplayInputStream(); + recordInfo.setContentStream(ris); try { - w.writeResourceRecord(curi.toString(), timestamp, mimetype, baseid, - namedFields, ris, - curi.getRecorder().getRecordedInput().getSize()); + w.writeRecord(recordInfo); } finally { IOUtils.closeQuietly(ris); } - return baseid; + + return recordInfo.getRecordId(); } protected URI writeRevisitDigest(final WARCWriter w, @@ -521,20 +568,33 @@ public class WARCWriterProcessor extends WriterPoolProcessor implements WARCWrit final String timestamp, final String mimetype, final URI baseid, final CrawlURI curi, final ANVLRecord namedFields, long contentLength) throws IOException { + WARCRecordInfo recordInfo = new WARCRecordInfo(); + recordInfo.setType(WARCRecordType.REVISIT); + recordInfo.setUrl(curi.toString()); + recordInfo.setCreate14DigitDate(timestamp); + recordInfo.setMimetype(mimetype); + recordInfo.setRecordId(baseid); + recordInfo.setContentLength(contentLength); + recordInfo.setEnforceLength(false); + namedFields.addLabelValue( HEADER_KEY_PROFILE, PROFILE_REVISIT_IDENTICAL_DIGEST); namedFields.addLabelValue( HEADER_KEY_TRUNCATED, NAMED_FIELD_TRUNCATED_VALUE_LENGTH); + recordInfo.setExtraHeaders(namedFields); + ReplayInputStream ris = curi.getRecorder().getRecordedInput().getReplayInputStream(); + recordInfo.setContentStream(ris); + try { - w.writeRevisitRecord(curi.toString(), timestamp, mimetype, baseid, - namedFields, ris, contentLength); + w.writeRecord(recordInfo); } finally { IOUtils.closeQuietly(ris); } curi.getAnnotations().add("warcRevisit:digest"); - return baseid; + + return recordInfo.getRecordId(); } protected URI writeRevisitNotModified(final WARCWriter w, @@ -542,10 +602,22 @@ public class WARCWriterProcessor extends WriterPoolProcessor implements WARCWrit final URI baseid, final CrawlURI puri, final ANVLRecord namedFields) throws IOException { - CrawlURI curi = (CrawlURI) puri; + CrawlURI curi = (CrawlURI) puri; + + WARCRecordInfo recordInfo = new WARCRecordInfo(); + recordInfo.setType(WARCRecordType.REVISIT); + recordInfo.setUrl(curi.toString()); + recordInfo.setCreate14DigitDate(timestamp); + recordInfo.setMimetype(null); + recordInfo.setRecordId(baseid); + recordInfo.setContentLength((long) 0); + recordInfo.setEnforceLength(false); + namedFields.addLabelValue( HEADER_KEY_PROFILE, PROFILE_REVISIT_NOT_MODIFIED); // save just enough context to understand basis of not-modified + recordInfo.setExtraHeaders(namedFields); + if(curi.isHttpTransaction()) { HttpMethod method = curi.getHttpMethod(); saveHeader(A_ETAG_HEADER,method,namedFields,HEADER_KEY_ETAG); @@ -557,14 +629,15 @@ public class WARCWriterProcessor extends WriterPoolProcessor implements WARCWrit NAMED_FIELD_TRUNCATED_VALUE_LENGTH); ReplayInputStream ris = curi.getRecorder().getRecordedInput().getReplayInputStream(); + recordInfo.setContentStream(ris); + try { - w.writeRevisitRecord(curi.toString(), timestamp, null, baseid, - namedFields, ris, 0); + w.writeRecord(recordInfo); } finally { IOUtils.closeQuietly(ris); } curi.getAnnotations().add("warcRevisit:notModified"); - return baseid; + return recordInfo.getRecordId(); } /** @@ -587,7 +660,16 @@ public class WARCWriterProcessor extends WriterPoolProcessor implements WARCWrit final URI baseid, final CrawlURI curi, final ANVLRecord namedFields) throws IOException { - final URI uid = qualifyRecordID(baseid, TYPE, WARCRecordType.METADATA.toString()); + WARCRecordInfo recordInfo = new WARCRecordInfo(); + recordInfo.setType(WARCRecordType.METADATA); + recordInfo.setUrl(curi.toString()); + recordInfo.setCreate14DigitDate(timestamp); + recordInfo.setMimetype(ANVLRecord.MIMETYPE); + recordInfo.setExtraHeaders(namedFields); + recordInfo.setEnforceLength(true); + + recordInfo.setRecordId(qualifyRecordID(baseid, TYPE, WARCRecordType.METADATA.toString())); + // Get some metadata from the curi. // TODO: Get all curi metadata. // TODO: Use other than ANVL (or rename ANVL as NameValue or use @@ -648,10 +730,12 @@ public class WARCWriterProcessor extends WriterPoolProcessor implements WARCWrit // Annotations. byte [] b = r.getUTF8Bytes(); - w.writeRecord(WARCRecordType.METADATA, curi.toString(), timestamp, - ANVLRecord.MIMETYPE, uid, namedFields, new ByteArrayInputStream(b), - (long) b.length, true); - return uid; + recordInfo.setContentStream(new ByteArrayInputStream(b)); + recordInfo.setContentLength((long) b.length); + + w.writeRecord(recordInfo); + + return recordInfo.getRecordId(); } protected URI getRecordID() throws IOException { diff --git a/modules/src/test/java/org/archive/modules/writer/WARCWriterProcessorTest.java b/modules/src/test/java/org/archive/modules/writer/WARCWriterProcessorTest.java index 8dad6ec5..83b1694c 100644 --- a/modules/src/test/java/org/archive/modules/writer/WARCWriterProcessorTest.java +++ b/modules/src/test/java/org/archive/modules/writer/WARCWriterProcessorTest.java @@ -21,8 +21,6 @@ package org.archive.modules.writer; import java.io.File; import java.io.IOException; -import java.io.InputStream; -import java.net.URI; import java.util.Arrays; import java.util.Collection; import java.util.Collections; @@ -32,6 +30,7 @@ import org.apache.commons.httpclient.methods.GetMethod; import org.archive.io.WriterPool; import org.archive.io.WriterPoolMember; import org.archive.io.WriterPoolSettings; +import org.archive.io.warc.WARCRecordInfo; import org.archive.io.warc.WARCWriter; import org.archive.io.warc.WARCWriterPoolSettingsData; import org.archive.modules.CrawlMetadata; @@ -44,7 +43,6 @@ import org.archive.uid.RecordIDGenerator; import org.archive.uid.UUIDGenerator; import org.archive.util.FileUtils; import org.archive.util.TmpDirTestCase; -import org.archive.util.anvl.ANVLRecord; /** * Unit test for {@link WARCWriterProcessor}. @@ -140,11 +138,9 @@ public class WARCWriterProcessorTest extends ProcessorTestBase { public FailWARCWriter(AtomicInteger serial, WARCWriterPoolSettingsData settings) { super(serial, settings); } + @Override - public void writeRecord(WARCRecordType type, String url, - String create14DigitDate, String mimetype, URI recordId, - ANVLRecord xtraHeaders, InputStream contentStream, - long contentLength, boolean enforceLength) throws IOException { + public void writeRecord(WARCRecordInfo recordInfo) throws IOException { throw new IOException("pretend no space left on device"); } } From a414f522f60e8002a1bf1b72b468ff70b907a00e Mon Sep 17 00:00:00 2001 From: Noah Levitt Date: Mon, 10 Sep 2012 18:03:25 -0700 Subject: [PATCH 03/19] Continuing to work on cleaning up WARC writer code: simply creation of WARC record headers. * ANVLRecord.java extend LinkedList instead of ArrayList (it was inane to guess the number of records everywhere, when it was never much more than ~10) * ANVLRecords.java remove (seemed to be an incomplete duplicate of ANVLRecord) * WARCRecordInfo.java addExtraHeader() - convenience method * Arc2Warc.java, WARCWriter.java, WARCWriter.java, WARCWriterProcessor.java use WARCRecordInfo.addExtraHeader() where appropriate, and use no-argument ANVLRecord constructor instead of deprecated ANVLRecord(int) --- .../main/java/org/archive/io/Arc2Warc.java | 2 +- .../org/archive/io/warc/WARCRecordInfo.java | 7 ++ .../java/org/archive/io/warc/WARCWriter.java | 8 +-- .../org/archive/util/anvl/ANVLRecord.java | 7 +- .../org/archive/util/anvl/ANVLRecords.java | 61 ------------------ .../org/archive/io/warc/WARCWriterTest.java | 4 +- .../modules/writer/WARCWriterProcessor.java | 64 +++++++++++++++---- 7 files changed, 65 insertions(+), 88 deletions(-) delete mode 100644 commons/src/main/java/org/archive/util/anvl/ANVLRecords.java diff --git a/commons/src/main/java/org/archive/io/Arc2Warc.java b/commons/src/main/java/org/archive/io/Arc2Warc.java index 96f1029b..72ddf192 100644 --- a/commons/src/main/java/org/archive/io/Arc2Warc.java +++ b/commons/src/main/java/org/archive/io/Arc2Warc.java @@ -104,7 +104,7 @@ public class Arc2Warc { getLength()); firstRecord.dump(baos); // Add ARC first record content as an ANVLRecord. - ANVLRecord ar = new ANVLRecord(1); + ANVLRecord ar = new ANVLRecord(); ar.addLabelValue("Filedesc", baos.toString()); List metadata = new ArrayList(1); metadata.add(ar.toString()); diff --git a/commons/src/main/java/org/archive/io/warc/WARCRecordInfo.java b/commons/src/main/java/org/archive/io/warc/WARCRecordInfo.java index 432323b4..be7cff5b 100644 --- a/commons/src/main/java/org/archive/io/warc/WARCRecordInfo.java +++ b/commons/src/main/java/org/archive/io/warc/WARCRecordInfo.java @@ -111,4 +111,11 @@ public class WARCRecordInfo { public String getUrl() { return url; } + + public void addExtraHeader(String label, String value) { + if (extraHeaders == null) { + extraHeaders = new ANVLRecord(); + } + extraHeaders.addLabelValue(label, value); + } } diff --git a/commons/src/main/java/org/archive/io/warc/WARCWriter.java b/commons/src/main/java/org/archive/io/warc/WARCWriter.java index 675f22d6..40abe4c5 100644 --- a/commons/src/main/java/org/archive/io/warc/WARCWriter.java +++ b/commons/src/main/java/org/archive/io/warc/WARCWriter.java @@ -23,7 +23,6 @@ import java.io.ByteArrayInputStream; import java.io.ByteArrayOutputStream; import java.io.File; import java.io.IOException; -import java.io.InputStream; import java.io.OutputStream; import java.net.URI; import java.util.HashMap; @@ -41,7 +40,6 @@ import org.archive.io.UTF8Bytes; import org.archive.io.WriterPoolMember; import org.archive.modules.writer.WARCWriterProcessor; import org.archive.util.ArchiveUtils; -import org.archive.util.anvl.ANVLRecord; import org.archive.util.anvl.Element; @@ -345,12 +343,10 @@ implements WARCConstants { filename = filename.substring(0, filename.length() - WriterPoolMember.OCCUPIED_SUFFIX.length()); } - ANVLRecord extraHeaders = new ANVLRecord(2); - extraHeaders.addLabelValue(HEADER_KEY_FILENAME, filename); + recordInfo.addExtraHeader(HEADER_KEY_FILENAME, filename); if (description != null && description.length() > 0) { - extraHeaders.addLabelValue(CONTENT_DESCRIPTION, description); + recordInfo.addExtraHeader(CONTENT_DESCRIPTION, description); } - recordInfo.setExtraHeaders(extraHeaders); // Add warcinfo body. byte [] warcinfoBody = null; diff --git a/commons/src/main/java/org/archive/util/anvl/ANVLRecord.java b/commons/src/main/java/org/archive/util/anvl/ANVLRecord.java index cfabb1dc..ad368fee 100644 --- a/commons/src/main/java/org/archive/util/anvl/ANVLRecord.java +++ b/commons/src/main/java/org/archive/util/anvl/ANVLRecord.java @@ -22,10 +22,10 @@ import java.io.ByteArrayOutputStream; import java.io.IOException; import java.io.InputStream; import java.io.UnsupportedEncodingException; -import java.util.ArrayList; import java.util.Collection; import java.util.HashMap; import java.util.Iterator; +import java.util.LinkedList; import java.util.List; import java.util.Map; import java.util.logging.Level; @@ -42,7 +42,7 @@ import org.archive.io.UTF8Bytes; * Language (ANVL) * @author stack */ -public class ANVLRecord extends ArrayList implements UTF8Bytes { +public class ANVLRecord extends LinkedList implements UTF8Bytes { private static final long serialVersionUID = -4610638888453052958L; private static final Logger logger = Logger.getLogger(ANVLRecord.class.getName()); @@ -73,8 +73,9 @@ public class ANVLRecord extends ArrayList implements UTF8Bytes { super(c); } + /** @deprecated */ public ANVLRecord(int initialCapacity) { - super(initialCapacity); + super(); } public boolean addLabel(final String l) { diff --git a/commons/src/main/java/org/archive/util/anvl/ANVLRecords.java b/commons/src/main/java/org/archive/util/anvl/ANVLRecords.java deleted file mode 100644 index 8ed5f43c..00000000 --- a/commons/src/main/java/org/archive/util/anvl/ANVLRecords.java +++ /dev/null @@ -1,61 +0,0 @@ -/* - * This file is part of the Heritrix web crawler (crawler.archive.org). - * - * Licensed to the Internet Archive (IA) by one or more individual - * contributors. - * - * The IA licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.archive.util.anvl; - -import java.io.UnsupportedEncodingException; -import java.util.ArrayList; -import java.util.Collection; -import java.util.Iterator; - -import org.archive.io.UTF8Bytes; - -/** - * List of {@link ANVLRecord}s. - * @author stack - * @version $Date$ $Version$ - */ -public class ANVLRecords extends ArrayList implements UTF8Bytes { - private static final long serialVersionUID = 5361551920550106113L; - - public ANVLRecords() { - super(); - } - - public ANVLRecords(int initialCapacity) { - super(initialCapacity); - } - - public ANVLRecords(Collection c) { - super(c); - } - - public byte[] getUTF8Bytes() throws UnsupportedEncodingException { - return toString().getBytes(UTF8); - } - - @Override - public String toString() { - StringBuilder sb = new StringBuilder(); - for (final Iterator i = iterator(); i.hasNext();) { - sb.append(i.next().toString()); - } - return super.toString(); - } -} \ No newline at end of file diff --git a/commons/src/test/java/org/archive/io/warc/WARCWriterTest.java b/commons/src/test/java/org/archive/io/warc/WARCWriterTest.java index b366d33f..fa19a5e2 100644 --- a/commons/src/test/java/org/archive/io/warc/WARCWriterTest.java +++ b/commons/src/test/java/org/archive/io/warc/WARCWriterTest.java @@ -227,9 +227,7 @@ extends TmpDirTestCase implements WARCConstants { // Add named fields for ip, checksum, and relate the metadata // and request to the resource field. - ANVLRecord r = new ANVLRecord(1); - r.addLabelValue(NAMED_FIELD_IP_LABEL, "127.0.0.1"); - recordInfo.setExtraHeaders(r); + recordInfo.addExtraHeader(NAMED_FIELD_IP_LABEL, "127.0.0.1"); w.writeRecord(recordInfo); return record.length; diff --git a/modules/src/main/java/org/archive/modules/writer/WARCWriterProcessor.java b/modules/src/main/java/org/archive/modules/writer/WARCWriterProcessor.java index 2dc6a109..c66bf3f2 100644 --- a/modules/src/main/java/org/archive/modules/writer/WARCWriterProcessor.java +++ b/modules/src/main/java/org/archive/modules/writer/WARCWriterProcessor.java @@ -320,24 +320,60 @@ public class WARCWriterProcessor extends WriterPoolProcessor implements WARCWrit private void writeDnsRecords(final CrawlURI curi, WARCWriter w, final URI baseid, final String timestamp) throws IOException { - ANVLRecord headers = null; + WARCRecordInfo recordInfo = new WARCRecordInfo(); + recordInfo.setType(WARCRecordType.RESPONSE); + recordInfo.setUrl(curi.toString()); + recordInfo.setCreate14DigitDate(timestamp); + recordInfo.setMimetype(curi.getContentType()); + recordInfo.setRecordId(baseid); + + recordInfo.setContentLength(curi.getRecorder().getRecordedInput().getSize()); + recordInfo.setEnforceLength(true); + String ip = (String)curi.getData().get(A_DNS_SERVER_IP_LABEL); if (ip != null && ip.length() > 0) { - headers = new ANVLRecord(1); - headers.addLabelValue(HEADER_KEY_IP, ip); + recordInfo.addExtraHeader(HEADER_KEY_IP, ip); } - writeResponse(w, timestamp, curi.getContentType(), baseid, - curi, headers); + + ReplayInputStream ris = + curi.getRecorder().getRecordedInput().getReplayInputStream(); + recordInfo.setContentStream(ris); + + try { + w.writeRecord(recordInfo); + } finally { + IOUtils.closeQuietly(ris); + } + + recordInfo.getRecordId(); } private void writeWhoisRecords(WARCWriter w, CrawlURI curi, URI baseid, String timestamp) throws IOException { - ANVLRecord headers = new ANVLRecord(1); + WARCRecordInfo recordInfo = new WARCRecordInfo(); + recordInfo.setType(WARCRecordType.RESPONSE); + recordInfo.setUrl(curi.toString()); + recordInfo.setCreate14DigitDate(timestamp); + recordInfo.setMimetype(curi.getContentType()); + recordInfo.setRecordId(baseid); + recordInfo.setContentLength(curi.getRecorder().getRecordedInput().getSize()); + recordInfo.setEnforceLength(true); + Object whoisServerIP = curi.getData().get(CoreAttributeConstants.A_WHOIS_SERVER_IP); if (whoisServerIP != null) { - headers.addLabelValue(HEADER_KEY_IP, whoisServerIP.toString()); + recordInfo.addExtraHeader(HEADER_KEY_IP, whoisServerIP.toString()); } - writeResponse(w, timestamp, curi.getContentType(), baseid, curi, headers); + + ReplayInputStream ris = + curi.getRecorder().getRecordedInput().getReplayInputStream(); + recordInfo.setContentStream(ris); + + try { + w.writeRecord(recordInfo); + } finally { + IOUtils.closeQuietly(ris); + } + recordInfo.getRecordId(); } private void writeHttpRecords(final CrawlURI curi, WARCWriter w, @@ -346,7 +382,7 @@ public class WARCWriterProcessor extends WriterPoolProcessor implements WARCWrit // and request to the resource field. // TODO: Use other than ANVL (or rename ANVL as NameValue or // use RFC822 (commons-httpclient?). - ANVLRecord headers = new ANVLRecord(5); + ANVLRecord headers = new ANVLRecord(); if (curi.getContentDigest() != null) { headers.addLabelValue(HEADER_KEY_PAYLOAD_DIGEST, curi.getContentDigestSchemeString()); @@ -381,7 +417,7 @@ public class WARCWriterProcessor extends WriterPoolProcessor implements WARCWrit baseid, curi, headers); } - headers = new ANVLRecord(1); + headers = new ANVLRecord(); headers.addLabelValue(HEADER_KEY_CONCURRENT_TO, '<' + rid.toString() + '>'); @@ -396,7 +432,7 @@ public class WARCWriterProcessor extends WriterPoolProcessor implements WARCWrit private void writeFtpRecords(WARCWriter w, final CrawlURI curi, final URI baseid, final String timestamp) throws IOException { - ANVLRecord headers = new ANVLRecord(3); + ANVLRecord headers = new ANVLRecord(); headers.addLabelValue(HEADER_KEY_IP, getHostAddress(curi)); String controlConversation = curi.getData().get(A_FTP_CONTROL_CONVERSATION).toString(); URI rid = writeFtpControlConversation(w, timestamp, baseid, curi, headers, controlConversation); @@ -412,7 +448,7 @@ public class WARCWriterProcessor extends WriterPoolProcessor implements WARCWrit rid = writeRevisitDigest(w, timestamp, null, baseid, curi, headers, 0); } else { - headers = new ANVLRecord(3); + headers = new ANVLRecord(); // Check for truncated annotation String value = null; Collection anno = curi.getAnnotations(); @@ -437,7 +473,7 @@ public class WARCWriterProcessor extends WriterPoolProcessor implements WARCWrit } } if (getWriteMetadata()) { - headers = new ANVLRecord(1); + headers = new ANVLRecord(); headers.addLabelValue(HEADER_KEY_CONCURRENT_TO, '<' + rid.toString() + '>'); writeMetadata(w, timestamp, baseid, curi, headers); } @@ -754,7 +790,7 @@ public class WARCWriterProcessor extends WriterPoolProcessor implements WARCWrit if (cachedMetadata != null) { return cachedMetadata; } - ANVLRecord record = new ANVLRecord(7); + ANVLRecord record = new ANVLRecord(); record.addLabelValue("software", "Heritrix/" + ArchiveUtils.VERSION + " http://crawler.archive.org"); try { From c4faa444a2f28a0909ec55260e4377f902f7e06a Mon Sep 17 00:00:00 2001 From: Noah Levitt Date: Mon, 10 Sep 2012 18:14:15 -0700 Subject: [PATCH 04/19] * ANVLRecord.java remove unneeded(?) serialVersionUID --- commons/src/main/java/org/archive/util/anvl/ANVLRecord.java | 1 - 1 file changed, 1 deletion(-) diff --git a/commons/src/main/java/org/archive/util/anvl/ANVLRecord.java b/commons/src/main/java/org/archive/util/anvl/ANVLRecord.java index ad368fee..de2d3101 100644 --- a/commons/src/main/java/org/archive/util/anvl/ANVLRecord.java +++ b/commons/src/main/java/org/archive/util/anvl/ANVLRecord.java @@ -43,7 +43,6 @@ import org.archive.io.UTF8Bytes; * @author stack */ public class ANVLRecord extends LinkedList implements UTF8Bytes { - private static final long serialVersionUID = -4610638888453052958L; private static final Logger logger = Logger.getLogger(ANVLRecord.class.getName()); From 2407fc5dd02367ece1f4d8e65a5fce51e232664c Mon Sep 17 00:00:00 2001 From: Noah Levitt Date: Mon, 10 Sep 2012 18:18:12 -0700 Subject: [PATCH 05/19] Beginning work on deduplication by content digest (irrespective of url) * ContentDigestHistory.java, ContentDigestHistoryLoader.java, ContentDigestHistoryStorer.java initial checkin --- .../modules/recrawl/ContentDigestHistory.java | 120 ++++++++++++++++++ .../recrawl/ContentDigestHistoryLoader.java | 43 +++++++ .../recrawl/ContentDigestHistoryStorer.java | 43 +++++++ 3 files changed, 206 insertions(+) create mode 100644 modules/src/main/java/org/archive/modules/recrawl/ContentDigestHistory.java create mode 100644 modules/src/main/java/org/archive/modules/recrawl/ContentDigestHistoryLoader.java create mode 100644 modules/src/main/java/org/archive/modules/recrawl/ContentDigestHistoryStorer.java diff --git a/modules/src/main/java/org/archive/modules/recrawl/ContentDigestHistory.java b/modules/src/main/java/org/archive/modules/recrawl/ContentDigestHistory.java new file mode 100644 index 00000000..92fba0e4 --- /dev/null +++ b/modules/src/main/java/org/archive/modules/recrawl/ContentDigestHistory.java @@ -0,0 +1,120 @@ +/* + * This file is part of the Heritrix web crawler (crawler.archive.org). + * + * Licensed to the Internet Archive (IA) by one or more individual + * contributors. + * + * The IA licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.archive.modules.recrawl; + +import java.util.Map; + +import org.archive.bdb.BdbModule; +import org.archive.modules.CrawlURI; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.context.Lifecycle; + +import com.sleepycat.bind.EntityBinding; +import com.sleepycat.bind.serial.SerialBinding; +import com.sleepycat.bind.serial.StoredClassCatalog; +import com.sleepycat.bind.tuple.StringBinding; +import com.sleepycat.collections.StoredSortedMap; +import com.sleepycat.je.Database; +import com.sleepycat.je.DatabaseException; + +/** Needs to be a toplevel bean for Lifecyle? */ +public class ContentDigestHistory implements Lifecycle { + + protected BdbModule bdb; + @Autowired + public void setBdbModule(BdbModule bdb) { + this.bdb = bdb; + } + + protected String historyDbName = "contentDigestHistory"; + public String getHistoryDbName() { + return this.historyDbName; + } + public void setHistoryDbName(String name) { + this.historyDbName = name; + } + + protected StoredSortedMap> store; + protected Database historyDb; + + protected String persistKeyFor(CrawlURI curi) { + return curi.getContentDigestSchemeString(); + } + + @Override + @SuppressWarnings({ "unchecked", "rawtypes" }) + public void start() { + if (isRunning()) { + return; + } + StoredSortedMap> historyMap; + try { + StoredClassCatalog classCatalog = bdb.getClassCatalog(); + historyDb = bdb.openDatabase(getHistoryDbName(), historyDbConfig(), true); + historyMap = new StoredSortedMap>( + historyDb, + new StringBinding(), + (EntityBinding>) new SerialBinding(classCatalog, Map.class), + true); + } catch (DatabaseException e) { + throw new RuntimeException(e); + } + store = historyMap; + } + + @Override + public boolean isRunning() { + return historyDb != null; + } + + @Override + public void stop() { + if (!isRunning()) { + return; + } + // leave other cleanup to BdbModule + historyDb = null; + } + + protected transient BdbModule.BdbConfig historyDbConfig; + protected BdbModule.BdbConfig historyDbConfig() { + if (historyDbConfig == null) { + historyDbConfig = new BdbModule.BdbConfig(); + historyDbConfig.setTransactional(false); + historyDbConfig.setAllowCreate(true); + historyDbConfig.setDeferredWrite(true); + } + + return historyDbConfig; + } + + public void load(CrawlURI curi) { + String pkey = persistKeyFor(curi); + Map prior = store.get(pkey); + if (prior != null) { + // merge in keys + prior.keySet().removeAll(curi.getData().keySet()); + curi.getData().putAll(prior); + } + } + + public void store(CrawlURI curi) { + store.put(persistKeyFor(curi), curi.getPersistentDataMap()); + } +} \ No newline at end of file diff --git a/modules/src/main/java/org/archive/modules/recrawl/ContentDigestHistoryLoader.java b/modules/src/main/java/org/archive/modules/recrawl/ContentDigestHistoryLoader.java new file mode 100644 index 00000000..4e858cbe --- /dev/null +++ b/modules/src/main/java/org/archive/modules/recrawl/ContentDigestHistoryLoader.java @@ -0,0 +1,43 @@ +/* + * This file is part of the Heritrix web crawler (crawler.archive.org). + * + * Licensed to the Internet Archive (IA) by one or more individual + * contributors. + * + * The IA licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.archive.modules.recrawl; + +import org.archive.modules.CrawlURI; +import org.archive.modules.Processor; +import org.springframework.beans.factory.annotation.Autowired; + +public class ContentDigestHistoryLoader extends Processor { + + protected ContentDigestHistory contentDigestHistory; + @Autowired + public void setContentDigestHistory( + ContentDigestHistory contentDigestHistory) { + this.contentDigestHistory = contentDigestHistory; + } + + @Override + protected boolean shouldProcess(CrawlURI uri) { + return uri.getContentDigest() != null; + } + + @Override + protected void innerProcess(CrawlURI curi) throws InterruptedException { + contentDigestHistory.load(curi); + } +} \ No newline at end of file diff --git a/modules/src/main/java/org/archive/modules/recrawl/ContentDigestHistoryStorer.java b/modules/src/main/java/org/archive/modules/recrawl/ContentDigestHistoryStorer.java new file mode 100644 index 00000000..e2a3027a --- /dev/null +++ b/modules/src/main/java/org/archive/modules/recrawl/ContentDigestHistoryStorer.java @@ -0,0 +1,43 @@ +/* + * This file is part of the Heritrix web crawler (crawler.archive.org). + * + * Licensed to the Internet Archive (IA) by one or more individual + * contributors. + * + * The IA licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.archive.modules.recrawl; + +import org.archive.modules.CrawlURI; +import org.archive.modules.Processor; +import org.springframework.beans.factory.annotation.Autowired; + +public class ContentDigestHistoryStorer extends Processor { + + protected ContentDigestHistory contentDigestHistory; + @Autowired + public void setContentDigestHistory( + ContentDigestHistory contentDigestHistory) { + this.contentDigestHistory = contentDigestHistory; + } + + @Override + protected boolean shouldProcess(CrawlURI uri) { + return uri.getContentDigest() != null; + } + + @Override + protected void innerProcess(CrawlURI curi) throws InterruptedException { + contentDigestHistory.store(curi); + } +} \ No newline at end of file From a04e76f00fa30739075e1031c566a0a87d104822 Mon Sep 17 00:00:00 2001 From: Noah Levitt Date: Tue, 11 Sep 2012 10:10:49 -0700 Subject: [PATCH 06/19] Load, store content digest history * WARCRecordInfo.java new fields warcFilename, warcFileOffset * WARCWriter.java temporarily remember warc records written * CrawlURI.java convenience method getContentDigestHistory() * ContentDigestHistory.java start up properly; load into and store from curi.getContentDigestHistory() * RecrawlAttributeConstants.java bunch of hash keys for curi.getContentDigestHistory() * WARCWriterProcessor.java save info in curi.getContentDigestHistory() --- .../org/archive/io/warc/WARCRecordInfo.java | 18 +++++ .../java/org/archive/io/warc/WARCWriter.java | 31 +++++++- .../java/org/archive/modules/CrawlURI.java | 14 ++++ .../modules/recrawl/ContentDigestHistory.java | 38 ++++++---- .../recrawl/RecrawlAttributeConstants.java | 20 ++++- .../modules/writer/WARCWriterProcessor.java | 76 +++++++++++++------ 6 files changed, 154 insertions(+), 43 deletions(-) diff --git a/commons/src/main/java/org/archive/io/warc/WARCRecordInfo.java b/commons/src/main/java/org/archive/io/warc/WARCRecordInfo.java index be7cff5b..d34f00b7 100644 --- a/commons/src/main/java/org/archive/io/warc/WARCRecordInfo.java +++ b/commons/src/main/java/org/archive/io/warc/WARCRecordInfo.java @@ -35,6 +35,8 @@ public class WARCRecordInfo { protected InputStream contentStream; protected long contentLength; protected boolean enforceLength; + protected String warcFilename; + protected Long warcFileOffset; public void setType(WARCRecordType type) { this.type = type; @@ -118,4 +120,20 @@ public class WARCRecordInfo { } extraHeaders.addLabelValue(label, value); } + + public void setWARCFilename(String warcFilenameWithoutOccupiedSuffix) { + this.warcFilename = warcFilenameWithoutOccupiedSuffix; + } + + public String getWARCFilename() { + return warcFilename; + } + + public void setWARCFileOffset(Long startPosition) { + this.warcFileOffset = startPosition; + } + + public Long getWARCFileOffset() { + return warcFileOffset; + } } diff --git a/commons/src/main/java/org/archive/io/warc/WARCWriter.java b/commons/src/main/java/org/archive/io/warc/WARCWriter.java index 40abe4c5..03366536 100644 --- a/commons/src/main/java/org/archive/io/warc/WARCWriter.java +++ b/commons/src/main/java/org/archive/io/warc/WARCWriter.java @@ -27,6 +27,7 @@ import java.io.OutputStream; import java.net.URI; import java.util.HashMap; import java.util.Iterator; +import java.util.LinkedList; import java.util.Map; import java.util.Map.Entry; import java.util.concurrent.ConcurrentMap; @@ -36,6 +37,7 @@ import java.util.logging.Level; import java.util.logging.Logger; import org.apache.commons.lang.StringUtils; +import org.archive.io.ArchiveFileConstants; import org.archive.io.UTF8Bytes; import org.archive.io.WriterPoolMember; import org.archive.modules.writer.WARCWriterProcessor; @@ -86,6 +88,9 @@ implements WARCConstants { */ private Map> tmpStats; + /** Temporarily accumulates info on written warc records for use externally. */ + private LinkedList tmpRecordLog = new LinkedList(); + /** * Constructor. * Takes a stream. Use with caution. There is no upperbound check on size. @@ -241,7 +246,6 @@ implements WARCConstants { write(bytes); totalBytes += bytes.length; - if (recordInfo.getContentStream() != null && recordInfo.getContentLength() > 0) { // Write out the header/body separator. write(CRLF_BYTES); // TODO: should this be written even for zero-length? @@ -256,12 +260,23 @@ implements WARCConstants { write(CRLF_BYTES); write(CRLF_BYTES); totalBytes += 2 * CRLF_BYTES.length; + + tally(recordInfo.getType(), contentBytes, totalBytes, getPosition() - startPosition); + + recordInfo.setWARCFilename(getFilenameWithoutOccupiedSuffix()); + recordInfo.setWARCFileOffset(startPosition); + tmpRecordLog.add(recordInfo); } finally { postWriteRecordTasks(); } - - // TODO: should this be in the finally block? - tally(recordInfo.getType(), contentBytes, totalBytes, getPosition() - startPosition); + } + + public String getFilenameWithoutOccupiedSuffix() { + String name = getFile().getName(); + if (name.endsWith(ArchiveFileConstants.OCCUPIED_SUFFIX)) { + name = name.substring(0, name.length() - ArchiveFileConstants.OCCUPIED_SUFFIX.length()); + } + return name; } // if compression is enabled, sizeOnDisk means compressed bytes; if not, it @@ -411,4 +426,12 @@ implements WARCConstants { return 0l; } } + + public void resetTmpRecordLog() { + tmpRecordLog.clear(); + } + + public Iterable getTmpRecordLog() { + return tmpRecordLog; + } } diff --git a/modules/src/main/java/org/archive/modules/CrawlURI.java b/modules/src/main/java/org/archive/modules/CrawlURI.java index 318b7fd7..0baaa20a 100644 --- a/modules/src/main/java/org/archive/modules/CrawlURI.java +++ b/modules/src/main/java/org/archive/modules/CrawlURI.java @@ -55,6 +55,7 @@ import static org.archive.modules.fetcher.FetchStatusCodes.S_TOO_MANY_LINK_HOPS; import static org.archive.modules.fetcher.FetchStatusCodes.S_TOO_MANY_RETRIES; import static org.archive.modules.fetcher.FetchStatusCodes.S_UNATTEMPTED; import static org.archive.modules.fetcher.FetchStatusCodes.S_UNFETCHABLE_URI; +import static org.archive.modules.recrawl.RecrawlAttributeConstants.A_CONTENT_DIGEST_HISTORY; import java.io.IOException; import java.io.ObjectInputStream; @@ -1884,4 +1885,17 @@ implements Reporter, Serializable, OverlayContext { public void setHttpAuthChallenges(Map httpAuthChallenges) { getData().put(A_HTTP_AUTH_CHALLENGES, httpAuthChallenges); } + + public Map getContentDigestHistory() { + @SuppressWarnings("unchecked") + Map contentDigestHistory = (Map) getData().get(A_CONTENT_DIGEST_HISTORY); + + if (contentDigestHistory == null) { + contentDigestHistory = new HashMap(); + getData().put(A_CONTENT_DIGEST_HISTORY, contentDigestHistory); + } + + return contentDigestHistory; + } + } diff --git a/modules/src/main/java/org/archive/modules/recrawl/ContentDigestHistory.java b/modules/src/main/java/org/archive/modules/recrawl/ContentDigestHistory.java index 92fba0e4..f0883d3c 100644 --- a/modules/src/main/java/org/archive/modules/recrawl/ContentDigestHistory.java +++ b/modules/src/main/java/org/archive/modules/recrawl/ContentDigestHistory.java @@ -19,13 +19,14 @@ package org.archive.modules.recrawl; import java.util.Map; +import java.util.logging.Level; +import java.util.logging.Logger; import org.archive.bdb.BdbModule; import org.archive.modules.CrawlURI; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.context.Lifecycle; -import com.sleepycat.bind.EntityBinding; import com.sleepycat.bind.serial.SerialBinding; import com.sleepycat.bind.serial.StoredClassCatalog; import com.sleepycat.bind.tuple.StringBinding; @@ -36,6 +37,9 @@ import com.sleepycat.je.DatabaseException; /** Needs to be a toplevel bean for Lifecyle? */ public class ContentDigestHistory implements Lifecycle { + private static final Logger logger = + Logger.getLogger(ContentDigestHistory.class.getName()); + protected BdbModule bdb; @Autowired public void setBdbModule(BdbModule bdb) { @@ -50,7 +54,8 @@ public class ContentDigestHistory implements Lifecycle { this.historyDbName = name; } - protected StoredSortedMap> store; + @SuppressWarnings("rawtypes") + protected StoredSortedMap store; protected Database historyDb; protected String persistKeyFor(CrawlURI curi) { @@ -58,19 +63,19 @@ public class ContentDigestHistory implements Lifecycle { } @Override - @SuppressWarnings({ "unchecked", "rawtypes" }) + @SuppressWarnings({"rawtypes"}) public void start() { if (isRunning()) { return; } - StoredSortedMap> historyMap; + StoredSortedMap historyMap; try { StoredClassCatalog classCatalog = bdb.getClassCatalog(); historyDb = bdb.openDatabase(getHistoryDbName(), historyDbConfig(), true); - historyMap = new StoredSortedMap>( + historyMap = new StoredSortedMap( historyDb, new StringBinding(), - (EntityBinding>) new SerialBinding(classCatalog, Map.class), + new SerialBinding(classCatalog, Map.class), true); } catch (DatabaseException e) { throw new RuntimeException(e); @@ -105,16 +110,23 @@ public class ContentDigestHistory implements Lifecycle { } public void load(CrawlURI curi) { - String pkey = persistKeyFor(curi); - Map prior = store.get(pkey); - if (prior != null) { - // merge in keys - prior.keySet().removeAll(curi.getData().keySet()); - curi.getData().putAll(prior); + @SuppressWarnings("unchecked") + Map loadedHistory = store.get(persistKeyFor(curi)); + if (loadedHistory != null) { + if (logger.isLoggable(Level.FINER)) { + logger.finer("loaded history by digest " + persistKeyFor(curi) + + " for uri " + curi + " - " + loadedHistory); + } + curi.getContentDigestHistory().putAll(loadedHistory); } } public void store(CrawlURI curi) { - store.put(persistKeyFor(curi), curi.getPersistentDataMap()); + if (logger.isLoggable(Level.FINER)) { + logger.finer("storing history by digest " + persistKeyFor(curi) + + " for uri " + curi + " - " + + curi.getContentDigestHistory()); + } + store.put(persistKeyFor(curi), curi.getContentDigestHistory()); } } \ No newline at end of file diff --git a/modules/src/main/java/org/archive/modules/recrawl/RecrawlAttributeConstants.java b/modules/src/main/java/org/archive/modules/recrawl/RecrawlAttributeConstants.java index f95e4b2e..e7c41dce 100644 --- a/modules/src/main/java/org/archive/modules/recrawl/RecrawlAttributeConstants.java +++ b/modules/src/main/java/org/archive/modules/recrawl/RecrawlAttributeConstants.java @@ -32,14 +32,30 @@ public interface RecrawlAttributeConstants { public static final String A_FETCH_HISTORY = "fetch-history"; /** content digest */ public static final String A_CONTENT_DIGEST = "content-digest"; - /** header name (and AList key) for last-modified timestamp */ + /** header name (and AList key) for last-modified timestamp */ public static final String A_LAST_MODIFIED_HEADER = "last-modified"; - /** header name (and AList key) for ETag */ + /** header name (and AList key) for ETag */ public static final String A_ETAG_HEADER = "etag"; /** key for status (when in history) */ public static final String A_STATUS = "status"; /** reference length (content length or virtual length */ public static final String A_REFERENCE_LENGTH = "reference-length"; + + // constants for uri-agnostic content digest based dedupe + /** content digest history map */ + public static final String A_CONTENT_DIGEST_HISTORY = "content-digest-history"; + /** url that the content payload was written for */ + public static final String A_ORIGINAL_URL = "original-url"; + /** warc record id of warc record with the content payload */ + public static final String A_WARC_RECORD_ID = "warc-record-id"; + /** warc filename containing the content payload */ + public static final String A_WARC_FILENAME = "warc-filename"; + /** offset into warc file of warc record with content payload */ + public static final String A_WARC_FILE_OFFSET = "warc-file-offset"; + /** date content payload was written */ + public static final String A_ORIGINAL_DATE = "content-written-date"; + /** number of times we've seen this content digest (1 original + n duplicates) */ + public static final String A_CONTENT_DIGEST_COUNT = "content-digest-count"; /** * Writer processors of all types are encouraged to put a 'writeTag' diff --git a/modules/src/main/java/org/archive/modules/writer/WARCWriterProcessor.java b/modules/src/main/java/org/archive/modules/writer/WARCWriterProcessor.java index c66bf3f2..6b0e54af 100644 --- a/modules/src/main/java/org/archive/modules/writer/WARCWriterProcessor.java +++ b/modules/src/main/java/org/archive/modules/writer/WARCWriterProcessor.java @@ -42,9 +42,15 @@ import static org.archive.modules.CoreAttributeConstants.A_SOURCE_TAG; import static org.archive.modules.CoreAttributeConstants.HEADER_TRUNC; import static org.archive.modules.CoreAttributeConstants.LENGTH_TRUNC; import static org.archive.modules.CoreAttributeConstants.TIMER_TRUNC; +import static org.archive.modules.recrawl.RecrawlAttributeConstants.A_CONTENT_DIGEST_COUNT; import static org.archive.modules.recrawl.RecrawlAttributeConstants.A_ETAG_HEADER; import static org.archive.modules.recrawl.RecrawlAttributeConstants.A_FETCH_HISTORY; import static org.archive.modules.recrawl.RecrawlAttributeConstants.A_LAST_MODIFIED_HEADER; +import static org.archive.modules.recrawl.RecrawlAttributeConstants.A_ORIGINAL_DATE; +import static org.archive.modules.recrawl.RecrawlAttributeConstants.A_ORIGINAL_URL; +import static org.archive.modules.recrawl.RecrawlAttributeConstants.A_WARC_FILENAME; +import static org.archive.modules.recrawl.RecrawlAttributeConstants.A_WARC_FILE_OFFSET; +import static org.archive.modules.recrawl.RecrawlAttributeConstants.A_WARC_RECORD_ID; import static org.archive.modules.recrawl.RecrawlAttributeConstants.A_WRITE_TAG; import java.io.ByteArrayInputStream; @@ -70,7 +76,6 @@ import org.apache.commons.httpclient.HttpMethod; import org.apache.commons.httpclient.HttpStatus; import org.apache.commons.io.IOUtils; import org.apache.commons.lang.StringUtils; -import org.archive.io.ArchiveFileConstants; import org.archive.io.ReplayInputStream; import org.archive.io.warc.WARCConstants.WARCRecordType; import org.archive.io.warc.WARCRecordInfo; @@ -243,6 +248,8 @@ public class WARCWriterProcessor extends WriterPoolProcessor implements WARCWrit // They'll be added to totals below, in finally block, after records // have been written. writer.resetTmpStats(); + writer.resetTmpRecordLog(); + // Write a request, response, and metadata all in the one // 'transaction'. final URI baseid = getRecordID(); @@ -269,34 +276,55 @@ public class WARCWriterProcessor extends WriterPoolProcessor implements WARCWrit throw e; } finally { if (writer != null) { - if (WARCWriter.getStat(writer.getTmpStats(), WARCWriter.TOTALS, WARCWriter.NUM_RECORDS) > 0l) { - addStats(writer.getTmpStats()); - urlsWritten.incrementAndGet(); - } - if (logger.isLoggable(Level.FINE)) { - logger.fine("wrote " - + WARCWriter.getStat(writer.getTmpStats(), WARCWriter.TOTALS, WARCWriter.SIZE_ON_DISK) - + " bytes to " + writer.getFile().getName() + " for " + curi); - } - setTotalBytesWritten(getTotalBytesWritten() + - (writer.getPosition() - position)); + updateMetadataAfterWrite(curi, writer, position); getPool().returnFile(writer); - - String filename = writer.getFile().getName(); - if (filename.endsWith(ArchiveFileConstants.OCCUPIED_SUFFIX)) { - filename = filename.substring(0, filename.length() - ArchiveFileConstants.OCCUPIED_SUFFIX.length()); - } - curi.addExtraInfo("warcFilename", filename); - - @SuppressWarnings("unchecked") - Map[] history = (Map[])curi.getData().get(A_FETCH_HISTORY); - if (history != null && history[0] != null) { - history[0].put(A_WRITE_TAG, filename); - } } } return checkBytesWritten(); } + + protected void updateMetadataAfterWrite(final CrawlURI curi, + WARCWriter writer, long startPosition) { + if (WARCWriter.getStat(writer.getTmpStats(), WARCWriter.TOTALS, WARCWriter.NUM_RECORDS) > 0l) { + addStats(writer.getTmpStats()); + urlsWritten.incrementAndGet(); + } + if (logger.isLoggable(Level.FINE)) { + logger.fine("wrote " + + WARCWriter.getStat(writer.getTmpStats(), WARCWriter.TOTALS, WARCWriter.SIZE_ON_DISK) + + " bytes to " + writer.getFile().getName() + " for " + curi); + } + setTotalBytesWritten(getTotalBytesWritten() + (writer.getPosition() - startPosition)); + + curi.addExtraInfo("warcFilename", writer.getFilenameWithoutOccupiedSuffix()); + // curi.addExtraInfo("warcOffset", startPosition); + + // history for uri-based dedupe + @SuppressWarnings("unchecked") + Map[] history = (Map[])curi.getData().get(A_FETCH_HISTORY); + if (history != null && history[0] != null) { + history[0].put(A_WRITE_TAG, writer.getFilenameWithoutOccupiedSuffix()); + } + + // history for uri-agnostic, content digest based dedupe + if (curi.getContentDigest() != null) { + for (WARCRecordInfo warcRecord: writer.getTmpRecordLog()) { + if ((warcRecord.getType() == WARCRecordType.RESPONSE + || warcRecord.getType() == WARCRecordType.RESOURCE) + && warcRecord.getContentStream() != null + && warcRecord.getContentLength() > 0) { + curi.getContentDigestHistory().put(A_ORIGINAL_URL, warcRecord.getUrl()); + curi.getContentDigestHistory().put(A_WARC_RECORD_ID, warcRecord.getRecordId()); + curi.getContentDigestHistory().put(A_WARC_FILENAME, warcRecord.getWARCFilename()); + curi.getContentDigestHistory().put(A_WARC_FILE_OFFSET, warcRecord.getWARCFileOffset()); + curi.getContentDigestHistory().put(A_ORIGINAL_DATE, warcRecord.getCreate14DigitDate()); + curi.getContentDigestHistory().put(A_CONTENT_DIGEST_COUNT, 1); + // } else if (warcRecord.getType() == WARCRecordType.REVISIT) { + // XXX add to content-digest-count IF it's a content digest based revisit record + } + } + } + } protected void addStats(Map> substats) { for (String key: substats.keySet()) { From 38aefe4d16bb19a0708e07fc796a31d85d23976d Mon Sep 17 00:00:00 2001 From: Noah Levitt Date: Tue, 11 Sep 2012 17:12:24 -0700 Subject: [PATCH 07/19] Write revisit records for URI-agnostic content digest duplicates. * WARCConstants.java new header keys "WARC-Refers-To" and "X-WARC-Reference-Location" * CrawlURI.java getContentDigestHistory() - return HashMap instead of Map so clone() can be called on the return value * WARCWriterProcessor.java writeHttpRecords(), writeRevisitUriAgnosticDigest() - write revisit record for URI-agnostic content digest duplicate if enabled --- .../org/archive/io/warc/WARCConstants.java | 2 + .../java/org/archive/modules/CrawlURI.java | 4 +- .../modules/writer/WARCWriterProcessor.java | 65 +++++++++++++++++-- 3 files changed, 64 insertions(+), 7 deletions(-) diff --git a/commons/src/main/java/org/archive/io/warc/WARCConstants.java b/commons/src/main/java/org/archive/io/warc/WARCConstants.java index 685c9364..e88894dc 100644 --- a/commons/src/main/java/org/archive/io/warc/WARCConstants.java +++ b/commons/src/main/java/org/archive/io/warc/WARCConstants.java @@ -193,6 +193,8 @@ public interface WARCConstants extends ArchiveFileConstants { public static final String HEADER_KEY_FILENAME = "WARC-Filename"; public static final String HEADER_KEY_ETAG = "WARC-Etag"; public static final String HEADER_KEY_LAST_MODIFIED = "WARC-Last-Modified"; + public static final String HEADER_KEY_REFERS_TO = "WARC-Refers-To"; + public static final String HEADER_KEY_REFERENCE_LOCATION = "X-WARC-Reference-Location"; public static final String PROFILE_REVISIT_IDENTICAL_DIGEST = "http://netpreserve.org/warc/1.0/revisit/identical-payload-digest"; diff --git a/modules/src/main/java/org/archive/modules/CrawlURI.java b/modules/src/main/java/org/archive/modules/CrawlURI.java index 0baaa20a..4525a978 100644 --- a/modules/src/main/java/org/archive/modules/CrawlURI.java +++ b/modules/src/main/java/org/archive/modules/CrawlURI.java @@ -1886,9 +1886,9 @@ implements Reporter, Serializable, OverlayContext { getData().put(A_HTTP_AUTH_CHALLENGES, httpAuthChallenges); } - public Map getContentDigestHistory() { + public HashMap getContentDigestHistory() { @SuppressWarnings("unchecked") - Map contentDigestHistory = (Map) getData().get(A_CONTENT_DIGEST_HISTORY); + HashMap contentDigestHistory = (HashMap) getData().get(A_CONTENT_DIGEST_HISTORY); if (contentDigestHistory == null) { contentDigestHistory = new HashMap(); diff --git a/modules/src/main/java/org/archive/modules/writer/WARCWriterProcessor.java b/modules/src/main/java/org/archive/modules/writer/WARCWriterProcessor.java index 6b0e54af..8c305335 100644 --- a/modules/src/main/java/org/archive/modules/writer/WARCWriterProcessor.java +++ b/modules/src/main/java/org/archive/modules/writer/WARCWriterProcessor.java @@ -26,6 +26,8 @@ import static org.archive.io.warc.WARCConstants.HEADER_KEY_IP; import static org.archive.io.warc.WARCConstants.HEADER_KEY_LAST_MODIFIED; import static org.archive.io.warc.WARCConstants.HEADER_KEY_PAYLOAD_DIGEST; import static org.archive.io.warc.WARCConstants.HEADER_KEY_PROFILE; +import static org.archive.io.warc.WARCConstants.HEADER_KEY_REFERENCE_LOCATION; +import static org.archive.io.warc.WARCConstants.HEADER_KEY_REFERS_TO; import static org.archive.io.warc.WARCConstants.HEADER_KEY_TRUNCATED; import static org.archive.io.warc.WARCConstants.HTTP_REQUEST_MIMETYPE; import static org.archive.io.warc.WARCConstants.HTTP_RESPONSE_MIMETYPE; @@ -93,6 +95,7 @@ import org.archive.uid.RecordIDGenerator; import org.archive.uid.UUIDGenerator; import org.archive.util.ArchiveUtils; import org.archive.util.anvl.ANVLRecord; +import org.json.JSONObject; /** * WARCWriterProcessor. @@ -151,6 +154,9 @@ public class WARCWriterProcessor extends WriterPoolProcessor implements WARCWrit /** * Whether to write 'revisit' type records when a URI's history indicates * the previous fetch had an identical content digest. Default is true. + * + * Decision applies to either URI-based fetch history or URI-agnostic + * content digest-based history. */ { setWriteRevisitForIdenticalDigests(true); @@ -346,7 +352,7 @@ public class WARCWriterProcessor extends WriterPoolProcessor implements WARCWrit } } - private void writeDnsRecords(final CrawlURI curi, WARCWriter w, + protected void writeDnsRecords(final CrawlURI curi, WARCWriter w, final URI baseid, final String timestamp) throws IOException { WARCRecordInfo recordInfo = new WARCRecordInfo(); recordInfo.setType(WARCRecordType.RESPONSE); @@ -376,7 +382,7 @@ public class WARCWriterProcessor extends WriterPoolProcessor implements WARCWrit recordInfo.getRecordId(); } - private void writeWhoisRecords(WARCWriter w, CrawlURI curi, URI baseid, + protected void writeWhoisRecords(WARCWriter w, CrawlURI curi, URI baseid, String timestamp) throws IOException { WARCRecordInfo recordInfo = new WARCRecordInfo(); recordInfo.setType(WARCRecordType.RESPONSE); @@ -404,7 +410,7 @@ public class WARCWriterProcessor extends WriterPoolProcessor implements WARCWrit recordInfo.getRecordId(); } - private void writeHttpRecords(final CrawlURI curi, WARCWriter w, + protected void writeHttpRecords(final CrawlURI curi, WARCWriter w, final URI baseid, final String timestamp) throws IOException { // Add named fields for ip, checksum, and relate the metadata // and request to the resource field. @@ -418,7 +424,11 @@ public class WARCWriterProcessor extends WriterPoolProcessor implements WARCWrit headers.addLabelValue(HEADER_KEY_IP, getHostAddress(curi)); URI rid; - if (IdenticalDigestDecideRule.hasIdenticalDigest(curi) && + if (getWriteRevisitForIdenticalDigests() + && curi.getContentDigestHistory().get(A_ORIGINAL_URL) != null) { + rid = writeRevisitUriAgnosticDigest(w, timestamp, + HTTP_RESPONSE_MIMETYPE, baseid, curi, headers); + } else if (IdenticalDigestDecideRule.hasIdenticalDigest(curi) && getWriteRevisitForIdenticalDigests()) { rid = writeRevisitDigest(w, timestamp, HTTP_RESPONSE_MIMETYPE, baseid, curi, headers); @@ -458,7 +468,7 @@ public class WARCWriterProcessor extends WriterPoolProcessor implements WARCWrit } } - private void writeFtpRecords(WARCWriter w, final CrawlURI curi, final URI baseid, + protected void writeFtpRecords(WARCWriter w, final CrawlURI curi, final URI baseid, final String timestamp) throws IOException { ANVLRecord headers = new ANVLRecord(); headers.addLabelValue(HEADER_KEY_IP, getHostAddress(curi)); @@ -661,6 +671,51 @@ public class WARCWriterProcessor extends WriterPoolProcessor implements WARCWrit return recordInfo.getRecordId(); } + protected URI writeRevisitUriAgnosticDigest(WARCWriter w, String timestamp, + String mimetype, URI baseid, CrawlURI curi, + ANVLRecord headers) throws IOException { + + WARCRecordInfo recordInfo = new WARCRecordInfo(); + recordInfo.setType(WARCRecordType.REVISIT); + recordInfo.setUrl(curi.toString()); + recordInfo.setCreate14DigitDate(timestamp); + recordInfo.setMimetype(mimetype); + recordInfo.setRecordId(baseid); + recordInfo.setEnforceLength(false); + + long revisedLength = curi.getRecorder().getRecordedInput().getContentBegin(); + revisedLength = revisedLength > 0 ? revisedLength : curi.getRecorder().getRecordedInput().getSize(); + recordInfo.setContentLength(revisedLength); + + headers.addLabelValue( + HEADER_KEY_PROFILE, PROFILE_REVISIT_IDENTICAL_DIGEST); + headers.addLabelValue( + HEADER_KEY_TRUNCATED, NAMED_FIELD_TRUNCATED_VALUE_LENGTH); + + headers.addLabelValue(HEADER_KEY_REFERS_TO, + curi.getContentDigestHistory().get(A_WARC_RECORD_ID).toString()); + + JSONObject refLoc = new JSONObject(curi.getContentDigestHistory().clone()); + refLoc.remove(A_CONTENT_DIGEST_COUNT); + refLoc.remove(A_WARC_RECORD_ID); + headers.addLabelValue(HEADER_KEY_REFERENCE_LOCATION, refLoc.toString()); + + recordInfo.setExtraHeaders(headers); + + ReplayInputStream ris = + curi.getRecorder().getRecordedInput().getReplayInputStream(); + recordInfo.setContentStream(ris); + + try { + w.writeRecord(recordInfo); + } finally { + IOUtils.closeQuietly(ris); + } + curi.getAnnotations().add("warcRevisit:uriAgnosticDigest"); + + return recordInfo.getRecordId(); + } + protected URI writeRevisitNotModified(final WARCWriter w, final String timestamp, final URI baseid, final CrawlURI puri, From 1048536a254bae674e2a66944356f4bd85041896 Mon Sep 17 00:00:00 2001 From: Noah Levitt Date: Tue, 11 Sep 2012 17:21:34 -0700 Subject: [PATCH 08/19] * WARCWriterProcessor.java writeRevisitUriAgnosticDigest() - fix construction of X-WARC-Reference-Location json value --- .../java/org/archive/modules/writer/WARCWriterProcessor.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/modules/src/main/java/org/archive/modules/writer/WARCWriterProcessor.java b/modules/src/main/java/org/archive/modules/writer/WARCWriterProcessor.java index 8c305335..6fbdc71b 100644 --- a/modules/src/main/java/org/archive/modules/writer/WARCWriterProcessor.java +++ b/modules/src/main/java/org/archive/modules/writer/WARCWriterProcessor.java @@ -695,7 +695,8 @@ public class WARCWriterProcessor extends WriterPoolProcessor implements WARCWrit headers.addLabelValue(HEADER_KEY_REFERS_TO, curi.getContentDigestHistory().get(A_WARC_RECORD_ID).toString()); - JSONObject refLoc = new JSONObject(curi.getContentDigestHistory().clone()); + @SuppressWarnings("unchecked") + JSONObject refLoc = new JSONObject((HashMap) curi.getContentDigestHistory().clone()); refLoc.remove(A_CONTENT_DIGEST_COUNT); refLoc.remove(A_WARC_RECORD_ID); headers.addLabelValue(HEADER_KEY_REFERENCE_LOCATION, refLoc.toString()); From a8aae91167133ec7e1399d9a7ee8a258c0036c1e Mon Sep 17 00:00:00 2001 From: Noah Levitt Date: Tue, 11 Sep 2012 18:21:20 -0700 Subject: [PATCH 09/19] * WARCConstants.java remove rfc6648-deprecated "X-" prefix to "WARC-Reference-Location", and document the field * WARCWriterProcessor.java writeRevisitUriAgnosticDigest() - add some comments --- .../java/org/archive/io/warc/WARCConstants.java | 16 +++++++++++++++- .../modules/writer/WARCWriterProcessor.java | 7 +++++++ 2 files changed, 22 insertions(+), 1 deletion(-) diff --git a/commons/src/main/java/org/archive/io/warc/WARCConstants.java b/commons/src/main/java/org/archive/io/warc/WARCConstants.java index e88894dc..0efa25b8 100644 --- a/commons/src/main/java/org/archive/io/warc/WARCConstants.java +++ b/commons/src/main/java/org/archive/io/warc/WARCConstants.java @@ -20,6 +20,7 @@ package org.archive.io.warc; import org.archive.io.ArchiveFileConstants; +import org.archive.modules.writer.WARCWriterProcessor; /** * WARC Constants used by WARC readers and writers. @@ -194,7 +195,20 @@ public interface WARCConstants extends ArchiveFileConstants { public static final String HEADER_KEY_ETAG = "WARC-Etag"; public static final String HEADER_KEY_LAST_MODIFIED = "WARC-Last-Modified"; public static final String HEADER_KEY_REFERS_TO = "WARC-Refers-To"; - public static final String HEADER_KEY_REFERENCE_LOCATION = "X-WARC-Reference-Location"; + + /** + * Information that helps to find the record that + * {@value #HEADER_KEY_REFERS_TO} refers to. This header is a heritrix + * extension not defined in the WARC spec version 1.0. Current + * implementation writes a JSON string that looks something like this: + * {"original-url":"http://archive.org/robots.txt" + * ,"warc-file-offset":1976,"warc-filename": + * "WEB-20120912001108855-00000-6882~desktop-nlevitt.sf.archive.org~6440.warc.gz" + * ,"content-written-date":"2012-09-12T00:11:10Z"}. + * + * @see WARCWriterProcessor + */ + public static final String HEADER_KEY_REFERENCE_LOCATION = "WARC-Reference-Location"; public static final String PROFILE_REVISIT_IDENTICAL_DIGEST = "http://netpreserve.org/warc/1.0/revisit/identical-payload-digest"; diff --git a/modules/src/main/java/org/archive/modules/writer/WARCWriterProcessor.java b/modules/src/main/java/org/archive/modules/writer/WARCWriterProcessor.java index 6fbdc71b..4985310a 100644 --- a/modules/src/main/java/org/archive/modules/writer/WARCWriterProcessor.java +++ b/modules/src/main/java/org/archive/modules/writer/WARCWriterProcessor.java @@ -692,9 +692,16 @@ public class WARCWriterProcessor extends WriterPoolProcessor implements WARCWrit headers.addLabelValue( HEADER_KEY_TRUNCATED, NAMED_FIELD_TRUNCATED_VALUE_LENGTH); + /* + * ISO 28500 WARC ISO standard draft says: "The WARC-Refers-To field may + * also be used to associate a record of type 'revisit' or 'conversion' + * with the preceding record which helped determine the present record + * content." + */ headers.addLabelValue(HEADER_KEY_REFERS_TO, curi.getContentDigestHistory().get(A_WARC_RECORD_ID).toString()); + // {"original-url":"http://archive.org/robots.txt","warc-file-offset":1976,"warc-filename":"WEB-20120912001108855-00000-6882~desktop-nlevitt.sf.archive.org~6440.warc.gz","content-written-date":"2012-09-12T00:11:10Z"} @SuppressWarnings("unchecked") JSONObject refLoc = new JSONObject((HashMap) curi.getContentDigestHistory().clone()); refLoc.remove(A_CONTENT_DIGEST_COUNT); From ef78e584c4d3c602826d7d4d3d9dbc9cc4c43cec Mon Sep 17 00:00:00 2001 From: Noah Levitt Date: Thu, 13 Sep 2012 17:21:29 -0700 Subject: [PATCH 10/19] * WARCWriterProcessor.java writeRevisitUriAgnosticDigest() - write several separate warc header fields for each value instead of one json blob * WARCConstants.java new header field names --- .../org/archive/io/warc/WARCConstants.java | 15 ++++++------- .../modules/writer/WARCWriterProcessor.java | 21 +++++++++++-------- 2 files changed, 18 insertions(+), 18 deletions(-) diff --git a/commons/src/main/java/org/archive/io/warc/WARCConstants.java b/commons/src/main/java/org/archive/io/warc/WARCConstants.java index 0efa25b8..eb4941c6 100644 --- a/commons/src/main/java/org/archive/io/warc/WARCConstants.java +++ b/commons/src/main/java/org/archive/io/warc/WARCConstants.java @@ -197,18 +197,15 @@ public interface WARCConstants extends ArchiveFileConstants { public static final String HEADER_KEY_REFERS_TO = "WARC-Refers-To"; /** - * Information that helps to find the record that - * {@value #HEADER_KEY_REFERS_TO} refers to. This header is a heritrix - * extension not defined in the WARC spec version 1.0. Current - * implementation writes a JSON string that looks something like this: - * {"original-url":"http://archive.org/robots.txt" - * ,"warc-file-offset":1976,"warc-filename": - * "WEB-20120912001108855-00000-6882~desktop-nlevitt.sf.archive.org~6440.warc.gz" - * ,"content-written-date":"2012-09-12T00:11:10Z"}. + * These fields help a consumer of the warc to locate the warc record that + * {@value #HEADER_KEY_REFERS_TO} refers to. * * @see WARCWriterProcessor */ - public static final String HEADER_KEY_REFERENCE_LOCATION = "WARC-Reference-Location"; + public static final String HEADER_KEY_REFERS_TO_TARGET_URI = "WARC-Refers-To-Target-URI"; + public static final String HEADER_KEY_REFERS_TO_DATE = "WARC-Refers-To-Date"; + public static final String HEADER_KEY_REFERS_TO_FILENAME = "WARC-Refers-To-Filename"; + public static final String HEADER_KEY_REFERS_TO_FILE_OFFSET = "WARC-Refers-To-File-Offset"; public static final String PROFILE_REVISIT_IDENTICAL_DIGEST = "http://netpreserve.org/warc/1.0/revisit/identical-payload-digest"; diff --git a/modules/src/main/java/org/archive/modules/writer/WARCWriterProcessor.java b/modules/src/main/java/org/archive/modules/writer/WARCWriterProcessor.java index 4985310a..acd229fe 100644 --- a/modules/src/main/java/org/archive/modules/writer/WARCWriterProcessor.java +++ b/modules/src/main/java/org/archive/modules/writer/WARCWriterProcessor.java @@ -26,8 +26,11 @@ import static org.archive.io.warc.WARCConstants.HEADER_KEY_IP; import static org.archive.io.warc.WARCConstants.HEADER_KEY_LAST_MODIFIED; import static org.archive.io.warc.WARCConstants.HEADER_KEY_PAYLOAD_DIGEST; import static org.archive.io.warc.WARCConstants.HEADER_KEY_PROFILE; -import static org.archive.io.warc.WARCConstants.HEADER_KEY_REFERENCE_LOCATION; import static org.archive.io.warc.WARCConstants.HEADER_KEY_REFERS_TO; +import static org.archive.io.warc.WARCConstants.HEADER_KEY_REFERS_TO_DATE; +import static org.archive.io.warc.WARCConstants.HEADER_KEY_REFERS_TO_FILENAME; +import static org.archive.io.warc.WARCConstants.HEADER_KEY_REFERS_TO_FILE_OFFSET; +import static org.archive.io.warc.WARCConstants.HEADER_KEY_REFERS_TO_TARGET_URI; import static org.archive.io.warc.WARCConstants.HEADER_KEY_TRUNCATED; import static org.archive.io.warc.WARCConstants.HTTP_REQUEST_MIMETYPE; import static org.archive.io.warc.WARCConstants.HTTP_RESPONSE_MIMETYPE; @@ -95,7 +98,6 @@ import org.archive.uid.RecordIDGenerator; import org.archive.uid.UUIDGenerator; import org.archive.util.ArchiveUtils; import org.archive.util.anvl.ANVLRecord; -import org.json.JSONObject; /** * WARCWriterProcessor. @@ -700,13 +702,14 @@ public class WARCWriterProcessor extends WriterPoolProcessor implements WARCWrit */ headers.addLabelValue(HEADER_KEY_REFERS_TO, curi.getContentDigestHistory().get(A_WARC_RECORD_ID).toString()); - - // {"original-url":"http://archive.org/robots.txt","warc-file-offset":1976,"warc-filename":"WEB-20120912001108855-00000-6882~desktop-nlevitt.sf.archive.org~6440.warc.gz","content-written-date":"2012-09-12T00:11:10Z"} - @SuppressWarnings("unchecked") - JSONObject refLoc = new JSONObject((HashMap) curi.getContentDigestHistory().clone()); - refLoc.remove(A_CONTENT_DIGEST_COUNT); - refLoc.remove(A_WARC_RECORD_ID); - headers.addLabelValue(HEADER_KEY_REFERENCE_LOCATION, refLoc.toString()); + headers.addLabelValue(HEADER_KEY_REFERS_TO_TARGET_URI, + curi.getContentDigestHistory().get(A_ORIGINAL_URL).toString()); + headers.addLabelValue(HEADER_KEY_REFERS_TO_DATE, + curi.getContentDigestHistory().get(A_ORIGINAL_DATE).toString()); + headers.addLabelValue(HEADER_KEY_REFERS_TO_FILENAME, + curi.getContentDigestHistory().get(A_WARC_FILENAME).toString()); + headers.addLabelValue(HEADER_KEY_REFERS_TO_FILE_OFFSET, + curi.getContentDigestHistory().get(A_WARC_FILE_OFFSET).toString()); recordInfo.setExtraHeaders(headers); From b8e14ba1df399138c36932e85ea18cf1b4c11d88 Mon Sep 17 00:00:00 2001 From: Noah Levitt Date: Tue, 25 Sep 2012 13:12:59 -0700 Subject: [PATCH 11/19] Rename ContentDigestHistory -> BdbContentDigestHistory, since it's bdb-specific --- ...ContentDigestHistory.java => BdbContentDigestHistory.java} | 4 ++-- .../archive/modules/recrawl/ContentDigestHistoryLoader.java | 4 ++-- .../archive/modules/recrawl/ContentDigestHistoryStorer.java | 4 ++-- 3 files changed, 6 insertions(+), 6 deletions(-) rename modules/src/main/java/org/archive/modules/recrawl/{ContentDigestHistory.java => BdbContentDigestHistory.java} (97%) diff --git a/modules/src/main/java/org/archive/modules/recrawl/ContentDigestHistory.java b/modules/src/main/java/org/archive/modules/recrawl/BdbContentDigestHistory.java similarity index 97% rename from modules/src/main/java/org/archive/modules/recrawl/ContentDigestHistory.java rename to modules/src/main/java/org/archive/modules/recrawl/BdbContentDigestHistory.java index f0883d3c..7d1c902a 100644 --- a/modules/src/main/java/org/archive/modules/recrawl/ContentDigestHistory.java +++ b/modules/src/main/java/org/archive/modules/recrawl/BdbContentDigestHistory.java @@ -35,10 +35,10 @@ import com.sleepycat.je.Database; import com.sleepycat.je.DatabaseException; /** Needs to be a toplevel bean for Lifecyle? */ -public class ContentDigestHistory implements Lifecycle { +public class BdbContentDigestHistory implements Lifecycle { private static final Logger logger = - Logger.getLogger(ContentDigestHistory.class.getName()); + Logger.getLogger(BdbContentDigestHistory.class.getName()); protected BdbModule bdb; @Autowired diff --git a/modules/src/main/java/org/archive/modules/recrawl/ContentDigestHistoryLoader.java b/modules/src/main/java/org/archive/modules/recrawl/ContentDigestHistoryLoader.java index 4e858cbe..8ec9a52a 100644 --- a/modules/src/main/java/org/archive/modules/recrawl/ContentDigestHistoryLoader.java +++ b/modules/src/main/java/org/archive/modules/recrawl/ContentDigestHistoryLoader.java @@ -24,10 +24,10 @@ import org.springframework.beans.factory.annotation.Autowired; public class ContentDigestHistoryLoader extends Processor { - protected ContentDigestHistory contentDigestHistory; + protected BdbContentDigestHistory contentDigestHistory; @Autowired public void setContentDigestHistory( - ContentDigestHistory contentDigestHistory) { + BdbContentDigestHistory contentDigestHistory) { this.contentDigestHistory = contentDigestHistory; } diff --git a/modules/src/main/java/org/archive/modules/recrawl/ContentDigestHistoryStorer.java b/modules/src/main/java/org/archive/modules/recrawl/ContentDigestHistoryStorer.java index e2a3027a..0462b90a 100644 --- a/modules/src/main/java/org/archive/modules/recrawl/ContentDigestHistoryStorer.java +++ b/modules/src/main/java/org/archive/modules/recrawl/ContentDigestHistoryStorer.java @@ -24,10 +24,10 @@ import org.springframework.beans.factory.annotation.Autowired; public class ContentDigestHistoryStorer extends Processor { - protected ContentDigestHistory contentDigestHistory; + protected BdbContentDigestHistory contentDigestHistory; @Autowired public void setContentDigestHistory( - ContentDigestHistory contentDigestHistory) { + BdbContentDigestHistory contentDigestHistory) { this.contentDigestHistory = contentDigestHistory; } From 25a91552987960650f9f7035329404d4aea005ba Mon Sep 17 00:00:00 2001 From: Noah Levitt Date: Tue, 25 Sep 2012 13:38:25 -0700 Subject: [PATCH 12/19] New parent class AbstractContentDigestHistory for implementations to extend * AbstractContentDigestHistory.java abstract methods load(), store(); non-abstract persistKeyFor(); javadocs * BdbContentDigestHistory.java extend AbstractContentDigestHistory * ContentDigestHistoryLoader.java, ContentDigestHistoryStorer.java use AbstractContentDigestHistory --- .../recrawl/AbstractContentDigestHistory.java | 59 +++++++++++++++++++ .../recrawl/BdbContentDigestHistory.java | 6 +- .../recrawl/ContentDigestHistoryLoader.java | 4 +- .../recrawl/ContentDigestHistoryStorer.java | 4 +- 4 files changed, 64 insertions(+), 9 deletions(-) create mode 100644 modules/src/main/java/org/archive/modules/recrawl/AbstractContentDigestHistory.java diff --git a/modules/src/main/java/org/archive/modules/recrawl/AbstractContentDigestHistory.java b/modules/src/main/java/org/archive/modules/recrawl/AbstractContentDigestHistory.java new file mode 100644 index 00000000..e0f39934 --- /dev/null +++ b/modules/src/main/java/org/archive/modules/recrawl/AbstractContentDigestHistory.java @@ -0,0 +1,59 @@ +/* + * This file is part of the Heritrix web crawler (crawler.archive.org). + * + * Licensed to the Internet Archive (IA) by one or more individual + * contributors. + * + * The IA licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.archive.modules.recrawl; + +import org.archive.modules.CrawlURI; + +/** + * Represents a store of information, presumably persistent, keyed by content + * digest. + * + * @contributor nlevitt + */ +public abstract class AbstractContentDigestHistory { + /** + * Looks up the history by key {@code persistKeyFor(curi)} and loads it into + * {@code curi.getContentDigestHistory()}. + * + * @param curi + */ + public abstract void load(CrawlURI curi); + + /** + * Stores {@code curi.getContentDigestHistory()} for the key + * {@code persistKeyFor(curi)}. + * + * @param curi + */ + public abstract void store(CrawlURI curi); + + /** + * + * @param curi + * @return {@code curi.getContentDigestSchemeString()} + * @throws IllegalStateException if {@code curi.getContentDigestSchemeString()} is null + */ + protected String persistKeyFor(CrawlURI curi) { + String key = curi.getContentDigestSchemeString(); + if (key == null) { + throw new IllegalStateException("cannot load content digest history, CrawlURI does not have content digest value for " + curi); + } + return key; + } +} diff --git a/modules/src/main/java/org/archive/modules/recrawl/BdbContentDigestHistory.java b/modules/src/main/java/org/archive/modules/recrawl/BdbContentDigestHistory.java index 7d1c902a..81fc9f35 100644 --- a/modules/src/main/java/org/archive/modules/recrawl/BdbContentDigestHistory.java +++ b/modules/src/main/java/org/archive/modules/recrawl/BdbContentDigestHistory.java @@ -35,7 +35,7 @@ import com.sleepycat.je.Database; import com.sleepycat.je.DatabaseException; /** Needs to be a toplevel bean for Lifecyle? */ -public class BdbContentDigestHistory implements Lifecycle { +public class BdbContentDigestHistory extends AbstractContentDigestHistory implements Lifecycle { private static final Logger logger = Logger.getLogger(BdbContentDigestHistory.class.getName()); @@ -58,10 +58,6 @@ public class BdbContentDigestHistory implements Lifecycle { protected StoredSortedMap store; protected Database historyDb; - protected String persistKeyFor(CrawlURI curi) { - return curi.getContentDigestSchemeString(); - } - @Override @SuppressWarnings({"rawtypes"}) public void start() { diff --git a/modules/src/main/java/org/archive/modules/recrawl/ContentDigestHistoryLoader.java b/modules/src/main/java/org/archive/modules/recrawl/ContentDigestHistoryLoader.java index 8ec9a52a..2f0f13cf 100644 --- a/modules/src/main/java/org/archive/modules/recrawl/ContentDigestHistoryLoader.java +++ b/modules/src/main/java/org/archive/modules/recrawl/ContentDigestHistoryLoader.java @@ -24,10 +24,10 @@ import org.springframework.beans.factory.annotation.Autowired; public class ContentDigestHistoryLoader extends Processor { - protected BdbContentDigestHistory contentDigestHistory; + protected AbstractContentDigestHistory contentDigestHistory; @Autowired public void setContentDigestHistory( - BdbContentDigestHistory contentDigestHistory) { + AbstractContentDigestHistory contentDigestHistory) { this.contentDigestHistory = contentDigestHistory; } diff --git a/modules/src/main/java/org/archive/modules/recrawl/ContentDigestHistoryStorer.java b/modules/src/main/java/org/archive/modules/recrawl/ContentDigestHistoryStorer.java index 0462b90a..f0c779ed 100644 --- a/modules/src/main/java/org/archive/modules/recrawl/ContentDigestHistoryStorer.java +++ b/modules/src/main/java/org/archive/modules/recrawl/ContentDigestHistoryStorer.java @@ -24,10 +24,10 @@ import org.springframework.beans.factory.annotation.Autowired; public class ContentDigestHistoryStorer extends Processor { - protected BdbContentDigestHistory contentDigestHistory; + protected AbstractContentDigestHistory contentDigestHistory; @Autowired public void setContentDigestHistory( - BdbContentDigestHistory contentDigestHistory) { + AbstractContentDigestHistory contentDigestHistory) { this.contentDigestHistory = contentDigestHistory; } From fe1df5883fedd77ae807ed90c46cdc366d057753 Mon Sep 17 00:00:00 2001 From: Noah Levitt Date: Tue, 25 Sep 2012 14:35:26 -0700 Subject: [PATCH 13/19] Some tweaks on maintaining content digest history * CrawlURI.java utility method hasContentDigestHistory() * BdbContentDigestHistory.java load() - make sure to call curi.getContentDigestHistory() in all cases so the value is initialized and WARCWriterProcessor knows it should put the info in there * WARCWriterProcessor.java updateMetadataAfterWrite() - update curi.getContentDigestHistory() only if curi.hasContentDigestHistory() for efficiency, like old uri-based fetch history; update the count after writing a revisit record --- .../src/main/java/org/archive/modules/CrawlURI.java | 4 ++++ .../modules/recrawl/BdbContentDigestHistory.java | 7 ++++++- .../archive/modules/writer/WARCWriterProcessor.java | 12 +++++++++--- 3 files changed, 19 insertions(+), 4 deletions(-) diff --git a/modules/src/main/java/org/archive/modules/CrawlURI.java b/modules/src/main/java/org/archive/modules/CrawlURI.java index 4525a978..93e591a6 100644 --- a/modules/src/main/java/org/archive/modules/CrawlURI.java +++ b/modules/src/main/java/org/archive/modules/CrawlURI.java @@ -1898,4 +1898,8 @@ implements Reporter, Serializable, OverlayContext { return contentDigestHistory; } + public boolean hasContentDigestHistory() { + return getData().get(A_CONTENT_DIGEST_HISTORY) != null; + } + } diff --git a/modules/src/main/java/org/archive/modules/recrawl/BdbContentDigestHistory.java b/modules/src/main/java/org/archive/modules/recrawl/BdbContentDigestHistory.java index 81fc9f35..c8de30e0 100644 --- a/modules/src/main/java/org/archive/modules/recrawl/BdbContentDigestHistory.java +++ b/modules/src/main/java/org/archive/modules/recrawl/BdbContentDigestHistory.java @@ -18,6 +18,7 @@ */ package org.archive.modules.recrawl; +import java.util.HashMap; import java.util.Map; import java.util.logging.Level; import java.util.logging.Logger; @@ -106,6 +107,10 @@ public class BdbContentDigestHistory extends AbstractContentDigestHistory implem } public void load(CrawlURI curi) { + // make this call in all cases so that the value is initialized and + // WARCWriterProcessor knows it should put the info in there + HashMap contentDigestHistory = curi.getContentDigestHistory(); + @SuppressWarnings("unchecked") Map loadedHistory = store.get(persistKeyFor(curi)); if (loadedHistory != null) { @@ -113,7 +118,7 @@ public class BdbContentDigestHistory extends AbstractContentDigestHistory implem logger.finer("loaded history by digest " + persistKeyFor(curi) + " for uri " + curi + " - " + loadedHistory); } - curi.getContentDigestHistory().putAll(loadedHistory); + contentDigestHistory.putAll(loadedHistory); } } diff --git a/modules/src/main/java/org/archive/modules/writer/WARCWriterProcessor.java b/modules/src/main/java/org/archive/modules/writer/WARCWriterProcessor.java index acd229fe..d432609f 100644 --- a/modules/src/main/java/org/archive/modules/writer/WARCWriterProcessor.java +++ b/modules/src/main/java/org/archive/modules/writer/WARCWriterProcessor.java @@ -315,7 +315,7 @@ public class WARCWriterProcessor extends WriterPoolProcessor implements WARCWrit } // history for uri-agnostic, content digest based dedupe - if (curi.getContentDigest() != null) { + if (curi.getContentDigest() != null && curi.hasContentDigestHistory()) { for (WARCRecordInfo warcRecord: writer.getTmpRecordLog()) { if ((warcRecord.getType() == WARCRecordType.RESPONSE || warcRecord.getType() == WARCRecordType.RESOURCE) @@ -327,8 +327,14 @@ public class WARCWriterProcessor extends WriterPoolProcessor implements WARCWrit curi.getContentDigestHistory().put(A_WARC_FILE_OFFSET, warcRecord.getWARCFileOffset()); curi.getContentDigestHistory().put(A_ORIGINAL_DATE, warcRecord.getCreate14DigitDate()); curi.getContentDigestHistory().put(A_CONTENT_DIGEST_COUNT, 1); - // } else if (warcRecord.getType() == WARCRecordType.REVISIT) { - // XXX add to content-digest-count IF it's a content digest based revisit record + } else if (warcRecord.getType() == WARCRecordType.REVISIT + && curi.getAnnotations().contains("warcRevisit:uriAgnosticDigest")) { + Integer oldCount = (Integer) curi.getContentDigestHistory().get(A_CONTENT_DIGEST_COUNT); + if (oldCount == null) { + // shouldn't happen, log a warning? + oldCount = 1; + } + curi.getContentDigestHistory().put(A_CONTENT_DIGEST_COUNT, oldCount + 1); } } } From 34252c4949e5460ff6ac82a02a6ccf028be7f659 Mon Sep 17 00:00:00 2001 From: Noah Levitt Date: Tue, 25 Sep 2012 15:26:56 -0700 Subject: [PATCH 14/19] * WARCWriterProcessor.java writeHttpRecords() - avoid inadvertently creating content digest history map --- .../java/org/archive/modules/writer/WARCWriterProcessor.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/modules/src/main/java/org/archive/modules/writer/WARCWriterProcessor.java b/modules/src/main/java/org/archive/modules/writer/WARCWriterProcessor.java index d432609f..e2b7e465 100644 --- a/modules/src/main/java/org/archive/modules/writer/WARCWriterProcessor.java +++ b/modules/src/main/java/org/archive/modules/writer/WARCWriterProcessor.java @@ -432,7 +432,8 @@ public class WARCWriterProcessor extends WriterPoolProcessor implements WARCWrit headers.addLabelValue(HEADER_KEY_IP, getHostAddress(curi)); URI rid; - if (getWriteRevisitForIdenticalDigests() + if (getWriteRevisitForIdenticalDigests() + && curi.hasContentDigestHistory() && curi.getContentDigestHistory().get(A_ORIGINAL_URL) != null) { rid = writeRevisitUriAgnosticDigest(w, timestamp, HTTP_RESPONSE_MIMETYPE, baseid, curi, headers); From 819371f6eb2401b6a12ff91098a3c9540af83bac Mon Sep 17 00:00:00 2001 From: Noah Levitt Date: Tue, 25 Sep 2012 15:28:19 -0700 Subject: [PATCH 15/19] * BdbContentDigestHistory.java store() - avoid clobbering good content digest history with empty one --- .../org/archive/modules/recrawl/BdbContentDigestHistory.java | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/modules/src/main/java/org/archive/modules/recrawl/BdbContentDigestHistory.java b/modules/src/main/java/org/archive/modules/recrawl/BdbContentDigestHistory.java index c8de30e0..03ed4d01 100644 --- a/modules/src/main/java/org/archive/modules/recrawl/BdbContentDigestHistory.java +++ b/modules/src/main/java/org/archive/modules/recrawl/BdbContentDigestHistory.java @@ -123,6 +123,11 @@ public class BdbContentDigestHistory extends AbstractContentDigestHistory implem } public void store(CrawlURI curi) { + if (!curi.hasContentDigestHistory() || curi.getContentDigestHistory().isEmpty()) { + logger.warning("not saving empty content digest history (do you " + + " have a ContentDigestHistoryLoader in your disposition chain?) - " + curi); + return; + } if (logger.isLoggable(Level.FINER)) { logger.finer("storing history by digest " + persistKeyFor(curi) + " for uri " + curi + " - " From 50c8a09b636ca940cc1a041a5050e5c9bc805f72 Mon Sep 17 00:00:00 2001 From: Noah Levitt Date: Tue, 25 Sep 2012 16:04:44 -0700 Subject: [PATCH 16/19] * BdbContentDigestHistory.java class javadoc --- .../modules/recrawl/BdbContentDigestHistory.java | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/modules/src/main/java/org/archive/modules/recrawl/BdbContentDigestHistory.java b/modules/src/main/java/org/archive/modules/recrawl/BdbContentDigestHistory.java index 03ed4d01..c2ab7a26 100644 --- a/modules/src/main/java/org/archive/modules/recrawl/BdbContentDigestHistory.java +++ b/modules/src/main/java/org/archive/modules/recrawl/BdbContentDigestHistory.java @@ -35,7 +35,13 @@ import com.sleepycat.collections.StoredSortedMap; import com.sleepycat.je.Database; import com.sleepycat.je.DatabaseException; -/** Needs to be a toplevel bean for Lifecyle? */ +/** + * Bdb content digest history store. Must be a toplevel bean in + * crawler-beans.cxml in order to receive {@link Lifecycle} events. + * + * @see AbstractContentDigestHistory + * @contributor nlevitt + */ public class BdbContentDigestHistory extends AbstractContentDigestHistory implements Lifecycle { private static final Logger logger = @@ -60,7 +66,7 @@ public class BdbContentDigestHistory extends AbstractContentDigestHistory implem protected Database historyDb; @Override - @SuppressWarnings({"rawtypes"}) + @SuppressWarnings("rawtypes") public void start() { if (isRunning()) { return; From 2e1d81fa8034e8a5a538c08043cd0b016dfbbe23 Mon Sep 17 00:00:00 2001 From: Noah Levitt Date: Wed, 26 Sep 2012 08:58:22 -0700 Subject: [PATCH 17/19] Fix build * WARCConstants.java, WARCWriter.java remove imports of WARCWriterProcessor from the modules/ area (was only used in javadoc) --- commons/src/main/java/org/archive/io/warc/WARCConstants.java | 1 - commons/src/main/java/org/archive/io/warc/WARCWriter.java | 1 - 2 files changed, 2 deletions(-) diff --git a/commons/src/main/java/org/archive/io/warc/WARCConstants.java b/commons/src/main/java/org/archive/io/warc/WARCConstants.java index eb4941c6..862f8a16 100644 --- a/commons/src/main/java/org/archive/io/warc/WARCConstants.java +++ b/commons/src/main/java/org/archive/io/warc/WARCConstants.java @@ -20,7 +20,6 @@ package org.archive.io.warc; import org.archive.io.ArchiveFileConstants; -import org.archive.modules.writer.WARCWriterProcessor; /** * WARC Constants used by WARC readers and writers. diff --git a/commons/src/main/java/org/archive/io/warc/WARCWriter.java b/commons/src/main/java/org/archive/io/warc/WARCWriter.java index 03366536..9c7b737e 100644 --- a/commons/src/main/java/org/archive/io/warc/WARCWriter.java +++ b/commons/src/main/java/org/archive/io/warc/WARCWriter.java @@ -40,7 +40,6 @@ import org.apache.commons.lang.StringUtils; import org.archive.io.ArchiveFileConstants; import org.archive.io.UTF8Bytes; import org.archive.io.WriterPoolMember; -import org.archive.modules.writer.WARCWriterProcessor; import org.archive.util.ArchiveUtils; import org.archive.util.anvl.Element; From 190e2ff0f32948a09abc5e3833c999dd436796c2 Mon Sep 17 00:00:00 2001 From: Noah Levitt Date: Wed, 26 Sep 2012 10:58:01 -0700 Subject: [PATCH 18/19] Fix unit test * WARCWriterTest.java writeRandomHTTPRecord() - fix method, had transposed some lines of code --- .../src/test/java/org/archive/io/warc/WARCWriterTest.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/commons/src/test/java/org/archive/io/warc/WARCWriterTest.java b/commons/src/test/java/org/archive/io/warc/WARCWriterTest.java index fa19a5e2..300e948a 100644 --- a/commons/src/test/java/org/archive/io/warc/WARCWriterTest.java +++ b/commons/src/test/java/org/archive/io/warc/WARCWriterTest.java @@ -218,12 +218,12 @@ extends TmpDirTestCase implements WARCConstants { String indexStr = Integer.toString(index); recordInfo.setUrl("http://www.one.net/id=" + indexStr); - ByteArrayOutputStream baos = new ByteArrayOutputStream(); - recordInfo.setContentStream(new ByteArrayInputStream(baos.toByteArray())); - byte[] record = (getContent(indexStr)).getBytes(); recordInfo.setContentLength((long) record.length); + + ByteArrayOutputStream baos = new ByteArrayOutputStream(); baos.write(record); + recordInfo.setContentStream(new ByteArrayInputStream(baos.toByteArray())); // Add named fields for ip, checksum, and relate the metadata // and request to the resource field. From 3612396ff3cb363c4405578da5f1e023b1eee233 Mon Sep 17 00:00:00 2001 From: Noah Levitt Date: Wed, 26 Sep 2012 12:03:45 -0700 Subject: [PATCH 19/19] Unit test for content digest history * ContentDigestHistoryTest.java test basics * BdbContentDigestHistory.java code formatting tweak --- .../recrawl/BdbContentDigestHistory.java | 8 +- .../recrawl/ContentDigestHistoryTest.java | 137 ++++++++++++++++++ 2 files changed, 142 insertions(+), 3 deletions(-) create mode 100644 modules/src/test/java/org/archive/modules/recrawl/ContentDigestHistoryTest.java diff --git a/modules/src/main/java/org/archive/modules/recrawl/BdbContentDigestHistory.java b/modules/src/main/java/org/archive/modules/recrawl/BdbContentDigestHistory.java index c2ab7a26..1d09e43e 100644 --- a/modules/src/main/java/org/archive/modules/recrawl/BdbContentDigestHistory.java +++ b/modules/src/main/java/org/archive/modules/recrawl/BdbContentDigestHistory.java @@ -129,9 +129,11 @@ public class BdbContentDigestHistory extends AbstractContentDigestHistory implem } public void store(CrawlURI curi) { - if (!curi.hasContentDigestHistory() || curi.getContentDigestHistory().isEmpty()) { - logger.warning("not saving empty content digest history (do you " + - " have a ContentDigestHistoryLoader in your disposition chain?) - " + curi); + if (!curi.hasContentDigestHistory() + || curi.getContentDigestHistory().isEmpty()) { + logger.warning("not saving empty content digest history (do you " + + " have a ContentDigestHistoryLoader in your disposition" + + " chain?) - " + curi); return; } if (logger.isLoggable(Level.FINER)) { diff --git a/modules/src/test/java/org/archive/modules/recrawl/ContentDigestHistoryTest.java b/modules/src/test/java/org/archive/modules/recrawl/ContentDigestHistoryTest.java new file mode 100644 index 00000000..371bcedd --- /dev/null +++ b/modules/src/test/java/org/archive/modules/recrawl/ContentDigestHistoryTest.java @@ -0,0 +1,137 @@ +/* + * This file is part of the Heritrix web crawler (crawler.archive.org). + * + * Licensed to the Internet Archive (IA) by one or more individual + * contributors. + * + * The IA licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.archive.modules.recrawl; + +import static org.archive.modules.recrawl.RecrawlAttributeConstants.A_ORIGINAL_URL; + +import java.io.IOException; +import java.util.logging.Logger; + +import org.apache.commons.io.FileUtils; +import org.archive.bdb.BdbModule; +import org.archive.modules.CrawlURI; +import org.archive.net.UURIFactory; +import org.archive.spring.ConfigPath; +import org.archive.util.Base32; +import org.archive.util.TmpDirTestCase; + +public class ContentDigestHistoryTest extends TmpDirTestCase { + + private static Logger logger = Logger.getLogger(ContentDigestHistoryTest.class.getName()); + + protected BdbModule bdb; + protected BdbContentDigestHistory historyStore; + protected ContentDigestHistoryStorer storer; + protected ContentDigestHistoryLoader loader; + + protected ContentDigestHistoryLoader loader() throws IOException { + if (loader == null) { + loader = new ContentDigestHistoryLoader(); + loader.setContentDigestHistory(historyStore()); + logger.info("created " + loader); + } + return loader; + } + + protected ContentDigestHistoryStorer storer() throws IOException { + if (storer == null) { + storer = new ContentDigestHistoryStorer(); + storer.setContentDigestHistory(historyStore()); + logger.info("created " + storer); + } + return storer; + } + + protected BdbContentDigestHistory historyStore() throws IOException { + if (historyStore == null) { + historyStore = new BdbContentDigestHistory(); + historyStore.setBdbModule(bdb()); + historyStore.start(); + logger.info("created " + historyStore); + } + return historyStore; + } + + protected BdbModule bdb() throws IOException { + if (bdb == null) { + ConfigPath basePath = new ConfigPath("testBase",getTmpDir().getAbsolutePath()); + ConfigPath bdbDir = new ConfigPath("bdb","bdb"); + bdbDir.setBase(basePath); + FileUtils.deleteDirectory(bdbDir.getFile()); + + bdb = new BdbModule(); + bdb.setDir(bdbDir); + bdb.start(); + logger.info("created " + bdb); + } + return bdb; + } + + public void testBasics() throws InterruptedException, IOException { + CrawlURI curi1 = new CrawlURI(UURIFactory.getInstance("http://example.org/1")); + + assertFalse(loader().shouldProcess(curi1)); + assertFalse(storer().shouldProcess(curi1)); + + // sha1 of "monkey\n", point is to have a value there + curi1.setContentDigest("sha1", Base32.decode("orfjublpcrnymm4seg5uk6vfoeu7kw6c")); + + assertTrue(loader().shouldProcess(curi1)); + assertTrue(storer().shouldProcess(curi1)); + + assertEquals("sha1:ORFJUBLPCRNYMM4SEG5UK6VFOEU7KW6C", historyStore().persistKeyFor(curi1)); + + assertFalse(curi1.hasContentDigestHistory()); + + loader().process(curi1); + + assertTrue(curi1.hasContentDigestHistory()); + assertTrue(curi1.getContentDigestHistory().isEmpty()); + + storer().process(curi1); + assertTrue(historyStore().store.isEmpty()); + + curi1.getContentDigestHistory().put(A_ORIGINAL_URL, "http://example.org/original"); + // curi1.getContentDigestHistory().put(A_WARC_RECORD_ID, ""); + // curi1.getContentDigestHistory().put(A_WARC_FILENAME, "test.warc.gz"); + // curi1.getContentDigestHistory().put(A_WARC_FILE_OFFSET, 98765432l); + // curi1.getContentDigestHistory().put(A_ORIGINAL_DATE, "20120101000000"); + // curi1.getContentDigestHistory().put(A_CONTENT_DIGEST_COUNT, 1); + + loader().process(curi1); + assertEquals("http://example.org/original", curi1.getContentDigestHistory().get(A_ORIGINAL_URL)); + + storer().process(curi1); + + assertFalse(historyStore().store.isEmpty()); + assertEquals(1, historyStore().store.size()); + + CrawlURI curi2 = new CrawlURI(UURIFactory.getInstance("http://example.org/2")); + curi2.setContentDigest("sha1", Base32.decode("orfjublpcrnymm4seg5uk6vfoeu7kw6c")); + + assertFalse(curi2.hasContentDigestHistory()); + + loader().process(curi2); + + assertTrue(curi2.hasContentDigestHistory()); + assertEquals("http://example.org/original", curi2.getContentDigestHistory().get(A_ORIGINAL_URL)); + } + + +}