lee/heritrix3
1
0
Fork 0
mirror of https://github.com/internetarchive/heritrix3.git synced 2026-09-26 07:35:39 +00:00
Code Issues Packages Projects Releases Wiki Activity

ArcRecord: Silently skip any header fields before a status line in arc record. This allows the arc reader to be fault-tolerant of 'bad' arcs that have extra headers

Browse Source 
inserted before the status line (but are otherwise valid)
This commit is contained in:
Ilya Kreymer
2013-07-18 17:08:38 -07:00
parent 3916175e4d
commit 1b67f5ce6b
1 changed files with 29 additions and 7 deletions
Download Patch File Download Diff File Expand all files Collapse all files
commons/src/main/java/org/archive/io/arc/ARCRecord.java
+29 -7
View File
@@ -33,7 +33,6 @@ import java.util.logging.Logger;
import java.util.regex.Matcher;
import org.apache.commons.httpclient.Header;
import org.apache.commons.httpclient.HttpParser;
import org.apache.commons.httpclient.StatusLine;
import org.apache.commons.httpclient.util.EncodingUtil;
import org.apache.commons.lang.StringUtils;
@@ -570,15 +569,38 @@ public class ARCRecord extends ArchiveRecord implements ARCConstants {
getHeader().getLength() <= MIN_HTTP_HEADER_LENGTH) {
return null;
}
byte [] statusBytes = LaxHttpParser.readRawLine(getIn());
int eolCharCount = getEolCharsCount(statusBytes);
if (eolCharCount <= 0) {
throw new RecoverableIOException(
String statusLine;
byte[] statusBytes;
int eolCharCount = 0;
// Read status line, skipping any errant http headers found before it
// This allows a larger number of 'corrupt' arcs -- where headers were accidentally
// inserted before the status line to be readable
while (true) {
statusBytes = LaxHttpParser.readRawLine(getIn());
eolCharCount = getEolCharsCount(statusBytes);
if (eolCharCount <= 0) {
throw new RecoverableIOException(
"Failed to read http status where one was expected: "
+ ((statusBytes == null) ? "" : new String(statusBytes)));
}
statusLine = EncodingUtil.getString(statusBytes, 0,
statusBytes.length - eolCharCount, ARCConstants.DEFAULT_ENCODING);
// If a null or DELETED break immediately
if ((statusLine == null) || statusLine.startsWith("DELETED")) {
break;
}
// If it's actually the status line, break, otherwise continue skipping any
// previous header values
if (!statusLine.contains(":") && StatusLine.startsWithHTTP(statusLine)) {
break;
}
}
String statusLine = EncodingUtil.getString(statusBytes, 0,
statusBytes.length - eolCharCount, ARCConstants.DEFAULT_ENCODING);
if ((statusLine == null) ||
!StatusLine.startsWithHTTP(statusLine)) {
if (statusLine.startsWith("DELETED")) {