Based on pointers in - * SSL - * Guide, - * and readings done in JSSE - * Guide. - * - *
TODO: Move to an ssl subpackage when we have other classes other than - * just this one. - * - * @author stack - * @version $Id$ - */ -public class ConfigurableX509TrustManager implements X509TrustManager -{ - /** - * Logging instance. - */ - protected static Logger logger = Logger.getLogger( - "org.archive.httpclient.ConfigurableX509TrustManager"); - - public static enum TrustLevel { - /** - * Trust anything given us. - * - * Default setting. - * - *
See - * e502. Disabling Certificate Validation in an HTTPS Connection from - * the java almanac for how to trust all. - */ - OPEN, - - /** - * Trust any valid cert including self-signed certificates. - */ - LOOSE, - - /** - * Normal jsse behavior. - * - * Seemingly any certificate that supplies valid chain of trust. - */ - NORMAL, - - /** - * Strict trust. - * - * Ensure server has same name as cert DN. - */ - STRICT, - } - - /** - * Default setting for trust level. - */ - public final static TrustLevel DEFAULT = TrustLevel.OPEN; - - /** - * Trust level. - */ - private TrustLevel trustLevel = DEFAULT; - - - /** - * An instance of the SUNX509TrustManager that we adapt variously - * depending upon passed configuration. - * - * We have it do all the work we don't want to. - */ - private X509TrustManager standardTrustManager = null; - - - public ConfigurableX509TrustManager() - throws NoSuchAlgorithmException, KeyStoreException { - this(DEFAULT); - } - - /** - * Constructor. - * - * @param level Level of trust to effect. - * - * @throws NoSuchAlgorithmException - * @throws KeyStoreException - */ - public ConfigurableX509TrustManager(TrustLevel level) - throws NoSuchAlgorithmException, KeyStoreException { - super(); - TrustManagerFactory factory = TrustManagerFactory. - getInstance(TrustManagerFactory.getDefaultAlgorithm()); - - // Pass in a null (Trust) KeyStore. Null says use the 'default' - // 'trust' keystore (KeyStore class is used to hold keys and to hold - // 'trusts' (certs)). See 'X509TrustManager Interface' in this doc: - // http://java.sun.com - // /j2se/1.4.2/docs/guide/security/jsse/JSSERefGuide.html#Introduction - factory.init((KeyStore)null); - TrustManager[] trustmanagers = factory.getTrustManagers(); - if (trustmanagers.length == 0) { - throw new NoSuchAlgorithmException(TrustManagerFactory. - getDefaultAlgorithm() + " trust manager not supported"); - } - this.standardTrustManager = (X509TrustManager)trustmanagers[0]; - - this.trustLevel = level; - } - - public void checkClientTrusted(X509Certificate[] certificates, String type) - throws CertificateException { - if (this.trustLevel.equals(TrustLevel.OPEN)) { - return; - } - - this.standardTrustManager.checkClientTrusted(certificates, type); - } - - public void checkServerTrusted(X509Certificate[] certificates, String type) - throws CertificateException { - if (this.trustLevel.equals(TrustLevel.OPEN)) { - return; - } - - try { - this.standardTrustManager.checkServerTrusted(certificates, type); - if (this.trustLevel.equals(TrustLevel.STRICT)) { - logger.severe(TrustLevel.STRICT + " not implemented."); - } - } catch (CertificateException e) { - if (this.trustLevel.equals(TrustLevel.LOOSE) && - certificates != null && certificates.length == 1) - { - // If only one cert and its valid and it caused a - // CertificateException, assume its selfsigned. - X509Certificate certificate = certificates[0]; - certificate.checkValidity(); - } else { - // If we got to here, then we're probably NORMAL. Rethrow. - throw e; - } - } - } - - public X509Certificate[] getAcceptedIssuers() { - return this.standardTrustManager.getAcceptedIssuers(); - } -} diff --git a/commons/src/main/java/org/archive/httpclient/package.html b/commons/src/main/java/org/archive/httpclient/package.html deleted file mode 100644 index 87ae77ed..00000000 --- a/commons/src/main/java/org/archive/httpclient/package.html +++ /dev/null @@ -1,24 +0,0 @@ - - -
-Class that the passed HttpRecorder w/ boundary between - HTTP header and content. Also forces a close on the response on - call to releaseConnection.
- -A protocol socket factory that allows setting of trust level on - construction.
- -JavaTM Secure Socket Extension (JSSE): Reference Guide
- - - diff --git a/commons/src/main/java/org/archive/io/ArchiveFileConstants.java b/commons/src/main/java/org/archive/io/ArchiveFileConstants.java deleted file mode 100644 index b1a39194..00000000 --- a/commons/src/main/java/org/archive/io/ArchiveFileConstants.java +++ /dev/null @@ -1,24 +0,0 @@ -/* - * This file is part of the Heritrix web crawler (crawler.archive.org). - * - * Licensed to the Internet Archive (IA) by one or more individual - * contributors. - * - * The IA licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.archive.io; - -@Deprecated -public interface ArchiveFileConstants extends org.archive.format.ArchiveFileConstants { -} diff --git a/commons/src/main/java/org/archive/io/ArchiveReader.java b/commons/src/main/java/org/archive/io/ArchiveReader.java deleted file mode 100644 index 66056d33..00000000 --- a/commons/src/main/java/org/archive/io/ArchiveReader.java +++ /dev/null @@ -1,761 +0,0 @@ -/* - * This file is part of the Heritrix web crawler (crawler.archive.org). - * - * Licensed to the Internet Archive (IA) by one or more individual - * contributors. - * - * The IA licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.archive.io; - - -import java.io.BufferedInputStream; -import java.io.BufferedWriter; -import java.io.Closeable; -import java.io.EOFException; -import java.io.File; -import java.io.FileInputStream; -import java.io.FileWriter; -import java.io.IOException; -import java.io.InputStream; -import java.util.ArrayList; -import java.util.Iterator; -import java.util.List; -import java.util.logging.Level; -import java.util.logging.Logger; - -import org.apache.commons.cli.Option; -import org.apache.commons.cli.Options; -import org.archive.util.MimetypeUtils; -import org.archive.util.zip.GZIPMembersInputStream; - -import com.google.common.io.CountingInputStream; - - -/** - * Reader for an Archive file of Archive {@link ArchiveRecord}s. - * @author stack - * @version $Date$ $Version$ - */ -public abstract class ArchiveReader implements ArchiveFileConstants, IterableSet in constructor. Should support at least 1 byte mark/reset.
- * Make it protected so subclasses have access.
- */
- protected InputStream in = null;
-
- /**
- * Maximum amount of recoverable exceptions in a row.
- * If more than this amount in a row, we'll let out the exception rather
- * than go back in for yet another retry.
- */
- public static final int MAX_ALLOWED_RECOVERABLES = 10;
-
-
- /**
- * The Record currently being read.
- *
- * Keep this ongoing reference so we'll close the record even if the caller
- * doesn't.
- */
- private ArchiveRecord currentRecord = null;
-
- /**
- * Descriptive string for the Archive file we're going against:
- * full path, url, etc. -- depends on context in which file was made.
- */
- private String identifier = null;
-
- /**
- * Archive file version.
- */
- private String version = null;
-
-
- protected ArchiveReader() {
- super();
- }
-
- /**
- * Convenience method used by subclass constructors.
- * @param i Identifier for Archive file this reader goes against.
- */
- protected void initialize(final String i) {
- setReaderIdentifier(i);
- }
-
- /**
- * Convenience method for constructors.
- *
- * @param f File to read.
- * @param offset Offset at which to start reading.
- * @return InputStream to read from.
- * @throws IOException If failed open or fail to get a memory
- * mapped byte buffer on file.
- */
- protected InputStream getInputStream(final File f, final long offset)
- throws IOException {
- FileInputStream fin = new FileInputStream(f);
- return new BufferedInputStream(fin);
- }
-
- public boolean isCompressed() {
- return this.compressed;
- }
-
- /**
- * Get record at passed offset.
- *
- * @param offset Byte index into file at which a record starts.
- * @return An Archive Record reference.
- * @throws IOException
- */
- public ArchiveRecord get(long offset) throws IOException {
- cleanupCurrentRecord();
- long posn = positionForRecord(in);
- if(offset>=posn) {
- in.skip(offset-posn);
- } else {
- throw new UnsupportedOperationException("no reverse seeking: at "+posn+" requested "+offset);
- }
- return createArchiveRecord(this.in, offset);
- }
-
- /**
- * @return Return Archive Record created against current offset.
- * @throws IOException
- */
- public ArchiveRecord get() throws IOException {
- return createArchiveRecord(this.in, positionForRecord(in));
- }
-
- public void close() throws IOException {
- if (this.in != null) {
- this.in.close();
- this.in = null;
- }
- }
-
- /**
- * Cleanout the current record if there is one.
- * @throws IOException
- */
- protected void cleanupCurrentRecord() throws IOException {
- if (this.currentRecord != null) {
- this.currentRecord.close();
- gotoEOR(this.currentRecord);
- this.currentRecord = null;
- }
- }
-
- /**
- * Return an Archive Record homed on offset into
- * is.
- * @param is Stream to read Record from.
- * @param offset Offset to find Record at.
- * @return ArchiveRecord instance.
- * @throws IOException
- */
- protected abstract ArchiveRecord createArchiveRecord(InputStream is,
- long offset)
- throws IOException;
-
- /**
- * Skip over any trailing new lines at end of the record so we're lined up
- * ready to read the next.
- * @param record
- * @throws IOException
- */
- protected abstract void gotoEOR(ArchiveRecord record) throws IOException;
-
- public abstract String getFileExtension();
- public abstract String getDotFileExtension();
-
- /**
- * @return Version of this Archive file.
- */
- public String getVersion() {
- return this.version;
- }
-
- /**
- * Validate the Archive file.
- *
- * This method iterates over the file throwing exception if it fails
- * to successfully parse any record.
- *
- *
Assumes the stream is at the start of the file.
- * @return List of all read Archive Headers.
- *
- * @throws IOException
- */
- public List We start validation from wherever we are in the stream.
- *
- * @param numRecords Number of records expected. Pass -1 if number is
- * unknown.
- *
- * @return List of all read metadatas. As we validate records, we add
- * a reference to the read metadata.
- *
- * @throws IOException
- */
- public List Streams can be markable or not. If they are, we'll be able to roll
- * back when we've read too far. If not markable, assumption is that
- * the underlying stream is managing our not reading too much (This pertains
- * to the skipping over the end of the ARCRecord. See {@link #skip()}.
- */
- protected InputStream in = null;
-
- /**
- * Position w/i the Record content, within Protected instead of private so subclasses can update and complete
- * the digest.
- */
- protected MessageDigest digest = null;
- private String digestStr = null;
-
- protected boolean strict = false;
-
-
- /**
- * Constructor.
- *
- * @param in Stream cue'd up to be at the start of the record this instance
- * is to represent.
- * @throws IOException
- */
- public ArchiveRecord(InputStream in)
- throws IOException {
- this(in, null, 0, true, false);
- }
-
- /**
- * Constructor.
- *
- * @param in Stream cue'd up to be at the start of the record this instance
- * is to represent.
- * @param header Header data.
- * @throws IOException
- */
- public ArchiveRecord(InputStream in, ArchiveRecordHeader header)
- throws IOException {
- this(in, header, 0, true, false);
- }
-
- /**
- * Constructor.
- *
- * @param in Stream cue'd up to be at the start of the record this instance
- * is to represent.
- * @param header Header data.
- * @param bodyOffset Offset into the body. Usually 0.
- * @param digest True if we're to calculate digest for this record. Not
- * digesting saves about ~15% of cpu during an ARC parse.
- * @param strict Be strict parsing (Parsing stops if ARC inproperly
- * formatted).
- * @throws IOException
- */
- public ArchiveRecord(InputStream in, ArchiveRecordHeader header,
- int bodyOffset, boolean digest, boolean strict)
- throws IOException {
- this.in = in;
- this.header = header;
- this.position = bodyOffset;
- if (digest) {
- try {
- this.digest = MessageDigest.getInstance("SHA1");
- } catch (NoSuchAlgorithmException e) {
- // Convert to IOE because thats more amenable to callers
- // -- they are dealing with it anyways.
- throw new IOException(e.getMessage());
- }
- }
- this.strict = strict;
- }
-
- public boolean markSupported() {
- return false;
- }
-
- /**
- * @return Header data for this record.
- */
- public ArchiveRecordHeader getHeader() {
- return this.header;
- }
-
- protected void setHeader(ArchiveRecordHeader header) {
- this.header = header;
- }
-
- /**
- * Calling close on a record skips us past this record to the next record
- * in the stream.
- *
- * It does not actually close the stream. The underlying steam is probably
- * being used by the next arc record.
- *
- * @throws IOException
- */
- public void close() throws IOException {
- if (this.in != null) {
- skip();
- this.in = null;
- if (this.digest != null) {
- this.digestStr = Base32.encode(this.digest.digest());
- }
- }
- }
-
- /**
- * @return Next character in this Record content else -1 if at EOR.
- * @throws IOException
- */
- public int read() throws IOException {
- int c = -1;
- if (available() > 0) {
- c = this.in.read();
- if (c == -1) {
- throw new IOException("Premature EOF before end-of-record.");
- }
- if (this.digest != null) {
- this.digest.update((byte) c);
- }
- incrementPosition();
- }
- return c;
- }
-
- public int read(byte[] b, int offset, int length) throws IOException {
- int read = Math.min(length, available());
- if (read == -1 || read == 0) {
- read = -1;
- } else {
- read = this.in.read(b, offset, read);
- if (read == -1) {
- String msg = "Premature EOF before end-of-record: "
- + getHeader().getHeaderFields();
- if (isStrict()) {
- throw new IOException(msg);
- }
- setEor(true);
- System.err.println(Level.WARNING.toString() + " " + msg);
- }
- if (this.digest != null && read >= 0) {
- this.digest.update(b, offset, read);
- }
- incrementPosition(read);
- }
- return read;
- }
-
- /**
- * This available is not the stream's available. Its an available based on
- * what the stated Archive record length is minus what we've read to date.
- *
- * @return True if bytes remaining in record content.
- */
- public int available() {
- long amount = getHeader().getLength() - getPosition();
- return (amount > Integer.MAX_VALUE? Integer.MAX_VALUE: (int)amount);
- }
-
- /**
- * Skip over this records content.
- *
- * @throws IOException
- */
- protected void skip() throws IOException {
- if (this.eor) {
- return;
- }
-
- // Read to the end of the body of the record. Exhaust the stream.
- // Can't skip direct to end because underlying stream may be compressed
- // and we're calculating the digest for the record.
- int r = available();
- while (r > 0 && !this.eor) {
- skip(r);
- r = available();
- }
- }
-
- public long skip(long n) throws IOException {
- final int SKIP_BUFFERSIZE = 1024 * 4;
- byte[] b = new byte[SKIP_BUFFERSIZE];
- long total = 0;
- for (int read = 0; (total < n) && (read != -1);) {
- read = Math.min(SKIP_BUFFERSIZE, (int) (n - total));
- // TODO: Interesting is that reading from compressed stream, we only
- // read about 500 characters at a time though we ask for 4k.
- // Look at this sometime.
- read = read(b, 0, read);
- if (read <= 0) {
- read = -1;
- } else {
- total += read;
- }
- }
- return total;
- }
-
- /**
- * @return Returns the strict.
- */
- public boolean isStrict() {
- return this.strict;
- }
-
- /**
- * @param strict The strict to set.
- */
- public void setStrict(boolean strict) {
- this.strict = strict;
- }
-
- protected InputStream getIn() {
- return this.in;
- }
-
- public String getDigestStr() {
- return this.digestStr;
- }
-
- protected void incrementPosition() {
- this.position++;
- }
-
- protected void incrementPosition(final long incr) {
- this.position += incr;
- }
-
- public long getPosition() {
- return this.position;
- }
-
- protected boolean isEor() {
- return eor;
- }
-
- protected void setEor(boolean eor) {
- this.eor = eor;
- }
-
- protected String getStatusCode4Cdx(final ArchiveRecordHeader h) {
- return "-";
- }
-
- protected String getIp4Cdx(final ArchiveRecordHeader h) {
- return "-";
- }
-
- protected String getDigest4Cdx(final ArchiveRecordHeader h) {
- return getDigestStr() == null? "-": getDigestStr();
- }
-
- protected String getMimetype4Cdx(final ArchiveRecordHeader h) {
- return h.getMimetype();
- }
-
- protected String outputCdx(final String strippedFileName)
- throws IOException {
- // Read the whole record so we get out a hash. Should be safe calling
- // close on already closed Record.
- close();
- ArchiveRecordHeader h = getHeader();
- StringBuilder buffer =
- new StringBuilder(ArchiveFileConstants.CDX_LINE_BUFFER_SIZE);
- buffer.append(h.getDate());
- buffer.append(ArchiveFileConstants.SINGLE_SPACE);
- buffer.append(getIp4Cdx(h));
- buffer.append(ArchiveFileConstants.SINGLE_SPACE);
- buffer.append(h.getUrl());
- buffer.append(ArchiveFileConstants.SINGLE_SPACE);
- buffer.append(getMimetype4Cdx(h));
- buffer.append(ArchiveFileConstants.SINGLE_SPACE);
- buffer.append(getStatusCode4Cdx(h));
- buffer.append(ArchiveFileConstants.SINGLE_SPACE);
- buffer.append(getDigest4Cdx(h));
- buffer.append(ArchiveFileConstants.SINGLE_SPACE);
- buffer.append(h.getOffset());
- buffer.append(ArchiveFileConstants.SINGLE_SPACE);
- buffer.append(h.getLength());
- buffer.append(ArchiveFileConstants.SINGLE_SPACE);
- buffer.append(strippedFileName != null? strippedFileName: '-');
- return buffer.toString();
- }
-
- /**
- * Writes output on STDOUT.
- * @throws IOException
- */
- public void dump()
- throws IOException {
- dump(System.out);
- }
-
- /**
- * Writes output on passed Call {@link close()} on this class when done to clean up resources.
- *
- * @contributor stack
- * @contributor nlevitt
- * @version $Revision$, $Date$
- */
-public class GenericReplayCharSequence implements ReplayCharSequence {
-
- protected static Logger logger = Logger
- .getLogger(GenericReplayCharSequence.class.getName());
-
- /**
- * Name of the encoding we use writing out concatenated decoded prefix
- * buffer and decoded backing file.
- *
- * This define is also used as suffix for the file that holds the
- * decodings. The name of the file that holds the decoding is the name
- * of the backing file w/ this encoding for a suffix.
- *
- * See Encoding.
- */
- public static final Charset WRITE_ENCODING = Charsets.UTF_16BE;
-
- private static final long MAP_MAX_BYTES = 64 * 1024 * 1024; // 64M
-
- /**
- * When the memory map moves away from the beginning of the file
- * (to the "right") in order to reach a certain index, it will
- * map up to this many bytes preceding (to the left of) the target character.
- * Consequently it will map up to
- * Calling this method in the midst of reading the header
- * will make for strange results. Otherwise, safe to call
- * at any time though before reading any of the record
- * content is only time that it makes sense.
- *
- * After calling this method, you can call
- * {@link #getContentHeaders()} to get the read http header.
- *
- * @throws IOException
- */
- public void skipHttpHeader() throws IOException {
- if (this.contentHeaderStream == null) {
- return;
- }
- // Empty the contentHeaderStream
- for (int available = this.contentHeaderStream.available();
- this.contentHeaderStream != null
- && (available = this.contentHeaderStream.available()) > 0;) {
- // We should be in this loop once only we should only do this
- // buffer allocation once.
- byte[] buffer = new byte[available];
- // The read nulls out httpHeaderStream when done with it so
- // need check for null in the loop control line.
- read(buffer, 0, available);
- }
- }
-
- public void dumpHttpHeader() throws IOException {
- dumpHttpHeader(System.out);
- }
-
- public void dumpHttpHeader(final PrintStream stream) throws IOException {
- if (this.contentHeaderStream == null) {
- return;
- }
- // Dump the httpHeaderStream to STDOUT
- for (int available = this.contentHeaderStream.available();
- this.contentHeaderStream != null
- && (available = this.contentHeaderStream.available()) > 0;) {
- // We should be in this loop only once and should do this
- // buffer allocation once.
- byte[] buffer = new byte[available];
- // The read nulls out httpHeaderStream when done with it so
- // need check for null in the loop control line.
- int read = read(buffer, 0, available);
- stream.write(buffer, 0, read);
- }
- }
-
- /**
- * Read header if present. Technique borrowed from HttpClient HttpParse
- * class. Using http parser code for now. Later move to more generic header
- * parsing code if there proves a need.
- *
- * @return ByteArrayInputStream with the http header in it or null if no
- * http header.
- * @throws IOException
- */
- private InputStream readContentHeaders() throws IOException {
- // If judged a record that doesn't have an http header, return
- // immediately.
- if (!hasContentHeaders()) {
- return null;
- }
- byte [] statusBytes = LaxHttpParser.readRawLine(getIn());
- int eolCharCount = getEolCharsCount(statusBytes);
- if (eolCharCount <= 0) {
- throw new IOException("Failed to read raw lie where one " +
- " was expected: " + new String(statusBytes));
- }
- String statusLine = EncodingUtil.getString(statusBytes, 0,
- statusBytes.length - eolCharCount, ARCConstants.DEFAULT_ENCODING);
- if (statusLine == null) {
- throw new NullPointerException("Expected status line is null");
- }
- // TODO: Tighten up this test.
- boolean isHttpResponse = StatusLine.startsWithHTTP(statusLine);
- boolean isHttpRequest = false;
- if (!isHttpResponse) {
- isHttpRequest = statusLine.toUpperCase().startsWith("GET") ||
- !statusLine.toUpperCase().startsWith("POST");
- }
- if (!isHttpResponse && !isHttpRequest) {
- throw new UnexpectedStartLineIOException("Failed parse of " +
- "status line: " + statusLine);
- }
- this.statusCode = isHttpResponse?
- (new StatusLine(statusLine)).getStatusCode(): -1;
-
- // Save off all bytes read. Keep them as bytes rather than
- // convert to strings so we don't have to worry about encodings
- // though this should never be a problem doing http headers since
- // its all supposed to be ascii.
- ByteArrayOutputStream baos =
- new ByteArrayOutputStream(statusBytes.length + 4 * 1024);
- baos.write(statusBytes);
-
- // Now read rest of the header lines looking for the separation
- // between header and body.
- for (byte [] lineBytes = null; true;) {
- lineBytes = LaxHttpParser.readRawLine(getIn());
- eolCharCount = getEolCharsCount(lineBytes);
- if (eolCharCount <= 0) {
- throw new IOException("Failed reading headers: " +
- ((lineBytes != null)? new String(lineBytes): null));
- }
- // Save the bytes read.
- baos.write(lineBytes);
- if ((lineBytes.length - eolCharCount) <= 0) {
- // We've finished reading the http header.
- break;
- }
- }
-
- byte [] headerBytes = baos.toByteArray();
- // Save off where content body, post content headers, starts.
- this.contentHeadersLength = headerBytes.length;
- ByteArrayInputStream bais =
- new ByteArrayInputStream(headerBytes);
- if (!bais.markSupported()) {
- throw new IOException("ByteArrayInputStream does not support mark");
- }
- bais.mark(headerBytes.length);
- // Read the status line. Don't let it into the parseHeaders function.
- // It doesn't know what to do with it.
- bais.read(statusBytes, 0, statusBytes.length);
- this.contentHeaders = LaxHttpParser.parseHeaders(bais,
- ARCConstants.DEFAULT_ENCODING);
- bais.reset();
- return bais;
- }
-
- public static class UnexpectedStartLineIOException
- extends RecoverableIOException {
- private static final long serialVersionUID = 1L;
-
- public UnexpectedStartLineIOException(final String reason) {
- super(reason);
- }
- }
-
- /**
- * @param bytes Array of bytes to examine for an EOL.
- * @return Count of end-of-line characters or zero if none.
- */
- private int getEolCharsCount(byte [] bytes) {
- int count = 0;
- if (bytes != null && bytes.length >=1 &&
- bytes[bytes.length - 1] == '\n') {
- count++;
- if (bytes.length >=2 && bytes[bytes.length -2] == '\r') {
- count++;
- }
- }
- return count;
- }
-
- /**
- * @return If headers are for a http response AND the headers have been
- * read, return status code. Else return -1.
- */
- public int getStatusCode() {
- return this.statusCode;
- }
-
- /**
- * @return Returns length of content headers or -1 if headers have
- * not yet been read.
- */
- public int getContentHeadersLength() {
- return this.contentHeadersLength;
- }
-
- public Header[] getContentHeaders() {
- return contentHeaders;
- }
-
- /**
- * @return Next character in this ARCRecord's content else -1 if at end of
- * this record.
- * @throws IOException
- */
- public int read() throws IOException {
- int c = -1;
- if (this.contentHeaderStream != null &&
- (this.contentHeaderStream.available() > 0)) {
- // If http header, return bytes from it before we go to underlying
- // stream.
- c = this.contentHeaderStream.read();
- // If done with the header stream, null it out.
- if (this.contentHeaderStream.available() <= 0) {
- this.contentHeaderStream = null;
- }
- // do not increment position -
- // the underlying ArchiveRecord stream allready did this
- // incrementPosition();
- } else {
- c = super.read();
- }
- return c;
- }
-
- public int read(byte [] b, int offset, int length) throws IOException {
- int read = -1;
- if (this.contentHeaderStream != null &&
- (this.contentHeaderStream.available() > 0)) {
- // If http header, return bytes from it before we go to underlying
- // stream.
- read = Math.min(length, this.contentHeaderStream.available());
- if (read == 0) {
- read = -1;
- } else {
- read = this.contentHeaderStream.read(b, offset, read);
- }
- // If done with the header stream, null it out.
- if (this.contentHeaderStream.available() <= 0) {
- this.contentHeaderStream = null;
- }
- // do not increment position -
- // the underlying ArchiveRecord stream allready did this
- //incrementPosition();
- } else {
- read = super.read(b, offset, length);
- }
- return read;
- }
-
- @Override
- public int available() {
- return ((ArchiveRecord)this.in).available();
- }
-
- @Override
- public void close() throws IOException {
- ((ArchiveRecord)this.in).close();
- }
-
- @Override
- public void dump() throws IOException {
- ((ArchiveRecord)this.in).dump();
- }
-
- @Override
- public void dump(OutputStream os) throws IOException {
- ((ArchiveRecord)this.in).dump(os);
- }
-
- @Override
- protected String getDigest4Cdx(ArchiveRecordHeader h) {
- return ((ArchiveRecord)this.in).getDigest4Cdx(h);
- }
-
- @Override
- public String getDigestStr() {
- return ((ArchiveRecord)this.in).getDigestStr();
- }
-
- @Override
- public ArchiveRecordHeader getHeader() {
- return ((ArchiveRecord)this.in).getHeader();
- }
-
- @Override
- protected String getIp4Cdx(ArchiveRecordHeader h) {
- return ((ArchiveRecord)this.in).getIp4Cdx(h);
- }
-
- @Override
- protected String getMimetype4Cdx(ArchiveRecordHeader h) {
- return ((ArchiveRecord)this.in).getMimetype4Cdx(h);
- }
-
- @Override
- public long getPosition() {
- return ((ArchiveRecord)this.in).getPosition();
- }
-
- @Override
- protected String getStatusCode4Cdx(ArchiveRecordHeader h) {
- return ((ArchiveRecord)this.in).getStatusCode4Cdx(h);
- }
-
- @Override
- public boolean hasContentHeaders() {
- return ((ArchiveRecord)this.in).hasContentHeaders();
- }
-
- @Override
- protected void incrementPosition() {
- ((ArchiveRecord)this.in).incrementPosition();
- }
-
- @Override
- protected void incrementPosition(long incr) {
- ((ArchiveRecord)this.in).incrementPosition(incr);
- }
-
- @Override
- protected boolean isEor() {
- return ((ArchiveRecord)this.in).isEor();
- }
-
- @Override
- public boolean isStrict() {
- return ((ArchiveRecord)this.in).isStrict();
- }
-
- @Override
- public boolean markSupported() {
- return ((ArchiveRecord)this.in).markSupported();
- }
-
- @Override
- protected String outputCdx(String strippedFileName) throws IOException {
- return ((ArchiveRecord)this.in).outputCdx(strippedFileName);
- }
-
- @Override
- protected void setEor(boolean eor) {
- ((ArchiveRecord)this.in).setEor(eor);
- }
-
- @Override
- protected void setHeader(ArchiveRecordHeader header) {
- ((ArchiveRecord)this.in).setHeader(header);
- }
-
- @Override
- public void setStrict(boolean strict) {
- ((ArchiveRecord)this.in).setStrict(strict);
- }
-
- @Override
- protected void skip() throws IOException {
- ((ArchiveRecord)this.in).skip();
- }
-
- @Override
- public long skip(long n) throws IOException {
- return ((ArchiveRecord)this.in).skip(n);
- }
-}
diff --git a/commons/src/main/java/org/archive/io/LoudObjectOutputStream.java b/commons/src/main/java/org/archive/io/LoudObjectOutputStream.java
deleted file mode 100644
index 959c2620..00000000
--- a/commons/src/main/java/org/archive/io/LoudObjectOutputStream.java
+++ /dev/null
@@ -1,63 +0,0 @@
-/*
- * This file is part of the Heritrix web crawler (crawler.archive.org).
- *
- * Licensed to the Internet Archive (IA) by one or more individual
- * contributors.
- *
- * The IA licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.archive.io;
-
-import java.io.IOException;
-import java.io.ObjectOutputStream;
-import java.io.OutputStream;
-import java.util.HashSet;
-import java.util.Set;
-import java.util.logging.Logger;
-
-/**
- * ObjectOutputStream that logs class name of each object that is written
- * to the stream. Useful for tracking down sources of NotSerializableException.
- *
- * @author pjack
- *
- */
-public class LoudObjectOutputStream extends ObjectOutputStream {
-
-
- final private static Logger LOGGER = Logger.getLogger(
- LoudObjectOutputStream.class.getName());
-
- // Only log each class name once
- private Set With the exception of {@link #position()} and {@link position(long)},
- * all of the methods in this class simply delegate to the underlying input
- * stream. The The RecordingOutputStream uses an in-memory buffer and
- * backing disk file to allow it to record streams of
- * arbitrary length limited only by available disk space.
- *
- * As long as the stream recorded is smaller than the
- * in-memory buffer, no disk access will occur.
- *
- * Recorded content can be recovered as a ReplayInputStream
- * (via getReplayInputStream() or, for only the content after
- * the content-begin-mark is set, getContentReplayInputStream() )
- * or as a ReplayCharSequence (via getReplayCharSequence()).
- *
- * This class is also used as a straight output stream
- * by {@link RecordingInputStream} to which it records all reads.
- * {@link RecordingInputStream} is exploiting the file backed buffer
- * facility of this class passing
- * This class does automatic detection of http message body begin (i.e. end
- * of http headers). Unfortunately httpcomponents did not want to add
- * functionality to help us with this, see
- * https://issues.apache.org/jira/browse/HTTPCORE-325
- *
- *
- * It works like this: while messageBodyBeginMark is not set, we remember
- * the last two bytes seen, and look at each byte we write. If the
- * lastTwoBytes+currentByte is "\n\r\n", or lastTwoBytes[1]+currentByte is
- * "\n\n" then we call markMessageBodyBegin() at the position after
- * currentByte.
- *
- *
- * An assumption here is that protocols other than http don't have headers,
- * and for those protocols the user of this class will call
- * markMessageBodyBegin() at position 0 before writing anything.
- */
- protected int[] lastTwoBytes = new int[] {-1, -1};
-
- /**
- * Stream to record.
- */
- private OutputStream out = null;
-
- // mark/reset support
- /** furthest position reached before any reset()s */
- private long maxPosition = 0;
- /** remembered position to reset() to */
- private long markPosition = 0;
-
- /**
- * Create a new RecordingOutputStream.
- *
- * @param bufferSize Buffer size to use.
- * @param backingFilename Name of backing file to use.
- */
- public RecordingOutputStream(int bufferSize, String backingFilename) {
- this.buffer = new byte[bufferSize];
- this.backingFilename = backingFilename;
- recording = true;
- }
-
- /**
- * Wrap the given stream, both recording and passing along any data written
- * to this RecordingOutputStream.
- *
- * @throws IOException If failed creation of backing file.
- */
- public void open() throws IOException {
- this.open(null);
- }
-
- /**
- * Wrap the given stream, both recording and passing along any data written
- * to this RecordingOutputStream.
- *
- * @param wrappedStream Stream to wrap. May be null for case where we
- * want to write to a file backed stream only.
- *
- * @throws IOException If failed creation of backing file.
- */
- public void open(OutputStream wrappedStream) throws IOException {
- if(isOpen()) {
- // error; should not be opening/wrapping in an unclosed
- // stream remains open
- throw new IOException("ROS already open for "
- +Thread.currentThread().getName());
- }
- this.out = wrappedStream;
- this.position = 0;
- this.markPosition = 0;
- this.maxPosition = 0;
- this.size = 0;
- this.messageBodyBeginMark = -1;
- // ensure recording turned on
- this.recording = true;
- // Always begins false; must use startDigest() to begin
- this.shouldDigest = false;
- if (this.diskStream != null) {
- closeDiskStream();
- }
- if (this.diskStream == null) {
- // TODO: Fix so we only make file when its actually needed.
- FileOutputStream fis = new FileOutputStream(this.backingFilename);
-
- this.diskStream = new RecyclingFastBufferedOutputStream(fis, bufStreamBuf);
- }
- startTime = System.currentTimeMillis();
- }
-
- public void write(int b) throws IOException {
- if(position TODO: More robust implementation. Tried to use the it.unimi.dsi.io
- * FastBufferdInputStream but relies on FileChannel ByteBuffers and if not
- * present -- as would be the case reading from a network stream, the main
- * application for this instance -- then it expects the underlying stream
- * implements RepositionableStream interface so chicken or egg problem.
- * @author stack
- */
-public class RepositionableInputStream extends BufferedInputStream implements
- RepositionableStream {
- private long position = 0;
- private long markPosition = -1;
-
- public RepositionableInputStream(InputStream in) {
- super(in);
- }
-
- public RepositionableInputStream(InputStream in, int size) {
- super(in, size);
- }
-
- public int read(byte[] b) throws IOException {
- int read = super.read(b);
- if (read != -1) {
- position += read;
- }
- return read;
- }
-
- public synchronized int read(byte[] b, int offset, int ct)
- throws IOException {
- // Mark the underlying stream so that we'll remember what we are about
- // to read unless a mark has been set in this RepositionableStream
- // (We have two levels of mark). In this latter case we want the
- // underlying stream to preserve its mark position so aligns with
- // this RS when eset is called.
- if (!isMarked()) {
- super.mark((ct > offset)? ct - offset: ct);
- }
- int read = super.read(b, offset, ct);
- if (read != -1) {
- position += read;
- }
- return read;
- }
-
- public int read() throws IOException {
- // Mark the underlying stream so that we'll remember what we are about
- // to read unless a mark has been set in this RepositionableStream
- // (We have two levels of mark). In this latter case we want the
- // underlying stream to preserve its mark position so aligns with
- // this RS when eset is called.
- if (!isMarked()) {
- super.mark(1);
- }
- int c = super.read();
- if (c != -1) {
- position++;
- }
- return c;
- }
-
- public void position(final long offset) {
- if (this.position == offset) {
- return;
- }
- int diff = (int)(offset - this.position);
- long lowerBound = this.position - this.pos;
- long upperBound = lowerBound + this.count;
- if (offset < lowerBound || offset >= upperBound) {
- throw new IllegalAccessError("Offset goes outside " +
- "current this.buf (TODO: Do buffer fills if positive)");
- }
- this.position = offset;
- this.pos += diff;
- // Clear any mark.
- this.markPosition = -1;
- }
-
- public void mark(int readlimit) {
- this.markPosition = this.position;
- super.mark(readlimit);
- }
-
- public void reset() throws IOException {
- super.reset();
- this.position = this.markPosition;
- this.markPosition = -1;
- }
-
- protected boolean isMarked() {
- return this.markPosition != -1;
- }
-
- public long position() {
- return this.position;
- }
-}
\ No newline at end of file
diff --git a/commons/src/main/java/org/archive/io/SafeSeekInputStream.java b/commons/src/main/java/org/archive/io/SafeSeekInputStream.java
deleted file mode 100644
index 0d8f83b1..00000000
--- a/commons/src/main/java/org/archive/io/SafeSeekInputStream.java
+++ /dev/null
@@ -1,124 +0,0 @@
-/*
- * This file is part of the Heritrix web crawler (crawler.archive.org).
- *
- * Licensed to the Internet Archive (IA) by one or more individual
- * contributors.
- *
- * The IA licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.archive.io;
-
-
-import java.io.IOException;
-
-
-/**
- * Enables multiple concurrent streams based on the same underlying stream.
- *
- * @author pjack
- */
-public class SafeSeekInputStream extends SeekInputStream {
-
-
- /**
- * The underlying stream.
- */
- private SeekInputStream input;
-
-
- /**
- * The expected position of the underlying stream.
- */
- private long expected;
-
-
- /**
- * Constructor. The given stream will be positioned to 0 so that an
- * accurate position can be tracked.
- *
- * @param input the underlying input stream
- * @throws IOException if an IO error occurs
- */
- public SafeSeekInputStream(SeekInputStream input) throws IOException {
- this.input = input;
- this.expected = input.position();
- }
-
-
- /**
- * Ensures that the underlying stream's position is what we expect to be.
- *
- * @throws IOException if an IO error occurs
- */
- private void ensure() throws IOException {
- if (expected != input.position()) {
- input.position(expected);
- }
- }
-
-
- @Override
- public int read() throws IOException {
- ensure();
- int c = input.read();
- if (c >= 0) {
- expected++;
- }
- return c;
- }
-
-
- @Override
- public int read(byte[] buf, int ofs, int len) throws IOException {
- ensure();
- int r = input.read(buf, ofs, len);
- if (r > 0) {
- expected += r;
- }
- return r;
- }
-
-
- @Override
- public int read(byte[] buf) throws IOException {
- ensure();
- int r = input.read(buf);
- if (r > 0) {
- expected += r;
- }
- return r;
- }
-
-
- @Override
- public long skip(long c) throws IOException {
- ensure();
- long r = input.skip(c);
- if (r > 0) {
- expected += r;
- }
- return r;
- }
-
-
- public void position(long p) throws IOException {
- input.position(p);
- expected = p;
- }
-
-
- public long position() throws IOException {
- return expected;
- }
-
-}
diff --git a/commons/src/main/java/org/archive/io/SeekInputStream.java b/commons/src/main/java/org/archive/io/SeekInputStream.java
deleted file mode 100644
index 177724ec..00000000
--- a/commons/src/main/java/org/archive/io/SeekInputStream.java
+++ /dev/null
@@ -1,81 +0,0 @@
-/*
- * This file is part of the Heritrix web crawler (crawler.archive.org).
- *
- * Licensed to the Internet Archive (IA) by one or more individual
- * contributors.
- *
- * The IA licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.archive.io;
-
-
-import it.unimi.dsi.fastutil.io.RepositionableStream;
-
-import java.io.IOException;
-import java.io.InputStream;
-
-
-/**
- * Base class for repositionable input streams.
- *
- * @author pjack
- */
-public abstract class SeekInputStream extends InputStream
-implements RepositionableStream {
-
-
- /**
- * The marked file position. A value less than zero
- * indicates that no mark has been set.
- */
- private long mark = -1;
-
-
- /**
- * Marks the current position of the stream. The limit parameter is
- * ignored; the mark will remain valid until reset is called or the
- * stream is closed.
- *
- * @param limit ignored
- */
- public void mark(int limit) {
- try {
- this.mark = position();
- } catch (IOException e) {
- mark = -1;
- }
- }
-
-
- /**
- * Resets this stream to its marked position.
- *
- * @throws IOException if there is no mark, or if an IO error occurs
- */
- public void reset() throws IOException {
- if (mark < 0) {
- throw new IOException("No mark.");
- }
- position(mark);
- }
-
-
- /**
- * Returns true, since SeekInputStreams support mark/reset by default.
- *
- * @return true
- */
- public boolean markSupported() {
- return true;
- }
-}
diff --git a/commons/src/main/java/org/archive/io/SeekReader.java b/commons/src/main/java/org/archive/io/SeekReader.java
deleted file mode 100644
index 4abf7847..00000000
--- a/commons/src/main/java/org/archive/io/SeekReader.java
+++ /dev/null
@@ -1,84 +0,0 @@
-/*
- * This file is part of the Heritrix web crawler (crawler.archive.org).
- *
- * Licensed to the Internet Archive (IA) by one or more individual
- * contributors.
- *
- * The IA licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.archive.io;
-
-
-import java.io.IOException;
-import java.io.Reader;
-
-import it.unimi.dsi.fastutil.io.RepositionableStream;
-
-
-/**
- * Base class for repositionable readers.
- *
- * @author pjack
- */
-public abstract class SeekReader extends Reader
-implements RepositionableStream {
-
-
- /**
- * The marked file position. A value less than zero
- * indicates that no mark has been set.
- */
- private long mark = -1;
-
-
- /**
- * Marks the current position of the stream. The limit parameter is
- * ignored; the mark will remain valid until reset is called or the
- * stream is closed.
- *
- * @param limit ignored
- */
- @Override
- public void mark(int limit) {
- try {
- this.mark = position();
- } catch (IOException e) {
- mark = -1;
- }
- }
-
-
- /**
- * Resets this stream to its marked position.
- *
- * @throws IOException if there is no mark, or if an IO error occurs
- */
- @Override
- public void reset() throws IOException {
- if (mark < 0) {
- throw new IOException("No mark.");
- }
- position(mark);
- }
-
-
- /**
- * Returns true, since SeekInputStreams support mark/reset by default.
- *
- * @return true
- */
- @Override
- public boolean markSupported() {
- return true;
- }
-}
diff --git a/commons/src/main/java/org/archive/io/SeekReaderCharSequence.java b/commons/src/main/java/org/archive/io/SeekReaderCharSequence.java
deleted file mode 100644
index a9b4880f..00000000
--- a/commons/src/main/java/org/archive/io/SeekReaderCharSequence.java
+++ /dev/null
@@ -1,56 +0,0 @@
-package org.archive.io;
-
-import java.io.IOException;
-
-public class SeekReaderCharSequence implements CharSequence {
-
-
- final private SeekReader reader;
- final private int size;
-
-
- public SeekReaderCharSequence(SeekReader reader, int size) {
- this.reader = reader;
- this.size = size;
- }
-
-
- public int length() {
- return size;
- }
-
-
- public char charAt(int index) {
- if ((index < 0) || (index >= length())) {
- throw new IndexOutOfBoundsException(Integer.toString(index));
- }
- try {
- reader.position(index);
- int r = reader.read();
- if (r < 0) {
- throw new IllegalStateException("EOF");
- }
- return (char)reader.read();
- } catch (IOException e) {
- throw new RuntimeException(e);
- }
- }
-
-
- public CharSequence subSequence(int start, int end) {
- return new CharSubSequence(this, start, end);
- }
-
- public String toString() {
- StringBuilder sb = new StringBuilder();
- try {
- reader.position(0);
- for (int ch = reader.read(); ch >= 0; ch = reader.read()) {
- sb.append((char)ch);
- }
- return sb.toString();
- } catch (IOException e) {
- throw new IllegalStateException(e);
- }
- }
-}
diff --git a/commons/src/main/java/org/archive/io/SinkHandlerLogThread.java b/commons/src/main/java/org/archive/io/SinkHandlerLogThread.java
deleted file mode 100644
index 0070785e..00000000
--- a/commons/src/main/java/org/archive/io/SinkHandlerLogThread.java
+++ /dev/null
@@ -1,34 +0,0 @@
-/*
- * This file is part of the Heritrix web crawler (crawler.archive.org).
- *
- * Licensed to the Internet Archive (IA) by one or more individual
- * contributors.
- *
- * The IA licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.archive.io;
-
-
-/**
- * Implemented by threads that provide extra information.
- *
- * TODO: rename class, rename getCurrentProcessorName()
- */
-public interface SinkHandlerLogThread {
-
- String getName();
- String getCurrentProcessorName();
- int getSerialNumber();
-
-}
diff --git a/commons/src/main/java/org/archive/io/UTF8Bytes.java b/commons/src/main/java/org/archive/io/UTF8Bytes.java
deleted file mode 100644
index c280b08d..00000000
--- a/commons/src/main/java/org/archive/io/UTF8Bytes.java
+++ /dev/null
@@ -1,37 +0,0 @@
-/*
- * This file is part of the Heritrix web crawler (crawler.archive.org).
- *
- * Licensed to the Internet Archive (IA) by one or more individual
- * contributors.
- *
- * The IA licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.archive.io;
-
-import java.io.UnsupportedEncodingException;
-
-/**
- * Marker Interface for instances that can be serialized as UTF8 bytes.
- * TODO: Do we need a UTF8Stream Marker Interface?
- * @author stack
- * @version $Date$ $Version$
- */
-public interface UTF8Bytes {
- public static final String UTF8 = "UTF-8";
-
- /**
- * @return Instance as UTF-8 bytes.
- * @throws UnsupportedEncodingException
- */
- public byte [] getUTF8Bytes() throws UnsupportedEncodingException;
-}
diff --git a/commons/src/main/java/org/archive/io/WriterPool.java b/commons/src/main/java/org/archive/io/WriterPool.java
deleted file mode 100644
index 11931516..00000000
--- a/commons/src/main/java/org/archive/io/WriterPool.java
+++ /dev/null
@@ -1,280 +0,0 @@
-/*
- * This file is part of the Heritrix web crawler (crawler.archive.org).
- *
- * Licensed to the Internet Archive (IA) by one or more individual
- * contributors.
- *
- * The IA licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.archive.io;
-
-import java.io.File;
-import java.io.IOException;
-import java.util.concurrent.ArrayBlockingQueue;
-import java.util.concurrent.BlockingQueue;
-import java.util.concurrent.TimeUnit;
-import java.util.concurrent.atomic.AtomicInteger;
-import java.util.logging.Level;
-import java.util.logging.Logger;
-
-/**
- * Pool of Writers.
- *
- * Abstract. Override and pass in the Constructor a factory that creates
- * {@link WriterPoolMember} implementations.
- *
- * @author stack
- */
-public abstract class WriterPool {
- private final Logger logger = Logger.getLogger(this.getClass().getName());
-
- /**
- * Used to generate unique filename sequences.
- */
- final protected AtomicInteger serialNo;
-
- /**
- * Default maximum active number of files in the pool.
- */
- public static final int DEFAULT_MAX_ACTIVE = 1;
-
- /** Assumed largest possible value of maxActive; pool will have this
- * maximum capacity, so dynamic changes beyond this number won't work. */
- protected static final int LARGEST_MAX_ACTIVE = 255;
-
- /**
- * Maximum time to wait on a free file before considering
- * making a new one (if not already at max)
- */
- public static final int DEFAULT_MAX_WAIT_FOR_IDLE = 500;
-
- /**
- * File settings.
- * Keep in data structure rather than as individual values.
- */
- protected final WriterPoolSettings settings;
-
- /** maximum number of writers to create at a time*/
- protected int maxActive;
- /** maximum ms to wait before considering creation of a writer */
- protected int maxWait;
- /** current count of active writers; only read/mutated in synchronized blocks */
- protected int currentActive = 0;
- /** round-robin queue of available writers */
- protected BlockingQueue Creates and opens a file if none already open. One use of this method
- * then is after construction, call this method to add the metadata, then
- * call {@link #getPosition()} to find offset of first record.
- *
- * TODO: perhaps this should be called checkForNewOpen? because it also
- * handles initial open, even when not rolling oversize
- *
- * @exception IOException
- */
- public void checkSize() throws IOException {
- if (this.out == null || isOversize()) {
- createFile();
- }
- }
-
- /** Check if underlying file has already reached its target size.
- * @return boolean true if file has reached target size and due to be closed
- */
- public boolean isOversize() {
- return settings.getMaxFileSizeBytes() != -1 && (this.getPosition() > settings.getMaxFileSizeBytes());
- }
-
- /**
- * Create a new file.
- * Rotates off the current Writer and creates a new in its place
- * to take subsequent writes. Usually called from {@link #checkSize()}.
- * @return Name of file created.
- * @throws IOException
- */
- protected String createFile() throws IOException {
- generateNewBasename();
- String name = currentBasename + '.' + this.extension +
- ((settings.getCompress())? DOT_COMPRESSED_FILE_EXTENSION: "") +
- OCCUPIED_SUFFIX;
- File dir = getNextDirectory(settings.calcOutputDirs());
- return createFile(new File(dir, name));
- }
-
- protected String createFile(final File file) throws IOException {
- close();
- this.f = file;
- FileOutputStream fos = new FileOutputStream(this.f);
- if(rebuf==null) {
- rebuf = new byte[settings.getWriteBufferSize()];
- }
- this.countOut = new MiserOutputStream(new RecyclingFastBufferedOutputStream(fos,rebuf),settings.getFrequentFlushes());
- this.out = this.countOut;
- logger.fine("Opened " + this.f.getAbsolutePath());
- return this.f.getName();
- }
-
- /**
- * @param dirs List of File objects that point at directories.
- * @return Find next directory to write an arc too. If more
- * than one, it tries to round-robin through each in turn.
- * @throws IOException
- */
- protected File getNextDirectory(List This class knows how to parse an ARC file. Pass it a file path
- * or an URL to an ARC. It can parse ARC Version 1 and 2.
- *
- * Iterator returns Profiling java.io vs. memory-mapped ByteBufferInputStream shows the
- * latter slightly slower -- but not by much. TODO: Test more. Just
- * change {@link #getInputStream(File, long)}.
- *
- * @author stack
- * @version $Date$ $Revision$
- */
-public abstract class ARCReader extends ArchiveReader
-implements ARCConstants, Closeable {
- private final Logger logger = Logger.getLogger(ARCReader.class.getName());
-
- /**
- * Set to true if we are aligned on first record of Archive file.
- * We used depend on offset. If offset was zero, then we were
- * aligned on first record. This is no longer necessarily the case when
- * Reader is created at an offset into an Archive file: The offset is zero
- * but its relative to where we started reading.
- */
- private boolean alignedOnFirstRecord = true;
-
- private boolean parseHttpHeaders = true;
-
- protected ARCReader() {
- super();
- }
-
- /**
- * Skip over any trailing new lines at end of the record so we're lined up
- * ready to read the next.
- * @param record
- * @throws IOException
- */
- protected void gotoEOR(ArchiveRecord record) throws IOException {
- if (getIn().available() <= 0) {
- return;
- }
-
- // Remove any trailing LINE_SEPARATOR
- int c = -1;
- while (getIn().available() > 0) {
- if (getIn().markSupported()) {
- getIn().mark(1);
- }
- c = getIn().read();
- if (c != -1) {
- if (c == LINE_SEPARATOR) {
- continue;
- }
- if (getIn().markSupported()) {
- // We've overread. We're probably in next record. There is
- // no way of telling for sure. It may be dross at end of
- // current record. Backup.
- getIn().reset();
- break;
- }
- ArchiveRecordHeader h = (getCurrentRecord() != null)?
- record.getHeader(): null;
- throw new IOException("Read " + (char)c +
- " when only " + LINE_SEPARATOR + " expected. " +
- getReaderIdentifier() + ((h != null)?
- h.getHeaderFields().toString(): ""));
- }
- }
- }
-
- /**
- * Create new arc record.
- *
- * Encapsulate housekeeping that has to do w/ creating a new record.
- *
- * Call this method at end of constructor to read in the
- * arcfile header. Will be problems reading subsequent arc records
- * if you don't since arcfile header has the list of metadata fields for
- * all records that follow.
- *
- * When parsing through ARCs writing out CDX info, we spend about
- * 38% of CPU in here -- about 30% of which is in getTokenizedHeaderLine
- * -- of which 16% is reading.
- *
- * @param is InputStream to use.
- * @param offset Absolute offset into arc file.
- * @return An arc record.
- * @throws IOException
- */
- protected ARCRecord createArchiveRecord(InputStream is, long offset)
- throws IOException {
- try {
- String version = super.getVersion();
- ARCRecord record = new ARCRecord(is, getReaderIdentifier(), offset,
- isDigest(), isStrict(), isParseHttpHeaders(),
- isAlignedOnFirstRecord(), version);
- if (version != null && super.getVersion() == null)
- super.setVersion(version);
- currentRecord(record);
- } catch (IOException e) {
- if (e instanceof RecoverableIOException) {
- // Don't mess with RecoverableIOExceptions. Let them out.
- throw e;
- }
- IOException newE = new IOException(e.getMessage() + " (Offset " +
- offset + ").");
- newE.setStackTrace(e.getStackTrace());
- throw newE;
- }
- return (ARCRecord)getCurrentRecord();
- }
-
- /**
- * Returns version of this ARC file. Usually read from first record of ARC.
- * If we're reading without having first read the first record -- e.g.
- * random access into middle of an ARC -- then version will not have been
- * set. For now, we return a default, version 1.1. Later, if more than
- * just one version of ARC, we could look at such as the meta line to see
- * what version of ARC this is.
- * @return Version of this ARC file.
- */
- public String getVersion() {
- return (super.getVersion() == null)? "1.1": super.getVersion();
- }
-
- protected boolean isAlignedOnFirstRecord() {
- return alignedOnFirstRecord;
- }
-
- protected void setAlignedOnFirstRecord(boolean alignedOnFirstRecord) {
- this.alignedOnFirstRecord = alignedOnFirstRecord;
- }
-
- /**
- * @return Returns the parseHttpHeaders.
- */
- public boolean isParseHttpHeaders() {
- return this.parseHttpHeaders;
- }
-
- /**
- * @param parse The parseHttpHeaders to set.
- */
- public void setParseHttpHeaders(boolean parse) {
- this.parseHttpHeaders = parse;
- }
-
- public String getFileExtension() {
- return ARC_FILE_EXTENSION;
- }
-
- public String getDotFileExtension() {
- return DOT_ARC_FILE_EXTENSION;
- }
-
- protected boolean output(final String format)
- throws IOException, java.text.ParseException {
- boolean result = super.output(format);
- if(!result && (format.equals(NOHEAD) || format.equals(HEADER))) {
- throw new IOException(format +
- " format only supported for single Records");
- }
- return result;
- }
-
- public boolean outputRecord(final String format) throws IOException {
- boolean result = super.outputRecord(format);
- if (result) {
- return result;
- }
- if (format.equals(NOHEAD)) {
- // No point digesting if dumping content.
- setDigest(false);
- ARCRecord r = (ARCRecord) get();
- r.skipHttpHeader();
- r.dump();
- result = true;
- } else if (format.equals(HEADER)) {
- // No point digesting if dumping content.
- setDigest(false);
- ARCRecord r = (ARCRecord) get();
- r.dumpHttpHeader();
- result = true;
- }
-
- return result;
- }
-
- public void dump(final boolean compress)
- throws IOException, java.text.ParseException {
- // No point digesting if we're doing a dump.
- setDigest(false);
- boolean firstRecord = true;
- ARCWriter writer = null;
- for (Iterator See in Outputs using a pseudo-CDX format as described here:
- * CDX
- * Legent and here
- * Example.
- * Legend used in below is: 'CDX b e a m s c V (or v if uncompressed) n g'.
- * Hash is hard-coded straight SHA-1 hash of content.
- *
- * @param args Command-line arguments.
- * @throws ParseException Failed parse of the command line.
- * @throws IOException
- * @throws java.text.ParseException
- */
- @SuppressWarnings("unchecked")
- public static void main(String [] args)
- throws ParseException, IOException, java.text.ParseException {
- Options options = getOptions();
- options.addOption(new Option("p","parse", false, "Parse headers."));
- PosixParser parser = new PosixParser();
- CommandLine cmdline = parser.parse(options, args, false);
- List Calling this method in the midst of reading the header
- * will make for strange results. Otherwise, safe to call
- * at any time though before reading any of the arc record
- * content is only time that it makes sense.
- *
- * After calling this method, you can call
- * {@link #getHttpHeaders()} to get the read http header.
- *
- * @throws IOException
- */
- public void skipHttpHeader() throws IOException {
- if (this.httpHeaderStream != null) {
- // Empty the httpHeaderStream
- for (int available = this.httpHeaderStream.available();
- this.httpHeaderStream != null &&
- (available = this.httpHeaderStream.available()) > 0;) {
- // We should be in this loop once only we should only do this
- // buffer allocation once.
- byte [] buffer = new byte[available];
- // The read nulls out httpHeaderStream when done with it so
- // need check for null in the loop control line.
- read(buffer, 0, available);
- }
- }
- }
-
- public void dumpHttpHeader() throws IOException {
- if (this.httpHeaderStream == null) {
- return;
- }
- // Dump the httpHeaderStream to STDOUT
- for (int available = this.httpHeaderStream.available();
- this.httpHeaderStream != null
- && (available = this.httpHeaderStream.available()) > 0;) {
- // We should be in this loop only once and should do this
- // buffer allocation once.
- byte[] buffer = new byte[available];
- // The read nulls out httpHeaderStream when done with it so
- // need check for null in the loop control line.
- int read = read(buffer, 0, available);
- System.out.write(buffer, 0, read);
- }
- }
-
- /**
- * Read http header if present. Technique borrowed from HttpClient HttpParse
- * class. set errors when found.
- *
- * @return ByteArrayInputStream with the http header in it or null if no
- * http header.
- * @throws IOException
- */
- private InputStream readHttpHeader() throws IOException {
-
- // this can be helpful when simply iterating over records,
- // looking for problems.
- Logger logger = Logger.getLogger(this.getClass().getName());
- ArchiveRecordHeader h = this.getHeader();
-
- // If judged a record that doesn't have an http header, return
- // immediately.
- String url = getHeader().getUrl();
- if(!url.startsWith("http") ||
- getHeader().getLength() <= MIN_HTTP_HEADER_LENGTH) {
- return null;
- }
-
- String statusLine;
- byte[] statusBytes;
- int eolCharCount = 0;
- int errOffset = 0;
-
- // Read status line, skipping any errant http headers found before it
- // This allows a larger number of 'corrupt' arcs -- where headers were accidentally
- // inserted before the status line to be readable
- while (true) {
- statusBytes = LaxHttpParser.readRawLine(getIn());
- eolCharCount = getEolCharsCount(statusBytes);
- if (eolCharCount <= 0) {
- throw new RecoverableIOException(
- "Failed to read http status where one was expected: "
- + ((statusBytes == null) ? "" : new String(statusBytes)));
- }
-
- statusLine = EncodingUtil.getString(statusBytes, 0,
- statusBytes.length - eolCharCount, ARCConstants.DEFAULT_ENCODING);
-
- // If a null or DELETED break immediately
- if ((statusLine == null) || statusLine.startsWith("DELETED")) {
- break;
- }
-
- // If it's actually the status line, break, otherwise continue skipping any
- // previous header values
- if (!statusLine.contains(":") && StatusLine.startsWithHTTP(statusLine)) {
- break;
- }
-
- // Add bytes read to error "offset" to add to position
- errOffset += statusBytes.length;
- }
-
- if (errOffset > 0) {
- this.incrementPosition(errOffset);
- }
-
- if ((statusLine == null) ||
- !StatusLine.startsWithHTTP(statusLine)) {
- if (statusLine.startsWith("DELETED")) {
- // Some old ARCs have deleted records like following:
- // http://vireo.gatech.edu:80/ebt-bin/nph-dweb/dynaweb/SGI_Developer/SGITCL_PG/@Generic__BookTocView/11108%3Btd%3D2 130.207.168.42 19991010131803 text/html 29202
- // DELETED_TIME=20000425001133_DELETER=Kurt_REASON=alexalist
- // (follows ~29K spaces)
- // For now, throw a RecoverableIOException so if iterating over
- // records, we keep going. TODO: Later make a legitimate
- // ARCRecord from the deleted record rather than throw
- // exception.
- throw new DeletedARCRecordIOException(statusLine);
- } else {
- this.errors.add(ArcRecordErrors.HTTP_STATUS_LINE_INVALID);
- }
- }
-
- try {
- this.httpStatus = new StatusLine(statusLine);
- } catch(IOException e) {
- logger.warning(e.getMessage() + " at offset: " + h.getOffset());
- this.errors.add(ArcRecordErrors.HTTP_STATUS_LINE_EXCEPTION);
- }
-
- // Save off all bytes read. Keep them as bytes rather than
- // convert to strings so we don't have to worry about encodings
- // though this should never be a problem doing http headers since
- // its all supposed to be ascii.
- ByteArrayOutputStream baos =
- new ByteArrayOutputStream(statusBytes.length + 4 * 1024);
- baos.write(statusBytes);
-
- // Now read rest of the header lines looking for the separation
- // between header and body.
- for (byte [] lineBytes = null; true;) {
- lineBytes = LaxHttpParser.readRawLine(getIn());
- eolCharCount = getEolCharsCount(lineBytes);
- if (eolCharCount <= 0) {
- if (getIn().available() == 0) {
- httpHeaderBytesRead += statusBytes.length;
- logger.warning("HTTP header truncated at offset: " + h.getOffset());
- this.errors.add(ArcRecordErrors.HTTP_HEADER_TRUNCATED);
- this.setEor(true);
- break;
- } else {
- throw new IOException("Failed reading http headers: " +
- ((lineBytes != null)? new String(lineBytes): null));
- }
- } else {
- httpHeaderBytesRead += lineBytes.length;
- }
- // Save the bytes read.
- baos.write(lineBytes);
- if ((lineBytes.length - eolCharCount) <= 0) {
- // We've finished reading the http header.
- break;
- }
- }
-
- byte [] headerBytes = baos.toByteArray();
- // Save off where body starts.
- this.getMetaData().setContentBegin(headerBytes.length);
- ByteArrayInputStream bais =
- new ByteArrayInputStream(headerBytes);
- if (!bais.markSupported()) {
- throw new IOException("ByteArrayInputStream does not support mark");
- }
- bais.mark(headerBytes.length);
- // Read the status line. Don't let it into the parseHeaders function.
- // It doesn't know what to do with it.
- bais.read(statusBytes, 0, statusBytes.length);
- this.httpHeaders = LaxHttpParser.parseHeaders(bais,
- ARCConstants.DEFAULT_ENCODING);
- this.getMetaData().setStatusCode(Integer.toString(getStatusCode()));
- bais.reset();
- return bais;
- }
-
- private static class DeletedARCRecordIOException
- extends RecoverableIOException {
- private static final long serialVersionUID = 1L;
-
- public DeletedARCRecordIOException(final String reason) {
- super(reason);
- }
- }
-
- /**
- * Return status code for this record.
- *
- * This method will return -1 until the http header has been read.
- * @return Status code.
- */
- public int getStatusCode() {
- return (this.httpStatus == null)? -1: this.httpStatus.getStatusCode();
- }
-
- /**
- * @param bytes Array of bytes to examine for an EOL.
- * @return Count of end-of-line characters or zero if none.
- */
- private int getEolCharsCount(byte [] bytes) {
- int count = 0;
- if (bytes != null && bytes.length >=1 &&
- bytes[bytes.length - 1] == '\n') {
- count++;
- if (bytes.length >=2 && bytes[bytes.length -2] == '\r') {
- count++;
- }
- }
- return count;
- }
-
- /**
- * @return Meta data for this record.
- */
- public ARCRecordMetaData getMetaData() {
- return (ARCRecordMetaData)getHeader();
- }
-
- /**
- * @return http headers (Only available after header has been read).
- */
- public Header [] getHttpHeaders() {
- return this.httpHeaders;
- }
-
- /**
- * @return ArcRecordErrors encountered when reading
- */
- public List Keys are lowercase.
- */
- protected Map
- * Returns the date in Heritrix 14 digit time format (UTC). See the
- * {@link org.archive.util.ArchiveUtils} class for converting to Java
- * dates.
- *
- * @return Header date in Heritrix 14 digit format.
- * @see org.archive.util.ArchiveUtils#parse14DigitDate(String)
- */
- public String getDate() {
- return (String) this.headerFields.get(DATE_FIELD_KEY);
- }
-
- /**
- * @return Return length of the record.
- */
- public long getLength() {
- return Long.parseLong((String)this.headerFields.
- get(LENGTH_FIELD_KEY));
- }
-
- /**
- * @return Return Content-Length of the contents of the record
- * Same as record length for arcs? TODO
- */
- public long getContentLength() {
- return getLength();
- }
-
- /**
- * @return Header url.
- */
- public String getUrl() {
- return (String)this.headerFields.get(URL_FIELD_KEY);
- }
-
- /**
- * @return IP.
- */
- public String getIp()
- {
- return (String)this.headerFields.get(IP_HEADER_FIELD_KEY);
- }
-
- /**
- * @return mimetype The mimetype that is in the ARC metaline -- NOT the http
- * content-type content.
- */
- public String getMimetype() {
- return (String)this.headerFields.get(MIMETYPE_FIELD_KEY);
- }
-
- /**
- * @return Arcfile version.
- */
- public String getVersion() {
- return (String)this.headerFields.get(VERSION_FIELD_KEY);
- }
-
- /**
- * @return Offset into arcfile at which this record begins.
- */
- public long getOffset() {
- return ((Long)this.headerFields.get(ABSOLUTE_OFFSET_KEY)).longValue();
- }
-
- /**
- * @param key Key to use looking up field value.
- * @return value for passed key of null if no such entry.
- */
- public Object getHeaderValue(String key) {
- return this.headerFields.get(key);
- }
-
- /**
- * @return Header field name keys.
- */
- public Set ARC files are described here:
- * Arc
- * File Format. This class does version 1 of the ARC file format. It also
- * writes version 1.1 which is version 1 with data stuffed into the body of the
- * first arc record in the file, the arc file meta record itself.
- *
- * An ARC file is three lines of meta data followed by an optional 'body' and
- * then a couple of '\n' and then: record, '\n', record, '\n', record, etc.
- * If we are writing compressed ARC files, then each of the ARC file records is
- * individually gzipped and concatenated together to make up a single ARC file.
- * In GZIP terms, each ARC record is a GZIP member of a total gzip'd
- * file.
- *
- * The GZIPping of the ARC file meta data is exceptional. It is GZIPped
- * w/ an extra GZIP header, a special Internet Archive (IA) extra header field
- * (e.g. FEXTRA is set in the GZIP header FLG field and an extra field is
- * appended to the GZIP header). The extra field has little in it but its
- * presence denotes this GZIP as an Internet Archive gzipped ARC. See RFC1952
- * to learn about the GZIP header structure.
- *
- * This class then does its GZIPping in the following fashion. Each GZIP
- * member is written w/ a new instance of GZIPOutputStream -- actually
- * ARCWriterGZIPOututStream so we can get access to the underlying stream.
- * The underlying stream stays open across GZIPoutputStream instantiations.
- * For the 'special' GZIPing of the ARC file meta data, we cheat by catching the
- * GZIPOutputStream output into a byte array, manipulating it adding the
- * IA GZIP header, before writing to the stream.
- *
- * I tried writing a resettable GZIPOutputStream and could make it work w/
- * the SUN JDK but the IBM JDK threw NPE inside in the deflate.reset -- its zlib
- * native call doesn't seem to like the notion of resetting -- so I gave up on
- * it.
- *
- * Because of such as the above and troubles with GZIPInputStream, we should
- * write our own GZIP*Streams, ones that resettable and consious of gzip
- * members.
- *
- * This class will write until we hit >= maxSize. The check is done at
- * record boundary. Records do not span ARC files. We will then close current
- * file and open another and then continue writing.
- *
- * TESTING: Here is how to test that produced ARC files are good
- * using the
- * alexa
- * ARC c-tools:
- * You can also do While being written, ARCs have a '.open' suffix appended.
- *
- * @author stack
- */
-public class ARCWriter extends WriterPoolMember implements ARCConstants, Closeable {
- private static final Logger logger =
- Logger.getLogger(ARCWriter.class.getName());
-
- /**
- * Metadata line pattern.
- */
- private static final Pattern METADATA_LINE_PATTERN =
- Pattern.compile("^\\S+ \\S+ \\S+ \\S+ \\S+(" + LINE_SEPARATOR + "?)$");
-
-
- /**
- * Constructor.
- * Takes a stream. Use with caution. There is no upperbound check on size.
- * Will just keep writing.
- *
- * @param serialNo used to generate unique file name sequences
- * @param out Where to write.
- * @param arc File the Generate ARC file meta data. Currently we only do version 1 of the
- * ARC file formats or version 1.1 when metadata has been supplied (We
- * write it into the body of the first record in the arc file).
- *
- * Version 1 metadata looks roughly like this:
- *
- * If compress is set, then we generate a header that has been gzipped
- * in the Internet Archive manner. Such a gzipping enables the FEXTRA
- * flag in the FLG field of the gzip header. It then appends an extra
- * header field: '8', '0', 'L', 'X', '0', '0', '0', '0'. The first two
- * bytes are the length of the field and the last 6 bytes the Internet
- * Archive header. To learn about GZIP format, see RFC1952. To learn
- * about the Internet Archive extra header field, read the source for
- * av_ziparc which can be found at
- * We do things in this roundabout manner because the java
- * GZIPOutputStream does not give access to GZIP header fields.
- *
- * @param date Date to put into the ARC metadata; if 17-digit will be
- * truncated to traditional 14-digits
- *
- * @return Byte array filled w/ the arc header.
- * @throws IOException
- */
- private byte [] generateARCFileMetaData(String date)
- throws IOException {
- if(date!=null && date.length()>14) {
- date = date.substring(0,14);
- }
- int metadataBodyLength = getMetadataLength();
- // If metadata body, then the minor part of the version is '1' rather
- // than '0'.
- String metadataHeaderLinesTwoAndThree =
- getMetadataHeaderLinesTwoAndThree("1 " +
- ((metadataBodyLength > 0)? "1": "0"));
- int recordLength = metadataBodyLength +
- metadataHeaderLinesTwoAndThree.getBytes(DEFAULT_ENCODING).length;
- String metadataHeaderStr = ARC_MAGIC_NUMBER + getBaseFilename() +
- " 0.0.0.0 " + date + " text/plain " + recordLength +
- metadataHeaderLinesTwoAndThree;
- ByteArrayOutputStream metabaos =
- new ByteArrayOutputStream(recordLength);
- // Write the metadata header.
- metabaos.write(metadataHeaderStr.getBytes(DEFAULT_ENCODING));
- // Write the metadata body, if anything to write.
- if (metadataBodyLength > 0) {
- writeMetaData(metabaos);
- }
-
- // Write out a LINE_SEPARATORs to end this record.
- metabaos.write(LINE_SEPARATOR);
-
- // Now get bytes of all just written and compress if flag set.
- byte [] bytes = metabaos.toByteArray();
-
- if(isCompressed()) {
- // GZIP the header but catch the gzipping into a byte array so we
- // can add the special IA GZIP header to the product. After
- // manipulations, write to the output stream (The JAVA GZIP
- // implementation does not give access to GZIP header. It
- // produces a 'default' header only). We can get away w/ these
- // maniupulations because the GZIP 'default' header doesn't
- // do the 'optional' CRC'ing of the header.
- byte [] gzippedMetaData = ArchiveUtils.gzip(bytes);
- if (gzippedMetaData[3] != 0) {
- throw new IOException("The GZIP FLG header is unexpectedly " +
- " non-zero. Need to add smarter code that can deal " +
- " when already extant extra GZIP header fields.");
- }
- // Set the GZIP FLG header to '4' which says that the GZIP header
- // has extra fields. Then insert the alex {'L', 'X', '0', '0', '0,
- // '0'} 'extra' field. The IA GZIP header will also set byte
- // 9 (zero-based), the OS byte, to 3 (Unix). We'll do the same.
- gzippedMetaData[3] = 4;
- gzippedMetaData[9] = 3;
- byte [] assemblyBuffer = new byte[gzippedMetaData.length +
- ARC_GZIP_EXTRA_FIELD.length];
- // '10' in the below is a pointer past the following bytes of the
- // GZIP header: ID1 ID2 CM FLG + MTIME(4-bytes) XFL OS. See
- // RFC1952 for explaination of the abbreviations just used.
- System.arraycopy(gzippedMetaData, 0, assemblyBuffer, 0, 10);
- System.arraycopy(ARC_GZIP_EXTRA_FIELD, 0, assemblyBuffer, 10,
- ARC_GZIP_EXTRA_FIELD.length);
- System.arraycopy(gzippedMetaData, 10, assemblyBuffer,
- 10 + ARC_GZIP_EXTRA_FIELD.length, gzippedMetaData.length - 10);
- bytes = assemblyBuffer;
- }
- return bytes;
- }
-
- public String getMetadataHeaderLinesTwoAndThree(String version) {
- StringBuffer buffer = new StringBuffer();
- buffer.append(LINE_SEPARATOR);
- buffer.append(version);
- buffer.append(" InternetArchive");
- buffer.append(LINE_SEPARATOR);
- buffer.append("URL IP-address Archive-date Content-type Archive-length");
- buffer.append(LINE_SEPARATOR);
- return buffer.toString();
- }
-
- /**
- * Write all metadata to passed Outputs using a pseudo-CDX format as described here:
- * CDX
- * Legent and here
- * Example.
- * Legend used in below is: 'CDX b e a m s c V (or v if uncompressed) n g'.
- * Hash is hard-coded straight SHA-1 hash of content.
- *
- * @param args Command-line arguments.
- * @throws ParseException Failed parse of the command line.
- * @throws IOException
- * @throws java.text.ParseException
- */
- public static void main(String [] args)
- throws ParseException, IOException, java.text.ParseException {
- Options options = getOptions();
- PosixParser parser = new PosixParser();
- CommandLine cmdline = parser.parse(options, args, false);
- @SuppressWarnings("unchecked")
- List Assumption is that the caller is managing access to this
- * WARCWriter ensuring only one thread accessing this WARC instance
- * at any one time.
- *
- * While being written, WARCs have a '.open' suffix appended.
- *
- * @contributor stack
- * @version $Revision: 4604 $ $Date: 2006-09-05 22:38:18 -0700 (Tue, 05 Sep 2006) $
- */
-public class WARCWriter extends WriterPoolMember
-implements WARCConstants {
- public static final String TOTALS = "totals";
- public static final String SIZE_ON_DISK = "sizeOnDisk";
- public static final String TOTAL_BYTES = "totalBytes";
- public static final String CONTENT_BYTES = "contentBytes";
- public static final String NUM_RECORDS = "numRecords";
-
- private static final Logger logger =
- Logger.getLogger(WARCWriter.class.getName());
-
- /**
- * NEWLINE as bytes.
- */
- public static byte [] CRLF_BYTES;
- static {
- try {
- CRLF_BYTES = CRLF.getBytes(DEFAULT_ENCODING);
- } catch(Exception e) {
- e.printStackTrace();
- }
- };
-
- /**
- * Temporarily accumulates stats managed externally by
- * {@link WARCWriterProcessor}. WARCWriterProcessor will call
- * {@link #resetTmpStats()}, write some records, then add
- * {@link #getTmpStats()} into its long-term running totals.
- */
- private Map Initial implementations of You need to define the system property
- * TODO: Move to an AddressParsingUtil class.
- * @param host Host name to examine.
- * @return InetAddress IF the passed name was an IP address, else null.
- */
- public static InetAddress getIPHostAddress(String host) {
- InetAddress result = null;
- Matcher matcher = IPV4_QUADS.matcher(host);
- if (matcher == null || !matcher.matches()) {
- return result;
- }
- try {
- // Doing an Inet.getByAddress() avoids a lookup.
- result = InetAddress.getByAddress(host,
- new byte[] {
- (byte)(new Integer(matcher.group(1)).intValue()),
- (byte)(new Integer(matcher.group(2)).intValue()),
- (byte)(new Integer(matcher.group(3)).intValue()),
- (byte)(new Integer(matcher.group(4)).intValue())});
- } catch (NumberFormatException e) {
- logger.warning(e.getMessage());
- } catch (UnknownHostException e) {
- logger.warning(e.getMessage());
- }
- return result;
- }
-
- /**
- * @return All known local names for this host or null if none found.
- */
- public static List Truncate at delimiters [;, ].
- * Truncate multi-part content type header at ';'.
- * Apache httpclient collapses values of multiple instances of the
- * header into one comma-separated value,therefore truncated at ','.
- * Current ia_tools that work with arc files expect 5-column
- * space-separated meta-lines, therefore truncate at ' '.
- *
- * @param contentType Raw content-type.
- *
- * @return Computed content-type made from passed content-type after
- * running it through a set of rules.
- */
- public static String truncate(String contentType) {
- if (contentType == null) {
- contentType = NO_TYPE_MIMETYPE;
- } else {
- Matcher matcher = TRUNCATION_REGEX.matcher(contentType);
- if (matcher.matches()) {
- contentType = matcher.group(1);
- } else {
- contentType = NO_TYPE_MIMETYPE;
- }
- }
-
- return contentType;
- }
-}
diff --git a/commons/src/main/java/org/archive/util/ProcessUtils.java b/commons/src/main/java/org/archive/util/ProcessUtils.java
deleted file mode 100644
index af792981..00000000
--- a/commons/src/main/java/org/archive/util/ProcessUtils.java
+++ /dev/null
@@ -1,151 +0,0 @@
-/*
- * This file is part of the Heritrix web crawler (crawler.archive.org).
- *
- * Licensed to the Internet Archive (IA) by one or more individual
- * contributors.
- *
- * The IA licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.archive.util;
-
-import java.io.BufferedReader;
-import java.io.IOException;
-import java.io.InputStream;
-import java.io.InputStreamReader;
-import java.util.Arrays;
-import java.util.logging.Level;
-import java.util.logging.Logger;
-
-/**
- * Class to run an external process.
- * @author stack
- * @version $Date$ $Revision$
- */
-public class ProcessUtils {
- private static final Logger LOGGER =
- Logger.getLogger(ProcessUtils.class.getName());
-
- protected ProcessUtils() {
- super();
- }
-
- /**
- * Thread to gobble up an output stream.
- * See http://www.javaworld.com/javaworld/jw-12-2000/jw-1229-traps.html
- */
- protected class StreamGobbler extends Thread {
- private final InputStream is;
- private final StringBuffer sink = new StringBuffer();
-
- protected StreamGobbler(InputStream is, String name) {
- this.is = is;
- setName(name);
- }
-
- public void run() {
- try {
- BufferedReader br =
- new BufferedReader(new InputStreamReader(this.is));
- for (String line = null; (line = br.readLine()) != null;) {
- this.sink.append(line);
- }
- } catch (IOException ioe) {
- ioe.printStackTrace();
- }
- }
-
- public String getSink() {
- return this.sink.toString();
- }
- }
-
- /**
- * Data structure to hold result of a process exec.
- * @author stack
- * @version $Date$ $Revision$
- */
- public class ProcessResult {
- private final String [] args;
- private final int result;
- private final String stdout;
- private final String stderr;
-
- protected ProcessResult(String [] args, int result, String stdout,
- String stderr) {
- this.args = args;
- this.result = result;
- this.stderr = stderr;
- this.stdout = stdout;
- }
-
- public int getResult() {
- return this.result;
- }
-
- public String getStdout() {
- return this.stdout;
- }
-
- public String getStderr() {
- return this.stderr;
- }
-
- public String toString() {
- StringBuffer sb = new StringBuffer();
- for (int i = 0; i < this.args.length; i++) {
- sb.append(this.args[i]);
- sb.append(", ");
- }
- return sb.toString() + " exit code: " + this.result +
- ((this.stderr != null && this.stderr.length() > 0)?
- "\nSTDERR: " + this.stderr: "") +
- ((this.stdout != null && this.stdout.length() > 0)?
- "\nSTDOUT: " + this.stdout: "");
- }
- }
-
- /**
- * Runs process.
- * @param args List of process args.
- * @return A ProcessResult data structure.
- * @throws IOException If interrupted, we throw an IOException. If non-zero
- * exit code, we throw an IOException (This may need to change).
- */
- public static ProcessUtils.ProcessResult exec(String [] args)
- throws IOException {
- Process p = Runtime.getRuntime().exec(args);
- ProcessUtils pu = new ProcessUtils();
- // Gobble up any output.
- StreamGobbler err = pu.new StreamGobbler(p.getErrorStream(), "stderr");
- err.setDaemon(true);
- err.start();
- StreamGobbler out = pu.new StreamGobbler(p.getInputStream(), "stdout");
- out.setDaemon(true);
- out.start();
- int exitVal;
- try {
- exitVal = p.waitFor();
- } catch (InterruptedException e) {
- throw new IOException("Wait on process " + Arrays.toString(args) + " interrupted: "
- + e.getMessage());
- }
- ProcessUtils.ProcessResult result =
- pu.new ProcessResult(args, exitVal, out.getSink(), err.getSink());
- if (exitVal != 0) {
- throw new IOException(result.toString());
- } else if (LOGGER.isLoggable(Level.INFO)) {
- LOGGER.info(result.toString());
- }
- return result;
- }
-}
diff --git a/commons/src/main/java/org/archive/util/ProgressStatisticsReporter.java b/commons/src/main/java/org/archive/util/ProgressStatisticsReporter.java
deleted file mode 100644
index dc1e51f7..00000000
--- a/commons/src/main/java/org/archive/util/ProgressStatisticsReporter.java
+++ /dev/null
@@ -1,36 +0,0 @@
-/*
- * This file is part of the Heritrix web crawler (crawler.archive.org).
- *
- * Licensed to the Internet Archive (IA) by one or more individual
- * contributors.
- *
- * The IA licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.archive.util;
-
-import java.io.IOException;
-import java.io.PrintWriter;
-
-public interface ProgressStatisticsReporter {
- /**
- * @param writer Where to write statistics.
- * @throws IOException
- */
- public void progressStatisticsLine(PrintWriter writer) throws IOException;
-
- /**
- * @param writer Where to write statistics legend.
- * @throws IOException
- */
- public void progressStatisticsLegend(PrintWriter writer) throws IOException;
-}
diff --git a/commons/src/main/java/org/archive/util/PropertyUtils.java b/commons/src/main/java/org/archive/util/PropertyUtils.java
deleted file mode 100644
index 083615f6..00000000
--- a/commons/src/main/java/org/archive/util/PropertyUtils.java
+++ /dev/null
@@ -1,114 +0,0 @@
-/*
- * This file is part of the Heritrix web crawler (crawler.archive.org).
- *
- * Licensed to the Internet Archive (IA) by one or more individual
- * contributors.
- *
- * The IA licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.archive.util;
-
-import java.util.Properties;
-import java.util.regex.Matcher;
-
-import org.apache.commons.lang.StringUtils;
-
-/**
- * Utilities for dealing with Java Properties (incl. System Properties)
- *
- * @contributor stack
- * @contributor gojomo
- * @version $Date$ $Revision$
- */
-public class PropertyUtils {
- /***
- * @param key Property key.
- * @return Named property or null if the property is null or empty.
- */
- public static String getPropertyOrNull(final String key) {
- String value = System.getProperty(key);
- return (value == null || value.length() <= 0)? null: value;
- }
-
- /***
- * @param key Property key.
- * @return Boolean value or false if null or unreadable.
- */
- public static boolean getBooleanProperty(final String key) {
- return (getPropertyOrNull(key) == null)?
- false: Boolean.valueOf(getPropertyOrNull(key)).booleanValue();
- }
-
- /**
- * @param key Key to use looking up system property.
- * @param fallback If no value found for passed The ANVL Internet-Draft of 14 February, 2005 is inspecific as to the
-definition of 'blank line' and 'newline'. This parser implementation
-assumes Says "An element consists of a label, a colon, and an optional value".
-Should that be: "An element consists of a label and an optional value, or a
-comment." Specification is unclear regards CR or NL in label or
-comment (This implementation disallows CR or NL in labels but lets
-them pass in comments). A grammar would help. Here is RFC822:
-
@@ -408,26 +408,46 @@ implements Lifecycle, Checkpointable, BeanNameAware, DisposableBean {
* grouping of urls that is feasible to forget in bulk. See
* {@link #createKey(CharSequence)}
*
- * @param schemeHost
+ *
+ * WARNING: Value collisions in this 24-bit schemeAuthority part are going
+ * to be fairly common, by 'birthday problem' over 50% likely to show up
+ * with as few as 2^12 unique schemeAuthority strings. So the forgetting may
+ * forget other hosts.
+ *
+ * @param url
+ * whose scheme+host+port should be forgotten (remainder of url
+ * is ignored)
*/
- public void forgetSchemeHost(String schemeHost) {
- long schemeHostKeyPart = calcSchemeHostKeyPart(schemeHost);
+ public void forgetAllSchemeAuthorityMatching(String url) {
+ long schemeAuthorityKeyLong = calcSchemeAuthorityKeyBytes(url);
DatabaseEntry key = new DatabaseEntry();
DatabaseEntry value = new DatabaseEntry();
+
+ LongBinding.longToEntry(schemeAuthorityKeyLong, key);
+ byte[] schemeAuthorityKeyBytes = key.getData();
+
Cursor cursor = alreadySeen.openCursor(null, null);
long forgottenCount = 0l;
- while (cursor.getNext(key, value, null) == OperationStatus.SUCCESS) {
- long alreadySeenKey = LongBinding.entryToLong(key);
- // System.out.printf("schemeHostKeyPart=%017x alreadySeenKey=%017x\n", schemeHostKeyPart, alreadySeenKey);
- if ((alreadySeenKey & 0xffffff0000000000l) == schemeHostKeyPart) {
+
+ for (OperationStatus status = cursor.getSearchKeyRange(key, value, null);
+ status == OperationStatus.SUCCESS;
+ status = cursor.getNext(key, value, null)) {
+
+ byte[] keyData = key.getData();
+ if (keyData[0] == schemeAuthorityKeyBytes[0]
+ && keyData[1] == schemeAuthorityKeyBytes[1]
+ && keyData[2] == schemeAuthorityKeyBytes[2]) {
cursor.delete();
- count.decrementAndGet();
forgottenCount++;
+ } else {
+ break;
}
}
+
cursor.close();
- logger.info("forgot " + forgottenCount + " urls from scheme+host+port " + schemeHost + " (" + count.get() + " urls left)");
+ long newCount = count.addAndGet(-forgottenCount);
+ logger.info("forgot " + forgottenCount + " urls from scheme+authority of url " + url + " (leaving " + newCount + " urls from other scheme+authorities)");
}
} //EOC
\ No newline at end of file
diff --git a/engine/src/main/java/org/archive/crawler/util/CrawledBytesHistotable.java b/engine/src/main/java/org/archive/crawler/util/CrawledBytesHistotable.java
index bff074e6..cf5fe10e 100644
--- a/engine/src/main/java/org/archive/crawler/util/CrawledBytesHistotable.java
+++ b/engine/src/main/java/org/archive/crawler/util/CrawledBytesHistotable.java
@@ -22,7 +22,6 @@ package org.archive.crawler.util;
import org.apache.commons.httpclient.HttpStatus;
import org.archive.modules.CoreAttributeConstants;
import org.archive.modules.CrawlURI;
-import org.archive.modules.deciderules.recrawl.IdenticalDigestDecideRule;
import org.archive.util.ArchiveUtils;
import org.archive.util.Histotable;
@@ -45,10 +44,7 @@ implements CoreAttributeConstants {
if(curi.getFetchStatus()==HttpStatus.SC_NOT_MODIFIED) {
tally(NOTMODIFIED, curi.getContentSize());
tally(NOTMODIFIEDCOUNT,1);
- } else if (IdenticalDigestDecideRule.hasIdenticalDigest(curi)) {
- tally(DUPLICATE,curi.getContentSize());
- tally(DUPLICATECOUNT,1);
- } else if (curi.getAnnotations().contains("duplicate:uriAgnosticDigest")) {
+ } else if (curi.getAnnotations().contains("duplicate:digest")) {
tally(DUPLICATE,curi.getContentSize());
tally(DUPLICATECOUNT,1);
} else {
diff --git a/engine/src/main/resources/org/archive/crawler/restlet/Beans.ftl b/engine/src/main/resources/org/archive/crawler/restlet/Beans.ftl
index cd74c3b8..8c0b39f4 100644
--- a/engine/src/main/resources/org/archive/crawler/restlet/Beans.ftl
+++ b/engine/src/main/resources/org/archive/crawler/restlet/Beans.ftl
@@ -1,44 +1,112 @@
strict
- * tries to move to next record if we get an
- * {@link IOException}.
- * @return Next object.
- * @exception RuntimeException Throws a runtime exception,
- * usually a wrapping of an IOException, if trouble getting
- * a record (Throws exception rather than return null).
- */
- public ArchiveRecord next() {
- long offset = -1;
- try {
- offset = positionForRecord(getIn());
- return exceptionNext();
- } catch (IOException e) {
- if (!isStrict()) {
- // Retry though an IOE. Maybe we will succeed reading
- // subsequent record.
- try {
- if (hasNext()) {
- getLogger().warning("Bad Record. Trying skip " +
- "(Record start " + offset + "): " +
- e.getMessage());
- return exceptionNext();
- }
- // Else we are at last record. Iterator#next is
- // expecting value. We do not have one. Throw exception.
- throw new RuntimeException("Retried but no next " +
- "record (Record start " + offset + ")", e);
- } catch (IOException e1) {
- throw new RuntimeException("After retry (Offset " +
- offset + ")", e1);
- }
- }
- throw new RuntimeException("(Record start " + offset + ")", e);
- }
- }
-
- /**
- * A next that throws exceptions and has handling of
- * recoverable exceptions moving us to next record. Can call
- * hasNext which itself may throw exceptions.
- * @return Next record.
- * @throws IOException
- * @throws RuntimeException Thrown when we've reached maximum
- * retries.
- */
- protected ArchiveRecord exceptionNext()
- throws IOException, RuntimeException {
- ArchiveRecord result = null;
- IOException ioe = null;
- for (int i = MAX_ALLOWED_RECOVERABLES; i > 0 &&
- result == null; i--) {
- ioe = null;
- try {
- result = innerNext();
- } catch (RecoverableIOException e) {
- ioe = e;
- getLogger().warning(e.getMessage());
- if (hasNext()) {
- continue;
- }
- // No records left. Throw exception rather than
- // return null. The caller is expecting to get
- // back a record since they've just called
- // hasNext.
- break;
- }
- }
- if (ioe != null) {
- // Then we did MAX_ALLOWED_RECOVERABLES retries. Throw
- // the recoverable ioe wrapped in a RuntimeException so
- // it goes out pass checks for IOE.
- throw new RuntimeException("Retried " +
- MAX_ALLOWED_RECOVERABLES + " times in a row", ioe);
- }
- return result;
- }
-
- protected ArchiveRecord innerNext() throws IOException {
- return get(positionForRecord(getIn()));
- }
-
- public void remove() {
- throw new UnsupportedOperationException();
- }
- }
-
- protected static long positionForRecord(InputStream in) {
- return (in instanceof GZIPMembersInputStream)
- ? ((GZIPMembersInputStream)in).getCurrentMemberStart()
- : ((CountingInputStream)in).getCount();
- }
-
- protected static String stripExtension(final String name,
- final String ext) {
- return (!name.endsWith(ext))? name:
- name.substring(0, name.length() - ext.length());
- }
-
- /**
- * @return short name of Archive file.
- */
- public String getFileName() {
- return (new File(getReaderIdentifier())).getName();
- }
-
- /**
- * @return short name of Archive file.
- */
- public String getStrippedFileName() {
- return getStrippedFileName(getFileName(),
- getDotFileExtension());
- }
-
- /**
- * @param name Name of ARCFile.
- * @param dotFileExtension '.arc' or '.warc', etc.
- * @return short name of Archive file.
- */
- public static String getStrippedFileName(String name,
- final String dotFileExtension) {
- name = stripExtension(name,
- ArchiveFileConstants.DOT_COMPRESSED_FILE_EXTENSION);
- return stripExtension(name, dotFileExtension);
- }
-
- /**
- * @param value Value to test.
- * @return True if value is 'true', else false.
- */
- protected static boolean getTrueOrFalse(final String value) {
- if (value == null || value.length() <= 0) {
- return false;
- }
- return Boolean.TRUE.toString().equals(value.toLowerCase());
- }
-
- /**
- * @param format Format to use outputting.
- * @throws IOException
- * @throws java.text.ParseException
- * @return True if handled.
- */
- protected boolean output(final String format)
- throws IOException, java.text.ParseException {
- boolean result = true;
- // long start = System.currentTimeMillis();
-
- // Write output as pseudo-CDX file. See
- // http://www.archive.org/web/researcher/cdx_legend.php
- // and http://www.archive.org/web/researcher/example_cdx.php.
- // Hash is hard-coded straight SHA-1 hash of content.
- if (format.equals(DUMP)) {
- // No point digesting dumping.
- setDigest(false);
- dump(false);
- } else if (format.equals(GZIP_DUMP)) {
- // No point digesting dumping.
- setDigest(false);
- dump(true);
- } else if (format.equals(CDX)) {
- cdxOutput(false);
- } else if (format.equals(CDX_FILE)) {
- cdxOutput(true);
- } else {
- result = false;
- }
- return result;
- }
-
- protected void cdxOutput(boolean toFile)
- throws IOException {
- BufferedWriter cdxWriter = null;
- if (toFile) {
- String cdxFilename = stripExtension(getReaderIdentifier(),
- DOT_COMPRESSED_FILE_EXTENSION);
- cdxFilename = stripExtension(cdxFilename, getDotFileExtension());
- cdxFilename += ('.' + CDX);
- cdxWriter = new BufferedWriter(new FileWriter(cdxFilename));
- }
-
- String header = "CDX b e a m s c " + ((isCompressed()) ? "V" : "v")
- + " n g";
- if (toFile) {
- cdxWriter.write(header);
- cdxWriter.newLine();
- } else {
- System.out.println(header);
- }
-
- String strippedFileName = getStrippedFileName();
- try {
- for (Iteratoroffset.
- * This version of get will not bring the file local but will try to
- * stream across the net making an HTTP 1.1 Range request on remote
- * http server (RFC1435 Section 14.35).
- * @param u HTTP URL for an Archive file.
- * @param offset Offset into file at which to start fetching.
- * @return An ArchiveReader aligned at offset.
- * @throws IOException
- */
- public static ArchiveReader get(final URL u, final long offset)
- throws IOException {
- return ArchiveReaderFactory.factory.getArchiveReader(u, offset);
- }
-
- protected ArchiveReader getArchiveReader(final URL f, final long offset)
- throws IOException {
- // Get URL connection.
- URLConnection connection = f.openConnection();
- if (connection instanceof HttpURLConnection) {
- addUserAgent((HttpURLConnection)connection);
- }
- if (offset != 0) {
- // Use a Range request (Assumes HTTP 1.1 on other end). If
- // length >= 0, add open-ended range header to the request. Else,
- // because end-byte is inclusive, subtract 1.
- connection.addRequestProperty("Range", "bytes=" + offset + "-");
- // TODO: should actually verify that server respected 'Range' request
- // (spec allows them to ignore; 206 response or Content-Range header
- // should be present if Range satisfied; multipart/byteranges could be
- // a problem).
- }
-
- return getArchiveReader(f.toString(), connection.getInputStream(), (offset == 0));
- }
-
- /**
- * Get an ARCReader.
- * Pulls the ARC local into whereever the System Property
- * java.io.tmpdir points. It then hands back an ARCReader that
- * points at this local copy. A close on this ARCReader instance will
- * remove the local copy.
- * @param u An URL that points at an ARC.
- * @return An ARCReader.
- * @throws IOException
- */
- public static ArchiveReader get(final URL u)
- throws IOException {
- return ArchiveReaderFactory.factory.getArchiveReader(u);
- }
-
- protected ArchiveReader getArchiveReader(final URL u)
- throws IOException {
- // If url represents a local file then return file it points to.
- if (u.getPath() != null) {
- // TODO: Add scheme check and host check.
- File f = new File(u.getPath());
- if (f.exists()) {
- return get(f, 0);
- }
- }
-
- String scheme = u.getProtocol();
- if (scheme.startsWith("http") || scheme.equals("s3")) {
- // Try streaming if http or s3 URLs rather than copying local
- // and then reading (Passing an offset will get us an Reader
- // that wraps a Stream).
- return get(u, 0);
- }
-
- return makeARCLocal(u.openConnection());
- }
-
- protected ArchiveReader makeARCLocal(final URLConnection connection)
- throws IOException {
- File localFile = null;
- if (connection instanceof HttpURLConnection) {
- // If http url connection, bring down the resource local.
- String p = connection.getURL().getPath();
- int index = p.lastIndexOf('/');
- if (index >= 0) {
- // Name file for the file we're making local.
- localFile = File.createTempFile("",p.substring(index + 1));
- if (localFile.exists()) {
- // If file of same name already exists in TMPDIR, then
- // clean it up (Assuming only reason a file of same name in
- // TMPDIR is because we failed a previous download).
- localFile.delete();
- }
- } else {
- localFile = File.createTempFile(ArchiveReader.class.getName(),
- ".tmp");
- }
- addUserAgent((HttpURLConnection)connection);
- connection.connect();
- try {
- FileUtils.readFullyToFile(connection.getInputStream(), localFile);
- } catch (IOException ioe) {
- localFile.delete();
- throw ioe;
- }
- } else if (connection instanceof RsyncURLConnection) {
- // Then, connect and this will create a local file.
- // See implementation of the rsync handler.
- connection.connect();
- localFile = ((RsyncURLConnection)connection).getFile();
- } else if (connection instanceof Md5URLConnection) {
- // Then, connect and this will create a local file.
- // See implementation of the md5 handler.
- connection.connect();
- localFile = ((Md5URLConnection)connection).getFile();
- } else {
- throw new UnsupportedOperationException("No support for " +
- connection);
- }
-
- ArchiveReader reader = null;
- try {
- reader = get(localFile, 0);
- } catch (IOException e) {
- localFile.delete();
- throw e;
- }
-
- // Return a delegate that does cleanup of downloaded file on close.
- return reader.getDeleteFileOnCloseReader(localFile);
- }
-
- protected void addUserAgent(final HttpURLConnection connection) {
- connection.addRequestProperty("User-Agent", this.getClass().getName());
- }
-
- /**
- * @param f File to test.
- * @return True if f is compressed.
- * @throws IOException
- */
- protected boolean isCompressed(final File f) throws IOException {
- return f.getName().toLowerCase().
- endsWith(DOT_COMPRESSED_FILE_EXTENSION);
- }
-}
\ No newline at end of file
diff --git a/commons/src/main/java/org/archive/io/ArchiveRecord.java b/commons/src/main/java/org/archive/io/ArchiveRecord.java
deleted file mode 100644
index 63bfe628..00000000
--- a/commons/src/main/java/org/archive/io/ArchiveRecord.java
+++ /dev/null
@@ -1,409 +0,0 @@
-/*
- * This file is part of the Heritrix web crawler (crawler.archive.org).
- *
- * Licensed to the Internet Archive (IA) by one or more individual
- * contributors.
- *
- * The IA licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.archive.io;
-
-import java.io.IOException;
-import java.io.InputStream;
-import java.io.OutputStream;
-import java.security.MessageDigest;
-import java.security.NoSuchAlgorithmException;
-import java.util.logging.Level;
-
-import org.archive.util.Base32;
-
-/**
- * Archive file Record.
- * @author stack
- * @version $Date$ $Version$
- */
-public abstract class ArchiveRecord extends InputStream {
-
- /**
- * Minimal http response or request header length.
- *
- * I've seen in arcs content length of 1 with no header.
- */
- protected static final long MIN_HTTP_HEADER_LENGTH =
- Math.min("HTTP/1.1 200 OK\r\n".length(), "GET / HTTP/1.0\n\r".length());
-
- protected ArchiveRecordHeader header = null;
-
- /**
- * Stream to read this record from.
- *
- * Stream can only be read sequentially. Will only return this records'
- * content returning a -1 if you try to read beyond the end of the current
- * record.
- *
- * in.
- * This position is relative within this Record. Its not same as the
- * Archive file position.
- */
- protected long position = 0;
-
- /**
- * Set flag when we've reached the end-of-record.
- */
- protected boolean eor = false;
-
- /**
- * Compute digest on what we read and add to metadata when done.
- *
- * Currently hardcoded as sha-1. TODO: Remove when archive records
- * digest or else, add a facility that allows the arc reader to
- * compare the calculated digest to that which is recorded in
- * the arc.
- *
- * os.
- * @throws IOException
- */
- public void dump(final OutputStream os)
- throws IOException {
- final byte [] outputBuffer = new byte [16*1024];
- int read = outputBuffer.length;
- while ((read = read(outputBuffer, 0, outputBuffer.length)) != -1) {
- os.write(outputBuffer, 0, read);
- }
- os.flush();
- }
-
- /**
- * Is it likely that this record contains headers?
- * This method will return true if the body is a http response that includes
- * http response headers or the body is a http request that includes request
- * headers, etc. Be aware that headers in content are distinct from
- * {@link ArchiveRecordHeader} 'headers'.
- * @return True if this Record's content has headers:
- */
- public boolean hasContentHeaders() {
- final String url = getHeader().getUrl();
- if (url == null) {
- return false;
- }
-
- if (!url.toLowerCase().startsWith("http")) {
- return false;
- }
-
- if (getHeader().getLength() <= MIN_HTTP_HEADER_LENGTH) {
- return false;
- }
-
- return true;
- }
-
- protected void setBodyOffset(int bodyOffset) {
- this.position = bodyOffset;
- }
-}
diff --git a/commons/src/main/java/org/archive/io/ArchiveRecordHeader.java b/commons/src/main/java/org/archive/io/ArchiveRecordHeader.java
deleted file mode 100644
index 953537b1..00000000
--- a/commons/src/main/java/org/archive/io/ArchiveRecordHeader.java
+++ /dev/null
@@ -1,111 +0,0 @@
-/*
- * This file is part of the Heritrix web crawler (crawler.archive.org).
- *
- * Licensed to the Internet Archive (IA) by one or more individual
- * contributors.
- *
- * The IA licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.archive.io;
-
-import java.util.Map;
-import java.util.Set;
-
-/**
- * Archive Record Header.
- * @author stack
- * @version $Date$ $Version$
- */
-public interface ArchiveRecordHeader {
- /**
- * Get the time when the record was created.
- * @return Date in 14 digit time format (UTC).
- * @see org.archive.util.ArchiveUtils#parse14DigitDate(String)
- */
- public abstract String getDate();
-
- /**
- * @return Return length of record.
- */
- public abstract long getLength();
-
- /**
- * @return Return Content-Length of the contents of the record
- */
- public abstract long getContentLength();
-
-
- /**
- * @return Record subject-url.
- */
- public abstract String getUrl();
-
- /**
- * @return Record mimetype.
- */
- public abstract String getMimetype();
-
- /**
- * @return Record version.
- */
- public abstract String getVersion();
-
- /**
- * @return Offset into Archive file at which this record begins.
- */
- public abstract long getOffset();
-
- /**
- * @param key Key to use looking up field value.
- * @return value for passed key of null if no such entry.
- */
- public abstract Object getHeaderValue(final String key);
-
- /**
- * @return Header field name keys.
- */
- public abstract SetactiveSuffix
- * @param activeSuffix Suffix to replace with storeSuffix.
- * @return GenerationFileHandler instance.
- * @throws IOException
- */
- public GenerationFileHandler rotate(String storeSuffix,
- String activeSuffix)
- throws IOException {
- close();
- String filename = (String)filenameSeries.getFirst();
- if (!filename.endsWith(activeSuffix)) {
- throw new FileNotFoundException("Active file does not have" +
- " expected suffix");
- }
- String storeFilename = filename.substring(0,
- filename.length() - activeSuffix.length()) +
- storeSuffix;
- File activeFile = new File(filename);
- File storeFile = new File(storeFilename);
- FileUtils.moveAsideIfExists(storeFile);
- if (!activeFile.renameTo(storeFile)) {
- throw new IOException("Unable to move " + filename + " to " +
- storeFilename);
- }
- filenameSeries.add(1, storeFilename);
- GenerationFileHandler newGfh =
- new GenerationFileHandler(filenameSeries, shouldManifest);
- newGfh.setFormatter(this.getFormatter());
- return newGfh;
- }
-
- /**
- * @return True if should manifest.
- */
- public boolean shouldManifest() {
- return this.shouldManifest;
- }
-
- /**
- * Constructor-helper that rather than clobbering any existing
- * file, moves it aside with a timestamp suffix.
- *
- * @param filename
- * @param append
- * @param shouldManifest
- * @return
- * @throws SecurityException
- * @throws IOException
- */
- public static GenerationFileHandler makeNew(String filename, boolean append, boolean shouldManifest) throws SecurityException, IOException {
- FileUtils.moveAsideIfExists(new File(filename));
- return new GenerationFileHandler(filename, append, shouldManifest);
- }
-
- @Override
- public void publish(LogRecord record) {
- // when possible preformat outside synchronized superclass method
- // (our most involved UriProcessingFormatter can cache result)
- Formatter f = getFormatter();
- if(!(f instanceof Preformatter)) {
- super.publish(record);
- } else {
- try {
- ((Preformatter)f).preformat(record);
- super.publish(record);
- } finally {
- ((Preformatter)f).clear();
- }
- }
- }
-//
-// TODO: determine if there's another way to have this optimization without
-// negative impact on log-following (esp. in web UI)
-// /**
-// * Flush only 1/100th of the usual once-per-record, to reduce the time
-// * spent holding the synchronization lock. (Flush is primarily called in
-// * a superclass's synchronized publish()).
-// *
-// * The eventual close calls a direct flush on the target writer, so all
-// * rotates/ends will ultimately be fully flushed.
-// *
-// * @see java.util.logging.StreamHandler#flush()
-// */
-// @Override
-// public synchronized void flush() {
-// flushCount++;
-// if(flushCount==100) {
-// super.flush();
-// flushCount=0;
-// }
-// }
-// int flushCount;
-
-}
\ No newline at end of file
diff --git a/commons/src/main/java/org/archive/io/GenericReplayCharSequence.java b/commons/src/main/java/org/archive/io/GenericReplayCharSequence.java
deleted file mode 100644
index 1af3922b..00000000
--- a/commons/src/main/java/org/archive/io/GenericReplayCharSequence.java
+++ /dev/null
@@ -1,412 +0,0 @@
-/*
- * This file is part of the Heritrix web crawler (crawler.archive.org).
- *
- * Licensed to the Internet Archive (IA) by one or more individual
- * contributors.
- *
- * The IA licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.archive.io;
-
-import java.io.BufferedReader;
-import java.io.File;
-import java.io.FileInputStream;
-import java.io.FileNotFoundException;
-import java.io.FileOutputStream;
-import java.io.IOException;
-import java.io.InputStream;
-import java.io.InputStreamReader;
-import java.io.OutputStreamWriter;
-import java.io.Writer;
-import java.nio.CharBuffer;
-import java.nio.channels.FileChannel;
-import java.nio.charset.CharacterCodingException;
-import java.nio.charset.Charset;
-import java.text.NumberFormat;
-import java.util.logging.Level;
-import java.util.logging.Logger;
-
-import org.apache.commons.io.IOUtils;
-import org.archive.util.DevUtils;
-
-import com.google.common.base.Charsets;
-import com.google.common.primitives.Ints;
-
-/**
- * (Replay)CharSequence view on recorded streams.
- *
- * For small streams, use {@link InMemoryReplayCharSequence}.
- *
- * MAP_MAX_BYTES - MAP_TARGET_LEFT_PADDING
- * bytes to the right of the target.
- */
- private static final long MAP_TARGET_LEFT_PADDING_BYTES = (long) (MAP_MAX_BYTES * 0.01);
-
- /**
- * Total length of character stream to replay minus the HTTP headers
- * if present.
- *
- * If the backing file is larger than Integer.MAX_VALUE (i.e. 2gb),
- * only the first Integer.MAX_VALUE characters are available through this API.
- * We're overriding java.lang.CharSequence so that we can use
- * java.util.regex directly on the data, and the CharSequence
- * API uses int for the length and index.
- */
- protected int length;
-
- /** counter of decoding exceptions for report at end */
- protected long decodingExceptions = 0;
- protected CharacterCodingException codingException = null;
-
- /**
- * Byte offset into the file where the memory mapped portion begins.
- */
- private long mapByteOffset;
-
- // XXX do we need to keep the input stream around?
- private FileInputStream backingFileIn = null;
-
- private FileChannel backingFileChannel = null;
-
- private long bytesPerChar;
-
- private CharBuffer mappedBuffer = null;
-
- /**
- * File that has decoded content.
- *
- * Keep it around so we can remove on close.
- */
- private File decodedFile = null;
-
- /*
- * This portion of the CharSequence precedes what's in the backing file. In
- * cases where we decodeToFile(), this is always empty, because we decode
- * the entire input stream.
- */
- private CharBuffer prefixBuffer = null;
-
- private boolean isOpen = true;
-
- protected Charset charset = null;
-
- /**
- * Constructor.
- *
- * @param contentReplayInputStream inputStream of content
- * @param charset Encoding to use reading the passed prefix
- * buffer and backing file. Must not be null.
- * @param backingFilename Path to backing file with content in excess of
- * whats in buffer.
- *
- * @throws IOException
- */
- public GenericReplayCharSequence(InputStream contentReplayInputStream,
- int prefixMax,
- String backingFilename,
- Charset charset) throws IOException {
- super();
- logger.fine("characterEncoding=" + charset + " backingFilename="
- + backingFilename);
-
- if(charset==null) {
- charset = ReplayCharSequence.FALLBACK_CHARSET;
- }
- // decodes only up to Integer.MAX_VALUE characters
- decode(contentReplayInputStream, prefixMax, backingFilename, charset);
-
- this.bytesPerChar = 2;
-
- if(length>prefixBuffer.position()) {
- this.backingFileIn = new FileInputStream(decodedFile);
- this.backingFileChannel = backingFileIn.getChannel();
- this.mapByteOffset = 0;
- updateMemoryMappedBuffer();
- }
- }
-
- private void updateMemoryMappedBuffer() {
- long charLength = (long) this.length() - (long) prefixBuffer.limit(); // in characters
- long mapSize = Math.min((charLength * bytesPerChar) - mapByteOffset, MAP_MAX_BYTES);
- logger.fine("updateMemoryMappedBuffer: mapOffset="
- + NumberFormat.getInstance().format(mapByteOffset)
- + " mapSize=" + NumberFormat.getInstance().format(mapSize));
- try {
- // TODO: stress-test without these possibly-costly requests!
-// System.gc();
-// System.runFinalization();
- // TODO: Confirm the READ_ONLY works. I recall it not working.
- // The buffers seem to always say that the buffer is writable.
- mappedBuffer = backingFileChannel.map(
- FileChannel.MapMode.READ_ONLY, mapByteOffset, mapSize)
- .asReadOnlyBuffer().asCharBuffer();
- } catch (IOException e) {
- // TODO convert this to a runtime error?
- DevUtils.logger.log(Level.SEVERE,
- " backingFileChannel.map() mapByteOffset=" + mapByteOffset
- + " mapSize=" + mapSize + "\n" + "decodedFile="
- + decodedFile + " length=" + length + "\n"
- + DevUtils.extraInfo(), e);
- throw new RuntimeException(e);
- }
- }
-
- /**
- * Converts the first Integer.MAX_VALUE characters from the
- * file backingFilename from encoding encoding to
- * encoding WRITE_ENCODING and saves as
- * this.decodedFile, which is named backingFilename
- * + "." + WRITE_ENCODING.
- *
- * @throws IOException
- */
- protected void decode(InputStream inStream, int prefixMax,
- String backingFilename, Charset charset) throws IOException {
-
- this.charset = charset;
-
- // TODO: consider if BufferedReader is helping any
- // TODO: consider adding TBW 'LimitReader' to stop reading at
- // Integer.MAX_VALUE characters because of charAt(int) limit
- BufferedReader reader = new BufferedReader(new InputStreamReader(
- inStream, charset));
-
- logger.fine("backingFilename=" + backingFilename + " encoding="
- + charset + " decodedFile=" + decodedFile);
-
- this.prefixBuffer = CharBuffer.allocate(prefixMax);
-
- long count = 0;
- while(count < prefixMax) {
- int read = reader.read(prefixBuffer);
- if(read<0) {
- break;
- }
- count += read;
- }
-
- int ch = reader.read();
- if(ch >= 0) {
- count++;
-
- // more to decode to file overflow
- this.decodedFile = new File(backingFilename + "." + WRITE_ENCODING);
-
- FileOutputStream fos;
- try {
- fos = new FileOutputStream(this.decodedFile);
- } catch (FileNotFoundException e) {
- // Windows workaround attempt
- System.gc();
- System.runFinalization();
- this.decodedFile = new File(decodedFile.getAbsolutePath()+".win");
- logger.info("Windows 'file with a user-mapped section open' "
- + "workaround gc/finalization/name-extension performed.");
- // try again
- fos = new FileOutputStream(this.decodedFile);
- }
-
- Writer writer = new OutputStreamWriter(fos,WRITE_ENCODING);
- writer.write(ch);
- count += IOUtils.copyLarge(reader, writer);
- writer.close();
- reader.close();
- }
-
- this.length = Ints.saturatedCast(count);
- if(count>Integer.MAX_VALUE) {
- logger.warning("input stream is longer than Integer.MAX_VALUE="
- + NumberFormat.getInstance().format(Integer.MAX_VALUE)
- + " characters -- only first "
- + NumberFormat.getInstance().format(Integer.MAX_VALUE)
- + " are accessible through this GenericReplayCharSequence");
- }
-
- logger.fine("decode: decoded " + count + " characters" +
- ((decodedFile==null) ? ""
- : " ("+(count-prefixBuffer.length())+" to "+decodedFile+")"));
- }
-
- /**
- * Get character at passed absolute position.
- * @param index Index into content
- * @return Character at offset index.
- */
- public char charAt(int index) {
- if (index < 0 || index >= this.length()) {
- throw new IndexOutOfBoundsException("index=" + index
- + " - should be between 0 and length()=" + this.length());
- }
-
- // is it in the buffer
- if (index < prefixBuffer.limit()) {
- return prefixBuffer.get(index);
- }
-
- // otherwise we gotta get it from disk via memory map
- long charFileIndex = (long) index - (long) prefixBuffer.limit();
- long charFileLength = (long) this.length() - (long) prefixBuffer.limit(); // in characters
- if (charFileIndex * bytesPerChar < mapByteOffset) {
- logger.log(Level.WARNING,"left-fault; probably don't want to use CharSequence that far backward");
- }
- if (charFileIndex * bytesPerChar < mapByteOffset
- || charFileIndex - (mapByteOffset / bytesPerChar) >= mappedBuffer.limit()) {
- // fault
- /*
- * mapByteOffset is bounded by 0 and file size +/- size of the map,
- * and starts as close to fileIndex -
- * MAP_TARGET_LEFT_PADDING_BYTES as it can while also not
- * being smaller than it needs to be.
- */
- mapByteOffset = Math.min(charFileIndex * bytesPerChar - MAP_TARGET_LEFT_PADDING_BYTES,
- charFileLength * bytesPerChar - MAP_MAX_BYTES);
- mapByteOffset = Math.max(0, mapByteOffset);
- updateMemoryMappedBuffer();
- }
-
- return mappedBuffer.get((int)(charFileIndex-(mapByteOffset/bytesPerChar)));
- }
-
- public CharSequence subSequence(int start, int end) {
- return new CharSubSequence(this, start, end);
- }
-
- private void deleteFile(File fileToDelete) {
- deleteFile(fileToDelete, null);
- }
-
- private void deleteFile(File fileToDelete, final Exception e) {
- if (e != null) {
- // Log why the delete to help with debug of
- // java.io.FileNotFoundException:
- // ....tt53http.ris.UTF-16BE.
- logger.severe("Deleting " + fileToDelete + " because of "
- + e.toString());
- }
- if (fileToDelete != null && fileToDelete.exists()) {
- logger.fine("deleting file: " + fileToDelete);
- fileToDelete.delete();
- }
- }
-
-
- @Override
- public boolean isOpen() {
- return this.isOpen;
- }
-
- public void close() throws IOException {
- this.isOpen = false;
-
- logger.fine("closing");
-
- if (this.backingFileChannel != null && this.backingFileChannel.isOpen()) {
- this.backingFileChannel.close();
- }
- if (backingFileIn != null) {
- backingFileIn.close();
- }
-
- deleteFile(this.decodedFile);
-
- // clear decodedFile -- so that double-close (as in finalize()) won't
- // delete a later instance with same name see bug [ 1218961 ]
- // "failed get of replay" in ExtractorHTML... usu: UTF-16BE
- this.decodedFile = null;
- }
-
- /*
- * (non-Javadoc)
- *
- * @see java.lang.Object#finalize()
- */
- protected void finalize() throws Throwable {
- super.finalize();
- logger.fine("finalizing");
- close();
- }
-
- /**
- * Convenience method for getting a substring.
- *
- * @deprecated please use subSequence() and then toString() directly
- */
- public String substring(int offset, int len) {
- return subSequence(offset, offset + len).toString();
- }
-
- public String toString() {
- StringBuilder sb = new StringBuilder(this.length());
- sb.append(this);
- return sb.toString();
- }
-
- public int length() {
- return length;
- }
-
- /* (non-Javadoc)
- * @see org.archive.io.ReplayCharSequence#getDecodeExceptionCount()
- */
- @Override
- public long getDecodeExceptionCount() {
- return decodingExceptions;
- }
-
-
- /* (non-Javadoc)
- * @see org.archive.io.ReplayCharSequence#getCodingException()
- */
- @Override
- public CharacterCodingException getCodingException() {
- return codingException;
- }
-
- /* (non-Javadoc)
- * @see org.archive.io.ReplayCharSequence#getCharset()
- */
- public Charset getCharset() {
- return charset;
- }
-}
\ No newline at end of file
diff --git a/commons/src/main/java/org/archive/io/GzipHeader.java b/commons/src/main/java/org/archive/io/GzipHeader.java
deleted file mode 100644
index 6b8263bc..00000000
--- a/commons/src/main/java/org/archive/io/GzipHeader.java
+++ /dev/null
@@ -1,26 +0,0 @@
-/*
- * This file is part of the Heritrix web crawler (crawler.archive.org).
- *
- * Licensed to the Internet Archive (IA) by one or more individual
- * contributors.
- *
- * The IA licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.archive.io;
-
-/**
- * @deprecated use {@link org.archive.util.zip.GzipHeader}
- */
-@Deprecated
-public class GzipHeader extends org.archive.util.zip.GzipHeader {
-}
diff --git a/commons/src/main/java/org/archive/io/HeaderedArchiveRecord.java b/commons/src/main/java/org/archive/io/HeaderedArchiveRecord.java
deleted file mode 100644
index 3cce595b..00000000
--- a/commons/src/main/java/org/archive/io/HeaderedArchiveRecord.java
+++ /dev/null
@@ -1,423 +0,0 @@
-/*
- * This file is part of the Heritrix web crawler (crawler.archive.org).
- *
- * Licensed to the Internet Archive (IA) by one or more individual
- * contributors.
- *
- * The IA licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.archive.io;
-
-import java.io.ByteArrayInputStream;
-import java.io.ByteArrayOutputStream;
-import java.io.IOException;
-import java.io.InputStream;
-import java.io.OutputStream;
-import java.io.PrintStream;
-
-import org.apache.commons.httpclient.Header;
-import org.apache.commons.httpclient.HttpParser;
-import org.apache.commons.httpclient.StatusLine;
-import org.apache.commons.httpclient.util.EncodingUtil;
-import org.archive.io.arc.ARCConstants;
-import org.archive.util.LaxHttpParser;
-
-/**
- * An ArchiveRecord whose content has a preamble of RFC822-like headers: e.g.
- * The ArchiveRecord is a http response that leads off with http response
- * headers. Use this ArchiveRecord Decorator to get at the content headers and
- * the header/content demarcation.
- *
- * @author stack
- * @author Olaf Freyer
- */
-public class HeaderedArchiveRecord extends ArchiveRecord {
- private int contentHeadersLength = -1;
- private int statusCode = -1;
-
- /**
- * Http header bytes.
- *
- * If non-null and bytes available, give out its contents before we
- * go back to the underlying stream.
- */
- private InputStream contentHeaderStream = null;
-
- /**
- * Content headers.
- *
- * Only available after the reading of headers.
- */
- private Header [] contentHeaders = null;
-
-
- public HeaderedArchiveRecord(final ArchiveRecord ar) throws IOException {
- super(ar);
- }
-
- public HeaderedArchiveRecord(final ArchiveRecord ar,
- final boolean readContentHeader) throws IOException {
- super(ar);
- if (readContentHeader) {
- this.contentHeaderStream = readContentHeaders();
- }
- }
-
- /**
- * Skip over the the content headers if present.
- *
- * Subsequent reads will get the body.
- *
- * position methods adjust the position of the
- * underlying stream relative to the origin specified at construction time.
- *
- * @author pjack
- */
-public class OriginSeekInputStream extends SeekInputStream {
-
-
- /**
- * The underlying stream.
- */
- final private SeekInputStream input;
-
-
- /**
- * The origin position. In other words, this.position(0)
- * resolves to input.position(start).
- */
- final private long origin;
-
-
- /**
- * Constructor.
- *
- * @param input the underlying stream
- * @param origin the origin position
- * @throws IOException if an IO error occurs
- */
- public OriginSeekInputStream(SeekInputStream input, long origin)
- throws IOException {
- this.input = input;
- this.origin = origin;
- input.position(origin);
- }
-
-
- @Override
- public int available() throws IOException {
- return input.available();
- }
-
-
- @Override
- public int read() throws IOException {
- return input.read();
- }
-
-
- @Override
- public int read(byte[] buf, int ofs, int len) throws IOException {
- return input.read(buf, ofs, len);
- }
-
-
- @Override
- public int read(byte[] buf) throws IOException {
- return input.read(buf);
- }
-
-
- @Override
- public long skip(long count) throws IOException {
- return input.skip(count);
- }
-
-
- /**
- * Returns the position of the underlying stream relative to the origin.
- *
- * @return the relative position
- * @throws IOException if an IO error occurs
- */
- public long position() throws IOException {
- return input.position() - origin;
- }
-
-
- /**
- * Positions the underlying stream relative to the origin.
- * In other words, this.position(0) resolves to input.position(origin),
- * where input is underlying stream and origin is the origin specified
- * at construction time.
- *
- * @param p the new position for this stream
- * @throws IOException if an IO error occurs
- */
- public void position(long p) throws IOException {
- input.position(p + origin);
- }
-}
diff --git a/commons/src/main/java/org/archive/io/Preformatter.java b/commons/src/main/java/org/archive/io/Preformatter.java
deleted file mode 100644
index dcd31bb6..00000000
--- a/commons/src/main/java/org/archive/io/Preformatter.java
+++ /dev/null
@@ -1,32 +0,0 @@
-/*
- * This file is part of the Heritrix web crawler (crawler.archive.org).
- *
- * Licensed to the Internet Archive (IA) by one or more individual
- * contributors.
- *
- * The IA licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.archive.io;
-
-import java.util.logging.LogRecord;
-
-/**
- * Interface indicating a logging Formatter can preformat a record (outside
- * the standard-implementation synchronized block) and cache it, returning it
- * for the next request for formatting from the same thread.
- * @contributor gojomo
- */
-public interface Preformatter {
- public void preformat(LogRecord record);
- public void clear();
-}
diff --git a/commons/src/main/java/org/archive/io/RandomAccessInputStream.java b/commons/src/main/java/org/archive/io/RandomAccessInputStream.java
deleted file mode 100644
index d8dd260b..00000000
--- a/commons/src/main/java/org/archive/io/RandomAccessInputStream.java
+++ /dev/null
@@ -1,180 +0,0 @@
-/*
- * This file is part of the Heritrix web crawler (crawler.archive.org).
- *
- * Licensed to the Internet Archive (IA) by one or more individual
- * contributors.
- *
- * The IA licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.archive.io;
-
-
-import java.io.File;
-import java.io.IOException;
-import java.io.RandomAccessFile;
-
-
-/**
- * Wraps a RandomAccessFile with an InputStream interface.
- *
- * @author gojomo
- */
-public class RandomAccessInputStream extends SeekInputStream {
-
- /**
- * Reference to the random access file this stream is reading from.
- */
- private RandomAccessFile raf = null;
-
- /**
- * When mark is called, save here the current position so we can go back
- * on reset.
- */
- private long markpos = -1;
-
- /**
- * True if we are to close the underlying random access file when this
- * stream is closed.
- */
- private boolean sympathyClose;
-
- /**
- * Constructor.
- *
- * If using this constructor, caller created the RAF and therefore
- * its assumed wants to control close of the RAF. The RAF.close
- * is not called if this constructor is used on close of this stream.
- *
- * @param raf RandomAccessFile to wrap.
- * @throws IOException
- */
- public RandomAccessInputStream(RandomAccessFile raf)
- throws IOException {
- this(raf, false, 0);
- }
-
- /**
- * Constructor.
- *
- * @param file File to get RAFIS on. Creates an RAF from passed file.
- * Closes the created RAF when this stream is closed.
- * @throws IOException
- */
- public RandomAccessInputStream(final File file)
- throws IOException {
- this(new RandomAccessFile(file, "r"), true, 0);
- }
-
- /**
- * Constructor.
- *
- * @param file File to get RAFIS on. Creates an RAF from passed file.
- * Closes the created RAF when this stream is closed.
- * @param offset
- * @throws IOException
- */
- public RandomAccessInputStream(final File file, final long offset)
- throws IOException {
- this(new RandomAccessFile(file, "r"), true, offset);
- }
-
- /**
- * @param raf RandomAccessFile to wrap.
- * @param sympathyClose Set to true if we are to close the RAF
- * file when this stream is closed.
- * @param offset
- * @throws IOException
- */
- public RandomAccessInputStream(final RandomAccessFile raf,
- final boolean sympathyClose, final long offset)
- throws IOException {
- super();
- this.sympathyClose = sympathyClose;
- this.raf = raf;
- if (offset > 0) {
- this.raf.seek(offset);
- }
- }
-
- /* (non-Javadoc)
- * @see java.io.InputStream#read()
- */
- public int read() throws IOException {
- return this.raf.read();
- }
-
- /* (non-Javadoc)
- * @see java.io.InputStream#read(byte[], int, int)
- */
- public int read(byte[] b, int off, int len) throws IOException {
- return this.raf.read(b, off, len);
- }
-
- /* (non-Javadoc)
- * @see java.io.InputStream#read(byte[])
- */
- public int read(byte[] b) throws IOException {
- return this.raf.read(b);
- }
-
- /* (non-Javadoc)
- * @see java.io.InputStream#skip(long)
- */
- public long skip(long n) throws IOException {
- this.raf.seek(this.raf.getFilePointer() + n);
- return n;
- }
-
- public long position() throws IOException {
- return this.raf.getFilePointer();
- }
-
- public void position(long position) throws IOException {
- this.raf.seek(position);
- }
-
- public int available() throws IOException {
- long amount = this.raf.length() - this.position();
- return (amount >= Integer.MAX_VALUE)? Integer.MAX_VALUE: (int)amount;
- }
-
- public boolean markSupported() {
- return true;
- }
-
- public synchronized void mark(int readlimit) {
- try {
- this.markpos = position();
- } catch (IOException e) {
- // Set markpos to -1. Will cause exception reset.
- this.markpos = -1;
- }
- }
-
- public synchronized void reset() throws IOException {
- if (this.markpos == -1) {
- throw new IOException("Mark has not been set.");
- }
- position(this.markpos);
- }
-
- public void close() throws IOException {
- try {
- super.close();
- } finally {
- if (this.sympathyClose) {
- this.raf.close();
- }
- }
- }
-}
\ No newline at end of file
diff --git a/commons/src/main/java/org/archive/io/RandomAccessOutputStream.java b/commons/src/main/java/org/archive/io/RandomAccessOutputStream.java
deleted file mode 100644
index 225f995f..00000000
--- a/commons/src/main/java/org/archive/io/RandomAccessOutputStream.java
+++ /dev/null
@@ -1,69 +0,0 @@
-/*
- * This file is part of the Heritrix web crawler (crawler.archive.org).
- *
- * Licensed to the Internet Archive (IA) by one or more individual
- * contributors.
- *
- * The IA licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.archive.io;
-
-import java.io.IOException;
-import java.io.OutputStream;
-import java.io.RandomAccessFile;
-
-
-/**
- * Wraps a RandomAccessFile with OutputStream interface.
- *
- * @author gojomo
- */
-public class RandomAccessOutputStream extends OutputStream {
- protected RandomAccessFile raf;
-
- /**
- * Wrap the given RandomAccessFile
- */
- public RandomAccessOutputStream(RandomAccessFile raf) {
- super();
- this.raf = raf;
- }
-
- /* (non-Javadoc)
- * @see java.io.OutputStream#write(int)
- */
- public void write(int b) throws IOException {
- raf.write(b);
- }
-
- /* (non-Javadoc)
- * @see java.io.OutputStream#close()
- */
- public void close() throws IOException {
- raf.close();
- }
-
- /* (non-Javadoc)
- * @see java.io.OutputStream#write(byte[], int, int)
- */
- public void write(byte[] b, int off, int len) throws IOException {
- raf.write(b, off, len);
- }
-
- /* (non-Javadoc)
- * @see java.io.OutputStream#write(byte[])
- */
- public void write(byte[] b) throws IOException {
- raf.write(b);
- }
-}
diff --git a/commons/src/main/java/org/archive/io/ReadSource.java b/commons/src/main/java/org/archive/io/ReadSource.java
deleted file mode 100644
index a3c29967..00000000
--- a/commons/src/main/java/org/archive/io/ReadSource.java
+++ /dev/null
@@ -1,37 +0,0 @@
-/*
- * This file is part of the Heritrix web crawler (crawler.archive.org).
- *
- * Licensed to the Internet Archive (IA) by one or more individual
- * contributors.
- *
- * The IA licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.archive.io;
-
-import java.io.Reader;
-
-/**
- * Interface for objects that can provide a Reader view of their
- * contents.
- *
- */
-public interface ReadSource {
- /**
- * Obtain a Reader. Not named 'getReader' so that it is not
- * considered a simple costless read-only property by
- * bean-convention introspection tools.
- * @return a Reader on this object
- */
- Reader obtainReader();
-}
diff --git a/commons/src/main/java/org/archive/io/RecorderIOException.java b/commons/src/main/java/org/archive/io/RecorderIOException.java
deleted file mode 100644
index 07b30061..00000000
--- a/commons/src/main/java/org/archive/io/RecorderIOException.java
+++ /dev/null
@@ -1,38 +0,0 @@
-/*
- * This file is part of the Heritrix web crawler (crawler.archive.org).
- *
- * Licensed to the Internet Archive (IA) by one or more individual
- * contributors.
- *
- * The IA licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.archive.io;
-
-import java.io.IOException;
-
-/**
- *
- * @author Gordon Mohr
- */
-public class RecorderIOException extends IOException {
-
- private static final long serialVersionUID = 5907470275350314277L;
-
- public RecorderIOException() {
- super();
- }
-
- public RecorderIOException(String msg) {
- super(msg);
- }
-}
diff --git a/commons/src/main/java/org/archive/io/RecorderLengthExceededException.java b/commons/src/main/java/org/archive/io/RecorderLengthExceededException.java
deleted file mode 100644
index 8c3e067d..00000000
--- a/commons/src/main/java/org/archive/io/RecorderLengthExceededException.java
+++ /dev/null
@@ -1,39 +0,0 @@
-/*
- * This file is part of the Heritrix web crawler (crawler.archive.org).
- *
- * Licensed to the Internet Archive (IA) by one or more individual
- * contributors.
- *
- * The IA licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.archive.io;
-
-
-/**
- * Indicates a length exception thrown by the Recorder.
- *
- * @author Gordon Mohr
- */
-public class RecorderLengthExceededException
-extends RecorderIOException {
-
- private static final long serialVersionUID = 6655419033414648444L;
-
- public RecorderLengthExceededException() {
- super();
- }
-
- public RecorderLengthExceededException(String msg) {
- super(msg);
- }
-}
diff --git a/commons/src/main/java/org/archive/io/RecorderTimeoutException.java b/commons/src/main/java/org/archive/io/RecorderTimeoutException.java
deleted file mode 100644
index 32be5b5d..00000000
--- a/commons/src/main/java/org/archive/io/RecorderTimeoutException.java
+++ /dev/null
@@ -1,37 +0,0 @@
-/*
- * This file is part of the Heritrix web crawler (crawler.archive.org).
- *
- * Licensed to the Internet Archive (IA) by one or more individual
- * contributors.
- *
- * The IA licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.archive.io;
-
-/**
- * Indicates a timeout thrown by the RecordingInputStream.
- *
- * @author Gordon Mohr
- */
-public class RecorderTimeoutException extends RecorderIOException {
-
- private static final long serialVersionUID = 7433214063765078269L;
-
- public RecorderTimeoutException() {
- super();
- }
-
- public RecorderTimeoutException(String msg) {
- super(msg);
- }
-}
diff --git a/commons/src/main/java/org/archive/io/RecorderTooMuchHeaderException.java b/commons/src/main/java/org/archive/io/RecorderTooMuchHeaderException.java
deleted file mode 100644
index 23f5d264..00000000
--- a/commons/src/main/java/org/archive/io/RecorderTooMuchHeaderException.java
+++ /dev/null
@@ -1,40 +0,0 @@
-/*
- * This file is part of the Heritrix web crawler (crawler.archive.org).
- *
- * Licensed to the Internet Archive (IA) by one or more individual
- * contributors.
- *
- * The IA licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.archive.io;
-
-
-/**
- * Indicates a too much header material exception thrown by the Recorder
- * (specificially the RecordingOutputStream)
- *
- * @author Gordon Mohr
- */
-public class RecorderTooMuchHeaderException
-extends RecorderIOException {
-
- private static final long serialVersionUID = 3528516034898129150L;
-
- public RecorderTooMuchHeaderException() {
- super();
- }
-
- public RecorderTooMuchHeaderException(String msg) {
- super(msg);
- }
-}
diff --git a/commons/src/main/java/org/archive/io/RecordingInputStream.java b/commons/src/main/java/org/archive/io/RecordingInputStream.java
deleted file mode 100644
index c654eb61..00000000
--- a/commons/src/main/java/org/archive/io/RecordingInputStream.java
+++ /dev/null
@@ -1,417 +0,0 @@
-/*
- * This file is part of the Heritrix web crawler (crawler.archive.org).
- *
- * Licensed to the Internet Archive (IA) by one or more individual
- * contributors.
- *
- * The IA licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.archive.io;
-
-import java.io.IOException;
-import java.io.InputStream;
-import java.net.SocketException;
-import java.net.SocketTimeoutException;
-import java.security.MessageDigest;
-import java.util.logging.Level;
-import java.util.logging.Logger;
-
-import org.apache.commons.io.IOUtils;
-
-
-/**
- * Stream which records all data read from it, which it acquires from a wrapped
- * input stream.
- *
- * Makes use of a RecordingOutputStream for recording because of its being
- * file backed so we can write massive amounts of data w/o worrying about
- * overflowing memory.
- *
- * @author gojomo
- *
- */
-public class RecordingInputStream
- extends InputStream {
-
- protected static Logger logger =
- Logger.getLogger("org.archive.io.RecordingInputStream");
-
- /**
- * Where we are recording to.
- */
- private RecordingOutputStream recordingOutputStream;
-
- /**
- * Stream to record.
- */
- private InputStream in = null;
-
- /**
- * Reusable buffer to avoid reallocation on each readFullyUntil
- */
- protected byte[] drainBuffer = new byte[16*1024];
-
- /**
- * Create a new RecordingInputStream.
- *
- * @param bufferSize Size of buffer to use.
- * @param backingFilename Name of backing file.
- */
- public RecordingInputStream(int bufferSize, String backingFilename)
- {
- this.recordingOutputStream = new RecordingOutputStream(bufferSize,
- backingFilename);
- }
-
- public void open(InputStream wrappedStream) throws IOException {
- if (logger.isLoggable(Level.FINE)) {
- logger.fine("wrapping " + wrappedStream + " in thread "
- + Thread.currentThread().getName());
- }
- if(isOpen()) {
- // error; should not be opening/wrapping in an unclosed
- // stream remains open
- throw new IOException("RIS already open for "
- +Thread.currentThread().getName());
- }
- try {
- this.in = wrappedStream;
- this.recordingOutputStream.open();
- } catch (IOException ioe) {
- close(); // ...and rethrow...
- throw ioe;
- }
- }
-
- public int read() throws IOException {
- if (!isOpen()) {
- throw new IOException("Stream closed " +
- Thread.currentThread().getName());
- }
- int b = this.in.read();
- if (b != -1) {
- assert this.recordingOutputStream != null: "ROS is null " +
- Thread.currentThread().getName();
- this.recordingOutputStream.write(b);
- }
- return b;
- }
-
- public int read(byte[] b, int off, int len) throws IOException {
- if (!isOpen()) {
- throw new IOException("Stream closed " +
- Thread.currentThread().getName());
- }
- int count = this.in.read(b,off,len);
- if (count > 0) {
- assert this.recordingOutputStream != null: "ROS is null " +
- Thread.currentThread().getName();
- this.recordingOutputStream.write(b,off,count);
- }
- return count;
- }
-
- public int read(byte[] b) throws IOException {
- if (!isOpen()) {
- throw new IOException("Stream closed " +
- Thread.currentThread().getName());
- }
- int count = this.in.read(b);
- if (count > 0) {
- assert this.recordingOutputStream != null: "ROS is null " +
- Thread.currentThread().getName();
- this.recordingOutputStream.write(b,0,count);
- }
- return count;
- }
-
- public void close() throws IOException {
- if (logger.isLoggable(Level.FINE)) {
- logger.fine("closing " + this.in + " in thread "
- + Thread.currentThread().getName());
- }
- IOUtils.closeQuietly(this.in);
- this.in = null;
- IOUtils.closeQuietly(this.recordingOutputStream);
- }
-
- public ReplayInputStream getReplayInputStream() throws IOException {
- return this.recordingOutputStream.getReplayInputStream();
- }
-
- public ReplayInputStream getMessageBodyReplayInputStream() throws IOException {
- return this.recordingOutputStream.getMessageBodyReplayInputStream();
- }
-
- public long readFully() throws IOException {
- while(read(drainBuffer) != -1) {
- // Empty out stream.
- continue;
- }
- return this.recordingOutputStream.getSize();
- }
-
- public void readToEndOfContent(long contentLength)
- throws IOException, InterruptedException {
- // Check we're open before proceeding.
- if (!isOpen()) {
- // TODO: should this be a noisier exception-raising error?
- return;
- }
-
- long totalBytes = recordingOutputStream.position - recordingOutputStream.getMessageBodyBegin();
- long bytesRead = -1L;
- long maxToRead = -1;
- while (contentLength <= 0 || totalBytes < contentLength) {
- try {
- // read no more than soft max
- maxToRead = (contentLength <= 0)
- ? drainBuffer.length
- : Math.min(drainBuffer.length, contentLength - totalBytes);
- // nor more than hard max
- maxToRead = Math.min(maxToRead, recordingOutputStream.getRemainingLength());
- // but always at least 1 (to trigger hard max exception) XXX wtf is this?
- maxToRead = Math.max(maxToRead, 1);
-
- bytesRead = read(drainBuffer,0,(int)maxToRead);
- if (bytesRead == -1) {
- break;
- }
- totalBytes += bytesRead;
-
- if (Thread.interrupted()) {
- throw new InterruptedException("Interrupted during IO");
- }
- } catch (SocketTimeoutException e) {
- // A socket timeout is just a transient problem, meaning
- // nothing was available in the configured timeout period,
- // but something else might become available later.
- // Take this opportunity to check the overall
- // timeout (below). One reason for this timeout is
- // servers that keep up the connection, 'keep-alive', even
- // though we asked them to not keep the connection open.
- if (logger.isLoggable(Level.FINE)) {
- logger.log(Level.FINE, "socket timeout", e);
- }
- // check for interrupt
- if (Thread.interrupted()) {
- throw new InterruptedException("Interrupted during IO");
- }
- // check for overall timeout
- recordingOutputStream.checkLimits();
- } catch (SocketException se) {
- throw se;
- } catch (NullPointerException e) {
- // [ 896757 ] NPEs in Andy's Th-Fri Crawl.
- // A crawl was showing NPE's in this part of the code but can
- // not reproduce. Adding this rethrowing catch block w/
- // diagnostics to help should we come across the problem in the
- // future.
- throw new NullPointerException("Stream " + this.in + ", " +
- e.getMessage() + " " + Thread.currentThread().getName());
- }
- }
- }
-
- /**
- * Read all of a stream (Or read until we timeout or have read to the max).
- * @param softMaxLength Maximum length to read; if zero or < 0, then no
- * limit. If met, return normally.
- * @throws IOException failed read.
- * @throws RecorderLengthExceededException
- * @throws RecorderTimeoutException
- * @throws InterruptedException
- * @deprecated
- */
- public void readFullyOrUntil(long softMaxLength)
- throws IOException, RecorderLengthExceededException,
- RecorderTimeoutException, InterruptedException {
- // Check we're open before proceeding.
- if (!isOpen()) {
- // TODO: should this be a noisier exception-raising error?
- return;
- }
-
- long totalBytes = 0L;
- long bytesRead = -1L;
- long maxToRead = -1;
- while (true) {
- try {
- // read no more than soft max
- maxToRead = (softMaxLength <= 0)
- ? drainBuffer.length
- : Math.min(drainBuffer.length, softMaxLength - totalBytes);
- // nor more than hard max
- maxToRead = Math.min(maxToRead, recordingOutputStream.getRemainingLength());
- // but always at least 1 (to trigger hard max exception
- maxToRead = Math.max(maxToRead, 1);
-
- bytesRead = read(drainBuffer,0,(int)maxToRead);
- if (bytesRead == -1) {
- break;
- }
- totalBytes += bytesRead;
-
- if (Thread.interrupted()) {
- throw new InterruptedException("Interrupted during IO");
- }
- } catch (SocketTimeoutException e) {
- // A socket timeout is just a transient problem, meaning
- // nothing was available in the configured timeout period,
- // but something else might become available later.
- // Take this opportunity to check the overall
- // timeout (below). One reason for this timeout is
- // servers that keep up the connection, 'keep-alive', even
- // though we asked them to not keep the connection open.
- if (logger.isLoggable(Level.FINE)) {
- logger.log(Level.FINE, "socket timeout", e);
- }
- // check for interrupt
- if (Thread.interrupted()) {
- throw new InterruptedException("Interrupted during IO");
- }
- // check for overall timeout
- recordingOutputStream.checkLimits();
- } catch (SocketException se) {
- throw se;
- } catch (NullPointerException e) {
- // [ 896757 ] NPEs in Andy's Th-Fri Crawl.
- // A crawl was showing NPE's in this part of the code but can
- // not reproduce. Adding this rethrowing catch block w/
- // diagnostics to help should we come across the problem in the
- // future.
- throw new NullPointerException("Stream " + this.in + ", " +
- e.getMessage() + " " + Thread.currentThread().getName());
- }
-
- // if have read 'enough', just finish
- if (softMaxLength > 0 && totalBytes >= softMaxLength) {
- break; // return
- }
- }
- }
-
- public long getSize() {
- return this.recordingOutputStream.getSize();
- }
-
- public void markContentBegin() {
- this.recordingOutputStream.markMessageBodyBegin();
- }
-
- public long getContentBegin() {
- return this.recordingOutputStream.getMessageBodyBegin();
- }
-
- public void startDigest() {
- this.recordingOutputStream.startDigest();
- }
-
- /**
- * Convenience method for setting SHA1 digest.
- */
- public void setSha1Digest() {
- this.recordingOutputStream.setSha1Digest();
- }
-
- /**
- * Sets a digest algorithm which may be applied to recorded data.
- * As usually only a subset of the recorded data should
- * be fed to the digest, you must also call startDigest()
- * to begin digesting.
- *
- * @param algorithm
- */
- public void setDigest(String algorithm) {
- this.recordingOutputStream.setDigest(algorithm);
- }
-
- /**
- * Sets a digest function which may be applied to recorded data.
- * As usually only a subset of the recorded data should
- * be fed to the digest, you must also call startDigest()
- * to begin digesting.
- *
- * @param md
- */
- public void setDigest(MessageDigest md) {
- this.recordingOutputStream.setDigest(md);
- }
-
- /**
- * Return the digest value for any recorded, digested data. Call
- * only after all data has been recorded; otherwise, the running
- * digest state is ruined.
- *
- * @return the digest final value
- */
- public byte[] getDigestValue() {
- return this.recordingOutputStream.getDigestValue();
- }
-
- public long getResponseContentLength() {
- return this.recordingOutputStream.getResponseContentLength();
- }
-
- public void closeRecorder() throws IOException {
- this.recordingOutputStream.closeRecorder();
- }
-
- /**
- * @return True if we've been opened.
- */
- public boolean isOpen()
- {
- return this.in != null;
- }
-
- @Override
- public synchronized void mark(int readlimit) {
- this.in.mark(readlimit);
- this.recordingOutputStream.mark();
- }
-
- @Override
- public boolean markSupported() {
- return this.in.markSupported();
- }
-
- @Override
- public synchronized void reset() throws IOException {
- this.in.reset();
- this.recordingOutputStream.reset();
- }
-
- /**
- * Set limits to be enforced by internal recording-out
- */
- public void setLimits(long hardMax, long timeoutMs, long maxRateKBps) {
- recordingOutputStream.setLimits(hardMax, timeoutMs, maxRateKBps);
- }
-
- /**
- * Expose the amount of in-memory buffering used by the internal
- * recording stream.
- * @return int buffer size
- */
- public int getRecordedBufferLength() {
- return recordingOutputStream.getBufferLength();
- }
-
- /**
- * See doc on {@link RecordingOutputStream#chopAtMessageBodyBegin()}
- */
- public void chopAtMessageBodyBegin() {
- recordingOutputStream.chopAtMessageBodyBegin();
- }
-}
diff --git a/commons/src/main/java/org/archive/io/RecordingOutputStream.java b/commons/src/main/java/org/archive/io/RecordingOutputStream.java
deleted file mode 100644
index dfec4f33..00000000
--- a/commons/src/main/java/org/archive/io/RecordingOutputStream.java
+++ /dev/null
@@ -1,628 +0,0 @@
-/*
- * This file is part of the Heritrix web crawler (crawler.archive.org).
- *
- * Licensed to the Internet Archive (IA) by one or more individual
- * contributors.
- *
- * The IA licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.archive.io;
-
-import it.unimi.dsi.fastutil.io.FastBufferedOutputStream;
-
-import java.io.FileOutputStream;
-import java.io.IOException;
-import java.io.OutputStream;
-import java.security.MessageDigest;
-import java.security.NoSuchAlgorithmException;
-import java.util.logging.Level;
-import java.util.logging.Logger;
-
-
-/**
- * An output stream that records all writes to wrapped output
- * stream.
- *
- * A RecordingOutputStream can be wrapped around any other
- * OutputStream to record all bytes written to it. You can
- * then request a ReplayInputStream to read those bytes.
- *
- * null for the stream
- * to wrap. TODO: Make a FileBackedOutputStream class that is
- * subclassed by RecordingInputStream.
- *
- * @author gojomo
- *
- */
-public class RecordingOutputStream extends OutputStream {
- protected static Logger logger =
- Logger.getLogger(RecordingOutputStream.class.getName());
-
- /**
- * Size of recording.
- *
- * Later passed to ReplayInputStream on creation. It uses it to know when
- * EOS.
- */
- protected long size = 0;
-
- protected String backingFilename;
- protected OutputStream diskStream = null;
-
- /**
- * Buffer we write recordings to.
- *
- * We write all recordings here first till its full. Thereafter we
- * write the backing file.
- */
- private byte[] buffer;
-
- /** current virtual position in the recording */
- long position;
-
- /** flag to disable recording */
- private boolean recording;
-
- /**
- * Reusable buffer for FastBufferedOutputStream
- */
- protected byte[] bufStreamBuf =
- new byte [ FastBufferedOutputStream.DEFAULT_BUFFER_SIZE ];
-
- /**
- * True if we're to digest content.
- */
- private boolean shouldDigest = false;
-
- /**
- * Digest instance.
- */
- private MessageDigest digest = null;
-
- /**
- * Define for SHA1 algarithm.
- */
- private static final String SHA1 = "SHA1";
-
- /**
- * Maximum amount of header material to accept without the content
- * body beginning -- if more, throw a RecorderTooMuchHeaderException.
- * TODO: make configurable? make smaller?
- */
- protected static final long MAX_HEADER_MATERIAL = 1024*1024; // 1MB
-
- // configurable max length, max time limits
- /** maximum length of material to record before throwing exception */
- protected long maxLength = Long.MAX_VALUE;
- /** maximum time to record before throwing exception */
- protected long timeoutMs = Long.MAX_VALUE;
- /** maximum rate to record (adds delays to hit target rate) */
- protected long maxRateBytesPerMs = Long.MAX_VALUE;
- /** time recording begins for timeout, rate calculations */
- protected long startTime = Long.MAX_VALUE;
-
- /**
- * When recording HTTP, where the content-body starts.
- */
- protected long messageBodyBeginMark;
-
- /**
- * While messageBodyBeginMark is not set, the last two bytes seen.
- *
- * size > than buffer then we go to backing file to read
- * data that is beyond buffer.length.
- *
- * @throws IOException If we fail to open an input stream on
- * backing file.
- */
- public ReplayInputStream(byte[] buffer, long size, long responseBodyStart,
- String backingFilename)
- throws IOException
- {
- this(buffer, size, backingFilename);
- this.responseBodyStart = responseBodyStart;
- }
-
- /**
- * Constructor.
- *
- * @param buffer Buffer to read from.
- * @param size Size of data to replay.
- * @param backingFilename Backing file that sits behind the buffer. If
- * size > than buffer then we go to backing file to read
- * data that is beyond buffer.length.
- * @throws IOException If we fail to open an input stream on
- * backing file.
- */
- public ReplayInputStream(byte[] buffer, long size, String backingFilename)
- throws IOException
- {
- this.buffer = buffer;
- this.size = size;
- if (size > buffer.length) {
- setupDiskStream(new File(backingFilename));
- }
- }
-
- protected void setupDiskStream(File backingFile) throws IOException {
- RandomAccessInputStream rais = new RandomAccessInputStream(backingFile);
- diskStream = new BufferedSeekInputStream(rais, 4096);
- }
-
- protected File backingFile;
-
- /**
- * Create a ReplayInputStream from the given source stream. Requires
- * reading the entire stream (and possibly overflowing to a temporary
- * file). Primary reason for doing so would be to have a repositionable
- * version of the original stream's contents.
- *
- * If created via this constructor, use the destroy() method to ensure
- * prompt deletion of any associated tmp file when done.
- *
- * @param fillStream
- * @throws IOException
- */
- public ReplayInputStream(InputStream fillStream) throws IOException {
- this.buffer = new byte[DEFAULT_BUFFER_SIZE];
- long count = ArchiveUtils.readFully(fillStream, buffer);
- if(fillStream.available()>0) {
- this.backingFile = File.createTempFile("tid"+Thread.currentThread().getId(), "ris");
- count += FileUtils.readFullyToFile(fillStream, backingFile);
- setupDiskStream(backingFile);
- }
- this.size = count;
- }
-
- /**
- * Close & destroy any internally-generated temporary files.
- */
- public void destroy() {
- IOUtils.closeQuietly(this);
- if(backingFile!=null) {
- FileUtils.deleteSoonerOrLater(backingFile);
- }
- }
-
- public long setToResponseBodyStart() throws IOException {
- position(responseBodyStart);
- return this.position;
- }
-
-
- /* (non-Javadoc)
- * @see java.io.InputStream#read()
- */
- public int read() throws IOException {
- if (position == size) {
- return -1; // EOF
- }
- if (position < buffer.length) {
- // Convert to unsigned int.
- int c = buffer[(int) position] & 0xFF;
- position++;
- return c;
- }
- int c = diskStream.read();
- if (c >= 0) {
- position++;
- }
- return c;
- }
-
- /*
- * (non-Javadoc)
- *
- * @see java.io.InputStream#read(byte[], int, int)
- */
- public int read(byte[] b, int off, int len) throws IOException {
- if (position == size) {
- return -1; // EOF
- }
- if (position < buffer.length) {
- int toCopy = (int)Math.min(size - position,
- Math.min(len, buffer.length - position));
- System.arraycopy(buffer, (int)position, b, off, toCopy);
- if (toCopy > 0) {
- position += toCopy;
- }
- return toCopy;
- }
- // into disk zone
- int read = diskStream.read(b,off,len);
- if(read>0) {
- position += read;
- }
- return read;
- }
-
- public void readFullyTo(OutputStream os) throws IOException {
- readFullyTo(this, os);
- }
-
- public static void readFullyTo(InputStream in, OutputStream os) throws IOException {
- byte[] buf = new byte[4096];
- int c = in.read(buf);
- while (c != -1) {
- os.write(buf,0,c);
- c = in.read(buf);
- }
- }
-
- /*
- * Like 'readFullyTo', but only reads the header-part.
- * Starts from the beginning each time it is called.
- */
- public void readHeaderTo(OutputStream os) throws IOException {
- position = 0;
- byte[] buf = new byte[(int)responseBodyStart];
- int c = read(buf,0,buf.length);
- if(c != -1) {
- os.write(buf,0,c);
- }
- }
-
- /*
- * Like 'readFullyTo', but only reads the content-part.
- */
- public void readContentTo(OutputStream os) throws IOException {
- setToResponseBodyStart();
- readFullyTo(os);
- }
-
- /**
- * Convenience method to copy content out to target stream.
- * @param os stream to write content to
- * @param maxSize maximum count of bytes to copy
- * @throws IOException
- */
- public void readContentTo(OutputStream os, long maxSize) throws IOException {
- setToResponseBodyStart();
- byte[] buf = new byte[4096];
- int c = read(buf);
- long tot = 0;
- while (c != -1 && tot < maxSize) {
- os.write(buf,0,c);
- c = read(buf);
- tot += c;
- }
- }
-
- /* (non-Javadoc)
- * @see java.io.InputStream#close()
- */
- public void close() throws IOException {
- super.close();
- if(diskStream != null) {
- diskStream.close();
- }
- }
-
- /**
- * Total size of stream content.
- * @return Returns the size.
- */
- public long getSize()
- {
- return size;
- }
-
- /**
- * Total size of header.
- * @return the size of the header.
- */
- public long getHeaderSize()
- {
- return responseBodyStart;
- }
-
- /**
- * Total size of content.
- * @return the size of the content.
- */
- public long getContentSize()
- {
- return size - responseBodyStart;
- }
-
- /**
- * @return Amount THEORETICALLY remaining (TODO: Its not theoretical
- * seemingly. The class implemetentation depends on it being exact).
- */
- public long remaining() {
- return size - position;
- }
-
-
- /**
- * Reposition the stream.
- *
- * @param p the new position for this stream
- * @throws IOException if an IO error occurs
- */
- public void position(long p) throws IOException {
- if (p < 0) {
- throw new IOException("Negative seek offset.");
- }
- if (p > size) {
- throw new IOException("Desired position exceeds size.");
- }
- if (p < buffer.length) {
- // Only seek file if necessary
- if (position > buffer.length) {
- diskStream.position(0);
- }
- } else {
- diskStream.position(p - buffer.length);
- }
- this.position = p;
- }
-
-
- public long position() throws IOException {
- return position;
- }
-
- protected byte[] getBuffer() {
- return buffer;
- }
-}
diff --git a/commons/src/main/java/org/archive/io/RepositionableInputStream.java b/commons/src/main/java/org/archive/io/RepositionableInputStream.java
deleted file mode 100644
index 6f885130..00000000
--- a/commons/src/main/java/org/archive/io/RepositionableInputStream.java
+++ /dev/null
@@ -1,133 +0,0 @@
-/*
- * This file is part of the Heritrix web crawler (crawler.archive.org).
- *
- * Licensed to the Internet Archive (IA) by one or more individual
- * contributors.
- *
- * The IA licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.archive.io;
-
-import it.unimi.dsi.fastutil.io.RepositionableStream;
-
-import java.io.BufferedInputStream;
-import java.io.IOException;
-import java.io.InputStream;
-
-/**
- * Wrapper around an {@link InputStream} to make a primitive Repositionable
- * stream. Uses a {@link BufferedInputStream}. Calls mark on every read so
- * we'll remember at least the last thing read (You can only backup on the
- * last thing read -- not last 2 or 3 things read). Used by
- * {@link GzippedInputStream} when reading streams over a network. Wraps a
- * HTTP, etc., stream so we can back it up if needs be after the
- * GZIP inflater has done a fill of its full buffer though it only needed
- * the first few bytes to finish decompressing the current GZIP member.
- *
- * out is connected to.
- * @param cmprs Compress the content written.
- * @param a14DigitDate If null, we'll write current time.
- * @throws IOException
- */
- protected WriterPoolMember(AtomicInteger serialNo,
- final OutputStream out, final File file,
- final WriterPoolSettings settings)
- throws IOException {
- this(serialNo, settings, null);
- this.countOut = (out instanceof MiserOutputStream)
- ? (MiserOutputStream)out
- : new MiserOutputStream(out, settings.getFrequentFlushes());
- this.out = this.countOut;
- this.f = file;
- }
-
- /**
- * Constructor.
- *
- * @param serialNo used to create unique filename sequences
- * @param dirs Where to drop files.
- * @param prefix File prefix to use.
- * @param cmprs Compress the records written.
- * @param maxSize Maximum size for ARC files written.
- * @param template filenaming template to use
- * @param extension Extension to give file.
- */
- public WriterPoolMember(AtomicInteger serialNo,
- final WriterPoolSettings settings, final String extension) {
- this.settings = settings;
- this.extension = extension;
- this.serialNo = serialNo;
- }
-
- /**
- * Call this method just before/after any significant write.
- *
- * Call at the end of the writing of a record or just before we start
- * writing a new record. Will close current file and open a new file
- * if file size has passed out maxSize.
- *
- * alexa/include/a_arcio.h:
- *
- * #define LINE_LENGTH (100*1024)
- *
- */
- public static final int MAX_HEADER_LINE_LENGTH = 1024 * 100;
-
- /**
- * Version 1 required metadata fields.
- */
- public static ListARCRecord
- * though {@link Iterator#next()} is returning
- * java.lang.Object. Cast the return.
- *
- *
- * usage: java org.archive.io.arc.ARCReader [--offset=#] ARCFILE
- * -h,--help Prints this message and exits.
- * -o,--offset Outputs record at this offset into arc file.
- *
- * $HERITRIX_HOME/bin/arcreader for a script that'll
- * take care of classpaths and the calling of ARCReader.
- *
- * false to open ARCs
- * with the .open or otherwise suffix.
- * @param offset Have returned ARCReader set to start reading at passed
- * offset.
- * @return An ARCReader.
- * @throws IOException
- */
- public static ARCReader get(final File f,
- final boolean skipSuffixTest, final long offset)
- throws IOException {
- return (ARCReader)ARCReaderFactory.factory.getArchiveReader(f,
- skipSuffixTest, offset);
- }
-
- protected ArchiveReader getArchiveReader(final File arcFile,
- final boolean skipSuffixTest, final long offset)
- throws IOException {
- boolean compressed = testCompressedARCFile(arcFile, skipSuffixTest);
- if (!compressed) {
- if (!FileUtils.isReadableWithExtensionAndMagic(arcFile,
- ARC_FILE_EXTENSION, ARC_MAGIC_NUMBER)) {
- throw new IOException(arcFile.getAbsolutePath() +
- " is not an Internet Archive ARC file.");
- }
- }
- return compressed?
- (ARCReader)ARCReaderFactory.factory.
- new CompressedARCReader(arcFile, offset):
- (ARCReader)ARCReaderFactory.factory.
- new UncompressedARCReader(arcFile, offset);
- }
-
- public static ArchiveReader get(final String s, final InputStream is,
- final boolean atFirstRecord)
- throws IOException {
- return ARCReaderFactory.factory.getArchiveReader(s, is,
- atFirstRecord);
- }
-
- protected ArchiveReader getArchiveReader(final String arc,
- final InputStream is, final boolean atFirstRecord)
- throws IOException {
-
- // We do this mark() reset() stuff, wrapping in a BufferedInputStream if
- // necessary to make it work, because testCompressedARCStream() consumes
- // some bytes from the input stream
- InputStream possiblyWrapped;
- if (is.markSupported()) {
- possiblyWrapped = is;
- } else {
- possiblyWrapped = new BufferedInputStream(is);
- }
-
- possiblyWrapped.mark(100);
- boolean compressed = testCompressedARCStream(possiblyWrapped);
- possiblyWrapped.reset();
-
- if (compressed) {
- return new CompressedARCReader(arc, possiblyWrapped, atFirstRecord);
- } else {
- return new UncompressedARCReader(arc, possiblyWrapped);
- }
- }
-
- /**
- * Get an ARCReader aligned at offset. This version of get
- * will not bring the ARC local but will try to stream across the net making
- * an HTTP 1.1 Range request on remote http server (RFC1435 Section 14.35).
- *
- * @param arcUrl HTTP URL for an ARC (All ARCs considered remote).
- * @param offset Offset into ARC at which to start fetching.
- * @return An ARCReader aligned at offset.
- * @throws IOException
- */
- public static ARCReader get(final URL arcUrl, final long offset)
- throws IOException {
- return (ARCReader)ARCReaderFactory.factory.getArchiveReader(arcUrl,
- offset);
- }
-
- /**
- * Get an ARCReader.
- * Pulls the ARC local into whereever the System Property
- * java.io.tmpdir points. It then hands back an ARCReader that
- * points at this local copy. A close on this ARCReader instance will
- * remove the local copy.
- * @param arcUrl An URL that points at an ARC.
- * @return An ARCReader.
- * @throws IOException
- */
- public static ARCReader get(final URL arcUrl)
- throws IOException {
- return (ARCReader)ARCReaderFactory.factory.getArchiveReader(arcUrl);
- }
-
- /**
- * @param arcFile File to test.
- * @return True if arcFile is compressed ARC.
- * @throws IOException
- */
- public boolean isCompressed(File arcFile) throws IOException {
- return testCompressedARCFile(arcFile);
- }
-
- /**
- * Check file is compressed and in ARC GZIP format.
- *
- * @param arcFile File to test if its Internet Archive ARC file
- * GZIP compressed.
- *
- * @return True if this is an Internet Archive GZIP'd ARC file (It begins
- * w/ the Internet Archive GZIP header and has the
- * COMPRESSED_ARC_FILE_EXTENSION suffix).
- *
- * @exception IOException If file does not exist or is not unreadable.
- */
- public static boolean testCompressedARCFile(File arcFile)
- throws IOException {
- return testCompressedARCFile(arcFile, false);
- }
-
- /**
- * Check file is compressed and in ARC GZIP format.
- *
- * @param arcFile File to test if its Internet Archive ARC file
- * GZIP compressed.
- * @param skipSuffixCheck Set to true if we're not to test on the
- * '.arc.gz' suffix.
- *
- * @return True if this is an Internet Archive GZIP'd ARC file (It begins
- * w/ the Internet Archive GZIP header).
- *
- * @exception IOException If file does not exist or is not unreadable.
- */
- public static boolean testCompressedARCFile(File arcFile,
- boolean skipSuffixCheck)
- throws IOException {
- boolean compressedARCFile = false;
- FileUtils.assertReadable(arcFile);
- if(!skipSuffixCheck && !arcFile.getName().toLowerCase()
- .endsWith(COMPRESSED_ARC_FILE_EXTENSION)) {
- return compressedARCFile;
- }
-
- final InputStream is = new FileInputStream(arcFile);
- try {
- compressedARCFile = testCompressedARCStream(is);
- } finally {
- is.close();
- }
- return compressedARCFile;
- }
-
- public static boolean isARCSuffix(final String arcName) {
- return (arcName == null)?
- false:
- (arcName.toLowerCase().endsWith(DOT_COMPRESSED_ARC_FILE_EXTENSION))?
- true:
- (arcName.toLowerCase().endsWith(DOT_ARC_FILE_EXTENSION))?
- true: false;
- }
-
- /**
- * Tests passed stream is gzip stream by reading in the HEAD.
- * Does not reposition the stream. That is left up to the caller.
- * @param is An InputStream.
- * @return True if compressed stream.
- * @throws IOException
- */
- public static boolean testCompressedARCStream(final InputStream is)
- throws IOException {
- boolean compressedARCFile = false;
- GzipHeader gh = null;
- try {
- gh = new GzipHeader(is);
- } catch (NoGzipMagicException e) {
- return false;
- }
-
- byte[] fextra = gh.getFextra();
- // Now make sure following bytes are IA GZIP comment.
- // First check length. ARC_GZIP_EXTRA_FIELD includes length
- // so subtract two and start compare to ARC_GZIP_EXTRA_FIELD
- // at +2.
- // some Alexa ARC files gzip extra fields have changed slightly
- // after the first two bytes, so we'll just look for the 'LX'
- // extension for valid IA ARC files.
- if (fextra != null) {
- if (fextra.length >= ARC_GZIP_EXTRA_FIELD.length - 2) {
- if (fextra[0] == ARC_GZIP_EXTRA_FIELD[2] &&
- fextra[1] == ARC_GZIP_EXTRA_FIELD[3]) {
- compressedARCFile = true;
- }
- }
- } else {
- // Some old arcs don't have an extra header at all, but they're still compressed
- compressedARCFile = true;
- }
-
- return compressedARCFile;
- }
-
- /**
- * Uncompressed arc file reader.
- * @author stack
- */
- public class UncompressedARCReader extends ARCReader {
- /**
- * Constructor.
- * @param f Uncompressed arcfile to read.
- * @throws IOException
- */
- public UncompressedARCReader(final File f)
- throws IOException {
- this(f, 0);
- }
-
- /**
- * Constructor.
- *
- * @param f Uncompressed arcfile to read.
- * @param offset Offset at which to position ARCReader.
- * @throws IOException
- */
- public UncompressedARCReader(final File f, final long offset)
- throws IOException {
- // Arc file has been tested for existence by time it has come
- // to here.
- setIn(new CountingInputStream(getInputStream(f, offset)));
- getIn().skip(offset);
- initialize(f.getAbsolutePath());
- }
-
- /**
- * Constructor.
- *
- * @param f Uncompressed arc to read.
- * @param is InputStream.
- */
- public UncompressedARCReader(final String f, final InputStream is) {
- // Arc file has been tested for existence by time it has come
- // to here.
- setIn(new CountingInputStream(is));
- initialize(f);
- }
- }
-
- /**
- * Compressed arc file reader.
- *
- * @author stack
- */
- public class CompressedARCReader extends ARCReader {
-
- /**
- * Constructor.
- *
- * @param f
- * Compressed arcfile to read.
- * @throws IOException
- */
- public CompressedARCReader(final File f) throws IOException {
- this(f, 0);
- }
-
- /**
- * Constructor.
- *
- * @param f Compressed arcfile to read.
- * @param offset Position at where to start reading file.
- * @throws IOException
- */
- public CompressedARCReader(final File f, final long offset)
- throws IOException {
- // Arc file has been tested for existence by time it has come
- // to here.
- setIn(new GZIPMembersInputStream(getInputStream(f, offset)));
- ((GZIPMembersInputStream)getIn()).compressedSeek(offset);
- setCompressed((offset == 0)); // TODO: does this make sense???
- initialize(f.getAbsolutePath());
- }
-
- /**
- * Constructor.
- *
- * @param f Compressed arcfile.
- * @param is InputStream to use.
- * @throws IOException
- */
- public CompressedARCReader(final String f, final InputStream is,
- final boolean atFirstRecord)
- throws IOException {
- // Arc file has been tested for existence by time it has come
- // to here.
- setIn(new GZIPMembersInputStream(is));
- setCompressed(true);
- setAlignedOnFirstRecord(atFirstRecord);
- initialize(f);
- }
-
- /**
- * Get record at passed offset.
- *
- * @param offset
- * Byte index into arcfile at which a record starts.
- * @return An ARCRecord reference.
- * @throws IOException
- */
- public ARCRecord get(long offset) throws IOException {
- cleanupCurrentRecord();
- ((GZIPMembersInputStream)getIn()).compressedSeek(offset);
- return createArchiveRecord(getIn(), offset);
- }
-
- public IteratorheaderFieldNameKeys.
- */
- private final String [] headerFieldNameKeysArray = {
- URL_FIELD_KEY,
- IP_HEADER_FIELD_KEY,
- DATE_FIELD_KEY,
- MIMETYPE_FIELD_KEY,
- LENGTH_FIELD_KEY
- };
-
- /**
- * An array of the header field names found in the ARC file header on
- * the 3rd line.
- *
- * We used to read these in from the arc file first record 3rd line but
- * now we hardcode them for sake of improved performance.
- */
- private final Listin (Used to keep
- * position properly aligned). Usually 0.
- * @param digest True if we're to calculate digest for this record. Not
- * digesting saves about ~15% of cpu during an ARC parse.
- * @param strict Be strict parsing (Parsing stops if ARC inproperly
- * formatted).
- * @param parseHttpHeaders True if we are to parse HTTP headers. Costs
- * about ~20% of CPU during an ARC parse.
- * @param isAllignedOnFirstRecord True if this is the first record to be
- * read from an archive
- * @param String version Version information to be returned to the
- * ARCReader constructing this record
- *
- * @throws IOException
- */
- public ARCRecord(InputStream in, final String identifier,
- final long offset, boolean digest, boolean strict,
- final boolean parseHttpHeaders,
- final boolean isAlignedOnFirstRecord, String version)
- throws IOException {
- super(in, null, 0, digest, strict);
- setHeader(parseHeaders(in, identifier, offset, strict, isAlignedOnFirstRecord, version));
- if (parseHttpHeaders) {
- this.httpHeaderStream = readHttpHeader();
- }
- }
-
- /**
- * Constructor.
- *
- * @param in Stream cue'd up to be at the start of the records metadata
- * this instance is to represent.
- * @param identifier Identifier for this the hosting Reader.
- * @param offset Current offset into in (Used to keep
- * position properly aligned). Usually 0.
- * @param digest True if we're to calculate digest for this record. Not
- * digesting saves about ~15% of cpu during an ARC parse.
- * @param strict Be strict parsing (Parsing stops if ARC inproperly
- * formatted).
- * @param parseHttpHeaders True if we are to parse HTTP headers. Costs
- * about ~20% of CPU during an ARC parse.
- *
- * @throws IOException
- */
- public ARCRecord(InputStream in, final String identifier,
- final long offset, boolean digest, boolean strict,
- final boolean parseHttpHeaders)
- throws IOException {
- this(in, identifier, offset, digest, strict, parseHttpHeaders,
- false, null);
- }
-
- private ArchiveRecordHeader parseHeaders(final InputStream in,
- final String identifier, final long offset, final boolean strict,
- final boolean isAlignedOnFirstRecord, String version)
- throws IOException {
-
- ArrayListarcFile is compressed ARC.
- * @throws IOException
- */
- public static boolean isCompressed(File arcFile) throws IOException {
- return testCompressedARCFile(arcFile);
- }
-
- /**
- * Check file is compressed and in ARC GZIP format.
- *
- * @param arcFile File to test if its Internet Archive ARC file
- * GZIP compressed.
- *
- * @return True if this is an Internet Archive GZIP'd ARC file (It begins
- * w/ the Internet Archive GZIP header and has the
- * COMPRESSED_ARC_FILE_EXTENSION suffix).
- *
- * @exception IOException If file does not exist or is not unreadable.
- */
- public static boolean testCompressedARCFile(File arcFile)
- throws IOException {
- return testCompressedARCFile(arcFile, false);
- }
-
- /**
- * Check file is compressed and in ARC GZIP format.
- *
- * @param arcFile File to test if its Internet Archive ARC file
- * GZIP compressed.
- * @param skipSuffixCheck Set to true if we're not to test on the
- * '.arc.gz' suffix.
- *
- * @return True if this is an Internet Archive GZIP'd ARC file (It begins
- * w/ the Internet Archive GZIP header).
- *
- * @exception IOException If file does not exist or is not unreadable.
- */
- public static boolean testCompressedARCFile(File arcFile,
- boolean skipSuffixCheck)
- throws IOException {
- boolean compressedARCFile = false;
- isReadable(arcFile);
- if(!skipSuffixCheck && !arcFile.getName().toLowerCase()
- .endsWith(COMPRESSED_ARC_FILE_EXTENSION)) {
- return compressedARCFile;
- }
-
- final InputStream is = new FileInputStream(arcFile);
- try {
- compressedARCFile = testCompressedARCStream(is);
- } finally {
- is.close();
- }
- return compressedARCFile;
- }
-
- /**
- * Tests passed stream is gzip stream by reading in the HEAD.
- * Does not reposition the stream. That is left up to the caller.
- * @param is An InputStream.
- * @return True if compressed stream.
- * @throws IOException
- */
- public static boolean testCompressedARCStream(final InputStream is)
- throws IOException {
- boolean compressedARCFile = false;
- GzipHeader gh = null;
- try {
- gh = new GzipHeader(is);
- } catch (NoGzipMagicException e ) {
- return compressedARCFile;
- }
-
- byte[] fextra = gh.getFextra();
- // Now make sure following bytes are IA GZIP comment.
- // First check length. ARC_GZIP_EXTRA_FIELD includes length
- // so subtract two and start compare to ARC_GZIP_EXTRA_FIELD
- // at +2.
- if (fextra != null &&
- ARC_GZIP_EXTRA_FIELD.length - 2 == fextra.length) {
- compressedARCFile = true;
- for (int i = 0; i < fextra.length; i++) {
- if (fextra[i] != ARC_GZIP_EXTRA_FIELD[i + 2]) {
- compressedARCFile = false;
- break;
- }
- }
- }
- return compressedARCFile;
- }
-
- /**
- * Tests passed stream is gzip stream by reading in the HEAD.
- * Does reposition of stream when done.
- * @param rs An InputStream that is Repositionable.
- * @return True if compressed stream.
- * @throws IOException
- */
- public static boolean testCompressedRepositionalStream(
- final RepositionableStream rs)
- throws IOException {
- boolean compressedARCFile = false;
- long p = rs.position();
- try {
- compressedARCFile = testCompressedStream((InputStream)rs);
- } finally {
- rs.position(p);
- }
- return compressedARCFile;
- }
-
- /**
- * Tests passed stream is gzip stream by reading in the HEAD.
- * Does reposition of stream when done.
- * @param is An InputStream.
- * @return True if compressed stream.
- * @throws IOException
- */
- public static boolean testCompressedStream(final InputStream is)
- throws IOException {
- boolean compressedARCFile = false;
- try {
- new GzipHeader(is);
- compressedARCFile = true;
- } catch (NoGzipMagicException e) {
- return compressedARCFile;
- }
- return compressedARCFile;
- }
-
- /**
- * Check file is uncompressed ARC file.
- *
- * @param arcFile
- * File to test if its Internet Archive ARC file uncompressed.
- *
- * @return True if this is an Internet Archive ARC file.
- *
- * @exception IOException
- * If file does not exist or is not unreadable.
- */
- public static boolean testUncompressedARCFile(File arcFile)
- throws IOException {
- boolean uncompressedARCFile = false;
- isReadable(arcFile);
- if(arcFile.getName().toLowerCase().endsWith(ARC_FILE_EXTENSION)) {
- FileInputStream fis = new FileInputStream(arcFile);
- try {
- byte [] b = new byte[ARC_MAGIC_NUMBER.length()];
- int read = fis.read(b, 0, ARC_MAGIC_NUMBER.length());
- fis.close();
- if (read == ARC_MAGIC_NUMBER.length()) {
- StringBuffer beginStr
- = new StringBuffer(ARC_MAGIC_NUMBER.length());
- for (int i = 0; i < ARC_MAGIC_NUMBER.length(); i++) {
- beginStr.append((char)b[i]);
- }
-
- if (beginStr.toString().
- equalsIgnoreCase(ARC_MAGIC_NUMBER)) {
- uncompressedARCFile = true;
- }
- }
- } finally {
- fis.close();
- }
- }
-
- return uncompressedARCFile;
- }
-
-
- /**
- * @param arcFile File to test.
- * @exception IOException If file does not exist or is not unreadable.
- */
- private static void isReadable(File arcFile) throws IOException {
- if (!arcFile.exists()) {
- throw new FileNotFoundException(arcFile.getAbsolutePath() +
- " does not exist.");
- }
-
- if (!arcFile.canRead()) {
- throw new FileNotFoundException(arcFile.getAbsolutePath() +
- " is not readable.");
- }
- }
-}
diff --git a/commons/src/main/java/org/archive/io/arc/ARCWriter.java b/commons/src/main/java/org/archive/io/arc/ARCWriter.java
deleted file mode 100644
index b5825d50..00000000
--- a/commons/src/main/java/org/archive/io/arc/ARCWriter.java
+++ /dev/null
@@ -1,459 +0,0 @@
-/*
- * This file is part of the Heritrix web crawler (crawler.archive.org).
- *
- * Licensed to the Internet Archive (IA) by one or more individual
- * contributors.
- *
- * The IA licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.archive.io.arc;
-
-import java.io.BufferedInputStream;
-import java.io.ByteArrayInputStream;
-import java.io.ByteArrayOutputStream;
-import java.io.Closeable;
-import java.io.File;
-import java.io.FileInputStream;
-import java.io.IOException;
-import java.io.InputStream;
-import java.io.PrintStream;
-import java.io.UnsupportedEncodingException;
-import java.util.Iterator;
-import java.util.concurrent.atomic.AtomicInteger;
-import java.util.logging.Logger;
-import java.util.regex.Matcher;
-import java.util.regex.Pattern;
-
-import org.archive.io.ReplayInputStream;
-import org.archive.io.WriterPoolMember;
-import org.archive.io.WriterPoolSettings;
-import org.archive.util.ArchiveUtils;
-import org.archive.util.DevUtils;
-import org.archive.util.MimetypeUtils;
-
-
-/**
- * Write ARC files.
- *
- * Assumption is that the caller is managing access to this ARCWriter ensuring
- * only one thread of control accessing this ARC file instance at any one time.
- *
- *
- * % av_procarc hx20040109230030-0.arc.gz | av_ziparc > \
- * /tmp/hx20040109230030-0.dat.gz
- * % av_ripdat /tmp/hx20040109230030-0.dat.gz > /tmp/hx20040109230030-0.cdx
- *
- * Examine the produced cdx file to make sure it makes sense. Search
- * for 'no-type 0'. If found, then we're opening a gzip record w/o data to
- * write. This is bad.
- *
- * gzip -t FILENAME and it will tell you if the
- * ARC makes sense to GZIP.
- *
- * out is connected to.
- * @param cmprs Compress the content written.
- * @param metadata File meta data. Can be null. Is list of File and/or
- * String objects.
- * @param a14DigitDate If null, we'll write current time.
- * @throws IOException
- */
- public ARCWriter(final AtomicInteger serialNo, final PrintStream out,
- final File arc, final WriterPoolSettings settings)
- throws IOException {
- super(serialNo, out, arc, settings);
- writeFirstRecord(ArchiveUtils.get14DigitDate());
- }
-
- /**
- * Constructor.
- *
- * @param serialNo used to generate unique file name sequences
- * @param settings all creation parameters
- */
- public ARCWriter(final AtomicInteger serialNo, final WriterPoolSettings settings) {
- super(serialNo, settings, ARC_FILE_EXTENSION);
-
- }
-
- protected String createFile()
- throws IOException {
- String name = super.createFile();
- writeFirstRecord(currentTimestamp);
- return name;
- }
-
- private void writeFirstRecord(final String ts)
- throws IOException {
- write(generateARCFileMetaData(ts));
- }
-
- /**
- * Write out the ARCMetaData.
- *
- * filedesc://testWriteRecord-JunitIAH20040110013326-2.arc 0.0.0.0 \\
- * 20040110013326 text/plain 77
- * 1 0 InternetArchive
- * URL IP-address Archive-date Content-type Archive-length
- *
- *
- * alexa/vista/alexa-tools-1.2/src/av_ziparc.cc.
- *
- * baos.
- *
- * @param baos Byte array to write to.
- * @throws UnsupportedEncodingException
- * @throws IOException
- */
- private void writeMetaData(ByteArrayOutputStream baos)
- throws UnsupportedEncodingException, IOException {
- if (settings.getMetadata() == null) {
- return;
- }
-
- for (Iterator
- * usage: java org.archive.io.arc.WARCReader [--offset=#] ARCFILE
- * -h,--help Prints this message and exits.
- * -o,--offset Outputs record at this offset into arc file.
- *
- * java.io.tmpdir points. It then hands back an ARCReader that
- * points at this local copy. A close on this ARCReader instance will
- * remove the local copy.
- * @param arcUrl An URL that points at an ARC.
- * @return An ARCReader.
- * @throws IOException
- */
- public static WARCReader get(final URL arcUrl)
- throws IOException {
- return (WARCReader)WARCReaderFactory.factory.getArchiveReader(arcUrl);
- }
-
- /**
- * Check file is compressed WARC.
- *
- * @param f File to test.
- *
- * @return True if this is compressed WARC (TODO: Just tests if file is
- * GZIP'd file (It begins w/ GZIP MAGIC)).
- *
- * @exception IOException If file does not exist or is not unreadable.
- */
- public static boolean testCompressedWARCFile(final File f)
- throws IOException {
- FileUtils.assertReadable(f);
- boolean compressed = false;
- final InputStream is = new FileInputStream(f);
- try {
- compressed = ArchiveUtils.isGzipped(is);
- } finally {
- is.close();
- }
- return compressed;
- }
-
- /**
- * Uncompressed WARC file reader.
- * @author stack
- */
- public class UncompressedWARCReader extends WARCReader {
- /**
- * Constructor.
- * @param f Uncompressed arcfile to read.
- * @throws IOException
- */
- public UncompressedWARCReader(final File f)
- throws IOException {
- this(f, 0);
- }
-
- /**
- * Constructor.
- *
- * @param f Uncompressed file to read.
- * @param offset Offset at which to position Reader.
- * @throws IOException
- */
- public UncompressedWARCReader(final File f, final long offset)
- throws IOException {
- // File has been tested for existence by time it has come to here.
- setIn(new CountingInputStream(getInputStream(f, offset)));
- getIn().skip(offset);
- initialize(f.getAbsolutePath());
- }
-
- /**
- * Constructor.
- *
- * @param f Uncompressed file to read.
- * @param is InputStream.
- */
- public UncompressedWARCReader(final String f, final InputStream is) {
- // Arc file has been tested for existence by time it has come
- // to here.
- setIn(new CountingInputStream(is));
- initialize(f);
- }
- }
-
- /**
- * Compressed WARC file reader.
- *
- * @author stack
- */
- public class CompressedWARCReader extends WARCReader {
- /**
- * Constructor.
- *
- * @param f Compressed file to read.
- * @throws IOException
- */
- public CompressedWARCReader(final File f) throws IOException {
- this(f, 0);
- }
-
- /**
- * Constructor.
- *
- * @param f Compressed arcfile to read.
- * @param offset Position at where to start reading file.
- * @throws IOException
- */
- public CompressedWARCReader(final File f, final long offset)
- throws IOException {
- // File has been tested for existence by time it has come to here.
- setIn(new GZIPMembersInputStream(getInputStream(f, offset)));
- ((GZIPMembersInputStream)getIn()).compressedSeek(offset);
- setCompressed((offset == 0)); // TODO: does this make sense?!?!
- initialize(f.getAbsolutePath());
- }
-
- /**
- * Constructor.
- *
- * @param f Compressed arcfile.
- * @param is InputStream to use.
- * @param atFirstRecord
- * @throws IOException
- */
- public CompressedWARCReader(final String f, final InputStream is,
- final boolean atFirstRecord)
- throws IOException {
- // Arc file has been tested for existence by time it has come
- // to here.
- setIn(new GZIPMembersInputStream(is));
- setCompressed(true);
- initialize(f);
- // TODO: Ignore atFirstRecord. Probably doesn't apply in WARC world.
- }
-
- /**
- * Get record at passed offset.
- *
- * @param offset Byte index into file at which a record starts.
- * @return A WARCRecord reference.
- * @throws IOException
- */
- public WARCRecord get(long offset) throws IOException {
- cleanupCurrentRecord();
- ((GZIPMembersInputStream)getIn()).compressedSeek(offset);
- return (WARCRecord) createArchiveRecord(getIn(), offset);
- }
-
- public Iteratorheaders is not null, just past the
- * Header Line and Named Fields.
- * @param identifier Identifier for this the hosting Reader.
- * @param offset Current offset into in (Used to keep
- * position properly aligned). Usually 0.
- * @param digest True if we're to calculate digest for this record. Not
- * digesting saves about ~15% of cpu during parse.
- * @param strict Be strict parsing (Parsing stops if file inproperly
- * formatted).
- * @throws IOException
- */
- public WARCRecord(final InputStream in, final String identifier,
- final long offset, boolean digest, boolean strict)
- throws IOException {
- super(in, null, 0, digest, strict);
- setHeader(parseHeaders(in, identifier, offset, strict));
- }
-
- /**
- * Parse WARC Header Line and Named Fields.
- * @param in Stream to read.
- * @param identifier Identifier for the hosting Reader.
- * @param offset Absolute offset into Reader.
- * @param strict Whether to be loose parsing or not.
- * @return An ArchiveRecordHeader.
- * @throws IOException
- */
- protected ArchiveRecordHeader parseHeaders(final InputStream in,
- final String identifier, final long offset, final boolean strict)
- throws IOException {
- final Mapout is connected to.
- * @param cmprs Compress the content written.
- * @param a14DigitDate If null, we'll write current time.
- * @throws IOException
- */
- public WARCWriter(final AtomicInteger serialNo,
- final OutputStream out, final File f,
- final WARCWriterPoolSettings settings)
- throws IOException {
- super(serialNo, out, f, settings);
- }
-
- /**
- * Constructor.
- *
- * @param dirs Where to drop files.
- * @param prefix File prefix to use.
- * @param cmprs Compress the records written.
- * @param maxSize Maximum size for ARC files written.
- * @param suffix File tail to use. If null, unused.
- * @param warcinfoData File metadata for warcinfo record.
- */
- public WARCWriter(final AtomicInteger serialNo,
- final WARCWriterPoolSettings settings) {
- super(serialNo, settings, WARC_FILE_EXTENSION);
- }
-
- @Override
- protected String createFile(File file) throws IOException {
- String filename = super.createFile(file);
- writeWarcinfoRecord(filename);
- return filename;
- }
-
- protected void baseCharacterCheck(final char c, final String parameter)
- throws IllegalArgumentException {
- // TODO: Too strict? UNICODE control characters?
- if (Character.isISOControl(c) || !Character.isValidCodePoint(c)) {
- throw new IllegalArgumentException("Contains illegal character 0x" +
- Integer.toHexString(c) + ": " + parameter);
- }
- }
-
- protected String checkHeaderValue(final String value)
- throws IllegalArgumentException {
- for (int i = 0; i < value.length(); i++) {
- final char c = value.charAt(i);
- baseCharacterCheck(c, value);
- if (Character.isWhitespace(c)) {
- throw new IllegalArgumentException("Contains disallowed white space 0x" +
- Integer.toHexString(c) + ": " + value);
- }
- }
- return value;
- }
-
- protected String checkHeaderLineMimetypeParameter(final String parameter)
- throws IllegalArgumentException {
- StringBuilder sb = new StringBuilder(parameter.length());
- boolean wasWhitespace = false;
- for (int i = 0; i < parameter.length(); i++) {
- char c = parameter.charAt(i);
- if (Character.isWhitespace(c)) {
- // Map all to ' ' and collapse multiples into one.
- // TODO: Make sure white space occurs in legal location --
- // before parameter or inside quoted-string.
- if (wasWhitespace) {
- continue;
- }
- wasWhitespace = true;
- c = ' ';
- } else {
- wasWhitespace = false;
- baseCharacterCheck(c, parameter);
- }
- sb.append(c);
- }
-
- return sb.toString();
- }
-
-// protected String createRecordHeader(final String type,
-// final String url, final String create14DigitDate,
-// final String mimetype, final URI recordId,
-// final ANVLRecord xtraHeaders, final long contentLength)
- protected String createRecordHeader(WARCRecordInfo metaRecord)
- throws IllegalArgumentException {
- final StringBuilder sb =
- new StringBuilder(2048/*A SWAG: TODO: Do analysis.*/);
- sb.append(WARC_ID).append(CRLF);
- sb.append(HEADER_KEY_TYPE).append(COLON_SPACE).append(metaRecord.getType()).
- append(CRLF);
- // Do not write a subject-uri if not one present.
- if (!StringUtils.isEmpty(metaRecord.getUrl())) {
- sb.append(HEADER_KEY_URI).append(COLON_SPACE).
- append(checkHeaderValue(metaRecord.getUrl())).append(CRLF);
- }
- sb.append(HEADER_KEY_DATE).append(COLON_SPACE).
- append(metaRecord.getCreate14DigitDate()).append(CRLF);
- if (metaRecord.getExtraHeaders() != null) {
- for (final IteratorImplementation Notes
-Tools
-Arc2Warc and Warc2Arc
-tools can be found in the package above this one, at
-{@link org.archive.io.Arc2Warc} and {@link org.archive.io.Warc2Arc}
-respectively. Pass --help to learn how to use each tool.
-TODO
-
-
-
-
-
diff --git a/commons/src/main/java/org/archive/net/DownloadURLConnection.java b/commons/src/main/java/org/archive/net/DownloadURLConnection.java
deleted file mode 100644
index fbcee421..00000000
--- a/commons/src/main/java/org/archive/net/DownloadURLConnection.java
+++ /dev/null
@@ -1,131 +0,0 @@
-/*
- * This file is part of the Heritrix web crawler (crawler.archive.org).
- *
- * Licensed to the Internet Archive (IA) by one or more individual
- * contributors.
- *
- * The IA licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.archive.net;
-
-import java.io.BufferedInputStream;
-import java.io.File;
-import java.io.FileInputStream;
-import java.io.IOException;
-import java.io.InputStream;
-import java.net.URL;
-import java.net.URLConnection;
-import java.util.Arrays;
-import java.util.logging.Level;
-import java.util.logging.Logger;
-
-import org.archive.util.ProcessUtils;
-import org.archive.util.ProcessUtils.ProcessResult;
-
-/**
- * An URL Connection that pre-downloads URL reference before passing back a
- * Stream reference. When closed, it removes the local download file.
- * @author stack
- * @version $Date$, $Revision$
- */
-public abstract class DownloadURLConnection extends URLConnection {
- private final String CLASSNAME = DownloadURLConnection.class.getName();
- private final Logger LOGGER = Logger.getLogger(CLASSNAME);
- private static final File TMPDIR =
- new File(System.getProperty("java.io.tmpdir", "/tmp"));
- private File downloadFile = null;
-
- protected DownloadURLConnection(URL u) {
- super(u);
- }
-
- protected String getScript() {
- return System.getProperty(this.getClass().getName() + ".path",
- "UNDEFINED");
- }
-
- protected String [] getCommand(final URL thisUrl,
- final File downloadFile) {
- return new String[] {getScript(), thisUrl.getPath(),
- downloadFile.getAbsolutePath()};
- }
-
- /**
- * Do script copy to local file.
- * File is available via {@link #getFile()}.
- * @throws IOException
- */
- public void connect() throws IOException {
- if (this.connected) {
- return;
- }
-
- this.downloadFile = File.createTempFile(CLASSNAME, null, TMPDIR);
- try {
- String [] cmd = getCommand(this.url, this.downloadFile);
- if (LOGGER.isLoggable(Level.FINE)) {
- StringBuffer buffer = new StringBuffer();
- for (int i = 0; i < cmd.length; i++) {
- if (i > 0) {
- buffer.append(" ");
- }
- buffer.append(cmd[i]);
- }
- LOGGER.fine("Command: " + buffer.toString());
- }
- ProcessResult pr = ProcessUtils.exec(cmd);
- if (pr.getResult() != 0) {
- LOGGER.info(Arrays.toString(cmd) + " returned non-null " + pr.getResult());
- }
- // Assume download went smoothly.
- this.connected = true;
- } catch (IOException ioe) {
- // Clean up my tmp file.
- this.downloadFile.delete();
- this.downloadFile = null;
- // Rethrow.
- throw ioe;
- }
- }
-
- public File getFile() {
- return this.downloadFile;
- }
-
- protected void setFile(final File f) {
- this.downloadFile = f;
- }
-
- public InputStream getInputStream() throws IOException {
- if (!this.connected) {
- connect();
- }
-
- // Return BufferedInputStream so 'delegation' is done for me, so
- // I don't have to implement all IS methods and pass to my
- // 'delegate' instance.
- final DownloadURLConnection connection = this;
- return new BufferedInputStream(new FileInputStream(this.downloadFile)) {
- private DownloadURLConnection ruc = connection;
-
- public void close() throws IOException {
- super.close();
- if (this.ruc != null && this.ruc.getFile()!= null &&
- this.ruc.getFile().exists()) {
- this.ruc.getFile().delete();
- this.ruc.setFile(null);
- }
- }
- };
- }
-}
\ No newline at end of file
diff --git a/commons/src/main/java/org/archive/net/FTPException.java b/commons/src/main/java/org/archive/net/FTPException.java
deleted file mode 100644
index 2d104390..00000000
--- a/commons/src/main/java/org/archive/net/FTPException.java
+++ /dev/null
@@ -1,56 +0,0 @@
-/*
- * This file is part of the Heritrix web crawler (crawler.archive.org).
- *
- * Licensed to the Internet Archive (IA) by one or more individual
- * contributors.
- *
- * The IA licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.archive.net;
-
-import java.io.IOException;
-
-/**
- * Indicates that a FTP operation failed due to a protocol violation.
- * For instance, if authentication fails.
- *
- * @author pjack
- */
-public class FTPException extends IOException {
- private static final long serialVersionUID = 1L;
-
- /**
- * The reply code from the FTP server.
- */
- private int code;
-
- /**
- * Constructs a new FTPException.
- *
- * @param code the error code from the FTP server
- */
- public FTPException(int code) {
- super("FTP error code: " + code);
- this.code = code;
- }
-
-
- /**
- * Returns the error code from the FTP server.
- *
- * @return the error code from the FTP server
- */
- public int getReplyCode() {
- return code;
- }
-}
diff --git a/commons/src/main/java/org/archive/net/PublicSuffixes.java b/commons/src/main/java/org/archive/net/PublicSuffixes.java
deleted file mode 100644
index eab8081a..00000000
--- a/commons/src/main/java/org/archive/net/PublicSuffixes.java
+++ /dev/null
@@ -1,363 +0,0 @@
-/*
- * This file is part of the Heritrix web crawler (crawler.archive.org).
- *
- * Licensed to the Internet Archive (IA) by one or more individual
- * contributors.
- *
- * The IA licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.archive.net;
-
-import java.io.BufferedReader;
-import java.io.BufferedWriter;
-import java.io.FileInputStream;
-import java.io.FileWriter;
-import java.io.IOException;
-import java.io.InputStream;
-import java.io.InputStreamReader;
-import java.io.OutputStreamWriter;
-import java.io.PrintWriter;
-import java.io.UnsupportedEncodingException;
-import java.util.ArrayList;
-import java.util.List;
-import java.util.regex.Matcher;
-import java.util.regex.Pattern;
-
-import org.apache.commons.io.IOUtils;
-import org.archive.util.TextUtils;
-
-/**
- * Utility class for making use of the information about 'public suffixes' at
- * http://publicsuffix.org.
- *
- * The public suffix list (once known as 'effective TLDs') was motivated by the
- * need to decide on which broader domains a subdomain was allowed to set
- * cookies. For example, a server at 'www.example.com' can set cookies for
- * 'www.example.com' or 'example.com' but not 'com'. 'www.example.co.uk' can set
- * cookies for 'www.example.co.uk' or 'example.co.uk' but not 'co.uk' or 'uk'.
- * The number of rules for all top-level-domains and 2nd- or 3rd- level domains
- * has become quite long; essentially the broadest domain a subdomain may assign
- * to is the one that was sold/registered to a specific name registrant.
- *
- * This concept should be useful in other contexts, too. Grouping URIs (or
- * queues of URIs to crawl) together with others sharing the same registered
- * suffix may be useful for applying the same rules to all, such as assigning
- * them to the same queue or crawler in a multi- machine setup.
- *
- * As of Heritrix3, we prefer the term 'Assignment Level Domain' (ALD)
- * for such domains, by analogy to 'Top Level Domain' (TLD) or '2nd Level
- * Domain' (2LD), etc.
- *
- * @author Gojomo
- *
- * this version of PublicSuffixes uses suffix-tree data structure for generating less
- * redundant regular expression. It may be even possible to write a light-weight,
- * thread-safe matcher based on this class.
- * @author Kenji Nagahashi
- */
-public class PublicSuffixes {
- protected static Pattern topmostAssignedSurtPrefixPattern;
- protected static String topmostAssignedSurtPrefixRegex;
-
- /**
- * prefix tree node. each Node represents sequence of letters (prefix)
- * and alternative sequences following it (list of Node's). Nodes in
- * {@code branches} are sorted for skip list like lookup and for generating
- * effective regular expression (see {@link #compareTo(Node)} and {@link #compareTo(char).)
- *
- * as is intended for internal use only, there's no access methods. procedures for updating
- * prefix tree with new input are defined within this class ({@link #addBranch(CharSequence)}).
- *
- * terminal node could be represented in two different form: 1) Node with zero branches,
- * or 2) Node with zero-length {@code cs}. So, root node must be initialized with empty (not null)
- * {@code branches} unless empty string matches the overall pattern.
- * {@code cs} must not be null except for root node.
- */
- public static class Node implements Comparablemd5:deadbeefdeadbeefdeadbeefdeadbeef
- * When this handler is invoked against an md5 URL, it passes the raw md5 to
- * the configured script as an argument. The configured script then does the
- * work to bring the item pointed to by the md5 local so we can open a Stream
- * on the local copy. Local file is deleted when we finish. Do
- * {@link org.archive.net.DownloadURLConnection#getFile()} to get name of
- * temporary file.
- *
- * -Djava.protocol.handler.pkgs=org.archive.net to add this handler
- * to the java.net.URL set. Also define system properties
- * -Dorg.archive.net.md5.Md5URLConnection.path=PATH_TO_SCRIPT to
- * pass path of script to run as well as
- * -Dorg.archive.net.md5.Md5URLConnection.options=OPTIONS for
- * any options you'd like to include. The pointed-to PATH_TO_SCRIPT
- * will be invoked as follows: PATH_TO_SCRIPT OPTIONS MD5
- * LOCAL_TMP_FILE. The LOCAL_TMP_FILE file is made in
- * java.io.tmpdir using java tmp name code.
- * @author stack
- */
-public class Handler extends URLStreamHandler {
- protected URLConnection openConnection(URL u) {
- return new Md5URLConnection(u);
- }
-
- /**
- * Main dumps rsync file to STDOUT.
- * @param args
- * @throws IOException
- */
- public static void main(String[] args)
- throws IOException {
- if (args.length != 1) {
- System.out.println("Usage: java java " +
- "-Djava.protocol.handler.pkgs=org.archive.net " +
- "org.archive.net.md5.Handler " +
- "md5:deadbeefdeadbeefdeadbeefdeadbeef");
- System.exit(1);
- }
- System.setProperty("org.archive.net.md5.Md5URLConnection.path",
- "/tmp/manifest");
- System.setProperty("java.protocol.handler.pkgs", "org.archive.net");
- URL u = new URL(args[0]);
- URLConnection connect = u.openConnection();
- // Write download to stdout.
- final int bufferlength = 4096;
- byte [] buffer = new byte [bufferlength];
- InputStream is = connect.getInputStream();
- try {
- for (int count = is.read(buffer, 0, bufferlength);
- (count = is.read(buffer, 0, bufferlength)) != -1;) {
- System.out.write(buffer, 0, count);
- }
- System.out.flush();
- } finally {
- is.close();
- }
- }
-}
diff --git a/commons/src/main/java/org/archive/net/md5/Md5URLConnection.java b/commons/src/main/java/org/archive/net/md5/Md5URLConnection.java
deleted file mode 100644
index e4fe98e3..00000000
--- a/commons/src/main/java/org/archive/net/md5/Md5URLConnection.java
+++ /dev/null
@@ -1,34 +0,0 @@
-/*
- * This file is part of the Heritrix web crawler (crawler.archive.org).
- *
- * Licensed to the Internet Archive (IA) by one or more individual
- * contributors.
- *
- * The IA licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.archive.net.md5;
-
-import java.net.URL;
-
-import org.archive.net.DownloadURLConnection;
-
-/**
- * Md5 URL connection.
- * @author stack
- * @version $Date$, $Revision$
- */
-public class Md5URLConnection extends DownloadURLConnection {
- protected Md5URLConnection(URL u) {
- super(u);
- }
-}
\ No newline at end of file
diff --git a/commons/src/main/java/org/archive/net/rsync/Handler.java b/commons/src/main/java/org/archive/net/rsync/Handler.java
deleted file mode 100644
index 9eb35f5d..00000000
--- a/commons/src/main/java/org/archive/net/rsync/Handler.java
+++ /dev/null
@@ -1,71 +0,0 @@
-/*
- * This file is part of the Heritrix web crawler (crawler.archive.org).
- *
- * Licensed to the Internet Archive (IA) by one or more individual
- * contributors.
- *
- * The IA licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.archive.net.rsync;
-
-import java.io.IOException;
-import java.io.InputStream;
-import java.net.URL;
-import java.net.URLConnection;
-import java.net.URLStreamHandler;
-
-/**
- * A protocol handler that uses native rsync client to do copy.
- * You need to define the system property
- * -Djava.protocol.handler.pkgs=org.archive.net to add this handler
- * to the java.net.URL set. Assumes rsync is in path. Define
- * system property
- * -Dorg.archive.net.rsync.RsyncUrlConnection.path=PATH_TO_RSYNC to
- * pass path to rsync. Downloads to java.io.tmpdir.
- * @author stack
- */
-public class Handler extends URLStreamHandler {
- protected URLConnection openConnection(URL u) {
- return new RsyncURLConnection(u);
- }
-
- /**
- * Main dumps rsync file to STDOUT.
- * @param args
- * @throws IOException
- */
- public static void main(String[] args)
- throws IOException {
- if (args.length != 1) {
- System.out.println("Usage: java java " +
- "-Djava.protocol.handler.pkgs=org.archive.net " +
- "org.archive.net.rsync.Handler RSYNC_URL");
- System.exit(1);
- }
- URL u = new URL(args[0]);
- URLConnection connect = u.openConnection();
- // Write download to stdout.
- final int bufferlength = 4096;
- byte [] buffer = new byte [bufferlength];
- InputStream is = connect.getInputStream();
- try {
- for (int count = is.read(buffer, 0, bufferlength);
- (count = is.read(buffer, 0, bufferlength)) != -1;) {
- System.out.write(buffer, 0, count);
- }
- System.out.flush();
- } finally {
- is.close();
- }
- }
-}
\ No newline at end of file
diff --git a/commons/src/main/java/org/archive/net/rsync/RsyncURLConnection.java b/commons/src/main/java/org/archive/net/rsync/RsyncURLConnection.java
deleted file mode 100644
index c6097e96..00000000
--- a/commons/src/main/java/org/archive/net/rsync/RsyncURLConnection.java
+++ /dev/null
@@ -1,51 +0,0 @@
-/*
- * This file is part of the Heritrix web crawler (crawler.archive.org).
- *
- * Licensed to the Internet Archive (IA) by one or more individual
- * contributors.
- *
- * The IA licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.archive.net.rsync;
-
-import java.io.File;
-import java.net.URL;
-
-import org.archive.net.DownloadURLConnection;
-
-/**
- * Rsync URL connection.
- * @author stack
- * @version $Date$, $Revision$
- */
-public class RsyncURLConnection extends DownloadURLConnection {
- private final String RSYNC_TIMEOUT =
- System.getProperty(RsyncURLConnection.class.getName() + ".timeout",
- "300");
-
- protected RsyncURLConnection(URL u) {
- super(u);
- }
-
- protected String getScript() {
- return System.getProperty(this.getClass().getName() + ".path",
- "rsync");
- }
-
- @Override
- protected String[] getCommand(final URL thisUrl,
- final File downloadFile) {
- return new String[] {getScript(), "--timeout=" + RSYNC_TIMEOUT,
- this.url.getPath(), downloadFile.getAbsolutePath()};
- }
-}
diff --git a/commons/src/main/java/org/archive/uid/RecordIDGenerator.java b/commons/src/main/java/org/archive/uid/RecordIDGenerator.java
deleted file mode 100644
index 97f1a022..00000000
--- a/commons/src/main/java/org/archive/uid/RecordIDGenerator.java
+++ /dev/null
@@ -1,72 +0,0 @@
-/*
- * This file is part of the Heritrix web crawler (crawler.archive.org).
- *
- * Licensed to the Internet Archive (IA) by one or more individual
- * contributors.
- *
- * The IA licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.archive.uid;
-
-import java.net.URI;
-import java.net.URISyntaxException;
-import java.util.Map;
-
-/**
- * A record-id generator.
- *
- * @contributor stack
- * @contributor gojomo
- * @version $Revision$ $Date$
- */
-public interface RecordIDGenerator {
- /**
- * @return A URI that can serve as a record-id.
- * @throws URISyntaxException
- */
- public URI getRecordID();
-
- /**
- * @param qualifiers Qualifiers to add.
- * @return A URI qualified with passed qualifiers that can
- * serve as a record-id, or, a new, unique record-id without qualifiers
- * (if qualifiers not easily implemented using passed URI scheme).
- */
- public URI getQualifiedRecordID(final Mapqualifiers that can
- * serve as a record-id, or, a new, unique record-id without qualifiers
- * (if qualifiers not easily implemented using passed URI scheme).
- */
- public URI getQualifiedRecordID(final String key, final String value);
-
- /**
- * Append (or if already present, update) qualifiers to passed
- * recordId. Use with caution. Guard against turning up a
- * result that already exists. Use when writing a group of records inside
- * a single transaction.
- *
- * How qualifiers are appended/updated varies with URI scheme. Its allowed
- * that an invocation of this method does nought but call
- * {@link #getRecordID()}, returning a new URI unrelated to the passed
- * recordId and passed qualifier.
- * @param recordId URI to append qualifier to.
- * @param qualifiers Map of qualifier values keyed by qualifier name.
- * @return New URI based off passed uri and passed qualifier.
- */
- public URI qualifyRecordID(final URI recordId,
- final Mapurn:uuid:0161811f-5da6-4c6e-9808-a2fab97114cf. Always makes a
- * new identifier even when passed qualifiers.
- *
- * @author stack
- * @version $Revision$ $Date$
- * @see RFC4122
- */
-public class UUIDGenerator implements RecordIDGenerator {
- private static final String SCHEME = "urn:uuid";
- private static final String SCHEME_COLON = SCHEME + ":";
-
- public UUIDGenerator() {
- super();
- }
-
- public URI qualifyRecordID(URI recordId,
- final Maporg.archive.uid.GeneratorFactory.generator to point
-at an alternate implementation of {@link org.archive.uid.Generator}.
-
-TODO
-
-
-
-
diff --git a/commons/src/main/java/org/archive/util/DevUtils.java b/commons/src/main/java/org/archive/util/DevUtils.java
deleted file mode 100644
index d630a0b1..00000000
--- a/commons/src/main/java/org/archive/util/DevUtils.java
+++ /dev/null
@@ -1,116 +0,0 @@
-/*
- * This file is part of the Heritrix web crawler (crawler.archive.org).
- *
- * Licensed to the Internet Archive (IA) by one or more individual
- * contributors.
- *
- * The IA licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.archive.util;
-
-import java.io.BufferedReader;
-import java.io.IOException;
-import java.io.InputStreamReader;
-import java.io.PrintWriter;
-import java.io.StringWriter;
-import java.util.logging.Logger;
-
-
-/**
- * Write a message and stack trace to the 'org.archive.util.DevUtils' logger.
- *
- * @author gojomo
- * @version $Revision$ $Date$
- */
-public class DevUtils {
- public static Logger logger =
- Logger.getLogger(DevUtils.class.getName());
-
- /**
- * Log a warning message to the logger 'org.archive.util.DevUtils' made of
- * the passed 'note' and a stack trace based off passed exception.
- *
- * @param ex Exception we print a stacktrace on.
- * @param note Message to print ahead of the stacktrace.
- */
- public static void warnHandle(Throwable ex, String note) {
- logger.warning(TextUtils.exceptionToString(note, ex));
- }
-
- /**
- * @return Extra information gotten from current Thread. May not
- * always be available in which case we return empty string.
- */
- public static String extraInfo() {
- StringWriter sw = new StringWriter();
- PrintWriter pw = new PrintWriter(sw);
- final Thread current = Thread.currentThread();
- if (current instanceof Reporter) {
- Reporter tt = (Reporter)current;
- try {
- tt.reportTo(pw);
- } catch (IOException e) {
- // Not really possible w/ a StringWriter
- e.printStackTrace();
- }
- }
- if (current instanceof ProgressStatisticsReporter) {
- ProgressStatisticsReporter tt = (ProgressStatisticsReporter)current;
- try {
- tt.progressStatisticsLegend(pw);
- tt.progressStatisticsLine(pw);
- } catch (IOException e) {
- // Not really possible w/ a StringWriter
- e.printStackTrace();
- }
- }
- pw.flush();
- return sw.toString();
- }
-
- /**
- * Nothing to see here, move along.
- * @deprecated This method was never used.
- */
- @Deprecated
- public static void betterPrintStack(RuntimeException re) {
- re.printStackTrace(System.err);
- }
-
- /**
- * Send this JVM process a SIGQUIT; giving a thread dump and possibly
- * a heap histogram (if using -XX:+PrintClassHistogram).
- *
- * Used to automatically dump info, for example when a serious error
- * is encountered. Would use 'jmap'/'jstack', but have seen JVM
- * lockups -- perhaps due to lost thread wake signals -- when using
- * those against Sun 1.5.0+03 64bit JVM.
- */
- public static void sigquitSelf() {
- try {
- Process p = Runtime.getRuntime().exec(
- new String[] {"perl", "-e", "print getppid(). \"\n\";"});
- BufferedReader br =
- new BufferedReader(new InputStreamReader(p.getInputStream()));
- String ppid = br.readLine();
- Runtime.getRuntime().exec(
- new String[] {"sh", "-c", "kill -3 "+ppid}).waitFor();
- } catch (IOException e) {
- // TODO Auto-generated catch block
- e.printStackTrace();
- } catch (InterruptedException e) {
- // TODO Auto-generated catch block
- e.printStackTrace();
- }
- }
-}
diff --git a/commons/src/main/java/org/archive/util/FileUtils.java b/commons/src/main/java/org/archive/util/FileUtils.java
deleted file mode 100644
index 3b5b93ad..00000000
--- a/commons/src/main/java/org/archive/util/FileUtils.java
+++ /dev/null
@@ -1,699 +0,0 @@
-/*
- * This file is part of the Heritrix web crawler (crawler.archive.org).
- *
- * Licensed to the Internet Archive (IA) by one or more individual
- * contributors.
- *
- * The IA licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.archive.util;
-
-import java.io.ByteArrayInputStream;
-import java.io.File;
-import java.io.FileFilter;
-import java.io.FileInputStream;
-import java.io.FileNotFoundException;
-import java.io.FileOutputStream;
-import java.io.IOException;
-import java.io.InputStream;
-import java.io.OutputStream;
-import java.nio.channels.ClosedByInterruptException;
-import java.nio.channels.FileChannel;
-import java.util.Iterator;
-import java.util.LinkedList;
-import java.util.List;
-import java.util.Properties;
-import java.util.logging.Level;
-import java.util.logging.Logger;
-import java.util.regex.Pattern;
-
-import org.apache.commons.io.IOUtils;
-import org.apache.commons.io.filefilter.IOFileFilter;
-import org.apache.commons.lang.math.LongRange;
-
-
-/** Utility methods for manipulating files and directories.
- *
- * @contributor John Erik Halse
- * @contributor gojomo
- */
-public class FileUtils {
- private static final Logger LOGGER =
- Logger.getLogger(FileUtils.class.getName());
-
- /**
- * Constructor made private because all methods of this class are static.
- */
- private FileUtils() {
- super();
- }
-
- /**
- * Copy the src file to the destination. Deletes any preexisting
- * file at destination.
- *
- * @param src
- * @param dest
- * @return True if the extent was greater than actual bytes copied.
- * @throws FileNotFoundException
- * @throws IOException
- */
- public static boolean copyFile(final File src, final File dest)
- throws FileNotFoundException, IOException {
- return copyFile(src, dest, -1, true);
- }
-
- /**
- * Copy up to extent bytes of the source file to the destination.
- * Deletes any preexisting file at destination.
- *
- * @param src
- * @param dest
- * @param extent Maximum number of bytes to copy
- * @return True if the extent was greater than actual bytes copied.
- * @throws FileNotFoundException
- * @throws IOException
- */
- public static boolean copyFile(final File src, final File dest,
- long extent)
- throws FileNotFoundException, IOException {
- return copyFile(src, dest, extent, true);
- }
-
- /**
- * Copy up to extent bytes of the source file to the destination
- *
- * @param src
- * @param dest
- * @param extent Maximum number of bytes to copy
- * @param overwrite If target file already exits, and this parameter is
- * true, overwrite target file (We do this by first deleting the target
- * file before we begin the copy).
- * @return True if the extent was greater than actual bytes copied.
- * @throws FileNotFoundException
- * @throws IOException
- */
- public static boolean copyFile(final File src, final File dest,
- long extent, final boolean overwrite)
- throws FileNotFoundException, IOException {
- boolean result = false;
- if (LOGGER.isLoggable(Level.FINE)) {
- LOGGER.fine("Copying file " + src + " to " + dest + " extent " +
- extent + " exists " + dest.exists());
- }
- if (dest.exists()) {
- if (overwrite) {
- dest.delete();
- LOGGER.finer(dest.getAbsolutePath() + " removed before copy.");
- } else {
- // Already in place and we're not to overwrite. Return.
- return result;
- }
- }
- FileInputStream fis = null;
- FileOutputStream fos = null;
- FileChannel fcin = null;
- FileChannel fcout = null;
- try {
- // Get channels
- fis = new FileInputStream(src);
- fos = new FileOutputStream(dest);
- fcin = fis.getChannel();
- fcout = fos.getChannel();
- if (extent < 0) {
- extent = fcin.size();
- }
-
- // Do the file copy
- long trans = fcin.transferTo(0, extent, fcout);
- if (trans < extent) {
- result = false;
- }
- result = true;
- } catch (IOException e) {
- // Add more info to the exception. Preserve old stacktrace.
- // We get 'Invalid argument' on some file copies. See
- // http://intellij.net/forums/thread.jsp?forum=13&thread=63027&message=853123
- // for related issue.
- String message = "Copying " + src.getAbsolutePath() + " to " +
- dest.getAbsolutePath() + " with extent " + extent +
- " got IOE: " + e.getMessage();
- if ((e instanceof ClosedByInterruptException) ||
- ((e.getMessage()!=null)
- &&e.getMessage().equals("Invalid argument"))) {
- LOGGER.severe("Failed copy, trying workaround: " + message);
- workaroundCopyFile(src, dest);
- } else {
- IOException newE = new IOException(message);
- newE.initCause(e);
- throw newE;
- }
- } finally {
- // finish up
- if (fcin != null) {
- fcin.close();
- }
- if (fcout != null) {
- fcout.close();
- }
- if (fis != null) {
- fis.close();
- }
- if (fos != null) {
- fos.close();
- }
- }
- return result;
- }
-
- protected static void workaroundCopyFile(final File src,
- final File dest)
- throws IOException {
- FileInputStream from = null;
- FileOutputStream to = null;
- try {
- from = new FileInputStream(src);
- to = new FileOutputStream(dest);
- byte[] buffer = new byte[4096];
- int bytesRead;
- while ((bytesRead = from.read(buffer)) != -1) {
- to.write(buffer, 0, bytesRead);
- }
- } finally {
- if (from != null) {
- try {
- from.close();
- } catch (IOException e) {
- e.printStackTrace();
- }
- }
- if (to != null) {
- try {
- to.close();
- } catch (IOException e) {
- e.printStackTrace();
- }
- }
- }
- }
-
- /**
- * Get a list of all files in directory that have passed prefix.
- *
- * @param dir Dir to look in.
- * @param prefix Basename of files to look for. Compare is case insensitive.
- *
- * @return List of files in dir that start w/ passed basename.
- */
- public static File [] getFilesWithPrefix(File dir, final String prefix) {
- FileFilter prefixFilter = new FileFilter() {
- public boolean accept(File pathname)
- {
- return pathname.getName().toLowerCase().
- startsWith(prefix.toLowerCase());
- }
- };
- return dir.listFiles(prefixFilter);
- }
-
- /** Get a @link java.io.FileFilter that filters files based on a regular
- * expression.
- *
- * @param regex the regular expression the files must match.
- * @return the newly created filter.
- */
- public static IOFileFilter getRegexFileFilter(String regex) {
- // Inner class defining the RegexFileFilter
- class RegexFileFilter implements IOFileFilter {
- Pattern pattern;
-
- protected RegexFileFilter(String re) {
- pattern = Pattern.compile(re);
- }
-
- public boolean accept(File pathname) {
- return pattern.matcher(pathname.getName()).matches();
- }
-
- public boolean accept(File dir, String name) {
- return accept(new File(dir,name));
- }
- }
-
- return new RegexFileFilter(regex);
- }
-
- /**
- * Test file exists and is readable.
- * @param f File to test.
- * @exception FileNotFoundException If file does not exist or is not unreadable.
- */
- public static File assertReadable(final File f) throws FileNotFoundException {
- if (!f.exists()) {
- throw new FileNotFoundException(f.getAbsolutePath() +
- " does not exist.");
- }
-
- if (!f.canRead()) {
- throw new FileNotFoundException(f.getAbsolutePath() +
- " is not readable.");
- }
-
- return f;
- }
-
- /**
- * @param f File to test.
- * @return True if file is readable, has uncompressed extension,
- * and magic string at file start.
- * @exception IOException If file not readable or other problem.
- */
- public static boolean isReadableWithExtensionAndMagic(final File f,
- final String uncompressedExtension, final String magic)
- throws IOException {
- boolean result = false;
- FileUtils.assertReadable(f);
- if(f.getName().toLowerCase().endsWith(uncompressedExtension)) {
- FileInputStream fis = new FileInputStream(f);
- try {
- byte [] b = new byte[magic.length()];
- int read = fis.read(b, 0, magic.length());
- fis.close();
- if (read == magic.length()) {
- StringBuffer beginStr
- = new StringBuffer(magic.length());
- for (int i = 0; i < magic.length(); i++) {
- beginStr.append((char)b[i]);
- }
-
- if (beginStr.toString().
- equalsIgnoreCase(magic)) {
- result = true;
- }
- }
- } finally {
- fis.close();
- }
- }
-
- return result;
- }
-
- /**
- * Turn path into a File, relative to context (which may be ignored
- * if path is absolute).
- *
- * @param context File context if path is relative
- * @param path String path to make into a File
- * @return File created
- */
- public static File maybeRelative(File context, String path) {
- File f = new File(path);
- if(f.isAbsolute()) {
- return f;
- }
- return new File(context, path);
- }
-
- /**
- * Load Properties instance from a File
- *
- * @param file
- * @return Properties
- * @throws IOException
- */
- public static Properties loadProperties(File file) throws IOException {
- FileInputStream finp = new FileInputStream(file);
- try {
- Properties p = new Properties();
- p.load(finp);
- return p;
- } finally {
- ArchiveUtils.closeQuietly(finp);
- }
- }
-
- /**
- * Store Properties instance to a File
- * @param p
- * @param file destination File
- * @throws IOException
- */
- public static void storeProperties(Properties p, File file) throws IOException {
- FileOutputStream fos = new FileOutputStream(file);
- try {
- p.store(fos,"");
- } finally {
- ArchiveUtils.closeQuietly(fos);
- }
- }
-
- // TODO: comment
- public static boolean moveAsideIfExists(File file) throws IOException {
- if(!file.exists()) {
- return true;
- }
- String newName =
- file.getCanonicalPath() + "."
- + ArchiveUtils.get14DigitDate(file.lastModified());
- boolean retVal = file.renameTo(new File(newName));
- if(!retVal) {
- LOGGER.warning("unable to move aside: "+file+" to "+newName);
- }
- return retVal;
-
- }
-
- /**
- * Retrieve a number of lines from the file around the given
- * position, as when paging forward or backward through a file.
- *
- * @param file File to retrieve lines
- * @param position offset to anchor lines
- * @param signedDesiredLineCount lines requested; if negative,
- * want this number of lines ending with a line containing
- * the position; if positive, want this number of lines,
- * all starting at or after position.
- * @param lines ListCRLFCRLF) and two leading hyphens. Add to
- {@link org.archive.uid.Generator}
- interface an upper-bound on generated ID length.http://archive.org/UID-SCHEME/ID
-where scheme might be UUID and an ID might be
-f9472055-fbb6-4810-90e8-68fd39e145a6;type=metadata or,
-using ARK:
-http://archive.org/ark:/13030/f9472055-fbb6-4810-90e8-68fd39e145a6;type=metadata.
-is when done or if an exception.
- * @param is Stream to read.
- * @param toFile File to write to.
- * @throws IOException
- */
- public static long readFullyToFile(InputStream is, File toFile)
- throws IOException {
- OutputStream os = org.apache.commons.io.FileUtils.openOutputStream(toFile);
- try {
- return IOUtils.copyLarge(is, os);
- } finally {
- IOUtils.closeQuietly(os);
- IOUtils.closeQuietly(is);
- }
- }
-
- /**
- * Ensure writeable directory.
- *
- * If doesn't exist, we attempt creation.
- *
- * @param dir Directory to test for exitence and is writeable.
- *
- * @return The passed dir.
- *
- * @exception IOException If passed directory does not exist and is not
- * createable, or directory is not writeable or is not a directory.
- */
- public static File ensureWriteableDirectory(String dir)
- throws IOException {
- return FileUtils.ensureWriteableDirectory(new File(dir));
- }
-
- /**
- * Ensure writeable directories.
- *
- * If doesn't exist, we attempt creation.
- *
- * @param dirs List of Files to test.
- *
- * @return The passed dirs.
- *
- * @exception IOException If passed directory does not exist and is not
- * createable, or directory is not writeable or is not a directory.
- */
- public static Listdir.
- *
- * @exception IOException If passed directory does not exist and is not
- * createable, or directory is not writeable or is not a directory.
- */
- public static File ensureWriteableDirectory(File dir)
- throws IOException {
- if (!dir.exists()) {
- boolean success = dir.mkdirs();
- if (!success) {
- throw new IOException("Failed to create directory: " + dir);
- }
- } else {
- if (!dir.canWrite()) {
- throw new IOException("Dir " + dir.getAbsolutePath() +
- " not writeable.");
- } else if (!dir.isDirectory()) {
- throw new IOException("Dir " + dir.getAbsolutePath() +
- " is not a directory.");
- }
- }
-
- return dir;
- }
-
- public static File tryToCanonicalize(File file) {
- try {
- return file.getCanonicalFile();
- } catch (IOException e) {
- return file;
- }
- }
-}
\ No newline at end of file
diff --git a/commons/src/main/java/org/archive/util/FilesystemLinkMaker.java b/commons/src/main/java/org/archive/util/FilesystemLinkMaker.java
index 9899a29c..c489d04d 100644
--- a/commons/src/main/java/org/archive/util/FilesystemLinkMaker.java
+++ b/commons/src/main/java/org/archive/util/FilesystemLinkMaker.java
@@ -20,6 +20,7 @@
package org.archive.util;
import java.io.File;
import java.io.IOException;
+import java.util.logging.Logger;
import com.sun.jna.Native;
import com.sun.jna.Platform;
@@ -34,6 +35,8 @@ import com.sun.jna.win32.StdCallLibrary;
*/
public class FilesystemLinkMaker {
+ private static final Logger logger = Logger.getLogger(FilesystemLinkMaker.class.getName());
+
// see https://github.com/twall/jna/blob/master/www/GettingStarted.md
public interface Kernel32Library extends StdCallLibrary {
Kernel32Library INSTANCE = (Kernel32Library) (Platform.isWindows()
@@ -75,26 +78,38 @@ public class FilesystemLinkMaker {
*/
// XXX could handle errors better (examine errno, throw exception...)
public static boolean makeHardLink(String existingPath, String newPath) {
- if (Platform.isWindows()) {
- return Kernel32Library.INSTANCE.CreateHardLinkA(newPath, existingPath, null);
- } else {
- int status = CLibrary.INSTANCE.link(existingPath, newPath);
- return status == 0;
+ try {
+ if (Platform.isWindows()) {
+ return Kernel32Library.INSTANCE.CreateHardLinkA(newPath, existingPath, null);
+ } else {
+ int status = CLibrary.INSTANCE.link(existingPath, newPath);
+ return status == 0;
+ }
+ } catch (UnsatisfiedLinkError e) {
+ // see https://webarchive.jira.com/browse/HER-1979
+ logger.warning("hard links not supported on this platform - " + e);
+ return false;
}
}
/**
- * Wrapper over platform-dependent system calls to create a symboic link.
+ * Wrapper over platform-dependent system calls to create a symbolic link.
*
* @return true on success
*/
// XXX could handle errors better (examine errno, throw exception...)
public static boolean makeSymbolicLink(String existingPath, String newPath) {
- if (Platform.isWindows()) {
- return Kernel32Library.INSTANCE.CreateSymbolicLinkA(newPath, existingPath, null);
- } else {
- int status = CLibrary.INSTANCE.symlink(existingPath, newPath);
- return status == 0;
+ try {
+ if (Platform.isWindows()) {
+ return Kernel32Library.INSTANCE.CreateSymbolicLinkA(newPath, existingPath, null);
+ } else {
+ int status = CLibrary.INSTANCE.symlink(existingPath, newPath);
+ return status == 0;
+ }
+ } catch (UnsatisfiedLinkError e) {
+ // see https://webarchive.jira.com/browse/HER-1979
+ logger.warning("symbolic links not supported on this platform - " + e);
+ return false;
}
}
diff --git a/commons/src/main/java/org/archive/util/InetAddressUtil.java b/commons/src/main/java/org/archive/util/InetAddressUtil.java
deleted file mode 100644
index 585ba772..00000000
--- a/commons/src/main/java/org/archive/util/InetAddressUtil.java
+++ /dev/null
@@ -1,116 +0,0 @@
-/*
- * This file is part of the Heritrix web crawler (crawler.archive.org).
- *
- * Licensed to the Internet Archive (IA) by one or more individual
- * contributors.
- *
- * The IA licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.archive.util;
-
-import java.net.InetAddress;
-import java.net.NetworkInterface;
-import java.net.SocketException;
-import java.net.UnknownHostException;
-import java.util.ArrayList;
-import java.util.Enumeration;
-import java.util.List;
-import java.util.logging.Logger;
-import java.util.regex.Matcher;
-import java.util.regex.Pattern;
-
-/**
- * InetAddress utility.
- * @author stack
- * @version $Date$, $Revision$
- */
-public class InetAddressUtil {
- private static Logger logger =
- Logger.getLogger(InetAddressUtil.class.getName());
-
- /**
- * ipv4 address.
- */
- public static Pattern IPV4_QUADS = Pattern.compile(
- "([0-9]{1,3})\\.([0-9]{1,3})\\.([0-9]{1,3})\\.([0-9]{1,3})");
-
- private InetAddressUtil () {
- super();
- }
-
- /**
- * Returns InetAddress for passed host IF its in
- * IPV4 quads format (e.g. 128.128.128.128).
- * null is returned.
- *
- * @param inputStream the stream to read from
- *
- * @throws IOException if an I/O problem occurs
- * @return a byte array from the stream
- */
- public static byte[] readRawLine(InputStream inputStream) throws IOException {
- LOG.trace("enter LaxHttpParser.readRawLine()");
-
- ByteArrayOutputStream buf = new ByteArrayOutputStream();
- int ch;
- while ((ch = inputStream.read()) >= 0) {
- buf.write(ch);
- if (ch == '\n') { // be tolerant (RFC-2616 Section 19.3)
- break;
- }
- }
- if (buf.size() == 0) {
- return null;
- }
- return buf.toByteArray();
- }
-
- /**
- * Read up to "\n" from an (unchunked) input stream.
- * If the stream ends before the line terminator is found,
- * the last part of the string will still be returned.
- * If no input data available, null is returned.
- *
- * @param inputStream the stream to read from
- * @param charset charset of HTTP protocol elements
- *
- * @throws IOException if an I/O problem occurs
- * @return a line from the stream
- *
- * @since 3.0
- */
- public static String readLine(InputStream inputStream, String charset) throws IOException {
- LOG.trace("enter LaxHttpParser.readLine(InputStream, String)");
- byte[] rawdata = readRawLine(inputStream);
- if (rawdata == null) {
- return null;
- }
- // strip CR and LF from the end
- int len = rawdata.length;
- int offset = 0;
- if (len > 0) {
- if (rawdata[len - 1] == '\n') {
- offset++;
- if (len > 1) {
- if (rawdata[len - 2] == '\r') {
- offset++;
- }
- }
- }
- }
- return EncodingUtil.getString(rawdata, 0, len - offset, charset);
- }
-
- /**
- * Read up to "\n" from an (unchunked) input stream.
- * If the stream ends before the line terminator is found,
- * the last part of the string will still be returned.
- * If no input data available, null is returned
- *
- * @param inputStream the stream to read from
- *
- * @throws IOException if an I/O problem occurs
- * @return a line from the stream
- *
- * @deprecated use #readLine(InputStream, String)
- */
-
- public static String readLine(InputStream inputStream) throws IOException {
- LOG.trace("enter LaxHttpParser.readLine(InputStream)");
- return readLine(inputStream, "US-ASCII");
- }
-
- /**
- * Parses headers from the given stream. Headers with the same name are not
- * combined.
- *
- * @param is the stream to read headers from
- * @param charset the charset to use for reading the data
- *
- * @return an array of headers in the order in which they were parsed
- *
- * @throws IOException if an IO error occurs while reading from the stream
- * @throws HttpException if there is an error parsing a header value
- *
- * @since 3.0
- */
- public static Header[] parseHeaders(InputStream is, String charset) throws IOException, HttpException {
- LOG.trace("enter HeaderParser.parseHeaders(InputStream, String)");
-
- ArrayListkey, return
- * fallback.
- * @return Value of property or fallback.
- */
- public static int getIntProperty(final String key, final int fallback) {
- return getPropertyOrNull(key) == null?
- fallback: Integer.parseInt(getPropertyOrNull(key));
- }
-
- /**
- * Given a string which may contain expressions of the form
- * ${key}, replace each expression with the value corresponding to the
- * given key in System Properties. If no value is present,
- * the expression is replaced with the empty-string.
- *
- * @param original String
- * @param properties Properties to try in order; first value found (if any) is used
- * @return modified String
- */
- public static String interpolateWithProperties(String original) {
- return interpolateWithProperties(original,System.getProperties());
- }
-
- protected static String propRefPattern = "\\$\\{([^{}]+)\\}";
-
- /**
- * Given a string which may contain expressions of the form
- * ${key}, replace each expression with the value corresponding to the
- * given key in the supplied Properties instance. If no value is present,
- * the expression is replaced with the empty-string.
- *
- * @param original String
- * @param props Properties to try in order; first value found (if any) is used
- * @return modified String
- */
- public static String interpolateWithProperties(String original,
- Properties... props) {
- String result = original;
- // cap number of interpolations as guard against unending loop
- inter: for(int i =0; i < original.length()*2; i++) {
- Matcher m = TextUtils.getMatcher(propRefPattern, result);
- while(m.find()) {
- String key = m.group(1);
- String value = "";
- for(Properties properties : props) {
- value = properties.getProperty(key, "");
- if(StringUtils.isNotEmpty(value)) {
- break;
- }
- }
- result = result.substring(0,m.start())
- + value
- + result.substring(m.end());
- continue inter;
- }
- // we only hit here if there were no interpolations last while loop
- break;
- }
- return result;
- }
-}
diff --git a/commons/src/main/java/org/archive/util/Recorder.java b/commons/src/main/java/org/archive/util/Recorder.java
deleted file mode 100644
index 298481ea..00000000
--- a/commons/src/main/java/org/archive/util/Recorder.java
+++ /dev/null
@@ -1,581 +0,0 @@
-/*
- * This file is part of the Heritrix web crawler (crawler.archive.org).
- *
- * Licensed to the Internet Archive (IA) by one or more individual
- * contributors.
- *
- * The IA licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.archive.util;
-
-import java.io.BufferedInputStream;
-import java.io.File;
-import java.io.IOException;
-import java.io.InputStream;
-import java.io.InputStreamReader;
-import java.io.OutputStream;
-import java.nio.charset.Charset;
-import java.util.HashSet;
-import java.util.Set;
-import java.util.logging.Level;
-import java.util.logging.Logger;
-import java.util.zip.DeflaterInputStream;
-import java.util.zip.GZIPInputStream;
-
-import org.apache.commons.httpclient.ChunkedInputStream;
-import org.apache.commons.io.FileUtils;
-import org.apache.commons.io.IOUtils;
-import org.apache.commons.lang.StringUtils;
-import org.archive.io.GenericReplayCharSequence;
-import org.archive.io.RecordingInputStream;
-import org.archive.io.RecordingOutputStream;
-import org.archive.io.ReplayCharSequence;
-import org.archive.io.ReplayInputStream;
-
-import com.google.common.base.Charsets;
-
-
-/**
- * Pairs together a RecordingInputStream and RecordingOutputStream
- * to capture exactly a single HTTP transaction.
- *
- * Initially only supports HTTP/1.0 (one request, one response per stream)
- *
- * Call {@link #markContentBegin()} to demarc the transition between HTTP
- * header and body.
- *
- * @author gojomo
- */
-public class Recorder {
- protected static Logger logger =
- Logger.getLogger("org.archive.util.HttpRecorder");
-
- private static final int DEFAULT_OUTPUT_BUFFER_SIZE = 16384;
- private static final int DEFAULT_INPUT_BUFFER_SIZE = 524288;
-
- private RecordingInputStream ris = null;
- private RecordingOutputStream ros = null;
-
- /**
- * Backing file basename.
- *
- * Keep it around so can clean up backing files left on disk.
- */
- private String backingFileBasename = null;
-
- /**
- * Backing file output stream suffix.
- */
- private static final String RECORDING_OUTPUT_STREAM_SUFFIX = ".ros";
-
- /**
- * Backing file input stream suffix.
- */
- private static final String RECORDING_INPUT_STREAM_SUFFIX = ".ris";
-
- /**
- * recording-input (ris) content character encoding.
- */
- protected String characterEncoding = null;
-
- /**
- * Charset to use for CharSequence provision. Will be UTF-8 if no
- * encoding ever requested; a Charset matching above characterEncoding
- * if possible; ISO_8859 if above characterEncoding is unsatisfiable.
- * TODO: unify to UTF-8 for unspecified and bad-specified cases?
- * (current behavior is for consistency with our prior but perhaps not
- * optimal behavior)
- */
- protected Charset charset = Charsets.UTF_8;
-
- /** whether recording-input (ris) message-body is chunked */
- protected boolean inputIsChunked = false;
-
- /** recording-input (ris) entity content-encoding (eg gzip, deflate), if any */
- protected String contentEncoding = null;
-
- private ReplayCharSequence replayCharSequence;
-
-
- /**
- * Create an HttpRecorder.
- *
- * @param tempDir Directory into which we drop backing files for
- * recorded input and output.
- * @param backingFilenameBase Backing filename base to which we'll append
- * suffices ris for recorded input stream and
- * ros for recorded output stream.
- * @param outBufferSize Size of output buffer to use.
- * @param inBufferSize Size of input buffer to use.
- */
- public Recorder(File tempDir, String backingFilenameBase,
- int outBufferSize, int inBufferSize) {
- this(new File(ensure(tempDir), backingFilenameBase),
- outBufferSize, inBufferSize);
- }
-
-
- private static File ensure(File tempDir) {
- try {
- org.archive.util.FileUtils.ensureWriteableDirectory(tempDir);
- } catch (IOException e) {
- throw new IllegalStateException(e);
- }
-
- return tempDir;
- }
-
- public Recorder(File file, int outBufferSize, int inBufferSize) {
- super();
- this.backingFileBasename = file.getAbsolutePath();
- this.ris = new RecordingInputStream(inBufferSize,
- this.backingFileBasename + RECORDING_INPUT_STREAM_SUFFIX);
- this.ros = new RecordingOutputStream(outBufferSize,
- this.backingFileBasename + RECORDING_OUTPUT_STREAM_SUFFIX);
- }
-
- /**
- * Create an HttpRecorder.
- *
- * @param tempDir
- * Directory into which we drop backing files for recorded input
- * and output.
- * @param backingFilenameBase
- * Backing filename base to which we'll append suffices
- * ris for recorded input stream and
- * ros for recorded output stream.
- */
- public Recorder(File tempDir, String backingFilenameBase) {
- this(tempDir, backingFilenameBase, DEFAULT_INPUT_BUFFER_SIZE,
- DEFAULT_OUTPUT_BUFFER_SIZE);
- }
-
-
- /**
- * Wrap the provided stream with the internal RecordingInputStream
- *
- * open() throws an exception if RecordingInputStream is already open.
- *
- * @param is InputStream to wrap.
- *
- * @return The input stream wrapper which itself is an input stream.
- * Pass this in place of the passed stream so input can be recorded.
- *
- * @throws IOException
- */
- public InputStream inputWrap(InputStream is)
- throws IOException {
- logger.fine(Thread.currentThread().getName() + " wrapping input");
-
- // discard any state from previously-recorded input
- this.characterEncoding = null;
- this.inputIsChunked = false;
- this.contentEncoding = null;
-
- this.ris.open(is);
- return this.ris;
- }
-
- /**
- * Wrap the provided stream with the internal RecordingOutputStream
- *
- * open() throws an exception if RecordingOutputStream is already open.
- *
- * @param os The output stream to wrap.
- *
- * @return The output stream wrapper which is itself an output stream.
- * Pass this in place of the passed stream so output can be recorded.
- *
- * @throws IOException
- */
- public OutputStream outputWrap(OutputStream os)
- throws IOException {
- this.ros.open(os);
- return this.ros;
- }
-
- /**
- * Close all streams.
- */
- public void close() {
- logger.fine(Thread.currentThread().getName() + " closing");
- try {
- this.ris.close();
- } catch (IOException e) {
- // TODO: Can we not let the exception out of here and report it
- // higher up in the caller?
- DevUtils.logger.log(Level.SEVERE, "close() ris" +
- DevUtils.extraInfo(), e);
- }
- try {
- this.ros.close();
- } catch (IOException e) {
- DevUtils.logger.log(Level.SEVERE, "close() ros" +
- DevUtils.extraInfo(), e);
- }
- }
-
- /**
- * Return the internal RecordingInputStream
- *
- * @return A RIS.
- */
- public RecordingInputStream getRecordedInput() {
- return this.ris;
- }
-
- /**
- * @return The RecordingOutputStream.
- */
- public RecordingOutputStream getRecordedOutput() {
- return this.ros;
- }
-
- /**
- * Mark current position as the point where the HTTP headers end.
- */
- public void markContentBegin() {
- this.ris.markContentBegin();
- }
-
- public long getResponseContentLength() {
- return this.ris.getResponseContentLength();
- }
-
- /**
- * Close both input and output recorders.
- *
- * Recorders are the output streams to which we are recording.
- * {@link #close()} closes the stream that is being recorded and the
- * recorder. This method explicitly closes the recorder only.
- */
- public void closeRecorders() {
- try {
- this.ris.closeRecorder();
- this.ros.closeRecorder();
- } catch (IOException e) {
- DevUtils.warnHandle(e, "Convert to runtime exception?");
- }
- }
-
- /**
- * Cleanup backing files.
- *
- * Call when completely done w/ recorder. Removes any backing files that
- * may have been dropped.
- */
- public void cleanup() {
- this.close();
- this.delete(this.backingFileBasename + RECORDING_OUTPUT_STREAM_SUFFIX);
- this.delete(this.backingFileBasename + RECORDING_INPUT_STREAM_SUFFIX);
- }
-
- /**
- * Delete file if exists.
- *
- * @param name Filename to delete.
- */
- private void delete(String name) {
- File f = new File(name);
- if (f.exists()) {
- f.delete();
- }
- }
-
-
- protected static ThreadLocalImplementation Details
-CRNL.
-
- field = field-name ":" [ field-body ] CRLF
-
- field-name = 1*<any CHAR, excluding CTLs, SPACE, and ":">
-
- field-body = field-body-contents
- [CRLF LWSP-char field-body]
-
- field-body-contents =
- <the ASCII characters making up the field-body, as
- defined in the following sections, and consisting
- of combinations of atom, quoted-string, and
- specials tokens, or else consisting of texts>
-
-Crawl beans in built job