Of Java - * and unsigned bytes. That is, its always a signed int in - * java no matter what the qualifier whether byte, char, etc. - * - *
Add accessors for optional filename, comment and MTIME.
- *
- * @author stack
+ * @deprecated use {@link org.archive.util.zip.GzipHeader}
*/
-public class GzipHeader {
- /**
- * Length of minimal GZIP header.
- *
- * See RFC1952 for explaination of value of 10.
- */
- public static final int MINIMAL_GZIP_HEADER_LENGTH = 10;
-
- /**
- * Total length of the gzip header.
- */
- protected int length = 0;
-
- /**
- * The GZIP header FLG byte.
- */
- protected int flg;
-
- /**
- * GZIP header XFL byte.
- */
- private int xfl;
-
- /**
- * GZIP header OS byte.
- */
- private int os;
-
- /**
- * Extra header field content.
- */
- private byte [] fextra = null;
-
- /**
- * GZIP header MTIME field.
- */
- private int mtime;
-
-
- /**
- * Shutdown constructor.
- *
- * Must pass an input stream.
- */
- public GzipHeader() {
- super();
- }
-
- /**
- * Constructor.
- *
- * This constructor advances the stream past any gzip header found.
- *
- * @param in InputStream to read from.
- * @throws IOException
- */
- public GzipHeader(InputStream in) throws IOException {
- super();
- readHeader(in);
- }
-
- /**
- * Read in gzip header.
- *
- * Advances the stream past the gzip header.
- * @param in InputStream.
- *
- * @throws IOException Throws if does not start with GZIP Header.
- */
- public void readHeader(InputStream in) throws IOException {
- CRC32 crc = new CRC32();
- crc.reset();
- if (!testGzipMagic(in, crc)) {
- throw new NoGzipMagicException();
- }
- this.length += 2;
- if (readByte(in, crc) != Deflater.DEFLATED) {
- throw new IOException("Unknown compression");
- }
- this.length++;
-
- // Get gzip header flag.
- this.flg = readByte(in, crc);
- this.length++;
-
- // Get MTIME.
- this.mtime = readInt(in, crc);
- this.length += 4;
-
- // Read XFL and OS.
- this.xfl = readByte(in, crc);
- this.length++;
- this.os = readByte(in, crc);
- this.length++;
-
- // Skip optional extra field -- stuff w/ alexa stuff in it.
- final int FLG_FEXTRA = 4;
- if ((this.flg & FLG_FEXTRA) == FLG_FEXTRA) {
- int count = readShort(in, crc);
- this.length +=2;
- this.fextra = new byte[count];
- readByte(in, crc, this.fextra, 0, count);
- this.length += count;
- }
-
- // Skip file name. It ends in null.
- final int FLG_FNAME = 8;
- if ((this.flg & FLG_FNAME) == FLG_FNAME) {
- while (readByte(in, crc) != 0) {
- this.length++;
- }
- }
-
- // Skip file comment. It ends in null.
- final int FLG_FCOMMENT = 16; // File comment
- if ((this.flg & FLG_FCOMMENT) == FLG_FCOMMENT) {
- while (readByte(in, crc) != 0) {
- this.length++;
- }
- }
-
- // Check optional CRC.
- final int FLG_FHCRC = 2;
- if ((this.flg & FLG_FHCRC) == FLG_FHCRC) {
- int calcCrc = (int)(crc.getValue() & 0xffff);
- if (readShort(in, crc) != calcCrc) {
- throw new IOException("Bad header CRC");
- }
- this.length += 2;
- }
- }
-
- /**
- * Test gzip magic is next in the stream.
- * Reads two bytes. Caller needs to manage resetting stream.
- * @param in InputStream to read.
- * @return true if found gzip magic. False otherwise
- * or an IOException (including EOFException).
- * @throws IOException
- */
- public boolean testGzipMagic(InputStream in) throws IOException {
- return testGzipMagic(in, null);
- }
-
- /**
- * Test gzip magic is next in the stream.
- * Reads two bytes. Caller needs to manage resetting stream.
- * @param in InputStream to read.
- * @param crc CRC to update.
- * @return true if found gzip magic. False otherwise
- * or an IOException (including EOFException).
- * @throws IOException
- */
- public boolean testGzipMagic(InputStream in, CRC32 crc)
- throws IOException {
- return readShort(in, crc) == GZIPInputStream.GZIP_MAGIC;
- }
-
- /**
- * Read an int.
- *
- * We do not expect to get a -1 reading. If we do, we throw exception.
- * Update the crc as we go.
- *
- * @param in InputStream to read.
- * @param crc CRC to update.
- * @return int read.
- *
- * @throws IOException
- */
- private int readInt(InputStream in, CRC32 crc) throws IOException {
- int s = readShort(in, crc);
- return ((readShort(in, crc) << 16) & 0xffff0000) | s;
- }
-
- /**
- * Read a short.
- *
- * We do not expect to get a -1 reading. If we do, we throw exception.
- * Update the crc as we go.
- *
- * @param in InputStream to read.
- * @param crc CRC to update.
- * @return Short read.
- *
- * @throws IOException
- */
- private int readShort(InputStream in, CRC32 crc) throws IOException {
- int b = readByte(in, crc);
- return ((readByte(in, crc) << 8) & 0x00ff00) | b;
- }
-
- /**
- * Read a byte.
- *
- * We do not expect to get a -1 reading. If we do, we throw exception.
- * Update the crc as we go.
- *
- * @param in InputStream to read.
- * @return Byte read.
- *
- * @throws IOException
- */
- protected int readByte(InputStream in) throws IOException {
- return readByte(in, null);
- }
-
- /**
- * Read a byte.
- *
- * We do not expect to get a -1 reading. If we do, we throw exception.
- * Update the crc as we go.
- *
- * @param in InputStream to read.
- * @param crc CRC to update.
- * @return Byte read.
- *
- * @throws IOException
- */
- protected int readByte(InputStream in, CRC32 crc) throws IOException {
- int b = in.read();
- if (b == -1) {
- throw new EOFException();
- }
- if (crc != null) {
- crc.update(b);
- }
- return b & 0xff;
- }
-
- /**
- * Read a byte.
- *
- * We do not expect to get a -1 reading. If we do, we throw exception.
- * Update the crc as we go.
- *
- * @param in InputStream to read.
- * @param crc CRC to update.
- * @param buffer Buffer to read into.
- * @param offset Offset to start filling buffer at.
- * @param length How much to read.
- * @return Bytes read.
- *
- * @throws IOException
- */
- protected int readByte(InputStream in, CRC32 crc, byte [] buffer,
- int offset, int length)
- throws IOException {
- for (int i = offset; i < length; i++) {
- buffer[offset + i] = (byte)readByte(in, crc);
- }
- return length;
- }
-
- /**
- * @return Returns the fextra.
- */
- public byte[] getFextra() {
- return this.fextra;
- }
-
- /**
- * @return Returns the flg.
- */
- public int getFlg() {
- return this.flg;
- }
-
- /**
- * @return Returns the os.
- */
- public int getOs() {
- return this.os;
- }
-
- /**
- * @return Returns the xfl.
- */
- public int getXfl() {
- return this.xfl;
- }
-
- /**
- * @return Returns the mtime.
- */
- public int getMtime() {
- return this.mtime;
- }
-
- /**
- * @return Returns the length.
- */
- public int getLength() {
- return length;
- }
+@Deprecated
+public class GzipHeader extends org.archive.util.zip.GzipHeader {
}
diff --git a/commons/src/main/java/org/archive/io/NoGzipMagicException.java b/commons/src/main/java/org/archive/io/NoGzipMagicException.java
index a65b78c4..27d1058a 100644
--- a/commons/src/main/java/org/archive/io/NoGzipMagicException.java
+++ b/commons/src/main/java/org/archive/io/NoGzipMagicException.java
@@ -18,13 +18,9 @@
*/
package org.archive.io;
-import java.io.IOException;
-
-public class NoGzipMagicException extends IOException {
-
- private static final long serialVersionUID = 3084169624430655013L;
-
- public NoGzipMagicException() {
- super();
- }
+/**
+ * @deprecated use {@link org.archive.util.zip.NoGzipMagicException}
+ */
+@Deprecated
+public class NoGzipMagicException extends org.archive.util.zip.NoGzipMagicException {
}
diff --git a/commons/src/main/java/org/archive/io/arc/ARCConstants.java b/commons/src/main/java/org/archive/io/arc/ARCConstants.java
index 8a5bbd31..23407236 100644
--- a/commons/src/main/java/org/archive/io/arc/ARCConstants.java
+++ b/commons/src/main/java/org/archive/io/arc/ARCConstants.java
@@ -24,7 +24,7 @@ import java.util.zip.Deflater;
import java.util.zip.GZIPInputStream;
import org.archive.io.ArchiveFileConstants;
-import org.archive.io.GzipHeader;
+import org.archive.util.zip.GzipHeader;
/**
* Constants used by ARC files and in ARC file processing.
diff --git a/commons/src/main/java/org/archive/io/arc/ARCReaderFactory.java b/commons/src/main/java/org/archive/io/arc/ARCReaderFactory.java
index 80e1d250..02173fb9 100644
--- a/commons/src/main/java/org/archive/io/arc/ARCReaderFactory.java
+++ b/commons/src/main/java/org/archive/io/arc/ARCReaderFactory.java
@@ -32,10 +32,10 @@ import org.archive.io.ArchiveReader;
import org.archive.io.ArchiveReaderFactory;
import org.archive.io.ArchiveRecord;
import org.archive.io.ArchiveRecordHeader;
-import org.archive.io.GZIPMembersInputStream;
-import org.archive.io.GzipHeader;
-import org.archive.io.NoGzipMagicException;
import org.archive.util.FileUtils;
+import org.archive.util.zip.GZIPMembersInputStream;
+import org.archive.util.zip.GzipHeader;
+import org.archive.util.zip.NoGzipMagicException;
import com.google.common.io.CountingInputStream;
diff --git a/commons/src/main/java/org/archive/io/arc/ARCUtils.java b/commons/src/main/java/org/archive/io/arc/ARCUtils.java
index ac0ce3ce..88e0e3a1 100644
--- a/commons/src/main/java/org/archive/io/arc/ARCUtils.java
+++ b/commons/src/main/java/org/archive/io/arc/ARCUtils.java
@@ -28,9 +28,9 @@ import java.io.InputStream;
import java.net.URI;
import java.net.URISyntaxException;
-import org.archive.io.GzipHeader;
-import org.archive.io.NoGzipMagicException;
import org.archive.net.UURI;
+import org.archive.util.zip.GzipHeader;
+import org.archive.util.zip.NoGzipMagicException;
public class ARCUtils implements ARCConstants {
/**
diff --git a/commons/src/main/java/org/archive/io/warc/WARCReaderFactory.java b/commons/src/main/java/org/archive/io/warc/WARCReaderFactory.java
index ef932a79..9c6c7e77 100644
--- a/commons/src/main/java/org/archive/io/warc/WARCReaderFactory.java
+++ b/commons/src/main/java/org/archive/io/warc/WARCReaderFactory.java
@@ -30,10 +30,10 @@ import java.util.Iterator;
import org.archive.io.ArchiveReader;
import org.archive.io.ArchiveReaderFactory;
import org.archive.io.ArchiveRecord;
-import org.archive.io.GZIPMembersInputStream;
import org.archive.io.warc.WARCConstants;
import org.archive.util.ArchiveUtils;
import org.archive.util.FileUtils;
+import org.archive.util.zip.GZIPMembersInputStream;
import com.google.common.io.CountingInputStream;
diff --git a/commons/src/main/java/org/archive/net/LaxURI.java b/commons/src/main/java/org/archive/net/LaxURI.java
deleted file mode 100644
index 200e9fb7..00000000
--- a/commons/src/main/java/org/archive/net/LaxURI.java
+++ /dev/null
@@ -1,459 +0,0 @@
-/*
- * This file is part of the Heritrix web crawler (crawler.archive.org).
- *
- * Licensed to the Internet Archive (IA) by one or more individual
- * contributors.
- *
- * The IA licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.archive.net;
-
-import java.util.Arrays;
-import java.util.BitSet;
-
-import org.apache.commons.httpclient.URI;
-import org.apache.commons.httpclient.URIException;
-import org.apache.commons.httpclient.util.EncodingUtil;
-
-/**
- * URI subclass which allows partial/inconsistent encoding, matching
- * the URIs which will be relayed in requests from popular web
- * browsers (esp. Mozilla Firefox and MS IE).
- *
- * @author gojomo
- */
-public class LaxURI extends URI {
-
- private static final long serialVersionUID = 5273922211722239537L;
-
- final protected static char[] HTTP_SCHEME = {'h','t','t','p'};
- final protected static char[] HTTPS_SCHEME = {'h','t','t','p','s'};
-
- protected static final BitSet lax_rel_segment = new BitSet(256);
- // Static initializer for lax_rel_segment
- static {
- lax_rel_segment.or(rel_segment);
- lax_rel_segment.set(':'); // allow ':'
- // TODO: add additional allowances as need is demonstrated
- }
-
- protected static final BitSet lax_abs_path = new BitSet(256);
- static {
- lax_abs_path.or(abs_path);
- lax_abs_path.set('|'); // tests indicate Firefox (1.0.6) doesn't escape.
- }
-
- protected static final BitSet lax_rel_path = new BitSet(256);
- // Static initializer for rel_path
- static {
- lax_rel_path.or(lax_rel_segment);
- lax_rel_path.or(lax_abs_path);
- }
-
- protected static final BitSet lax_query = new BitSet(256);
- static {
- lax_query.or(query);
- lax_query.set('{'); // tests indicate FF doesn't escape { in query
- lax_query.set('}'); // tests indicate FF doesn't escape } in query
- lax_query.set('|'); // tests indicate FF doesn't escape | in query
- lax_query.set('['); // tests indicate FF doesn't escape [ in query
- lax_query.set(']'); // tests indicate FF doesn't escape ] in query
- lax_query.set('^'); // tests indicate FF doesn't escape ^ in query
- }
-
- // passthrough initializers
- public LaxURI(String uri, boolean escaped, String charset)
- throws URIException {
- super(uri,escaped,charset);
- }
- public LaxURI(URI base, URI relative) throws URIException {
- super(base,relative);
- }
- public LaxURI(String uri, boolean escaped) throws URIException {
- super(uri,escaped);
- }
- public LaxURI() {
- super();
- }
-
- // overridden to use this class's static decode()
- public String getURI() throws URIException {
- return (_uri == null) ? null : decode(_uri, getProtocolCharset());
- }
-
- // overridden to use this class's static decode()
- public String getPath() throws URIException {
- char[] p = getRawPath();
- return (p == null) ? null : decode(p, getProtocolCharset());
- }
-
- // overridden to use this class's static decode()
- public String getPathQuery() throws URIException {
- char[] rawPathQuery = getRawPathQuery();
- return (rawPathQuery == null) ? null : decode(rawPathQuery,
- getProtocolCharset());
- }
- // overridden to use this class's static decode()
- protected static String decode(char[] component, String charset)
- throws URIException {
- if (component == null) {
- throw new IllegalArgumentException(
- "Component array of chars may not be null");
- }
- return decode(new String(component), charset);
- }
-
- // overridden to use IA's LaxURLCodec, which never throws DecoderException
- protected static String decode(String component, String charset)
- throws URIException {
- if (component == null) {
- throw new IllegalArgumentException(
- "Component array of chars may not be null");
- }
- byte[] rawdata = null;
- // try {
- rawdata = LaxURLCodec.decodeUrlLoose(EncodingUtil
- .getAsciiBytes(component));
- // } catch (DecoderException e) {
- // throw new URIException(e.getMessage());
- // }
- return EncodingUtil.getString(rawdata, charset);
- }
-
- // overidden to lax() the acceptable-char BitSet passed in
- protected boolean validate(char[] component, BitSet generous) {
- return super.validate(component, lax(generous));
- }
-
- // overidden to lax() the acceptable-char BitSet passed in
- protected boolean validate(char[] component, int soffset, int eoffset,
- BitSet generous) {
- return super.validate(component, soffset, eoffset, lax(generous));
- }
-
- /**
- * Given a BitSet -- typically one of the URI superclass's
- * predefined static variables -- possibly replace it with
- * a more-lax version to better match the character sets
- * actually left unencoded in web browser requests
- *
- * @param generous original BitSet
- * @return (possibly more lax) BitSet to use
- */
- protected BitSet lax(BitSet generous) {
- if (generous == rel_segment) {
- // Swap in more lax allowable set
- return lax_rel_segment;
- }
- if (generous == abs_path) {
- return lax_abs_path;
- }
- if (generous == query) {
- return lax_query;
- }
- if (generous == rel_path) {
- return lax_rel_path;
- }
- // otherwise, leave as is
- return generous;
- }
-
- /**
- * Coalesce the _host and _authority fields where
- * possible.
- *
- * In the web crawl/http domain, most URIs have an
- * identical _host and _authority. (There is no port
- * or user info.) However, the superclass always
- * creates two separate char[] instances.
- *
- * Notably, the lengths of these char[] fields are
- * equal if and only if their values are identical.
- * This method makes use of this fact to reduce the
- * two instances to one where possible, slimming
- * instances.
- *
- * @see org.apache.commons.httpclient.URI#parseAuthority(java.lang.String, boolean)
- */
- protected void parseAuthority(String original, boolean escaped)
- throws URIException {
- super.parseAuthority(original, escaped);
- if (_host != null && _authority != null
- && _host.length == _authority.length) {
- _host = _authority;
- }
- }
-
-
- /**
- * Coalesce _scheme to existing instances, where appropriate.
- *
- * In the web-crawl domain, most _schemes are 'http' or 'https',
- * but the superclass always creates a new char[] instance. For
- * these two cases, we replace the created instance with a
- * long-lived instance from a static field, saving 12-14 bytes
- * per instance.
- *
- * @see org.apache.commons.httpclient.URI#setURI()
- */
- protected void setURI() {
- if (_scheme != null) {
- if (_scheme.length == 4 && Arrays.equals(_scheme, HTTP_SCHEME)) {
- _scheme = HTTP_SCHEME;
- } else if (_scheme.length == 5
- && Arrays.equals(_scheme, HTTP_SCHEME)) {
- _scheme = HTTPS_SCHEME;
- }
- }
- super.setURI();
- }
-
- /**
- * IA OVERRIDDEN IN LaxURI TO INCLUDE FIX FOR
- * http://issues.apache.org/jira/browse/HTTPCLIENT-588
- * AND
- * http://webteam.archive.org/jira/browse/HER-1268
- *
- * In order to avoid any possilbity of conflict with non-ASCII characters,
- * Parse a URI reference as a String with the character
- * encoding of the local system or the document.
- *
- * The following line is the regular expression for breaking-down a URI - * reference into its components. - *
- * ^(([^:/?#]+):)?(//([^/?#]*))?([^?#]*)(\?([^#]*))?(#(.*))? - * 12 3 4 5 6 7 8 9 - *
- * For example, matching the above expression to - * http://jakarta.apache.org/ietf/uri/#Related - * results in the following subexpression matches: - *
- * $1 = http: - * scheme = $2 = http - * $3 = //jakarta.apache.org - * authority = $4 = jakarta.apache.org - * path = $5 = /ietf/uri/ - * $6 =- * query = $7 = - * $8 = #Related - * fragment = $9 = Related - *
- *
- * @param original the original character sequence
- * @param escaped true if original is escaped
- * @throws URIException If an error occurs.
- */
- protected void parseUriReference(String original, boolean escaped)
- throws URIException {
-
- // validate and contruct the URI character sequence
- if (original == null) {
- throw new URIException("URI-Reference required");
- }
-
- /* @
- * ^(([^:/?#]+):)?(//([^/?#]*))?([^?#]*)(\?([^#]*))?(#(.*))?
- */
- String tmp = original.trim();
-
- /*
- * The length of the string sequence of characters.
- * It may not be equal to the length of the byte array.
- */
- int length = tmp.length();
-
- /*
- * Remove the delimiters like angle brackets around an URI.
- */
- if (length > 0) {
- char[] firstDelimiter = { tmp.charAt(0) };
- if (validate(firstDelimiter, delims)) {
- if (length >= 2) {
- char[] lastDelimiter = { tmp.charAt(length - 1) };
- if (validate(lastDelimiter, delims)) {
- tmp = tmp.substring(1, length - 1);
- length = length - 2;
- }
- }
- }
- }
-
- /*
- * The starting index
- */
- int from = 0;
-
- /*
- * The test flag whether the URI is started from the path component.
- */
- boolean isStartedFromPath = false;
- int atColon = tmp.indexOf(':');
- int atSlash = tmp.indexOf('/');
- if (!tmp.startsWith("//")
- && (atColon <= 0 || (atSlash >= 0 && atSlash < atColon))) {
- isStartedFromPath = true;
- }
-
- /*
- *
- * @@@@@@@@ - * ^(([^:/?#]+):)?(//([^/?#]*))?([^?#]*)(\?([^#]*))?(#(.*))? - *
- */ - int at = indexFirstOf(tmp, isStartedFromPath ? "/?#" : ":/?#", from); - if (at == -1) { - at = 0; - } - - /* - * Parse the scheme. - *
- * scheme = $2 = http - * @ - * ^(([^:/?#]+):)?(//([^/?#]*))?([^?#]*)(\?([^#]*))?(#(.*))? - *
- */ - if (at > 0 && at < length && tmp.charAt(at) == ':') { - char[] target = tmp.substring(0, at).toLowerCase().toCharArray(); - if (validate(target, scheme)) { - _scheme = target; - from = ++at; - } else { - // IA CHANGE: - // do nothing; allow interpretation as URI with - // later colon in other syntactical component - } - } - - /* - * Parse the authority component. - *
- * authority = $4 = jakarta.apache.org - * @@ - * ^(([^:/?#]+):)?(//([^/?#]*))?([^?#]*)(\?([^#]*))?(#(.*))? - *
- */ - // Reset flags - _is_net_path = _is_abs_path = _is_rel_path = _is_hier_part = false; - if (0 <= at && at < length && tmp.charAt(at) == '/') { - // Set flag - _is_hier_part = true; - if (at + 2 < length && tmp.charAt(at + 1) == '/' - && !isStartedFromPath) { - // the temporary index to start the search from - int next = indexFirstOf(tmp, "/?#", at + 2); - if (next == -1) { - next = (tmp.substring(at + 2).length() == 0) ? at + 2 - : tmp.length(); - } - parseAuthority(tmp.substring(at + 2, next), escaped); - from = at = next; - // Set flag - _is_net_path = true; - } - if (from == at) { - // Set flag - _is_abs_path = true; - } - } - - /* - * Parse the path component. - *
- * path = $5 = /ietf/uri/ - * @@@@@@ - * ^(([^:/?#]+):)?(//([^/?#]*))?([^?#]*)(\?([^#]*))?(#(.*))? - *
- */ - if (from < length) { - // rel_path = rel_segment [ abs_path ] - int next = indexFirstOf(tmp, "?#", from); - if (next == -1) { - next = tmp.length(); - } - if (!_is_abs_path) { - if (!escaped - && prevalidate(tmp.substring(from, next), disallowed_rel_path) - || escaped - && validate(tmp.substring(from, next).toCharArray(), rel_path)) { - // Set flag - _is_rel_path = true; - } else if (!escaped - && prevalidate(tmp.substring(from, next), disallowed_opaque_part) - || escaped - && validate(tmp.substring(from, next).toCharArray(), opaque_part)) { - // Set flag - _is_opaque_part = true; - } else { - // the path component may be empty - _path = null; - } - } - String s = tmp.substring(from, next); - if (escaped) { - setRawPath(s.toCharArray()); - } else { - setPath(s); - } - at = next; - } - - // set the charset to do escape encoding - String charset = getProtocolCharset(); - - /* - * Parse the query component. - *
- * query = $7 =- * @@@@@@@@@ - * ^(([^:/?#]+):)?(//([^/?#]*))?([^?#]*)(\?([^#]*))?(#(.*))? - *
- */ - if (0 <= at && at + 1 < length && tmp.charAt(at) == '?') { - int next = tmp.indexOf('#', at + 1); - if (next == -1) { - next = tmp.length(); - } - if (escaped) { - _query = tmp.substring(at + 1, next).toCharArray(); - if (!validate(_query, query)) { - throw new URIException("Invalid query"); - } - } else { - _query = encode(tmp.substring(at + 1, next), allowed_query, charset); - } - at = next; - } - - /* - * Parse the fragment component. - *
- * fragment = $9 = Related - * @@@@@@@@ - * ^(([^:/?#]+):)?(//([^/?#]*))?([^?#]*)(\?([^#]*))?(#(.*))? - *
- */
- if (0 <= at && at + 1 <= length && tmp.charAt(at) == '#') {
- if (at + 1 == length) { // empty fragment
- _fragment = "".toCharArray();
- } else {
- _fragment = (escaped) ? tmp.substring(at + 1).toCharArray()
- : encode(tmp.substring(at + 1), allowed_fragment, charset);
- }
- }
-
- // set this URI.
- setURI();
- }
-
-}
diff --git a/commons/src/main/java/org/archive/net/LaxURLCodec.java b/commons/src/main/java/org/archive/net/LaxURLCodec.java
deleted file mode 100644
index ef2567bc..00000000
--- a/commons/src/main/java/org/archive/net/LaxURLCodec.java
+++ /dev/null
@@ -1,160 +0,0 @@
-/*
- * This file is part of the Heritrix web crawler (crawler.archive.org).
- *
- * Licensed to the Internet Archive (IA) by one or more individual
- * contributors.
- *
- * The IA licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.archive.net;
-
-import java.io.ByteArrayOutputStream;
-import java.io.UnsupportedEncodingException;
-import java.util.BitSet;
-
-import org.apache.commons.codec.net.URLCodec;
-
-import com.google.common.base.Charsets;
-
-/**
- * @author gojomo
- */
-public class LaxURLCodec extends URLCodec {
- public static LaxURLCodec DEFAULT = new LaxURLCodec("UTF-8");
-
- // passthrough constructor
- public LaxURLCodec(String encoding) {
- super(encoding);
- }
-
- /**
- * Decodes an array of URL safe 7-bit characters into an array of
- * original bytes. Escaped characters are converted back to their
- * original representation.
- *
- * Differs from URLCodec.decodeUrl() in that it throws no
- * exceptions; bad or incomplete escape sequences are ignored
- * and passed into result undecoded. This matches the behavior
- * of browsers, which will use inconsistently-encoded URIs
- * in HTTP request-lines.
- *
- * @param bytes array of URL safe characters
- * @return array of original bytes
- */
- public static final byte[] decodeUrlLoose(byte[] bytes)
- {
- if (bytes == null) {
- return null;
- }
- ByteArrayOutputStream buffer = new ByteArrayOutputStream();
- for (int i = 0; i < bytes.length; i++) {
- int b = bytes[i];
- if (b == '+') {
- buffer.write(' ');
- continue;
- }
- if (b == '%') {
- if(i+2 We used to use {@link java.net.URI} for parsing URIs but ran across
- * quirky behaviors and bugs. {@link java.net.URI} is not subclassable --
- * its final -- and its unlikely that java.net.URI will change any time soon
- * (See Gordon's considered petition here:
- * java.net.URI
- * should have loose/tolerant/compatibility option (or allow reuse)).
- *
- * This class tries to cache calculated strings such as the extracted host
- * and this class as a string rather than have the parent class rerun its
- * calculation everytime.
- *
- * @author gojomo
- * @author stack
- *
- * @see org.apache.commons.httpclient.URI
+ * Usable URI. The bulk of the functionality of this class has moved to
+ * {@link UsableURI} in the archive-commons project. This class adds Kryo
+ * serialization.
*/
-public class UURI extends LaxURI
-implements CharSequence, Serializable, CustomSerialization {
+public class UURI extends UsableURI implements CustomSerialization {
- private static final long serialVersionUID = -1277570889914647093L;
+ private static final long serialVersionUID = -8946640480772772310L;
- //private static Logger LOGGER =
- // Logger.getLogger(UURI.class.getName());
-
- /**
- * Consider URIs too long for IE as illegal.
- */
- public final static int MAX_URL_LENGTH = 2083;
-
- public static final String MASSAGEHOST_PATTERN = "^www\\d*\\.";
-
- /**
- * Cache of the host name.
- *
- * Super class calculates on every call. Profiling shows us spend 30% of
- * total elapsed time in URI class.
- */
- private transient String cachedHost = null;
-
- /**
- * Cache of this uuri escaped as a string.
- *
- * Super class calculates on every call. Profiling shows us spend 30% of
- * total elapsed time in URI class.
- */
- private transient String cachedEscapedURI = null;
-
- /**
- * Cache of this uuri escaped as a string.
- *
- * Super class calculates on every call. Profiling shows us spend 30% of
- * total elapsed time in URI class.
- */
- private transient String cachedString = null;
-
- /**
- * Cached authority minus userinfo.
- */
- private transient String cachedAuthorityMinusUserinfo = null;
-
- /**
- * Cache of this uuri in SURT format
- */
- private transient String surtForm = null;
-
- // Technically, underscores are disallowed in the domainlabel
- // portion of hostname according to rfc2396 but we'll be more
- // loose and allow them. See: [ 1072035 ] [uuri] Underscore in
- // host messes up port parsing.
- static {
- hostname.set('_');
+ public UURI(String fixup, boolean b, String charset) throws URIException {
+ super(fixup, b, charset);
}
-
- /**
- * Shutdown access to default constructor.
- */
+ public UURI(UsableURI base, UsableURI relative) throws URIException {
+ super(base, relative);
+ }
+
+ /* needed for kryo serialization */
protected UURI() {
super();
}
-
- /**
- * @param uri String representation of an absolute URI.
- * @param escaped If escaped.
- * @param charset Charset to use.
- * @throws org.apache.commons.httpclient.URIException
- */
- protected UURI(String uri, boolean escaped, String charset)
- throws URIException {
- super(uri, escaped, charset);
- normalize();
- }
-
- /**
- * @param relative String representation of URI.
- * @param base Parent UURI to use derelativizing.
- * @throws org.apache.commons.httpclient.URIException
- */
- protected UURI(UURI base, UURI relative) throws URIException {
- super(base, relative);
- normalize();
- }
- /**
- * @param uri String representation of a URI.
- * @param escaped If escaped.
- * @throws NullPointerException
- * @throws URIException
- */
- protected UURI(String uri, boolean escaped) throws URIException, NullPointerException {
- super(uri,escaped);
- normalize();
- }
-
- /**
- * @param uri URI as string that is resolved relative to this UURI.
- * @return UURI that uses this UURI as base.
- * @throws URIException
- */
- public UURI resolve(String uri)
- throws URIException {
- return resolve(uri, false, // assume not escaped
- this.getProtocolCharset());
- }
-
- /**
- * @param uri URI as string that is resolved relative to this UURI.
- * @param e True if escaped.
- * @return UURI that uses this UURI as base.
- * @throws URIException
- */
- public UURI resolve(String uri, boolean e)
- throws URIException {
- return resolve(uri, e, this.getProtocolCharset());
- }
-
- /**
- * @param uri URI as string that is resolved relative to this UURI.
- * @param e True if uri is escaped.
- * @param charset Charset to use.
- * @return UURI that uses this UURI as base.
- * @throws URIException
- */
- public UURI resolve(String uri, boolean e, String charset)
- throws URIException {
- return new UURI(this, new UURI(uri, e, charset));
- }
-
- /**
- * Test an object if this UURI is equal to another.
- *
- * @param obj an object to compare
- * @return true if two URI objects are equal
- */
- public boolean equals(Object obj) {
-
- // normalize and test each components
- if (obj == this) {
- return true;
- }
- if (!(obj instanceof UURI)) {
- return false;
- }
- UURI another = (UURI) obj;
- // scheme
- if (!equals(this._scheme, another._scheme)) {
- return false;
- }
- // is_opaque_part or is_hier_part? and opaque
- if (!equals(this._opaque, another._opaque)) {
- return false;
- }
- // is_hier_part
- // has_authority
- if (!equals(this._authority, another._authority)) {
- return false;
- }
- // path
- if (!equals(this._path, another._path)) {
- return false;
- }
- // has_query
- if (!equals(this._query, another._query)) {
- return false;
- }
- // UURIs do not have fragments
- return true;
- }
-
- /**
- * Strips www variants from the host.
- *
- * Strips www[0-9]*\. from the host. If calling getHostBaseName becomes a
- * performance issue we should consider adding the hostBasename member that
- * is set on initialization.
- *
- * @return Host's basename.
- * @throws URIException
- */
- public String getHostBasename() throws URIException {
- // caching eliminated because this is rarely used
- // (only benefits legacy DomainScope, which should
- // be retired). Saves 4-byte object pointer in UURI
- // instances.
- return (this.getReferencedHost() == null)
- ? null
- : TextUtils.replaceFirst(MASSAGEHOST_PATTERN,
- this.getReferencedHost(), UURIFactory.EMPTY_STRING);
- }
-
- /**
- * Returns an alternate, functional String representation -- in this
- * case, a String of the URI represented by this UURI instance.
- *
- * @return
- */
- public synchronized String toCustomString() {
- if (this.cachedString == null) {
- this.cachedString = super.toString();
- coalesceUriStrings();
- }
- return this.cachedString;
- }
-
- /**
- * Override to cache result
- *
- * TODO: eliminate, moving most callers to toCustomString, to avoid
- * overloading/diluting toString()
- * (see http://webteam.archive.org/confluence/display/Heritrix/Preserve+toString%28%29 )
- * @return String representation of this URI
- */
- public String toString() {
- return toCustomString();
- }
-
- public synchronized String getEscapedURI() {
- if (this.cachedEscapedURI == null) {
- this.cachedEscapedURI = super.getEscapedURI();
- coalesceUriStrings();
- }
- return this.cachedEscapedURI;
- }
-
- /**
- * The two String fields cachedString and cachedEscapedURI are
- * usually identical; if so, coalesce into a single instance.
- */
- protected void coalesceUriStrings() {
- if (this.cachedString != null && this.cachedEscapedURI != null
- && this.cachedString.length() == this.cachedEscapedURI.length()) {
- // lengths will only be identical if contents are identical
- // (deescaping will always shrink length), so coalesce to
- // use only single cached instance
- this.cachedString = this.cachedEscapedURI;
- }
- }
-
- public synchronized String getHost() throws URIException {
- if (this.cachedHost == null) {
- // If this._host is null, 3.0 httpclient throws
- // illegalargumentexception. Don't go there.
- if (this._host != null) {
- this.cachedHost = super.getHost();
- coalesceHostAuthorityStrings();
- }
- }
- return this.cachedHost;
- }
-
- /**
- * The two String fields cachedHost and cachedAuthorityMinusUserInfo are
- * usually identical; if so, coalesce into a single instance.
- */
- protected void coalesceHostAuthorityStrings() {
- if (this.cachedAuthorityMinusUserinfo != null
- && this.cachedHost != null
- && this.cachedHost.length() ==
- this.cachedAuthorityMinusUserinfo.length()) {
- // lengths can only be identical if contents
- // are identical; use only one instance
- this.cachedAuthorityMinusUserinfo = this.cachedHost;
- }
- }
-
- /**
- * Return the referenced host in the UURI, if any, also extracting the
- * host of a DNS-lookup URI where necessary.
- *
- * @return the target or topic host of the URI
- * @throws URIException
- */
- public String getReferencedHost() throws URIException {
- String referencedHost = this.getHost();
- if(referencedHost==null && this.getScheme().equals("dns")) {
- // extract target domain of DNS lookup
- String possibleHost = this.getCurrentHierPath();
- if(possibleHost != null && possibleHost.matches("[-_\\w\\.:]+")) {
- referencedHost = possibleHost;
- }
- }
- return referencedHost;
- }
-
- /**
- * @return Return the 'SURT' format of this UURI
- */
- public String getSurtForm() {
- if (surtForm == null) {
- surtForm = SURT.fromURI(this.toString());
- }
- return surtForm;
- }
-
- /**
- * Return the authority minus userinfo (if any).
- *
- * If no userinfo present, just returns the authority.
- *
- * @return The authority stripped of any userinfo if present.
- * @throws URIException
- */
- public String getAuthorityMinusUserinfo()
- throws URIException {
- if (this.cachedAuthorityMinusUserinfo == null) {
- String tmp = getAuthority();
- if (tmp != null && tmp.length() > 0) {
- int index = tmp.indexOf('@');
- if (index >= 0 && index < tmp.length()) {
- tmp = tmp.substring(index + 1);
- }
- }
- this.cachedAuthorityMinusUserinfo = tmp;
- coalesceHostAuthorityStrings();
- }
- return this.cachedAuthorityMinusUserinfo;
- }
-
- /* (non-Javadoc)
- * @see java.lang.CharSequence#length()
- */
- public int length() {
- return getEscapedURI().length();
- }
-
- /* (non-Javadoc)
- * @see java.lang.CharSequence#charAt(int)
- */
- public char charAt(int index) {
- return getEscapedURI().charAt(index);
- }
-
- /* (non-Javadoc)
- * @see java.lang.CharSequence#subSequence(int, int)
- */
- public CharSequence subSequence(int start, int end) {
- return getEscapedURI().subSequence(start,end);
- }
-
- /* (non-Javadoc)
- * @see java.lang.Comparable#compareTo(java.lang.Object)
- */
- public int compareTo(Object arg0) {
- return getEscapedURI().compareTo(arg0.toString());
- }
-
-
-
- /**
- * Test if passed String has likely URI scheme prefix.
- * @param possibleUrl URL string to examine.
- * @return True if passed string looks like it could be an URL.
- */
- public static boolean hasScheme(String possibleUrl) {
- boolean result = false;
- for (int i = 0; i < possibleUrl.length(); i++) {
- char c = possibleUrl.charAt(i);
- if (c == ':') {
- if (i != 0) {
- result = true;
- }
- break;
- }
- if (!scheme.get(c)) {
- break;
- }
- }
- return result;
- }
-
- /**
- * @param pathOrUri A file path or a URI.
- * @return Path parsed from passed
- * TODO: Test logging.
- *
- * @author stack
*/
-public class UURIFactory extends URI {
-
- private static final long serialVersionUID = -6146295130382209042L;
-
- /**
- * Logging instance.
- */
- private static Logger logger =
- Logger.getLogger(UURIFactory.class.getName());
+public class UURIFactory extends UsableURIFactory {
+
+ private static final long serialVersionUID = -7969477276065915936L;
/**
* The single instance of this factory.
*/
private static final UURIFactory factory = new UURIFactory();
-
- /**
- * RFC 2396-inspired regex.
- *
- * From the RFC Appendix B:
- * Below differs from the rfc regex in that...
- * (1) it has java escaping of regex characters
- * (2) we allow a URI made of a fragment only (Added extra
- * group so indexing is off by one after scheme).
- * (3) scheme is limited to legal scheme characters
- */
- final public static Pattern RFC2396REGEX = Pattern.compile(
- "^(([a-zA-Z][a-zA-Z0-9\\+\\-\\.]*):)?((//([^/?#]*))?([^?#]*)(\\?([^#]*))?)?(#(.*))?");
- // 12 34 5 6 7 8 9 A
- // 2 1 54 6 87 3 A9 // 1: scheme
- // 2: scheme:
- // 3: //authority/path
- // 4: //authority
- // 5: authority
- // 6: path
- // 7: ?query
- // 8: query
- // 9: #fragment
- // A: fragment
- public static final String SLASHDOTDOTSLASH = "^(/\\.\\./)+";
- public static final String SLASH = "/";
- public static final String HTTP = "http";
- public static final String HTTP_PORT = ":80";
- public static final String HTTPS = "https";
- public static final String HTTPS_PORT = ":443";
- public static final String DOT = ".";
- public static final String EMPTY_STRING = "";
- public static final String NBSP = "\u00A0";
- public static final String SPACE = " ";
- public static final String ESCAPED_SPACE = "%20";
- public static final String TRAILING_ESCAPED_SPACE = "^(.*)(%20)+$";
- public static final String PIPE = "|";
- public static final String PIPE_PATTERN = "\\|";
- public static final String ESCAPED_PIPE = "%7C";
- public static final String CIRCUMFLEX = "^";
- public static final String CIRCUMFLEX_PATTERN = "\\^";
- public static final String ESCAPED_CIRCUMFLEX = "%5E";
- public static final String QUOT = "\"";
- public static final String ESCAPED_QUOT = "%22";
- public static final String SQUOT = "'";
- public static final String ESCAPED_SQUOT = "%27";
- public static final String APOSTROPH = "`";
- public static final String ESCAPED_APOSTROPH = "%60";
- public static final String LSQRBRACKET = "[";
- public static final String LSQRBRACKET_PATTERN = "\\[";
- public static final String ESCAPED_LSQRBRACKET = "%5B";
- public static final String RSQRBRACKET = "]";
- public static final String RSQRBRACKET_PATTERN = "\\]";
- public static final String ESCAPED_RSQRBRACKET = "%5D";
- public static final String LCURBRACKET = "{";
- public static final String LCURBRACKET_PATTERN = "\\{";
- public static final String ESCAPED_LCURBRACKET = "%7B";
- public static final String RCURBRACKET = "}";
- public static final String RCURBRACKET_PATTERN = "\\}";
- public static final String ESCAPED_RCURBRACKET = "%7D";
- public static final String BACKSLASH = "\\";
- public static final String ESCAPED_BACKSLASH = "%5C";
- public static final String STRAY_SPACING = "[\n\r\t]+";
- public static final String IMPROPERESC_REPLACE = "%25$1";
- public static final String IMPROPERESC =
- "%((?:[^\\p{XDigit}])|(?:.[^\\p{XDigit}])|(?:\\z))";
- public static final String COMMERCIAL_AT = "@";
- public static final char PERCENT_SIGN = '%';
- public static final char COLON = ':';
-
- /**
- * First percent sign in string followed by two hex chars.
- */
- public static final String URI_HEX_ENCODING =
- "^[^%]*%[\\p{XDigit}][\\p{XDigit}].*";
-
- /**
- * Authority port number regex.
- */
- final static Pattern PORTREGEX = Pattern.compile("(.*:)([0-9]+)$");
-
- /**
- * Characters we'll accept in the domain label part of a URI
- * authority: ASCII letters-digits-hyphen (LDH) plus underscore,
- * with single intervening '.' characters.
- *
- * (We accept '_' because DNS servers have tolerated for many
- * years counter to spec; we also accept dash patterns and ACE
- * prefixes that will be rejected by IDN-punycoding attempt.)
- */
- final static String ACCEPTABLE_ASCII_DOMAIN =
- "^(?:[a-zA-Z0-9_-]++(?:\\.)?)++$";
-
- /**
- * Pattern that looks for case of three or more slashes after the
- * scheme. If found, we replace them with two only as mozilla does.
- */
- final static Pattern HTTP_SCHEME_SLASHES =
- Pattern.compile("^(https?://)/+(.*)");
-
- /**
- * Pattern that looks for case of two or more slashes in a path.
- */
- final static Pattern MULTIPLE_SLASHES = Pattern.compile("//+");
-
- /**
- * Protected constructor.
- */
- private UURIFactory() {
- super();
- }
-
/**
* @param uri URI as string.
* @return An instance of UURI
* @throws URIException
*/
public static UURI getInstance(String uri) throws URIException {
- return UURIFactory.factory.create(uri);
+ return (UURI) UURIFactory.factory.create(uri);
}
-
- /**
- * @param uri URI as string.
- * @param charset Character encoding of the passed uri string.
- * @return An instance of UURI
- * @throws URIException
- */
- public static UURI getInstance(String uri, String charset)
- throws URIException {
- return UURIFactory.factory.create(uri, charset);
- }
-
+
/**
* @param base Base uri to use resolving passed relative uri.
* @param relative URI as string.
@@ -251,552 +51,20 @@ public class UURIFactory extends URI {
* @throws URIException
*/
public static UURI getInstance(UURI base, String relative)
- throws URIException {
-// return base.resolve(relative);
- return UURIFactory.factory.create(base, relative);
+ throws URIException {
+ return (UURI) UURIFactory.factory.create(base, relative);
}
- /**
- * @param uri URI as string.
- * @return Instance of UURI.
- * @throws URIException
- */
- private UURI create(String uri) throws URIException {
- return create(uri, UURI.getDefaultProtocolCharset());
+ @Override
+ protected UURI makeOne(String fixedUpUri, boolean escaped, String charset)
+ throws URIException {
+ return new UURI(fixedUpUri, escaped, charset);
}
- /**
- * @param uri URI as string.
- * @param charset Original encoding of the string.
- * @return Instance of UURI.
- * @throws URIException
- */
- private UURI create(String uri, String charset) throws URIException {
- UURI uuri = new UURI(fixup(uri, null, charset), true, charset);
- if (logger.isLoggable(Level.FINE)) {
- logger.fine("URI " + uri +
- " PRODUCT " + uuri.toString() +
- " CHARSET " + charset);
- }
- return validityCheck(uuri);
- }
-
- /**
- * @param base UURI to use as a base resolving
- * Additionally, at least 2 significant digits are always displayed.
- *
- * Negative numbers will be returned as '0 B'.
- *
- * @param amount the amount of bytes
- * @return A string containing the amount, properly formated.
- */
- public static String formatBytesForDisplay(long amount) {
- double displayAmount = (double) amount;
- int unitPowerOf1024 = 0;
-
- if(amount <= 0){
- return "0 B";
- }
-
- while(displayAmount>=1024 && unitPowerOf1024 < 4) {
- displayAmount = displayAmount / 1024;
- unitPowerOf1024++;
- }
-
- final String[] units = { " B", " KiB", " MiB", " GiB", " TiB" };
-
- // ensure at least 2 significant digits (#.#) for small displayValues
- int fractionDigits = (displayAmount < 10) ? 1 : 0;
- return doubleToString(displayAmount, fractionDigits, fractionDigits)
- + units[unitPowerOf1024];
- }
-
- /**
- * Convert milliseconds value to a human-readable duration
- * @param time
- * @return Human readable string version of passed pString.
- * @throws UnsupportedEncodingException
- */
- public String encode(BitSet safe, String pString, String cs)
- throws UnsupportedEncodingException {
- if (pString == null) {
- return null;
- }
- return new String(encodeUrl(safe,pString.getBytes(cs)), Charsets.US_ASCII);
- }
-}
diff --git a/commons/src/main/java/org/archive/net/UURI.java b/commons/src/main/java/org/archive/net/UURI.java
index 3556760d..95885aa9 100644
--- a/commons/src/main/java/org/archive/net/UURI.java
+++ b/commons/src/main/java/org/archive/net/UURI.java
@@ -18,442 +18,49 @@
*/
package org.archive.net;
-import java.io.File;
import java.io.IOException;
import java.io.ObjectInputStream;
import java.io.ObjectOutputStream;
-import java.io.Serializable;
-import java.net.URI;
-import java.net.URISyntaxException;
import java.nio.ByteBuffer;
import org.apache.commons.httpclient.URIException;
-import org.archive.util.SURT;
-import org.archive.util.TextUtils;
+import org.archive.url.UsableURI;
import com.esotericsoftware.kryo.CustomSerialization;
import com.esotericsoftware.kryo.Kryo;
import com.esotericsoftware.kryo.serialize.StringSerializer;
-
/**
- * Usable URI.
- *
- * This class wraps {@link org.apache.commons.httpclient.URI} adding caching
- * and methods. It cannot be instantiated directly. Go via UURIFactory.
- *
- * pathOrUri.
- * @throws URISyntaxException
- */
- public static String parseFilename(final String pathOrUri)
- throws URISyntaxException {
- String path = pathOrUri;
- if (UURI.hasScheme(pathOrUri)) {
- URI url = new URI(pathOrUri);
- path = url.getPath();
- }
- return (new File(path)).getName();
- }
-
+ @Override
public void writeObjectData(Kryo kryo, ByteBuffer buffer) {
StringSerializer.put(buffer, toCustomString());
}
- public void readObjectData (Kryo kryo, ByteBuffer buffer) {
+ @Override
+ public void readObjectData(Kryo kryo, ByteBuffer buffer) {
try {
- parseUriReference(StringSerializer.get(buffer),true);
+ parseUriReference(StringSerializer.get(buffer), true);
} catch (URIException e) {
// TODO Auto-generated catch block
e.printStackTrace();
@@ -462,10 +69,10 @@ implements CharSequence, Serializable, CustomSerialization {
private void writeObject(ObjectOutputStream stream) throws IOException {
stream.writeUTF(toCustomString());
- }
-
+ }
+
private void readObject(ObjectInputStream stream) throws IOException,
- ClassNotFoundException {
- parseUriReference(stream.readUTF(),true);
+ ClassNotFoundException {
+ parseUriReference(stream.readUTF(), true);
}
}
diff --git a/commons/src/main/java/org/archive/net/UURIFactory.java b/commons/src/main/java/org/archive/net/UURIFactory.java
index 75810e70..b6db06fc 100644
--- a/commons/src/main/java/org/archive/net/UURIFactory.java
+++ b/commons/src/main/java/org/archive/net/UURIFactory.java
@@ -18,232 +18,32 @@
*/
package org.archive.net;
-import gnu.inet.encoding.IDNA;
-import gnu.inet.encoding.IDNAException;
-import it.unimi.dsi.mg4j.util.MutableString;
-
-import java.io.UnsupportedEncodingException;
-import java.util.BitSet;
-import java.util.logging.Level;
-import java.util.logging.Logger;
-import java.util.regex.Matcher;
-import java.util.regex.Pattern;
-
-import org.apache.commons.httpclient.URI;
import org.apache.commons.httpclient.URIException;
-import org.archive.util.TextUtils;
+import org.archive.url.UsableURI;
+import org.archive.url.UsableURIFactory;
/**
- * Factory that returns UURIs.
+ * Factory that returns UURIs. Mostly wraps {@link UsableURIFactory}.
*
- * Does escaping and fixup on URIs massaging in accordance with RFC2396 and to
- * match browser practice. For example, it removes any '..' if first thing in
- * the path as per IE, converts backslashes preceding the query string to
- * forward slashes, and discards any 'fragment'/anchor portion of the URI. This
- * class will also fail URIs if they are longer than IE's allowed maximum
- * length.
- *
- *
- * URI Generic Syntax August 1998
- *
- * B. Parsing a URI Reference with a Regular Expression
- *
- * As described in Section 4.3, the generic URI syntax is not sufficient
- * to disambiguate the components of some forms of URI. Since the
- * "greedy algorithm" described in that section is identical to the
- * disambiguation method used by POSIX regular expressions, it is
- * natural and commonplace to use a regular expression for parsing the
- * potential four components and fragment identifier of a URI reference.
- *
- * The following line is the regular expression for breaking-down a URI
- * reference into its components.
- *
- * ^(([^:/?#]+):)?(//([^/?#]*))?([^?#]*)(\?([^#]*))?(#(.*))?
- * 12 3 4 5 6 7 8 9
- *
- * The numbers in the second line above are only to assist readability;
- * they indicate the reference points for each subexpression (i.e., each
- * paired parenthesis). We refer to the value matched for subexpression
- *
- *
- * --
- * relative.
- * @param relative Relative URI.
- * @return Instance of UURI.
- * @throws URIException
- */
- private UURI create(UURI base, String relative) throws URIException {
- UURI uuri = new UURI(base, new UURI(fixup(relative, base, base.getProtocolCharset()),
- true, base.getProtocolCharset()));
- if (logger.isLoggable(Level.FINE)) {
- logger.fine(" URI " + relative +
- " PRODUCT " + uuri.toString() +
- " CHARSET " + base.getProtocolCharset() +
- " BASE " + base);
- }
- return validityCheck(uuri);
+ @Override
+ protected UsableURI makeOne(UsableURI base, UsableURI relative) throws URIException {
+ // return new UURI(base, relative);
+ return new UURI(base, relative);
}
- /**
- * Check the generated UURI.
- *
- * At the least look at length of uuri string. We were seeing case
- * where before escaping, string was < MAX_URL_LENGTH but after was
- * >. Letting out a too-big message was causing us troubles later
- * down the processing chain.
- * @param uuri Created uuri to check.
- * @return The passed uuri so can easily inline this check.
- * @throws URIException
- */
- protected UURI validityCheck(UURI uuri) throws URIException {
- if (uuri.getRawURI().length > UURI.MAX_URL_LENGTH) {
- throw new URIException("Created (escaped) uuri > " +
- UURI.MAX_URL_LENGTH +": "+uuri.toString());
- }
- return uuri;
- }
-
- /**
- * Do heritrix fix-up on passed uri string.
- *
- * Does heritrix escaping; usually escaping done to make our behavior align
- * with IEs. This method codifies our experience pulling URIs from the
- * wilds. Its does all the escaping we want; its output can always be
- * assumed to be 'escaped' (though perhaps to a laxer standard than the
- * vanilla HttpClient URI class or official specs might suggest).
- *
- * @param uri URI as string.
- * @param base May be null.
- * @param e True if the uri is already escaped.
- * @return A fixed up URI string.
- * @throws URIException
- */
- private String fixup(String uri, final URI base, final String charset)
- throws URIException {
- if (uri == null) {
- throw new NullPointerException();
- } else if (uri.length() == 0 && base == null) {
- throw new URIException("URI length is zero (and not relative).");
- }
-
- if (uri.length() > UURI.MAX_URL_LENGTH) {
- // We check length here and again later after all convertions.
- throw new URIException("URI length > " + UURI.MAX_URL_LENGTH +
- ": " + uri);
- }
-
- // Replace nbsp with normal spaces (so that they get stripped if at
- // ends, or encoded if in middle)
- if (uri.indexOf(NBSP) >= 0) {
- uri = TextUtils.replaceAll(NBSP, uri, SPACE);
- }
-
- // Get rid of any trailing spaces or new-lines.
- uri = uri.trim();
-
- // IE converts backslashes preceding the query string to slashes, rather
- // than to %5C. Since URIs that have backslashes usually work only with
- // IE, we will convert backslashes to slashes as well.
- int nextBackslash = uri.indexOf(BACKSLASH);
- if (nextBackslash >= 0) {
- int queryStart = uri.indexOf('?');
- StringBuilder tmp = new StringBuilder(uri);
- while (nextBackslash >= 0
- && (queryStart < 0 || nextBackslash < queryStart)) {
- tmp.setCharAt(nextBackslash, '/');
- nextBackslash = uri.indexOf(BACKSLASH, nextBackslash + 1);
- }
- uri = tmp.toString();
- }
-
- // Remove stray TAB/CR/LF
- uri = TextUtils.replaceAll(STRAY_SPACING, uri, EMPTY_STRING);
-
- // Test for the case of more than two slashes after the http(s) scheme.
- // Replace with two slashes as mozilla does if found.
- // See [ 788219 ] URI Syntax Errors stop page parsing.
-// Matcher matcher = HTTP_SCHEME_SLASHES.matcher(uri);
- Matcher matcher = TextUtils.getMatcher(HTTP_SCHEME_SLASHES.pattern(), uri);
- if (matcher.matches()) {
- uri = matcher.group(1) + matcher.group(2);
- }
- TextUtils.recycleMatcher(matcher);
-
- // now, minimally escape any whitespace
- uri = escapeWhitespace(uri);
-
- // For further processing, get uri elements. See the RFC2396REGEX
- // comment above for explanation of group indices used in the below.
-// matcher = RFC2396REGEX.matcher(uri);
- matcher = TextUtils.getMatcher(RFC2396REGEX.pattern(), uri);
- if (!matcher.matches()) {
- throw new URIException("Failed parse of " + uri);
- }
- String uriScheme = checkUriElementAndLowerCase(matcher.group(2));
- String uriSchemeSpecificPart = checkUriElement(matcher.group(3));
- String uriAuthority = checkUriElement(matcher.group(5));
- String uriPath = checkUriElement(matcher.group(6));
- String uriQuery = checkUriElement(matcher.group(8));
- // UNUSED String uriFragment = checkUriElement(matcher.group(10));
- TextUtils.recycleMatcher(matcher); matcher = null;
-
- // Test if relative URI. If so, need a base to resolve against.
- if (uriScheme == null || uriScheme.length() <= 0) {
- if (base == null) {
- throw new URIException("Relative URI but no base: " + uri);
- }
- } else {
- checkHttpSchemeSpecificPartSlashPrefix(base, uriScheme,
- uriSchemeSpecificPart);
- }
-
- // fixup authority portion: lowercase/IDN-punycode any domain;
- // remove stray trailing spaces
- uriAuthority = fixupAuthority(uriAuthority,charset);
-
- // Do some checks if absolute path.
- if (uriSchemeSpecificPart != null &&
- uriSchemeSpecificPart.startsWith(SLASH)) {
- if (uriPath != null) {
- // Eliminate '..' if its first thing in the path. IE does this.
- uriPath = TextUtils.replaceFirst(SLASHDOTDOTSLASH, uriPath,
- SLASH);
- }
- // Ensure root URLs end with '/': browsers always send "/"
- // on the request-line, so we should consider "http://host"
- // to be "http://host/".
- if (uriPath == null || EMPTY_STRING.equals(uriPath)) {
- uriPath = SLASH;
- }
- }
-
- if (uriAuthority != null) {
- if (uriScheme != null && uriScheme.length() > 0 &&
- uriScheme.equals(HTTP)) {
- uriAuthority = checkPort(uriAuthority);
- uriAuthority = stripTail(uriAuthority, HTTP_PORT);
- } else if (uriScheme != null && uriScheme.length() > 0 &&
- uriScheme.equals(HTTPS)) {
- uriAuthority = checkPort(uriAuthority);
- uriAuthority = stripTail(uriAuthority, HTTPS_PORT);
- }
- // Strip any prefix dot or tail dots from the authority.
- uriAuthority = stripTail(uriAuthority, DOT);
- uriAuthority = stripPrefix(uriAuthority, DOT);
- } else {
- // no authority; may be relative. consider stripping scheme
- // to work-around org.apache.commons.httpclient.URI bug
- // ( http://issues.apache.org/jira/browse/HTTPCLIENT-587 )
- if (uriScheme != null && base != null
- && uriScheme.equals(base.getScheme())) {
- // uriScheme redundant and will only confound httpclient.URI
- uriScheme = null;
- }
- }
-
- // Ensure minimal escaping. Use of 'lax' URI and URLCodec
- // means minimal escaping isn't necessarily complete/consistent.
- // There is a chance such lax encoding will throw exceptions
- // later at inconvenient times.
- //
- // One reason for these bad escapings -- though not the only --
- // is that the page is using an encoding other than the ASCII or the
- // UTF-8 that is our default URI encoding. In this case the parent
- // class is burping on the passed URL encoding. If the page encoding
- // was passed into this factory, the encoding seems to be parsed
- // correctly (See the testEscapedEncoding unit test).
- //
- // This fixup may cause us to miss content. There is the charset case
- // noted above. TODO: Look out for cases where we fail other than for
- // the above given reason which will be fixed when we address
- // '[ 913687 ] Make extractors interrogate for charset'.
-
- uriPath = ensureMinimalEscaping(uriPath, charset);
- uriQuery = ensureMinimalEscaping(uriQuery, charset,
- LaxURLCodec.QUERY_SAFE);
-
- // Preallocate. The '1's and '2's in below are space for ':',
- // '//', etc. URI characters.
- MutableString s = new MutableString(
- ((uriScheme != null)? uriScheme.length(): 0)
- + 1 // ';'
- + ((uriAuthority != null)? uriAuthority.length(): 0)
- + 2 // '//'
- + ((uriPath != null)? uriPath.length(): 0)
- + 1 // '?'
- + ((uriQuery != null)? uriQuery.length(): 0));
- appendNonNull(s, uriScheme, ":", true);
- appendNonNull(s, uriAuthority, "//", false);
- appendNonNull(s, uriPath, "", false);
- appendNonNull(s, uriQuery, "?", false);
- return s.toString();
- }
-
- /**
- * If http(s) scheme, check scheme specific part begins '//'.
- * @throws URIException
- * @see http://www.faqs.org/rfcs/rfc1738.html Section 3.1. Common Internet
- * Scheme Syntax
- */
- protected void checkHttpSchemeSpecificPartSlashPrefix(final URI base,
- final String scheme, final String schemeSpecificPart)
- throws URIException {
- if (scheme == null || scheme.length() <= 0) {
- return;
- }
- if (!scheme.equals("http") && !scheme.equals("https")) {
- return;
- }
- if ( schemeSpecificPart == null
- || !schemeSpecificPart.startsWith("//")) {
- // only acceptable if schemes match
- if (base == null || !scheme.equals(base.getScheme())) {
- throw new URIException(
- "relative URI with scheme only allowed for " +
- "scheme matching base");
- }
- return;
- }
- if (schemeSpecificPart.length() <= 2) {
- throw new URIException("http scheme specific part is " +
- "too short: " + schemeSpecificPart);
- }
- }
-
- /**
- * Fixup 'authority' portion of URI, by removing any stray
- * encoded spaces, lowercasing any domain names, and applying
- * IDN-punycoding to Unicode domains.
- *
- * @param uriAuthority the authority string to fix
- * @return fixed version
- * @throws URIException
- */
- private String fixupAuthority(String uriAuthority, String charset) throws URIException {
- // Lowercase the host part of the uriAuthority; don't destroy any
- // userinfo capitalizations. Make sure no illegal characters in
- // domainlabel substring of the uri authority.
- if (uriAuthority != null) {
- // Get rid of any trailing escaped spaces:
- // http://www.archive.org%20. Rare but happens.
- // TODO: reevaluate: do IE or firefox do such mid-URI space-removal?
- // if not, we shouldn't either.
- while(uriAuthority.endsWith(ESCAPED_SPACE)) {
- uriAuthority = uriAuthority.substring(0,uriAuthority.length()-3);
- }
-
- // lowercase & IDN-punycode only the domain portion
- int atIndex = uriAuthority.indexOf(COMMERCIAL_AT);
- int portColonIndex = uriAuthority.indexOf(COLON,(atIndex<0)?0:atIndex);
- if(atIndex<0 && portColonIndex<0) {
- // most common case: neither userinfo nor port
- return fixupDomainlabel(uriAuthority);
- } else if (atIndex<0 && portColonIndex>-1) {
- // next most common: port but no userinfo
- String domain = fixupDomainlabel(uriAuthority.substring(0,portColonIndex));
- String port = uriAuthority.substring(portColonIndex);
- return domain + port;
- } else if (atIndex>-1 && portColonIndex<0) {
- // uncommon: userinfo, no port
- String userinfo = ensureMinimalEscaping(uriAuthority.substring(0,atIndex+1),charset);
- String domain = fixupDomainlabel(uriAuthority.substring(atIndex+1));
- return userinfo + domain;
- } else {
- // uncommon: userinfo, port
- String userinfo = ensureMinimalEscaping(uriAuthority.substring(0,atIndex+1),charset);
- String domain = fixupDomainlabel(uriAuthority.substring(atIndex+1,portColonIndex));
- String port = uriAuthority.substring(portColonIndex);
- return userinfo + domain + port;
- }
- }
- return uriAuthority;
- }
-
- /**
- * Fixup the domain label part of the authority.
- *
- * We're more lax than the spec. in that we allow underscores.
- *
- * @param label Domain label to fix.
- * @return Return fixed domain label.
- * @throws URIException
- */
- private String fixupDomainlabel(String label)
- throws URIException {
-
- // apply IDN-punycoding, as necessary
- try {
- // TODO: optimize: only apply when necessary, or
- // keep cache of recent encodings
- label = IDNA.toASCII(label);
- } catch (IDNAException e) {
- if(TextUtils.matches(ACCEPTABLE_ASCII_DOMAIN,label)) {
- // domain name has ACE prefix, leading/trailing dash, or
- // underscore -- but is still a name we wish to tolerate;
- // simply continue
- } else {
- // problematic domain: neither ASCII acceptable characters
- // nor IDN-punycodable, so throw exception
- // TODO: change to HeritrixURIException so distinguishable
- // from URIExceptions in library code
- URIException ue = new URIException(e+" "+label);
- ue.initCause(e);
- throw ue;
- }
- }
- label = label.toLowerCase();
- return label;
- }
-
- /**
- * Ensure that there all characters needing escaping
- * in the passed-in String are escaped. Stray '%' characters
- * are *not* escaped, as per browser behavior.
- *
- * @param u String to escape
- * @param charset
- * @return string with any necessary escaping applied
- */
- private String ensureMinimalEscaping(String u, final String charset) {
- return ensureMinimalEscaping(u, charset, LaxURLCodec.EXPANDED_URI_SAFE);
- }
-
- /**
- * Ensure that there all characters needing escaping
- * in the passed-in String are escaped. Stray '%' characters
- * are *not* escaped, as per browser behavior.
- *
- * @param u String to escape
- * @param charset
- * @param bitset
- * @return string with any necessary escaping applied
- */
- private String ensureMinimalEscaping(String u, final String charset,
- final BitSet bitset) {
- if (u == null) {
- return null;
- }
- for (int i = 0; i < u.length(); i++) {
- char c = u.charAt(i);
- if (!bitset.get(c)) {
- try {
- u = LaxURLCodec.DEFAULT.encode(bitset, u, charset);
- } catch (UnsupportedEncodingException e) {
- e.printStackTrace();
- }
- break;
- }
- }
- return u;
- }
-
- /**
- * Escape any whitespace found.
- *
- * The parent class takes care of the bulk of escaping. But if any
- * instance of escaping is found in the URI, then we ask for parent
- * to do NO escaping. Here we escape any whitespace found irrespective
- * of whether the uri has already been escaped. We do this for
- * case where uri has been judged already-escaped only, its been
- * incompletly done and whitespace remains. Spaces, etc., in the URI are
- * a real pain. Their presence will break log file and ARC parsing.
- * @param uri URI string to check.
- * @return uri with spaces escaped if any found.
- */
- protected String escapeWhitespace(String uri) {
- // Just write a new string anyways. The perl '\s' is not
- // as inclusive as the Character.isWhitespace so there are
- // whitespace characters we could miss. So, rather than
- // write some awkward regex, just go through the string
- // a character at a time. Only create buffer first time
- // we find a space.
- MutableString buffer = null;
- for (int i = 0; i < uri.length(); i++) {
- char c = uri.charAt(i);
- if (Character.isWhitespace(c)) {
- if (buffer == null) {
- buffer = new MutableString(uri.length() +
- 2 /*If space, two extra characters (at least)*/);
- buffer.append(uri.substring(0, i));
- }
- buffer.append("%");
- String hexStr = Integer.toHexString(c);
- if ((hexStr.length() % 2) > 0) {
- buffer.append("0");
- }
- buffer.append(hexStr);
-
- } else {
- if (buffer != null) {
- buffer.append(c);
- }
- }
- }
- return (buffer != null)? buffer.toString(): uri;
- }
-
- /**
- * Check port on passed http authority. Make sure the size is not larger
- * than allowed: See the 'port' definition on this
- * page, http://www.kerio.com/manual/wrp/en/418.htm.
- * Also, we've seen port numbers of '0080' whose leading zeros confuse
- * the parent class. Strip the leading zeros.
- *
- * @param uriAuthority
- * @return Null or an amended port number.
- * @throws URIException
- */
- private String checkPort(String uriAuthority)
- throws URIException {
-// Matcher m = PORTREGEX.matcher(uriAuthority);
- Matcher m = TextUtils.getMatcher(PORTREGEX.pattern(), uriAuthority);
- if (m.matches()) {
- String no = m.group(2);
- if (no != null && no.length() > 0) {
- // First check if the port has leading zeros
- // as in '0080'. Strip them if it has and
- // then reconstitute the uriAuthority. Be careful
- // of cases where port is '0' or '000'.
- while (no.charAt(0) == '0' && no.length() > 1) {
- no = no.substring(1);
- }
- uriAuthority = m.group(1) + no;
- // Now makesure the number is legit.
- int portNo = 0;
- try {
- portNo = Integer.parseInt(no);
- } catch (NumberFormatException nfe) {
- // just catch and leave portNo at illegal 0
- }
- if (portNo <= 0 || portNo > 65535) {
- throw new URIException("Port out of bounds: " +
- uriAuthority);
- }
- }
- }
- TextUtils.recycleMatcher(m);
- return uriAuthority;
- }
-
- /**
- * @param b Buffer to append to.
- * @param str String to append if not null.
- * @param substr Suffix or prefix to use if str is not null.
- * @param suffix True if substr is a suffix.
- */
- private void appendNonNull(MutableString b, String str, String substr,
- boolean suffix) {
- if (str != null && str.length() > 0) {
- if (!suffix) {
- b.append(substr);
- }
- b.append(str);
- if (suffix) {
- b.append(substr);
- }
- }
- }
-
- /**
- * @param str String to work on.
- * @param prefix Prefix to strip if present.
- * @return str w/o prefix.
- */
- private String stripPrefix(String str, String prefix) {
- return str.startsWith(prefix)?
- str.substring(prefix.length(), str.length()):
- str;
- }
-
- /**
- * @param str String to work on.
- * @param tail Tail to strip if present.
- * @return str w/o tail.
- */
- private static String stripTail(String str, String tail) {
- return str.endsWith(tail)?
- str.substring(0, str.length() - tail.length()):
- str;
- }
-
- /**
- * @param element to examine.
- * @return Null if passed null or an empty string otherwise
- * element.
- */
- private String checkUriElement(String element) {
- return (element == null || element.length() <= 0)? null: element;
- }
-
- /**
- * @param element to examine and lowercase if non-null.
- * @return Null if passed null or an empty string otherwise
- * element lowercased.
- */
- private String checkUriElementAndLowerCase(String element) {
- String tmp = checkUriElement(element);
- return (tmp != null)? tmp.toLowerCase(): tmp;
- }
}
diff --git a/commons/src/main/java/org/archive/util/ArchiveUtils.java b/commons/src/main/java/org/archive/util/ArchiveUtils.java
deleted file mode 100644
index fe2da96f..00000000
--- a/commons/src/main/java/org/archive/util/ArchiveUtils.java
+++ /dev/null
@@ -1,1008 +0,0 @@
-/*
- * This file is part of the Heritrix web crawler (crawler.archive.org).
- *
- * Licensed to the Internet Archive (IA) by one or more individual
- * contributors.
- *
- * The IA licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.archive.util;
-
-import java.io.BufferedInputStream;
-import java.io.BufferedReader;
-import java.io.ByteArrayOutputStream;
-import java.io.Closeable;
-import java.io.EOFException;
-import java.io.File;
-import java.io.FileInputStream;
-import java.io.IOException;
-import java.io.InputStream;
-import java.io.InputStreamReader;
-import java.io.PrintWriter;
-import java.io.StringWriter;
-import java.net.URL;
-import java.net.URLConnection;
-import java.text.NumberFormat;
-import java.text.ParseException;
-import java.text.SimpleDateFormat;
-import java.util.Date;
-import java.util.HashSet;
-import java.util.Locale;
-import java.util.Map;
-import java.util.Set;
-import java.util.TimeZone;
-import java.util.logging.Level;
-import java.util.logging.Logger;
-import java.util.zip.GZIPInputStream;
-import java.util.zip.GZIPOutputStream;
-
-import org.apache.commons.io.IOUtils;
-import org.archive.io.GzipHeader;
-import org.archive.io.NoGzipMagicException;
-
-
-/**
- * Miscellaneous useful methods.
- *
- * @author gojomo & others
- */
-public class ArchiveUtils {
- private static final Logger LOGGER = Logger.getLogger(ArchiveUtils.class.getName());
-
-
- final public static String VERSION = loadVersion();
-
- /**
- * Arc-style date stamp in the format yyyyMMddHHmm and UTC time zone.
- */
- private static final ThreadLocalyyyy to
- * yyyyMMddHHmmssSSS) formatting.
- * @return A Date object representing the passed String.
- * @throws ParseException
- */
- public static Date getDate(String d) throws ParseException {
- Date date = null;
- if (d == null) {
- throw new IllegalArgumentException("Passed date is null");
- }
- switch (d.length()) {
- case 14:
- date = ArchiveUtils.parse14DigitDate(d);
- break;
-
- case 17:
- date = ArchiveUtils.parse17DigitDate(d);
- break;
-
- case 12:
- date = ArchiveUtils.parse12DigitDate(d);
- break;
-
- case 0:
- case 1:
- case 2:
- case 3:
- throw new ParseException("Date string must at least contain a" +
- "year: " + d, d.length());
-
- default:
- if (!(d.startsWith("19") || d.startsWith("20"))) {
- throw new ParseException("Unrecognized century: " + d, 0);
- }
- if (d.length() < 8 && (d.length() % 2) != 0) {
- throw new ParseException("Incomplete month/date: " + d,
- d.length());
- }
- StringBuilder sb = new StringBuilder(d);
- while (sb.length() < 8) {
- sb.append("01");
- }
- while (sb.length() < 12) {
- sb.append("0");
- }
- date = ArchiveUtils.parse12DigitDate(sb.toString());
- }
-
- return date;
- }
-
- /**
- * Utility function for parsing arc-style date stamps
- * in the format yyyMMddHHmmssSSS.
- * Date stamps are in the UTC time zone. The whole string will not be
- * parsed, only the first 17 digits.
- *
- * @param date an arc-style formatted date stamp
- * @return the Date corresponding to the date stamp string
- * @throws ParseException if the inputstring was malformed
- */
- public static Date parse17DigitDate(String date) throws ParseException {
- return TIMESTAMP17.get().parse(date);
- }
-
- /**
- * Utility function for parsing arc-style date stamps
- * in the format yyyMMddHHmmss.
- * Date stamps are in the UTC time zone. The whole string will not be
- * parsed, only the first 14 digits.
- *
- * @param date an arc-style formatted date stamp
- * @return the Date corresponding to the date stamp string
- * @throws ParseException if the inputstring was malformed
- */
- public static Date parse14DigitDate(String date) throws ParseException{
- return TIMESTAMP14.get().parse(date);
- }
-
- /**
- * Utility function for parsing arc-style date stamps
- * in the format yyyMMddHHmm.
- * Date stamps are in the UTC time zone. The whole string will not be
- * parsed, only the first 12 digits.
- *
- * @param date an arc-style formatted date stamp
- * @return the Date corresponding to the date stamp string
- * @throws ParseException if the inputstring was malformed
- */
- public static Date parse12DigitDate(String date) throws ParseException{
- return TIMESTAMP12.get().parse(date);
- }
-
- /**
- * @param timestamp A 14-digit timestamp or the suffix for a 14-digit
- * timestamp: E.g. '20010909014640' or '20010101' or '1970'.
- * @return Seconds since the epoch as a string zero-pre-padded so always
- * Integer.MAX_VALUE wide (Makes it so sorting of resultant string works
- * properly).
- * @throws ParseException
- */
- public static String secondsSinceEpoch(String timestamp)
- throws ParseException {
- return zeroPadInteger((int)
- (getSecondsSinceEpoch(timestamp).getTime()/1000));
- }
-
- /**
- * @param timestamp A 14-digit timestamp or the suffix for a 14-digit
- * timestamp: E.g. '20010909014640' or '20010101' or '1970'.
- * @return A date.
- * @see #secondsSinceEpoch(String)
- * @throws ParseException
- */
- public static Date getSecondsSinceEpoch(String timestamp)
- throws ParseException {
- if (timestamp.length() < 14) {
- if (timestamp.length() < 10 && (timestamp.length() % 2) == 1) {
- throw new IllegalArgumentException("Must have year, " +
- "month, date, hour or second granularity: " + timestamp);
- }
- if (timestamp.length() == 4) {
- // Add first month and first date.
- timestamp = timestamp + "01010000";
- }
- if (timestamp.length() == 6) {
- // Add a date of the first.
- timestamp = timestamp + "010000";
- }
- if (timestamp.length() < 14) {
- timestamp = timestamp +
- ArchiveUtils.padTo("", 14 - timestamp.length(), '0');
- }
- }
- return ArchiveUtils.parse14DigitDate(timestamp);
- }
-
- /**
- * @param i Integer to add prefix of zeros too. If passed
- * 2005, will return the String 0000002005. String
- * width is the width of Integer.MAX_VALUE as a string (10
- * digits).
- * @return Padded String version of i.
- */
- public static String zeroPadInteger(int i) {
- return ArchiveUtils.padTo(Integer.toString(i),
- MAX_INT_CHAR_WIDTH, '0');
- }
-
- /**
- * Convert an int to a String, and pad it to
- * pad spaces.
- * @param i the int
- * @param pad the width to pad to.
- * @return String w/ padding.
- */
- public static String padTo(final int i, final int pad) {
- String n = Integer.toString(i);
- return padTo(n, pad);
- }
-
- /**
- * Pad the given String to pad characters wide
- * by pre-pending spaces. s should not be null.
- * If s is already wider than pad no change is
- * done.
- *
- * @param s the String to pad
- * @param pad the width to pad to.
- * @return String w/ padding.
- */
- public static String padTo(final String s, final int pad) {
- return padTo(s, pad, DEFAULT_PAD_CHAR);
- }
-
- /**
- * Pad the given String to pad characters wide
- * by pre-pending padChar.
- *
- * s should not be null. If s is
- * already wider than pad no change is done.
- *
- * @param s the String to pad
- * @param pad the width to pad to.
- * @param padChar The pad character to use.
- * @return String w/ padding.
- */
- public static String padTo(final String s, final int pad,
- final char padChar) {
- String result = s;
- int l = s.length();
- if (l < pad) {
- StringBuffer sb = new StringBuffer(pad);
- while(l < pad) {
- sb.append(padChar);
- l++;
- }
- sb.append(s);
- result = sb.toString();
- }
- return result;
- }
-
- /** check that two byte arrays are equal. They may be null.
- *
- * @param lhs a byte array
- * @param rhs another byte array.
- * @return true if they are both equal (or both
- * null)
- */
- public static boolean byteArrayEquals(final byte[] lhs, final byte[] rhs) {
- if (lhs == null && rhs != null || lhs != null && rhs == null) {
- return false;
- }
- if (lhs==rhs) {
- return true;
- }
- if (lhs.length != rhs.length) {
- return false;
- }
- for(int i = 0; itime
- */
- public static String formatMillisecondsToConventional(long time) {
- return formatMillisecondsToConventional(time,5);
- }
-
- /**
- * Convert milliseconds value to a human-readable duration of
- * mixed units, using units no larger than days. For example,
- * "5d12h13m12s113ms" or "19h51m".
- *
- * @param duration
- * @param unitCount how many significant units to show, at most
- * for example, a value of 2 would show days+hours or hours+seconds
- * but not hours+second+milliseconds
- * @return Human readable string version of passed time
- */
- public static String formatMillisecondsToConventional(long duration, int unitCount) {
- if(unitCount <=0) {
- unitCount = 5;
- }
- if(duration==0) {
- return "0ms";
- }
- StringBuffer sb = new StringBuffer();
- if(duration<0) {
- sb.append("-");
- }
- long absTime = Math.abs(duration);
- long[] thresholds = {DAY_IN_MS, HOUR_IN_MS, 60000, 1000, 1};
- String[] units = {"d","h","m","s","ms"};
-
- for(int i = 0; i < thresholds.length; i++) {
- if(absTime >= thresholds[i]) {
- sb.append(absTime / thresholds[i] + units[i]);
- absTime = absTime % thresholds[i];
- unitCount--;
- }
- if(unitCount==0) {
- break;
- }
- }
- return sb.toString();
- }
-
- /**
- * Copy the raw bytes of a long into a byte array, starting at
- * the specified offset.
- *
- * @param l
- * @param array
- * @param offset
- */
- public static void longIntoByteArray(long l, byte[] array, int offset) {
- int i, shift;
-
- for(i = 0, shift = 56; i < 8; i++, shift -= 8)
- array[offset+i] = (byte)(0xFF & (l >> shift));
- }
-
- public static long byteArrayIntoLong(byte [] bytearray) {
- return byteArrayIntoLong(bytearray, 0);
- }
-
- /**
- * Byte array into long.
- * @param bytearray Array to convert to a long.
- * @param offset Offset into array at which we start decoding the long.
- * @return Long made of the bytes of array beginning at
- * offset offset.
- * @see #longIntoByteArray(long, byte[], int)
- */
- public static long byteArrayIntoLong(byte [] bytearray,
- int offset) {
- long result = 0;
- for (int i = offset; i < 8 /*Bytes in long*/; i++) {
- result = (result << 8 /*Bits in byte*/) |
- (0xff & (byte)(bytearray[i] & 0xff));
- }
- return result;
- }
-
- /**
- * Given a string that may be a plain host or host/path (without
- * URI scheme), add an implied http:// if necessary.
- *
- * @param u string to evaluate
- * @return string with http:// added if no scheme already present
- */
- public static String addImpliedHttpIfNecessary(String u) {
- int colon = u.indexOf(':');
- int period = u.indexOf('.');
- if (colon == -1 || (period >= 0) && (period < colon)) {
- // No scheme present; prepend "http://"
- u = "http://" + u;
- }
- return u;
- }
-
- /**
- * Verify that the array begins with the prefix.
- *
- * @param array
- * @param prefix
- * @return true if array is identical to prefix for the first prefix.length
- * positions
- */
- public static boolean startsWith(byte[] array, byte[] prefix) {
- if(prefix.length>array.length) {
- return false;
- }
- for(int i = 0; i < prefix.length; i++) {
- if(array[i]!=prefix[i]) {
- return false;
- }
- }
- return true;
- }
-
- /**
- * Utility method to get a String shortReportLine from Reporter
- * @param rep Reporter to get shortReportLine from
- * @return String of report
- */
- public static String shortReportLine(Reporter rep) {
- StringWriter sw = new StringWriter();
- PrintWriter pw = new PrintWriter(sw);
- try {
- rep.shortReportLineTo(pw);
- } catch (IOException e) {
- // not really possible
- e.printStackTrace();
- }
- pw.flush();
- return sw.toString();
- }
-
- /**
- * Enhance given object's default String display for appearing
- * nested in a pretty Map String.
- *
- * @param obj Object to prettify
- * @return prettified String
- */
- public static String prettyString(Object obj) {
- // these things have to checked and casted unfortunately
- if (obj instanceof Object[]) {
- return prettyString((Object[]) obj);
- } else if (obj instanceof Map) {
- return prettyString((Map) obj);
- } else {
- return "<"+obj+">";
- }
- }
-
- /**
- * Provide a improved String of a Map's entries
- *
- * @param Map
- * @return prettified (in curly brackets) string of Map contents
- */
- public static String prettyString(Map map) {
- StringBuilder builder = new StringBuilder();
- builder.append("{ ");
- boolean needsComma = false;
- for( Object key : map.keySet()) {
- if(needsComma) {
- builder.append(", ");
- }
- builder.append(key);
- builder.append(": ");
- builder.append(prettyString(map.get(key)));
- needsComma = true;
- }
- builder.append(" }");
- return builder.toString();
- }
-
- /**
- * Provide a slightly-improved String of Object[]
- *
- * @param Object[]
- * @return prettified (in square brackets) of Object[]
- */
- public static String prettyString(Object[] array) {
- StringBuilder builder = new StringBuilder();
- builder.append("[ ");
- boolean needsComma = false;
- for (Object o: array) {
- if(o==null) continue;
- if(needsComma) {
- builder.append(", ");
- }
- builder.append(prettyString(o));
- needsComma = true;
- }
- builder.append(" ]");
- return builder.toString();
- }
-
-
- private static String loadVersion() {
- InputStream input = ArchiveUtils.class.getResourceAsStream(
- "/org/archive/util/version.txt");
- if (input == null) {
- return "UNKNOWN";
- }
- BufferedReader br = null;
- String version;
- try {
- br = new BufferedReader(new InputStreamReader(input));
- version = br.readLine();
- br.readLine();
- } catch (IOException e) {
- return e.getMessage();
- } finally {
- closeQuietly(br);
- }
-
- version = version.trim();
- if (!version.endsWith("SNAPSHOT")) {
- return version;
- }
-
- input = ArchiveUtils.class.getResourceAsStream("/org/archive/util/timestamp.txt");
- if (input == null) {
- return version;
- }
-
- br = null;
- String timestamp;
- try {
- br = new BufferedReader(new InputStreamReader(input));
- timestamp = br.readLine();
- } catch (IOException e) {
- return version;
- } finally {
- closeQuietly(br);
- }
-
- if (timestamp.startsWith("timestamp=")) {
- timestamp = timestamp.substring(10);
- }
-
- return version.trim() + "-" + timestamp.trim();
- }
-
- public static Set