diff --git a/.classpath b/.classpath
index 5d03c472..0c8f1ff7 100644
--- a/.classpath
+++ b/.classpath
@@ -14,12 +14,11 @@
-
-
+
@@ -59,5 +58,6 @@
+
diff --git a/commons/pom.xml b/commons/pom.xml
index 790e31bb..4d3ae25c 100644
--- a/commons/pom.xml
+++ b/commons/pom.xml
@@ -191,12 +191,6 @@
5.0.7compile
-
- org.gnu.inet
- libidn
- 0.6.5
- compile
- net.java.dev.jets3tjets3t
@@ -282,6 +276,12 @@
3.2.3
+
+ org.archive
+ archive-commons
+ 1.0-SNAPSHOT
+
+
diff --git a/commons/src/main/java/org/apache/commons/httpclient/HttpMethodBase.java b/commons/src/main/java/org/apache/commons/httpclient/HttpMethodBase.java
index cda1b9d2..aa171b04 100644
--- a/commons/src/main/java/org/apache/commons/httpclient/HttpMethodBase.java
+++ b/commons/src/main/java/org/apache/commons/httpclient/HttpMethodBase.java
@@ -217,7 +217,7 @@ public abstract class HttpMethodBase implements HttpMethod {
}
// BEGIN IA/HERITRIX CHANGES
// setURI(new URI(uri, true));
- setURI(new org.archive.net.LaxURI(uri, true));
+ setURI(new org.archive.url.LaxURI(uri, true));
// END IA/HERITRIX CHANGES
} catch (URIException e) {
throw new IllegalArgumentException("Invalid uri '"
@@ -264,7 +264,7 @@ public abstract class HttpMethodBase implements HttpMethod {
}
// BEGIN IA/HERITRIX CHANGES
// return new URI(buffer.toString(), true);
- return new org.archive.net.LaxURI(buffer.toString(), true);
+ return new org.archive.url.LaxURI(buffer.toString(), true);
// END IA/HERITRIX CHANGES
}
diff --git a/commons/src/main/java/org/archive/net/LaxURI.java b/commons/src/main/java/org/archive/net/LaxURI.java
deleted file mode 100644
index 200e9fb7..00000000
--- a/commons/src/main/java/org/archive/net/LaxURI.java
+++ /dev/null
@@ -1,459 +0,0 @@
-/*
- * This file is part of the Heritrix web crawler (crawler.archive.org).
- *
- * Licensed to the Internet Archive (IA) by one or more individual
- * contributors.
- *
- * The IA licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.archive.net;
-
-import java.util.Arrays;
-import java.util.BitSet;
-
-import org.apache.commons.httpclient.URI;
-import org.apache.commons.httpclient.URIException;
-import org.apache.commons.httpclient.util.EncodingUtil;
-
-/**
- * URI subclass which allows partial/inconsistent encoding, matching
- * the URIs which will be relayed in requests from popular web
- * browsers (esp. Mozilla Firefox and MS IE).
- *
- * @author gojomo
- */
-public class LaxURI extends URI {
-
- private static final long serialVersionUID = 5273922211722239537L;
-
- final protected static char[] HTTP_SCHEME = {'h','t','t','p'};
- final protected static char[] HTTPS_SCHEME = {'h','t','t','p','s'};
-
- protected static final BitSet lax_rel_segment = new BitSet(256);
- // Static initializer for lax_rel_segment
- static {
- lax_rel_segment.or(rel_segment);
- lax_rel_segment.set(':'); // allow ':'
- // TODO: add additional allowances as need is demonstrated
- }
-
- protected static final BitSet lax_abs_path = new BitSet(256);
- static {
- lax_abs_path.or(abs_path);
- lax_abs_path.set('|'); // tests indicate Firefox (1.0.6) doesn't escape.
- }
-
- protected static final BitSet lax_rel_path = new BitSet(256);
- // Static initializer for rel_path
- static {
- lax_rel_path.or(lax_rel_segment);
- lax_rel_path.or(lax_abs_path);
- }
-
- protected static final BitSet lax_query = new BitSet(256);
- static {
- lax_query.or(query);
- lax_query.set('{'); // tests indicate FF doesn't escape { in query
- lax_query.set('}'); // tests indicate FF doesn't escape } in query
- lax_query.set('|'); // tests indicate FF doesn't escape | in query
- lax_query.set('['); // tests indicate FF doesn't escape [ in query
- lax_query.set(']'); // tests indicate FF doesn't escape ] in query
- lax_query.set('^'); // tests indicate FF doesn't escape ^ in query
- }
-
- // passthrough initializers
- public LaxURI(String uri, boolean escaped, String charset)
- throws URIException {
- super(uri,escaped,charset);
- }
- public LaxURI(URI base, URI relative) throws URIException {
- super(base,relative);
- }
- public LaxURI(String uri, boolean escaped) throws URIException {
- super(uri,escaped);
- }
- public LaxURI() {
- super();
- }
-
- // overridden to use this class's static decode()
- public String getURI() throws URIException {
- return (_uri == null) ? null : decode(_uri, getProtocolCharset());
- }
-
- // overridden to use this class's static decode()
- public String getPath() throws URIException {
- char[] p = getRawPath();
- return (p == null) ? null : decode(p, getProtocolCharset());
- }
-
- // overridden to use this class's static decode()
- public String getPathQuery() throws URIException {
- char[] rawPathQuery = getRawPathQuery();
- return (rawPathQuery == null) ? null : decode(rawPathQuery,
- getProtocolCharset());
- }
- // overridden to use this class's static decode()
- protected static String decode(char[] component, String charset)
- throws URIException {
- if (component == null) {
- throw new IllegalArgumentException(
- "Component array of chars may not be null");
- }
- return decode(new String(component), charset);
- }
-
- // overridden to use IA's LaxURLCodec, which never throws DecoderException
- protected static String decode(String component, String charset)
- throws URIException {
- if (component == null) {
- throw new IllegalArgumentException(
- "Component array of chars may not be null");
- }
- byte[] rawdata = null;
- // try {
- rawdata = LaxURLCodec.decodeUrlLoose(EncodingUtil
- .getAsciiBytes(component));
- // } catch (DecoderException e) {
- // throw new URIException(e.getMessage());
- // }
- return EncodingUtil.getString(rawdata, charset);
- }
-
- // overidden to lax() the acceptable-char BitSet passed in
- protected boolean validate(char[] component, BitSet generous) {
- return super.validate(component, lax(generous));
- }
-
- // overidden to lax() the acceptable-char BitSet passed in
- protected boolean validate(char[] component, int soffset, int eoffset,
- BitSet generous) {
- return super.validate(component, soffset, eoffset, lax(generous));
- }
-
- /**
- * Given a BitSet -- typically one of the URI superclass's
- * predefined static variables -- possibly replace it with
- * a more-lax version to better match the character sets
- * actually left unencoded in web browser requests
- *
- * @param generous original BitSet
- * @return (possibly more lax) BitSet to use
- */
- protected BitSet lax(BitSet generous) {
- if (generous == rel_segment) {
- // Swap in more lax allowable set
- return lax_rel_segment;
- }
- if (generous == abs_path) {
- return lax_abs_path;
- }
- if (generous == query) {
- return lax_query;
- }
- if (generous == rel_path) {
- return lax_rel_path;
- }
- // otherwise, leave as is
- return generous;
- }
-
- /**
- * Coalesce the _host and _authority fields where
- * possible.
- *
- * In the web crawl/http domain, most URIs have an
- * identical _host and _authority. (There is no port
- * or user info.) However, the superclass always
- * creates two separate char[] instances.
- *
- * Notably, the lengths of these char[] fields are
- * equal if and only if their values are identical.
- * This method makes use of this fact to reduce the
- * two instances to one where possible, slimming
- * instances.
- *
- * @see org.apache.commons.httpclient.URI#parseAuthority(java.lang.String, boolean)
- */
- protected void parseAuthority(String original, boolean escaped)
- throws URIException {
- super.parseAuthority(original, escaped);
- if (_host != null && _authority != null
- && _host.length == _authority.length) {
- _host = _authority;
- }
- }
-
-
- /**
- * Coalesce _scheme to existing instances, where appropriate.
- *
- * In the web-crawl domain, most _schemes are 'http' or 'https',
- * but the superclass always creates a new char[] instance. For
- * these two cases, we replace the created instance with a
- * long-lived instance from a static field, saving 12-14 bytes
- * per instance.
- *
- * @see org.apache.commons.httpclient.URI#setURI()
- */
- protected void setURI() {
- if (_scheme != null) {
- if (_scheme.length == 4 && Arrays.equals(_scheme, HTTP_SCHEME)) {
- _scheme = HTTP_SCHEME;
- } else if (_scheme.length == 5
- && Arrays.equals(_scheme, HTTP_SCHEME)) {
- _scheme = HTTPS_SCHEME;
- }
- }
- super.setURI();
- }
-
- /**
- * IA OVERRIDDEN IN LaxURI TO INCLUDE FIX FOR
- * http://issues.apache.org/jira/browse/HTTPCLIENT-588
- * AND
- * http://webteam.archive.org/jira/browse/HER-1268
- *
- * In order to avoid any possilbity of conflict with non-ASCII characters,
- * Parse a URI reference as a String with the character
- * encoding of the local system or the document.
- *
- * The following line is the regular expression for breaking-down a URI
- * reference into its components.
- *
- * For example, matching the above expression to
- * http://jakarta.apache.org/ietf/uri/#Related
- * results in the following subexpression matches:
- *
- */
- if (0 <= at && at + 1 < length && tmp.charAt(at) == '?') {
- int next = tmp.indexOf('#', at + 1);
- if (next == -1) {
- next = tmp.length();
- }
- if (escaped) {
- _query = tmp.substring(at + 1, next).toCharArray();
- if (!validate(_query, query)) {
- throw new URIException("Invalid query");
- }
- } else {
- _query = encode(tmp.substring(at + 1, next), allowed_query, charset);
- }
- at = next;
- }
-
- /*
- * Parse the fragment component.
- *
- * fragment = $9 = Related
- * @@@@@@@@
- * ^(([^:/?#]+):)?(//([^/?#]*))?([^?#]*)(\?([^#]*))?(#(.*))?
- *
- */
- if (0 <= at && at + 1 <= length && tmp.charAt(at) == '#') {
- if (at + 1 == length) { // empty fragment
- _fragment = "".toCharArray();
- } else {
- _fragment = (escaped) ? tmp.substring(at + 1).toCharArray()
- : encode(tmp.substring(at + 1), allowed_fragment, charset);
- }
- }
-
- // set this URI.
- setURI();
- }
-
-}
diff --git a/commons/src/main/java/org/archive/net/LaxURLCodec.java b/commons/src/main/java/org/archive/net/LaxURLCodec.java
deleted file mode 100644
index ef2567bc..00000000
--- a/commons/src/main/java/org/archive/net/LaxURLCodec.java
+++ /dev/null
@@ -1,160 +0,0 @@
-/*
- * This file is part of the Heritrix web crawler (crawler.archive.org).
- *
- * Licensed to the Internet Archive (IA) by one or more individual
- * contributors.
- *
- * The IA licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.archive.net;
-
-import java.io.ByteArrayOutputStream;
-import java.io.UnsupportedEncodingException;
-import java.util.BitSet;
-
-import org.apache.commons.codec.net.URLCodec;
-
-import com.google.common.base.Charsets;
-
-/**
- * @author gojomo
- */
-public class LaxURLCodec extends URLCodec {
- public static LaxURLCodec DEFAULT = new LaxURLCodec("UTF-8");
-
- // passthrough constructor
- public LaxURLCodec(String encoding) {
- super(encoding);
- }
-
- /**
- * Decodes an array of URL safe 7-bit characters into an array of
- * original bytes. Escaped characters are converted back to their
- * original representation.
- *
- * Differs from URLCodec.decodeUrl() in that it throws no
- * exceptions; bad or incomplete escape sequences are ignored
- * and passed into result undecoded. This matches the behavior
- * of browsers, which will use inconsistently-encoded URIs
- * in HTTP request-lines.
- *
- * @param bytes array of URL safe characters
- * @return array of original bytes
- */
- public static final byte[] decodeUrlLoose(byte[] bytes)
- {
- if (bytes == null) {
- return null;
- }
- ByteArrayOutputStream buffer = new ByteArrayOutputStream();
- for (int i = 0; i < bytes.length; i++) {
- int b = bytes[i];
- if (b == '+') {
- buffer.write(' ');
- continue;
- }
- if (b == '%') {
- if(i+2 -1 && l > -1) {
- // good encoding
- int c = ((u << 4) + l);
- buffer.write((char)c);
- i += 2;
- continue;
- } // else: bad encoding digits, leave '%' in place
- } // else: insufficient encoding digits, leave '%' in place
- }
- buffer.write(b);
- }
- return buffer.toByteArray();
- }
-
- /**
- * A more expansive set of ASCII URI characters to consider as 'safe' to
- * leave unencoded, based on actual browser behavior.
- */
- public static BitSet EXPANDED_URI_SAFE = new BitSet(256);
- static {
- // alpha characters
- for (int i = 'a'; i <= 'z'; i++) {
- EXPANDED_URI_SAFE.set(i);
- }
- for (int i = 'A'; i <= 'Z'; i++) {
- EXPANDED_URI_SAFE.set(i);
- }
- // numeric characters
- for (int i = '0'; i <= '9'; i++) {
- EXPANDED_URI_SAFE.set(i);
- }
- // special chars
- EXPANDED_URI_SAFE.set('-');
- EXPANDED_URI_SAFE.set('~');
- EXPANDED_URI_SAFE.set('_');
- EXPANDED_URI_SAFE.set('.');
- EXPANDED_URI_SAFE.set('*');
- EXPANDED_URI_SAFE.set('/');
- EXPANDED_URI_SAFE.set('=');
- EXPANDED_URI_SAFE.set('&');
- EXPANDED_URI_SAFE.set('+');
- EXPANDED_URI_SAFE.set(',');
- EXPANDED_URI_SAFE.set(':');
- EXPANDED_URI_SAFE.set(';');
- EXPANDED_URI_SAFE.set('@');
- EXPANDED_URI_SAFE.set('$');
- EXPANDED_URI_SAFE.set('!');
- EXPANDED_URI_SAFE.set(')');
- EXPANDED_URI_SAFE.set('(');
- // experiments indicate: Firefox (1.0.6) never escapes '%'
- EXPANDED_URI_SAFE.set('%');
- // experiments indicate: Firefox (1.0.6) does not escape '|' or '''
- EXPANDED_URI_SAFE.set('|');
- EXPANDED_URI_SAFE.set('\'');
- }
-
- public static BitSet QUERY_SAFE = new BitSet(256);
- static {
- QUERY_SAFE.or(EXPANDED_URI_SAFE);
- // Tests indicate Firefox (1.0.7-1) doesn't escape curlies in query str.
- QUERY_SAFE.set('{');
- QUERY_SAFE.set('}');
- // nor any of these: [ ] ^ ?
- QUERY_SAFE.set('[');
- QUERY_SAFE.set(']');
- QUERY_SAFE.set('^');
- QUERY_SAFE.set('?');
- }
-
- /**
- * Encodes a string into its URL safe form using the specified
- * string charset. Unsafe characters are escaped.
- *
- * This method is analogous to superclass encode() methods,
- * additionally offering the ability to specify a different
- * 'safe' character set (such as EXPANDED_URI_SAFE).
- *
- * @param safe BitSet of characters that don't need to be encoded
- * @param pString String to encode
- * @param cs Name of character set to use
- * @return Encoded version of pString.
- * @throws UnsupportedEncodingException
- */
- public String encode(BitSet safe, String pString, String cs)
- throws UnsupportedEncodingException {
- if (pString == null) {
- return null;
- }
- return new String(encodeUrl(safe,pString.getBytes(cs)), Charsets.US_ASCII);
- }
-}
diff --git a/commons/src/main/java/org/archive/net/UURI.java b/commons/src/main/java/org/archive/net/UURI.java
index 3556760d..7995736c 100644
--- a/commons/src/main/java/org/archive/net/UURI.java
+++ b/commons/src/main/java/org/archive/net/UURI.java
@@ -18,18 +18,13 @@
*/
package org.archive.net;
-import java.io.File;
import java.io.IOException;
import java.io.ObjectInputStream;
import java.io.ObjectOutputStream;
-import java.io.Serializable;
-import java.net.URI;
-import java.net.URISyntaxException;
import java.nio.ByteBuffer;
import org.apache.commons.httpclient.URIException;
-import org.archive.util.SURT;
-import org.archive.util.TextUtils;
+import org.archive.url.UsableURI;
import com.esotericsoftware.kryo.CustomSerialization;
import com.esotericsoftware.kryo.Kryo;
@@ -37,423 +32,31 @@ import com.esotericsoftware.kryo.serialize.StringSerializer;
/**
- * Usable URI.
- *
- * This class wraps {@link org.apache.commons.httpclient.URI} adding caching
- * and methods. It cannot be instantiated directly. Go via UURIFactory.
- *
- *
We used to use {@link java.net.URI} for parsing URIs but ran across
- * quirky behaviors and bugs. {@link java.net.URI} is not subclassable --
- * its final -- and its unlikely that java.net.URI will change any time soon
- * (See Gordon's considered petition here:
- * java.net.URI
- * should have loose/tolerant/compatibility option (or allow reuse)).
- *
- *
This class tries to cache calculated strings such as the extracted host
- * and this class as a string rather than have the parent class rerun its
- * calculation everytime.
- *
- * @author gojomo
- * @author stack
- *
- * @see org.apache.commons.httpclient.URI
+ * Usable URI. The bulk of the functionality of this class has moved to
+ * {@link UsableURI} in the archive-commons project. This class adds Kryo
+ * serialization.
*/
-public class UURI extends LaxURI
-implements CharSequence, Serializable, CustomSerialization {
+public class UURI extends UsableURI implements CustomSerialization {
- private static final long serialVersionUID = -1277570889914647093L;
+ private static final long serialVersionUID = -8946640480772772310L;
- //private static Logger LOGGER =
- // Logger.getLogger(UURI.class.getName());
-
- /**
- * Consider URIs too long for IE as illegal.
- */
- public final static int MAX_URL_LENGTH = 2083;
-
- public static final String MASSAGEHOST_PATTERN = "^www\\d*\\.";
-
- /**
- * Cache of the host name.
- *
- * Super class calculates on every call. Profiling shows us spend 30% of
- * total elapsed time in URI class.
- */
- private transient String cachedHost = null;
-
- /**
- * Cache of this uuri escaped as a string.
- *
- * Super class calculates on every call. Profiling shows us spend 30% of
- * total elapsed time in URI class.
- */
- private transient String cachedEscapedURI = null;
-
- /**
- * Cache of this uuri escaped as a string.
- *
- * Super class calculates on every call. Profiling shows us spend 30% of
- * total elapsed time in URI class.
- */
- private transient String cachedString = null;
-
- /**
- * Cached authority minus userinfo.
- */
- private transient String cachedAuthorityMinusUserinfo = null;
-
- /**
- * Cache of this uuri in SURT format
- */
- private transient String surtForm = null;
-
- // Technically, underscores are disallowed in the domainlabel
- // portion of hostname according to rfc2396 but we'll be more
- // loose and allow them. See: [ 1072035 ] [uuri] Underscore in
- // host messes up port parsing.
- static {
- hostname.set('_');
+ public UURI(String fixup, boolean b, String charset) throws URIException {
+ super(fixup, b, charset);
}
-
- /**
- * Shutdown access to default constructor.
- */
- protected UURI() {
- super();
- }
-
- /**
- * @param uri String representation of an absolute URI.
- * @param escaped If escaped.
- * @param charset Charset to use.
- * @throws org.apache.commons.httpclient.URIException
- */
- protected UURI(String uri, boolean escaped, String charset)
- throws URIException {
- super(uri, escaped, charset);
- normalize();
- }
-
- /**
- * @param relative String representation of URI.
- * @param base Parent UURI to use derelativizing.
- * @throws org.apache.commons.httpclient.URIException
- */
- protected UURI(UURI base, UURI relative) throws URIException {
- super(base, relative);
- normalize();
+ public UURI(UURI base, UURI uuri) throws URIException {
+ super(base, uuri);
}
- /**
- * @param uri String representation of a URI.
- * @param escaped If escaped.
- * @throws NullPointerException
- * @throws URIException
- */
- protected UURI(String uri, boolean escaped) throws URIException, NullPointerException {
- super(uri,escaped);
- normalize();
- }
-
- /**
- * @param uri URI as string that is resolved relative to this UURI.
- * @return UURI that uses this UURI as base.
- * @throws URIException
- */
- public UURI resolve(String uri)
- throws URIException {
- return resolve(uri, false, // assume not escaped
- this.getProtocolCharset());
- }
-
- /**
- * @param uri URI as string that is resolved relative to this UURI.
- * @param e True if escaped.
- * @return UURI that uses this UURI as base.
- * @throws URIException
- */
- public UURI resolve(String uri, boolean e)
- throws URIException {
- return resolve(uri, e, this.getProtocolCharset());
- }
-
- /**
- * @param uri URI as string that is resolved relative to this UURI.
- * @param e True if uri is escaped.
- * @param charset Charset to use.
- * @return UURI that uses this UURI as base.
- * @throws URIException
- */
- public UURI resolve(String uri, boolean e, String charset)
- throws URIException {
- return new UURI(this, new UURI(uri, e, charset));
- }
-
- /**
- * Test an object if this UURI is equal to another.
- *
- * @param obj an object to compare
- * @return true if two URI objects are equal
- */
- public boolean equals(Object obj) {
-
- // normalize and test each components
- if (obj == this) {
- return true;
- }
- if (!(obj instanceof UURI)) {
- return false;
- }
- UURI another = (UURI) obj;
- // scheme
- if (!equals(this._scheme, another._scheme)) {
- return false;
- }
- // is_opaque_part or is_hier_part? and opaque
- if (!equals(this._opaque, another._opaque)) {
- return false;
- }
- // is_hier_part
- // has_authority
- if (!equals(this._authority, another._authority)) {
- return false;
- }
- // path
- if (!equals(this._path, another._path)) {
- return false;
- }
- // has_query
- if (!equals(this._query, another._query)) {
- return false;
- }
- // UURIs do not have fragments
- return true;
- }
-
- /**
- * Strips www variants from the host.
- *
- * Strips www[0-9]*\. from the host. If calling getHostBaseName becomes a
- * performance issue we should consider adding the hostBasename member that
- * is set on initialization.
- *
- * @return Host's basename.
- * @throws URIException
- */
- public String getHostBasename() throws URIException {
- // caching eliminated because this is rarely used
- // (only benefits legacy DomainScope, which should
- // be retired). Saves 4-byte object pointer in UURI
- // instances.
- return (this.getReferencedHost() == null)
- ? null
- : TextUtils.replaceFirst(MASSAGEHOST_PATTERN,
- this.getReferencedHost(), UURIFactory.EMPTY_STRING);
- }
-
- /**
- * Returns an alternate, functional String representation -- in this
- * case, a String of the URI represented by this UURI instance.
- *
- * @return
- */
- public synchronized String toCustomString() {
- if (this.cachedString == null) {
- this.cachedString = super.toString();
- coalesceUriStrings();
- }
- return this.cachedString;
- }
-
- /**
- * Override to cache result
- *
- * TODO: eliminate, moving most callers to toCustomString, to avoid
- * overloading/diluting toString()
- * (see http://webteam.archive.org/confluence/display/Heritrix/Preserve+toString%28%29 )
- * @return String representation of this URI
- */
- public String toString() {
- return toCustomString();
- }
-
- public synchronized String getEscapedURI() {
- if (this.cachedEscapedURI == null) {
- this.cachedEscapedURI = super.getEscapedURI();
- coalesceUriStrings();
- }
- return this.cachedEscapedURI;
- }
-
- /**
- * The two String fields cachedString and cachedEscapedURI are
- * usually identical; if so, coalesce into a single instance.
- */
- protected void coalesceUriStrings() {
- if (this.cachedString != null && this.cachedEscapedURI != null
- && this.cachedString.length() == this.cachedEscapedURI.length()) {
- // lengths will only be identical if contents are identical
- // (deescaping will always shrink length), so coalesce to
- // use only single cached instance
- this.cachedString = this.cachedEscapedURI;
- }
- }
-
- public synchronized String getHost() throws URIException {
- if (this.cachedHost == null) {
- // If this._host is null, 3.0 httpclient throws
- // illegalargumentexception. Don't go there.
- if (this._host != null) {
- this.cachedHost = super.getHost();
- coalesceHostAuthorityStrings();
- }
- }
- return this.cachedHost;
- }
-
- /**
- * The two String fields cachedHost and cachedAuthorityMinusUserInfo are
- * usually identical; if so, coalesce into a single instance.
- */
- protected void coalesceHostAuthorityStrings() {
- if (this.cachedAuthorityMinusUserinfo != null
- && this.cachedHost != null
- && this.cachedHost.length() ==
- this.cachedAuthorityMinusUserinfo.length()) {
- // lengths can only be identical if contents
- // are identical; use only one instance
- this.cachedAuthorityMinusUserinfo = this.cachedHost;
- }
- }
-
- /**
- * Return the referenced host in the UURI, if any, also extracting the
- * host of a DNS-lookup URI where necessary.
- *
- * @return the target or topic host of the URI
- * @throws URIException
- */
- public String getReferencedHost() throws URIException {
- String referencedHost = this.getHost();
- if(referencedHost==null && this.getScheme().equals("dns")) {
- // extract target domain of DNS lookup
- String possibleHost = this.getCurrentHierPath();
- if(possibleHost != null && possibleHost.matches("[-_\\w\\.:]+")) {
- referencedHost = possibleHost;
- }
- }
- return referencedHost;
- }
-
- /**
- * @return Return the 'SURT' format of this UURI
- */
- public String getSurtForm() {
- if (surtForm == null) {
- surtForm = SURT.fromURI(this.toString());
- }
- return surtForm;
- }
-
- /**
- * Return the authority minus userinfo (if any).
- *
- * If no userinfo present, just returns the authority.
- *
- * @return The authority stripped of any userinfo if present.
- * @throws URIException
- */
- public String getAuthorityMinusUserinfo()
- throws URIException {
- if (this.cachedAuthorityMinusUserinfo == null) {
- String tmp = getAuthority();
- if (tmp != null && tmp.length() > 0) {
- int index = tmp.indexOf('@');
- if (index >= 0 && index < tmp.length()) {
- tmp = tmp.substring(index + 1);
- }
- }
- this.cachedAuthorityMinusUserinfo = tmp;
- coalesceHostAuthorityStrings();
- }
- return this.cachedAuthorityMinusUserinfo;
- }
-
- /* (non-Javadoc)
- * @see java.lang.CharSequence#length()
- */
- public int length() {
- return getEscapedURI().length();
- }
-
- /* (non-Javadoc)
- * @see java.lang.CharSequence#charAt(int)
- */
- public char charAt(int index) {
- return getEscapedURI().charAt(index);
- }
-
- /* (non-Javadoc)
- * @see java.lang.CharSequence#subSequence(int, int)
- */
- public CharSequence subSequence(int start, int end) {
- return getEscapedURI().subSequence(start,end);
- }
-
- /* (non-Javadoc)
- * @see java.lang.Comparable#compareTo(java.lang.Object)
- */
- public int compareTo(Object arg0) {
- return getEscapedURI().compareTo(arg0.toString());
- }
-
-
-
- /**
- * Test if passed String has likely URI scheme prefix.
- * @param possibleUrl URL string to examine.
- * @return True if passed string looks like it could be an URL.
- */
- public static boolean hasScheme(String possibleUrl) {
- boolean result = false;
- for (int i = 0; i < possibleUrl.length(); i++) {
- char c = possibleUrl.charAt(i);
- if (c == ':') {
- if (i != 0) {
- result = true;
- }
- break;
- }
- if (!scheme.get(c)) {
- break;
- }
- }
- return result;
- }
-
- /**
- * @param pathOrUri A file path or a URI.
- * @return Path parsed from passed pathOrUri.
- * @throws URISyntaxException
- */
- public static String parseFilename(final String pathOrUri)
- throws URISyntaxException {
- String path = pathOrUri;
- if (UURI.hasScheme(pathOrUri)) {
- URI url = new URI(pathOrUri);
- path = url.getPath();
- }
- return (new File(path)).getName();
- }
-
+ @Override
public void writeObjectData(Kryo kryo, ByteBuffer buffer) {
StringSerializer.put(buffer, toCustomString());
}
- public void readObjectData (Kryo kryo, ByteBuffer buffer) {
+ @Override
+ public void readObjectData(Kryo kryo, ByteBuffer buffer) {
try {
- parseUriReference(StringSerializer.get(buffer),true);
+ parseUriReference(StringSerializer.get(buffer), true);
} catch (URIException e) {
// TODO Auto-generated catch block
e.printStackTrace();
@@ -462,10 +65,10 @@ implements CharSequence, Serializable, CustomSerialization {
private void writeObject(ObjectOutputStream stream) throws IOException {
stream.writeUTF(toCustomString());
- }
-
+ }
+
private void readObject(ObjectInputStream stream) throws IOException,
- ClassNotFoundException {
- parseUriReference(stream.readUTF(),true);
+ ClassNotFoundException {
+ parseUriReference(stream.readUTF(), true);
}
}
diff --git a/commons/src/main/java/org/archive/net/UURIFactory.java b/commons/src/main/java/org/archive/net/UURIFactory.java
index 75810e70..d44aa093 100644
--- a/commons/src/main/java/org/archive/net/UURIFactory.java
+++ b/commons/src/main/java/org/archive/net/UURIFactory.java
@@ -18,232 +18,26 @@
*/
package org.archive.net;
-import gnu.inet.encoding.IDNA;
-import gnu.inet.encoding.IDNAException;
-import it.unimi.dsi.mg4j.util.MutableString;
-
-import java.io.UnsupportedEncodingException;
-import java.util.BitSet;
-import java.util.logging.Level;
-import java.util.logging.Logger;
-import java.util.regex.Matcher;
-import java.util.regex.Pattern;
-
-import org.apache.commons.httpclient.URI;
import org.apache.commons.httpclient.URIException;
-import org.archive.util.TextUtils;
-/**
- * Factory that returns UURIs.
- *
- * Does escaping and fixup on URIs massaging in accordance with RFC2396 and to
- * match browser practice. For example, it removes any '..' if first thing in
- * the path as per IE, converts backslashes preceding the query string to
- * forward slashes, and discards any 'fragment'/anchor portion of the URI. This
- * class will also fail URIs if they are longer than IE's allowed maximum
- * length.
- *
- *
- * TODO: Test logging.
- *
- * @author stack
- */
-public class UURIFactory extends URI {
+public class UURIFactory extends org.archive.url.UsableURIFactory {
- private static final long serialVersionUID = -6146295130382209042L;
-
- /**
- * Logging instance.
- */
- private static Logger logger =
- Logger.getLogger(UURIFactory.class.getName());
+ private static final long serialVersionUID = -2615498094356088387L;
/**
* The single instance of this factory.
*/
private static final UURIFactory factory = new UURIFactory();
-
- /**
- * RFC 2396-inspired regex.
- *
- * From the RFC Appendix B:
- *
- * URI Generic Syntax August 1998
- *
- * B. Parsing a URI Reference with a Regular Expression
- *
- * As described in Section 4.3, the generic URI syntax is not sufficient
- * to disambiguate the components of some forms of URI. Since the
- * "greedy algorithm" described in that section is identical to the
- * disambiguation method used by POSIX regular expressions, it is
- * natural and commonplace to use a regular expression for parsing the
- * potential four components and fragment identifier of a URI reference.
- *
- * The following line is the regular expression for breaking-down a URI
- * reference into its components.
- *
- * ^(([^:/?#]+):)?(//([^/?#]*))?([^?#]*)(\?([^#]*))?(#(.*))?
- * 12 3 4 5 6 7 8 9
- *
- * The numbers in the second line above are only to assist readability;
- * they indicate the reference points for each subexpression (i.e., each
- * paired parenthesis). We refer to the value matched for subexpression
- * as $. For example, matching the above expression to
- *
- * http://www.ics.uci.edu/pub/ietf/uri/#Related
- *
- * results in the following subexpression matches:
- *
- * $1 = http:
- * $2 = http
- * $3 = //www.ics.uci.edu
- * $4 = www.ics.uci.edu
- * $5 = /pub/ietf/uri/
- * $6 =
- * $7 =
- * $8 = #Related
- * $9 = Related
- *
- * where indicates that the component is not present, as is
- * the case for the query component in the above example. Therefore, we
- * can determine the value of the four components and fragment as
- *
- * scheme = $2
- * authority = $4
- * path = $5
- * query = $7
- * fragment = $9
- *
- *
- * --
- *
Below differs from the rfc regex in that...
- * (1) it has java escaping of regex characters
- * (2) we allow a URI made of a fragment only (Added extra
- * group so indexing is off by one after scheme).
- * (3) scheme is limited to legal scheme characters
- */
- final public static Pattern RFC2396REGEX = Pattern.compile(
- "^(([a-zA-Z][a-zA-Z0-9\\+\\-\\.]*):)?((//([^/?#]*))?([^?#]*)(\\?([^#]*))?)?(#(.*))?");
- // 12 34 5 6 7 8 9 A
- // 2 1 54 6 87 3 A9 // 1: scheme
- // 2: scheme:
- // 3: //authority/path
- // 4: //authority
- // 5: authority
- // 6: path
- // 7: ?query
- // 8: query
- // 9: #fragment
- // A: fragment
- public static final String SLASHDOTDOTSLASH = "^(/\\.\\./)+";
- public static final String SLASH = "/";
- public static final String HTTP = "http";
- public static final String HTTP_PORT = ":80";
- public static final String HTTPS = "https";
- public static final String HTTPS_PORT = ":443";
- public static final String DOT = ".";
- public static final String EMPTY_STRING = "";
- public static final String NBSP = "\u00A0";
- public static final String SPACE = " ";
- public static final String ESCAPED_SPACE = "%20";
- public static final String TRAILING_ESCAPED_SPACE = "^(.*)(%20)+$";
- public static final String PIPE = "|";
- public static final String PIPE_PATTERN = "\\|";
- public static final String ESCAPED_PIPE = "%7C";
- public static final String CIRCUMFLEX = "^";
- public static final String CIRCUMFLEX_PATTERN = "\\^";
- public static final String ESCAPED_CIRCUMFLEX = "%5E";
- public static final String QUOT = "\"";
- public static final String ESCAPED_QUOT = "%22";
- public static final String SQUOT = "'";
- public static final String ESCAPED_SQUOT = "%27";
- public static final String APOSTROPH = "`";
- public static final String ESCAPED_APOSTROPH = "%60";
- public static final String LSQRBRACKET = "[";
- public static final String LSQRBRACKET_PATTERN = "\\[";
- public static final String ESCAPED_LSQRBRACKET = "%5B";
- public static final String RSQRBRACKET = "]";
- public static final String RSQRBRACKET_PATTERN = "\\]";
- public static final String ESCAPED_RSQRBRACKET = "%5D";
- public static final String LCURBRACKET = "{";
- public static final String LCURBRACKET_PATTERN = "\\{";
- public static final String ESCAPED_LCURBRACKET = "%7B";
- public static final String RCURBRACKET = "}";
- public static final String RCURBRACKET_PATTERN = "\\}";
- public static final String ESCAPED_RCURBRACKET = "%7D";
- public static final String BACKSLASH = "\\";
- public static final String ESCAPED_BACKSLASH = "%5C";
- public static final String STRAY_SPACING = "[\n\r\t]+";
- public static final String IMPROPERESC_REPLACE = "%25$1";
- public static final String IMPROPERESC =
- "%((?:[^\\p{XDigit}])|(?:.[^\\p{XDigit}])|(?:\\z))";
- public static final String COMMERCIAL_AT = "@";
- public static final char PERCENT_SIGN = '%';
- public static final char COLON = ':';
-
- /**
- * First percent sign in string followed by two hex chars.
- */
- public static final String URI_HEX_ENCODING =
- "^[^%]*%[\\p{XDigit}][\\p{XDigit}].*";
-
- /**
- * Authority port number regex.
- */
- final static Pattern PORTREGEX = Pattern.compile("(.*:)([0-9]+)$");
-
- /**
- * Characters we'll accept in the domain label part of a URI
- * authority: ASCII letters-digits-hyphen (LDH) plus underscore,
- * with single intervening '.' characters.
- *
- * (We accept '_' because DNS servers have tolerated for many
- * years counter to spec; we also accept dash patterns and ACE
- * prefixes that will be rejected by IDN-punycoding attempt.)
- */
- final static String ACCEPTABLE_ASCII_DOMAIN =
- "^(?:[a-zA-Z0-9_-]++(?:\\.)?)++$";
-
- /**
- * Pattern that looks for case of three or more slashes after the
- * scheme. If found, we replace them with two only as mozilla does.
- */
- final static Pattern HTTP_SCHEME_SLASHES =
- Pattern.compile("^(https?://)/+(.*)");
-
- /**
- * Pattern that looks for case of two or more slashes in a path.
- */
- final static Pattern MULTIPLE_SLASHES = Pattern.compile("//+");
-
- /**
- * Protected constructor.
- */
- private UURIFactory() {
- super();
- }
-
/**
* @param uri URI as string.
* @return An instance of UURI
* @throws URIException
*/
public static UURI getInstance(String uri) throws URIException {
- return UURIFactory.factory.create(uri);
+ return (UURI) UURIFactory.factory.create(uri);
}
-
- /**
- * @param uri URI as string.
- * @param charset Character encoding of the passed uri string.
- * @return An instance of UURI
- * @throws URIException
- */
- public static UURI getInstance(String uri, String charset)
- throws URIException {
- return UURIFactory.factory.create(uri, charset);
- }
-
+
/**
* @param base Base uri to use resolving passed relative uri.
* @param relative URI as string.
@@ -251,552 +45,13 @@ public class UURIFactory extends URI {
* @throws URIException
*/
public static UURI getInstance(UURI base, String relative)
- throws URIException {
-// return base.resolve(relative);
- return UURIFactory.factory.create(base, relative);
+ throws URIException {
+ return (UURI) UURIFactory.factory.create(base, relative);
}
- /**
- * @param uri URI as string.
- * @return Instance of UURI.
- * @throws URIException
- */
- private UURI create(String uri) throws URIException {
- return create(uri, UURI.getDefaultProtocolCharset());
- }
-
- /**
- * @param uri URI as string.
- * @param charset Original encoding of the string.
- * @return Instance of UURI.
- * @throws URIException
- */
- private UURI create(String uri, String charset) throws URIException {
- UURI uuri = new UURI(fixup(uri, null, charset), true, charset);
- if (logger.isLoggable(Level.FINE)) {
- logger.fine("URI " + uri +
- " PRODUCT " + uuri.toString() +
- " CHARSET " + charset);
- }
- return validityCheck(uuri);
- }
-
- /**
- * @param base UURI to use as a base resolving relative.
- * @param relative Relative URI.
- * @return Instance of UURI.
- * @throws URIException
- */
- private UURI create(UURI base, String relative) throws URIException {
- UURI uuri = new UURI(base, new UURI(fixup(relative, base, base.getProtocolCharset()),
- true, base.getProtocolCharset()));
- if (logger.isLoggable(Level.FINE)) {
- logger.fine(" URI " + relative +
- " PRODUCT " + uuri.toString() +
- " CHARSET " + base.getProtocolCharset() +
- " BASE " + base);
- }
- return validityCheck(uuri);
+ protected UURI newUURI(String charset, boolean escaped, String fixedUpUri)
+ throws URIException {
+ return new UURI(fixedUpUri, escaped, charset);
}
- /**
- * Check the generated UURI.
- *
- * At the least look at length of uuri string. We were seeing case
- * where before escaping, string was < MAX_URL_LENGTH but after was
- * >. Letting out a too-big message was causing us troubles later
- * down the processing chain.
- * @param uuri Created uuri to check.
- * @return The passed uuri so can easily inline this check.
- * @throws URIException
- */
- protected UURI validityCheck(UURI uuri) throws URIException {
- if (uuri.getRawURI().length > UURI.MAX_URL_LENGTH) {
- throw new URIException("Created (escaped) uuri > " +
- UURI.MAX_URL_LENGTH +": "+uuri.toString());
- }
- return uuri;
- }
-
- /**
- * Do heritrix fix-up on passed uri string.
- *
- * Does heritrix escaping; usually escaping done to make our behavior align
- * with IEs. This method codifies our experience pulling URIs from the
- * wilds. Its does all the escaping we want; its output can always be
- * assumed to be 'escaped' (though perhaps to a laxer standard than the
- * vanilla HttpClient URI class or official specs might suggest).
- *
- * @param uri URI as string.
- * @param base May be null.
- * @param e True if the uri is already escaped.
- * @return A fixed up URI string.
- * @throws URIException
- */
- private String fixup(String uri, final URI base, final String charset)
- throws URIException {
- if (uri == null) {
- throw new NullPointerException();
- } else if (uri.length() == 0 && base == null) {
- throw new URIException("URI length is zero (and not relative).");
- }
-
- if (uri.length() > UURI.MAX_URL_LENGTH) {
- // We check length here and again later after all convertions.
- throw new URIException("URI length > " + UURI.MAX_URL_LENGTH +
- ": " + uri);
- }
-
- // Replace nbsp with normal spaces (so that they get stripped if at
- // ends, or encoded if in middle)
- if (uri.indexOf(NBSP) >= 0) {
- uri = TextUtils.replaceAll(NBSP, uri, SPACE);
- }
-
- // Get rid of any trailing spaces or new-lines.
- uri = uri.trim();
-
- // IE converts backslashes preceding the query string to slashes, rather
- // than to %5C. Since URIs that have backslashes usually work only with
- // IE, we will convert backslashes to slashes as well.
- int nextBackslash = uri.indexOf(BACKSLASH);
- if (nextBackslash >= 0) {
- int queryStart = uri.indexOf('?');
- StringBuilder tmp = new StringBuilder(uri);
- while (nextBackslash >= 0
- && (queryStart < 0 || nextBackslash < queryStart)) {
- tmp.setCharAt(nextBackslash, '/');
- nextBackslash = uri.indexOf(BACKSLASH, nextBackslash + 1);
- }
- uri = tmp.toString();
- }
-
- // Remove stray TAB/CR/LF
- uri = TextUtils.replaceAll(STRAY_SPACING, uri, EMPTY_STRING);
-
- // Test for the case of more than two slashes after the http(s) scheme.
- // Replace with two slashes as mozilla does if found.
- // See [ 788219 ] URI Syntax Errors stop page parsing.
-// Matcher matcher = HTTP_SCHEME_SLASHES.matcher(uri);
- Matcher matcher = TextUtils.getMatcher(HTTP_SCHEME_SLASHES.pattern(), uri);
- if (matcher.matches()) {
- uri = matcher.group(1) + matcher.group(2);
- }
- TextUtils.recycleMatcher(matcher);
-
- // now, minimally escape any whitespace
- uri = escapeWhitespace(uri);
-
- // For further processing, get uri elements. See the RFC2396REGEX
- // comment above for explanation of group indices used in the below.
-// matcher = RFC2396REGEX.matcher(uri);
- matcher = TextUtils.getMatcher(RFC2396REGEX.pattern(), uri);
- if (!matcher.matches()) {
- throw new URIException("Failed parse of " + uri);
- }
- String uriScheme = checkUriElementAndLowerCase(matcher.group(2));
- String uriSchemeSpecificPart = checkUriElement(matcher.group(3));
- String uriAuthority = checkUriElement(matcher.group(5));
- String uriPath = checkUriElement(matcher.group(6));
- String uriQuery = checkUriElement(matcher.group(8));
- // UNUSED String uriFragment = checkUriElement(matcher.group(10));
- TextUtils.recycleMatcher(matcher); matcher = null;
-
- // Test if relative URI. If so, need a base to resolve against.
- if (uriScheme == null || uriScheme.length() <= 0) {
- if (base == null) {
- throw new URIException("Relative URI but no base: " + uri);
- }
- } else {
- checkHttpSchemeSpecificPartSlashPrefix(base, uriScheme,
- uriSchemeSpecificPart);
- }
-
- // fixup authority portion: lowercase/IDN-punycode any domain;
- // remove stray trailing spaces
- uriAuthority = fixupAuthority(uriAuthority,charset);
-
- // Do some checks if absolute path.
- if (uriSchemeSpecificPart != null &&
- uriSchemeSpecificPart.startsWith(SLASH)) {
- if (uriPath != null) {
- // Eliminate '..' if its first thing in the path. IE does this.
- uriPath = TextUtils.replaceFirst(SLASHDOTDOTSLASH, uriPath,
- SLASH);
- }
- // Ensure root URLs end with '/': browsers always send "/"
- // on the request-line, so we should consider "http://host"
- // to be "http://host/".
- if (uriPath == null || EMPTY_STRING.equals(uriPath)) {
- uriPath = SLASH;
- }
- }
-
- if (uriAuthority != null) {
- if (uriScheme != null && uriScheme.length() > 0 &&
- uriScheme.equals(HTTP)) {
- uriAuthority = checkPort(uriAuthority);
- uriAuthority = stripTail(uriAuthority, HTTP_PORT);
- } else if (uriScheme != null && uriScheme.length() > 0 &&
- uriScheme.equals(HTTPS)) {
- uriAuthority = checkPort(uriAuthority);
- uriAuthority = stripTail(uriAuthority, HTTPS_PORT);
- }
- // Strip any prefix dot or tail dots from the authority.
- uriAuthority = stripTail(uriAuthority, DOT);
- uriAuthority = stripPrefix(uriAuthority, DOT);
- } else {
- // no authority; may be relative. consider stripping scheme
- // to work-around org.apache.commons.httpclient.URI bug
- // ( http://issues.apache.org/jira/browse/HTTPCLIENT-587 )
- if (uriScheme != null && base != null
- && uriScheme.equals(base.getScheme())) {
- // uriScheme redundant and will only confound httpclient.URI
- uriScheme = null;
- }
- }
-
- // Ensure minimal escaping. Use of 'lax' URI and URLCodec
- // means minimal escaping isn't necessarily complete/consistent.
- // There is a chance such lax encoding will throw exceptions
- // later at inconvenient times.
- //
- // One reason for these bad escapings -- though not the only --
- // is that the page is using an encoding other than the ASCII or the
- // UTF-8 that is our default URI encoding. In this case the parent
- // class is burping on the passed URL encoding. If the page encoding
- // was passed into this factory, the encoding seems to be parsed
- // correctly (See the testEscapedEncoding unit test).
- //
- // This fixup may cause us to miss content. There is the charset case
- // noted above. TODO: Look out for cases where we fail other than for
- // the above given reason which will be fixed when we address
- // '[ 913687 ] Make extractors interrogate for charset'.
-
- uriPath = ensureMinimalEscaping(uriPath, charset);
- uriQuery = ensureMinimalEscaping(uriQuery, charset,
- LaxURLCodec.QUERY_SAFE);
-
- // Preallocate. The '1's and '2's in below are space for ':',
- // '//', etc. URI characters.
- MutableString s = new MutableString(
- ((uriScheme != null)? uriScheme.length(): 0)
- + 1 // ';'
- + ((uriAuthority != null)? uriAuthority.length(): 0)
- + 2 // '//'
- + ((uriPath != null)? uriPath.length(): 0)
- + 1 // '?'
- + ((uriQuery != null)? uriQuery.length(): 0));
- appendNonNull(s, uriScheme, ":", true);
- appendNonNull(s, uriAuthority, "//", false);
- appendNonNull(s, uriPath, "", false);
- appendNonNull(s, uriQuery, "?", false);
- return s.toString();
- }
-
- /**
- * If http(s) scheme, check scheme specific part begins '//'.
- * @throws URIException
- * @see http://www.faqs.org/rfcs/rfc1738.html Section 3.1. Common Internet
- * Scheme Syntax
- */
- protected void checkHttpSchemeSpecificPartSlashPrefix(final URI base,
- final String scheme, final String schemeSpecificPart)
- throws URIException {
- if (scheme == null || scheme.length() <= 0) {
- return;
- }
- if (!scheme.equals("http") && !scheme.equals("https")) {
- return;
- }
- if ( schemeSpecificPart == null
- || !schemeSpecificPart.startsWith("//")) {
- // only acceptable if schemes match
- if (base == null || !scheme.equals(base.getScheme())) {
- throw new URIException(
- "relative URI with scheme only allowed for " +
- "scheme matching base");
- }
- return;
- }
- if (schemeSpecificPart.length() <= 2) {
- throw new URIException("http scheme specific part is " +
- "too short: " + schemeSpecificPart);
- }
- }
-
- /**
- * Fixup 'authority' portion of URI, by removing any stray
- * encoded spaces, lowercasing any domain names, and applying
- * IDN-punycoding to Unicode domains.
- *
- * @param uriAuthority the authority string to fix
- * @return fixed version
- * @throws URIException
- */
- private String fixupAuthority(String uriAuthority, String charset) throws URIException {
- // Lowercase the host part of the uriAuthority; don't destroy any
- // userinfo capitalizations. Make sure no illegal characters in
- // domainlabel substring of the uri authority.
- if (uriAuthority != null) {
- // Get rid of any trailing escaped spaces:
- // http://www.archive.org%20. Rare but happens.
- // TODO: reevaluate: do IE or firefox do such mid-URI space-removal?
- // if not, we shouldn't either.
- while(uriAuthority.endsWith(ESCAPED_SPACE)) {
- uriAuthority = uriAuthority.substring(0,uriAuthority.length()-3);
- }
-
- // lowercase & IDN-punycode only the domain portion
- int atIndex = uriAuthority.indexOf(COMMERCIAL_AT);
- int portColonIndex = uriAuthority.indexOf(COLON,(atIndex<0)?0:atIndex);
- if(atIndex<0 && portColonIndex<0) {
- // most common case: neither userinfo nor port
- return fixupDomainlabel(uriAuthority);
- } else if (atIndex<0 && portColonIndex>-1) {
- // next most common: port but no userinfo
- String domain = fixupDomainlabel(uriAuthority.substring(0,portColonIndex));
- String port = uriAuthority.substring(portColonIndex);
- return domain + port;
- } else if (atIndex>-1 && portColonIndex<0) {
- // uncommon: userinfo, no port
- String userinfo = ensureMinimalEscaping(uriAuthority.substring(0,atIndex+1),charset);
- String domain = fixupDomainlabel(uriAuthority.substring(atIndex+1));
- return userinfo + domain;
- } else {
- // uncommon: userinfo, port
- String userinfo = ensureMinimalEscaping(uriAuthority.substring(0,atIndex+1),charset);
- String domain = fixupDomainlabel(uriAuthority.substring(atIndex+1,portColonIndex));
- String port = uriAuthority.substring(portColonIndex);
- return userinfo + domain + port;
- }
- }
- return uriAuthority;
- }
-
- /**
- * Fixup the domain label part of the authority.
- *
- * We're more lax than the spec. in that we allow underscores.
- *
- * @param label Domain label to fix.
- * @return Return fixed domain label.
- * @throws URIException
- */
- private String fixupDomainlabel(String label)
- throws URIException {
-
- // apply IDN-punycoding, as necessary
- try {
- // TODO: optimize: only apply when necessary, or
- // keep cache of recent encodings
- label = IDNA.toASCII(label);
- } catch (IDNAException e) {
- if(TextUtils.matches(ACCEPTABLE_ASCII_DOMAIN,label)) {
- // domain name has ACE prefix, leading/trailing dash, or
- // underscore -- but is still a name we wish to tolerate;
- // simply continue
- } else {
- // problematic domain: neither ASCII acceptable characters
- // nor IDN-punycodable, so throw exception
- // TODO: change to HeritrixURIException so distinguishable
- // from URIExceptions in library code
- URIException ue = new URIException(e+" "+label);
- ue.initCause(e);
- throw ue;
- }
- }
- label = label.toLowerCase();
- return label;
- }
-
- /**
- * Ensure that there all characters needing escaping
- * in the passed-in String are escaped. Stray '%' characters
- * are *not* escaped, as per browser behavior.
- *
- * @param u String to escape
- * @param charset
- * @return string with any necessary escaping applied
- */
- private String ensureMinimalEscaping(String u, final String charset) {
- return ensureMinimalEscaping(u, charset, LaxURLCodec.EXPANDED_URI_SAFE);
- }
-
- /**
- * Ensure that there all characters needing escaping
- * in the passed-in String are escaped. Stray '%' characters
- * are *not* escaped, as per browser behavior.
- *
- * @param u String to escape
- * @param charset
- * @param bitset
- * @return string with any necessary escaping applied
- */
- private String ensureMinimalEscaping(String u, final String charset,
- final BitSet bitset) {
- if (u == null) {
- return null;
- }
- for (int i = 0; i < u.length(); i++) {
- char c = u.charAt(i);
- if (!bitset.get(c)) {
- try {
- u = LaxURLCodec.DEFAULT.encode(bitset, u, charset);
- } catch (UnsupportedEncodingException e) {
- e.printStackTrace();
- }
- break;
- }
- }
- return u;
- }
-
- /**
- * Escape any whitespace found.
- *
- * The parent class takes care of the bulk of escaping. But if any
- * instance of escaping is found in the URI, then we ask for parent
- * to do NO escaping. Here we escape any whitespace found irrespective
- * of whether the uri has already been escaped. We do this for
- * case where uri has been judged already-escaped only, its been
- * incompletly done and whitespace remains. Spaces, etc., in the URI are
- * a real pain. Their presence will break log file and ARC parsing.
- * @param uri URI string to check.
- * @return uri with spaces escaped if any found.
- */
- protected String escapeWhitespace(String uri) {
- // Just write a new string anyways. The perl '\s' is not
- // as inclusive as the Character.isWhitespace so there are
- // whitespace characters we could miss. So, rather than
- // write some awkward regex, just go through the string
- // a character at a time. Only create buffer first time
- // we find a space.
- MutableString buffer = null;
- for (int i = 0; i < uri.length(); i++) {
- char c = uri.charAt(i);
- if (Character.isWhitespace(c)) {
- if (buffer == null) {
- buffer = new MutableString(uri.length() +
- 2 /*If space, two extra characters (at least)*/);
- buffer.append(uri.substring(0, i));
- }
- buffer.append("%");
- String hexStr = Integer.toHexString(c);
- if ((hexStr.length() % 2) > 0) {
- buffer.append("0");
- }
- buffer.append(hexStr);
-
- } else {
- if (buffer != null) {
- buffer.append(c);
- }
- }
- }
- return (buffer != null)? buffer.toString(): uri;
- }
-
- /**
- * Check port on passed http authority. Make sure the size is not larger
- * than allowed: See the 'port' definition on this
- * page, http://www.kerio.com/manual/wrp/en/418.htm.
- * Also, we've seen port numbers of '0080' whose leading zeros confuse
- * the parent class. Strip the leading zeros.
- *
- * @param uriAuthority
- * @return Null or an amended port number.
- * @throws URIException
- */
- private String checkPort(String uriAuthority)
- throws URIException {
-// Matcher m = PORTREGEX.matcher(uriAuthority);
- Matcher m = TextUtils.getMatcher(PORTREGEX.pattern(), uriAuthority);
- if (m.matches()) {
- String no = m.group(2);
- if (no != null && no.length() > 0) {
- // First check if the port has leading zeros
- // as in '0080'. Strip them if it has and
- // then reconstitute the uriAuthority. Be careful
- // of cases where port is '0' or '000'.
- while (no.charAt(0) == '0' && no.length() > 1) {
- no = no.substring(1);
- }
- uriAuthority = m.group(1) + no;
- // Now makesure the number is legit.
- int portNo = 0;
- try {
- portNo = Integer.parseInt(no);
- } catch (NumberFormatException nfe) {
- // just catch and leave portNo at illegal 0
- }
- if (portNo <= 0 || portNo > 65535) {
- throw new URIException("Port out of bounds: " +
- uriAuthority);
- }
- }
- }
- TextUtils.recycleMatcher(m);
- return uriAuthority;
- }
-
- /**
- * @param b Buffer to append to.
- * @param str String to append if not null.
- * @param substr Suffix or prefix to use if str is not null.
- * @param suffix True if substr is a suffix.
- */
- private void appendNonNull(MutableString b, String str, String substr,
- boolean suffix) {
- if (str != null && str.length() > 0) {
- if (!suffix) {
- b.append(substr);
- }
- b.append(str);
- if (suffix) {
- b.append(substr);
- }
- }
- }
-
- /**
- * @param str String to work on.
- * @param prefix Prefix to strip if present.
- * @return str w/o prefix.
- */
- private String stripPrefix(String str, String prefix) {
- return str.startsWith(prefix)?
- str.substring(prefix.length(), str.length()):
- str;
- }
-
- /**
- * @param str String to work on.
- * @param tail Tail to strip if present.
- * @return str w/o tail.
- */
- private static String stripTail(String str, String tail) {
- return str.endsWith(tail)?
- str.substring(0, str.length() - tail.length()):
- str;
- }
-
- /**
- * @param element to examine.
- * @return Null if passed null or an empty string otherwise
- * element.
- */
- private String checkUriElement(String element) {
- return (element == null || element.length() <= 0)? null: element;
- }
-
- /**
- * @param element to examine and lowercase if non-null.
- * @return Null if passed null or an empty string otherwise
- * element lowercased.
- */
- private String checkUriElementAndLowerCase(String element) {
- String tmp = checkUriElement(element);
- return (tmp != null)? tmp.toLowerCase(): tmp;
- }
}
diff --git a/commons/src/main/java/org/archive/util/ArchiveUtils.java b/commons/src/main/java/org/archive/util/ArchiveUtils.java
deleted file mode 100644
index fe2da96f..00000000
--- a/commons/src/main/java/org/archive/util/ArchiveUtils.java
+++ /dev/null
@@ -1,1008 +0,0 @@
-/*
- * This file is part of the Heritrix web crawler (crawler.archive.org).
- *
- * Licensed to the Internet Archive (IA) by one or more individual
- * contributors.
- *
- * The IA licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.archive.util;
-
-import java.io.BufferedInputStream;
-import java.io.BufferedReader;
-import java.io.ByteArrayOutputStream;
-import java.io.Closeable;
-import java.io.EOFException;
-import java.io.File;
-import java.io.FileInputStream;
-import java.io.IOException;
-import java.io.InputStream;
-import java.io.InputStreamReader;
-import java.io.PrintWriter;
-import java.io.StringWriter;
-import java.net.URL;
-import java.net.URLConnection;
-import java.text.NumberFormat;
-import java.text.ParseException;
-import java.text.SimpleDateFormat;
-import java.util.Date;
-import java.util.HashSet;
-import java.util.Locale;
-import java.util.Map;
-import java.util.Set;
-import java.util.TimeZone;
-import java.util.logging.Level;
-import java.util.logging.Logger;
-import java.util.zip.GZIPInputStream;
-import java.util.zip.GZIPOutputStream;
-
-import org.apache.commons.io.IOUtils;
-import org.archive.io.GzipHeader;
-import org.archive.io.NoGzipMagicException;
-
-
-/**
- * Miscellaneous useful methods.
- *
- * @author gojomo & others
- */
-public class ArchiveUtils {
- private static final Logger LOGGER = Logger.getLogger(ArchiveUtils.class.getName());
-
-
- final public static String VERSION = loadVersion();
-
- /**
- * Arc-style date stamp in the format yyyyMMddHHmm and UTC time zone.
- */
- private static final ThreadLocal
- TIMESTAMP12 = threadLocalDateFormat("yyyyMMddHHmm");;
-
- /**
- * Arc-style date stamp in the format yyyyMMddHHmmss and UTC time zone.
- */
- private static final ThreadLocal
- TIMESTAMP14 = threadLocalDateFormat("yyyyMMddHHmmss");
- /**
- * Arc-style date stamp in the format yyyyMMddHHmmssSSS and UTC time zone.
- */
- private static final ThreadLocal
- TIMESTAMP17 = threadLocalDateFormat("yyyyMMddHHmmssSSS");
-
- /**
- * Log-style date stamp in the format yyyy-MM-dd'T'HH:mm:ss.SSS'Z'
- * UTC time zone is assumed.
- */
- private static final ThreadLocal
- TIMESTAMP17ISO8601Z = threadLocalDateFormat("yyyy-MM-dd'T'HH:mm:ss.SSS'Z'");
-
- /**
- * Log-style date stamp in the format yyyy-MM-dd'T'HH:mm:ss'Z'
- * UTC time zone is assumed.
- */
- private static final ThreadLocal
- TIMESTAMP14ISO8601Z = threadLocalDateFormat("yyyy-MM-dd'T'HH:mm:ss'Z'");
-
- /**
- * Default character to use padding strings.
- */
- private static final char DEFAULT_PAD_CHAR = ' ';
-
- /** milliseconds in an hour */
- private static final int HOUR_IN_MS = 60 * 60 * 1000;
- /** milliseconds in a day */
- private static final int DAY_IN_MS = 24 * HOUR_IN_MS;
-
- private static ThreadLocal threadLocalDateFormat(final String pattern) {
- ThreadLocal tl = new ThreadLocal() {
- protected SimpleDateFormat initialValue() {
- SimpleDateFormat df = new SimpleDateFormat(pattern);
- df.setTimeZone(TimeZone.getTimeZone("GMT"));
- return df;
- }
- };
- return tl;
- }
-
- public static int MAX_INT_CHAR_WIDTH =
- Integer.toString(Integer.MAX_VALUE).length();
-
- /**
- * Utility function for creating arc-style date stamps
- * in the format yyyMMddHHmmssSSS.
- * Date stamps are in the UTC time zone
- * @return the date stamp
- */
- public static String get17DigitDate(){
- return TIMESTAMP17.get().format(new Date());
- }
-
- protected static long LAST_UNIQUE_NOW17 = 0;
- protected static String LAST_TIMESTAMP17 = "";
- /**
- * Utility function for creating UNIQUE-from-this-class
- * arc-style date stamps in the format yyyMMddHHmmssSSS.
- * Rather than giving a duplicate datestamp on a
- * subsequent call, will increment the milliseconds until a
- * unique value is returned.
- *
- * Date stamps are in the UTC time zone
- * @return the date stamp
- */
- public synchronized static String getUnique17DigitDate(){
- long effectiveNow = System.currentTimeMillis();
- effectiveNow = Math.max(effectiveNow, LAST_UNIQUE_NOW17+1);
- String candidate = get17DigitDate(effectiveNow);
- while(candidate.equals(LAST_TIMESTAMP17)) {
- effectiveNow++;
- candidate = get17DigitDate(effectiveNow);
- }
- LAST_UNIQUE_NOW17 = effectiveNow;
- LAST_TIMESTAMP17 = candidate;
- return candidate;
- }
-
- /**
- * Utility function for creating arc-style date stamps
- * in the format yyyyMMddHHmmss.
- * Date stamps are in the UTC time zone
- * @return the date stamp
- */
- public static String get14DigitDate(){
- return TIMESTAMP14.get().format(new Date());
- }
-
- protected static long LAST_UNIQUE_NOW14 = 0;
- protected static String LAST_TIMESTAMP14 = "";
- /**
- * Utility function for creating UNIQUE-from-this-class
- * arc-style date stamps in the format yyyMMddHHmmss.
- * Rather than giving a duplicate datestamp on a
- * subsequent call, will increment the seconds until a
- * unique value is returned.
- *
- * Date stamps are in the UTC time zone
- * @return the date stamp
- */
- public synchronized static String getUnique14DigitDate(){
- long effectiveNow = System.currentTimeMillis();
- effectiveNow = Math.max(effectiveNow, LAST_UNIQUE_NOW14+1);
- String candidate = get14DigitDate(effectiveNow);
- while(candidate.equals(LAST_TIMESTAMP14)) {
- effectiveNow += 1000;
- candidate = get14DigitDate(effectiveNow);
- }
- LAST_UNIQUE_NOW14 = effectiveNow;
- LAST_TIMESTAMP14 = candidate;
- return candidate;
- }
-
- /**
- * Utility function for creating arc-style date stamps
- * in the format yyyyMMddHHmm.
- * Date stamps are in the UTC time zone
- * @return the date stamp
- */
- public static String get12DigitDate(){
- return TIMESTAMP12.get().format(new Date());
- }
-
- /**
- * Utility function for creating log timestamps, in
- * W3C/ISO8601 format, assuming UTC. Use current time.
- *
- * Format is yyyy-MM-dd'T'HH:mm:ss.SSS'Z'
- *
- * @return the date stamp
- */
- public static String getLog17Date(){
- return TIMESTAMP17ISO8601Z.get().format(new Date());
- }
-
- /**
- * Utility function for creating log timestamps, in
- * W3C/ISO8601 format, assuming UTC.
- *
- * Format is yyyy-MM-dd'T'HH:mm:ss.SSS'Z'
- * @param date Date to format.
- *
- * @return the date stamp
- */
- public static String getLog17Date(long date){
- return TIMESTAMP17ISO8601Z.get().format(new Date(date));
- }
-
- /**
- * Utility function for creating log timestamps, in
- * W3C/ISO8601 format, assuming UTC. Use current time.
- *
- * Format is yyyy-MM-dd'T'HH:mm:ss'Z'
- *
- * @return the date stamp
- */
- public static String getLog14Date(){
- return TIMESTAMP14ISO8601Z.get().format(new Date());
- }
-
- /**
- * Utility function for creating log timestamps, in
- * W3C/ISO8601 format, assuming UTC.
- *
- * Format is yyyy-MM-dd'T'HH:mm:ss'Z'
- * @param date long timestamp to format.
- *
- * @return the date stamp
- */
- public static String getLog14Date(long date){
- return TIMESTAMP14ISO8601Z.get().format(new Date(date));
- }
-
- /**
- * Utility function for creating log timestamps, in
- * W3C/ISO8601 format, assuming UTC.
- *
- * Format is yyyy-MM-dd'T'HH:mm:ss'Z'
- * @param date Date to format.
- *
- * @return the date stamp
- */
- public static String getLog14Date(Date date){
- return TIMESTAMP14ISO8601Z.get().format(date);
- }
-
- /**
- * Utility function for creating arc-style date stamps
- * in the format yyyyMMddHHmmssSSS.
- * Date stamps are in the UTC time zone
- *
- * @param date milliseconds since epoc
- * @return the date stamp
- */
- public static String get17DigitDate(long date){
- return TIMESTAMP17.get().format(new Date(date));
- }
-
- public static String get17DigitDate(Date date){
- return TIMESTAMP17.get().format(date);
- }
-
- /**
- * Utility function for creating arc-style date stamps
- * in the format yyyyMMddHHmmss.
- * Date stamps are in the UTC time zone
- *
- * @param date milliseconds since epoc
- * @return the date stamp
- */
- public static String get14DigitDate(long date){
- return TIMESTAMP14.get().format(new Date(date));
- }
-
- public static String get14DigitDate(Date d) {
- return TIMESTAMP14.get().format(d);
- }
-
- /**
- * Utility function for creating arc-style date stamps
- * in the format yyyyMMddHHmm.
- * Date stamps are in the UTC time zone
- *
- * @param date milliseconds since epoc
- * @return the date stamp
- */
- public static String get12DigitDate(long date){
- return TIMESTAMP12.get().format(new Date(date));
- }
-
- public static String get12DigitDate(Date d) {
- return TIMESTAMP12.get().format(d);
- }
-
- /**
- * Parses an ARC-style date. If passed String is < 12 characters in length,
- * we pad. At a minimum, String should contain a year (>=4 characters).
- * Parse will also fail if day or month are incompletely specified. Depends
- * on the above getXXDigitDate methods.
- * @param A 4-17 digit date in ARC style (yyyy to
- * yyyyMMddHHmmssSSS) formatting.
- * @return A Date object representing the passed String.
- * @throws ParseException
- */
- public static Date getDate(String d) throws ParseException {
- Date date = null;
- if (d == null) {
- throw new IllegalArgumentException("Passed date is null");
- }
- switch (d.length()) {
- case 14:
- date = ArchiveUtils.parse14DigitDate(d);
- break;
-
- case 17:
- date = ArchiveUtils.parse17DigitDate(d);
- break;
-
- case 12:
- date = ArchiveUtils.parse12DigitDate(d);
- break;
-
- case 0:
- case 1:
- case 2:
- case 3:
- throw new ParseException("Date string must at least contain a" +
- "year: " + d, d.length());
-
- default:
- if (!(d.startsWith("19") || d.startsWith("20"))) {
- throw new ParseException("Unrecognized century: " + d, 0);
- }
- if (d.length() < 8 && (d.length() % 2) != 0) {
- throw new ParseException("Incomplete month/date: " + d,
- d.length());
- }
- StringBuilder sb = new StringBuilder(d);
- while (sb.length() < 8) {
- sb.append("01");
- }
- while (sb.length() < 12) {
- sb.append("0");
- }
- date = ArchiveUtils.parse12DigitDate(sb.toString());
- }
-
- return date;
- }
-
- /**
- * Utility function for parsing arc-style date stamps
- * in the format yyyMMddHHmmssSSS.
- * Date stamps are in the UTC time zone. The whole string will not be
- * parsed, only the first 17 digits.
- *
- * @param date an arc-style formatted date stamp
- * @return the Date corresponding to the date stamp string
- * @throws ParseException if the inputstring was malformed
- */
- public static Date parse17DigitDate(String date) throws ParseException {
- return TIMESTAMP17.get().parse(date);
- }
-
- /**
- * Utility function for parsing arc-style date stamps
- * in the format yyyMMddHHmmss.
- * Date stamps are in the UTC time zone. The whole string will not be
- * parsed, only the first 14 digits.
- *
- * @param date an arc-style formatted date stamp
- * @return the Date corresponding to the date stamp string
- * @throws ParseException if the inputstring was malformed
- */
- public static Date parse14DigitDate(String date) throws ParseException{
- return TIMESTAMP14.get().parse(date);
- }
-
- /**
- * Utility function for parsing arc-style date stamps
- * in the format yyyMMddHHmm.
- * Date stamps are in the UTC time zone. The whole string will not be
- * parsed, only the first 12 digits.
- *
- * @param date an arc-style formatted date stamp
- * @return the Date corresponding to the date stamp string
- * @throws ParseException if the inputstring was malformed
- */
- public static Date parse12DigitDate(String date) throws ParseException{
- return TIMESTAMP12.get().parse(date);
- }
-
- /**
- * @param timestamp A 14-digit timestamp or the suffix for a 14-digit
- * timestamp: E.g. '20010909014640' or '20010101' or '1970'.
- * @return Seconds since the epoch as a string zero-pre-padded so always
- * Integer.MAX_VALUE wide (Makes it so sorting of resultant string works
- * properly).
- * @throws ParseException
- */
- public static String secondsSinceEpoch(String timestamp)
- throws ParseException {
- return zeroPadInteger((int)
- (getSecondsSinceEpoch(timestamp).getTime()/1000));
- }
-
- /**
- * @param timestamp A 14-digit timestamp or the suffix for a 14-digit
- * timestamp: E.g. '20010909014640' or '20010101' or '1970'.
- * @return A date.
- * @see #secondsSinceEpoch(String)
- * @throws ParseException
- */
- public static Date getSecondsSinceEpoch(String timestamp)
- throws ParseException {
- if (timestamp.length() < 14) {
- if (timestamp.length() < 10 && (timestamp.length() % 2) == 1) {
- throw new IllegalArgumentException("Must have year, " +
- "month, date, hour or second granularity: " + timestamp);
- }
- if (timestamp.length() == 4) {
- // Add first month and first date.
- timestamp = timestamp + "01010000";
- }
- if (timestamp.length() == 6) {
- // Add a date of the first.
- timestamp = timestamp + "010000";
- }
- if (timestamp.length() < 14) {
- timestamp = timestamp +
- ArchiveUtils.padTo("", 14 - timestamp.length(), '0');
- }
- }
- return ArchiveUtils.parse14DigitDate(timestamp);
- }
-
- /**
- * @param i Integer to add prefix of zeros too. If passed
- * 2005, will return the String 0000002005. String
- * width is the width of Integer.MAX_VALUE as a string (10
- * digits).
- * @return Padded String version of i.
- */
- public static String zeroPadInteger(int i) {
- return ArchiveUtils.padTo(Integer.toString(i),
- MAX_INT_CHAR_WIDTH, '0');
- }
-
- /**
- * Convert an int to a String, and pad it to
- * pad spaces.
- * @param i the int
- * @param pad the width to pad to.
- * @return String w/ padding.
- */
- public static String padTo(final int i, final int pad) {
- String n = Integer.toString(i);
- return padTo(n, pad);
- }
-
- /**
- * Pad the given String to pad characters wide
- * by pre-pending spaces. s should not be null.
- * If s is already wider than pad no change is
- * done.
- *
- * @param s the String to pad
- * @param pad the width to pad to.
- * @return String w/ padding.
- */
- public static String padTo(final String s, final int pad) {
- return padTo(s, pad, DEFAULT_PAD_CHAR);
- }
-
- /**
- * Pad the given String to pad characters wide
- * by pre-pending padChar.
- *
- * s should not be null. If s is
- * already wider than pad no change is done.
- *
- * @param s the String to pad
- * @param pad the width to pad to.
- * @param padChar The pad character to use.
- * @return String w/ padding.
- */
- public static String padTo(final String s, final int pad,
- final char padChar) {
- String result = s;
- int l = s.length();
- if (l < pad) {
- StringBuffer sb = new StringBuffer(pad);
- while(l < pad) {
- sb.append(padChar);
- l++;
- }
- sb.append(s);
- result = sb.toString();
- }
- return result;
- }
-
- /** check that two byte arrays are equal. They may be null.
- *
- * @param lhs a byte array
- * @param rhs another byte array.
- * @return true if they are both equal (or both
- * null)
- */
- public static boolean byteArrayEquals(final byte[] lhs, final byte[] rhs) {
- if (lhs == null && rhs != null || lhs != null && rhs == null) {
- return false;
- }
- if (lhs==rhs) {
- return true;
- }
- if (lhs.length != rhs.length) {
- return false;
- }
- for(int i = 0; i
- * This involves converting it to the largest unit
- * (of B, KiB, MiB, GiB, TiB) for which the amount will be > 1.
- *
- * Additionally, at least 2 significant digits are always displayed.
- *
- * Negative numbers will be returned as '0 B'.
- *
- * @param amount the amount of bytes
- * @return A string containing the amount, properly formated.
- */
- public static String formatBytesForDisplay(long amount) {
- double displayAmount = (double) amount;
- int unitPowerOf1024 = 0;
-
- if(amount <= 0){
- return "0 B";
- }
-
- while(displayAmount>=1024 && unitPowerOf1024 < 4) {
- displayAmount = displayAmount / 1024;
- unitPowerOf1024++;
- }
-
- final String[] units = { " B", " KiB", " MiB", " GiB", " TiB" };
-
- // ensure at least 2 significant digits (#.#) for small displayValues
- int fractionDigits = (displayAmount < 10) ? 1 : 0;
- return doubleToString(displayAmount, fractionDigits, fractionDigits)
- + units[unitPowerOf1024];
- }
-
- /**
- * Convert milliseconds value to a human-readable duration
- * @param time
- * @return Human readable string version of passed time
- */
- public static String formatMillisecondsToConventional(long time) {
- return formatMillisecondsToConventional(time,5);
- }
-
- /**
- * Convert milliseconds value to a human-readable duration of
- * mixed units, using units no larger than days. For example,
- * "5d12h13m12s113ms" or "19h51m".
- *
- * @param duration
- * @param unitCount how many significant units to show, at most
- * for example, a value of 2 would show days+hours or hours+seconds
- * but not hours+second+milliseconds
- * @return Human readable string version of passed time
- */
- public static String formatMillisecondsToConventional(long duration, int unitCount) {
- if(unitCount <=0) {
- unitCount = 5;
- }
- if(duration==0) {
- return "0ms";
- }
- StringBuffer sb = new StringBuffer();
- if(duration<0) {
- sb.append("-");
- }
- long absTime = Math.abs(duration);
- long[] thresholds = {DAY_IN_MS, HOUR_IN_MS, 60000, 1000, 1};
- String[] units = {"d","h","m","s","ms"};
-
- for(int i = 0; i < thresholds.length; i++) {
- if(absTime >= thresholds[i]) {
- sb.append(absTime / thresholds[i] + units[i]);
- absTime = absTime % thresholds[i];
- unitCount--;
- }
- if(unitCount==0) {
- break;
- }
- }
- return sb.toString();
- }
-
- /**
- * Copy the raw bytes of a long into a byte array, starting at
- * the specified offset.
- *
- * @param l
- * @param array
- * @param offset
- */
- public static void longIntoByteArray(long l, byte[] array, int offset) {
- int i, shift;
-
- for(i = 0, shift = 56; i < 8; i++, shift -= 8)
- array[offset+i] = (byte)(0xFF & (l >> shift));
- }
-
- public static long byteArrayIntoLong(byte [] bytearray) {
- return byteArrayIntoLong(bytearray, 0);
- }
-
- /**
- * Byte array into long.
- * @param bytearray Array to convert to a long.
- * @param offset Offset into array at which we start decoding the long.
- * @return Long made of the bytes of array beginning at
- * offset offset.
- * @see #longIntoByteArray(long, byte[], int)
- */
- public static long byteArrayIntoLong(byte [] bytearray,
- int offset) {
- long result = 0;
- for (int i = offset; i < 8 /*Bytes in long*/; i++) {
- result = (result << 8 /*Bits in byte*/) |
- (0xff & (byte)(bytearray[i] & 0xff));
- }
- return result;
- }
-
- /**
- * Given a string that may be a plain host or host/path (without
- * URI scheme), add an implied http:// if necessary.
- *
- * @param u string to evaluate
- * @return string with http:// added if no scheme already present
- */
- public static String addImpliedHttpIfNecessary(String u) {
- int colon = u.indexOf(':');
- int period = u.indexOf('.');
- if (colon == -1 || (period >= 0) && (period < colon)) {
- // No scheme present; prepend "http://"
- u = "http://" + u;
- }
- return u;
- }
-
- /**
- * Verify that the array begins with the prefix.
- *
- * @param array
- * @param prefix
- * @return true if array is identical to prefix for the first prefix.length
- * positions
- */
- public static boolean startsWith(byte[] array, byte[] prefix) {
- if(prefix.length>array.length) {
- return false;
- }
- for(int i = 0; i < prefix.length; i++) {
- if(array[i]!=prefix[i]) {
- return false;
- }
- }
- return true;
- }
-
- /**
- * Utility method to get a String shortReportLine from Reporter
- * @param rep Reporter to get shortReportLine from
- * @return String of report
- */
- public static String shortReportLine(Reporter rep) {
- StringWriter sw = new StringWriter();
- PrintWriter pw = new PrintWriter(sw);
- try {
- rep.shortReportLineTo(pw);
- } catch (IOException e) {
- // not really possible
- e.printStackTrace();
- }
- pw.flush();
- return sw.toString();
- }
-
- /**
- * Enhance given object's default String display for appearing
- * nested in a pretty Map String.
- *
- * @param obj Object to prettify
- * @return prettified String
- */
- public static String prettyString(Object obj) {
- // these things have to checked and casted unfortunately
- if (obj instanceof Object[]) {
- return prettyString((Object[]) obj);
- } else if (obj instanceof Map) {
- return prettyString((Map) obj);
- } else {
- return "<"+obj+">";
- }
- }
-
- /**
- * Provide a improved String of a Map's entries
- *
- * @param Map
- * @return prettified (in curly brackets) string of Map contents
- */
- public static String prettyString(Map map) {
- StringBuilder builder = new StringBuilder();
- builder.append("{ ");
- boolean needsComma = false;
- for( Object key : map.keySet()) {
- if(needsComma) {
- builder.append(", ");
- }
- builder.append(key);
- builder.append(": ");
- builder.append(prettyString(map.get(key)));
- needsComma = true;
- }
- builder.append(" }");
- return builder.toString();
- }
-
- /**
- * Provide a slightly-improved String of Object[]
- *
- * @param Object[]
- * @return prettified (in square brackets) of Object[]
- */
- public static String prettyString(Object[] array) {
- StringBuilder builder = new StringBuilder();
- builder.append("[ ");
- boolean needsComma = false;
- for (Object o: array) {
- if(o==null) continue;
- if(needsComma) {
- builder.append(", ");
- }
- builder.append(prettyString(o));
- needsComma = true;
- }
- builder.append(" ]");
- return builder.toString();
- }
-
-
- private static String loadVersion() {
- InputStream input = ArchiveUtils.class.getResourceAsStream(
- "/org/archive/util/version.txt");
- if (input == null) {
- return "UNKNOWN";
- }
- BufferedReader br = null;
- String version;
- try {
- br = new BufferedReader(new InputStreamReader(input));
- version = br.readLine();
- br.readLine();
- } catch (IOException e) {
- return e.getMessage();
- } finally {
- closeQuietly(br);
- }
-
- version = version.trim();
- if (!version.endsWith("SNAPSHOT")) {
- return version;
- }
-
- input = ArchiveUtils.class.getResourceAsStream("/org/archive/util/timestamp.txt");
- if (input == null) {
- return version;
- }
-
- br = null;
- String timestamp;
- try {
- br = new BufferedReader(new InputStreamReader(input));
- timestamp = br.readLine();
- } catch (IOException e) {
- return version;
- } finally {
- closeQuietly(br);
- }
-
- if (timestamp.startsWith("timestamp=")) {
- timestamp = timestamp.substring(10);
- }
-
- return version.trim() + "-" + timestamp.trim();
- }
-
- public static Set TLDS;
-
- static {
- TLDS = new HashSet();
- InputStream is = ArchiveUtils.class.getResourceAsStream("tlds-alpha-by-domain.txt");
- try {
- BufferedReader reader = new BufferedReader(new InputStreamReader(is));
- String line;
- while((line = reader.readLine())!=null) {
- if (line.startsWith("#")) {
- continue;
- }
- TLDS.add(line.trim().toLowerCase());
- }
- } catch (Exception e) {
- LOGGER.log(Level.SEVERE,"TLD list unavailable",e);
- } finally {
- IOUtils.closeQuietly(is);
- }
- }
- /**
- * Return whether the given string represents a known
- * top-level-domain (like "com", "org", etc.) per IANA
- * as of 20100419
- *
- * @param dom candidate string
- * @return boolean true if recognized as TLD
- */
- public static boolean isTld(String dom) {
- return TLDS.contains(dom.toLowerCase());
- }
-
- public static void closeQuietly(Object input) {
- if(input == null || ! (input instanceof Closeable)) {
- return;
- }
- try {
- ((Closeable)input).close();
- } catch (IOException ioe) {
- // ignore
- }
- }
-
- /**
- * Perform checks as to whether normal execution should proceed.
- *
- * If an external interrupt is detected, throw an interrupted exception.
- * Used before anything that should not be attempted by a 'zombie' thread
- * that the Frontier/Crawl has given up on.
- *
- * @throws InterruptedException
- */
- public static void continueCheck() throws InterruptedException {
- if(Thread.interrupted()) {
- throw new InterruptedException("interrupt detected");
- }
- }
-
- /**
- * Read stream into buf until EOF or buf full.
- *
- * @param input
- * @param buf
- * @throws IOException
- */
- public static int readFully(InputStream input, byte[] buf)
- throws IOException {
- int max = buf.length;
- int ofs = 0;
- while (ofs < max) {
- int l = input.read(buf, ofs, max - ofs);
- if (l == 0) {
- throw new EOFException();
- }
- ofs += l;
- }
- return ofs;
- }
-
- /** suffix to recognize gzipped files */
- public static final String GZIP_SUFFIX = ".gz";
-
- /**
- * Get a BufferedReader on the crawler journal given
- *
- * TODO: move to a general utils class
- *
- * @param source File journal
- * @return journal buffered reader.
- * @throws IOException
- */
- public static BufferedReader getBufferedReader(File source) throws IOException {
- InputStream is = new BufferedInputStream(new FileInputStream(source));
- boolean isGzipped = source.getName().toLowerCase().
- endsWith(GZIP_SUFFIX);
- if(isGzipped) {
- is = new GZIPInputStream(is);
- }
- return new BufferedReader(new InputStreamReader(is));
- }
-
- /**
- * Get a BufferedReader on the crawler journal given.
- *
- * @param source URL journal
- * @return journal buffered reader.
- * @throws IOException
- */
- public static BufferedReader getBufferedReader(URL source) throws IOException {
- URLConnection conn = source.openConnection();
- boolean isGzipped = conn.getContentType() != null && conn.getContentType().equalsIgnoreCase("application/x-gzip")
- || conn.getContentEncoding() != null && conn.getContentEncoding().equalsIgnoreCase("gzip");
- InputStream uis = conn.getInputStream();
- return new BufferedReader(isGzipped?
- new InputStreamReader(new GZIPInputStream(uis)):
- new InputStreamReader(uis));
- }
-
- /**
- * Gzip passed bytes.
- * Use only when bytes is small.
- * @param bytes What to gzip.
- * @return A gzip member of bytes.
- * @throws IOException
- */
- public static byte [] gzip(byte [] bytes) throws IOException {
- ByteArrayOutputStream baos = new ByteArrayOutputStream();
- GZIPOutputStream gzipOS = new GZIPOutputStream(baos);
- gzipOS.write(bytes, 0, bytes.length);
- gzipOS.close();
- return baos.toByteArray();
- }
-
- /**
- * Tests passed stream is gzip stream by reading in the HEAD.
- * Does not mark/reset stream -- so this test actually makes
- * stream unopenable within GZIP streams, unless reset.
- * @param is An InputStream.
- * @return True if compressed stream.
- * @throws IOException
- */
- public static boolean isGzipped(final InputStream is)
- throws IOException {
- try {
- new GzipHeader(is);
- } catch (NoGzipMagicException e) {
- return false;
- }
- return true;
- }
-
-// public static long doubleMurmur(byte[] data) {
-// int first = MurmurHash.hash(data, 7);
-// int second = MurmurHash.hash(data, 13);
-// return (((long)first)<<32) | ((long)second & 0x00000000ffffffffl);
-// }
-}
-
diff --git a/commons/src/main/java/org/archive/util/InterruptibleCharSequence.java b/commons/src/main/java/org/archive/util/InterruptibleCharSequence.java
deleted file mode 100644
index f608c49c..00000000
--- a/commons/src/main/java/org/archive/util/InterruptibleCharSequence.java
+++ /dev/null
@@ -1,56 +0,0 @@
-/*
- * This file is part of the Heritrix web crawler (crawler.archive.org).
- *
- * Licensed to the Internet Archive (IA) by one or more individual
- * contributors.
- *
- * The IA licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.archive.util;
-
-/**
- * CharSequence that noticed thread interrupts -- as might be necessary
- * to recover from a loose regex on unexpected challenging input.
- *
- * @author gojomo
- */
-public class InterruptibleCharSequence implements CharSequence {
- protected CharSequence inner;
- // public long counter = 0;
-
- public InterruptibleCharSequence(CharSequence inner) {
- super();
- this.inner = inner;
- }
-
- public char charAt(int index) {
- if (Thread.interrupted()) { // clears flag if set
- throw new RuntimeException(new InterruptedException());
- }
- // counter++;
- return inner.charAt(index);
- }
-
- public int length() {
- return inner.length();
- }
-
- public CharSequence subSequence(int start, int end) {
- return new InterruptibleCharSequence(inner.subSequence(start, end));
- }
-
- @Override
- public String toString() {
- return inner.toString();
- }
-}
diff --git a/commons/src/main/java/org/archive/util/InterruptibleCharSequenceTest.java b/commons/src/main/java/org/archive/util/InterruptibleCharSequenceTest.java
deleted file mode 100644
index a3a5f180..00000000
--- a/commons/src/main/java/org/archive/util/InterruptibleCharSequenceTest.java
+++ /dev/null
@@ -1,120 +0,0 @@
-/*
- * This file is part of the Heritrix web crawler (crawler.archive.org).
- *
- * Licensed to the Internet Archive (IA) by one or more individual
- * contributors.
- *
- * The IA licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.archive.util;
-
-import java.util.concurrent.BlockingQueue;
-import java.util.concurrent.LinkedBlockingQueue;
-import java.util.regex.Pattern;
-
-import junit.framework.TestCase;
-
-/**
- * Tests (and
- * @author gojomo
- */
-public class InterruptibleCharSequenceTest extends TestCase {
- // this regex takes many seconds to fail on the input
- // (~20 seconds on 2Ghz Athlon64 JDK 1.6)
- public static String BACKTRACKER = "^(((((a+)*)*)*)*)*$";
- public static String INPUT = "aaaaab";
-
- /**
- * Development-time benchmarking of InterruptibleCharSequence in
- * regex use. (Rename 'xest' to 'test' if wanted as unit test,
- * but never actually fails anything -- just measures.)
- *
- * For reference the regex "^(((((a+)*)*)*)*)*$" requires
- * 239,286,636 charAt(s) to fail on "aaaaab", which takes
- * around 20 seconds on a 2Ghz Athlon64(x2) with JDK 1.6.
- * The runtime overhead of checking interrupt status in this
- * extreme case is around 5% in my tests.
- */
- public void xestOverhead() {
- String regex = BACKTRACKER;
- String inputNormal = INPUT;
- InterruptibleCharSequence inputWrapped = new InterruptibleCharSequence(inputNormal);
- // warm up
- tryMatch(inputNormal,regex);
- tryMatch(inputWrapped,regex);
- // inputWrapped.counter=0;
- int trials = 5;
- long stringTally = 0;
- long icsTally = 0;
- for(int i = 1; i <= trials; i++) {
- System.out.println("trial "+i+" of "+trials);
- long start = System.currentTimeMillis();
- System.out.print("String ");
- tryMatch(inputNormal,regex);
- long end = System.currentTimeMillis();
- System.out.println(end-start);
- stringTally += (end-start);
- start = System.currentTimeMillis();
- System.out.print("InterruptibleCharSequence ");
- tryMatch(inputWrapped,regex);
- end = System.currentTimeMillis();
- System.out.println(end-start);
- //System.out.println(inputWrapped.counter+" steps");
- //inputWrapped.counter=0;
- icsTally += (end-start);
- }
- System.out.println("InterruptibleCharSequence took "+((float)icsTally)/stringTally+" longer.");
- }
-
- public boolean tryMatch(CharSequence input, String regex) {
- return Pattern.matches(regex,input);
- }
-
- public Thread tryMatchInThread(final CharSequence input, final String regex, final BlockingQueue