From 3a65fa25d79225532222455eff8ce7e12a4693ce Mon Sep 17 00:00:00 2001 From: Noah Levitt Date: Tue, 18 Dec 2012 14:46:52 -0800 Subject: [PATCH] Working on refactoring some stuff into archive-commons --- .classpath | 4 +- commons/pom.xml | 12 +- .../commons/httpclient/HttpMethodBase.java | 4 +- .../src/main/java/org/archive/net/LaxURI.java | 459 ------- .../java/org/archive/net/LaxURLCodec.java | 160 --- .../src/main/java/org/archive/net/UURI.java | 433 +----- .../java/org/archive/net/UURIFactory.java | 763 +---------- .../java/org/archive/util/ArchiveUtils.java | 1008 -------------- .../util/InterruptibleCharSequence.java | 56 - .../util/InterruptibleCharSequenceTest.java | 120 -- .../main/java/org/archive/util/PrefixSet.java | 74 - .../java/org/archive/util/ReportUtils.java | 45 + .../src/main/java/org/archive/util/SURT.java | 256 ---- .../java/org/archive/util/SurtPrefixSet.java | 360 ----- .../main/java/org/archive/util/TextUtils.java | 308 ----- .../main/java/org/archive/util/UriUtils.java | 2 +- .../util/iterator/LineReadingIterator.java | 59 - .../util/iterator/LookaheadIterator.java | 73 - .../util/iterator/RegexLineIterator.java | 91 -- .../iterator/TransformingIteratorWrapper.java | 65 - .../util/zip/OpenJDK7GZIPInputStream.java | 295 ---- .../util/zip/OpenJDK7InflaterInputStream.java | 291 ---- .../java/org/archive/net/UURIFactoryTest.java | 1200 ----------------- .../test/java/org/archive/net/UURITest.java | 55 - .../crawler/framework/CrawlController.java | 5 +- .../archive/crawler/framework/ToeThread.java | 3 +- .../crawler/frontier/AbstractFrontier.java | 3 +- .../archive/crawler/frontier/WorkQueue.java | 3 +- .../precedence/PrecedenceProvider.java | 3 +- .../java/org/archive/modules/CrawlURI.java | 4 +- .../modules/extractor/ExtractorURI.java | 2 +- .../archive/modules/fetcher/FetchStats.java | 3 +- 32 files changed, 99 insertions(+), 6120 deletions(-) delete mode 100644 commons/src/main/java/org/archive/net/LaxURI.java delete mode 100644 commons/src/main/java/org/archive/net/LaxURLCodec.java delete mode 100644 commons/src/main/java/org/archive/util/ArchiveUtils.java delete mode 100644 commons/src/main/java/org/archive/util/InterruptibleCharSequence.java delete mode 100644 commons/src/main/java/org/archive/util/InterruptibleCharSequenceTest.java delete mode 100644 commons/src/main/java/org/archive/util/PrefixSet.java create mode 100644 commons/src/main/java/org/archive/util/ReportUtils.java delete mode 100644 commons/src/main/java/org/archive/util/SURT.java delete mode 100644 commons/src/main/java/org/archive/util/SurtPrefixSet.java delete mode 100644 commons/src/main/java/org/archive/util/TextUtils.java delete mode 100644 commons/src/main/java/org/archive/util/iterator/LineReadingIterator.java delete mode 100644 commons/src/main/java/org/archive/util/iterator/LookaheadIterator.java delete mode 100644 commons/src/main/java/org/archive/util/iterator/RegexLineIterator.java delete mode 100644 commons/src/main/java/org/archive/util/iterator/TransformingIteratorWrapper.java delete mode 100644 commons/src/main/java/org/archive/util/zip/OpenJDK7GZIPInputStream.java delete mode 100644 commons/src/main/java/org/archive/util/zip/OpenJDK7InflaterInputStream.java delete mode 100644 commons/src/test/java/org/archive/net/UURIFactoryTest.java delete mode 100644 commons/src/test/java/org/archive/net/UURITest.java diff --git a/.classpath b/.classpath index 5d03c472..0c8f1ff7 100644 --- a/.classpath +++ b/.classpath @@ -14,12 +14,11 @@ - - + @@ -59,5 +58,6 @@ + diff --git a/commons/pom.xml b/commons/pom.xml index 790e31bb..4d3ae25c 100644 --- a/commons/pom.xml +++ b/commons/pom.xml @@ -191,12 +191,6 @@ 5.0.7 compile - - org.gnu.inet - libidn - 0.6.5 - compile - net.java.dev.jets3t jets3t @@ -282,6 +276,12 @@ 3.2.3 + + org.archive + archive-commons + 1.0-SNAPSHOT + + diff --git a/commons/src/main/java/org/apache/commons/httpclient/HttpMethodBase.java b/commons/src/main/java/org/apache/commons/httpclient/HttpMethodBase.java index cda1b9d2..aa171b04 100644 --- a/commons/src/main/java/org/apache/commons/httpclient/HttpMethodBase.java +++ b/commons/src/main/java/org/apache/commons/httpclient/HttpMethodBase.java @@ -217,7 +217,7 @@ public abstract class HttpMethodBase implements HttpMethod { } // BEGIN IA/HERITRIX CHANGES // setURI(new URI(uri, true)); - setURI(new org.archive.net.LaxURI(uri, true)); + setURI(new org.archive.url.LaxURI(uri, true)); // END IA/HERITRIX CHANGES } catch (URIException e) { throw new IllegalArgumentException("Invalid uri '" @@ -264,7 +264,7 @@ public abstract class HttpMethodBase implements HttpMethod { } // BEGIN IA/HERITRIX CHANGES // return new URI(buffer.toString(), true); - return new org.archive.net.LaxURI(buffer.toString(), true); + return new org.archive.url.LaxURI(buffer.toString(), true); // END IA/HERITRIX CHANGES } diff --git a/commons/src/main/java/org/archive/net/LaxURI.java b/commons/src/main/java/org/archive/net/LaxURI.java deleted file mode 100644 index 200e9fb7..00000000 --- a/commons/src/main/java/org/archive/net/LaxURI.java +++ /dev/null @@ -1,459 +0,0 @@ -/* - * This file is part of the Heritrix web crawler (crawler.archive.org). - * - * Licensed to the Internet Archive (IA) by one or more individual - * contributors. - * - * The IA licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.archive.net; - -import java.util.Arrays; -import java.util.BitSet; - -import org.apache.commons.httpclient.URI; -import org.apache.commons.httpclient.URIException; -import org.apache.commons.httpclient.util.EncodingUtil; - -/** - * URI subclass which allows partial/inconsistent encoding, matching - * the URIs which will be relayed in requests from popular web - * browsers (esp. Mozilla Firefox and MS IE). - * - * @author gojomo - */ -public class LaxURI extends URI { - - private static final long serialVersionUID = 5273922211722239537L; - - final protected static char[] HTTP_SCHEME = {'h','t','t','p'}; - final protected static char[] HTTPS_SCHEME = {'h','t','t','p','s'}; - - protected static final BitSet lax_rel_segment = new BitSet(256); - // Static initializer for lax_rel_segment - static { - lax_rel_segment.or(rel_segment); - lax_rel_segment.set(':'); // allow ':' - // TODO: add additional allowances as need is demonstrated - } - - protected static final BitSet lax_abs_path = new BitSet(256); - static { - lax_abs_path.or(abs_path); - lax_abs_path.set('|'); // tests indicate Firefox (1.0.6) doesn't escape. - } - - protected static final BitSet lax_rel_path = new BitSet(256); - // Static initializer for rel_path - static { - lax_rel_path.or(lax_rel_segment); - lax_rel_path.or(lax_abs_path); - } - - protected static final BitSet lax_query = new BitSet(256); - static { - lax_query.or(query); - lax_query.set('{'); // tests indicate FF doesn't escape { in query - lax_query.set('}'); // tests indicate FF doesn't escape } in query - lax_query.set('|'); // tests indicate FF doesn't escape | in query - lax_query.set('['); // tests indicate FF doesn't escape [ in query - lax_query.set(']'); // tests indicate FF doesn't escape ] in query - lax_query.set('^'); // tests indicate FF doesn't escape ^ in query - } - - // passthrough initializers - public LaxURI(String uri, boolean escaped, String charset) - throws URIException { - super(uri,escaped,charset); - } - public LaxURI(URI base, URI relative) throws URIException { - super(base,relative); - } - public LaxURI(String uri, boolean escaped) throws URIException { - super(uri,escaped); - } - public LaxURI() { - super(); - } - - // overridden to use this class's static decode() - public String getURI() throws URIException { - return (_uri == null) ? null : decode(_uri, getProtocolCharset()); - } - - // overridden to use this class's static decode() - public String getPath() throws URIException { - char[] p = getRawPath(); - return (p == null) ? null : decode(p, getProtocolCharset()); - } - - // overridden to use this class's static decode() - public String getPathQuery() throws URIException { - char[] rawPathQuery = getRawPathQuery(); - return (rawPathQuery == null) ? null : decode(rawPathQuery, - getProtocolCharset()); - } - // overridden to use this class's static decode() - protected static String decode(char[] component, String charset) - throws URIException { - if (component == null) { - throw new IllegalArgumentException( - "Component array of chars may not be null"); - } - return decode(new String(component), charset); - } - - // overridden to use IA's LaxURLCodec, which never throws DecoderException - protected static String decode(String component, String charset) - throws URIException { - if (component == null) { - throw new IllegalArgumentException( - "Component array of chars may not be null"); - } - byte[] rawdata = null; - // try { - rawdata = LaxURLCodec.decodeUrlLoose(EncodingUtil - .getAsciiBytes(component)); - // } catch (DecoderException e) { - // throw new URIException(e.getMessage()); - // } - return EncodingUtil.getString(rawdata, charset); - } - - // overidden to lax() the acceptable-char BitSet passed in - protected boolean validate(char[] component, BitSet generous) { - return super.validate(component, lax(generous)); - } - - // overidden to lax() the acceptable-char BitSet passed in - protected boolean validate(char[] component, int soffset, int eoffset, - BitSet generous) { - return super.validate(component, soffset, eoffset, lax(generous)); - } - - /** - * Given a BitSet -- typically one of the URI superclass's - * predefined static variables -- possibly replace it with - * a more-lax version to better match the character sets - * actually left unencoded in web browser requests - * - * @param generous original BitSet - * @return (possibly more lax) BitSet to use - */ - protected BitSet lax(BitSet generous) { - if (generous == rel_segment) { - // Swap in more lax allowable set - return lax_rel_segment; - } - if (generous == abs_path) { - return lax_abs_path; - } - if (generous == query) { - return lax_query; - } - if (generous == rel_path) { - return lax_rel_path; - } - // otherwise, leave as is - return generous; - } - - /** - * Coalesce the _host and _authority fields where - * possible. - * - * In the web crawl/http domain, most URIs have an - * identical _host and _authority. (There is no port - * or user info.) However, the superclass always - * creates two separate char[] instances. - * - * Notably, the lengths of these char[] fields are - * equal if and only if their values are identical. - * This method makes use of this fact to reduce the - * two instances to one where possible, slimming - * instances. - * - * @see org.apache.commons.httpclient.URI#parseAuthority(java.lang.String, boolean) - */ - protected void parseAuthority(String original, boolean escaped) - throws URIException { - super.parseAuthority(original, escaped); - if (_host != null && _authority != null - && _host.length == _authority.length) { - _host = _authority; - } - } - - - /** - * Coalesce _scheme to existing instances, where appropriate. - * - * In the web-crawl domain, most _schemes are 'http' or 'https', - * but the superclass always creates a new char[] instance. For - * these two cases, we replace the created instance with a - * long-lived instance from a static field, saving 12-14 bytes - * per instance. - * - * @see org.apache.commons.httpclient.URI#setURI() - */ - protected void setURI() { - if (_scheme != null) { - if (_scheme.length == 4 && Arrays.equals(_scheme, HTTP_SCHEME)) { - _scheme = HTTP_SCHEME; - } else if (_scheme.length == 5 - && Arrays.equals(_scheme, HTTP_SCHEME)) { - _scheme = HTTPS_SCHEME; - } - } - super.setURI(); - } - - /** - * IA OVERRIDDEN IN LaxURI TO INCLUDE FIX FOR - * http://issues.apache.org/jira/browse/HTTPCLIENT-588 - * AND - * http://webteam.archive.org/jira/browse/HER-1268 - * - * In order to avoid any possilbity of conflict with non-ASCII characters, - * Parse a URI reference as a String with the character - * encoding of the local system or the document. - *

- * The following line is the regular expression for breaking-down a URI - * reference into its components. - * 

-     *   ^(([^:/?#]+):)?(//([^/?#]*))?([^?#]*)(\?([^#]*))?(#(.*))?
-     *    12            3  4          5       6  7        8 9
-     *

- * For example, matching the above expression to - * http://jakarta.apache.org/ietf/uri/#Related - * results in the following subexpression matches: - * 

-     *               $1 = http:
-     *  scheme    =  $2 = http
-     *               $3 = //jakarta.apache.org
-     *  authority =  $4 = jakarta.apache.org
-     *  path      =  $5 = /ietf/uri/
-     *               $6 = 
-     *  query     =  $7 = 
-     *               $8 = #Related
-     *  fragment  =  $9 = Related
-     *

- * - * @param original the original character sequence - * @param escaped true if original is escaped - * @throws URIException If an error occurs. - */ - protected void parseUriReference(String original, boolean escaped) - throws URIException { - - // validate and contruct the URI character sequence - if (original == null) { - throw new URIException("URI-Reference required"); - } - - /* @ - * ^(([^:/?#]+):)?(//([^/?#]*))?([^?#]*)(\?([^#]*))?(#(.*))? - */ - String tmp = original.trim(); - - /* - * The length of the string sequence of characters. - * It may not be equal to the length of the byte array. - */ - int length = tmp.length(); - - /* - * Remove the delimiters like angle brackets around an URI. - */ - if (length > 0) { - char[] firstDelimiter = { tmp.charAt(0) }; - if (validate(firstDelimiter, delims)) { - if (length >= 2) { - char[] lastDelimiter = { tmp.charAt(length - 1) }; - if (validate(lastDelimiter, delims)) { - tmp = tmp.substring(1, length - 1); - length = length - 2; - } - } - } - } - - /* - * The starting index - */ - int from = 0; - - /* - * The test flag whether the URI is started from the path component. - */ - boolean isStartedFromPath = false; - int atColon = tmp.indexOf(':'); - int atSlash = tmp.indexOf('/'); - if (!tmp.startsWith("//") - && (atColon <= 0 || (atSlash >= 0 && atSlash < atColon))) { - isStartedFromPath = true; - } - - /* - * 

-         *     @@@@@@@@
-         *  ^(([^:/?#]+):)?(//([^/?#]*))?([^?#]*)(\?([^#]*))?(#(.*))?
-         *

- */ - int at = indexFirstOf(tmp, isStartedFromPath ? "/?#" : ":/?#", from); - if (at == -1) { - at = 0; - } - - /* - * Parse the scheme. - * 

-         *  scheme    =  $2 = http
-         *              @
-         *  ^(([^:/?#]+):)?(//([^/?#]*))?([^?#]*)(\?([^#]*))?(#(.*))?
-         *

- */ - if (at > 0 && at < length && tmp.charAt(at) == ':') { - char[] target = tmp.substring(0, at).toLowerCase().toCharArray(); - if (validate(target, scheme)) { - _scheme = target; - from = ++at; - } else { - // IA CHANGE: - // do nothing; allow interpretation as URI with - // later colon in other syntactical component - } - } - - /* - * Parse the authority component. - * 

-         *  authority =  $4 = jakarta.apache.org
-         *                  @@
-         *  ^(([^:/?#]+):)?(//([^/?#]*))?([^?#]*)(\?([^#]*))?(#(.*))?
-         *

- */ - // Reset flags - _is_net_path = _is_abs_path = _is_rel_path = _is_hier_part = false; - if (0 <= at && at < length && tmp.charAt(at) == '/') { - // Set flag - _is_hier_part = true; - if (at + 2 < length && tmp.charAt(at + 1) == '/' - && !isStartedFromPath) { - // the temporary index to start the search from - int next = indexFirstOf(tmp, "/?#", at + 2); - if (next == -1) { - next = (tmp.substring(at + 2).length() == 0) ? at + 2 - : tmp.length(); - } - parseAuthority(tmp.substring(at + 2, next), escaped); - from = at = next; - // Set flag - _is_net_path = true; - } - if (from == at) { - // Set flag - _is_abs_path = true; - } - } - - /* - * Parse the path component. - * 

-         *  path      =  $5 = /ietf/uri/
-         *                                @@@@@@
-         *  ^(([^:/?#]+):)?(//([^/?#]*))?([^?#]*)(\?([^#]*))?(#(.*))?
-         *

- */ - if (from < length) { - // rel_path = rel_segment [ abs_path ] - int next = indexFirstOf(tmp, "?#", from); - if (next == -1) { - next = tmp.length(); - } - if (!_is_abs_path) { - if (!escaped - && prevalidate(tmp.substring(from, next), disallowed_rel_path) - || escaped - && validate(tmp.substring(from, next).toCharArray(), rel_path)) { - // Set flag - _is_rel_path = true; - } else if (!escaped - && prevalidate(tmp.substring(from, next), disallowed_opaque_part) - || escaped - && validate(tmp.substring(from, next).toCharArray(), opaque_part)) { - // Set flag - _is_opaque_part = true; - } else { - // the path component may be empty - _path = null; - } - } - String s = tmp.substring(from, next); - if (escaped) { - setRawPath(s.toCharArray()); - } else { - setPath(s); - } - at = next; - } - - // set the charset to do escape encoding - String charset = getProtocolCharset(); - - /* - * Parse the query component. - * 

-         *  query     =  $7 = 
-         *                                        @@@@@@@@@
-         *  ^(([^:/?#]+):)?(//([^/?#]*))?([^?#]*)(\?([^#]*))?(#(.*))?
-         *

- */ - if (0 <= at && at + 1 < length && tmp.charAt(at) == '?') { - int next = tmp.indexOf('#', at + 1); - if (next == -1) { - next = tmp.length(); - } - if (escaped) { - _query = tmp.substring(at + 1, next).toCharArray(); - if (!validate(_query, query)) { - throw new URIException("Invalid query"); - } - } else { - _query = encode(tmp.substring(at + 1, next), allowed_query, charset); - } - at = next; - } - - /* - * Parse the fragment component. - * 

-         *  fragment  =  $9 = Related
-         *                                                   @@@@@@@@
-         *  ^(([^:/?#]+):)?(//([^/?#]*))?([^?#]*)(\?([^#]*))?(#(.*))?
-         *

- */ - if (0 <= at && at + 1 <= length && tmp.charAt(at) == '#') { - if (at + 1 == length) { // empty fragment - _fragment = "".toCharArray(); - } else { - _fragment = (escaped) ? tmp.substring(at + 1).toCharArray() - : encode(tmp.substring(at + 1), allowed_fragment, charset); - } - } - - // set this URI. - setURI(); - } - -} diff --git a/commons/src/main/java/org/archive/net/LaxURLCodec.java b/commons/src/main/java/org/archive/net/LaxURLCodec.java deleted file mode 100644 index ef2567bc..00000000 --- a/commons/src/main/java/org/archive/net/LaxURLCodec.java +++ /dev/null @@ -1,160 +0,0 @@ -/* - * This file is part of the Heritrix web crawler (crawler.archive.org). - * - * Licensed to the Internet Archive (IA) by one or more individual - * contributors. - * - * The IA licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.archive.net; - -import java.io.ByteArrayOutputStream; -import java.io.UnsupportedEncodingException; -import java.util.BitSet; - -import org.apache.commons.codec.net.URLCodec; - -import com.google.common.base.Charsets; - -/** - * @author gojomo - */ -public class LaxURLCodec extends URLCodec { - public static LaxURLCodec DEFAULT = new LaxURLCodec("UTF-8"); - - // passthrough constructor - public LaxURLCodec(String encoding) { - super(encoding); - } - - /** - * Decodes an array of URL safe 7-bit characters into an array of - * original bytes. Escaped characters are converted back to their - * original representation. - * - * Differs from URLCodec.decodeUrl() in that it throws no - * exceptions; bad or incomplete escape sequences are ignored - * and passed into result undecoded. This matches the behavior - * of browsers, which will use inconsistently-encoded URIs - * in HTTP request-lines. - * - * @param bytes array of URL safe characters - * @return array of original bytes - */ - public static final byte[] decodeUrlLoose(byte[] bytes) - { - if (bytes == null) { - return null; - } - ByteArrayOutputStream buffer = new ByteArrayOutputStream(); - for (int i = 0; i < bytes.length; i++) { - int b = bytes[i]; - if (b == '+') { - buffer.write(' '); - continue; - } - if (b == '%') { - if(i+2 -1 && l > -1) { - // good encoding - int c = ((u << 4) + l); - buffer.write((char)c); - i += 2; - continue; - } // else: bad encoding digits, leave '%' in place - } // else: insufficient encoding digits, leave '%' in place - } - buffer.write(b); - } - return buffer.toByteArray(); - } - - /** - * A more expansive set of ASCII URI characters to consider as 'safe' to - * leave unencoded, based on actual browser behavior. - */ - public static BitSet EXPANDED_URI_SAFE = new BitSet(256); - static { - // alpha characters - for (int i = 'a'; i <= 'z'; i++) { - EXPANDED_URI_SAFE.set(i); - } - for (int i = 'A'; i <= 'Z'; i++) { - EXPANDED_URI_SAFE.set(i); - } - // numeric characters - for (int i = '0'; i <= '9'; i++) { - EXPANDED_URI_SAFE.set(i); - } - // special chars - EXPANDED_URI_SAFE.set('-'); - EXPANDED_URI_SAFE.set('~'); - EXPANDED_URI_SAFE.set('_'); - EXPANDED_URI_SAFE.set('.'); - EXPANDED_URI_SAFE.set('*'); - EXPANDED_URI_SAFE.set('/'); - EXPANDED_URI_SAFE.set('='); - EXPANDED_URI_SAFE.set('&'); - EXPANDED_URI_SAFE.set('+'); - EXPANDED_URI_SAFE.set(','); - EXPANDED_URI_SAFE.set(':'); - EXPANDED_URI_SAFE.set(';'); - EXPANDED_URI_SAFE.set('@'); - EXPANDED_URI_SAFE.set('$'); - EXPANDED_URI_SAFE.set('!'); - EXPANDED_URI_SAFE.set(')'); - EXPANDED_URI_SAFE.set('('); - // experiments indicate: Firefox (1.0.6) never escapes '%' - EXPANDED_URI_SAFE.set('%'); - // experiments indicate: Firefox (1.0.6) does not escape '|' or ''' - EXPANDED_URI_SAFE.set('|'); - EXPANDED_URI_SAFE.set('\''); - } - - public static BitSet QUERY_SAFE = new BitSet(256); - static { - QUERY_SAFE.or(EXPANDED_URI_SAFE); - // Tests indicate Firefox (1.0.7-1) doesn't escape curlies in query str. - QUERY_SAFE.set('{'); - QUERY_SAFE.set('}'); - // nor any of these: [ ] ^ ? - QUERY_SAFE.set('['); - QUERY_SAFE.set(']'); - QUERY_SAFE.set('^'); - QUERY_SAFE.set('?'); - } - - /** - * Encodes a string into its URL safe form using the specified - * string charset. Unsafe characters are escaped. - * - * This method is analogous to superclass encode() methods, - * additionally offering the ability to specify a different - * 'safe' character set (such as EXPANDED_URI_SAFE). - * - * @param safe BitSet of characters that don't need to be encoded - * @param pString String to encode - * @param cs Name of character set to use - * @return Encoded version of pString. - * @throws UnsupportedEncodingException - */ - public String encode(BitSet safe, String pString, String cs) - throws UnsupportedEncodingException { - if (pString == null) { - return null; - } - return new String(encodeUrl(safe,pString.getBytes(cs)), Charsets.US_ASCII); - } -} diff --git a/commons/src/main/java/org/archive/net/UURI.java b/commons/src/main/java/org/archive/net/UURI.java index 3556760d..7995736c 100644 --- a/commons/src/main/java/org/archive/net/UURI.java +++ b/commons/src/main/java/org/archive/net/UURI.java @@ -18,18 +18,13 @@ */ package org.archive.net; -import java.io.File; import java.io.IOException; import java.io.ObjectInputStream; import java.io.ObjectOutputStream; -import java.io.Serializable; -import java.net.URI; -import java.net.URISyntaxException; import java.nio.ByteBuffer; import org.apache.commons.httpclient.URIException; -import org.archive.util.SURT; -import org.archive.util.TextUtils; +import org.archive.url.UsableURI; import com.esotericsoftware.kryo.CustomSerialization; import com.esotericsoftware.kryo.Kryo; @@ -37,423 +32,31 @@ import com.esotericsoftware.kryo.serialize.StringSerializer; /** - * Usable URI. - * - * This class wraps {@link org.apache.commons.httpclient.URI} adding caching - * and methods. It cannot be instantiated directly. Go via UURIFactory. - * - *

We used to use {@link java.net.URI} for parsing URIs but ran across - * quirky behaviors and bugs. {@link java.net.URI} is not subclassable -- - * its final -- and its unlikely that java.net.URI will change any time soon - * (See Gordon's considered petition here: - * java.net.URI - * should have loose/tolerant/compatibility option (or allow reuse)). - * - *

This class tries to cache calculated strings such as the extracted host - * and this class as a string rather than have the parent class rerun its - * calculation everytime. - * - * @author gojomo - * @author stack - * - * @see org.apache.commons.httpclient.URI + * Usable URI. The bulk of the functionality of this class has moved to + * {@link UsableURI} in the archive-commons project. This class adds Kryo + * serialization. */ -public class UURI extends LaxURI -implements CharSequence, Serializable, CustomSerialization { +public class UURI extends UsableURI implements CustomSerialization { - private static final long serialVersionUID = -1277570889914647093L; + private static final long serialVersionUID = -8946640480772772310L; - //private static Logger LOGGER = - // Logger.getLogger(UURI.class.getName()); - - /** - * Consider URIs too long for IE as illegal. - */ - public final static int MAX_URL_LENGTH = 2083; - - public static final String MASSAGEHOST_PATTERN = "^www\\d*\\."; - - /** - * Cache of the host name. - * - * Super class calculates on every call. Profiling shows us spend 30% of - * total elapsed time in URI class. - */ - private transient String cachedHost = null; - - /** - * Cache of this uuri escaped as a string. - * - * Super class calculates on every call. Profiling shows us spend 30% of - * total elapsed time in URI class. - */ - private transient String cachedEscapedURI = null; - - /** - * Cache of this uuri escaped as a string. - * - * Super class calculates on every call. Profiling shows us spend 30% of - * total elapsed time in URI class. - */ - private transient String cachedString = null; - - /** - * Cached authority minus userinfo. - */ - private transient String cachedAuthorityMinusUserinfo = null; - - /** - * Cache of this uuri in SURT format - */ - private transient String surtForm = null; - - // Technically, underscores are disallowed in the domainlabel - // portion of hostname according to rfc2396 but we'll be more - // loose and allow them. See: [ 1072035 ] [uuri] Underscore in - // host messes up port parsing. - static { - hostname.set('_'); + public UURI(String fixup, boolean b, String charset) throws URIException { + super(fixup, b, charset); } - - /** - * Shutdown access to default constructor. - */ - protected UURI() { - super(); - } - - /** - * @param uri String representation of an absolute URI. - * @param escaped If escaped. - * @param charset Charset to use. - * @throws org.apache.commons.httpclient.URIException - */ - protected UURI(String uri, boolean escaped, String charset) - throws URIException { - super(uri, escaped, charset); - normalize(); - } - - /** - * @param relative String representation of URI. - * @param base Parent UURI to use derelativizing. - * @throws org.apache.commons.httpclient.URIException - */ - protected UURI(UURI base, UURI relative) throws URIException { - super(base, relative); - normalize(); + public UURI(UURI base, UURI uuri) throws URIException { + super(base, uuri); } - /** - * @param uri String representation of a URI. - * @param escaped If escaped. - * @throws NullPointerException - * @throws URIException - */ - protected UURI(String uri, boolean escaped) throws URIException, NullPointerException { - super(uri,escaped); - normalize(); - } - - /** - * @param uri URI as string that is resolved relative to this UURI. - * @return UURI that uses this UURI as base. - * @throws URIException - */ - public UURI resolve(String uri) - throws URIException { - return resolve(uri, false, // assume not escaped - this.getProtocolCharset()); - } - - /** - * @param uri URI as string that is resolved relative to this UURI. - * @param e True if escaped. - * @return UURI that uses this UURI as base. - * @throws URIException - */ - public UURI resolve(String uri, boolean e) - throws URIException { - return resolve(uri, e, this.getProtocolCharset()); - } - - /** - * @param uri URI as string that is resolved relative to this UURI. - * @param e True if uri is escaped. - * @param charset Charset to use. - * @return UURI that uses this UURI as base. - * @throws URIException - */ - public UURI resolve(String uri, boolean e, String charset) - throws URIException { - return new UURI(this, new UURI(uri, e, charset)); - } - - /** - * Test an object if this UURI is equal to another. - * - * @param obj an object to compare - * @return true if two URI objects are equal - */ - public boolean equals(Object obj) { - - // normalize and test each components - if (obj == this) { - return true; - } - if (!(obj instanceof UURI)) { - return false; - } - UURI another = (UURI) obj; - // scheme - if (!equals(this._scheme, another._scheme)) { - return false; - } - // is_opaque_part or is_hier_part? and opaque - if (!equals(this._opaque, another._opaque)) { - return false; - } - // is_hier_part - // has_authority - if (!equals(this._authority, another._authority)) { - return false; - } - // path - if (!equals(this._path, another._path)) { - return false; - } - // has_query - if (!equals(this._query, another._query)) { - return false; - } - // UURIs do not have fragments - return true; - } - - /** - * Strips www variants from the host. - * - * Strips www[0-9]*\. from the host. If calling getHostBaseName becomes a - * performance issue we should consider adding the hostBasename member that - * is set on initialization. - * - * @return Host's basename. - * @throws URIException - */ - public String getHostBasename() throws URIException { - // caching eliminated because this is rarely used - // (only benefits legacy DomainScope, which should - // be retired). Saves 4-byte object pointer in UURI - // instances. - return (this.getReferencedHost() == null) - ? null - : TextUtils.replaceFirst(MASSAGEHOST_PATTERN, - this.getReferencedHost(), UURIFactory.EMPTY_STRING); - } - - /** - * Returns an alternate, functional String representation -- in this - * case, a String of the URI represented by this UURI instance. - * - * @return - */ - public synchronized String toCustomString() { - if (this.cachedString == null) { - this.cachedString = super.toString(); - coalesceUriStrings(); - } - return this.cachedString; - } - - /** - * Override to cache result - * - * TODO: eliminate, moving most callers to toCustomString, to avoid - * overloading/diluting toString() - * (see http://webteam.archive.org/confluence/display/Heritrix/Preserve+toString%28%29 ) - * @return String representation of this URI - */ - public String toString() { - return toCustomString(); - } - - public synchronized String getEscapedURI() { - if (this.cachedEscapedURI == null) { - this.cachedEscapedURI = super.getEscapedURI(); - coalesceUriStrings(); - } - return this.cachedEscapedURI; - } - - /** - * The two String fields cachedString and cachedEscapedURI are - * usually identical; if so, coalesce into a single instance. - */ - protected void coalesceUriStrings() { - if (this.cachedString != null && this.cachedEscapedURI != null - && this.cachedString.length() == this.cachedEscapedURI.length()) { - // lengths will only be identical if contents are identical - // (deescaping will always shrink length), so coalesce to - // use only single cached instance - this.cachedString = this.cachedEscapedURI; - } - } - - public synchronized String getHost() throws URIException { - if (this.cachedHost == null) { - // If this._host is null, 3.0 httpclient throws - // illegalargumentexception. Don't go there. - if (this._host != null) { - this.cachedHost = super.getHost(); - coalesceHostAuthorityStrings(); - } - } - return this.cachedHost; - } - - /** - * The two String fields cachedHost and cachedAuthorityMinusUserInfo are - * usually identical; if so, coalesce into a single instance. - */ - protected void coalesceHostAuthorityStrings() { - if (this.cachedAuthorityMinusUserinfo != null - && this.cachedHost != null - && this.cachedHost.length() == - this.cachedAuthorityMinusUserinfo.length()) { - // lengths can only be identical if contents - // are identical; use only one instance - this.cachedAuthorityMinusUserinfo = this.cachedHost; - } - } - - /** - * Return the referenced host in the UURI, if any, also extracting the - * host of a DNS-lookup URI where necessary. - * - * @return the target or topic host of the URI - * @throws URIException - */ - public String getReferencedHost() throws URIException { - String referencedHost = this.getHost(); - if(referencedHost==null && this.getScheme().equals("dns")) { - // extract target domain of DNS lookup - String possibleHost = this.getCurrentHierPath(); - if(possibleHost != null && possibleHost.matches("[-_\\w\\.:]+")) { - referencedHost = possibleHost; - } - } - return referencedHost; - } - - /** - * @return Return the 'SURT' format of this UURI - */ - public String getSurtForm() { - if (surtForm == null) { - surtForm = SURT.fromURI(this.toString()); - } - return surtForm; - } - - /** - * Return the authority minus userinfo (if any). - * - * If no userinfo present, just returns the authority. - * - * @return The authority stripped of any userinfo if present. - * @throws URIException - */ - public String getAuthorityMinusUserinfo() - throws URIException { - if (this.cachedAuthorityMinusUserinfo == null) { - String tmp = getAuthority(); - if (tmp != null && tmp.length() > 0) { - int index = tmp.indexOf('@'); - if (index >= 0 && index < tmp.length()) { - tmp = tmp.substring(index + 1); - } - } - this.cachedAuthorityMinusUserinfo = tmp; - coalesceHostAuthorityStrings(); - } - return this.cachedAuthorityMinusUserinfo; - } - - /* (non-Javadoc) - * @see java.lang.CharSequence#length() - */ - public int length() { - return getEscapedURI().length(); - } - - /* (non-Javadoc) - * @see java.lang.CharSequence#charAt(int) - */ - public char charAt(int index) { - return getEscapedURI().charAt(index); - } - - /* (non-Javadoc) - * @see java.lang.CharSequence#subSequence(int, int) - */ - public CharSequence subSequence(int start, int end) { - return getEscapedURI().subSequence(start,end); - } - - /* (non-Javadoc) - * @see java.lang.Comparable#compareTo(java.lang.Object) - */ - public int compareTo(Object arg0) { - return getEscapedURI().compareTo(arg0.toString()); - } - - - - /** - * Test if passed String has likely URI scheme prefix. - * @param possibleUrl URL string to examine. - * @return True if passed string looks like it could be an URL. - */ - public static boolean hasScheme(String possibleUrl) { - boolean result = false; - for (int i = 0; i < possibleUrl.length(); i++) { - char c = possibleUrl.charAt(i); - if (c == ':') { - if (i != 0) { - result = true; - } - break; - } - if (!scheme.get(c)) { - break; - } - } - return result; - } - - /** - * @param pathOrUri A file path or a URI. - * @return Path parsed from passed pathOrUri. - * @throws URISyntaxException - */ - public static String parseFilename(final String pathOrUri) - throws URISyntaxException { - String path = pathOrUri; - if (UURI.hasScheme(pathOrUri)) { - URI url = new URI(pathOrUri); - path = url.getPath(); - } - return (new File(path)).getName(); - } - + @Override public void writeObjectData(Kryo kryo, ByteBuffer buffer) { StringSerializer.put(buffer, toCustomString()); } - public void readObjectData (Kryo kryo, ByteBuffer buffer) { + @Override + public void readObjectData(Kryo kryo, ByteBuffer buffer) { try { - parseUriReference(StringSerializer.get(buffer),true); + parseUriReference(StringSerializer.get(buffer), true); } catch (URIException e) { // TODO Auto-generated catch block e.printStackTrace(); @@ -462,10 +65,10 @@ implements CharSequence, Serializable, CustomSerialization { private void writeObject(ObjectOutputStream stream) throws IOException { stream.writeUTF(toCustomString()); - } - + } + private void readObject(ObjectInputStream stream) throws IOException, - ClassNotFoundException { - parseUriReference(stream.readUTF(),true); + ClassNotFoundException { + parseUriReference(stream.readUTF(), true); } } diff --git a/commons/src/main/java/org/archive/net/UURIFactory.java b/commons/src/main/java/org/archive/net/UURIFactory.java index 75810e70..d44aa093 100644 --- a/commons/src/main/java/org/archive/net/UURIFactory.java +++ b/commons/src/main/java/org/archive/net/UURIFactory.java @@ -18,232 +18,26 @@ */ package org.archive.net; -import gnu.inet.encoding.IDNA; -import gnu.inet.encoding.IDNAException; -import it.unimi.dsi.mg4j.util.MutableString; - -import java.io.UnsupportedEncodingException; -import java.util.BitSet; -import java.util.logging.Level; -import java.util.logging.Logger; -import java.util.regex.Matcher; -import java.util.regex.Pattern; - -import org.apache.commons.httpclient.URI; import org.apache.commons.httpclient.URIException; -import org.archive.util.TextUtils; -/** - * Factory that returns UURIs. - * - * Does escaping and fixup on URIs massaging in accordance with RFC2396 and to - * match browser practice. For example, it removes any '..' if first thing in - * the path as per IE, converts backslashes preceding the query string to - * forward slashes, and discards any 'fragment'/anchor portion of the URI. This - * class will also fail URIs if they are longer than IE's allowed maximum - * length. - * - *

- * TODO: Test logging. - * - * @author stack - */ -public class UURIFactory extends URI { +public class UURIFactory extends org.archive.url.UsableURIFactory { - private static final long serialVersionUID = -6146295130382209042L; - - /** - * Logging instance. - */ - private static Logger logger = - Logger.getLogger(UURIFactory.class.getName()); + private static final long serialVersionUID = -2615498094356088387L; /** * The single instance of this factory. */ private static final UURIFactory factory = new UURIFactory(); - - /** - * RFC 2396-inspired regex. - * - * From the RFC Appendix B: - * 

-     * URI Generic Syntax                August 1998
-     *
-     * B. Parsing a URI Reference with a Regular Expression
-     *
-     * As described in Section 4.3, the generic URI syntax is not sufficient
-     * to disambiguate the components of some forms of URI.  Since the
-     * "greedy algorithm" described in that section is identical to the
-     * disambiguation method used by POSIX regular expressions, it is
-     * natural and commonplace to use a regular expression for parsing the
-     * potential four components and fragment identifier of a URI reference.
-     *
-     * The following line is the regular expression for breaking-down a URI
-     * reference into its components.
-     *
-     * ^(([^:/?#]+):)?(//([^/?#]*))?([^?#]*)(\?([^#]*))?(#(.*))?
-     * 12            3  4          5       6  7        8 9
-     *
-     * The numbers in the second line above are only to assist readability;
-     * they indicate the reference points for each subexpression (i.e., each
-     * paired parenthesis).  We refer to the value matched for subexpression
-     *  as $.  For example, matching the above expression to
-     *
-     * http://www.ics.uci.edu/pub/ietf/uri/#Related
-     *
-     * results in the following subexpression matches:
-     *
-     * $1 = http:
-     * $2 = http
-     * $3 = //www.ics.uci.edu
-     * $4 = www.ics.uci.edu
-     * $5 = /pub/ietf/uri/
-     * $6 = 
-     * $7 = 
-     * $8 = #Related
-     * $9 = Related
-     *
-     * where  indicates that the component is not present, as is
-     * the case for the query component in the above example.  Therefore, we
-     * can determine the value of the four components and fragment as
-     *
-     * scheme    = $2
-     * authority = $4
-     * path      = $5
-     * query     = $7
-     * fragment  = $9
-     *
- * - * -- - *

Below differs from the rfc regex in that... - * (1) it has java escaping of regex characters - * (2) we allow a URI made of a fragment only (Added extra - * group so indexing is off by one after scheme). - * (3) scheme is limited to legal scheme characters - */ - final public static Pattern RFC2396REGEX = Pattern.compile( - "^(([a-zA-Z][a-zA-Z0-9\\+\\-\\.]*):)?((//([^/?#]*))?([^?#]*)(\\?([^#]*))?)?(#(.*))?"); - // 12 34 5 6 7 8 9 A - // 2 1 54 6 87 3 A9 // 1: scheme - // 2: scheme: - // 3: //authority/path - // 4: //authority - // 5: authority - // 6: path - // 7: ?query - // 8: query - // 9: #fragment - // A: fragment - public static final String SLASHDOTDOTSLASH = "^(/\\.\\./)+"; - public static final String SLASH = "/"; - public static final String HTTP = "http"; - public static final String HTTP_PORT = ":80"; - public static final String HTTPS = "https"; - public static final String HTTPS_PORT = ":443"; - public static final String DOT = "."; - public static final String EMPTY_STRING = ""; - public static final String NBSP = "\u00A0"; - public static final String SPACE = " "; - public static final String ESCAPED_SPACE = "%20"; - public static final String TRAILING_ESCAPED_SPACE = "^(.*)(%20)+$"; - public static final String PIPE = "|"; - public static final String PIPE_PATTERN = "\\|"; - public static final String ESCAPED_PIPE = "%7C"; - public static final String CIRCUMFLEX = "^"; - public static final String CIRCUMFLEX_PATTERN = "\\^"; - public static final String ESCAPED_CIRCUMFLEX = "%5E"; - public static final String QUOT = "\""; - public static final String ESCAPED_QUOT = "%22"; - public static final String SQUOT = "'"; - public static final String ESCAPED_SQUOT = "%27"; - public static final String APOSTROPH = "`"; - public static final String ESCAPED_APOSTROPH = "%60"; - public static final String LSQRBRACKET = "["; - public static final String LSQRBRACKET_PATTERN = "\\["; - public static final String ESCAPED_LSQRBRACKET = "%5B"; - public static final String RSQRBRACKET = "]"; - public static final String RSQRBRACKET_PATTERN = "\\]"; - public static final String ESCAPED_RSQRBRACKET = "%5D"; - public static final String LCURBRACKET = "{"; - public static final String LCURBRACKET_PATTERN = "\\{"; - public static final String ESCAPED_LCURBRACKET = "%7B"; - public static final String RCURBRACKET = "}"; - public static final String RCURBRACKET_PATTERN = "\\}"; - public static final String ESCAPED_RCURBRACKET = "%7D"; - public static final String BACKSLASH = "\\"; - public static final String ESCAPED_BACKSLASH = "%5C"; - public static final String STRAY_SPACING = "[\n\r\t]+"; - public static final String IMPROPERESC_REPLACE = "%25$1"; - public static final String IMPROPERESC = - "%((?:[^\\p{XDigit}])|(?:.[^\\p{XDigit}])|(?:\\z))"; - public static final String COMMERCIAL_AT = "@"; - public static final char PERCENT_SIGN = '%'; - public static final char COLON = ':'; - - /** - * First percent sign in string followed by two hex chars. - */ - public static final String URI_HEX_ENCODING = - "^[^%]*%[\\p{XDigit}][\\p{XDigit}].*"; - - /** - * Authority port number regex. - */ - final static Pattern PORTREGEX = Pattern.compile("(.*:)([0-9]+)$"); - - /** - * Characters we'll accept in the domain label part of a URI - * authority: ASCII letters-digits-hyphen (LDH) plus underscore, - * with single intervening '.' characters. - * - * (We accept '_' because DNS servers have tolerated for many - * years counter to spec; we also accept dash patterns and ACE - * prefixes that will be rejected by IDN-punycoding attempt.) - */ - final static String ACCEPTABLE_ASCII_DOMAIN = - "^(?:[a-zA-Z0-9_-]++(?:\\.)?)++$"; - - /** - * Pattern that looks for case of three or more slashes after the - * scheme. If found, we replace them with two only as mozilla does. - */ - final static Pattern HTTP_SCHEME_SLASHES = - Pattern.compile("^(https?://)/+(.*)"); - - /** - * Pattern that looks for case of two or more slashes in a path. - */ - final static Pattern MULTIPLE_SLASHES = Pattern.compile("//+"); - - /** - * Protected constructor. - */ - private UURIFactory() { - super(); - } - /** * @param uri URI as string. * @return An instance of UURI * @throws URIException */ public static UURI getInstance(String uri) throws URIException { - return UURIFactory.factory.create(uri); + return (UURI) UURIFactory.factory.create(uri); } - - /** - * @param uri URI as string. - * @param charset Character encoding of the passed uri string. - * @return An instance of UURI - * @throws URIException - */ - public static UURI getInstance(String uri, String charset) - throws URIException { - return UURIFactory.factory.create(uri, charset); - } - + /** * @param base Base uri to use resolving passed relative uri. * @param relative URI as string. @@ -251,552 +45,13 @@ public class UURIFactory extends URI { * @throws URIException */ public static UURI getInstance(UURI base, String relative) - throws URIException { -// return base.resolve(relative); - return UURIFactory.factory.create(base, relative); + throws URIException { + return (UURI) UURIFactory.factory.create(base, relative); } - /** - * @param uri URI as string. - * @return Instance of UURI. - * @throws URIException - */ - private UURI create(String uri) throws URIException { - return create(uri, UURI.getDefaultProtocolCharset()); - } - - /** - * @param uri URI as string. - * @param charset Original encoding of the string. - * @return Instance of UURI. - * @throws URIException - */ - private UURI create(String uri, String charset) throws URIException { - UURI uuri = new UURI(fixup(uri, null, charset), true, charset); - if (logger.isLoggable(Level.FINE)) { - logger.fine("URI " + uri + - " PRODUCT " + uuri.toString() + - " CHARSET " + charset); - } - return validityCheck(uuri); - } - - /** - * @param base UURI to use as a base resolving relative. - * @param relative Relative URI. - * @return Instance of UURI. - * @throws URIException - */ - private UURI create(UURI base, String relative) throws URIException { - UURI uuri = new UURI(base, new UURI(fixup(relative, base, base.getProtocolCharset()), - true, base.getProtocolCharset())); - if (logger.isLoggable(Level.FINE)) { - logger.fine(" URI " + relative + - " PRODUCT " + uuri.toString() + - " CHARSET " + base.getProtocolCharset() + - " BASE " + base); - } - return validityCheck(uuri); + protected UURI newUURI(String charset, boolean escaped, String fixedUpUri) + throws URIException { + return new UURI(fixedUpUri, escaped, charset); } - /** - * Check the generated UURI. - * - * At the least look at length of uuri string. We were seeing case - * where before escaping, string was < MAX_URL_LENGTH but after was - * >. Letting out a too-big message was causing us troubles later - * down the processing chain. - * @param uuri Created uuri to check. - * @return The passed uuri so can easily inline this check. - * @throws URIException - */ - protected UURI validityCheck(UURI uuri) throws URIException { - if (uuri.getRawURI().length > UURI.MAX_URL_LENGTH) { - throw new URIException("Created (escaped) uuri > " + - UURI.MAX_URL_LENGTH +": "+uuri.toString()); - } - return uuri; - } - - /** - * Do heritrix fix-up on passed uri string. - * - * Does heritrix escaping; usually escaping done to make our behavior align - * with IEs. This method codifies our experience pulling URIs from the - * wilds. Its does all the escaping we want; its output can always be - * assumed to be 'escaped' (though perhaps to a laxer standard than the - * vanilla HttpClient URI class or official specs might suggest). - * - * @param uri URI as string. - * @param base May be null. - * @param e True if the uri is already escaped. - * @return A fixed up URI string. - * @throws URIException - */ - private String fixup(String uri, final URI base, final String charset) - throws URIException { - if (uri == null) { - throw new NullPointerException(); - } else if (uri.length() == 0 && base == null) { - throw new URIException("URI length is zero (and not relative)."); - } - - if (uri.length() > UURI.MAX_URL_LENGTH) { - // We check length here and again later after all convertions. - throw new URIException("URI length > " + UURI.MAX_URL_LENGTH + - ": " + uri); - } - - // Replace nbsp with normal spaces (so that they get stripped if at - // ends, or encoded if in middle) - if (uri.indexOf(NBSP) >= 0) { - uri = TextUtils.replaceAll(NBSP, uri, SPACE); - } - - // Get rid of any trailing spaces or new-lines. - uri = uri.trim(); - - // IE converts backslashes preceding the query string to slashes, rather - // than to %5C. Since URIs that have backslashes usually work only with - // IE, we will convert backslashes to slashes as well. - int nextBackslash = uri.indexOf(BACKSLASH); - if (nextBackslash >= 0) { - int queryStart = uri.indexOf('?'); - StringBuilder tmp = new StringBuilder(uri); - while (nextBackslash >= 0 - && (queryStart < 0 || nextBackslash < queryStart)) { - tmp.setCharAt(nextBackslash, '/'); - nextBackslash = uri.indexOf(BACKSLASH, nextBackslash + 1); - } - uri = tmp.toString(); - } - - // Remove stray TAB/CR/LF - uri = TextUtils.replaceAll(STRAY_SPACING, uri, EMPTY_STRING); - - // Test for the case of more than two slashes after the http(s) scheme. - // Replace with two slashes as mozilla does if found. - // See [ 788219 ] URI Syntax Errors stop page parsing. -// Matcher matcher = HTTP_SCHEME_SLASHES.matcher(uri); - Matcher matcher = TextUtils.getMatcher(HTTP_SCHEME_SLASHES.pattern(), uri); - if (matcher.matches()) { - uri = matcher.group(1) + matcher.group(2); - } - TextUtils.recycleMatcher(matcher); - - // now, minimally escape any whitespace - uri = escapeWhitespace(uri); - - // For further processing, get uri elements. See the RFC2396REGEX - // comment above for explanation of group indices used in the below. -// matcher = RFC2396REGEX.matcher(uri); - matcher = TextUtils.getMatcher(RFC2396REGEX.pattern(), uri); - if (!matcher.matches()) { - throw new URIException("Failed parse of " + uri); - } - String uriScheme = checkUriElementAndLowerCase(matcher.group(2)); - String uriSchemeSpecificPart = checkUriElement(matcher.group(3)); - String uriAuthority = checkUriElement(matcher.group(5)); - String uriPath = checkUriElement(matcher.group(6)); - String uriQuery = checkUriElement(matcher.group(8)); - // UNUSED String uriFragment = checkUriElement(matcher.group(10)); - TextUtils.recycleMatcher(matcher); matcher = null; - - // Test if relative URI. If so, need a base to resolve against. - if (uriScheme == null || uriScheme.length() <= 0) { - if (base == null) { - throw new URIException("Relative URI but no base: " + uri); - } - } else { - checkHttpSchemeSpecificPartSlashPrefix(base, uriScheme, - uriSchemeSpecificPart); - } - - // fixup authority portion: lowercase/IDN-punycode any domain; - // remove stray trailing spaces - uriAuthority = fixupAuthority(uriAuthority,charset); - - // Do some checks if absolute path. - if (uriSchemeSpecificPart != null && - uriSchemeSpecificPart.startsWith(SLASH)) { - if (uriPath != null) { - // Eliminate '..' if its first thing in the path. IE does this. - uriPath = TextUtils.replaceFirst(SLASHDOTDOTSLASH, uriPath, - SLASH); - } - // Ensure root URLs end with '/': browsers always send "/" - // on the request-line, so we should consider "http://host" - // to be "http://host/". - if (uriPath == null || EMPTY_STRING.equals(uriPath)) { - uriPath = SLASH; - } - } - - if (uriAuthority != null) { - if (uriScheme != null && uriScheme.length() > 0 && - uriScheme.equals(HTTP)) { - uriAuthority = checkPort(uriAuthority); - uriAuthority = stripTail(uriAuthority, HTTP_PORT); - } else if (uriScheme != null && uriScheme.length() > 0 && - uriScheme.equals(HTTPS)) { - uriAuthority = checkPort(uriAuthority); - uriAuthority = stripTail(uriAuthority, HTTPS_PORT); - } - // Strip any prefix dot or tail dots from the authority. - uriAuthority = stripTail(uriAuthority, DOT); - uriAuthority = stripPrefix(uriAuthority, DOT); - } else { - // no authority; may be relative. consider stripping scheme - // to work-around org.apache.commons.httpclient.URI bug - // ( http://issues.apache.org/jira/browse/HTTPCLIENT-587 ) - if (uriScheme != null && base != null - && uriScheme.equals(base.getScheme())) { - // uriScheme redundant and will only confound httpclient.URI - uriScheme = null; - } - } - - // Ensure minimal escaping. Use of 'lax' URI and URLCodec - // means minimal escaping isn't necessarily complete/consistent. - // There is a chance such lax encoding will throw exceptions - // later at inconvenient times. - // - // One reason for these bad escapings -- though not the only -- - // is that the page is using an encoding other than the ASCII or the - // UTF-8 that is our default URI encoding. In this case the parent - // class is burping on the passed URL encoding. If the page encoding - // was passed into this factory, the encoding seems to be parsed - // correctly (See the testEscapedEncoding unit test). - // - // This fixup may cause us to miss content. There is the charset case - // noted above. TODO: Look out for cases where we fail other than for - // the above given reason which will be fixed when we address - // '[ 913687 ] Make extractors interrogate for charset'. - - uriPath = ensureMinimalEscaping(uriPath, charset); - uriQuery = ensureMinimalEscaping(uriQuery, charset, - LaxURLCodec.QUERY_SAFE); - - // Preallocate. The '1's and '2's in below are space for ':', - // '//', etc. URI characters. - MutableString s = new MutableString( - ((uriScheme != null)? uriScheme.length(): 0) - + 1 // ';' - + ((uriAuthority != null)? uriAuthority.length(): 0) - + 2 // '//' - + ((uriPath != null)? uriPath.length(): 0) - + 1 // '?' - + ((uriQuery != null)? uriQuery.length(): 0)); - appendNonNull(s, uriScheme, ":", true); - appendNonNull(s, uriAuthority, "//", false); - appendNonNull(s, uriPath, "", false); - appendNonNull(s, uriQuery, "?", false); - return s.toString(); - } - - /** - * If http(s) scheme, check scheme specific part begins '//'. - * @throws URIException - * @see http://www.faqs.org/rfcs/rfc1738.html Section 3.1. Common Internet - * Scheme Syntax - */ - protected void checkHttpSchemeSpecificPartSlashPrefix(final URI base, - final String scheme, final String schemeSpecificPart) - throws URIException { - if (scheme == null || scheme.length() <= 0) { - return; - } - if (!scheme.equals("http") && !scheme.equals("https")) { - return; - } - if ( schemeSpecificPart == null - || !schemeSpecificPart.startsWith("//")) { - // only acceptable if schemes match - if (base == null || !scheme.equals(base.getScheme())) { - throw new URIException( - "relative URI with scheme only allowed for " + - "scheme matching base"); - } - return; - } - if (schemeSpecificPart.length() <= 2) { - throw new URIException("http scheme specific part is " + - "too short: " + schemeSpecificPart); - } - } - - /** - * Fixup 'authority' portion of URI, by removing any stray - * encoded spaces, lowercasing any domain names, and applying - * IDN-punycoding to Unicode domains. - * - * @param uriAuthority the authority string to fix - * @return fixed version - * @throws URIException - */ - private String fixupAuthority(String uriAuthority, String charset) throws URIException { - // Lowercase the host part of the uriAuthority; don't destroy any - // userinfo capitalizations. Make sure no illegal characters in - // domainlabel substring of the uri authority. - if (uriAuthority != null) { - // Get rid of any trailing escaped spaces: - // http://www.archive.org%20. Rare but happens. - // TODO: reevaluate: do IE or firefox do such mid-URI space-removal? - // if not, we shouldn't either. - while(uriAuthority.endsWith(ESCAPED_SPACE)) { - uriAuthority = uriAuthority.substring(0,uriAuthority.length()-3); - } - - // lowercase & IDN-punycode only the domain portion - int atIndex = uriAuthority.indexOf(COMMERCIAL_AT); - int portColonIndex = uriAuthority.indexOf(COLON,(atIndex<0)?0:atIndex); - if(atIndex<0 && portColonIndex<0) { - // most common case: neither userinfo nor port - return fixupDomainlabel(uriAuthority); - } else if (atIndex<0 && portColonIndex>-1) { - // next most common: port but no userinfo - String domain = fixupDomainlabel(uriAuthority.substring(0,portColonIndex)); - String port = uriAuthority.substring(portColonIndex); - return domain + port; - } else if (atIndex>-1 && portColonIndex<0) { - // uncommon: userinfo, no port - String userinfo = ensureMinimalEscaping(uriAuthority.substring(0,atIndex+1),charset); - String domain = fixupDomainlabel(uriAuthority.substring(atIndex+1)); - return userinfo + domain; - } else { - // uncommon: userinfo, port - String userinfo = ensureMinimalEscaping(uriAuthority.substring(0,atIndex+1),charset); - String domain = fixupDomainlabel(uriAuthority.substring(atIndex+1,portColonIndex)); - String port = uriAuthority.substring(portColonIndex); - return userinfo + domain + port; - } - } - return uriAuthority; - } - - /** - * Fixup the domain label part of the authority. - * - * We're more lax than the spec. in that we allow underscores. - * - * @param label Domain label to fix. - * @return Return fixed domain label. - * @throws URIException - */ - private String fixupDomainlabel(String label) - throws URIException { - - // apply IDN-punycoding, as necessary - try { - // TODO: optimize: only apply when necessary, or - // keep cache of recent encodings - label = IDNA.toASCII(label); - } catch (IDNAException e) { - if(TextUtils.matches(ACCEPTABLE_ASCII_DOMAIN,label)) { - // domain name has ACE prefix, leading/trailing dash, or - // underscore -- but is still a name we wish to tolerate; - // simply continue - } else { - // problematic domain: neither ASCII acceptable characters - // nor IDN-punycodable, so throw exception - // TODO: change to HeritrixURIException so distinguishable - // from URIExceptions in library code - URIException ue = new URIException(e+" "+label); - ue.initCause(e); - throw ue; - } - } - label = label.toLowerCase(); - return label; - } - - /** - * Ensure that there all characters needing escaping - * in the passed-in String are escaped. Stray '%' characters - * are *not* escaped, as per browser behavior. - * - * @param u String to escape - * @param charset - * @return string with any necessary escaping applied - */ - private String ensureMinimalEscaping(String u, final String charset) { - return ensureMinimalEscaping(u, charset, LaxURLCodec.EXPANDED_URI_SAFE); - } - - /** - * Ensure that there all characters needing escaping - * in the passed-in String are escaped. Stray '%' characters - * are *not* escaped, as per browser behavior. - * - * @param u String to escape - * @param charset - * @param bitset - * @return string with any necessary escaping applied - */ - private String ensureMinimalEscaping(String u, final String charset, - final BitSet bitset) { - if (u == null) { - return null; - } - for (int i = 0; i < u.length(); i++) { - char c = u.charAt(i); - if (!bitset.get(c)) { - try { - u = LaxURLCodec.DEFAULT.encode(bitset, u, charset); - } catch (UnsupportedEncodingException e) { - e.printStackTrace(); - } - break; - } - } - return u; - } - - /** - * Escape any whitespace found. - * - * The parent class takes care of the bulk of escaping. But if any - * instance of escaping is found in the URI, then we ask for parent - * to do NO escaping. Here we escape any whitespace found irrespective - * of whether the uri has already been escaped. We do this for - * case where uri has been judged already-escaped only, its been - * incompletly done and whitespace remains. Spaces, etc., in the URI are - * a real pain. Their presence will break log file and ARC parsing. - * @param uri URI string to check. - * @return uri with spaces escaped if any found. - */ - protected String escapeWhitespace(String uri) { - // Just write a new string anyways. The perl '\s' is not - // as inclusive as the Character.isWhitespace so there are - // whitespace characters we could miss. So, rather than - // write some awkward regex, just go through the string - // a character at a time. Only create buffer first time - // we find a space. - MutableString buffer = null; - for (int i = 0; i < uri.length(); i++) { - char c = uri.charAt(i); - if (Character.isWhitespace(c)) { - if (buffer == null) { - buffer = new MutableString(uri.length() + - 2 /*If space, two extra characters (at least)*/); - buffer.append(uri.substring(0, i)); - } - buffer.append("%"); - String hexStr = Integer.toHexString(c); - if ((hexStr.length() % 2) > 0) { - buffer.append("0"); - } - buffer.append(hexStr); - - } else { - if (buffer != null) { - buffer.append(c); - } - } - } - return (buffer != null)? buffer.toString(): uri; - } - - /** - * Check port on passed http authority. Make sure the size is not larger - * than allowed: See the 'port' definition on this - * page, http://www.kerio.com/manual/wrp/en/418.htm. - * Also, we've seen port numbers of '0080' whose leading zeros confuse - * the parent class. Strip the leading zeros. - * - * @param uriAuthority - * @return Null or an amended port number. - * @throws URIException - */ - private String checkPort(String uriAuthority) - throws URIException { -// Matcher m = PORTREGEX.matcher(uriAuthority); - Matcher m = TextUtils.getMatcher(PORTREGEX.pattern(), uriAuthority); - if (m.matches()) { - String no = m.group(2); - if (no != null && no.length() > 0) { - // First check if the port has leading zeros - // as in '0080'. Strip them if it has and - // then reconstitute the uriAuthority. Be careful - // of cases where port is '0' or '000'. - while (no.charAt(0) == '0' && no.length() > 1) { - no = no.substring(1); - } - uriAuthority = m.group(1) + no; - // Now makesure the number is legit. - int portNo = 0; - try { - portNo = Integer.parseInt(no); - } catch (NumberFormatException nfe) { - // just catch and leave portNo at illegal 0 - } - if (portNo <= 0 || portNo > 65535) { - throw new URIException("Port out of bounds: " + - uriAuthority); - } - } - } - TextUtils.recycleMatcher(m); - return uriAuthority; - } - - /** - * @param b Buffer to append to. - * @param str String to append if not null. - * @param substr Suffix or prefix to use if str is not null. - * @param suffix True if substr is a suffix. - */ - private void appendNonNull(MutableString b, String str, String substr, - boolean suffix) { - if (str != null && str.length() > 0) { - if (!suffix) { - b.append(substr); - } - b.append(str); - if (suffix) { - b.append(substr); - } - } - } - - /** - * @param str String to work on. - * @param prefix Prefix to strip if present. - * @return str w/o prefix. - */ - private String stripPrefix(String str, String prefix) { - return str.startsWith(prefix)? - str.substring(prefix.length(), str.length()): - str; - } - - /** - * @param str String to work on. - * @param tail Tail to strip if present. - * @return str w/o tail. - */ - private static String stripTail(String str, String tail) { - return str.endsWith(tail)? - str.substring(0, str.length() - tail.length()): - str; - } - - /** - * @param element to examine. - * @return Null if passed null or an empty string otherwise - * element. - */ - private String checkUriElement(String element) { - return (element == null || element.length() <= 0)? null: element; - } - - /** - * @param element to examine and lowercase if non-null. - * @return Null if passed null or an empty string otherwise - * element lowercased. - */ - private String checkUriElementAndLowerCase(String element) { - String tmp = checkUriElement(element); - return (tmp != null)? tmp.toLowerCase(): tmp; - } } diff --git a/commons/src/main/java/org/archive/util/ArchiveUtils.java b/commons/src/main/java/org/archive/util/ArchiveUtils.java deleted file mode 100644 index fe2da96f..00000000 --- a/commons/src/main/java/org/archive/util/ArchiveUtils.java +++ /dev/null @@ -1,1008 +0,0 @@ -/* - * This file is part of the Heritrix web crawler (crawler.archive.org). - * - * Licensed to the Internet Archive (IA) by one or more individual - * contributors. - * - * The IA licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.archive.util; - -import java.io.BufferedInputStream; -import java.io.BufferedReader; -import java.io.ByteArrayOutputStream; -import java.io.Closeable; -import java.io.EOFException; -import java.io.File; -import java.io.FileInputStream; -import java.io.IOException; -import java.io.InputStream; -import java.io.InputStreamReader; -import java.io.PrintWriter; -import java.io.StringWriter; -import java.net.URL; -import java.net.URLConnection; -import java.text.NumberFormat; -import java.text.ParseException; -import java.text.SimpleDateFormat; -import java.util.Date; -import java.util.HashSet; -import java.util.Locale; -import java.util.Map; -import java.util.Set; -import java.util.TimeZone; -import java.util.logging.Level; -import java.util.logging.Logger; -import java.util.zip.GZIPInputStream; -import java.util.zip.GZIPOutputStream; - -import org.apache.commons.io.IOUtils; -import org.archive.io.GzipHeader; -import org.archive.io.NoGzipMagicException; - - -/** - * Miscellaneous useful methods. - * - * @author gojomo & others - */ -public class ArchiveUtils { - private static final Logger LOGGER = Logger.getLogger(ArchiveUtils.class.getName()); - - - final public static String VERSION = loadVersion(); - - /** - * Arc-style date stamp in the format yyyyMMddHHmm and UTC time zone. - */ - private static final ThreadLocal - TIMESTAMP12 = threadLocalDateFormat("yyyyMMddHHmm");; - - /** - * Arc-style date stamp in the format yyyyMMddHHmmss and UTC time zone. - */ - private static final ThreadLocal - TIMESTAMP14 = threadLocalDateFormat("yyyyMMddHHmmss"); - /** - * Arc-style date stamp in the format yyyyMMddHHmmssSSS and UTC time zone. - */ - private static final ThreadLocal - TIMESTAMP17 = threadLocalDateFormat("yyyyMMddHHmmssSSS"); - - /** - * Log-style date stamp in the format yyyy-MM-dd'T'HH:mm:ss.SSS'Z' - * UTC time zone is assumed. - */ - private static final ThreadLocal - TIMESTAMP17ISO8601Z = threadLocalDateFormat("yyyy-MM-dd'T'HH:mm:ss.SSS'Z'"); - - /** - * Log-style date stamp in the format yyyy-MM-dd'T'HH:mm:ss'Z' - * UTC time zone is assumed. - */ - private static final ThreadLocal - TIMESTAMP14ISO8601Z = threadLocalDateFormat("yyyy-MM-dd'T'HH:mm:ss'Z'"); - - /** - * Default character to use padding strings. - */ - private static final char DEFAULT_PAD_CHAR = ' '; - - /** milliseconds in an hour */ - private static final int HOUR_IN_MS = 60 * 60 * 1000; - /** milliseconds in a day */ - private static final int DAY_IN_MS = 24 * HOUR_IN_MS; - - private static ThreadLocal threadLocalDateFormat(final String pattern) { - ThreadLocal tl = new ThreadLocal() { - protected SimpleDateFormat initialValue() { - SimpleDateFormat df = new SimpleDateFormat(pattern); - df.setTimeZone(TimeZone.getTimeZone("GMT")); - return df; - } - }; - return tl; - } - - public static int MAX_INT_CHAR_WIDTH = - Integer.toString(Integer.MAX_VALUE).length(); - - /** - * Utility function for creating arc-style date stamps - * in the format yyyMMddHHmmssSSS. - * Date stamps are in the UTC time zone - * @return the date stamp - */ - public static String get17DigitDate(){ - return TIMESTAMP17.get().format(new Date()); - } - - protected static long LAST_UNIQUE_NOW17 = 0; - protected static String LAST_TIMESTAMP17 = ""; - /** - * Utility function for creating UNIQUE-from-this-class - * arc-style date stamps in the format yyyMMddHHmmssSSS. - * Rather than giving a duplicate datestamp on a - * subsequent call, will increment the milliseconds until a - * unique value is returned. - * - * Date stamps are in the UTC time zone - * @return the date stamp - */ - public synchronized static String getUnique17DigitDate(){ - long effectiveNow = System.currentTimeMillis(); - effectiveNow = Math.max(effectiveNow, LAST_UNIQUE_NOW17+1); - String candidate = get17DigitDate(effectiveNow); - while(candidate.equals(LAST_TIMESTAMP17)) { - effectiveNow++; - candidate = get17DigitDate(effectiveNow); - } - LAST_UNIQUE_NOW17 = effectiveNow; - LAST_TIMESTAMP17 = candidate; - return candidate; - } - - /** - * Utility function for creating arc-style date stamps - * in the format yyyyMMddHHmmss. - * Date stamps are in the UTC time zone - * @return the date stamp - */ - public static String get14DigitDate(){ - return TIMESTAMP14.get().format(new Date()); - } - - protected static long LAST_UNIQUE_NOW14 = 0; - protected static String LAST_TIMESTAMP14 = ""; - /** - * Utility function for creating UNIQUE-from-this-class - * arc-style date stamps in the format yyyMMddHHmmss. - * Rather than giving a duplicate datestamp on a - * subsequent call, will increment the seconds until a - * unique value is returned. - * - * Date stamps are in the UTC time zone - * @return the date stamp - */ - public synchronized static String getUnique14DigitDate(){ - long effectiveNow = System.currentTimeMillis(); - effectiveNow = Math.max(effectiveNow, LAST_UNIQUE_NOW14+1); - String candidate = get14DigitDate(effectiveNow); - while(candidate.equals(LAST_TIMESTAMP14)) { - effectiveNow += 1000; - candidate = get14DigitDate(effectiveNow); - } - LAST_UNIQUE_NOW14 = effectiveNow; - LAST_TIMESTAMP14 = candidate; - return candidate; - } - - /** - * Utility function for creating arc-style date stamps - * in the format yyyyMMddHHmm. - * Date stamps are in the UTC time zone - * @return the date stamp - */ - public static String get12DigitDate(){ - return TIMESTAMP12.get().format(new Date()); - } - - /** - * Utility function for creating log timestamps, in - * W3C/ISO8601 format, assuming UTC. Use current time. - * - * Format is yyyy-MM-dd'T'HH:mm:ss.SSS'Z' - * - * @return the date stamp - */ - public static String getLog17Date(){ - return TIMESTAMP17ISO8601Z.get().format(new Date()); - } - - /** - * Utility function for creating log timestamps, in - * W3C/ISO8601 format, assuming UTC. - * - * Format is yyyy-MM-dd'T'HH:mm:ss.SSS'Z' - * @param date Date to format. - * - * @return the date stamp - */ - public static String getLog17Date(long date){ - return TIMESTAMP17ISO8601Z.get().format(new Date(date)); - } - - /** - * Utility function for creating log timestamps, in - * W3C/ISO8601 format, assuming UTC. Use current time. - * - * Format is yyyy-MM-dd'T'HH:mm:ss'Z' - * - * @return the date stamp - */ - public static String getLog14Date(){ - return TIMESTAMP14ISO8601Z.get().format(new Date()); - } - - /** - * Utility function for creating log timestamps, in - * W3C/ISO8601 format, assuming UTC. - * - * Format is yyyy-MM-dd'T'HH:mm:ss'Z' - * @param date long timestamp to format. - * - * @return the date stamp - */ - public static String getLog14Date(long date){ - return TIMESTAMP14ISO8601Z.get().format(new Date(date)); - } - - /** - * Utility function for creating log timestamps, in - * W3C/ISO8601 format, assuming UTC. - * - * Format is yyyy-MM-dd'T'HH:mm:ss'Z' - * @param date Date to format. - * - * @return the date stamp - */ - public static String getLog14Date(Date date){ - return TIMESTAMP14ISO8601Z.get().format(date); - } - - /** - * Utility function for creating arc-style date stamps - * in the format yyyyMMddHHmmssSSS. - * Date stamps are in the UTC time zone - * - * @param date milliseconds since epoc - * @return the date stamp - */ - public static String get17DigitDate(long date){ - return TIMESTAMP17.get().format(new Date(date)); - } - - public static String get17DigitDate(Date date){ - return TIMESTAMP17.get().format(date); - } - - /** - * Utility function for creating arc-style date stamps - * in the format yyyyMMddHHmmss. - * Date stamps are in the UTC time zone - * - * @param date milliseconds since epoc - * @return the date stamp - */ - public static String get14DigitDate(long date){ - return TIMESTAMP14.get().format(new Date(date)); - } - - public static String get14DigitDate(Date d) { - return TIMESTAMP14.get().format(d); - } - - /** - * Utility function for creating arc-style date stamps - * in the format yyyyMMddHHmm. - * Date stamps are in the UTC time zone - * - * @param date milliseconds since epoc - * @return the date stamp - */ - public static String get12DigitDate(long date){ - return TIMESTAMP12.get().format(new Date(date)); - } - - public static String get12DigitDate(Date d) { - return TIMESTAMP12.get().format(d); - } - - /** - * Parses an ARC-style date. If passed String is < 12 characters in length, - * we pad. At a minimum, String should contain a year (>=4 characters). - * Parse will also fail if day or month are incompletely specified. Depends - * on the above getXXDigitDate methods. - * @param A 4-17 digit date in ARC style (yyyy to - * yyyyMMddHHmmssSSS) formatting. - * @return A Date object representing the passed String. - * @throws ParseException - */ - public static Date getDate(String d) throws ParseException { - Date date = null; - if (d == null) { - throw new IllegalArgumentException("Passed date is null"); - } - switch (d.length()) { - case 14: - date = ArchiveUtils.parse14DigitDate(d); - break; - - case 17: - date = ArchiveUtils.parse17DigitDate(d); - break; - - case 12: - date = ArchiveUtils.parse12DigitDate(d); - break; - - case 0: - case 1: - case 2: - case 3: - throw new ParseException("Date string must at least contain a" + - "year: " + d, d.length()); - - default: - if (!(d.startsWith("19") || d.startsWith("20"))) { - throw new ParseException("Unrecognized century: " + d, 0); - } - if (d.length() < 8 && (d.length() % 2) != 0) { - throw new ParseException("Incomplete month/date: " + d, - d.length()); - } - StringBuilder sb = new StringBuilder(d); - while (sb.length() < 8) { - sb.append("01"); - } - while (sb.length() < 12) { - sb.append("0"); - } - date = ArchiveUtils.parse12DigitDate(sb.toString()); - } - - return date; - } - - /** - * Utility function for parsing arc-style date stamps - * in the format yyyMMddHHmmssSSS. - * Date stamps are in the UTC time zone. The whole string will not be - * parsed, only the first 17 digits. - * - * @param date an arc-style formatted date stamp - * @return the Date corresponding to the date stamp string - * @throws ParseException if the inputstring was malformed - */ - public static Date parse17DigitDate(String date) throws ParseException { - return TIMESTAMP17.get().parse(date); - } - - /** - * Utility function for parsing arc-style date stamps - * in the format yyyMMddHHmmss. - * Date stamps are in the UTC time zone. The whole string will not be - * parsed, only the first 14 digits. - * - * @param date an arc-style formatted date stamp - * @return the Date corresponding to the date stamp string - * @throws ParseException if the inputstring was malformed - */ - public static Date parse14DigitDate(String date) throws ParseException{ - return TIMESTAMP14.get().parse(date); - } - - /** - * Utility function for parsing arc-style date stamps - * in the format yyyMMddHHmm. - * Date stamps are in the UTC time zone. The whole string will not be - * parsed, only the first 12 digits. - * - * @param date an arc-style formatted date stamp - * @return the Date corresponding to the date stamp string - * @throws ParseException if the inputstring was malformed - */ - public static Date parse12DigitDate(String date) throws ParseException{ - return TIMESTAMP12.get().parse(date); - } - - /** - * @param timestamp A 14-digit timestamp or the suffix for a 14-digit - * timestamp: E.g. '20010909014640' or '20010101' or '1970'. - * @return Seconds since the epoch as a string zero-pre-padded so always - * Integer.MAX_VALUE wide (Makes it so sorting of resultant string works - * properly). - * @throws ParseException - */ - public static String secondsSinceEpoch(String timestamp) - throws ParseException { - return zeroPadInteger((int) - (getSecondsSinceEpoch(timestamp).getTime()/1000)); - } - - /** - * @param timestamp A 14-digit timestamp or the suffix for a 14-digit - * timestamp: E.g. '20010909014640' or '20010101' or '1970'. - * @return A date. - * @see #secondsSinceEpoch(String) - * @throws ParseException - */ - public static Date getSecondsSinceEpoch(String timestamp) - throws ParseException { - if (timestamp.length() < 14) { - if (timestamp.length() < 10 && (timestamp.length() % 2) == 1) { - throw new IllegalArgumentException("Must have year, " + - "month, date, hour or second granularity: " + timestamp); - } - if (timestamp.length() == 4) { - // Add first month and first date. - timestamp = timestamp + "01010000"; - } - if (timestamp.length() == 6) { - // Add a date of the first. - timestamp = timestamp + "010000"; - } - if (timestamp.length() < 14) { - timestamp = timestamp + - ArchiveUtils.padTo("", 14 - timestamp.length(), '0'); - } - } - return ArchiveUtils.parse14DigitDate(timestamp); - } - - /** - * @param i Integer to add prefix of zeros too. If passed - * 2005, will return the String 0000002005. String - * width is the width of Integer.MAX_VALUE as a string (10 - * digits). - * @return Padded String version of i. - */ - public static String zeroPadInteger(int i) { - return ArchiveUtils.padTo(Integer.toString(i), - MAX_INT_CHAR_WIDTH, '0'); - } - - /** - * Convert an int to a String, and pad it to - * pad spaces. - * @param i the int - * @param pad the width to pad to. - * @return String w/ padding. - */ - public static String padTo(final int i, final int pad) { - String n = Integer.toString(i); - return padTo(n, pad); - } - - /** - * Pad the given String to pad characters wide - * by pre-pending spaces. s should not be null. - * If s is already wider than pad no change is - * done. - * - * @param s the String to pad - * @param pad the width to pad to. - * @return String w/ padding. - */ - public static String padTo(final String s, final int pad) { - return padTo(s, pad, DEFAULT_PAD_CHAR); - } - - /** - * Pad the given String to pad characters wide - * by pre-pending padChar. - * - * s should not be null. If s is - * already wider than pad no change is done. - * - * @param s the String to pad - * @param pad the width to pad to. - * @param padChar The pad character to use. - * @return String w/ padding. - */ - public static String padTo(final String s, final int pad, - final char padChar) { - String result = s; - int l = s.length(); - if (l < pad) { - StringBuffer sb = new StringBuffer(pad); - while(l < pad) { - sb.append(padChar); - l++; - } - sb.append(s); - result = sb.toString(); - } - return result; - } - - /** check that two byte arrays are equal. They may be null. - * - * @param lhs a byte array - * @param rhs another byte array. - * @return true if they are both equal (or both - * null) - */ - public static boolean byteArrayEquals(final byte[] lhs, final byte[] rhs) { - if (lhs == null && rhs != null || lhs != null && rhs == null) { - return false; - } - if (lhs==rhs) { - return true; - } - if (lhs.length != rhs.length) { - return false; - } - for(int i = 0; i - * This involves converting it to the largest unit - * (of B, KiB, MiB, GiB, TiB) for which the amount will be > 1. - *

- * Additionally, at least 2 significant digits are always displayed. - *

- * Negative numbers will be returned as '0 B'. - * - * @param amount the amount of bytes - * @return A string containing the amount, properly formated. - */ - public static String formatBytesForDisplay(long amount) { - double displayAmount = (double) amount; - int unitPowerOf1024 = 0; - - if(amount <= 0){ - return "0 B"; - } - - while(displayAmount>=1024 && unitPowerOf1024 < 4) { - displayAmount = displayAmount / 1024; - unitPowerOf1024++; - } - - final String[] units = { " B", " KiB", " MiB", " GiB", " TiB" }; - - // ensure at least 2 significant digits (#.#) for small displayValues - int fractionDigits = (displayAmount < 10) ? 1 : 0; - return doubleToString(displayAmount, fractionDigits, fractionDigits) - + units[unitPowerOf1024]; - } - - /** - * Convert milliseconds value to a human-readable duration - * @param time - * @return Human readable string version of passed time - */ - public static String formatMillisecondsToConventional(long time) { - return formatMillisecondsToConventional(time,5); - } - - /** - * Convert milliseconds value to a human-readable duration of - * mixed units, using units no larger than days. For example, - * "5d12h13m12s113ms" or "19h51m". - * - * @param duration - * @param unitCount how many significant units to show, at most - * for example, a value of 2 would show days+hours or hours+seconds - * but not hours+second+milliseconds - * @return Human readable string version of passed time - */ - public static String formatMillisecondsToConventional(long duration, int unitCount) { - if(unitCount <=0) { - unitCount = 5; - } - if(duration==0) { - return "0ms"; - } - StringBuffer sb = new StringBuffer(); - if(duration<0) { - sb.append("-"); - } - long absTime = Math.abs(duration); - long[] thresholds = {DAY_IN_MS, HOUR_IN_MS, 60000, 1000, 1}; - String[] units = {"d","h","m","s","ms"}; - - for(int i = 0; i < thresholds.length; i++) { - if(absTime >= thresholds[i]) { - sb.append(absTime / thresholds[i] + units[i]); - absTime = absTime % thresholds[i]; - unitCount--; - } - if(unitCount==0) { - break; - } - } - return sb.toString(); - } - - /** - * Copy the raw bytes of a long into a byte array, starting at - * the specified offset. - * - * @param l - * @param array - * @param offset - */ - public static void longIntoByteArray(long l, byte[] array, int offset) { - int i, shift; - - for(i = 0, shift = 56; i < 8; i++, shift -= 8) - array[offset+i] = (byte)(0xFF & (l >> shift)); - } - - public static long byteArrayIntoLong(byte [] bytearray) { - return byteArrayIntoLong(bytearray, 0); - } - - /** - * Byte array into long. - * @param bytearray Array to convert to a long. - * @param offset Offset into array at which we start decoding the long. - * @return Long made of the bytes of array beginning at - * offset offset. - * @see #longIntoByteArray(long, byte[], int) - */ - public static long byteArrayIntoLong(byte [] bytearray, - int offset) { - long result = 0; - for (int i = offset; i < 8 /*Bytes in long*/; i++) { - result = (result << 8 /*Bits in byte*/) | - (0xff & (byte)(bytearray[i] & 0xff)); - } - return result; - } - - /** - * Given a string that may be a plain host or host/path (without - * URI scheme), add an implied http:// if necessary. - * - * @param u string to evaluate - * @return string with http:// added if no scheme already present - */ - public static String addImpliedHttpIfNecessary(String u) { - int colon = u.indexOf(':'); - int period = u.indexOf('.'); - if (colon == -1 || (period >= 0) && (period < colon)) { - // No scheme present; prepend "http://" - u = "http://" + u; - } - return u; - } - - /** - * Verify that the array begins with the prefix. - * - * @param array - * @param prefix - * @return true if array is identical to prefix for the first prefix.length - * positions - */ - public static boolean startsWith(byte[] array, byte[] prefix) { - if(prefix.length>array.length) { - return false; - } - for(int i = 0; i < prefix.length; i++) { - if(array[i]!=prefix[i]) { - return false; - } - } - return true; - } - - /** - * Utility method to get a String shortReportLine from Reporter - * @param rep Reporter to get shortReportLine from - * @return String of report - */ - public static String shortReportLine(Reporter rep) { - StringWriter sw = new StringWriter(); - PrintWriter pw = new PrintWriter(sw); - try { - rep.shortReportLineTo(pw); - } catch (IOException e) { - // not really possible - e.printStackTrace(); - } - pw.flush(); - return sw.toString(); - } - - /** - * Enhance given object's default String display for appearing - * nested in a pretty Map String. - * - * @param obj Object to prettify - * @return prettified String - */ - public static String prettyString(Object obj) { - // these things have to checked and casted unfortunately - if (obj instanceof Object[]) { - return prettyString((Object[]) obj); - } else if (obj instanceof Map) { - return prettyString((Map) obj); - } else { - return "<"+obj+">"; - } - } - - /** - * Provide a improved String of a Map's entries - * - * @param Map - * @return prettified (in curly brackets) string of Map contents - */ - public static String prettyString(Map map) { - StringBuilder builder = new StringBuilder(); - builder.append("{ "); - boolean needsComma = false; - for( Object key : map.keySet()) { - if(needsComma) { - builder.append(", "); - } - builder.append(key); - builder.append(": "); - builder.append(prettyString(map.get(key))); - needsComma = true; - } - builder.append(" }"); - return builder.toString(); - } - - /** - * Provide a slightly-improved String of Object[] - * - * @param Object[] - * @return prettified (in square brackets) of Object[] - */ - public static String prettyString(Object[] array) { - StringBuilder builder = new StringBuilder(); - builder.append("[ "); - boolean needsComma = false; - for (Object o: array) { - if(o==null) continue; - if(needsComma) { - builder.append(", "); - } - builder.append(prettyString(o)); - needsComma = true; - } - builder.append(" ]"); - return builder.toString(); - } - - - private static String loadVersion() { - InputStream input = ArchiveUtils.class.getResourceAsStream( - "/org/archive/util/version.txt"); - if (input == null) { - return "UNKNOWN"; - } - BufferedReader br = null; - String version; - try { - br = new BufferedReader(new InputStreamReader(input)); - version = br.readLine(); - br.readLine(); - } catch (IOException e) { - return e.getMessage(); - } finally { - closeQuietly(br); - } - - version = version.trim(); - if (!version.endsWith("SNAPSHOT")) { - return version; - } - - input = ArchiveUtils.class.getResourceAsStream("/org/archive/util/timestamp.txt"); - if (input == null) { - return version; - } - - br = null; - String timestamp; - try { - br = new BufferedReader(new InputStreamReader(input)); - timestamp = br.readLine(); - } catch (IOException e) { - return version; - } finally { - closeQuietly(br); - } - - if (timestamp.startsWith("timestamp=")) { - timestamp = timestamp.substring(10); - } - - return version.trim() + "-" + timestamp.trim(); - } - - public static Set TLDS; - - static { - TLDS = new HashSet(); - InputStream is = ArchiveUtils.class.getResourceAsStream("tlds-alpha-by-domain.txt"); - try { - BufferedReader reader = new BufferedReader(new InputStreamReader(is)); - String line; - while((line = reader.readLine())!=null) { - if (line.startsWith("#")) { - continue; - } - TLDS.add(line.trim().toLowerCase()); - } - } catch (Exception e) { - LOGGER.log(Level.SEVERE,"TLD list unavailable",e); - } finally { - IOUtils.closeQuietly(is); - } - } - /** - * Return whether the given string represents a known - * top-level-domain (like "com", "org", etc.) per IANA - * as of 20100419 - * - * @param dom candidate string - * @return boolean true if recognized as TLD - */ - public static boolean isTld(String dom) { - return TLDS.contains(dom.toLowerCase()); - } - - public static void closeQuietly(Object input) { - if(input == null || ! (input instanceof Closeable)) { - return; - } - try { - ((Closeable)input).close(); - } catch (IOException ioe) { - // ignore - } - } - - /** - * Perform checks as to whether normal execution should proceed. - * - * If an external interrupt is detected, throw an interrupted exception. - * Used before anything that should not be attempted by a 'zombie' thread - * that the Frontier/Crawl has given up on. - * - * @throws InterruptedException - */ - public static void continueCheck() throws InterruptedException { - if(Thread.interrupted()) { - throw new InterruptedException("interrupt detected"); - } - } - - /** - * Read stream into buf until EOF or buf full. - * - * @param input - * @param buf - * @throws IOException - */ - public static int readFully(InputStream input, byte[] buf) - throws IOException { - int max = buf.length; - int ofs = 0; - while (ofs < max) { - int l = input.read(buf, ofs, max - ofs); - if (l == 0) { - throw new EOFException(); - } - ofs += l; - } - return ofs; - } - - /** suffix to recognize gzipped files */ - public static final String GZIP_SUFFIX = ".gz"; - - /** - * Get a BufferedReader on the crawler journal given - * - * TODO: move to a general utils class - * - * @param source File journal - * @return journal buffered reader. - * @throws IOException - */ - public static BufferedReader getBufferedReader(File source) throws IOException { - InputStream is = new BufferedInputStream(new FileInputStream(source)); - boolean isGzipped = source.getName().toLowerCase(). - endsWith(GZIP_SUFFIX); - if(isGzipped) { - is = new GZIPInputStream(is); - } - return new BufferedReader(new InputStreamReader(is)); - } - - /** - * Get a BufferedReader on the crawler journal given. - * - * @param source URL journal - * @return journal buffered reader. - * @throws IOException - */ - public static BufferedReader getBufferedReader(URL source) throws IOException { - URLConnection conn = source.openConnection(); - boolean isGzipped = conn.getContentType() != null && conn.getContentType().equalsIgnoreCase("application/x-gzip") - || conn.getContentEncoding() != null && conn.getContentEncoding().equalsIgnoreCase("gzip"); - InputStream uis = conn.getInputStream(); - return new BufferedReader(isGzipped? - new InputStreamReader(new GZIPInputStream(uis)): - new InputStreamReader(uis)); - } - - /** - * Gzip passed bytes. - * Use only when bytes is small. - * @param bytes What to gzip. - * @return A gzip member of bytes. - * @throws IOException - */ - public static byte [] gzip(byte [] bytes) throws IOException { - ByteArrayOutputStream baos = new ByteArrayOutputStream(); - GZIPOutputStream gzipOS = new GZIPOutputStream(baos); - gzipOS.write(bytes, 0, bytes.length); - gzipOS.close(); - return baos.toByteArray(); - } - - /** - * Tests passed stream is gzip stream by reading in the HEAD. - * Does not mark/reset stream -- so this test actually makes - * stream unopenable within GZIP streams, unless reset. - * @param is An InputStream. - * @return True if compressed stream. - * @throws IOException - */ - public static boolean isGzipped(final InputStream is) - throws IOException { - try { - new GzipHeader(is); - } catch (NoGzipMagicException e) { - return false; - } - return true; - } - -// public static long doubleMurmur(byte[] data) { -// int first = MurmurHash.hash(data, 7); -// int second = MurmurHash.hash(data, 13); -// return (((long)first)<<32) | ((long)second & 0x00000000ffffffffl); -// } -} - diff --git a/commons/src/main/java/org/archive/util/InterruptibleCharSequence.java b/commons/src/main/java/org/archive/util/InterruptibleCharSequence.java deleted file mode 100644 index f608c49c..00000000 --- a/commons/src/main/java/org/archive/util/InterruptibleCharSequence.java +++ /dev/null @@ -1,56 +0,0 @@ -/* - * This file is part of the Heritrix web crawler (crawler.archive.org). - * - * Licensed to the Internet Archive (IA) by one or more individual - * contributors. - * - * The IA licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.archive.util; - -/** - * CharSequence that noticed thread interrupts -- as might be necessary - * to recover from a loose regex on unexpected challenging input. - * - * @author gojomo - */ -public class InterruptibleCharSequence implements CharSequence { - protected CharSequence inner; - // public long counter = 0; - - public InterruptibleCharSequence(CharSequence inner) { - super(); - this.inner = inner; - } - - public char charAt(int index) { - if (Thread.interrupted()) { // clears flag if set - throw new RuntimeException(new InterruptedException()); - } - // counter++; - return inner.charAt(index); - } - - public int length() { - return inner.length(); - } - - public CharSequence subSequence(int start, int end) { - return new InterruptibleCharSequence(inner.subSequence(start, end)); - } - - @Override - public String toString() { - return inner.toString(); - } -} diff --git a/commons/src/main/java/org/archive/util/InterruptibleCharSequenceTest.java b/commons/src/main/java/org/archive/util/InterruptibleCharSequenceTest.java deleted file mode 100644 index a3a5f180..00000000 --- a/commons/src/main/java/org/archive/util/InterruptibleCharSequenceTest.java +++ /dev/null @@ -1,120 +0,0 @@ -/* - * This file is part of the Heritrix web crawler (crawler.archive.org). - * - * Licensed to the Internet Archive (IA) by one or more individual - * contributors. - * - * The IA licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.archive.util; - -import java.util.concurrent.BlockingQueue; -import java.util.concurrent.LinkedBlockingQueue; -import java.util.regex.Pattern; - -import junit.framework.TestCase; - -/** - * Tests (and - * @author gojomo - */ -public class InterruptibleCharSequenceTest extends TestCase { - // this regex takes many seconds to fail on the input - // (~20 seconds on 2Ghz Athlon64 JDK 1.6) - public static String BACKTRACKER = "^(((((a+)*)*)*)*)*$"; - public static String INPUT = "aaaaab"; - - /** - * Development-time benchmarking of InterruptibleCharSequence in - * regex use. (Rename 'xest' to 'test' if wanted as unit test, - * but never actually fails anything -- just measures.) - * - * For reference the regex "^(((((a+)*)*)*)*)*$" requires - * 239,286,636 charAt(s) to fail on "aaaaab", which takes - * around 20 seconds on a 2Ghz Athlon64(x2) with JDK 1.6. - * The runtime overhead of checking interrupt status in this - * extreme case is around 5% in my tests. - */ - public void xestOverhead() { - String regex = BACKTRACKER; - String inputNormal = INPUT; - InterruptibleCharSequence inputWrapped = new InterruptibleCharSequence(inputNormal); - // warm up - tryMatch(inputNormal,regex); - tryMatch(inputWrapped,regex); - // inputWrapped.counter=0; - int trials = 5; - long stringTally = 0; - long icsTally = 0; - for(int i = 1; i <= trials; i++) { - System.out.println("trial "+i+" of "+trials); - long start = System.currentTimeMillis(); - System.out.print("String "); - tryMatch(inputNormal,regex); - long end = System.currentTimeMillis(); - System.out.println(end-start); - stringTally += (end-start); - start = System.currentTimeMillis(); - System.out.print("InterruptibleCharSequence "); - tryMatch(inputWrapped,regex); - end = System.currentTimeMillis(); - System.out.println(end-start); - //System.out.println(inputWrapped.counter+" steps"); - //inputWrapped.counter=0; - icsTally += (end-start); - } - System.out.println("InterruptibleCharSequence took "+((float)icsTally)/stringTally+" longer."); - } - - public boolean tryMatch(CharSequence input, String regex) { - return Pattern.matches(regex,input); - } - - public Thread tryMatchInThread(final CharSequence input, final String regex, final BlockingQueue atFinish) { - Thread t = new Thread() { - public void run() { - boolean result; - try { - result = tryMatch(input,regex); - } catch (Exception e) { - atFinish.offer(e); - return; - } - atFinish.offer(result); - } - }; - t.start(); - return t; - } - - public void testNoninterruptible() throws InterruptedException { - BlockingQueue q = new LinkedBlockingQueue(); - Thread t = tryMatchInThread(INPUT, BACKTRACKER, q); - Thread.sleep(1000); - t.interrupt(); - Object result = q.take(); - assertTrue("mismatch uncompleted",Boolean.FALSE.equals(result)); - } - - public void testInterruptibility() throws InterruptedException { - BlockingQueue q = new LinkedBlockingQueue(); - Thread t = tryMatchInThread(new InterruptibleCharSequence(INPUT), BACKTRACKER, q); - Thread.sleep(500); - t.interrupt(); - Object result = q.take(); - if(result instanceof Boolean) { - System.err.println(result+" match beat interrupt"); - } - assertTrue("exception not thrown",result instanceof RuntimeException); - } -} diff --git a/commons/src/main/java/org/archive/util/PrefixSet.java b/commons/src/main/java/org/archive/util/PrefixSet.java deleted file mode 100644 index 382b154a..00000000 --- a/commons/src/main/java/org/archive/util/PrefixSet.java +++ /dev/null @@ -1,74 +0,0 @@ -/* - * This file is part of the Heritrix web crawler (crawler.archive.org). - * - * Licensed to the Internet Archive (IA) by one or more individual - * contributors. - * - * The IA licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.archive.util; - -import java.util.SortedSet; -import java.util.concurrent.ConcurrentSkipListSet; - -/** - * Utility class for maintaining sorted set of string prefixes. - * Redundant prefixes are coalesced into the shorter prefix. - */ -public class PrefixSet extends ConcurrentSkipListSet { - private static final long serialVersionUID = -6054697706348411992L; - - public PrefixSet() { - super(); - } - - /** - * Test whether the given String is prefixed by one - * of this set's entries. - * - * @param s - * @return True if contains prefix. - */ - public boolean containsPrefixOf(String s) { - SortedSet sub = headSet(s); - // because redundant prefixes have been eliminated, - // only a test against last item in headSet is necessary - if (!sub.isEmpty() && s.startsWith((String)sub.last())) { - return true; // prefix substring exists - } // else: might still exist exactly (headSet does not contain boundary) - return contains(s); // exact string exists, or no prefix is there - } - - /** - * Maintains additional invariant: if one entry is a - * prefix of another, keep only the prefix. - * - * @see java.util.Collection#add(java.lang.Object) - */ - public boolean add(String s) { - SortedSet sub = headSet(s); - if (!sub.isEmpty() && s.startsWith((String)sub.last())) { - // no need to add; prefix is already present - return false; - } - boolean retVal = super.add(s); - sub = tailSet(s+"\0"); - while(!sub.isEmpty() && ((String)sub.first()).startsWith(s)) { - // remove redundant entries - sub.remove(sub.first()); - } - return retVal; - } - -} \ No newline at end of file diff --git a/commons/src/main/java/org/archive/util/ReportUtils.java b/commons/src/main/java/org/archive/util/ReportUtils.java new file mode 100644 index 00000000..877b8376 --- /dev/null +++ b/commons/src/main/java/org/archive/util/ReportUtils.java @@ -0,0 +1,45 @@ +/* + * This file is part of the Heritrix web crawler (crawler.archive.org). + * + * Licensed to the Internet Archive (IA) by one or more individual + * contributors. + * + * The IA licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.archive.util; + +import java.io.IOException; +import java.io.PrintWriter; +import java.io.StringWriter; + +public class ReportUtils { + /** + * Utility method to get a String shortReportLine from Reporter + * @param rep Reporter to get shortReportLine from + * @return String of report + */ + public static String shortReportLine(Reporter rep) { + StringWriter sw = new StringWriter(); + PrintWriter pw = new PrintWriter(sw); + try { + rep.shortReportLineTo(pw); + } catch (IOException e) { + // not really possible + e.printStackTrace(); + } + pw.flush(); + return sw.toString(); + } + +} diff --git a/commons/src/main/java/org/archive/util/SURT.java b/commons/src/main/java/org/archive/util/SURT.java deleted file mode 100644 index e5797935..00000000 --- a/commons/src/main/java/org/archive/util/SURT.java +++ /dev/null @@ -1,256 +0,0 @@ -/* - * This file is part of the Heritrix web crawler (crawler.archive.org). - * - * Licensed to the Internet Archive (IA) by one or more individual - * contributors. - * - * The IA licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.archive.util; - -import java.io.BufferedInputStream; -import java.io.BufferedOutputStream; -import java.io.BufferedReader; -import java.io.FileInputStream; -import java.io.FileOutputStream; -import java.io.IOException; -import java.io.InputStream; -import java.io.InputStreamReader; -import java.io.PrintStream; -import java.util.regex.Matcher; - -import org.apache.commons.httpclient.URIException; -import org.archive.net.UURIFactory; - -/** - * Sort-friendly URI Reordering Transform. - * - * Converts URIs of the form: - * - * scheme://userinfo@domain.tld:port/path?query#fragment - * - * ...into... - * - * scheme://(tld,domain,:port@userinfo)/path?query#fragment - * - * The '(' ')' characters serve as an unambiguous notice that the so-called - * 'authority' portion of the URI ([userinfo@]host[:port] in http URIs) has - * been transformed; the commas prevent confusion with regular hostnames. - * - * This remedies the 'problem' with standard URIs that the host portion of a - * regular URI, with its dotted-domains, is actually in reverse order from - * the natural hierarchy that's usually helpful for grouping and sorting. - * - * The value of respecting URI case variance is considered negligible: it - * is vanishingly rare for case-variance to be meaningful, while URI case- - * variance often arises from people's confusion or sloppiness, and they - * only correct it insofar as necessary to avoid blatant problems. Thus - * the usual SURT form is considered to be flattened to all lowercase, and - * not completely reversible. - * - * @author gojomo - */ -public class SURT { - protected static char DOT = '.'; - protected static String BEGIN_TRANSFORMED_AUTHORITY = "("; - protected static String TRANSFORMED_HOST_DELIM = ","; - protected static String END_TRANSFORMED_AUTHORITY = ")"; - - // 1: scheme:// - // 2: userinfo (if present) - // 3: @ (if present) - // 4: dotted-quad host - // 5: other host - // 6: :port - // 7: path - protected static String URI_SPLITTER = - "^(\\w+://)(?:([-\\w\\.!~\\*'\\(\\)%;:&=+$,]+?)(@))?"+ - // 1 2 3 - "(?:((?:\\d{1,3}\\.){3}\\d{1,3})|(\\S+?))(:\\d+)?(/\\S*)?$"; - // 4 5 6 7 - - // RFC2396 - // reserved = ";" | "/" | "?" | ":" | "@" | "&" | "=" | "+" | - // "$" | "," - // unreserved = alphanum | mark - // mark = "-" | "_" | "." | "!" | "~" | "*" | "'" | "(" | ")" - // userinfo = *( unreserved | escaped | - // ";" | ":" | "&" | "=" | "+" | "$" | "," ) - // escaped = "%" hex hex - - - /** - * Utility method for creating the SURT form of the URI in the - * given String. - * - * By default, does not preserve casing. - * - * @param s String URI to be converted to SURT form - * @return SURT form - */ - public static String fromURI(String s) { - return fromURI(s,false); - } - - /** - * Utility method for creating the SURT form of the URI in the - * given String. - * - * If it appears a bit convoluted in its approach, note that it was - * optimized to minimize object-creation after allocation-sites profiling - * indicated this method was a top source of garbage in long-running crawls. - * - * Assumes that the String URI has already been cleaned/fixed (eg - * by UURI fixup) in ways that put it in its crawlable form for - * evaluation. - * - * @param s String URI to be converted to SURT form - * @param preserveCase whether original case should be preserved - * @return SURT form - */ - public static String fromURI(String s, boolean preserveCase) { - Matcher m = TextUtils.getMatcher(URI_SPLITTER,s); - if(!m.matches()) { - // not an authority-based URI scheme; return unchanged - TextUtils.recycleMatcher(m); - return s; - } - // preallocate enough space for SURT form, which includes - // 3 extra characters ('(', ')', and one more ',' than '.'s - // in original) - StringBuffer builder = new StringBuffer(s.length()+3); - append(builder,s,m.start(1),m.end(1)); // scheme:// - builder.append(BEGIN_TRANSFORMED_AUTHORITY); // '(' - - if(m.start(4)>-1) { - // dotted-quad ip match: don't reverse - append(builder,s,m.start(4),m.end(4)); - } else { - // other hostname match: do reverse - int hostSegEnd = m.end(5); - int hostStart = m.start(5); - for(int i = m.end(5)-1; i>=hostStart; i--) { - if(s.charAt(i-1)!=DOT && i > hostStart) { - continue; - } - append(builder,s,i,hostSegEnd); // rev host segment - builder.append(TRANSFORMED_HOST_DELIM); // ',' - hostSegEnd = i-1; - } - } - - append(builder,s,m.start(6),m.end(6)); // :port - append(builder,s,m.start(3),m.end(3)); // at - append(builder,s,m.start(2),m.end(2)); // userinfo - builder.append(END_TRANSFORMED_AUTHORITY); // ')' - append(builder,s,m.start(7),m.end(7)); // path - if (!preserveCase) { - for(int i = 0; i < builder.length(); i++) { - builder.setCharAt(i,Character.toLowerCase(builder.charAt((i)))); - } - } - TextUtils.recycleMatcher(m); - return builder.toString(); - } - - private static void append(StringBuffer b, CharSequence cs, int start, - int end) { - if (start < 0) { - return; - } - b.append(cs, start, end); - } - - /** - * Given a plain URI or hostname/hostname+path, deduce an implied SURT - * prefix from it. Results may be unpredictable on strings that cannot - * be interpreted as URIs. - * - * UURI 'fixup' is applied to the URI that is built. - * - * @param u URI or almost-URI to consider - * @return implied SURT prefix form - */ - public static String prefixFromPlain(String u) { - u = fromPlain(u); - // truncate to implied prefix - u = SurtPrefixSet.asPrefix(u); - return u; - } - - /** - * Given a plain URI or hostname/hostname+path, give its SURT form. - * Results may be unpredictable on strings that cannot - * be interpreted as URIs. - * - * UURI 'fixup' is applied to the URI before conversion to SURT - * form. - * - * @param u URI or almost-URI to consider - * @return implied SURT prefix form - */ - public static String fromPlain(String u) { - u = ArchiveUtils.addImpliedHttpIfNecessary(u); - boolean trailingSlash = u.endsWith("/"); - // ensure all typical UURI cleanup (incl. IDN-punycoding) is done - try { - u = UURIFactory.getInstance(u).toString(); - } catch (URIException e) { - e.printStackTrace(); - // allow to continue with original string uri - } - // except: don't let UURI-fixup add a trailing slash - // if it wasn't already there (presence or absence of - // such slash has special meaning specifying implied - // SURT prefixes) - if(!trailingSlash && u.endsWith("/")) { - u = u.substring(0,u.length()-1); - } - // convert to full SURT - u = SURT.fromURI(u); - return u; - } - - /** - * Allow class to be used as a command-line tool for converting - * URL lists (or naked host or host/path fragments implied - * to be HTTP URLs) to SURT form. Lines that cannot be converted - * are returned unchanged. - * - * - * Read from stdin or first file argument. Writes to stdout or - * second argument filename - * - * @param args cmd-line arguments - * @throws IOException - */ - public static void main(String[] args) throws IOException { - InputStream in = args.length > 0 ? new BufferedInputStream( - new FileInputStream(args[0])) : System.in; - PrintStream out = args.length > 1 ? new PrintStream( - new BufferedOutputStream(new FileOutputStream(args[1]))) - : System.out; - BufferedReader br = - new BufferedReader(new InputStreamReader(in)); - String line; - while((line = br.readLine())!=null) { - if(line.indexOf("#")>0) line=line.substring(0,line.indexOf("#")); - line = line.trim(); - if(line.length()==0) continue; - line = ArchiveUtils.addImpliedHttpIfNecessary(line); - out.println(SURT.fromURI(line)); - } - br.close(); - out.close(); - } -} diff --git a/commons/src/main/java/org/archive/util/SurtPrefixSet.java b/commons/src/main/java/org/archive/util/SurtPrefixSet.java deleted file mode 100644 index 0db32b24..00000000 --- a/commons/src/main/java/org/archive/util/SurtPrefixSet.java +++ /dev/null @@ -1,360 +0,0 @@ -/* - * This file is part of the Heritrix web crawler (crawler.archive.org). - * - * Licensed to the Internet Archive (IA) by one or more individual - * contributors. - * - * The IA licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.archive.util; - -import java.io.BufferedInputStream; -import java.io.BufferedOutputStream; -import java.io.BufferedReader; -import java.io.FileInputStream; -import java.io.FileOutputStream; -import java.io.FileWriter; -import java.io.IOException; -import java.io.InputStream; -import java.io.InputStreamReader; -import java.io.PrintStream; -import java.io.Reader; -import java.util.Iterator; - -import org.archive.net.UURI; -import org.archive.util.iterator.LineReadingIterator; -import org.archive.util.iterator.RegexLineIterator; - -/** - * Specialized TreeSet for keeping a set of String prefixes. - * - * Redundant prefixes (those that are themselves prefixed - * by other set entries) are eliminated. - * - * @author gojomo - */ -public class SurtPrefixSet extends PrefixSet { - - private static final long serialVersionUID = 2598365040524933110L; - - private static final String SURT_PREFIX_DIRECTIVE = "+"; - - - /** - * Read a set of SURT prefixes from a reader source; keep sorted and - * with redundant entries removed. - * - * @param r reader over file of SURT_format strings - * @throws IOException - */ - public void importFrom(Reader r) { - BufferedReader reader = new BufferedReader(r); - String s; - - Iterator iter = - new RegexLineIterator( - new LineReadingIterator(reader), - RegexLineIterator.COMMENT_LINE, - RegexLineIterator.NONWHITESPACE_ENTRY_TRAILING_COMMENT, - RegexLineIterator.ENTRY); - - while (iter.hasNext()) { - s = (String) iter.next(); - add(s.toLowerCase()); - } - } - - /** - * @param r Where to read from. - */ - public void importFromUris(Reader r) { - BufferedReader reader = new BufferedReader(r); - String s; - - Iterator iter = - new RegexLineIterator( - new LineReadingIterator(reader), - RegexLineIterator.COMMENT_LINE, - RegexLineIterator.NONWHITESPACE_ENTRY_TRAILING_COMMENT, - RegexLineIterator.ENTRY); - - while (iter.hasNext()) { - s = (String) iter.next(); - // s is a URI (or even fragmentary hostname), not a SURT - addFromPlain(s); - } - } - - /** - * Import SURT prefixes from a reader with mixed URI and SURT prefix - * format. - * - * @param r the reader to import the prefixes from - * @param deduceFromSeeds true to also import SURT prefixes implied - * from normal URIs/hostname seeds - */ - public void importFromMixed(Reader r, boolean deduceFromSeeds) { - BufferedReader reader = new BufferedReader(r); - String s; - - Iterator iter = - new RegexLineIterator( - new LineReadingIterator(reader), - RegexLineIterator.COMMENT_LINE, - RegexLineIterator.NONWHITESPACE_ENTRY_TRAILING_COMMENT, - RegexLineIterator.ENTRY); - - while (iter.hasNext()) { - s = (String) iter.next(); - if(s.startsWith(SURT_PREFIX_DIRECTIVE)) { - considerAsAddDirective(s.substring(SURT_PREFIX_DIRECTIVE.length())); - continue; - } else { - if(deduceFromSeeds) { - // also deducing 'implied' SURT prefixes - // from normal URIs/hostname seeds - addFromPlain(s); - } - } - } - } - - - /** - * Interpret the given SURT/URI/host as a directive, returning true - * if it was meaningful. - * - * @param suri potential directive - * @return boolean true if applied as directive, false otherwise - */ - public boolean considerAsAddDirective(String suri) { - String u = suri.trim(); - if(u.length()==0) { - // empty string: consider a mistake - return false; - } - if(u.indexOf("(")>0) { - // formal SURT prefix; toLowerCase just in case - add(u.toLowerCase()); - } else { - // hostname/normal form URI from which - // to deduce SURT prefix - addFromPlain(u); - } - return true; - - } - - /** - * Given a plain URI or hostname, deduce an implied SURT prefix from - * it and add to active prefixes. - * - * @param u String of URI or hostname - */ - public void addFromPlain(String u) { - u = prefixFromPlainForceHttp(u); - add(u); - } - - /** - * Given a plain URI or hostname/hostname+path, deduce an implied SURT - * prefix from it. Results may be unpredictable on strings that cannot - * be interpreted as URIs. - * - * UURI 'fixup' is applied to the URI that is built. - * - * HTTPS URIs are changed to HTTP as a convenience for the usual - * preferred common-treatment. - * - * @param u URI or almost-URI to consider - * @return implied SURT prefix form - */ - public static String prefixFromPlainForceHttp(String u) { - u = SURT.prefixFromPlain(u); - u = coerceFromHttpsForComparison(u); - return u; - } - - /** - * For SURT comparisons -- prefixes or candidates being checked against - * those prefixes -- we treat https URIs as if they were http. - * - * @param u string to coerce if it has https scheme - * @return string converted to http scheme, or original if not necessary - */ - private static String coerceFromHttpsForComparison(String u) { - if (u.startsWith("https://")) { - u = "http" + u.substring("https".length()); - } - return u; - } - - /** - * Utility method for truncating a SURT that came from a - * full URI (as a seed, for example) into a prefix - * for determining inclusion. - * - * This involves: - * 
-     *    (1) removing the last path component, if any
-     *        (anything after the last '/', if there are
-     *        at least 3 '/'s)
-     *    (2) removing a trailing ')', if present, opening
-     *        the possibility of proper subdomains. (This
-     *        means that the presence or absence of a
-     *        trailing '/' after a hostname in a seed list
-     *        is significant for the how the SURT prefix is 
-     *        created, even though it is not signficant for 
-     *        the URI's treatment as a seed.)
-     *
- * - * @param s String to work on. - * @return As prefix. - */ - public static String asPrefix(String s) { - // Strip last path-segment, if more than 3 slashes - s = s.replaceAll("^(.*//.*/)[^/]*","$1"); - // Strip trailing ")", if present and NO path (no 3rd slash). - if (!s.endsWith("/")) { - s = s.replaceAll("^(.*)\\)","$1"); - } - return s; - } - - /** - * Calculate the SURT form URI to use as a candidate against prefixes - * from the given Object (CandidateURI or UURI) - * - * @param object CandidateURI or UURI - * @return SURT form of URI for evaluation, or null if unavailable - */ - public static String getCandidateSurt(UURI u) { - if (u == null) { - return null; - } - String candidateSurt = u.getSurtForm(); - // also want to treat https as http - candidateSurt = coerceFromHttpsForComparison(candidateSurt); - return candidateSurt; - } - /** - * @param fw - * @throws IOException - */ - public void exportTo(FileWriter fw) throws IOException { - Iterator iter = this.iterator(); - while(iter.hasNext()) { - fw.write((String)iter.next() + "\n"); - } - } - - /** - * Changes all prefixes so that they enforce an exact host. For - * prefixes that already include a ')', this means discarding - * anything after ')' (path info). For prefixes that don't include - * a ')' -- domain prefixes open to subdomains -- add the closing - * ')' (or ",)"). - */ - public void convertAllPrefixesToHosts() { - SurtPrefixSet iterCopy = (SurtPrefixSet) this.clone(); - Iterator iter = iterCopy.iterator(); - while (iter.hasNext()) { - String prefix = (String) iter.next(); - String convPrefix = convertPrefixToHost(prefix); - if(prefix!=convPrefix) { - // if returned value not unchanged, update set - this.remove(prefix); - this.add(convPrefix); - } - } - } - - public static String convertPrefixToHost(String prefix) { - if(prefix.endsWith(")")) { - return prefix; // no change necessary - } - if(prefix.indexOf(')')<0) { - // open-ended domain prefix - if(!prefix.endsWith(",")) { - prefix += ","; - } - prefix += ")"; - } else { - // prefix with excess path-info - prefix = prefix.substring(0,prefix.indexOf(')')+1); - } - return prefix; - } - - /** - * Changes all prefixes so that they only enforce a general - * domain (allowing subdomains).For prefixes that don't include - * a ')', no change is necessary. For others, truncate everything - * from the ')' onward. Additionally, truncate off "www," if it - * appears. - */ - public void convertAllPrefixesToDomains() { - SurtPrefixSet iterCopy = (SurtPrefixSet) this.clone(); - Iterator iter = iterCopy.iterator(); - while (iter.hasNext()) { - String prefix = (String) iter.next(); - String convPrefix = convertPrefixToDomain(prefix); - if(prefix!=convPrefix) { - // if returned value not unchanged, update set - this.remove(prefix); - this.add(convPrefix); - } - } - } - - public static String convertPrefixToDomain(String prefix) { - if(prefix.indexOf(')')>=0) { - prefix = prefix.substring(0,prefix.indexOf(')')); - } - // strip 'www,' when present - if(prefix.endsWith("www,")) { - prefix = prefix.substring(0,prefix.length()-4); - } - return prefix; - } - - /** - * Allow class to be used as a command-line tool for converting - * URL lists (or naked host or host/path fragments implied - * to be HTTP URLs) to implied SURT prefix form. - * - * Read from stdin or first file argument. Writes to stdout. - * - * @param args cmd-line arguments: may include input file - * @throws IOException - */ - public static void main(String[] args) throws IOException { - InputStream in = args.length > 0 ? new BufferedInputStream( - new FileInputStream(args[0])) : System.in; - PrintStream out = args.length > 1 ? new PrintStream( - new BufferedOutputStream(new FileOutputStream(args[1]))) - : System.out; - BufferedReader br = - new BufferedReader(new InputStreamReader(in)); - String line; - while((line = br.readLine())!=null) { - if(line.indexOf("#")>0) line=line.substring(0,line.indexOf("#")); - line = line.trim(); - if(line.length()==0) continue; - out.println(prefixFromPlainForceHttp(line)); - } - br.close(); - out.close(); - } -} diff --git a/commons/src/main/java/org/archive/util/TextUtils.java b/commons/src/main/java/org/archive/util/TextUtils.java deleted file mode 100644 index 707f93c7..00000000 --- a/commons/src/main/java/org/archive/util/TextUtils.java +++ /dev/null @@ -1,308 +0,0 @@ -/* - * This file is part of the Heritrix web crawler (crawler.archive.org). - * - * Licensed to the Internet Archive (IA) by one or more individual - * contributors. - * - * The IA licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.archive.util; - -import java.io.BufferedReader; -import java.io.IOException; -import java.io.PrintWriter; -import java.io.StringReader; -import java.io.StringWriter; -import java.io.UnsupportedEncodingException; -import java.io.Writer; -import java.net.URLDecoder; -import java.net.URLEncoder; -import java.util.HashMap; -import java.util.Map; -import java.util.concurrent.ConcurrentMap; -import java.util.regex.Matcher; -import java.util.regex.Pattern; - -import org.apache.commons.lang.StringEscapeUtils; - -import com.google.common.base.Function; -import com.google.common.collect.MapMaker; - -public class TextUtils { - private static final String FIRSTWORD = "^([^\\s]*).*$"; - - /** thread-local cached matchers, by string key */ - private static final ThreadLocal> TL_MATCHER_MAP - = new ThreadLocal>() { - protected Map initialValue() { - return new HashMap(50); - } - }; - - /** global soft-cache of Patterns, by string key */ - private static final ConcurrentMap PATTERNS = new MapMaker() - .concurrencyLevel(16) - .softValues() - .makeComputingMap(new Function() { - public Pattern apply(String regex) { - return Pattern.compile(regex); - } - }); - - /** - * Get a matcher object for a precompiled regex pattern. - * - * This method tries to reuse Matcher objects for efficiency. - * It can hold for recycling one Matcher per pattern per thread. - * - * Matchers retrieved should be returned for reuse via the - * recycleMatcher() method, but no errors will occur if they - * are not. - * - * This method is a hotspot frequently accessed. - * - * @param pattern the string pattern to use - * @param input the character sequence the matcher should be using - * @return a matcher object loaded with the submitted character sequence - */ - public static Matcher getMatcher(String pattern, CharSequence input) { - if (pattern == null) { - throw new IllegalArgumentException("String 'pattern' must not be null"); - } - input = new InterruptibleCharSequence(input); - final Map matchers = TL_MATCHER_MAP.get(); - Matcher m = (Matcher)matchers.get(pattern); - if(m == null) { - m = PATTERNS.get(pattern).matcher(input); - } else { - matchers.put(pattern,null); - m.reset(input); - } - return m; - } - - public static void recycleMatcher(Matcher m) { - // while cached, eliminate reference to potentially-large prior 'input' - m.reset(""); - final Map matchers = TL_MATCHER_MAP.get(); - matchers.put(m.pattern().pattern(),m); - } - - /** - * Utility method using a precompiled pattern instead of using the - * replaceAll method of the String class. This method will also be reusing - * Matcher objects. - * - * @see java.util.regex.Pattern - * @param pattern precompiled Pattern to match against - * @param input the character sequence to check - * @param replacement the String to substitute every match with - * @return the String with all the matches substituted - */ - public static String replaceAll( - String pattern, CharSequence input, String replacement) { - input = new InterruptibleCharSequence(input); - Matcher m = getMatcher(pattern, input); - String res = m.replaceAll(replacement); - recycleMatcher(m); - return res; - } - - /** - * Utility method using a precompiled pattern instead of using the - * replaceFirst method of the String class. This method will also be reusing - * Matcher objects. - * - * @see java.util.regex.Pattern - * @param pattern precompiled Pattern to match against - * @param input the character sequence to check - * @param replacement the String to substitute the first match with - * @return the String with the first match substituted - */ - public static String replaceFirst( - String pattern, CharSequence input, String replacement) { - input = new InterruptibleCharSequence(input); - Matcher m = getMatcher(pattern, input); - String res = m.replaceFirst(replacement); - recycleMatcher(m); - return res; - } - - /** - * Utility method using a precompiled pattern instead of using the matches - * method of the String class. This method will also be reusing Matcher - * objects. - * - * @see java.util.regex.Pattern - * @param pattern precompiled Pattern to match against - * @param input the character sequence to check - * @return true if character sequence matches - */ - public static boolean matches(String pattern, CharSequence input) { - input = new InterruptibleCharSequence(input); - Matcher m = getMatcher(pattern, input); - boolean res = m.matches(); - recycleMatcher(m); - return res; - } - - /** - * Utility method using a precompiled pattern instead of using the split - * method of the String class. - * - * @see java.util.regex.Pattern - * @param pattern precompiled Pattern to split by - * @param input the character sequence to split - * @return array of Strings split by pattern - */ - public static String[] split(String pattern, CharSequence input) { - input = new InterruptibleCharSequence(input); - Matcher m = getMatcher(pattern,input); - String[] retVal = m.pattern().split(input); - recycleMatcher(m); - return retVal; - } - - /** - * @param s String to find first word in (Words are delimited by - * whitespace). - * @return First word in the passed string else null if no word found. - */ - public static String getFirstWord(String s) { - Matcher m = getMatcher(FIRSTWORD, s); - String retVal = (m != null && m.matches())? m.group(1): null; - recycleMatcher(m); - return retVal; - } - - /** - * Escapes a string so that it can be passed as an argument to a javscript - * in a JSP page. This method takes a string and returns the same string - * with any single quote escaped by prepending the character with a - * backslash. Linebreaks are also replaced with '\n'. Also, - * less-than signs and ampersands are replaced with HTML entities. - * - * @param s The string to escape - * @return The same string escaped. - */ - public static String escapeForHTMLJavascript(String s) { - return escapeForHTML(StringEscapeUtils.escapeJavaScript(s)); - } - - /** - * Escapes a string so that it can be placed inside XML/HTML attribute. - * Replaces ampersand, less-than, greater-than, single-quote, and - * double-quote with escaped versions. - * @param s The string to escape - * @return The same string escaped. - */ - public static String escapeForMarkupAttribute(String s) { - return StringEscapeUtils.escapeXml(s); - } - - /** - * Minimally escapes a string so that it can be placed inside XML/HTML - * attribute. - * Escapes lt and amp. - * @param s The string to escape - * @return The same string escaped. - */ - public static String escapeForHTML(String s) { - // TODO: do this in a single pass instead of creating 5 junk strings - String escaped = s.replaceAll("&","&"); - return escaped.replaceAll("<","<"); - } - - /** - * Utility method for writing a (potentially large) String to a JspWriter, - * escaping it for HTML display, without constructing another large String - * of the whole content. - * @param s String to write - * @param out destination JspWriter - * @throws IOException - */ - public static void writeEscapedForHTML(String s, Writer w) - throws IOException { - PrintWriter out = new PrintWriter(w); - BufferedReader reader = new BufferedReader(new StringReader(s)); - String line; - while((line=reader.readLine()) != null){ - out.println(StringEscapeUtils.escapeHtml(line)); - } - } - - /** - * Replaces HTML Entity Encodings. - * @param cs The CharSequence to remove html codes from - * @return the same CharSequence or an escaped String. - */ - public static CharSequence unescapeHtml(final CharSequence cs) { - if (cs == null) { - return cs; - } - - return StringEscapeUtils.unescapeHtml(cs.toString()); - } - - /** - * @param message Message to put at top of the string returned. May be - * null. - * @param e Exception to write into a string. - * @return Return formatted string made of passed message and stack trace - * of passed exception. - */ - public static String exceptionToString(String message, Throwable e) { - StringWriter sw = new StringWriter(); - if (message == null || message.length() == 0) { - sw.write(message); - sw.write("\n"); - } - e.printStackTrace(new PrintWriter(sw)); - return sw.toString(); - } - - /** - * Exception- and warning-free URL-escaping utility method. - * - * @param s String to escape - * @return URL-escaped string - */ - @SuppressWarnings("deprecation") - public static String urlEscape(String s) { - try { - return URLEncoder.encode(s,"UTF8"); - } catch (UnsupportedEncodingException e) { - // should be impossible; all JVMs must support UTF8 - // but have a fallback just in case - return URLEncoder.encode(s); - } - } - - /** - * Exception- and warning-free URL-unescaping utility method. - * - * @param s String do unescape - * @return URL-unescaped String - */ - @SuppressWarnings("deprecation") - public static String urlUnescape(String s) { - try { - return URLDecoder.decode(s, "UTF8"); - } catch (UnsupportedEncodingException e) { - // should be impossible; all JVMs must support UTF8 - // but have a fallback just in case - return URLDecoder.decode(s); - } - } -} \ No newline at end of file diff --git a/commons/src/main/java/org/archive/util/UriUtils.java b/commons/src/main/java/org/archive/util/UriUtils.java index 37512102..5dfd2327 100644 --- a/commons/src/main/java/org/archive/util/UriUtils.java +++ b/commons/src/main/java/org/archive/util/UriUtils.java @@ -33,8 +33,8 @@ import java.util.regex.Matcher; import org.apache.commons.codec.DecoderException; import org.apache.commons.httpclient.URIException; -import org.archive.net.LaxURLCodec; import org.archive.net.UURI; +import org.archive.url.LaxURLCodec; /** diff --git a/commons/src/main/java/org/archive/util/iterator/LineReadingIterator.java b/commons/src/main/java/org/archive/util/iterator/LineReadingIterator.java deleted file mode 100644 index 41c634bb..00000000 --- a/commons/src/main/java/org/archive/util/iterator/LineReadingIterator.java +++ /dev/null @@ -1,59 +0,0 @@ -/* - * This file is part of the Heritrix web crawler (crawler.archive.org). - * - * Licensed to the Internet Archive (IA) by one or more individual - * contributors. - * - * The IA licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.archive.util.iterator; - -import java.io.BufferedReader; -import java.io.IOException; -import java.util.logging.Logger; - -/** - * Utility class providing an Iterator interface over line-oriented - * text input, as a thin wrapper over a BufferedReader. - * - * @author gojomo - */ -public class LineReadingIterator extends LookaheadIterator { - private static final Logger logger = - Logger.getLogger(LineReadingIterator.class.getName()); - - protected BufferedReader reader = null; - - public LineReadingIterator(BufferedReader r) { - reader = r; - } - - /** - * Loads next line into lookahead spot - * - * @return whether any item was loaded into next field - */ - protected boolean lookahead() { - try { - next = this.reader.readLine(); - if(next == null) { - // TODO: make this close-on-exhaust optional? - reader.close(); - } - return (next!=null); - } catch (IOException e) { - logger.warning(e.toString()); - return false; - } - } -} diff --git a/commons/src/main/java/org/archive/util/iterator/LookaheadIterator.java b/commons/src/main/java/org/archive/util/iterator/LookaheadIterator.java deleted file mode 100644 index 26ec768f..00000000 --- a/commons/src/main/java/org/archive/util/iterator/LookaheadIterator.java +++ /dev/null @@ -1,73 +0,0 @@ -/* - * This file is part of the Heritrix web crawler (crawler.archive.org). - * - * Licensed to the Internet Archive (IA) by one or more individual - * contributors. - * - * The IA licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.archive.util.iterator; - -import java.util.Iterator; -import java.util.NoSuchElementException; - -/** - * Superclass for Iterators which must probe ahead to know if - * a 'next' exists, and thus have a cached next between a call - * to hasNext() and next(). - * - * @author gojomo - * - */ -public abstract class LookaheadIterator implements Iterator { - protected T next; - - /** - * Test whether any items remain; loads next item into - * holding 'next' field. - * - * @see java.util.Iterator#hasNext() - */ - public boolean hasNext() { - return (this.next != null)? true: lookahead(); - } - - /** - * Caches the next item if available. - * - * @return true if there was a next item to cache, false otherwise - */ - protected abstract boolean lookahead(); - - /** - * Return the next item. - * - * @see java.util.Iterator#next() - */ - public T next() { - if (!hasNext()) { - throw new NoSuchElementException(); - } - // 'next' is guaranteed non-null by a hasNext() which returned true - T returnObj = this.next; - this.next = null; - return returnObj; - } - - /* (non-Javadoc) - * @see java.util.Iterator#remove() - */ - public void remove() { - throw new UnsupportedOperationException(); - } -} diff --git a/commons/src/main/java/org/archive/util/iterator/RegexLineIterator.java b/commons/src/main/java/org/archive/util/iterator/RegexLineIterator.java deleted file mode 100644 index b6200780..00000000 --- a/commons/src/main/java/org/archive/util/iterator/RegexLineIterator.java +++ /dev/null @@ -1,91 +0,0 @@ -/* - * This file is part of the Heritrix web crawler (crawler.archive.org). - * - * Licensed to the Internet Archive (IA) by one or more individual - * contributors. - * - * The IA licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.archive.util.iterator; - -import java.util.Iterator; -import java.util.logging.Logger; -import java.util.regex.Matcher; -import java.util.regex.Pattern; - -/** - * Utility class providing an Iterator interface over line-oriented - * text input. By providing regexs indicating lines to ignore - * (such as pure whitespace or comments), lines to consider input, and - * what to return from the input lines (such as a whitespace-trimmed - * non-whitespace token with optional trailing comment), this can - * be configured to handle a number of formats. - * - * The public static members provide pattern configurations that will - * be helpful in a wide variety of contexts. - * - * @author gojomo - */ -public class RegexLineIterator -extends TransformingIteratorWrapper { - private static final Logger logger = - Logger.getLogger(RegexLineIterator.class.getName()); - - public static final String COMMENT_LINE = "\\s*(#.*)?"; - public static final String NONWHITESPACE_ENTRY_TRAILING_COMMENT = - "^[\\s\ufeff]*(\\S+)\\s*(#.*)?$"; - public static final String TRIMMED_ENTRY_TRAILING_COMMENT = - "^\\s*([^#]+?)\\s*(#.*)?$"; - - public static final String ENTRY = "$1"; - - protected Matcher ignoreLine = null; - protected Matcher extractLine = null; - protected String outputTemplate = null; - - - public RegexLineIterator(Iterator inner, String ignore, - String extract, String replace) { - this.inner = inner; - ignoreLine = Pattern.compile(ignore).matcher(""); - extractLine = Pattern.compile(extract).matcher(""); - outputTemplate = replace; - } - - /** - * Loads next item into lookahead spot, if available. Skips - * lines matching ignoreLine; extracts desired portion of - * lines matching extractLine; informationally reports any - * lines matching neither. - * - * @return whether any item was loaded into next field - */ - protected String transform(String line) { - ignoreLine.reset(line); - if(ignoreLine.matches()) { - return null; - } - extractLine.reset(line); - if(extractLine.matches()) { - StringBuffer output = new StringBuffer(); - // TODO: consider if a loop that find()s all is more - // generally useful here - extractLine.appendReplacement(output,outputTemplate); - return output.toString(); - } - // no match; possibly error - logger.warning("line not extracted nor no-op: "+line); - return null; - } -} diff --git a/commons/src/main/java/org/archive/util/iterator/TransformingIteratorWrapper.java b/commons/src/main/java/org/archive/util/iterator/TransformingIteratorWrapper.java deleted file mode 100644 index 1b8bcab5..00000000 --- a/commons/src/main/java/org/archive/util/iterator/TransformingIteratorWrapper.java +++ /dev/null @@ -1,65 +0,0 @@ -/* - * This file is part of the Heritrix web crawler (crawler.archive.org). - * - * Licensed to the Internet Archive (IA) by one or more individual - * contributors. - * - * The IA licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.archive.util.iterator; - -import java.util.Iterator; - -/** - * Superclass for Iterators which transform and/or filter results - * from a wrapped Iterator. Because transform() has the option of - * discarding an item from the inner Iterator (by returning null), - * this is a kind of LookaheadIterator. - * - * @author gojomo - */ -public abstract class TransformingIteratorWrapper -extends LookaheadIterator { - protected Iterator inner; - - - /* (non-Javadoc) - * @see org.archive.util.iterator.LookaheadIterator#lookahead() - */ - protected boolean lookahead() { - assert next == null : "looking ahead when next is already loaded"; - while(inner.hasNext()) { - next = transform(inner.next()); - if(next!=null) { - return true; - } - } - noteExhausted(); - return false; - } - - /** - * Any cleanup to occur when hasNext() is about to return false - */ - protected void noteExhausted() { - // by default, do nothing - - } - - /** - * @param object Object to transform. - * @return Transfomed object. - */ - protected abstract Transformed transform(Original object); - -} diff --git a/commons/src/main/java/org/archive/util/zip/OpenJDK7GZIPInputStream.java b/commons/src/main/java/org/archive/util/zip/OpenJDK7GZIPInputStream.java deleted file mode 100644 index 044597f0..00000000 --- a/commons/src/main/java/org/archive/util/zip/OpenJDK7GZIPInputStream.java +++ /dev/null @@ -1,295 +0,0 @@ -/* - * Copyright (c) 1996, 2010, Oracle and/or its affiliates. All rights reserved. - * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. - * - * This code is free software; you can redistribute it and/or modify it - * under the terms of the GNU General Public License version 2 only, as - * published by the Free Software Foundation. Oracle designates this - * particular file as subject to the "Classpath" exception as provided - * by Oracle in the LICENSE file that accompanied this code. - * - * This code is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License - * version 2 for more details (a copy is included in the LICENSE file that - * accompanied this code). - * - * You should have received a copy of the GNU General Public License version - * 2 along with this work; if not, write to the Free Software Foundation, - * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. - * - * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA - * or visit www.oracle.com if you need additional information or have any - * questions. - */ - -// ALL INTERNETARCHIVE CHANGES INCLUDE A COMMENT STARTING "// IA " - -package org.archive.util.zip; - -import java.io.SequenceInputStream; -import java.io.ByteArrayInputStream; -import java.io.InputStream; -import java.io.IOException; -import java.io.EOFException; -import java.util.zip.CRC32; // IA ADDED IMPORT -import java.util.zip.CheckedInputStream; // IA ADDED IMPORT -import java.util.zip.Inflater; // IA ADDED IMPORT -import java.util.zip.ZipException; // IA ADDED IMPORT - -/** - * This class implements a stream filter for reading compressed data in - * the GZIP file format. - * - * @see InflaterInputStream - * @author David Connelly - * - */ -public -class OpenJDK7GZIPInputStream extends OpenJDK7InflaterInputStream { // IA RENAMINGS - /** - * CRC-32 for uncompressed data. - */ - protected CRC32 crc = new CRC32(); - - /** - * Indicates end of input stream. - */ - protected boolean eos; - - private boolean closed = false; - - /** - * Check to make sure that this stream has not been closed - */ - private void ensureOpen() throws IOException { - if (closed) { - throw new IOException("Stream closed"); - } - } - - /** - * Creates a new input stream with the specified buffer size. - * @param in the input stream - * @param size the input buffer size - * - * @exception ZipException if a GZIP format error has occurred or the - * compression method used is unsupported - * @exception IOException if an I/O error has occurred - * @exception IllegalArgumentException if size is <= 0 - */ - public OpenJDK7GZIPInputStream(InputStream in, int size) throws IOException { - super(in, new Inflater(true), size); - usesDefaultInflater = true; - readHeader(in); - } - - /** - * Creates a new input stream with a default buffer size. - * @param in the input stream - * - * @exception ZipException if a GZIP format error has occurred or the - * compression method used is unsupported - * @exception IOException if an I/O error has occurred - */ - public OpenJDK7GZIPInputStream(InputStream in) throws IOException { - this(in, 512); - } - - /** - * Reads uncompressed data into an array of bytes. If len is not - * zero, the method will block until some input can be decompressed; otherwise, - * no bytes are read and 0 is returned. - * @param buf the buffer into which the data is read - * @param off the start offset in the destination array b - * @param len the maximum number of bytes read - * @return the actual number of bytes read, or -1 if the end of the - * compressed input stream is reached - * - * @exception NullPointerException If buf is null. - * @exception IndexOutOfBoundsException If off is negative, - * len is negative, or len is greater than - * buf.length - off - * @exception ZipException if the compressed input data is corrupt. - * @exception IOException if an I/O error has occurred. - * - */ - public int read(byte[] buf, int off, int len) throws IOException { - ensureOpen(); - if (eos) { - return -1; - } - int n = super.read(buf, off, len); - if (n == -1) { - if (readTrailer()) - eos = true; - else - return this.read(buf, off, len); - } else { - crc.update(buf, off, n); - } - return n; - } - - /** - * Closes this input stream and releases any system resources associated - * with the stream. - * @exception IOException if an I/O error has occurred - */ - public void close() throws IOException { - if (!closed) { - super.close(); - eos = true; - closed = true; - } - } - - /** - * GZIP header magic number. - */ - public final static int GZIP_MAGIC = 0x8b1f; - - /* - * File header flags. - */ - protected final static int FTEXT = 1; // Extra text // IA VISIBILITY CHANGE FOR SUBCLASS USE - protected final static int FHCRC = 2; // Header CRC // IA VISIBILITY CHANGE FOR SUBCLASS USE - protected final static int FEXTRA = 4; // Extra field // IA VISIBILITY CHANGE FOR SUBCLASS USE - protected final static int FNAME = 8; // File name // IA VISIBILITY CHANGE FOR SUBCLASS USE - protected final static int FCOMMENT = 16; // File comment // IA VISIBILITY CHANGE FOR SUBCLASS USE - - /* - * Reads GZIP member header and returns the total byte number - * of this member header. - */ - protected int readHeader(InputStream this_in) throws IOException { // IA VISIBILITY CHANGE FOR OVERRIDING - CheckedInputStream in = new CheckedInputStream(this_in, crc); - crc.reset(); - // Check header magic - if (readUShort(in) != GZIP_MAGIC) { - throw new ZipException("Not in GZIP format"); - } - // Check compression method - if (readUByte(in) != 8) { - throw new ZipException("Unsupported compression method"); - } - // Read flags - int flg = readUByte(in); - // Skip MTIME, XFL, and OS fields - skipBytes(in, 6); - int n = 2 + 2 + 6; - // Skip optional extra field - if ((flg & FEXTRA) == FEXTRA) { - int m = readUShort(in); - skipBytes(in, m); - n += m + 2; - } - // Skip optional file name - if ((flg & FNAME) == FNAME) { - do { - n++; - } while (readUByte(in) != 0); - } - // Skip optional file comment - if ((flg & FCOMMENT) == FCOMMENT) { - do { - n++; - } while (readUByte(in) != 0); - } - // Check optional header CRC - if ((flg & FHCRC) == FHCRC) { - int v = (int)crc.getValue() & 0xffff; - if (readUShort(in) != v) { - throw new ZipException("Corrupt GZIP header"); - } - n += 2; - } - crc.reset(); - return n; - } - - /* - * Reads GZIP member trailer and returns true if the eos - * reached, false if there are more (concatenated gzip - * data set) - */ - protected boolean readTrailer() throws IOException { // IA CHANGE VISIBILITY FOR OVERRIDING - InputStream in = this.in; - int n = inf.getRemaining(); - if (n > 0) { - in = new SequenceInputStream( - new ByteArrayInputStream(buf, len - n, n), in); - } - // Uses left-to-right evaluation order - if ((readUInt(in) != crc.getValue()) || - // rfc1952; ISIZE is the input size modulo 2^32 - (readUInt(in) != (inf.getBytesWritten() & 0xffffffffL))) - throw new ZipException("Corrupt GZIP trailer"); - - // If there are more bytes available in "in" or - // the leftover in the "inf" is > 26 bytes: - // this.trailer(8) + next.header.min(10) + next.trailer(8) - // try concatenated case - if (this.in.available() > 0 || n > 26) { - int m = 8; // this.trailer - try { - m += readHeader(in); // next.header - } catch (IOException ze) { - return true; // ignore any malformed, do nothing - } - inf.reset(); - if (n > m) - inf.setInput(buf, len - n + m, n - m); - return false; - } - return true; - } - - /* - * Reads unsigned integer in Intel byte order. - */ - protected long readUInt(InputStream in) throws IOException { // IA VISIBILITY CHANGE FOR SUBCLASS USE - long s = readUShort(in); - return ((long)readUShort(in) << 16) | s; - } - - /* - * Reads unsigned short in Intel byte order. - */ - protected int readUShort(InputStream in) throws IOException { // IA VISIBILITY CHANGE FOR SUBCLASS USE - int b = readUByte(in); - return ((int)readUByte(in) << 8) | b; - } - - /* - * Reads unsigned byte. - */ - protected int readUByte(InputStream in) throws IOException { // IA VISIBILITY CHANGE FOR SUBCLASS USE - int b = in.read(); - if (b == -1) { - throw new EOFException(); - } - if (b < -1 || b > 255) { - // Report on this.in, not argument in; see read{Header, Trailer}. - throw new IOException(this.in.getClass().getName() - + ".read() returned value out of range -1..255: " + b); - } - return b; - } - - private byte[] tmpbuf = new byte[128]; - - /* - * Skips bytes of input data blocking until all bytes are skipped. - * Does not assume that the input stream is capable of seeking. - */ - protected void skipBytes(InputStream in, int n) throws IOException { // IA VISIBILITY CHANGE FOR SUBCLASS USE - while (n > 0) { - int len = in.read(tmpbuf, 0, n < tmpbuf.length ? n : tmpbuf.length); - if (len == -1) { - throw new EOFException(); - } - n -= len; - } - } -} diff --git a/commons/src/main/java/org/archive/util/zip/OpenJDK7InflaterInputStream.java b/commons/src/main/java/org/archive/util/zip/OpenJDK7InflaterInputStream.java deleted file mode 100644 index 42e9ea18..00000000 --- a/commons/src/main/java/org/archive/util/zip/OpenJDK7InflaterInputStream.java +++ /dev/null @@ -1,291 +0,0 @@ -/* - * Copyright (c) 1996, 2006, Oracle and/or its affiliates. All rights reserved. - * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. - * - * This code is free software; you can redistribute it and/or modify it - * under the terms of the GNU General Public License version 2 only, as - * published by the Free Software Foundation. Oracle designates this - * particular file as subject to the "Classpath" exception as provided - * by Oracle in the LICENSE file that accompanied this code. - * - * This code is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License - * version 2 for more details (a copy is included in the LICENSE file that - * accompanied this code). - * - * You should have received a copy of the GNU General Public License version - * 2 along with this work; if not, write to the Free Software Foundation, - * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. - * - * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA - * or visit www.oracle.com if you need additional information or have any - * questions. - */ - -package org.archive.util.zip; - -import java.io.FilterInputStream; -import java.io.InputStream; -import java.io.IOException; -import java.io.EOFException; -import java.util.zip.DataFormatException; // IA ADDED IMPORT -import java.util.zip.Inflater; // IA ADDED IMPORT -import java.util.zip.ZipException; // IA ADDED IMPORT - -/** - * This class implements a stream filter for uncompressing data in the - * "deflate" compression format. It is also used as the basis for other - * decompression filters, such as GZIPInputStream. - * - * @see Inflater - * @author David Connelly - */ -public -class OpenJDK7InflaterInputStream extends FilterInputStream { // IA RENAMED CLASS - /** - * Decompressor for this stream. - */ - protected Inflater inf; - - /** - * Input buffer for decompression. - */ - protected byte[] buf; - - /** - * Length of input buffer. - */ - protected int len; - - private boolean closed = false; - // this flag is set to true after EOF has reached - private boolean reachEOF = false; - - /** - * Check to make sure that this stream has not been closed - */ - private void ensureOpen() throws IOException { - if (closed) { - throw new IOException("Stream closed"); - } - } - - - /** - * Creates a new input stream with the specified decompressor and - * buffer size. - * @param in the input stream - * @param inf the decompressor ("inflater") - * @param size the input buffer size - * @exception IllegalArgumentException if size is <= 0 - */ - public OpenJDK7InflaterInputStream(InputStream in, Inflater inf, int size) { - super(in); - if (in == null || inf == null) { - throw new NullPointerException(); - } else if (size <= 0) { - throw new IllegalArgumentException("buffer size <= 0"); - } - this.inf = inf; - buf = new byte[size]; - } - - /** - * Creates a new input stream with the specified decompressor and a - * default buffer size. - * @param in the input stream - * @param inf the decompressor ("inflater") - */ - public OpenJDK7InflaterInputStream(InputStream in, Inflater inf) { - this(in, inf, 512); - } - - protected boolean usesDefaultInflater = false; - - /** - * Creates a new input stream with a default decompressor and buffer size. - * @param in the input stream - */ - public OpenJDK7InflaterInputStream(InputStream in) { - this(in, new Inflater()); - usesDefaultInflater = true; - } - - private byte[] singleByteBuf = new byte[1]; - - /** - * Reads a byte of uncompressed data. This method will block until - * enough input is available for decompression. - * @return the byte read, or -1 if end of compressed input is reached - * @exception IOException if an I/O error has occurred - */ - public int read() throws IOException { - ensureOpen(); - return read(singleByteBuf, 0, 1) == -1 ? -1 : singleByteBuf[0] & 0xff; - } - - /** - * Reads uncompressed data into an array of bytes. If len is not - * zero, the method will block until some input can be decompressed; otherwise, - * no bytes are read and 0 is returned. - * @param b the buffer into which the data is read - * @param off the start offset in the destination array b - * @param len the maximum number of bytes read - * @return the actual number of bytes read, or -1 if the end of the - * compressed input is reached or a preset dictionary is needed - * @exception NullPointerException If b is null. - * @exception IndexOutOfBoundsException If off is negative, - * len is negative, or len is greater than - * b.length - off - * @exception ZipException if a ZIP format error has occurred - * @exception IOException if an I/O error has occurred - */ - public int read(byte[] b, int off, int len) throws IOException { - ensureOpen(); - if (b == null) { - throw new NullPointerException(); - } else if (off < 0 || len < 0 || len > b.length - off) { - throw new IndexOutOfBoundsException(); - } else if (len == 0) { - return 0; - } - try { - int n; - while ((n = inf.inflate(b, off, len)) == 0) { - if (inf.finished() || inf.needsDictionary()) { - reachEOF = true; - return -1; - } - if (inf.needsInput()) { - fill(); - } - } - return n; - } catch (DataFormatException e) { - String s = e.getMessage(); - throw new ZipException(s != null ? s : "Invalid ZLIB data format"); - } - } - - /** - * Returns 0 after EOF has been reached, otherwise always return 1. - *

- * Programs should not count on this method to return the actual number - * of bytes that could be read without blocking. - * - * @return 1 before EOF and 0 after EOF. - * @exception IOException if an I/O error occurs. - * - */ - public int available() throws IOException { - ensureOpen(); - if (reachEOF) { - return 0; - } else { - return 1; - } - } - - private byte[] b = new byte[512]; - - /** - * Skips specified number of bytes of uncompressed data. - * @param n the number of bytes to skip - * @return the actual number of bytes skipped. - * @exception IOException if an I/O error has occurred - * @exception IllegalArgumentException if n < 0 - */ - public long skip(long n) throws IOException { - if (n < 0) { - throw new IllegalArgumentException("negative skip length"); - } - ensureOpen(); - int max = (int)Math.min(n, Integer.MAX_VALUE); - int total = 0; - while (total < max) { - int len = max - total; - if (len > b.length) { - len = b.length; - } - len = read(b, 0, len); - if (len == -1) { - reachEOF = true; - break; - } - total += len; - } - return total; - } - - /** - * Closes this input stream and releases any system resources associated - * with the stream. - * @exception IOException if an I/O error has occurred - */ - public void close() throws IOException { - if (!closed) { - if (usesDefaultInflater) - inf.end(); - in.close(); - closed = true; - } - } - - /** - * Fills input buffer with more data to decompress. - * @exception IOException if an I/O error has occurred - */ - protected void fill() throws IOException { - ensureOpen(); - len = in.read(buf, 0, buf.length); - if (len == -1) { - throw new EOFException("Unexpected end of ZLIB input stream"); - } - inf.setInput(buf, 0, len); - } - - /** - * Tests if this input stream supports the mark and - * reset methods. The markSupported - * method of InflaterInputStream returns - * false. - * - * @return a boolean indicating if this stream type supports - * the mark and reset methods. - * @see java.io.InputStream#mark(int) - * @see java.io.InputStream#reset() - */ - public boolean markSupported() { - return false; - } - - /** - * Marks the current position in this input stream. - * - *

The mark method of InflaterInputStream - * does nothing. - * - * @param readlimit the maximum limit of bytes that can be read before - * the mark position becomes invalid. - * @see java.io.InputStream#reset() - */ - public synchronized void mark(int readlimit) { - } - - /** - * Repositions this stream to the position at the time the - * mark method was last called on this input stream. - * - *

The method reset for class - * InflaterInputStream does nothing except throw an - * IOException. - * - * @exception IOException if this method is invoked. - * @see java.io.InputStream#mark(int) - * @see java.io.IOException - */ - public synchronized void reset() throws IOException { - throw new IOException("mark/reset not supported"); - } -} diff --git a/commons/src/test/java/org/archive/net/UURIFactoryTest.java b/commons/src/test/java/org/archive/net/UURIFactoryTest.java deleted file mode 100644 index b0827352..00000000 --- a/commons/src/test/java/org/archive/net/UURIFactoryTest.java +++ /dev/null @@ -1,1200 +0,0 @@ -/* - * This file is part of the Heritrix web crawler (crawler.archive.org). - * - * Licensed to the Internet Archive (IA) by one or more individual - * contributors. - * - * The IA licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.archive.net; - -import java.util.Iterator; -import java.util.TreeMap; - -import junit.framework.TestCase; - -import org.apache.commons.httpclient.URIException; -import org.apache.commons.lang.SerializationUtils; - -/** - * Test UURIFactory for proper UURI creation across variety of - * important/tricky cases. - * - * Be careful writing this file. Make sure you write it with UTF-8 encoding. - * - * @author igor stack gojomo - */ -public class UURIFactoryTest extends TestCase { - - public final void testEscaping() throws URIException { - // Note: single quote is not being escaped by URI class. - final String ESCAPED_URISTR = "http://archive.org/" + - UURIFactory.ESCAPED_SPACE + - UURIFactory.ESCAPED_SPACE + - UURIFactory.ESCAPED_CIRCUMFLEX + - UURIFactory.ESCAPED_QUOT + - UURIFactory.SQUOT + - UURIFactory.ESCAPED_APOSTROPH + - UURIFactory.ESCAPED_LSQRBRACKET + - UURIFactory.ESCAPED_RSQRBRACKET + - UURIFactory.ESCAPED_LCURBRACKET + - UURIFactory.ESCAPED_RCURBRACKET + - UURIFactory.SLASH + "a.gif"; // NBSP and SPACE should be trimmed; - - final String URISTR = "http://archive.org/.././" + "\u00A0" + - UURIFactory.SPACE + UURIFactory.CIRCUMFLEX + - UURIFactory.QUOT + UURIFactory.SQUOT + - UURIFactory.APOSTROPH + UURIFactory.LSQRBRACKET + - UURIFactory.RSQRBRACKET + UURIFactory.LCURBRACKET + - UURIFactory.RCURBRACKET + UURIFactory.BACKSLASH + - "test/../a.gif" + "\u00A0" + UURIFactory.SPACE; - - UURI uuri = UURIFactory.getInstance(URISTR); - final String uuriStr = uuri.toString(); - assertEquals("expected escaping", ESCAPED_URISTR, uuriStr); - } - - public final void testUnderscoreMakesPortParseFail() throws URIException { - UURI uuri = UURIFactory.getInstance("http://one-two_three:8080/index.html"); - int port = uuri.getPort(); - assertTrue("Failed find of port " + uuri, port == 8080); - } - - public final void testRelativeURIWithTwoSlashes() throws URIException { - UURI base = UURIFactory.getInstance("http://www.archive.org"); - UURI uuri = UURIFactory.getInstance(base, "one//index.html"); - assertTrue("Doesn't do right thing with two slashes " + uuri, - uuri.toString().equals( - "http://www.archive.org/one//index.html")); - } - - public final void testSchemelessURI() throws URIException { - UURI base = UURIFactory.getInstance("https://www.archive.org"); - UURI uuri = UURIFactory.getInstance(base, "//example.com/monkey?this:uri:has:colons"); - assertTrue("Doesn't do right thing with a schemeless URI " + uuri, - uuri.toString().equals( - "https://example.com/monkey?this:uri:has:colons")); - } - - public final void testTrailingEncodedSpace() throws URIException { - UURI uuri = UURIFactory.getInstance("http://www.nps-shoes.co.uk%20"); - assertTrue("Doesn't strip trailing encoded space 1 " + uuri, - uuri.toString().equals("http://www.nps-shoes.co.uk/")); - uuri = UURIFactory.getInstance("http://www.nps-shoes.co.uk%20%20%20"); - assertTrue("Doesn't strip trailing encoded space 2 " + uuri, - uuri.toString().equals("http://www.nps-shoes.co.uk/")); - } - - public final void testPort0080is80() throws URIException { - UURI uuri = UURIFactory.getInstance("http://archive.org:0080"); - assertTrue("Doesn't strip leading zeros " + uuri, - uuri.toString().equals("http://archive.org/")); - } - -// DISABLING TEST AS PRECURSOR TO ELIMINATION -// the problematic input given -- specifically the "%6s" incomplete uri-escape, -// shouldn't necessarily be rejected as a bad URI. IE and Firefox, at least, -// will attempt to fetch such an URL (getting, in this case against that ad -// server, a bad-request error). Ideally, we'd generate exactly the same -// request against the server as they do. However, with the most recent -// fixup for stray '%' signs, we come close, but not exactly. That's enough -// to cause this test to fail (it's not getting the expected exception) but -// our almost-URI, which might be what was intended, is better than trying -// nothing. -// public final void testBadPath() { -// String message = null; -// try { -// UURIFactory.getInstance("http://ads.as4x.tmcs.net/" + -// "html.ng/site=cs&pagepos=102&page=home&adsize=1x1&context=" + -// "generic&Params.richmedia=yes%26city%3Dseattle%26" + -// "rstid%3D2415%26market_id%3D86%26brand%3Dcitysearch" + -// "%6state%3DWA"); -// } catch (URIException e) { -// message = e.getMessage(); -// } -// assertNotNull("Didn't get expected exception.", message); -// } - - public final void testEscapeEncoding() throws URIException { - UURI uuri = UURIFactory.getInstance("http://www.y1y1.com/" + - "albums/userpics/11111/normal_%E3%E4%EC%EC%EC.jpg", "windows-1256"); - uuri.getPath(); - } - - public final void testTooLongAfterEscaping() { - StringBuffer buffer = new StringBuffer("http://www.archive.org/a/"); - // Append bunch of spaces. When escaped, they'll triple in size. - for (int i = 0; i < 1024; i++) { - buffer.append(" "); - } - buffer.append("/index.html"); - String message = null; - try { - UURIFactory.getInstance(buffer.toString()); - } catch (URIException e) { - message = e.getMessage(); - } - assertTrue("Wrong or no exception: " + message, (message != null) && - message.startsWith("Created (escaped) uuri >")); - } - - public final void testFtpUris() throws URIException { - final String FTP = "ftp"; - final String AUTHORITY = "pfbuser:pfbuser@mprsrv.agri.gov.cn"; - final String PATH = "/clzreceive/"; - final String uri = FTP + "://" + AUTHORITY + PATH; - UURI uuri = UURIFactory.getInstance(uri); - assertTrue("Failed to get matching scheme: " + uuri.getScheme(), - (uuri.getScheme()).equals(FTP)); - assertTrue("Failed to get matching authority: " + - uuri.getAuthority(), (uuri.getAuthority()).equals(AUTHORITY)); - assertTrue("Failed to get matching path: " + - uuri.getPath(), (uuri.getPath()).equals(PATH)); - } - - public final void testWhitespaceEscaped() throws URIException { - // Test that we get all whitespace even if the uri is - // already escaped. - String uri = "http://archive.org/index%25 .html"; - String tgtUri = "http://archive.org/index%25%20.html"; - UURI uuri = UURIFactory.getInstance(uri); - assertTrue("Not equal " + uuri.toString(), - uuri.toString().equals(tgtUri)); - uri = "http://archive.org/index%25\u001D.html"; - tgtUri = "http://archive.org/index%25%1D.html".toLowerCase(); - uuri = UURIFactory.getInstance(uri); - assertEquals("whitespace escaping", tgtUri, uuri.toString()); - uri = "http://gemini.info.usaid.gov/directory/" + - "pbResults.cfm?&urlNameLast=Rumplestiltskin"; - tgtUri = "http://gemini.info.usaid.gov/directory/faxResults.cfm?" + - "name=Ebenezer%20+Rumplestiltskin,&location=RRB%20%20%20%205%2E08%2D006"; - uuri = UURIFactory.getInstance(UURIFactory.getInstance(uri), - "faxResults.cfm?name=Ebenezer +Rumplestiltskin,&location=" + - "RRB%20%20%20%205%2E08%2D006"); - assertEquals("whitespace escaping", tgtUri, uuri.toString()); - } - -// public final void testFailedGetPath() throws URIException { -// final String path = "/RealMedia/ads/" + -// "click_lx.ads/%%PAGE%%/%%RAND%%/%%POS%%/%%CAMP%%/empty"; -// // decoding in getPath will interpret %CA as 8-bit escaped char, -// // possibly incomplete -// final String uri = "http://ads.nandomedia.com" + path; -// final UURI uuri = UURIFactory.getInstance(uri); -// String foundPath = uuri.getPath(); -// assertEquals("unexpected path", path, foundPath); -// } - - public final void testDnsHost() throws URIException { - String uri = "dns://ads.nandomedia.com:81/one.html"; - UURI uuri = UURIFactory.getInstance(uri); - String host = uuri.getReferencedHost(); - assertTrue("Host is wrong " + host, host.equals("ads.nandomedia.com")); - uri = "dns:ads.nandomedia.com"; - uuri = UURIFactory.getInstance(uri); - host = uuri.getReferencedHost(); - assertTrue("Host is wrong " + host, host.equals("ads.nandomedia.com")); - uri = "dns:ads.nandomedia.com?a=b"; - uuri = UURIFactory.getInstance(uri); - host = uuri.getReferencedHost(); - assertTrue("Host is wrong " + host, host.equals("ads.nandomedia.com")); - } - - public final void testPercentEscaping() throws URIException { - final String uri = "http://archive.org/%a%%%%%.html"; - // tests indicate firefox (1.0.6) does not encode '%' at all - final String tgtUri = "http://archive.org/%a%%%%%.html"; - UURI uuri = UURIFactory.getInstance(uri); - assertEquals("Not equal",tgtUri, uuri.toString()); - } - - public final void testRelativeDblPathSlashes() throws URIException { - UURI base = UURIFactory.getInstance("http://www.archive.org/index.html"); - UURI uuri = UURIFactory.getInstance(base, "JIGOU//KYC//INDEX.HTM"); - assertTrue("Double slash not working " + uuri.toString(), - uuri.getPath().equals("/JIGOU//KYC//INDEX.HTM")); - } - - public final void testRelativeWithScheme() throws URIException { - UURI base = UURIFactory.getInstance("http://www.example.com/some/page"); - UURI uuri = UURIFactory.getInstance(base, "http:boo"); - assertTrue("Relative with scheme not working " + uuri.toString(), - uuri.toString().equals("http://www.example.com/some/boo")); - } - - public final void testBadBaseResolve() throws URIException { - UURI base = UURIFactory.getInstance("http://license.joins.com/board/" + - "etc_board_list.asp?board_name=new_main&b_type=&nPage=" + - "2&category=G&lic_id=70&site=changeup&g_page=changeup&g_sPage=" + - "notice&gate=02"); - UURIFactory.getInstance(base, "http://www.changeup.com/...[ 1010966 ] crawl.log has URIs with spaces in them. - * @throws URIException - */ - public final void testSpaceDoubleEncoding() throws URIException { - final String uri = "http://www.brook.edu/i.html? %20taxonomy=Politics"; - final String encodedUri = - "http://www.brook.edu/i.html?%20%20taxonomy=Politics"; - UURI uuri = UURIFactory.getInstance(uri, "ISO-8859-1"); - assertTrue("Not equal " + uuri.toString(), - uuri.toString().equals(encodedUri)); - } - - /** - * Test for doubly-encoded sequences. - * See [ 966219 ] UURI doubly-encodes %XX sequences. - * @throws URIException - */ - public final void testDoubleEncoding() throws URIException { - final char ae = '\u00E6'; - final String uri = "http://archive.org/DIR WITH SPACES/home" + - ae + ".html"; - final String encodedUri = - "http://archive.org/DIR%20WITH%20SPACES/home%E6.html"; - UURI uuri = UURIFactory.getInstance(uri, "ISO-8859-1"); - assertEquals("single encoding", encodedUri, uuri.toString()); - // Dbl-encodes. - uuri = UURIFactory.getInstance(uuri.toString(), "ISO-8859-1"); - uuri = UURIFactory.getInstance(uuri.toString(), "ISO-8859-1"); - assertEquals("double encoding", encodedUri, uuri.toString()); - // Do default utf-8 test. - uuri = UURIFactory.getInstance(uri); - final String encodedUtf8Uri = - "http://archive.org/DIR%20WITH%20SPACES/home%C3%A6.html"; - assertEquals("Not equal utf8", encodedUtf8Uri, uuri.toString()); - // Now dbl-encode. - uuri = UURIFactory.getInstance(uuri.toString()); - uuri = UURIFactory.getInstance(uuri.toString()); - assertEquals("Not equal (dbl-encoding) utf8", encodedUtf8Uri, uuri.toString()); - } - - /** - * Test for syntax errors stop page parsing. - * @see [ 788219 ] URI Syntax Errors stop page parsing - * @throws URIException - */ - public final void testThreeSlashes() throws URIException { - UURI goodURI = UURIFactory. - getInstance("http://lcweb.loc.gov/rr/goodtwo.html"); - String uuri = "http:///lcweb.loc.gov/rr/goodtwo.html"; - UURI rewrittenURI = UURIFactory.getInstance(uuri); - assertTrue("Not equal " + goodURI + ", " + uuri, - goodURI.toString().equals(rewrittenURI.toString())); - uuri = "http:////lcweb.loc.gov/rr/goodtwo.html"; - rewrittenURI = UURIFactory.getInstance(uuri); - assertTrue("Not equal " + goodURI + ", " + uuri, - goodURI.toString().equals(rewrittenURI.toString())); - // Check https. - goodURI = UURIFactory. - getInstance("https://lcweb.loc.gov/rr/goodtwo.html"); - uuri = "https:////lcweb.loc.gov/rr/goodtwo.html"; - rewrittenURI = UURIFactory.getInstance(uuri); - assertTrue("Not equal " + goodURI + ", " + uuri, - goodURI.toString().equals(rewrittenURI.toString())); - } - - public final void testNoScheme() { - boolean expectedException = false; - String uuri = "www.loc.gov/rr/european/egw/polishex.html"; - try { - UURIFactory.getInstance(uuri); - } catch (URIException e) { - // Expected exception. - expectedException = true; - } - assertTrue("Didn't get expected exception: " + uuri, - expectedException); - } - - public final void testRelative() throws URIException { - UURI uuriTgt = UURIFactory. - getInstance("http://archive.org:83/home.html"); - UURI uri = UURIFactory. - getInstance("http://archive.org:83/one/two/three.html"); - UURI uuri = UURIFactory. - getInstance(uri, "/home.html"); - assertTrue("Not equal", - uuriTgt.toString().equals(uuri.toString())); - } - - public void testSchemelessRelative() throws URIException { - UURI base = UURIFactory.getInstance("http://www.itsnicethat.com/articles/laura-hobson"); - UURI test1 = UURIFactory.getInstance(base, "//www.facebook.com/plugins/like.php"); - assertEquals("schemaless relative 1", "http://www.facebook.com/plugins/like.php", test1.toString()); - // reported by Erin Staniland - UURI test2 = UURIFactory.getInstance(base, "//www.facebook.com/plugins/like.php?href=http://www.itsnicethat.com/articles/laura-hobson"); - assertEquals("schemeless relative 2", "http://www.facebook.com/plugins/like.php?href=http://www.itsnicethat.com/articles/laura-hobson", - test2.toString()); - } - - /** - * Test that an empty uuri does the right thing -- that we get back the - * base. - * - * @throws URIException - */ - public final void testRelativeEmpty() throws URIException { - UURI uuriTgt = UURIFactory. - getInstance("http://archive.org:83/one/two/three.html"); - UURI uri = UURIFactory. - getInstance("http://archive.org:83/one/two/three.html"); - UURI uuri = UURIFactory. - getInstance(uri, ""); - assertTrue("Empty length don't work", - uuriTgt.toString().equals(uuri.toString())); - } - - public final void testAbsolute() throws URIException { - UURI uuriTgt = UURIFactory. - getInstance("http://archive.org:83/home.html"); - UURI uri = UURIFactory. - getInstance("http://archive.org:83/one/two/three.html"); - UURI uuri = UURIFactory. - getInstance(uri, "http://archive.org:83/home.html"); - assertTrue("Not equal", - uuriTgt.toString().equals(uuri.toString())); - } - - /** - * Test for [ 962892 ] UURI accepting/creating unUsable URIs (bad hosts). - * @see [ 962892 ] UURI accepting/creating unUsable URIs (bad hosts) - */ - public final void testHostWithLessThan() { - checkExceptionOnIllegalDomainlabel("http://www.betamobile.com[ 1012520 ] UURI.length() > 2k - */ - public final void test2kURI() throws URIException { - final StringBuffer buffer = new StringBuffer("http://a.b"); - final String subPath = "/123456789"; - for (int i = 0; i < 207; i++) { - buffer.append(subPath); - } - // String should be 2080 characters long. Legal. - UURIFactory.getInstance(buffer.toString()); - boolean gotException = false; - // Add ten more characters and make size illegal. - buffer.append(subPath); - try { - UURIFactory.getInstance(buffer.toString()); - } catch (URIException e) { - gotException = true; - } - assertTrue("No expected exception complaining about long URI", - gotException); - } - - private void checkExceptionOnIllegalDomainlabel(String uuri) { - boolean expectedException = false; - try { - UURIFactory.getInstance(uuri); - } catch (URIException e) { - // Expected exception. - expectedException = true; - } - assertTrue("Didn't get expected exception: " + uuri, - expectedException); - } - - /** - * Test for doing separate DNS lookup for same host - * - * @see [ 788277 ] Doing separate DNS lookup for same host - * @throws URIException - */ - public final void testHostWithPeriod() throws URIException { - UURI uuri1 = UURIFactory. - getInstance("http://www.loc.gov./index.html"); - UURI uuri2 = UURIFactory. - getInstance("http://www.loc.gov/index.html"); - assertEquals("Failed equating hosts with dot", - uuri1.getHost(), uuri2.getHost()); - } - - /** - * Test for NPE in java.net.URI.encode - * - * @see [ 874220 ] NPE in java.net.URI.encode - * @throws URIException - */ - public final void testHostEncodedChars() throws URIException { - String s = "http://g.msn.co.kr/0nwkokr0/00/19??" + - "PS=10274&NC=10009&CE=42&CP=949&HL=" + - "���?��"; - assertNotNull("Encoded chars " + s, - UURIFactory.getInstance(s)); - } - - /** - * Test for java.net.URI parses %20 but getHost null - * - * See [ 927940 ] java.net.URI parses %20 but getHost null - */ - public final void testSpaceInHost() { - boolean expectedException = false; - try { - UURIFactory.getInstance( - "http://www.local-regions.odpm%20.gov.uk" + - "/lpsa/challenge/pdf/propect.pdf"); - } catch (URIException e) { - expectedException = true; - } - assertTrue("Did not fail with escaped space.", expectedException); - - expectedException = false; - try { - UURIFactory.getInstance( - "http://www.local-regions.odpm .gov.uk" + - "/lpsa/challenge/pdf/propect.pdf"); - } catch (URIException e) { - expectedException = true; - } - assertTrue("Did not fail with real space.", expectedException); - } - - /** - * Test for java.net.URI chokes on hosts_with_underscores. - * - * @see [ 808270 ] java.net.URI chokes on hosts_with_underscores - * @throws URIException - */ - public final void testHostWithUnderscores() throws URIException { - UURI uuri = UURIFactory.getInstance( - "http://x_underscore_underscore.2u.com.tw/nonexistent_page.html"); - assertEquals("Failed get of host with underscore", - "x_underscore_underscore.2u.com.tw", uuri.getHost()); - } - - - /** - * Two dots for igor. - */ - public final void testTwoDots() { - boolean expectedException = false; - try { - UURIFactory.getInstance( - "http://x_underscore_underscore..2u.com/nonexistent_page.html"); - } catch (URIException e) { - expectedException = true; - } - assertTrue("Two dots did not throw exception", expectedException); - } - - /** - * Test for java.net.URI#getHost fails when leading digit. - * - * @see [ 910120 ] java.net.URI#getHost fails when leading digit. - * @throws URIException - */ - public final void testHostWithDigit() throws URIException { - UURI uuri = UURIFactory. - getInstance("http://0204chat.2u.com.tw/nonexistent_page.html"); - assertEquals("Failed get of host with digit", - "0204chat.2u.com.tw", uuri.getHost()); - } - - /** - * Test for Constraining java URI class. - * - * @see [ 949548 ] Constraining java URI class - */ - public final void testPort() { - checkBadPort("http://www.tyopaikat.com:a/robots.txt"); - checkBadPort("http://158.144.21.3:80808/robots.txt"); - checkBadPort("http://pdb.rutgers.edu:81.rutgers.edu/robots.txt"); - checkBadPort( - "https://webmail.gse.harvard.edu:9100robots.txt/robots.txt"); - checkBadPort( - "https://webmail.gse.harvard.edu:0/robots.txt/robots.txt"); - } - - /** - * Test bad port throws exception. - * @param uri URI with bad port to check. - */ - private void checkBadPort(String uri) { - boolean exception = false; - try { - UURIFactory.getInstance(uri); - } - catch (URIException e) { - exception = true; - } - assertTrue("Didn't throw exception: " + uri, exception); - } - - /** - * Preserve userinfo capitalization. - * @throws URIException - */ - public final void testUserinfo() throws URIException { - final String authority = "stack:StAcK@www.tyopaikat.com"; - final String uri = "http://" + authority + "/robots.txt"; - UURI uuri = UURIFactory.getInstance(uri); - assertEquals("Authority not equal", uuri.getAuthority(), - authority); - /* - String tmp = uuri.toString(); - assertTrue("URI not equal", tmp.equals(uri)); - */ - } - - /** - * Test user info + port - * @throws URIException - */ - public final void testUserinfoPlusPort() throws URIException { - final String userInfo = "stack:StAcK"; - final String authority = "www.tyopaikat.com"; - final int port = 8080; - final String uri = "http://" + userInfo + "@" + authority + ":" + port - + "/robots.txt"; - UURI uuri = UURIFactory.getInstance(uri); - assertEquals("Host not equal", authority,uuri.getHost()); - assertEquals("Userinfo Not equal",userInfo,uuri.getUserinfo()); - assertEquals("Port not equal",port,uuri.getPort()); - assertEquals("Authority wrong","stack:StAcK@www.tyopaikat.com:8080", - uuri.getAuthority()); - assertEquals("AuthorityMinusUserinfo wrong","www.tyopaikat.com:8080", - uuri.getAuthorityMinusUserinfo()); - - } - - public final void testRFC3986RelativeChange() throws URIException { - UURI base = UURIFactory.getInstance("http://a/b/c/d;p?q"); - tryRelative(base, "?y", "http://a/b/c/d;p?y"); - } - - /** - * Tests from rfc3986 - * - * 

-     *       "g:h"           =  "g:h"
-     *       "g"             =  "http://a/b/c/g"
-     *       "./g"           =  "http://a/b/c/g"
-     *       "g/"            =  "http://a/b/c/g/"
-     *       "/g"            =  "http://a/g"
-     *       "//g"           =  "http://g"
-     *       "?y"            =  "http://a/b/c/d;p?y"
-     *       "g?y"           =  "http://a/b/c/g?y"
-     *       "#s"            =  "http://a/b/c/d;p?q#s"
-     *       "g#s"           =  "http://a/b/c/g#s"
-     *       "g?y#s"         =  "http://a/b/c/g?y#s"
-     *       ";x"            =  "http://a/b/c/;x"
-     *       "g;x"           =  "http://a/b/c/g;x"
-     *       "g;x?y#s"       =  "http://a/b/c/g;x?y#s"
-     *       ""              =  "http://a/b/c/d;p?q"
-     *       "."             =  "http://a/b/c/"
-     *       "./"            =  "http://a/b/c/"
-     *       ".."            =  "http://a/b/"
-     *       "../"           =  "http://a/b/"
-     *       "../g"          =  "http://a/b/g"
-     *       "../.."         =  "http://a/"
-     *       "../../"        =  "http://a/"
-     *       "../../g"       =  "http://a/g"
-     *
- * - * @throws URIException - */ - public final void testRFC3986Relative() throws URIException { - UURI base = UURIFactory.getInstance("http://a/b/c/d;p?q"); - tryRelative(base, "g:h", "g:h"); - tryRelative(base, "g", "http://a/b/c/g"); - tryRelative(base, "./g", "http://a/b/c/g"); - tryRelative(base, "g/", "http://a/b/c/g/"); - tryRelative(base, "/g", "http://a/g"); - tryRelative(base, "//g", "http://g"); - tryRelative(base, "?y", "http://a/b/c/d;p?y"); - tryRelative(base, "g?y", "http://a/b/c/g?y"); - tryRelative(base, "#s", "http://a/b/c/d;p?q#s"); - tryRelative(base, "g#s", "http://a/b/c/g#s"); - tryRelative(base, "g?y#s", "http://a/b/c/g?y#s"); - tryRelative(base, ";x", "http://a/b/c/;x"); - tryRelative(base, "g;x", "http://a/b/c/g;x"); - tryRelative(base, "g;x?y#s","http://a/b/c/g;x?y#s"); - tryRelative(base, "", "http://a/b/c/d;p?q"); - tryRelative(base, ".", "http://a/b/c/"); - tryRelative(base, "./", "http://a/b/c/"); - tryRelative(base, "..", "http://a/b/"); - tryRelative(base, "../", "http://a/b/"); - tryRelative(base, "../g", "http://a/b/g"); - tryRelative(base, "../..", "http://a/"); - tryRelative(base, "../../", "http://a/"); - tryRelative(base, "../../g","http://a/g"); - } - - protected void tryRelative(UURI base, String relative, String expected) - throws URIException { - UURI uuri = UURIFactory.getInstance(base, relative); - assertEquals("Derelativized " + relative + " gave " - + uuri + " not " + expected, - UURIFactory.getInstance(expected),uuri); - } - - /** - * Tests from rfc2396 with amendments to accomodate differences - * intentionally added to make our URI handling like IEs. - * - * 
-	 *       g:h           =  g:h
-	 *       g             =  http://a/b/c/g
-	 *       ./g           =  http://a/b/c/g
-	 *       g/            =  http://a/b/c/g/
-	 *       /g            =  http://a/g
-	 *       //g           =  http://g
-	 *       ?y            =  http://a/b/c/?y
-	 *       g?y           =  http://a/b/c/g?y
-	 *       #s            =  (current document)#s
-	 *       g#s           =  http://a/b/c/g#s
-	 *       g?y#s         =  http://a/b/c/g?y#s
-	 *       ;x            =  http://a/b/c/;x
-	 *       g;x           =  http://a/b/c/g;x
-	 *       g;x?y#s       =  http://a/b/c/g;x?y#s
-	 *       .             =  http://a/b/c/
-	 *       ./            =  http://a/b/c/
-	 *       ..            =  http://a/b/
-	 *       ../           =  http://a/b/
-	 *       ../g          =  http://a/b/g
-	 *       ../..         =  http://a/
-	 *       ../../        =  http://a/
-	 *       ../../g       =  http://a/g
-	 *
- * - * @throws URIException - */ - public final void testRFC2396Relative() throws URIException { - UURI base = UURIFactory. - getInstance("http://a/b/c/d;p?q"); - TreeMap m = new TreeMap(); - m.put("..", "http://a/b/"); - m.put("../", "http://a/b/"); - m.put("../g", "http://a/b/g"); - m.put("../..", "http://a/"); - m.put("../../", "http://a/"); - m.put("../../g", "http://a/g"); - m.put("g#s", "http://a/b/c/g#s"); - m.put("g?y#s ", "http://a/b/c/g?y#s"); - m.put(";x", "http://a/b/c/;x"); - m.put("g;x", "http://a/b/c/g;x"); - m.put("g;x?y#s", "http://a/b/c/g;x?y#s"); - m.put(".", "http://a/b/c/"); - m.put("./", "http://a/b/c/"); - m.put("g", "http://a/b/c/g"); - m.put("./g", "http://a/b/c/g"); - m.put("g/", "http://a/b/c/g/"); - m.put("/g", "http://a/g"); - m.put("//g", "http://g"); - // CHANGED BY RFC3986 - // m.put("?y", "http://a/b/c/?y"); - m.put("g?y", "http://a/b/c/g?y"); - // EXTRAS beyond the RFC set. - // TODO: That these resolve to a path of /a/g might be wrong. Perhaps - // it should be '/g'?. - m.put("/../../../../../../../../g", "http://a/g"); - m.put("../../../../../../../../g", "http://a/g"); - m.put("../G", "http://a/b/G"); - for (Iterator i = m.keySet().iterator(); i.hasNext();) { - String key = (String)i.next(); - String value = (String)m.get(key); - UURI uuri = UURIFactory.getInstance(base, key); - assertTrue("Unexpected " + key + " " + value + " " + uuri, - uuri.equals(UURIFactory.getInstance(value))); - } - } - - /** - * A UURI should always be without a 'fragment' segment, which is - * unused and irrelevant for network fetches. - * - * See [ 970666 ] #anchor links not trimmed, and thus recrawled - * - * @throws URIException - */ - public final void testAnchors() throws URIException { - UURI uuri = UURIFactory. - getInstance("http://www.example.com/path?query#anchor"); - assertEquals("Not equal", "http://www.example.com/path?query", - uuri.toString()); - } - - - /** - * Ensure that URI strings beginning with a colon are treated - * the same as browsers do (as relative, rather than as absolute - * with zero-length scheme). - * - * @throws URIException - */ - public void testStartsWithColon() throws URIException { - UURI base = UURIFactory.getInstance("http://www.example.com/path/page"); - UURI uuri = UURIFactory.getInstance(base,":foo"); - assertEquals("derelativize starsWithColon", - uuri.getURI(), - "http://www.example.com/path/:foo"); - } - - /** - * Ensure that relative URIs with colons in late positions - * aren't mistakenly interpreted as absolute URIs with long, - * illegal schemes. - * - * @throws URIException - */ - public void testLateColon() throws URIException { - UURI base = UURIFactory.getInstance("http://www.example.com/path/page"); - UURI uuri1 = UURIFactory.getInstance(base,"example.html;jsessionid=deadbeef:deadbeed?parameter=this:value"); - assertEquals("derelativize lateColon", - uuri1.getURI(), - "http://www.example.com/path/example.html;jsessionid=deadbeef:deadbeed?parameter=this:value"); - UURI uuri2 = UURIFactory.getInstance(base,"example.html?parameter=this:value"); - assertEquals("derelativize lateColon", - uuri2.getURI(), - "http://www.example.com/path/example.html?parameter=this:value"); - } - - /** - * Ensure that stray trailing '%' characters do not prevent - * UURI instances from being created, and are reasonably - * escaped when encountered. - * - * @throws URIException - */ - public void testTrailingPercents() throws URIException { - String plainPath = "http://www.example.com/path%"; - UURI plainPathUuri = UURIFactory.getInstance(plainPath); - assertEquals("plainPath getURI", plainPath, plainPathUuri.getURI()); - assertEquals("plainPath getEscapedURI", - "http://www.example.com/path%", // browsers don't escape '%' - plainPathUuri.getEscapedURI()); - - String partiallyEscapedPath = "http://www.example.com/pa%20th%"; - UURI partiallyEscapedPathUuri = UURIFactory.getInstance( - partiallyEscapedPath); -// assertEquals("partiallyEscapedPath getURI", -// "http://www.example.com/pa th%", // TODO: is this desirable? -//// partiallyEscapedPath, -// partiallyEscapedPathUuri.getURI()); - assertEquals("partiallyEscapedPath getEscapedURI", - "http://www.example.com/pa%20th%", - partiallyEscapedPathUuri.getEscapedURI()); - - String plainQueryString = "http://www.example.com/path?q=foo%"; - UURI plainQueryStringUuri = UURIFactory.getInstance( - plainQueryString); -// assertEquals("plainQueryString getURI", -// plainQueryString, -// plainQueryStringUuri.getURI()); - assertEquals("plainQueryString getEscapedURI", - "http://www.example.com/path?q=foo%", - plainQueryStringUuri.getEscapedURI()); - - String partiallyEscapedQueryString = - "http://www.example.com/pa%20th?q=foo%"; - UURI partiallyEscapedQueryStringUuri = UURIFactory.getInstance( - partiallyEscapedQueryString); - assertEquals("partiallyEscapedQueryString getURI", - "http://www.example.com/pa th?q=foo%", - partiallyEscapedQueryStringUuri.getURI()); - assertEquals("partiallyEscapedQueryString getEscapedURI", - "http://www.example.com/pa%20th?q=foo%", - partiallyEscapedQueryStringUuri.getEscapedURI()); - } - - /** - * Ensure that stray '%' characters do not prevent - * UURI instances from being created, and are reasonably - * escaped when encountered. - * - * @throws URIException - */ - public void testStrayPercents() throws URIException { - String oneStray = "http://www.example.com/pa%th"; - UURI oneStrayUuri = UURIFactory.getInstance(oneStray); - assertEquals("oneStray getURI", oneStray, oneStrayUuri.getURI()); - assertEquals("oneStray getEscapedURI", - "http://www.example.com/pa%th", // browsers don't escape '%' - oneStrayUuri.getEscapedURI()); - - String precededByValidEscape = "http://www.example.com/pa%20th%way"; - UURI precededByValidEscapeUuri = UURIFactory.getInstance( - precededByValidEscape); - assertEquals("precededByValidEscape getURI", - "http://www.example.com/pa th%way", // getURI interprets escapes - precededByValidEscapeUuri.getURI()); - assertEquals("precededByValidEscape getEscapedURI", - "http://www.example.com/pa%20th%way", - precededByValidEscapeUuri.getEscapedURI()); - - String followedByValidEscape = "http://www.example.com/pa%th%20way"; - UURI followedByValidEscapeUuri = UURIFactory.getInstance( - followedByValidEscape); - assertEquals("followedByValidEscape getURI", - "http://www.example.com/pa%th way", // getURI interprets escapes - followedByValidEscapeUuri.getURI()); - assertEquals("followedByValidEscape getEscapedURI", - "http://www.example.com/pa%th%20way", - followedByValidEscapeUuri.getEscapedURI()); - } - - public void testEscapingNotNecessary() throws URIException { - String escapesUnnecessary = - "http://www.example.com/misc;reserved:chars@that&don't=need" - +"+escaping$even,though!you(might)initially?think#so"; - // expect everything but the #fragment - String expected = escapesUnnecessary.substring(0, escapesUnnecessary - .length() - 3); - assertEquals("escapes unnecessary", - expected, - UURIFactory.getInstance(escapesUnnecessary).toString()); - } - - public void testIdn() throws URIException { - // See http://www.josefsson.org/idn.php. - // http://räksmörgås.josefßon.org/ - String idn1 = "http://r\u00e4ksm\u00f6rg\u00e5s.josef\u00dfon.org/"; - String puny1 = "http://xn--rksmrgs-5wao1o.josefsson.org/"; - assertEquals("encoding of " + idn1, puny1, UURIFactory - .getInstance(idn1).toString()); - // http://www.pølse.dk/ - String idn2 = "http://www.p\u00f8lse.dk/"; - String puny2 = "http://www.xn--plse-gra.dk/"; - assertEquals("encoding of " + idn2, puny2, UURIFactory - .getInstance(idn2).toString()); - // http://例子.測試 - String idn3 = "http://\u4F8B\u5B50.\u6E2C\u8A66"; - String puny3 = "http://xn--fsqu00a.xn--g6w251d/"; - assertEquals("encoding of " + idn3, puny3, UURIFactory - .getInstance(idn3).toString()); - } - - public void testNewLineInURL() throws URIException { - UURI uuri = UURIFactory.getInstance("http://www.ar\rchive\n." + - "org/i\n\n\r\rndex.html"); - assertEquals("http://www.archive.org/index.html", uuri.toString()); - } - - public void testTabsInURL() throws URIException { - UURI uuri = UURIFactory.getInstance("http://www.ar\tchive\t." + - "org/i\t\r\n\tndex.html"); - assertEquals("http://www.archive.org/index.html", uuri.toString()); - } - - public void testQueryEscaping() throws URIException { - UURI uuri = UURIFactory.getInstance( - "http://www.yahoo.com/foo?somechars!@$%^&*()_-+={[}]|\'\";:/?.>,<"); - assertEquals( - // tests in FF1.5 indicate it only escapes " < > - "http://www.yahoo.com/foo?somechars!@$%^&*()_-+={[}]|\'%22;:/?.%3E,%3C", - uuri.toString()); - } - - /** - * Check that our 'normalization' does same as Nutch's - * Below before-and-afters were taken from the nutch urlnormalizer-basic - * TestBasicURLNormalizer class (December 2006, Nutch 0.9-dev). - * @throws URIException - */ - public void testSameAsNutchURLFilterBasic() throws URIException { - assertEquals(UURIFactory.getInstance(" http://foo.com/ ").toString(), - "http://foo.com/"); - - // check that protocol is lower cased - assertEquals(UURIFactory.getInstance("HTTP://foo.com/").toString(), - "http://foo.com/"); - - // check that host is lower cased - assertEquals(UURIFactory. - getInstance("http://Foo.Com/index.html").toString(), - "http://foo.com/index.html"); - assertEquals(UURIFactory. - getInstance("http://Foo.Com/index.html").toString(), - "http://foo.com/index.html"); - - // check that port number is normalized - assertEquals(UURIFactory. - getInstance("http://foo.com:80/index.html").toString(), - "http://foo.com/index.html"); - assertEquals(UURIFactory.getInstance("http://foo.com:81/").toString(), - "http://foo.com:81/"); - - // check that null path is normalized - assertEquals(UURIFactory.getInstance("http://foo.com").toString(), - "http://foo.com/"); - - // check that references are removed - assertEquals(UURIFactory. - getInstance("http://foo.com/foo.html#ref").toString(), - "http://foo.com/foo.html"); - - // // check that encoding is normalized - // normalizeTest("http://foo.com/%66oo.html", "http://foo.com/foo.html"); - - // check that unnecessary "../" are removed - assertEquals(UURIFactory. - getInstance("http://foo.com/aa/../").toString(), - "http://foo.com/" ); - assertEquals(UURIFactory. - getInstance("http://foo.com/aa/bb/../").toString(), - "http://foo.com/aa/"); - - /* We fail this one. Here we produce: 'http://foo.com/'. - assertEquals(UURIFactory. - getInstance("http://foo.com/aa/..").toString(), - "http://foo.com/aa/.."); - */ - - assertEquals(UURIFactory. - getInstance("http://foo.com/aa/bb/cc/../../foo.html").toString(), - "http://foo.com/aa/foo.html"); - assertEquals(UURIFactory. - getInstance("http://foo.com/aa/bb/../cc/dd/../ee/foo.html"). - toString(), - "http://foo.com/aa/cc/ee/foo.html"); - assertEquals(UURIFactory. - getInstance("http://foo.com/../foo.html").toString(), - "http://foo.com/foo.html" ); - assertEquals(UURIFactory. - getInstance("http://foo.com/../../foo.html").toString(), - "http://foo.com/foo.html" ); - assertEquals(UURIFactory. - getInstance("http://foo.com/../aa/../foo.html").toString(), - "http://foo.com/foo.html" ); - assertEquals(UURIFactory. - getInstance("http://foo.com/aa/../../foo.html").toString(), - "http://foo.com/foo.html" ); - assertEquals(UURIFactory. - getInstance("http://foo.com/aa/../bb/../foo.html/../../"). - toString(), - "http://foo.com/" ); - assertEquals(UURIFactory.getInstance("http://foo.com/../aa/foo.html"). - toString(), "http://foo.com/aa/foo.html" ); - assertEquals(UURIFactory. - getInstance("http://foo.com/../aa/../foo.html").toString(), - "http://foo.com/foo.html" ); - assertEquals(UURIFactory. - getInstance("http://foo.com/a..a/foo.html").toString(), - "http://foo.com/a..a/foo.html" ); - assertEquals(UURIFactory. - getInstance("http://foo.com/a..a/../foo.html").toString(), - "http://foo.com/foo.html" ); - assertEquals(UURIFactory. - getInstance("http://foo.com/foo.foo/../foo.html").toString(), - "http://foo.com/foo.html" ); - } - - public void testHttpSchemeColonSlash() { - boolean exception = false; - try { - UURIFactory.getInstance("https:/"); - } catch (URIException e) { - exception = true; - } - assertTrue("Didn't throw exception when one expected", exception); - exception = false; - try { - UURIFactory.getInstance("http://"); - } catch (URIException e) { - exception = true; - } - assertTrue("Didn't throw exception when one expected", exception); - } - - public void testNakedHttpsSchemeColon() { - boolean exception = false; - try { - UURIFactory.getInstance("https:"); - } catch (URIException e) { - exception = true; - } - assertTrue("Didn't throw exception when one expected", exception); - exception = false; - try { - UURI base = UURIFactory.getInstance("http://www.example.com"); - UURIFactory.getInstance(base, "https:"); - } catch (URIException e) { - exception = true; - } - assertTrue("Didn't throw exception when one expected", exception); - } - - /** - * Test motivated by [#HER-616] The UURI class may throw - * NullPointerException in getReferencedHost() - * - * @throws URIException - */ - public void testMissingHttpColon() throws URIException { - String suspectUri = "http//www.test.foo"; - UURI base = UURIFactory.getInstance("http://www.example.com"); - boolean exceptionThrown = false; - try { - UURI badUuri = UURIFactory.getInstance(suspectUri); - badUuri.getReferencedHost(); // not reached - } catch (URIException e) { - // should get relative-uri-no-base exception - exceptionThrown = true; - } finally { - assertTrue("expected exception not thrown",exceptionThrown); - } - UURI goodUuri = UURIFactory.getInstance(base,suspectUri); - goodUuri.getReferencedHost(); - } - - /** - * A UURI's string representation should be same after a - * serialization roundtrip. - * - * @throws URIException - */ - public final void testSerializationRoundtrip() throws URIException { - UURI uuri = UURIFactory. - getInstance("http://www.example.com/path?query#anchor"); - UURI uuri2 = (UURI) SerializationUtils.deserialize( - SerializationUtils.serialize(uuri)); - assertEquals("Not equal", uuri.toString(), uuri2.toString()); - uuri = UURIFactory. - getInstance("file://///boo_hoo/wwwroot/CMS/Images1/Banner.gif"); - uuri2 = (UURI) SerializationUtils.deserialize( - SerializationUtils.serialize(uuri)); - assertEquals("Not equal", uuri.toString(), uuri2.toString()); - } - - /** - * A UURI's string representation should be same after a - * toCustomString-getInstance roundtrip. - * - * @throws URIException - */ - public final void testToCustomStringRoundtrip() throws URIException { - UURI uuri = UURIFactory. - getInstance("http://www.example.com/path?query#anchor"); - UURI uuri2 = UURIFactory.getInstance(uuri.toCustomString()); - assertEquals("Not equal", uuri.toString(), uuri2.toString()); - // TODO: fix - // see [HER-1470] UURI String roundtrip (UURIFactory.getInstance(uuri.toString()) results in different URI for file: (and perhaps other) URIs - // http://webteam.archive.org/jira/browse/HER-1470 -// uuri = UURIFactory. -// getInstance("file://///boo_hoo/wwwroot/CMS/Images1/Banner.gif"); -// uuri2 = UURIFactory.getInstance(uuri.toCustomString()); -// assertEquals("Not equal", uuri.toString(), uuri2.toString()); - } - - /** - * A UURI's string representation should be same after a - * toCustomString-getInstance roundtrip. - * - * @throws URIException - */ - public final void testHostnamePortRoundtrip() throws URIException { - UURI base = UURIFactory. - getInstance("http://www.example.com/path?query#anchor"); - UURI test = UURIFactory.getInstance(base,"boom1.hostname.com:9999"); - System.out.println("scheme:"+test.getScheme()); - System.out.println(test.toCustomString()); - UURI roundtrip = UURIFactory.getInstance(test.toCustomString()); - assertEquals("Not equal", test.toString(), roundtrip.toString()); - } - - - /** - * Test bad port throws URIException not NumberFormatException - */ - public void testExtremePort() { - try { - UURI uuri = UURIFactory.getInstance("http://Tel.:010101010101"); - System.out.println(uuri); - fail("expected exception not thrown"); - } catch (URIException ue){ - // expected - } - } - - /** - * Bars ('|') in path-segments aren't encoded by FF, preferred by some - * RESTful-URI-ideas guides, so should work without error. - * - * @throws URIException - */ - public void testBarsInRelativePath() throws URIException { - UURI base = UURIFactory.getInstance("http://www.example.com"); - String relative = "foo/bar|baz|yorple"; - base.resolve(relative); - UURIFactory.getInstance(base,relative); - } - - /** - * To match IE behavior, backslashes in path-info (really, anywhere before - * query string) assumed to be slashes, to match IE behavior. In - * query-string, they are escaped to %5C. - * - * @throws URIException - */ - public void testBackslashes() throws URIException { - UURI uuri = UURIFactory.getInstance("http:\\/www.example.com\\a/b\\c/d?q\\r\\|s/t\\v"); - String expected = "http://www.example.com/a/b/c/d?q%5Cr%5C|s/t%5Cv"; - assertEquals(expected, uuri.toString()); - } -} diff --git a/commons/src/test/java/org/archive/net/UURITest.java b/commons/src/test/java/org/archive/net/UURITest.java deleted file mode 100644 index 8e5084a5..00000000 --- a/commons/src/test/java/org/archive/net/UURITest.java +++ /dev/null @@ -1,55 +0,0 @@ -/* - * This file is part of the Heritrix web crawler (crawler.archive.org). - * - * Licensed to the Internet Archive (IA) by one or more individual - * contributors. - * - * The IA licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.archive.net; - -import java.net.URISyntaxException; - -import org.apache.commons.httpclient.URIException; - -import junit.framework.TestCase; - -public class UURITest extends TestCase { - public void testHasScheme() { - assertTrue(UURI.hasScheme("http://www.archive.org")); - assertTrue(UURI.hasScheme("http:")); - assertFalse(UURI.hasScheme("ht/tp://www.archive.org")); - assertFalse(UURI.hasScheme("/tmp")); - } - - public void testGetFileName() throws URISyntaxException { - final String filename = "x.arc.gz"; - assertEquals(filename, - UURI.parseFilename("/tmp/one.two/" + filename)); - assertEquals(filename, - UURI.parseFilename("http://archive.org/tmp/one.two/" + - filename)); - assertEquals(filename, - UURI.parseFilename("rsync://archive.org/tmp/one.two/" + - filename)); - } - - public void testSchemalessRelative() throws URIException { - UURI base = new UURI("http://www.archive.org/a", true, "UTF-8"); - UURI relative = new UURI("//www.facebook.com/?href=http://www.archive.org/a", true, "UTF-8"); - assertEquals(null, relative.getScheme()); - assertEquals("www.facebook.com", relative.getAuthority()); - UURI test = new UURI(base, relative); - assertEquals("http://www.facebook.com/?href=http://www.archive.org/a", test.toString()); - } -} diff --git a/engine/src/main/java/org/archive/crawler/framework/CrawlController.java b/engine/src/main/java/org/archive/crawler/framework/CrawlController.java index 81504977..af8ef44d 100644 --- a/engine/src/main/java/org/archive/crawler/framework/CrawlController.java +++ b/engine/src/main/java/org/archive/crawler/framework/CrawlController.java @@ -43,6 +43,7 @@ import org.archive.modules.net.ServerCache; import org.archive.modules.seeds.SeedModule; import org.archive.spring.ConfigPath; import org.archive.util.ArchiveUtils; +import org.archive.util.ReportUtils; import org.springframework.beans.BeansException; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Value; @@ -615,7 +616,7 @@ implements Serializable, } public String getToeThreadReportShort() { - return (toePool == null) ? "" : ArchiveUtils.shortReportLine(toePool); + return (toePool == null) ? "" : ReportUtils.shortReportLine(toePool); } public Map getToeThreadReportShortData() { @@ -624,7 +625,7 @@ implements Serializable, } public String getFrontierReportShort() { - return ArchiveUtils.shortReportLine(getFrontier()); + return ReportUtils.shortReportLine(getFrontier()); } /** diff --git a/engine/src/main/java/org/archive/crawler/framework/ToeThread.java b/engine/src/main/java/org/archive/crawler/framework/ToeThread.java index 155d95cc..8a13e2a5 100644 --- a/engine/src/main/java/org/archive/crawler/framework/ToeThread.java +++ b/engine/src/main/java/org/archive/crawler/framework/ToeThread.java @@ -44,6 +44,7 @@ import org.archive.util.ArchiveUtils; import org.archive.util.DevUtils; import org.archive.util.ProgressStatisticsReporter; import org.archive.util.Recorder; +import org.archive.util.ReportUtils; import org.archive.util.Reporter; import com.sleepycat.util.RuntimeExceptionWrapper; @@ -573,7 +574,7 @@ implements Reporter, ProgressStatisticsReporter, } public String shortReportLine() { - return ArchiveUtils.shortReportLine(this); + return ReportUtils.shortReportLine(this); } public void progressStatisticsLine(PrintWriter writer) { diff --git a/engine/src/main/java/org/archive/crawler/frontier/AbstractFrontier.java b/engine/src/main/java/org/archive/crawler/frontier/AbstractFrontier.java index 95d0b632..7f63d5ac 100644 --- a/engine/src/main/java/org/archive/crawler/frontier/AbstractFrontier.java +++ b/engine/src/main/java/org/archive/crawler/frontier/AbstractFrontier.java @@ -69,6 +69,7 @@ import org.archive.modules.seeds.SeedModule; import org.archive.spring.HasKeyedProperties; import org.archive.spring.KeyedProperties; import org.archive.util.ArchiveUtils; +import org.archive.util.ReportUtils; import org.archive.util.iterator.LineReadingIterator; import org.archive.util.iterator.RegexLineIterator; import org.json.JSONException; @@ -1128,7 +1129,7 @@ public abstract class AbstractFrontier // Reporter implementation // public String shortReportLine() { - return ArchiveUtils.shortReportLine(this); + return ReportUtils.shortReportLine(this); } @Override diff --git a/engine/src/main/java/org/archive/crawler/frontier/WorkQueue.java b/engine/src/main/java/org/archive/crawler/frontier/WorkQueue.java index 363e085a..edd56b67 100644 --- a/engine/src/main/java/org/archive/crawler/frontier/WorkQueue.java +++ b/engine/src/main/java/org/archive/crawler/frontier/WorkQueue.java @@ -39,6 +39,7 @@ import org.archive.modules.fetcher.FetchStats.Stage; import org.archive.util.ArchiveUtils; import org.archive.util.IdentityCacheable; import org.archive.util.ObjectIdentityCache; +import org.archive.util.ReportUtils; import org.archive.util.Reporter; /** @@ -541,7 +542,7 @@ public abstract class WorkQueue implements Frontier.FrontierGroup, } public String shortReportLine() { - return ArchiveUtils.shortReportLine(this); + return ReportUtils.shortReportLine(this); } /** diff --git a/engine/src/main/java/org/archive/crawler/frontier/precedence/PrecedenceProvider.java b/engine/src/main/java/org/archive/crawler/frontier/precedence/PrecedenceProvider.java index 65ab891d..85b2fc28 100644 --- a/engine/src/main/java/org/archive/crawler/frontier/precedence/PrecedenceProvider.java +++ b/engine/src/main/java/org/archive/crawler/frontier/precedence/PrecedenceProvider.java @@ -27,6 +27,7 @@ import org.archive.modules.CrawlURI; import org.archive.modules.fetcher.FetchStats; import org.archive.modules.fetcher.FetchStats.Stage; import org.archive.util.ArchiveUtils; +import org.archive.util.ReportUtils; import org.archive.util.Reporter; /** @@ -62,7 +63,7 @@ FetchStats.CollectsFetchStats, Serializable { } public String shortReportLine() { - return ArchiveUtils.shortReportLine(this); + return ReportUtils.shortReportLine(this); } @Override diff --git a/modules/src/main/java/org/archive/modules/CrawlURI.java b/modules/src/main/java/org/archive/modules/CrawlURI.java index 754e3417..50d6dba5 100644 --- a/modules/src/main/java/org/archive/modules/CrawlURI.java +++ b/modules/src/main/java/org/archive/modules/CrawlURI.java @@ -92,9 +92,9 @@ import org.archive.net.UURI; import org.archive.net.UURIFactory; import org.archive.spring.OverlayContext; import org.archive.spring.OverlayMapsSource; -import org.archive.util.ArchiveUtils; import org.archive.util.Base32; import org.archive.util.Recorder; +import org.archive.util.ReportUtils; import org.archive.util.Reporter; import org.json.JSONException; import org.json.JSONObject; @@ -1418,7 +1418,7 @@ implements Reporter, Serializable, OverlayContext { // public String shortReportLine() { - return ArchiveUtils.shortReportLine(this); + return ReportUtils.shortReportLine(this); } @Override diff --git a/modules/src/main/java/org/archive/modules/extractor/ExtractorURI.java b/modules/src/main/java/org/archive/modules/extractor/ExtractorURI.java index 54185401..b856f95b 100644 --- a/modules/src/main/java/org/archive/modules/extractor/ExtractorURI.java +++ b/modules/src/main/java/org/archive/modules/extractor/ExtractorURI.java @@ -27,7 +27,7 @@ import java.util.regex.Matcher; import org.apache.commons.codec.DecoderException; import org.apache.commons.httpclient.URIException; import org.archive.modules.CrawlURI; -import org.archive.net.LaxURLCodec; +import org.archive.url.LaxURLCodec; import org.archive.net.UURI; import org.archive.net.UURIFactory; import org.archive.util.TextUtils; diff --git a/modules/src/main/java/org/archive/modules/fetcher/FetchStats.java b/modules/src/main/java/org/archive/modules/fetcher/FetchStats.java index 85f489a5..a9b53534 100644 --- a/modules/src/main/java/org/archive/modules/fetcher/FetchStats.java +++ b/modules/src/main/java/org/archive/modules/fetcher/FetchStats.java @@ -27,6 +27,7 @@ import org.apache.commons.httpclient.HttpStatus; import org.archive.modules.CrawlURI; import org.archive.modules.deciderules.recrawl.IdenticalDigestDecideRule; import org.archive.util.ArchiveUtils; +import org.archive.util.ReportUtils; import org.archive.util.Reporter; /** @@ -204,7 +205,7 @@ public class FetchStats implements Serializable, FetchStatusCodes, Reporter { } public String shortReportLine() { - return ArchiveUtils.shortReportLine(this); + return ReportUtils.shortReportLine(this); } @Override