lee/heritrix3
1
0
Fork 0
mirror of https://github.com/internetarchive/heritrix3.git synced 2026-09-23 14:15:47 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'master' into hc43

Browse Source 
Conflicts:
	.classpath
	commons/src/main/java/org/apache/commons/httpclient/HttpMethodBase.java
This commit is contained in:
Noah Levitt
2013-01-03 18:18:22 -08:00
parent b6133aee5f 3a6ade1e09
commit 7d3fe554af
42 changed files with 133 additions and 7668 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.classpath
+2 -2
View File
@@ -14,12 +14,11 @@
<classpathentry kind="var" path="M2_REPO/poi/poi/2.5.1-final-20040804/poi-2.5.1-final-20040804.jar" sourcepath="M2_REPO/poi/poi/2.5.1-final-20040804/poi-2.5.1-final-20040804-sources.jar"/>
<classpathentry kind="var" path="M2_REPO/commons-cli/commons-cli/1.1/commons-cli-1.1.jar" sourcepath="/M2_REPO/commons-cli/commons-cli/1.1/commons-cli-1.1-sources.jar"/>
<classpathentry kind="var" path="M2_REPO/net/htmlparser/jericho/jericho-html/2.6.1/jericho-html-2.6.1.jar"/>
<classpathentry kind="var" path="M2_REPO/org/gnu/inet/libidn/0.6.5/libidn-0.6.5.jar"/>
<classpathentry kind="var" path="M2_REPO/commons-lang/commons-lang/2.6/commons-lang-2.6.jar" sourcepath="/M2_REPO/commons-lang/commons-lang/2.6/commons-lang-2.6-sources.jar"/>
<classpathentry kind="var" path="M2_REPO/com/lowagie/itext/1.3/itext-1.3.jar" sourcepath="M2_REPO/com/lowagie/itext/1.3/itext-1.3-sources.jar"/>
<classpathentry kind="var" path="M2_REPO/com/anotherbigidea/javaswf/CVS-SNAPSHOT-1/javaswf-CVS-SNAPSHOT-1.jar"/>
<classpathentry kind="var" path="M2_REPO/it/unimi/dsi/mg4j/1.0.1/mg4j-1.0.1.jar"/>
<classpathentry kind="var" path="M2_REPO/fastutil/fastutil/5.0.7/fastutil-5.0.7.jar"/>
<classpathentry kind="var" path="M2_REPO/it/unimi/dsi/mg4j/1.0.1/mg4j-1.0.1.jar"/>
<classpathentry kind="var" path="M2_REPO/net/java/dev/jets3t/jets3t/0.5.0/jets3t-0.5.0.jar"/>
<classpathentry kind="var" path="M2_REPO/commons-net/commons-net/2.0/commons-net-2.0.jar" sourcepath="/M2_REPO/commons-net/commons-net/2.0/commons-net-2.0-sources.jar"/>
<classpathentry kind="var" path="M2_REPO/poi/poi-scratchpad/2.5.1-final-20040804/poi-scratchpad-2.5.1-final-20040804.jar"/>
@@ -61,5 +60,6 @@
<classpathentry kind="var" path="M2_REPO/junit/junit/3.8.2/junit-3.8.2.jar" sourcepath="/M2_REPO/junit/junit/3.8.2/junit-3.8.2-sources.jar"/>
<classpathentry kind="var" path="M2_REPO/org/mortbay/jetty/jetty/6.1.26/jetty-6.1.26.jar" sourcepath="M2_REPO/org/mortbay/jetty/jetty/6.1.26/jetty-6.1.26-sources.jar"/>
<classpathentry combineaccessrules="false" kind="src" path="/httpcomponents"/>
<classpathentry combineaccessrules="false" kind="src" path="/archive-commons"/>
<classpathentry kind="output" path="eclipse-build"/>
</classpath>
commons/pom.xml
+6 -6
View File
@@ -159,12 +159,6 @@
<version>5.0.7</version>
<scope>compile</scope>
</dependency>
<dependency>
<groupId>org.gnu.inet</groupId>
<artifactId>libidn</artifactId>
<version>0.6.5</version>
<scope>compile</scope>
</dependency>
<dependency>
<groupId>net.java.dev.jets3t</groupId>
<artifactId>jets3t</artifactId>
@@ -250,6 +244,12 @@
<version>3.2.3</version>
</dependency>
<dependency>
<groupId>org.archive</groupId>
<artifactId>archive-commons</artifactId>
<version>1.0-SNAPSHOT</version>
</dependency>
</dependencies>
<build>
<resources>
commons/src/main/java/org/archive/io/ArchiveReader.java
+1
View File
@@ -38,6 +38,7 @@ import java.util.logging.Logger;
import org.apache.commons.cli.Option;
import org.apache.commons.cli.Options;
import org.archive.util.MimetypeUtils;
import org.archive.util.zip.GZIPMembersInputStream;
import com.google.common.io.CountingInputStream;
commons/src/main/java/org/archive/io/GZIPMembersInputStream.java
+8 -293
View File
@@ -17,307 +17,22 @@
* limitations under the License.
*/
package org.archive.io;
import java.io.ByteArrayInputStream;
import java.io.EOFException;
import java.io.IOException;
import java.io.InputStream;
import java.io.SequenceInputStream;
import java.util.Iterator;
import java.util.NoSuchElementException;
import java.util.zip.Inflater;
import java.util.zip.ZipException;
import org.archive.util.zip.OpenJDK7GZIPInputStream;
import com.google.common.io.ByteStreams;
import com.google.common.io.CountingInputStream;
/**
* A replacement for GZIPInputStream; offers GZIP decompression, without any
* artificial stop after the first member in a concatenated series (in
* pre-JDK6u23), and offers direct access to discovered GZIP member
* boundaries (in compressed offsets) via the getMemberNumber(),
* getCurrentMemberStart(), getCurrentMemberEnd() accessors, both pre- and
* post- JDK6u23 (but see below for caveat about getCurrentMemberEnd()).
*
* (This replaces our previous workaround, 'GzippedInputStream', for
* pre-JDK6u23 GZIPInputStream behavior.)
*
* By default, will read straight through members, returning all uncompressed
* data from concatenated compressed members as one stream, per the
* JDK6u23-and-higher behavior. The data returned from a single
* read() will not straddle a member boundary, *but* only after reading
* the first byte of the next member can certainty be offered as to
* whether the previous member ended. Thus, in this default mode, until
* the end of all input, the getAtMemberEnd() method will always return
* false, and getCurrentMemberEnd() will always return -1, because any
* read that discovered a definitive member-end will have begun the next
* member. In this mode, member-ends should be deduced by watching the
* increment of getMemberNumber(), and using the start of the current
* record as the (exclusive) end-position of the previous record.
*
* The setEofEachMember() method may be used to change behavior to mimic that
* of pre-6u23 GZIPInputStream: reaching the end of a GZIP member will result
* in a returned EOF. When receiving this EOF, getAtMemberEnd() will return
* true and getCurrentMemberEnd() will return the (exclusive) member-end
* position. Calling nextMember() after receiving an EOF will allow reading
* to proceed into the next member (if any).
*
* @contributor gojomo
* @deprecated use {@link org.archive.util.zip.GZIPMembersInputStream}
*/
public class GZIPMembersInputStream extends OpenJDK7GZIPInputStream {
protected long memberNumber = 0;
protected long holdAtMemberNumber = Long.MAX_VALUE;
protected long currentMemberStart = 0;
protected long currentMemberEnd = -1;
protected InputStream originalIn;
@Deprecated
public class GZIPMembersInputStream extends org.archive.util.zip.GZIPMembersInputStream {
public GZIPMembersInputStream(InputStream in) throws IOException {
this(in,512);
super(in);
}
public GZIPMembersInputStream(InputStream in, int size)
throws IOException {
super(countingStream(in,size), size);
originalIn = in;
}
/**
* A CountingInputStream is inserted to read compressed-offsets.
*
* @param in stream to wrap
* @param lookback tolerance of initial mark
* @return original stream wrapped in CountingInputStream
* @throws IOException
*/
protected static InputStream countingStream(InputStream in, int lookback) throws IOException {
CountingInputStream cin = new CountingInputStream(in);
cin.mark(lookback);
return cin;
}
protected void updateInnerMark() {
this.in.mark(buf.length);
}
@Override
public int read(byte[] buf, int off, int len) throws IOException {
if(currentMemberEnd>0) {
if(memberNumber>=holdAtMemberNumber) {
// only advance if allowed
return -1;
}
// note read past member boundary
memberNumber++;
currentMemberStart = currentMemberEnd;
currentMemberEnd = -1;
}
return super.read(buf, off, len);
public GZIPMembersInputStream(InputStream in, int size) throws IOException {
super(in, size);
}
@Override
protected boolean readTrailer() throws IOException {
int c = inf.getRemaining();
currentMemberEnd = ((CountingInputStream)in).getCount()-(c-8);
// return super.readTrailer();
// REIMPLEMENTED TO FIX MISUSE OF available()
InputStream in = this.in;
int n = inf.getRemaining();
if (n > 0) {
in = new SequenceInputStream(
new ByteArrayInputStream(buf, len - n, n), in);
}
// Uses left-to-right evaluation order
if ((readUInt(in) != crc.getValue()) ||
// rfc1952; ISIZE is the input size modulo 2^32
(readUInt(in) != (inf.getBytesWritten() & 0xffffffffL)))
throw new ZipException("Corrupt GZIP trailer");
// always try concatenated case; EOF or other IOException
// will let us know if we're wrong
int m = 8; // this.trailer
try {
m += readHeader(in); // next.header
} catch (IOException ze) {
return true; // ignore any malformed, do nothing
}
inf.reset();
if (n > m)
inf.setInput(buf, len - n + m, n - m);
return false;
}
/**
* Seek forward to a particular offset in the compressed stream. Note
* that after any seek/skip the memberNumbers may not reflect a member's
* true ordinal position from the beginning of the stream.
*
* @param position target position
* @throws IOException
*/
public void compressedSeek(long position) throws IOException {
in.reset();
long count = ((CountingInputStream)in).getCount();
long delta = position - count;
if(delta<0) {
throw new IllegalArgumentException("can't seek backwards: seeked "+position+" already at "+count);
}
compressedSkip(delta);
}
/**
* Skip forward the given number of bytes in the compressed stream. Note
* that after any seek/skip the memberNumbers may not reflect a member's
* true ordinal position from the beginning of the stream.
*
* @param offset bytes to skip
* @throws IOException
* @throws EOFException
*/
public void compressedSkip(long offset) throws IOException {
ByteStreams.skipFully(in, offset);
updateInnerMark();
currentMemberStart = ((CountingInputStream)in).getCount();
currentMemberEnd = -1;
startNewMember();
}
protected void startNewMember() throws IOException {
new GzipHeader(in); // consume header
inf.reset();
crc.reset();
eos = false;
}
/**
* Test whether last read resulted in reaching the exact end of one GZIP
* member.
*
* @return true if exactly at member end
*/
public boolean getAtMemberEnd() {
return currentMemberEnd>0;
}
/**
* Get the ordinal number, starting at zero, of the currently-being-read
* GZIP member, counting from the creation of this stream. If reading
* straight through, this will be an accurate index relative to all
* members in the underlying stream. If any seeks/skips have been used,
* the number will only be relative to the members actually read.
*
* @return ordinal number of member-in-progres
*/
public long getMemberNumber() {
return memberNumber;
}
/**
* Get the compressed offset where the current member began.
*
* @return position in compressed stream where current member began
*/
public long getCurrentMemberStart() {
return currentMemberStart;
}
/**
* Get the compressed offset where the current, just-completed member
* ends. Only accurate after the read which finishes a member (when
* getAtMemberEnd returns true). Otherwise, returns -1 to indicate
* not-yet-found.
*
* @return position in compressed stream where member just finished, or -1
* if member end not yet reached
*/
public long getCurrentMemberEnd() {
return currentMemberEnd;
}
/**
* Set stream behavior to match JDK 6u22-and-earlier behavior, where
* reaching the end of any one GZIP member results in EOFs from all
* read()s as if no more data is available. (However, nextMember() may
* be used to advance to the next member.)
*
* @param eofPerMember true to set EOF-each-member behavior
*/
public void setEofEachMember(boolean eofPerMember) {
holdAtMemberNumber = eofPerMember ? memberNumber : Long.MAX_VALUE;
}
/**
* Advance to next member (if the stream has been set to return EOF at the
* end of each member). Each call before reaching the end of a member will
* cause one additional member boundary to be passed. (Has no effect if not
* in EOF-each-member mode.)
*/
public void nextMember() {
if(holdAtMemberNumber<Long.MAX_VALUE) {
holdAtMemberNumber++;
}
}
/**
* Helpful for testing/debugging
*
* @return Inflater
*/
public Inflater getInflater() {
return inf;
}
/**
* Get an Iterator-ish interface to each member in turn. Has the effect of
* putting stream in EOF-each-member mode; thereafter reading should occur
* through the stream returned by the iterator's next(). Reading of one
* stream from next() should finish (reaching EOF) before the iterator's
* hasNext() or next() is called.
*
* @return Iterator<GZIPMembersInputStream> of
* @deprecated for backward compatibility; better to use direct facilities in future
*/
public Iterator<GZIPMembersInputStream> memberIterator() {
return new GZIPEnvelopeIterator();
}
/**
* Provides iterator-ish interface to members in a concatenated multi-member
* GZIP stream for backward compatibility with our prior workaround. Not
* exactly like a real iterator: hasNext() will only return an accurate
* result when the stream returned by the previous next() is read until EOF.
* Previous next() values can not be retained/reused (they are in fact the
* same object as subsequent next() returns.)
*/
public class GZIPEnvelopeIterator implements
Iterator<GZIPMembersInputStream> {
{
setEofEachMember(true);
}
@Override
public boolean hasNext() {
// because readTrailer also reads into next header
// resetting inflater when there's more content, this works
return !inf.finished();
}
@Override
public GZIPMembersInputStream next() {
if(getAtMemberEnd()) {
nextMember();
}
if(hasNext()) {
return GZIPMembersInputStream.this;
} else {
throw new NoSuchElementException();
}
}
@Override
public void remove() {
throw new UnsupportedOperationException();
}
}
}
commons/src/main/java/org/archive/io/GzipHeader.java
+3 -315
View File
@@ -18,321 +18,9 @@
*/
package org.archive.io;
import java.io.EOFException;
import java.io.IOException;
import java.io.InputStream;
import java.util.zip.CRC32;
import java.util.zip.Deflater;
import java.util.zip.GZIPInputStream;
/**
* Read in the GZIP header.
*
* See RFC1952 for specification on what the header looks like.
* Assumption is that stream is cued-up with the gzip header as the
* next thing to be read.
*
* <p>Of <a href="http://jguru.com/faq/view.jsp?EID=13647">Java
* and unsigned bytes</a>. That is, its always a signed int in
* java no matter what the qualifier whether byte, char, etc.
*
* <p>Add accessors for optional filename, comment and MTIME.
*
* @author stack
* @deprecated use {@link org.archive.util.zip.GzipHeader}
*/
public class GzipHeader {
/**
* Length of minimal GZIP header.
*
* See RFC1952 for explaination of value of 10.
*/
public static final int MINIMAL_GZIP_HEADER_LENGTH = 10;
/**
* Total length of the gzip header.
*/
protected int length = 0;
/**
* The GZIP header FLG byte.
*/
protected int flg;
/**
* GZIP header XFL byte.
*/
private int xfl;
/**
* GZIP header OS byte.
*/
private int os;
/**
* Extra header field content.
*/
private byte [] fextra = null;
/**
* GZIP header MTIME field.
*/
private int mtime;
/**
* Shutdown constructor.
*
* Must pass an input stream.
*/
public GzipHeader() {
super();
}
/**
* Constructor.
*
* This constructor advances the stream past any gzip header found.
*
* @param in InputStream to read from.
* @throws IOException
*/
public GzipHeader(InputStream in) throws IOException {
super();
readHeader(in);
}
/**
* Read in gzip header.
*
* Advances the stream past the gzip header.
* @param in InputStream.
*
* @throws IOException Throws if does not start with GZIP Header.
*/
public void readHeader(InputStream in) throws IOException {
CRC32 crc = new CRC32();
crc.reset();
if (!testGzipMagic(in, crc)) {
throw new NoGzipMagicException();
}
this.length += 2;
if (readByte(in, crc) != Deflater.DEFLATED) {
throw new IOException("Unknown compression");
}
this.length++;
// Get gzip header flag.
this.flg = readByte(in, crc);
this.length++;
// Get MTIME.
this.mtime = readInt(in, crc);
this.length += 4;
// Read XFL and OS.
this.xfl = readByte(in, crc);
this.length++;
this.os = readByte(in, crc);
this.length++;
// Skip optional extra field -- stuff w/ alexa stuff in it.
final int FLG_FEXTRA = 4;
if ((this.flg & FLG_FEXTRA) == FLG_FEXTRA) {
int count = readShort(in, crc);
this.length +=2;
this.fextra = new byte[count];
readByte(in, crc, this.fextra, 0, count);
this.length += count;
}
// Skip file name. It ends in null.
final int FLG_FNAME = 8;
if ((this.flg & FLG_FNAME) == FLG_FNAME) {
while (readByte(in, crc) != 0) {
this.length++;
}
}
// Skip file comment. It ends in null.
final int FLG_FCOMMENT = 16; // File comment
if ((this.flg & FLG_FCOMMENT) == FLG_FCOMMENT) {
while (readByte(in, crc) != 0) {
this.length++;
}
}
// Check optional CRC.
final int FLG_FHCRC = 2;
if ((this.flg & FLG_FHCRC) == FLG_FHCRC) {
int calcCrc = (int)(crc.getValue() & 0xffff);
if (readShort(in, crc) != calcCrc) {
throw new IOException("Bad header CRC");
}
this.length += 2;
}
}
/**
* Test gzip magic is next in the stream.
* Reads two bytes. Caller needs to manage resetting stream.
* @param in InputStream to read.
* @return true if found gzip magic. False otherwise
* or an IOException (including EOFException).
* @throws IOException
*/
public boolean testGzipMagic(InputStream in) throws IOException {
return testGzipMagic(in, null);
}
/**
* Test gzip magic is next in the stream.
* Reads two bytes. Caller needs to manage resetting stream.
* @param in InputStream to read.
* @param crc CRC to update.
* @return true if found gzip magic. False otherwise
* or an IOException (including EOFException).
* @throws IOException
*/
public boolean testGzipMagic(InputStream in, CRC32 crc)
throws IOException {
return readShort(in, crc) == GZIPInputStream.GZIP_MAGIC;
}
/**
* Read an int.
*
* We do not expect to get a -1 reading. If we do, we throw exception.
* Update the crc as we go.
*
* @param in InputStream to read.
* @param crc CRC to update.
* @return int read.
*
* @throws IOException
*/
private int readInt(InputStream in, CRC32 crc) throws IOException {
int s = readShort(in, crc);
return ((readShort(in, crc) << 16) & 0xffff0000) | s;
}
/**
* Read a short.
*
* We do not expect to get a -1 reading. If we do, we throw exception.
* Update the crc as we go.
*
* @param in InputStream to read.
* @param crc CRC to update.
* @return Short read.
*
* @throws IOException
*/
private int readShort(InputStream in, CRC32 crc) throws IOException {
int b = readByte(in, crc);
return ((readByte(in, crc) << 8) & 0x00ff00) | b;
}
/**
* Read a byte.
*
* We do not expect to get a -1 reading. If we do, we throw exception.
* Update the crc as we go.
*
* @param in InputStream to read.
* @return Byte read.
*
* @throws IOException
*/
protected int readByte(InputStream in) throws IOException {
return readByte(in, null);
}
/**
* Read a byte.
*
* We do not expect to get a -1 reading. If we do, we throw exception.
* Update the crc as we go.
*
* @param in InputStream to read.
* @param crc CRC to update.
* @return Byte read.
*
* @throws IOException
*/
protected int readByte(InputStream in, CRC32 crc) throws IOException {
int b = in.read();
if (b == -1) {
throw new EOFException();
}
if (crc != null) {
crc.update(b);
}
return b & 0xff;
}
/**
* Read a byte.
*
* We do not expect to get a -1 reading. If we do, we throw exception.
* Update the crc as we go.
*
* @param in InputStream to read.
* @param crc CRC to update.
* @param buffer Buffer to read into.
* @param offset Offset to start filling buffer at.
* @param length How much to read.
* @return Bytes read.
*
* @throws IOException
*/
protected int readByte(InputStream in, CRC32 crc, byte [] buffer,
int offset, int length)
throws IOException {
for (int i = offset; i < length; i++) {
buffer[offset + i] = (byte)readByte(in, crc);
}
return length;
}
/**
* @return Returns the fextra.
*/
public byte[] getFextra() {
return this.fextra;
}
/**
* @return Returns the flg.
*/
public int getFlg() {
return this.flg;
}
/**
* @return Returns the os.
*/
public int getOs() {
return this.os;
}
/**
* @return Returns the xfl.
*/
public int getXfl() {
return this.xfl;
}
/**
* @return Returns the mtime.
*/
public int getMtime() {
return this.mtime;
}
/**
* @return Returns the length.
*/
public int getLength() {
return length;
}
@Deprecated
public class GzipHeader extends org.archive.util.zip.GzipHeader {
}
commons/src/main/java/org/archive/io/NoGzipMagicException.java
+5 -9
View File
@@ -18,13 +18,9 @@
*/
package org.archive.io;
import java.io.IOException;
public class NoGzipMagicException extends IOException {
private static final long serialVersionUID = 3084169624430655013L;
public NoGzipMagicException() {
super();
}
/**
* @deprecated use {@link org.archive.util.zip.NoGzipMagicException}
*/
@Deprecated
public class NoGzipMagicException extends org.archive.util.zip.NoGzipMagicException {
}
commons/src/main/java/org/archive/io/arc/ARCConstants.java
+1 -1
View File
@@ -24,7 +24,7 @@ import java.util.zip.Deflater;
import java.util.zip.GZIPInputStream;
import org.archive.io.ArchiveFileConstants;
import org.archive.io.GzipHeader;
import org.archive.util.zip.GzipHeader;
/**
* Constants used by ARC files and in ARC file processing.
commons/src/main/java/org/archive/io/arc/ARCReaderFactory.java
+3 -3
View File
@@ -32,10 +32,10 @@ import org.archive.io.ArchiveReader;
import org.archive.io.ArchiveReaderFactory;
import org.archive.io.ArchiveRecord;
import org.archive.io.ArchiveRecordHeader;
import org.archive.io.GZIPMembersInputStream;
import org.archive.io.GzipHeader;
import org.archive.io.NoGzipMagicException;
import org.archive.util.FileUtils;
import org.archive.util.zip.GZIPMembersInputStream;
import org.archive.util.zip.GzipHeader;
import org.archive.util.zip.NoGzipMagicException;
import com.google.common.io.CountingInputStream;
commons/src/main/java/org/archive/io/arc/ARCUtils.java
+2 -2
View File
@@ -28,9 +28,9 @@ import java.io.InputStream;
import java.net.URI;
import java.net.URISyntaxException;
import org.archive.io.GzipHeader;
import org.archive.io.NoGzipMagicException;
import org.archive.net.UURI;
import org.archive.util.zip.GzipHeader;
import org.archive.util.zip.NoGzipMagicException;
public class ARCUtils implements ARCConstants {
/**
commons/src/main/java/org/archive/io/warc/WARCReaderFactory.java
+1 -1
View File
@@ -30,10 +30,10 @@ import java.util.Iterator;
import org.archive.io.ArchiveReader;
import org.archive.io.ArchiveReaderFactory;
import org.archive.io.ArchiveRecord;
import org.archive.io.GZIPMembersInputStream;
import org.archive.io.warc.WARCConstants;
import org.archive.util.ArchiveUtils;
import org.archive.util.FileUtils;
import org.archive.util.zip.GZIPMembersInputStream;
import com.google.common.io.CountingInputStream;
commons/src/main/java/org/archive/net/LaxURI.java
-459
View File
@@ -1,459 +0,0 @@
/*
* This file is part of the Heritrix web crawler (crawler.archive.org).
*
* Licensed to the Internet Archive (IA) by one or more individual
* contributors.
*
* The IA licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.archive.net;
import java.util.Arrays;
import java.util.BitSet;
import org.apache.commons.httpclient.URI;
import org.apache.commons.httpclient.URIException;
import org.apache.commons.httpclient.util.EncodingUtil;
/**
* URI subclass which allows partial/inconsistent encoding, matching
* the URIs which will be relayed in requests from popular web
* browsers (esp. Mozilla Firefox and MS IE).
*
* @author gojomo
*/
public class LaxURI extends URI {
private static final long serialVersionUID = 5273922211722239537L;
final protected static char[] HTTP_SCHEME = {'h','t','t','p'};
final protected static char[] HTTPS_SCHEME = {'h','t','t','p','s'};
protected static final BitSet lax_rel_segment = new BitSet(256);
// Static initializer for lax_rel_segment
static {
lax_rel_segment.or(rel_segment);
lax_rel_segment.set(':'); // allow ':'
// TODO: add additional allowances as need is demonstrated
}
protected static final BitSet lax_abs_path = new BitSet(256);
static {
lax_abs_path.or(abs_path);
lax_abs_path.set('|'); // tests indicate Firefox (1.0.6) doesn't escape.
}
protected static final BitSet lax_rel_path = new BitSet(256);
// Static initializer for rel_path
static {
lax_rel_path.or(lax_rel_segment);
lax_rel_path.or(lax_abs_path);
}
protected static final BitSet lax_query = new BitSet(256);
static {
lax_query.or(query);
lax_query.set('{'); // tests indicate FF doesn't escape { in query
lax_query.set('}'); // tests indicate FF doesn't escape } in query
lax_query.set('|'); // tests indicate FF doesn't escape | in query
lax_query.set('['); // tests indicate FF doesn't escape [ in query
lax_query.set(']'); // tests indicate FF doesn't escape ] in query
lax_query.set('^'); // tests indicate FF doesn't escape ^ in query
}
// passthrough initializers
public LaxURI(String uri, boolean escaped, String charset)
throws URIException {
super(uri,escaped,charset);
}
public LaxURI(URI base, URI relative) throws URIException {
super(base,relative);
}
public LaxURI(String uri, boolean escaped) throws URIException {
super(uri,escaped);
}
public LaxURI() {
super();
}
// overridden to use this class's static decode()
public String getURI() throws URIException {
return (_uri == null) ? null : decode(_uri, getProtocolCharset());
}
// overridden to use this class's static decode()
public String getPath() throws URIException {
char[] p = getRawPath();
return (p == null) ? null : decode(p, getProtocolCharset());
}
// overridden to use this class's static decode()
public String getPathQuery() throws URIException {
char[] rawPathQuery = getRawPathQuery();
return (rawPathQuery == null) ? null : decode(rawPathQuery,
getProtocolCharset());
}
// overridden to use this class's static decode()
protected static String decode(char[] component, String charset)
throws URIException {
if (component == null) {
throw new IllegalArgumentException(
"Component array of chars may not be null");
}
return decode(new String(component), charset);
}
// overridden to use IA's LaxURLCodec, which never throws DecoderException
protected static String decode(String component, String charset)
throws URIException {
if (component == null) {
throw new IllegalArgumentException(
"Component array of chars may not be null");
}
byte[] rawdata = null;
// try {
rawdata = LaxURLCodec.decodeUrlLoose(EncodingUtil
.getAsciiBytes(component));
// } catch (DecoderException e) {
// throw new URIException(e.getMessage());
// }
return EncodingUtil.getString(rawdata, charset);
}
// overidden to lax() the acceptable-char BitSet passed in
protected boolean validate(char[] component, BitSet generous) {
return super.validate(component, lax(generous));
}
// overidden to lax() the acceptable-char BitSet passed in
protected boolean validate(char[] component, int soffset, int eoffset,
BitSet generous) {
return super.validate(component, soffset, eoffset, lax(generous));
}
/**
* Given a BitSet -- typically one of the URI superclass's
* predefined static variables -- possibly replace it with
* a more-lax version to better match the character sets
* actually left unencoded in web browser requests
*
* @param generous original BitSet
* @return (possibly more lax) BitSet to use
*/
protected BitSet lax(BitSet generous) {
if (generous == rel_segment) {
// Swap in more lax allowable set
return lax_rel_segment;
}
if (generous == abs_path) {
return lax_abs_path;
}
if (generous == query) {
return lax_query;
}
if (generous == rel_path) {
return lax_rel_path;
}
// otherwise, leave as is
return generous;
}
/**
* Coalesce the _host and _authority fields where
* possible.
*
* In the web crawl/http domain, most URIs have an
* identical _host and _authority. (There is no port
* or user info.) However, the superclass always
* creates two separate char[] instances.
*
* Notably, the lengths of these char[] fields are
* equal if and only if their values are identical.
* This method makes use of this fact to reduce the
* two instances to one where possible, slimming
* instances.
*
* @see org.apache.commons.httpclient.URI#parseAuthority(java.lang.String, boolean)
*/
protected void parseAuthority(String original, boolean escaped)
throws URIException {
super.parseAuthority(original, escaped);
if (_host != null && _authority != null
&& _host.length == _authority.length) {
_host = _authority;
}
}
/**
* Coalesce _scheme to existing instances, where appropriate.
*
* In the web-crawl domain, most _schemes are 'http' or 'https',
* but the superclass always creates a new char[] instance. For
* these two cases, we replace the created instance with a
* long-lived instance from a static field, saving 12-14 bytes
* per instance.
*
* @see org.apache.commons.httpclient.URI#setURI()
*/
protected void setURI() {
if (_scheme != null) {
if (_scheme.length == 4 && Arrays.equals(_scheme, HTTP_SCHEME)) {
_scheme = HTTP_SCHEME;
} else if (_scheme.length == 5
&& Arrays.equals(_scheme, HTTP_SCHEME)) {
_scheme = HTTPS_SCHEME;
}
}
super.setURI();
}
/**
* IA OVERRIDDEN IN LaxURI TO INCLUDE FIX FOR
* http://issues.apache.org/jira/browse/HTTPCLIENT-588
* AND
* http://webteam.archive.org/jira/browse/HER-1268
*
* In order to avoid any possilbity of conflict with non-ASCII characters,
* Parse a URI reference as a <code>String</code> with the character
* encoding of the local system or the document.
* <p>
* The following line is the regular expression for breaking-down a URI
* reference into its components.
* <p><blockquote><pre>
* ^(([^:/?#]+):)?(//([^/?#]*))?([^?#]*)(\?([^#]*))?(#(.*))?
* 12 3 4 5 6 7 8 9
* </pre></blockquote><p>
* For example, matching the above expression to
* http://jakarta.apache.org/ietf/uri/#Related
* results in the following subexpression matches:
* <p><blockquote><pre>
* $1 = http:
* scheme = $2 = http
* $3 = //jakarta.apache.org
* authority = $4 = jakarta.apache.org
* path = $5 = /ietf/uri/
* $6 = <undefined>
* query = $7 = <undefined>
* $8 = #Related
* fragment = $9 = Related
* </pre></blockquote><p>
*
* @param original the original character sequence
* @param escaped <code>true</code> if <code>original</code> is escaped
* @throws URIException If an error occurs.
*/
protected void parseUriReference(String original, boolean escaped)
throws URIException {
// validate and contruct the URI character sequence
if (original == null) {
throw new URIException("URI-Reference required");
}
/* @
* ^(([^:/?#]+):)?(//([^/?#]*))?([^?#]*)(\?([^#]*))?(#(.*))?
*/
String tmp = original.trim();
/*
* The length of the string sequence of characters.
* It may not be equal to the length of the byte array.
*/
int length = tmp.length();
/*
* Remove the delimiters like angle brackets around an URI.
*/
if (length > 0) {
char[] firstDelimiter = { tmp.charAt(0) };
if (validate(firstDelimiter, delims)) {
if (length >= 2) {
char[] lastDelimiter = { tmp.charAt(length - 1) };
if (validate(lastDelimiter, delims)) {
tmp = tmp.substring(1, length - 1);
length = length - 2;
}
}
}
}
/*
* The starting index
*/
int from = 0;
/*
* The test flag whether the URI is started from the path component.
*/
boolean isStartedFromPath = false;
int atColon = tmp.indexOf(':');
int atSlash = tmp.indexOf('/');
if (!tmp.startsWith("//")
&& (atColon <= 0 || (atSlash >= 0 && atSlash < atColon))) {
isStartedFromPath = true;
}
/*
* <p><blockquote><pre>
* @@@@@@@@
* ^(([^:/?#]+):)?(//([^/?#]*))?([^?#]*)(\?([^#]*))?(#(.*))?
* </pre></blockquote><p>
*/
int at = indexFirstOf(tmp, isStartedFromPath ? "/?#" : ":/?#", from);
if (at == -1) {
at = 0;
}
/*
* Parse the scheme.
* <p><blockquote><pre>
* scheme = $2 = http
* @
* ^(([^:/?#]+):)?(//([^/?#]*))?([^?#]*)(\?([^#]*))?(#(.*))?
* </pre></blockquote><p>
*/
if (at > 0 && at < length && tmp.charAt(at) == ':') {
char[] target = tmp.substring(0, at).toLowerCase().toCharArray();
if (validate(target, scheme)) {
_scheme = target;
from = ++at;
} else {
// IA CHANGE:
// do nothing; allow interpretation as URI with
// later colon in other syntactical component
}
}
/*
* Parse the authority component.
* <p><blockquote><pre>
* authority = $4 = jakarta.apache.org
* @@
* ^(([^:/?#]+):)?(//([^/?#]*))?([^?#]*)(\?([^#]*))?(#(.*))?
* </pre></blockquote><p>
*/
// Reset flags
_is_net_path = _is_abs_path = _is_rel_path = _is_hier_part = false;
if (0 <= at && at < length && tmp.charAt(at) == '/') {
// Set flag
_is_hier_part = true;
if (at + 2 < length && tmp.charAt(at + 1) == '/'
&& !isStartedFromPath) {
// the temporary index to start the search from
int next = indexFirstOf(tmp, "/?#", at + 2);
if (next == -1) {
next = (tmp.substring(at + 2).length() == 0) ? at + 2
: tmp.length();
}
parseAuthority(tmp.substring(at + 2, next), escaped);
from = at = next;
// Set flag
_is_net_path = true;
}
if (from == at) {
// Set flag
_is_abs_path = true;
}
}
/*
* Parse the path component.
* <p><blockquote><pre>
* path = $5 = /ietf/uri/
* @@@@@@
* ^(([^:/?#]+):)?(//([^/?#]*))?([^?#]*)(\?([^#]*))?(#(.*))?
* </pre></blockquote><p>
*/
if (from < length) {
// rel_path = rel_segment [ abs_path ]
int next = indexFirstOf(tmp, "?#", from);
if (next == -1) {
next = tmp.length();
}
if (!_is_abs_path) {
if (!escaped
&& prevalidate(tmp.substring(from, next), disallowed_rel_path)
|| escaped
&& validate(tmp.substring(from, next).toCharArray(), rel_path)) {
// Set flag
_is_rel_path = true;
} else if (!escaped
&& prevalidate(tmp.substring(from, next), disallowed_opaque_part)
|| escaped
&& validate(tmp.substring(from, next).toCharArray(), opaque_part)) {
// Set flag
_is_opaque_part = true;
} else {
// the path component may be empty
_path = null;
}
}
String s = tmp.substring(from, next);
if (escaped) {
setRawPath(s.toCharArray());
} else {
setPath(s);
}
at = next;
}
// set the charset to do escape encoding
String charset = getProtocolCharset();
/*
* Parse the query component.
* <p><blockquote><pre>
* query = $7 = <undefined>
* @@@@@@@@@
* ^(([^:/?#]+):)?(//([^/?#]*))?([^?#]*)(\?([^#]*))?(#(.*))?
* </pre></blockquote><p>
*/
if (0 <= at && at + 1 < length && tmp.charAt(at) == '?') {
int next = tmp.indexOf('#', at + 1);
if (next == -1) {
next = tmp.length();
}
if (escaped) {
_query = tmp.substring(at + 1, next).toCharArray();
if (!validate(_query, query)) {
throw new URIException("Invalid query");
}
} else {
_query = encode(tmp.substring(at + 1, next), allowed_query, charset);
}
at = next;
}
/*
* Parse the fragment component.
* <p><blockquote><pre>
* fragment = $9 = Related
* @@@@@@@@
* ^(([^:/?#]+):)?(//([^/?#]*))?([^?#]*)(\?([^#]*))?(#(.*))?
* </pre></blockquote><p>
*/
if (0 <= at && at + 1 <= length && tmp.charAt(at) == '#') {
if (at + 1 == length) { // empty fragment
_fragment = "".toCharArray();
} else {
_fragment = (escaped) ? tmp.substring(at + 1).toCharArray()
: encode(tmp.substring(at + 1), allowed_fragment, charset);
}
}
// set this URI.
setURI();
}
}
commons/src/main/java/org/archive/net/LaxURLCodec.java
-160
View File
@@ -1,160 +0,0 @@
/*
* This file is part of the Heritrix web crawler (crawler.archive.org).
*
* Licensed to the Internet Archive (IA) by one or more individual
* contributors.
*
* The IA licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.archive.net;
import java.io.ByteArrayOutputStream;
import java.io.UnsupportedEncodingException;
import java.util.BitSet;
import org.apache.commons.codec.net.URLCodec;
import com.google.common.base.Charsets;
/**
* @author gojomo
*/
public class LaxURLCodec extends URLCodec {
public static LaxURLCodec DEFAULT = new LaxURLCodec("UTF-8");
// passthrough constructor
public LaxURLCodec(String encoding) {
super(encoding);
}
/**
* Decodes an array of URL safe 7-bit characters into an array of
* original bytes. Escaped characters are converted back to their
* original representation.
*
* Differs from URLCodec.decodeUrl() in that it throws no
* exceptions; bad or incomplete escape sequences are ignored
* and passed into result undecoded. This matches the behavior
* of browsers, which will use inconsistently-encoded URIs
* in HTTP request-lines.
*
* @param bytes array of URL safe characters
* @return array of original bytes
*/
public static final byte[] decodeUrlLoose(byte[] bytes)
{
if (bytes == null) {
return null;
}
ByteArrayOutputStream buffer = new ByteArrayOutputStream();
for (int i = 0; i < bytes.length; i++) {
int b = bytes[i];
if (b == '+') {
buffer.write(' ');
continue;
}
if (b == '%') {
if(i+2<bytes.length) {
int u = Character.digit((char)bytes[i+1], 16);
int l = Character.digit((char)bytes[i+2], 16);
if (u > -1 && l > -1) {
// good encoding
int c = ((u << 4) + l);
buffer.write((char)c);
i += 2;
continue;
} // else: bad encoding digits, leave '%' in place
} // else: insufficient encoding digits, leave '%' in place
}
buffer.write(b);
}
return buffer.toByteArray();
}
/**
* A more expansive set of ASCII URI characters to consider as 'safe' to
* leave unencoded, based on actual browser behavior.
*/
public static BitSet EXPANDED_URI_SAFE = new BitSet(256);
static {
// alpha characters
for (int i = 'a'; i <= 'z'; i++) {
EXPANDED_URI_SAFE.set(i);
}
for (int i = 'A'; i <= 'Z'; i++) {
EXPANDED_URI_SAFE.set(i);
}
// numeric characters
for (int i = '0'; i <= '9'; i++) {
EXPANDED_URI_SAFE.set(i);
}
// special chars
EXPANDED_URI_SAFE.set('-');
EXPANDED_URI_SAFE.set('~');
EXPANDED_URI_SAFE.set('_');
EXPANDED_URI_SAFE.set('.');
EXPANDED_URI_SAFE.set('*');
EXPANDED_URI_SAFE.set('/');
EXPANDED_URI_SAFE.set('=');
EXPANDED_URI_SAFE.set('&');
EXPANDED_URI_SAFE.set('+');
EXPANDED_URI_SAFE.set(',');
EXPANDED_URI_SAFE.set(':');
EXPANDED_URI_SAFE.set(';');
EXPANDED_URI_SAFE.set('@');
EXPANDED_URI_SAFE.set('$');
EXPANDED_URI_SAFE.set('!');
EXPANDED_URI_SAFE.set(')');
EXPANDED_URI_SAFE.set('(');
// experiments indicate: Firefox (1.0.6) never escapes '%'
EXPANDED_URI_SAFE.set('%');
// experiments indicate: Firefox (1.0.6) does not escape '|' or '''
EXPANDED_URI_SAFE.set('|');
EXPANDED_URI_SAFE.set('\'');
}
public static BitSet QUERY_SAFE = new BitSet(256);
static {
QUERY_SAFE.or(EXPANDED_URI_SAFE);
// Tests indicate Firefox (1.0.7-1) doesn't escape curlies in query str.
QUERY_SAFE.set('{');
QUERY_SAFE.set('}');
// nor any of these: [ ] ^ ?
QUERY_SAFE.set('[');
QUERY_SAFE.set(']');
QUERY_SAFE.set('^');
QUERY_SAFE.set('?');
}
/**
* Encodes a string into its URL safe form using the specified
* string charset. Unsafe characters are escaped.
*
* This method is analogous to superclass encode() methods,
* additionally offering the ability to specify a different
* 'safe' character set (such as EXPANDED_URI_SAFE).
*
* @param safe BitSet of characters that don't need to be encoded
* @param pString String to encode
* @param cs Name of character set to use
* @return Encoded version of <code>pString</code>.
* @throws UnsupportedEncodingException
*/
public String encode(BitSet safe, String pString, String cs)
throws UnsupportedEncodingException {
if (pString == null) {
return null;
}
return new String(encodeUrl(safe,pString.getBytes(cs)), Charsets.US_ASCII);
}
}
commons/src/main/java/org/archive/net/UURI.java
+21 -414
View File
@@ -18,442 +18,49 @@
*/
package org.archive.net;
import java.io.File;
import java.io.IOException;
import java.io.ObjectInputStream;
import java.io.ObjectOutputStream;
import java.io.Serializable;
import java.net.URI;
import java.net.URISyntaxException;
import java.nio.ByteBuffer;
import org.apache.commons.httpclient.URIException;
import org.archive.util.SURT;
import org.archive.util.TextUtils;
import org.archive.url.UsableURI;
import com.esotericsoftware.kryo.CustomSerialization;
import com.esotericsoftware.kryo.Kryo;
import com.esotericsoftware.kryo.serialize.StringSerializer;
/**
* Usable URI.
*
* This class wraps {@link org.apache.commons.httpclient.URI} adding caching
* and methods. It cannot be instantiated directly. Go via UURIFactory.
*
* <p>We used to use {@link java.net.URI} for parsing URIs but ran across
* quirky behaviors and bugs. {@link java.net.URI} is not subclassable --
* its final -- and its unlikely that java.net.URI will change any time soon
* (See Gordon's considered petition here:
* <a href="http://developer.java.sun.com/developer/bugParade/bugs/4939847.html">java.net.URI
* should have loose/tolerant/compatibility option (or allow reuse)</a>).
*
* <p>This class tries to cache calculated strings such as the extracted host
* and this class as a string rather than have the parent class rerun its
* calculation everytime.
*
* @author gojomo
* @author stack
*
* @see org.apache.commons.httpclient.URI
* Usable URI. The bulk of the functionality of this class has moved to
* {@link UsableURI} in the archive-commons project. This class adds Kryo
* serialization.
*/
public class UURI extends LaxURI
implements CharSequence, Serializable, CustomSerialization {
public class UURI extends UsableURI implements CustomSerialization {
private static final long serialVersionUID = -1277570889914647093L;
private static final long serialVersionUID = -8946640480772772310L;
//private static Logger LOGGER =
// Logger.getLogger(UURI.class.getName());
/**
* Consider URIs too long for IE as illegal.
*/
public final static int MAX_URL_LENGTH = 2083;
public static final String MASSAGEHOST_PATTERN = "^www\\d*\\.";
/**
* Cache of the host name.
*
* Super class calculates on every call. Profiling shows us spend 30% of
* total elapsed time in URI class.
*/
private transient String cachedHost = null;
/**
* Cache of this uuri escaped as a string.
*
* Super class calculates on every call. Profiling shows us spend 30% of
* total elapsed time in URI class.
*/
private transient String cachedEscapedURI = null;
/**
* Cache of this uuri escaped as a string.
*
* Super class calculates on every call. Profiling shows us spend 30% of
* total elapsed time in URI class.
*/
private transient String cachedString = null;
/**
* Cached authority minus userinfo.
*/
private transient String cachedAuthorityMinusUserinfo = null;
/**
* Cache of this uuri in SURT format
*/
private transient String surtForm = null;
// Technically, underscores are disallowed in the domainlabel
// portion of hostname according to rfc2396 but we'll be more
// loose and allow them. See: [ 1072035 ] [uuri] Underscore in
// host messes up port parsing.
static {
hostname.set('_');
public UURI(String fixup, boolean b, String charset) throws URIException {
super(fixup, b, charset);
}
/**
* Shutdown access to default constructor.
*/
public UURI(UsableURI base, UsableURI relative) throws URIException {
super(base, relative);
}
/* needed for kryo serialization */
protected UURI() {
super();
}
/**
* @param uri String representation of an absolute URI.
* @param escaped If escaped.
* @param charset Charset to use.
* @throws org.apache.commons.httpclient.URIException
*/
protected UURI(String uri, boolean escaped, String charset)
throws URIException {
super(uri, escaped, charset);
normalize();
}
/**
* @param relative String representation of URI.
* @param base Parent UURI to use derelativizing.
* @throws org.apache.commons.httpclient.URIException
*/
protected UURI(UURI base, UURI relative) throws URIException {
super(base, relative);
normalize();
}
/**
* @param uri String representation of a URI.
* @param escaped If escaped.
* @throws NullPointerException
* @throws URIException
*/
protected UURI(String uri, boolean escaped) throws URIException, NullPointerException {
super(uri,escaped);
normalize();
}
/**
* @param uri URI as string that is resolved relative to this UURI.
* @return UURI that uses this UURI as base.
* @throws URIException
*/
public UURI resolve(String uri)
throws URIException {
return resolve(uri, false, // assume not escaped
this.getProtocolCharset());
}
/**
* @param uri URI as string that is resolved relative to this UURI.
* @param e True if escaped.
* @return UURI that uses this UURI as base.
* @throws URIException
*/
public UURI resolve(String uri, boolean e)
throws URIException {
return resolve(uri, e, this.getProtocolCharset());
}
/**
* @param uri URI as string that is resolved relative to this UURI.
* @param e True if uri is escaped.
* @param charset Charset to use.
* @return UURI that uses this UURI as base.
* @throws URIException
*/
public UURI resolve(String uri, boolean e, String charset)
throws URIException {
return new UURI(this, new UURI(uri, e, charset));
}
/**
* Test an object if this UURI is equal to another.
*
* @param obj an object to compare
* @return true if two URI objects are equal
*/
public boolean equals(Object obj) {
// normalize and test each components
if (obj == this) {
return true;
}
if (!(obj instanceof UURI)) {
return false;
}
UURI another = (UURI) obj;
// scheme
if (!equals(this._scheme, another._scheme)) {
return false;
}
// is_opaque_part or is_hier_part? and opaque
if (!equals(this._opaque, another._opaque)) {
return false;
}
// is_hier_part
// has_authority
if (!equals(this._authority, another._authority)) {
return false;
}
// path
if (!equals(this._path, another._path)) {
return false;
}
// has_query
if (!equals(this._query, another._query)) {
return false;
}
// UURIs do not have fragments
return true;
}
/**
* Strips www variants from the host.
*
* Strips www[0-9]*\. from the host. If calling getHostBaseName becomes a
* performance issue we should consider adding the hostBasename member that
* is set on initialization.
*
* @return Host's basename.
* @throws URIException
*/
public String getHostBasename() throws URIException {
// caching eliminated because this is rarely used
// (only benefits legacy DomainScope, which should
// be retired). Saves 4-byte object pointer in UURI
// instances.
return (this.getReferencedHost() == null)
? null
: TextUtils.replaceFirst(MASSAGEHOST_PATTERN,
this.getReferencedHost(), UURIFactory.EMPTY_STRING);
}
/**
* Returns an alternate, functional String representation -- in this
* case, a String of the URI represented by this UURI instance.
*
* @return
*/
public synchronized String toCustomString() {
if (this.cachedString == null) {
this.cachedString = super.toString();
coalesceUriStrings();
}
return this.cachedString;
}
/**
* Override to cache result
*
* TODO: eliminate, moving most callers to toCustomString, to avoid
* overloading/diluting toString()
* (see http://webteam.archive.org/confluence/display/Heritrix/Preserve+toString%28%29 )
* @return String representation of this URI
*/
public String toString() {
return toCustomString();
}
public synchronized String getEscapedURI() {
if (this.cachedEscapedURI == null) {
this.cachedEscapedURI = super.getEscapedURI();
coalesceUriStrings();
}
return this.cachedEscapedURI;
}
/**
* The two String fields cachedString and cachedEscapedURI are
* usually identical; if so, coalesce into a single instance.
*/
protected void coalesceUriStrings() {
if (this.cachedString != null && this.cachedEscapedURI != null
&& this.cachedString.length() == this.cachedEscapedURI.length()) {
// lengths will only be identical if contents are identical
// (deescaping will always shrink length), so coalesce to
// use only single cached instance
this.cachedString = this.cachedEscapedURI;
}
}
public synchronized String getHost() throws URIException {
if (this.cachedHost == null) {
// If this._host is null, 3.0 httpclient throws
// illegalargumentexception. Don't go there.
if (this._host != null) {
this.cachedHost = super.getHost();
coalesceHostAuthorityStrings();
}
}
return this.cachedHost;
}
/**
* The two String fields cachedHost and cachedAuthorityMinusUserInfo are
* usually identical; if so, coalesce into a single instance.
*/
protected void coalesceHostAuthorityStrings() {
if (this.cachedAuthorityMinusUserinfo != null
&& this.cachedHost != null
&& this.cachedHost.length() ==
this.cachedAuthorityMinusUserinfo.length()) {
// lengths can only be identical if contents
// are identical; use only one instance
this.cachedAuthorityMinusUserinfo = this.cachedHost;
}
}
/**
* Return the referenced host in the UURI, if any, also extracting the
* host of a DNS-lookup URI where necessary.
*
* @return the target or topic host of the URI
* @throws URIException
*/
public String getReferencedHost() throws URIException {
String referencedHost = this.getHost();
if(referencedHost==null && this.getScheme().equals("dns")) {
// extract target domain of DNS lookup
String possibleHost = this.getCurrentHierPath();
if(possibleHost != null && possibleHost.matches("[-_\\w\\.:]+")) {
referencedHost = possibleHost;
}
}
return referencedHost;
}
/**
* @return Return the 'SURT' format of this UURI
*/
public String getSurtForm() {
if (surtForm == null) {
surtForm = SURT.fromURI(this.toString());
}
return surtForm;
}
/**
* Return the authority minus userinfo (if any).
*
* If no userinfo present, just returns the authority.
*
* @return The authority stripped of any userinfo if present.
* @throws URIException
*/
public String getAuthorityMinusUserinfo()
throws URIException {
if (this.cachedAuthorityMinusUserinfo == null) {
String tmp = getAuthority();
if (tmp != null && tmp.length() > 0) {
int index = tmp.indexOf('@');
if (index >= 0 && index < tmp.length()) {
tmp = tmp.substring(index + 1);
}
}
this.cachedAuthorityMinusUserinfo = tmp;
coalesceHostAuthorityStrings();
}
return this.cachedAuthorityMinusUserinfo;
}
/* (non-Javadoc)
* @see java.lang.CharSequence#length()
*/
public int length() {
return getEscapedURI().length();
}
/* (non-Javadoc)
* @see java.lang.CharSequence#charAt(int)
*/
public char charAt(int index) {
return getEscapedURI().charAt(index);
}
/* (non-Javadoc)
* @see java.lang.CharSequence#subSequence(int, int)
*/
public CharSequence subSequence(int start, int end) {
return getEscapedURI().subSequence(start,end);
}
/* (non-Javadoc)
* @see java.lang.Comparable#compareTo(java.lang.Object)
*/
public int compareTo(Object arg0) {
return getEscapedURI().compareTo(arg0.toString());
}
/**
* Test if passed String has likely URI scheme prefix.
* @param possibleUrl URL string to examine.
* @return True if passed string looks like it could be an URL.
*/
public static boolean hasScheme(String possibleUrl) {
boolean result = false;
for (int i = 0; i < possibleUrl.length(); i++) {
char c = possibleUrl.charAt(i);
if (c == ':') {
if (i != 0) {
result = true;
}
break;
}
if (!scheme.get(c)) {
break;
}
}
return result;
}
/**
* @param pathOrUri A file path or a URI.
* @return Path parsed from passed <code>pathOrUri</code>.
* @throws URISyntaxException
*/
public static String parseFilename(final String pathOrUri)
throws URISyntaxException {
String path = pathOrUri;
if (UURI.hasScheme(pathOrUri)) {
URI url = new URI(pathOrUri);
path = url.getPath();
}
return (new File(path)).getName();
}
@Override
public void writeObjectData(Kryo kryo, ByteBuffer buffer) {
StringSerializer.put(buffer, toCustomString());
}
public void readObjectData (Kryo kryo, ByteBuffer buffer) {
@Override
public void readObjectData(Kryo kryo, ByteBuffer buffer) {
try {
parseUriReference(StringSerializer.get(buffer),true);
parseUriReference(StringSerializer.get(buffer), true);
} catch (URIException e) {
// TODO Auto-generated catch block
e.printStackTrace();
@@ -462,10 +69,10 @@ implements CharSequence, Serializable, CustomSerialization {
private void writeObject(ObjectOutputStream stream) throws IOException {
stream.writeUTF(toCustomString());
}
}
private void readObject(ObjectInputStream stream) throws IOException,
ClassNotFoundException {
parseUriReference(stream.readUTF(),true);
ClassNotFoundException {
parseUriReference(stream.readUTF(), true);
}
}
commons/src/main/java/org/archive/net/UURIFactory.java
+18 -750
View File
@@ -18,232 +18,32 @@
*/
package org.archive.net;
import gnu.inet.encoding.IDNA;
import gnu.inet.encoding.IDNAException;
import it.unimi.dsi.mg4j.util.MutableString;
import java.io.UnsupportedEncodingException;
import java.util.BitSet;
import java.util.logging.Level;
import java.util.logging.Logger;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import org.apache.commons.httpclient.URI;
import org.apache.commons.httpclient.URIException;
import org.archive.util.TextUtils;
import org.archive.url.UsableURI;
import org.archive.url.UsableURIFactory;
/**
* Factory that returns UURIs.
* Factory that returns UURIs. Mostly wraps {@link UsableURIFactory}.
*
* Does escaping and fixup on URIs massaging in accordance with RFC2396 and to
* match browser practice. For example, it removes any '..' if first thing in
* the path as per IE, converts backslashes preceding the query string to
* forward slashes, and discards any 'fragment'/anchor portion of the URI. This
* class will also fail URIs if they are longer than IE's allowed maximum
* length.
*
* <p>
* TODO: Test logging.
*
* @author stack
*/
public class UURIFactory extends URI {
private static final long serialVersionUID = -6146295130382209042L;
/**
* Logging instance.
*/
private static Logger logger =
Logger.getLogger(UURIFactory.class.getName());
public class UURIFactory extends UsableURIFactory {
private static final long serialVersionUID = -7969477276065915936L;
/**
* The single instance of this factory.
*/
private static final UURIFactory factory = new UURIFactory();
/**
* RFC 2396-inspired regex.
*
* From the RFC Appendix B:
* <pre>
* URI Generic Syntax August 1998
*
* B. Parsing a URI Reference with a Regular Expression
*
* As described in Section 4.3, the generic URI syntax is not sufficient
* to disambiguate the components of some forms of URI. Since the
* "greedy algorithm" described in that section is identical to the
* disambiguation method used by POSIX regular expressions, it is
* natural and commonplace to use a regular expression for parsing the
* potential four components and fragment identifier of a URI reference.
*
* The following line is the regular expression for breaking-down a URI
* reference into its components.
*
* ^(([^:/?#]+):)?(//([^/?#]*))?([^?#]*)(\?([^#]*))?(#(.*))?
* 12 3 4 5 6 7 8 9
*
* The numbers in the second line above are only to assist readability;
* they indicate the reference points for each subexpression (i.e., each
* paired parenthesis). We refer to the value matched for subexpression
* <n> as $<n>. For example, matching the above expression to
*
* http://www.ics.uci.edu/pub/ietf/uri/#Related
*
* results in the following subexpression matches:
*
* $1 = http:
* $2 = http
* $3 = //www.ics.uci.edu
* $4 = www.ics.uci.edu
* $5 = /pub/ietf/uri/
* $6 = <undefined>
* $7 = <undefined>
* $8 = #Related
* $9 = Related
*
* where <undefined> indicates that the component is not present, as is
* the case for the query component in the above example. Therefore, we
* can determine the value of the four components and fragment as
*
* scheme = $2
* authority = $4
* path = $5
* query = $7
* fragment = $9
* </pre>
*
* --
* <p>Below differs from the rfc regex in that...
* (1) it has java escaping of regex characters
* (2) we allow a URI made of a fragment only (Added extra
* group so indexing is off by one after scheme).
* (3) scheme is limited to legal scheme characters
*/
final public static Pattern RFC2396REGEX = Pattern.compile(
"^(([a-zA-Z][a-zA-Z0-9\\+\\-\\.]*):)?((//([^/?#]*))?([^?#]*)(\\?([^#]*))?)?(#(.*))?");
// 12 34 5 6 7 8 9 A
// 2 1 54 6 87 3 A9 // 1: scheme
// 2: scheme:
// 3: //authority/path
// 4: //authority
// 5: authority
// 6: path
// 7: ?query
// 8: query
// 9: #fragment
// A: fragment
public static final String SLASHDOTDOTSLASH = "^(/\\.\\./)+";
public static final String SLASH = "/";
public static final String HTTP = "http";
public static final String HTTP_PORT = ":80";
public static final String HTTPS = "https";
public static final String HTTPS_PORT = ":443";
public static final String DOT = ".";
public static final String EMPTY_STRING = "";
public static final String NBSP = "\u00A0";
public static final String SPACE = " ";
public static final String ESCAPED_SPACE = "%20";
public static final String TRAILING_ESCAPED_SPACE = "^(.*)(%20)+$";
public static final String PIPE = "|";
public static final String PIPE_PATTERN = "\\|";
public static final String ESCAPED_PIPE = "%7C";
public static final String CIRCUMFLEX = "^";
public static final String CIRCUMFLEX_PATTERN = "\\^";
public static final String ESCAPED_CIRCUMFLEX = "%5E";
public static final String QUOT = "\"";
public static final String ESCAPED_QUOT = "%22";
public static final String SQUOT = "'";
public static final String ESCAPED_SQUOT = "%27";
public static final String APOSTROPH = "`";
public static final String ESCAPED_APOSTROPH = "%60";
public static final String LSQRBRACKET = "[";
public static final String LSQRBRACKET_PATTERN = "\\[";
public static final String ESCAPED_LSQRBRACKET = "%5B";
public static final String RSQRBRACKET = "]";
public static final String RSQRBRACKET_PATTERN = "\\]";
public static final String ESCAPED_RSQRBRACKET = "%5D";
public static final String LCURBRACKET = "{";
public static final String LCURBRACKET_PATTERN = "\\{";
public static final String ESCAPED_LCURBRACKET = "%7B";
public static final String RCURBRACKET = "}";
public static final String RCURBRACKET_PATTERN = "\\}";
public static final String ESCAPED_RCURBRACKET = "%7D";
public static final String BACKSLASH = "\\";
public static final String ESCAPED_BACKSLASH = "%5C";
public static final String STRAY_SPACING = "[\n\r\t]+";
public static final String IMPROPERESC_REPLACE = "%25$1";
public static final String IMPROPERESC =
"%((?:[^\\p{XDigit}])|(?:.[^\\p{XDigit}])|(?:\\z))";
public static final String COMMERCIAL_AT = "@";
public static final char PERCENT_SIGN = '%';
public static final char COLON = ':';
/**
* First percent sign in string followed by two hex chars.
*/
public static final String URI_HEX_ENCODING =
"^[^%]*%[\\p{XDigit}][\\p{XDigit}].*";
/**
* Authority port number regex.
*/
final static Pattern PORTREGEX = Pattern.compile("(.*:)([0-9]+)$");
/**
* Characters we'll accept in the domain label part of a URI
* authority: ASCII letters-digits-hyphen (LDH) plus underscore,
* with single intervening '.' characters.
*
* (We accept '_' because DNS servers have tolerated for many
* years counter to spec; we also accept dash patterns and ACE
* prefixes that will be rejected by IDN-punycoding attempt.)
*/
final static String ACCEPTABLE_ASCII_DOMAIN =
"^(?:[a-zA-Z0-9_-]++(?:\\.)?)++$";
/**
* Pattern that looks for case of three or more slashes after the
* scheme. If found, we replace them with two only as mozilla does.
*/
final static Pattern HTTP_SCHEME_SLASHES =
Pattern.compile("^(https?://)/+(.*)");
/**
* Pattern that looks for case of two or more slashes in a path.
*/
final static Pattern MULTIPLE_SLASHES = Pattern.compile("//+");
/**
* Protected constructor.
*/
private UURIFactory() {
super();
}
/**
* @param uri URI as string.
* @return An instance of UURI
* @throws URIException
*/
public static UURI getInstance(String uri) throws URIException {
return UURIFactory.factory.create(uri);
return (UURI) UURIFactory.factory.create(uri);
}
/**
* @param uri URI as string.
* @param charset Character encoding of the passed uri string.
* @return An instance of UURI
* @throws URIException
*/
public static UURI getInstance(String uri, String charset)
throws URIException {
return UURIFactory.factory.create(uri, charset);
}
/**
* @param base Base uri to use resolving passed relative uri.
* @param relative URI as string.
@@ -251,552 +51,20 @@ public class UURIFactory extends URI {
* @throws URIException
*/
public static UURI getInstance(UURI base, String relative)
throws URIException {
// return base.resolve(relative);
return UURIFactory.factory.create(base, relative);
throws URIException {
return (UURI) UURIFactory.factory.create(base, relative);
}
/**
* @param uri URI as string.
* @return Instance of UURI.
* @throws URIException
*/
private UURI create(String uri) throws URIException {
return create(uri, UURI.getDefaultProtocolCharset());
@Override
protected UURI makeOne(String fixedUpUri, boolean escaped, String charset)
throws URIException {
return new UURI(fixedUpUri, escaped, charset);
}
/**
* @param uri URI as string.
* @param charset Original encoding of the string.
* @return Instance of UURI.
* @throws URIException
*/
private UURI create(String uri, String charset) throws URIException {
UURI uuri = new UURI(fixup(uri, null, charset), true, charset);
if (logger.isLoggable(Level.FINE)) {
logger.fine("URI " + uri +
" PRODUCT " + uuri.toString() +
" CHARSET " + charset);
}
return validityCheck(uuri);
}
/**
* @param base UURI to use as a base resolving <code>relative</code>.
* @param relative Relative URI.
* @return Instance of UURI.
* @throws URIException
*/
private UURI create(UURI base, String relative) throws URIException {
UURI uuri = new UURI(base, new UURI(fixup(relative, base, base.getProtocolCharset()),
true, base.getProtocolCharset()));
if (logger.isLoggable(Level.FINE)) {
logger.fine(" URI " + relative +
" PRODUCT " + uuri.toString() +
" CHARSET " + base.getProtocolCharset() +
" BASE " + base);
}
return validityCheck(uuri);
@Override
protected UsableURI makeOne(UsableURI base, UsableURI relative) throws URIException {
// return new UURI(base, relative);
return new UURI(base, relative);
}
/**
* Check the generated UURI.
*
* At the least look at length of uuri string. We were seeing case
* where before escaping, string was &lt; MAX_URL_LENGTH but after was
* &gt;. Letting out a too-big message was causing us troubles later
* down the processing chain.
* @param uuri Created uuri to check.
* @return The passed <code>uuri</code> so can easily inline this check.
* @throws URIException
*/
protected UURI validityCheck(UURI uuri) throws URIException {
if (uuri.getRawURI().length > UURI.MAX_URL_LENGTH) {
throw new URIException("Created (escaped) uuri > " +
UURI.MAX_URL_LENGTH +": "+uuri.toString());
}
return uuri;
}
/**
* Do heritrix fix-up on passed uri string.
*
* Does heritrix escaping; usually escaping done to make our behavior align
* with IEs. This method codifies our experience pulling URIs from the
* wilds. Its does all the escaping we want; its output can always be
* assumed to be 'escaped' (though perhaps to a laxer standard than the
* vanilla HttpClient URI class or official specs might suggest).
*
* @param uri URI as string.
* @param base May be null.
* @param e True if the uri is already escaped.
* @return A fixed up URI string.
* @throws URIException
*/
private String fixup(String uri, final URI base, final String charset)
throws URIException {
if (uri == null) {
throw new NullPointerException();
} else if (uri.length() == 0 && base == null) {
throw new URIException("URI length is zero (and not relative).");
}
if (uri.length() > UURI.MAX_URL_LENGTH) {
// We check length here and again later after all convertions.
throw new URIException("URI length > " + UURI.MAX_URL_LENGTH +
": " + uri);
}
// Replace nbsp with normal spaces (so that they get stripped if at
// ends, or encoded if in middle)
if (uri.indexOf(NBSP) >= 0) {
uri = TextUtils.replaceAll(NBSP, uri, SPACE);
}
// Get rid of any trailing spaces or new-lines.
uri = uri.trim();
// IE converts backslashes preceding the query string to slashes, rather
// than to %5C. Since URIs that have backslashes usually work only with
// IE, we will convert backslashes to slashes as well.
int nextBackslash = uri.indexOf(BACKSLASH);
if (nextBackslash >= 0) {
int queryStart = uri.indexOf('?');
StringBuilder tmp = new StringBuilder(uri);
while (nextBackslash >= 0
&& (queryStart < 0 || nextBackslash < queryStart)) {
tmp.setCharAt(nextBackslash, '/');
nextBackslash = uri.indexOf(BACKSLASH, nextBackslash + 1);
}
uri = tmp.toString();
}
// Remove stray TAB/CR/LF
uri = TextUtils.replaceAll(STRAY_SPACING, uri, EMPTY_STRING);
// Test for the case of more than two slashes after the http(s) scheme.
// Replace with two slashes as mozilla does if found.
// See [ 788219 ] URI Syntax Errors stop page parsing.
// Matcher matcher = HTTP_SCHEME_SLASHES.matcher(uri);
Matcher matcher = TextUtils.getMatcher(HTTP_SCHEME_SLASHES.pattern(), uri);
if (matcher.matches()) {
uri = matcher.group(1) + matcher.group(2);
}
TextUtils.recycleMatcher(matcher);
// now, minimally escape any whitespace
uri = escapeWhitespace(uri);
// For further processing, get uri elements. See the RFC2396REGEX
// comment above for explanation of group indices used in the below.
// matcher = RFC2396REGEX.matcher(uri);
matcher = TextUtils.getMatcher(RFC2396REGEX.pattern(), uri);
if (!matcher.matches()) {
throw new URIException("Failed parse of " + uri);
}
String uriScheme = checkUriElementAndLowerCase(matcher.group(2));
String uriSchemeSpecificPart = checkUriElement(matcher.group(3));
String uriAuthority = checkUriElement(matcher.group(5));
String uriPath = checkUriElement(matcher.group(6));
String uriQuery = checkUriElement(matcher.group(8));
// UNUSED String uriFragment = checkUriElement(matcher.group(10));
TextUtils.recycleMatcher(matcher); matcher = null;
// Test if relative URI. If so, need a base to resolve against.
if (uriScheme == null || uriScheme.length() <= 0) {
if (base == null) {
throw new URIException("Relative URI but no base: " + uri);
}
} else {
checkHttpSchemeSpecificPartSlashPrefix(base, uriScheme,
uriSchemeSpecificPart);
}
// fixup authority portion: lowercase/IDN-punycode any domain;
// remove stray trailing spaces
uriAuthority = fixupAuthority(uriAuthority,charset);
// Do some checks if absolute path.
if (uriSchemeSpecificPart != null &&
uriSchemeSpecificPart.startsWith(SLASH)) {
if (uriPath != null) {
// Eliminate '..' if its first thing in the path. IE does this.
uriPath = TextUtils.replaceFirst(SLASHDOTDOTSLASH, uriPath,
SLASH);
}
// Ensure root URLs end with '/': browsers always send "/"
// on the request-line, so we should consider "http://host"
// to be "http://host/".
if (uriPath == null || EMPTY_STRING.equals(uriPath)) {
uriPath = SLASH;
}
}
if (uriAuthority != null) {
if (uriScheme != null && uriScheme.length() > 0 &&
uriScheme.equals(HTTP)) {
uriAuthority = checkPort(uriAuthority);
uriAuthority = stripTail(uriAuthority, HTTP_PORT);
} else if (uriScheme != null && uriScheme.length() > 0 &&
uriScheme.equals(HTTPS)) {
uriAuthority = checkPort(uriAuthority);
uriAuthority = stripTail(uriAuthority, HTTPS_PORT);
}
// Strip any prefix dot or tail dots from the authority.
uriAuthority = stripTail(uriAuthority, DOT);
uriAuthority = stripPrefix(uriAuthority, DOT);
} else {
// no authority; may be relative. consider stripping scheme
// to work-around org.apache.commons.httpclient.URI bug
// ( http://issues.apache.org/jira/browse/HTTPCLIENT-587 )
if (uriScheme != null && base != null
&& uriScheme.equals(base.getScheme())) {
// uriScheme redundant and will only confound httpclient.URI
uriScheme = null;
}
}
// Ensure minimal escaping. Use of 'lax' URI and URLCodec
// means minimal escaping isn't necessarily complete/consistent.
// There is a chance such lax encoding will throw exceptions
// later at inconvenient times.
//
// One reason for these bad escapings -- though not the only --
// is that the page is using an encoding other than the ASCII or the
// UTF-8 that is our default URI encoding. In this case the parent
// class is burping on the passed URL encoding. If the page encoding
// was passed into this factory, the encoding seems to be parsed
// correctly (See the testEscapedEncoding unit test).
//
// This fixup may cause us to miss content. There is the charset case
// noted above. TODO: Look out for cases where we fail other than for
// the above given reason which will be fixed when we address
// '[ 913687 ] Make extractors interrogate for charset'.
uriPath = ensureMinimalEscaping(uriPath, charset);
uriQuery = ensureMinimalEscaping(uriQuery, charset,
LaxURLCodec.QUERY_SAFE);
// Preallocate. The '1's and '2's in below are space for ':',
// '//', etc. URI characters.
MutableString s = new MutableString(
((uriScheme != null)? uriScheme.length(): 0)
+ 1 // ';'
+ ((uriAuthority != null)? uriAuthority.length(): 0)
+ 2 // '//'
+ ((uriPath != null)? uriPath.length(): 0)
+ 1 // '?'
+ ((uriQuery != null)? uriQuery.length(): 0));
appendNonNull(s, uriScheme, ":", true);
appendNonNull(s, uriAuthority, "//", false);
appendNonNull(s, uriPath, "", false);
appendNonNull(s, uriQuery, "?", false);
return s.toString();
}
/**
* If http(s) scheme, check scheme specific part begins '//'.
* @throws URIException
* @see http://www.faqs.org/rfcs/rfc1738.html Section 3.1. Common Internet
* Scheme Syntax
*/
protected void checkHttpSchemeSpecificPartSlashPrefix(final URI base,
final String scheme, final String schemeSpecificPart)
throws URIException {
if (scheme == null || scheme.length() <= 0) {
return;
}
if (!scheme.equals("http") && !scheme.equals("https")) {
return;
}
if ( schemeSpecificPart == null
|| !schemeSpecificPart.startsWith("//")) {
// only acceptable if schemes match
if (base == null || !scheme.equals(base.getScheme())) {
throw new URIException(
"relative URI with scheme only allowed for " +
"scheme matching base");
}
return;
}
if (schemeSpecificPart.length() <= 2) {
throw new URIException("http scheme specific part is " +
"too short: " + schemeSpecificPart);
}
}
/**
* Fixup 'authority' portion of URI, by removing any stray
* encoded spaces, lowercasing any domain names, and applying
* IDN-punycoding to Unicode domains.
*
* @param uriAuthority the authority string to fix
* @return fixed version
* @throws URIException
*/
private String fixupAuthority(String uriAuthority, String charset) throws URIException {
// Lowercase the host part of the uriAuthority; don't destroy any
// userinfo capitalizations. Make sure no illegal characters in
// domainlabel substring of the uri authority.
if (uriAuthority != null) {
// Get rid of any trailing escaped spaces:
// http://www.archive.org%20. Rare but happens.
// TODO: reevaluate: do IE or firefox do such mid-URI space-removal?
// if not, we shouldn't either.
while(uriAuthority.endsWith(ESCAPED_SPACE)) {
uriAuthority = uriAuthority.substring(0,uriAuthority.length()-3);
}
// lowercase & IDN-punycode only the domain portion
int atIndex = uriAuthority.indexOf(COMMERCIAL_AT);
int portColonIndex = uriAuthority.indexOf(COLON,(atIndex<0)?0:atIndex);
if(atIndex<0 && portColonIndex<0) {
// most common case: neither userinfo nor port
return fixupDomainlabel(uriAuthority);
} else if (atIndex<0 && portColonIndex>-1) {
// next most common: port but no userinfo
String domain = fixupDomainlabel(uriAuthority.substring(0,portColonIndex));
String port = uriAuthority.substring(portColonIndex);
return domain + port;
} else if (atIndex>-1 && portColonIndex<0) {
// uncommon: userinfo, no port
String userinfo = ensureMinimalEscaping(uriAuthority.substring(0,atIndex+1),charset);
String domain = fixupDomainlabel(uriAuthority.substring(atIndex+1));
return userinfo + domain;
} else {
// uncommon: userinfo, port
String userinfo = ensureMinimalEscaping(uriAuthority.substring(0,atIndex+1),charset);
String domain = fixupDomainlabel(uriAuthority.substring(atIndex+1,portColonIndex));
String port = uriAuthority.substring(portColonIndex);
return userinfo + domain + port;
}
}
return uriAuthority;
}
/**
* Fixup the domain label part of the authority.
*
* We're more lax than the spec. in that we allow underscores.
*
* @param label Domain label to fix.
* @return Return fixed domain label.
* @throws URIException
*/
private String fixupDomainlabel(String label)
throws URIException {
// apply IDN-punycoding, as necessary
try {
// TODO: optimize: only apply when necessary, or
// keep cache of recent encodings
label = IDNA.toASCII(label);
} catch (IDNAException e) {
if(TextUtils.matches(ACCEPTABLE_ASCII_DOMAIN,label)) {
// domain name has ACE prefix, leading/trailing dash, or
// underscore -- but is still a name we wish to tolerate;
// simply continue
} else {
// problematic domain: neither ASCII acceptable characters
// nor IDN-punycodable, so throw exception
// TODO: change to HeritrixURIException so distinguishable
// from URIExceptions in library code
URIException ue = new URIException(e+" "+label);
ue.initCause(e);
throw ue;
}
}
label = label.toLowerCase();
return label;
}
/**
* Ensure that there all characters needing escaping
* in the passed-in String are escaped. Stray '%' characters
* are *not* escaped, as per browser behavior.
*
* @param u String to escape
* @param charset
* @return string with any necessary escaping applied
*/
private String ensureMinimalEscaping(String u, final String charset) {
return ensureMinimalEscaping(u, charset, LaxURLCodec.EXPANDED_URI_SAFE);
}
/**
* Ensure that there all characters needing escaping
* in the passed-in String are escaped. Stray '%' characters
* are *not* escaped, as per browser behavior.
*
* @param u String to escape
* @param charset
* @param bitset
* @return string with any necessary escaping applied
*/
private String ensureMinimalEscaping(String u, final String charset,
final BitSet bitset) {
if (u == null) {
return null;
}
for (int i = 0; i < u.length(); i++) {
char c = u.charAt(i);
if (!bitset.get(c)) {
try {
u = LaxURLCodec.DEFAULT.encode(bitset, u, charset);
} catch (UnsupportedEncodingException e) {
e.printStackTrace();
}
break;
}
}
return u;
}
/**
* Escape any whitespace found.
*
* The parent class takes care of the bulk of escaping. But if any
* instance of escaping is found in the URI, then we ask for parent
* to do NO escaping. Here we escape any whitespace found irrespective
* of whether the uri has already been escaped. We do this for
* case where uri has been judged already-escaped only, its been
* incompletly done and whitespace remains. Spaces, etc., in the URI are
* a real pain. Their presence will break log file and ARC parsing.
* @param uri URI string to check.
* @return uri with spaces escaped if any found.
*/
protected String escapeWhitespace(String uri) {
// Just write a new string anyways. The perl '\s' is not
// as inclusive as the Character.isWhitespace so there are
// whitespace characters we could miss. So, rather than
// write some awkward regex, just go through the string
// a character at a time. Only create buffer first time
// we find a space.
MutableString buffer = null;
for (int i = 0; i < uri.length(); i++) {
char c = uri.charAt(i);
if (Character.isWhitespace(c)) {
if (buffer == null) {
buffer = new MutableString(uri.length() +
2 /*If space, two extra characters (at least)*/);
buffer.append(uri.substring(0, i));
}
buffer.append("%");
String hexStr = Integer.toHexString(c);
if ((hexStr.length() % 2) > 0) {
buffer.append("0");
}
buffer.append(hexStr);
} else {
if (buffer != null) {
buffer.append(c);
}
}
}
return (buffer != null)? buffer.toString(): uri;
}
/**
* Check port on passed http authority. Make sure the size is not larger
* than allowed: See the 'port' definition on this
* page, http://www.kerio.com/manual/wrp/en/418.htm.
* Also, we've seen port numbers of '0080' whose leading zeros confuse
* the parent class. Strip the leading zeros.
*
* @param uriAuthority
* @return Null or an amended port number.
* @throws URIException
*/
private String checkPort(String uriAuthority)
throws URIException {
// Matcher m = PORTREGEX.matcher(uriAuthority);
Matcher m = TextUtils.getMatcher(PORTREGEX.pattern(), uriAuthority);
if (m.matches()) {
String no = m.group(2);
if (no != null && no.length() > 0) {
// First check if the port has leading zeros
// as in '0080'. Strip them if it has and
// then reconstitute the uriAuthority. Be careful
// of cases where port is '0' or '000'.
while (no.charAt(0) == '0' && no.length() > 1) {
no = no.substring(1);
}
uriAuthority = m.group(1) + no;
// Now makesure the number is legit.
int portNo = 0;
try {
portNo = Integer.parseInt(no);
} catch (NumberFormatException nfe) {
// just catch and leave portNo at illegal 0
}
if (portNo <= 0 || portNo > 65535) {
throw new URIException("Port out of bounds: " +
uriAuthority);
}
}
}
TextUtils.recycleMatcher(m);
return uriAuthority;
}
/**
* @param b Buffer to append to.
* @param str String to append if not null.
* @param substr Suffix or prefix to use if <code>str</code> is not null.
* @param suffix True if <code>substr</code> is a suffix.
*/
private void appendNonNull(MutableString b, String str, String substr,
boolean suffix) {
if (str != null && str.length() > 0) {
if (!suffix) {
b.append(substr);
}
b.append(str);
if (suffix) {
b.append(substr);
}
}
}
/**
* @param str String to work on.
* @param prefix Prefix to strip if present.
* @return <code>str</code> w/o <code>prefix</code>.
*/
private String stripPrefix(String str, String prefix) {
return str.startsWith(prefix)?
str.substring(prefix.length(), str.length()):
str;
}
/**
* @param str String to work on.
* @param tail Tail to strip if present.
* @return <code>str</code> w/o <code>tail</code>.
*/
private static String stripTail(String str, String tail) {
return str.endsWith(tail)?
str.substring(0, str.length() - tail.length()):
str;
}
/**
* @param element to examine.
* @return Null if passed null or an empty string otherwise
* <code>element</code>.
*/
private String checkUriElement(String element) {
return (element == null || element.length() <= 0)? null: element;
}
/**
* @param element to examine and lowercase if non-null.
* @return Null if passed null or an empty string otherwise
* <code>element</code> lowercased.
*/
private String checkUriElementAndLowerCase(String element) {
String tmp = checkUriElement(element);
return (tmp != null)? tmp.toLowerCase(): tmp;
}
}
commons/src/main/java/org/archive/util/ArchiveUtils.java
-1008
View File
File diff suppressed because it is too large Load Diff
commons/src/main/java/org/archive/util/InterruptibleCharSequence.java
-56
View File
@@ -1,56 +0,0 @@
/*
* This file is part of the Heritrix web crawler (crawler.archive.org).
*
* Licensed to the Internet Archive (IA) by one or more individual
* contributors.
*
* The IA licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.archive.util;
/**
* CharSequence that noticed thread interrupts -- as might be necessary
* to recover from a loose regex on unexpected challenging input.
*
* @author gojomo
*/
public class InterruptibleCharSequence implements CharSequence {
protected CharSequence inner;
// public long counter = 0;
public InterruptibleCharSequence(CharSequence inner) {
super();
this.inner = inner;
}
public char charAt(int index) {
if (Thread.interrupted()) { // clears flag if set
throw new RuntimeException(new InterruptedException());
}
// counter++;
return inner.charAt(index);
}
public int length() {
return inner.length();
}
public CharSequence subSequence(int start, int end) {
return new InterruptibleCharSequence(inner.subSequence(start, end));
}
@Override
public String toString() {
return inner.toString();
}
}
commons/src/main/java/org/archive/util/InterruptibleCharSequenceTest.java
-120
View File
@@ -1,120 +0,0 @@
/*
* This file is part of the Heritrix web crawler (crawler.archive.org).
*
* Licensed to the Internet Archive (IA) by one or more individual
* contributors.
*
* The IA licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.archive.util;
import java.util.concurrent.BlockingQueue;
import java.util.concurrent.LinkedBlockingQueue;
import java.util.regex.Pattern;
import junit.framework.TestCase;
/**
* Tests (and
* @author gojomo
*/
public class InterruptibleCharSequenceTest extends TestCase {
// this regex takes many seconds to fail on the input
// (~20 seconds on 2Ghz Athlon64 JDK 1.6)
public static String BACKTRACKER = "^(((((a+)*)*)*)*)*$";
public static String INPUT = "aaaaab";
/**
* Development-time benchmarking of InterruptibleCharSequence in
* regex use. (Rename 'xest' to 'test' if wanted as unit test,
* but never actually fails anything -- just measures.)
*
* For reference the regex "^(((((a+)*)*)*)*)*$" requires
* 239,286,636 charAt(s) to fail on "aaaaab", which takes
* around 20 seconds on a 2Ghz Athlon64(x2) with JDK 1.6.
* The runtime overhead of checking interrupt status in this
* extreme case is around 5% in my tests.
*/
public void xestOverhead() {
String regex = BACKTRACKER;
String inputNormal = INPUT;
InterruptibleCharSequence inputWrapped = new InterruptibleCharSequence(inputNormal);
// warm up
tryMatch(inputNormal,regex);
tryMatch(inputWrapped,regex);
// inputWrapped.counter=0;
int trials = 5;
long stringTally = 0;
long icsTally = 0;
for(int i = 1; i <= trials; i++) {
System.out.println("trial "+i+" of "+trials);
long start = System.currentTimeMillis();
System.out.print("String ");
tryMatch(inputNormal,regex);
long end = System.currentTimeMillis();
System.out.println(end-start);
stringTally += (end-start);
start = System.currentTimeMillis();
System.out.print("InterruptibleCharSequence ");
tryMatch(inputWrapped,regex);
end = System.currentTimeMillis();
System.out.println(end-start);
//System.out.println(inputWrapped.counter+" steps");
//inputWrapped.counter=0;
icsTally += (end-start);
}
System.out.println("InterruptibleCharSequence took "+((float)icsTally)/stringTally+" longer.");
}
public boolean tryMatch(CharSequence input, String regex) {
return Pattern.matches(regex,input);
}
public Thread tryMatchInThread(final CharSequence input, final String regex, final BlockingQueue<Object> atFinish) {
Thread t = new Thread() {
public void run() {
boolean result;
try {
result = tryMatch(input,regex);
} catch (Exception e) {
atFinish.offer(e);
return;
}
atFinish.offer(result);
}
};
t.start();
return t;
}
public void testNoninterruptible() throws InterruptedException {
BlockingQueue<Object> q = new LinkedBlockingQueue<Object>();
Thread t = tryMatchInThread(INPUT, BACKTRACKER, q);
Thread.sleep(1000);
t.interrupt();
Object result = q.take();
assertTrue("mismatch uncompleted",Boolean.FALSE.equals(result));
}
public void testInterruptibility() throws InterruptedException {
BlockingQueue<Object> q = new LinkedBlockingQueue<Object>();
Thread t = tryMatchInThread(new InterruptibleCharSequence(INPUT), BACKTRACKER, q);
Thread.sleep(500);
t.interrupt();
Object result = q.take();
if(result instanceof Boolean) {
System.err.println(result+" match beat interrupt");
}
assertTrue("exception not thrown",result instanceof RuntimeException);
}
}
commons/src/main/java/org/archive/util/PrefixSet.java
-74
View File
@@ -1,74 +0,0 @@
/*
* This file is part of the Heritrix web crawler (crawler.archive.org).
*
* Licensed to the Internet Archive (IA) by one or more individual
* contributors.
*
* The IA licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.archive.util;
import java.util.SortedSet;
import java.util.concurrent.ConcurrentSkipListSet;
/**
* Utility class for maintaining sorted set of string prefixes.
* Redundant prefixes are coalesced into the shorter prefix.
*/
public class PrefixSet extends ConcurrentSkipListSet<String> {
private static final long serialVersionUID = -6054697706348411992L;
public PrefixSet() {
super();
}
/**
* Test whether the given String is prefixed by one
* of this set's entries.
*
* @param s
* @return True if contains prefix.
*/
public boolean containsPrefixOf(String s) {
SortedSet<String> sub = headSet(s);
// because redundant prefixes have been eliminated,
// only a test against last item in headSet is necessary
if (!sub.isEmpty() && s.startsWith((String)sub.last())) {
return true; // prefix substring exists
} // else: might still exist exactly (headSet does not contain boundary)
return contains(s); // exact string exists, or no prefix is there
}
/**
* Maintains additional invariant: if one entry is a
* prefix of another, keep only the prefix.
*
* @see java.util.Collection#add(java.lang.Object)
*/
public boolean add(String s) {
SortedSet<String> sub = headSet(s);
if (!sub.isEmpty() && s.startsWith((String)sub.last())) {
// no need to add; prefix is already present
return false;
}
boolean retVal = super.add(s);
sub = tailSet(s+"\0");
while(!sub.isEmpty() && ((String)sub.first()).startsWith(s)) {
// remove redundant entries
sub.remove(sub.first());
}
return retVal;
}
}
commons/src/main/java/org/archive/util/ReportUtils.java
+45
View File
@@ -0,0 +1,45 @@
/*
* This file is part of the Heritrix web crawler (crawler.archive.org).
*
* Licensed to the Internet Archive (IA) by one or more individual
* contributors.
*
* The IA licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.archive.util;
import java.io.IOException;
import java.io.PrintWriter;
import java.io.StringWriter;
public class ReportUtils {
/**
* Utility method to get a String shortReportLine from Reporter
* @param rep Reporter to get shortReportLine from
* @return String of report
*/
public static String shortReportLine(Reporter rep) {
StringWriter sw = new StringWriter();
PrintWriter pw = new PrintWriter(sw);
try {
rep.shortReportLineTo(pw);
} catch (IOException e) {
// not really possible
e.printStackTrace();
}
pw.flush();
return sw.toString();
}
}
commons/src/main/java/org/archive/util/SURT.java
-256
View File
@@ -1,256 +0,0 @@
/*
* This file is part of the Heritrix web crawler (crawler.archive.org).
*
* Licensed to the Internet Archive (IA) by one or more individual
* contributors.
*
* The IA licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.archive.util;
import java.io.BufferedInputStream;
import java.io.BufferedOutputStream;
import java.io.BufferedReader;
import java.io.FileInputStream;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.io.PrintStream;
import java.util.regex.Matcher;
import org.apache.commons.httpclient.URIException;
import org.archive.net.UURIFactory;
/**
* Sort-friendly URI Reordering Transform.
*
* Converts URIs of the form:
*
* scheme://userinfo@domain.tld:port/path?query#fragment
*
* ...into...
*
* scheme://(tld,domain,:port@userinfo)/path?query#fragment
*
* The '(' ')' characters serve as an unambiguous notice that the so-called
* 'authority' portion of the URI ([userinfo@]host[:port] in http URIs) has
* been transformed; the commas prevent confusion with regular hostnames.
*
* This remedies the 'problem' with standard URIs that the host portion of a
* regular URI, with its dotted-domains, is actually in reverse order from
* the natural hierarchy that's usually helpful for grouping and sorting.
*
* The value of respecting URI case variance is considered negligible: it
* is vanishingly rare for case-variance to be meaningful, while URI case-
* variance often arises from people's confusion or sloppiness, and they
* only correct it insofar as necessary to avoid blatant problems. Thus
* the usual SURT form is considered to be flattened to all lowercase, and
* not completely reversible.
*
* @author gojomo
*/
public class SURT {
protected static char DOT = '.';
protected static String BEGIN_TRANSFORMED_AUTHORITY = "(";
protected static String TRANSFORMED_HOST_DELIM = ",";
protected static String END_TRANSFORMED_AUTHORITY = ")";
// 1: scheme://
// 2: userinfo (if present)
// 3: @ (if present)
// 4: dotted-quad host
// 5: other host
// 6: :port
// 7: path
protected static String URI_SPLITTER =
"^(\\w+://)(?:([-\\w\\.!~\\*'\\(\\)%;:&=+$,]+?)(@))?"+
// 1 2 3
"(?:((?:\\d{1,3}\\.){3}\\d{1,3})|(\\S+?))(:\\d+)?(/\\S*)?$";
// 4 5 6 7
// RFC2396
// reserved = ";" | "/" | "?" | ":" | "@" | "&" | "=" | "+" |
// "$" | ","
// unreserved = alphanum | mark
// mark = "-" | "_" | "." | "!" | "~" | "*" | "'" | "(" | ")"
// userinfo = *( unreserved | escaped |
// ";" | ":" | "&" | "=" | "+" | "$" | "," )
// escaped = "%" hex hex
/**
* Utility method for creating the SURT form of the URI in the
* given String.
*
* By default, does not preserve casing.
*
* @param s String URI to be converted to SURT form
* @return SURT form
*/
public static String fromURI(String s) {
return fromURI(s,false);
}
/**
* Utility method for creating the SURT form of the URI in the
* given String.
*
* If it appears a bit convoluted in its approach, note that it was
* optimized to minimize object-creation after allocation-sites profiling
* indicated this method was a top source of garbage in long-running crawls.
*
* Assumes that the String URI has already been cleaned/fixed (eg
* by UURI fixup) in ways that put it in its crawlable form for
* evaluation.
*
* @param s String URI to be converted to SURT form
* @param preserveCase whether original case should be preserved
* @return SURT form
*/
public static String fromURI(String s, boolean preserveCase) {
Matcher m = TextUtils.getMatcher(URI_SPLITTER,s);
if(!m.matches()) {
// not an authority-based URI scheme; return unchanged
TextUtils.recycleMatcher(m);
return s;
}
// preallocate enough space for SURT form, which includes
// 3 extra characters ('(', ')', and one more ',' than '.'s
// in original)
StringBuffer builder = new StringBuffer(s.length()+3);
append(builder,s,m.start(1),m.end(1)); // scheme://
builder.append(BEGIN_TRANSFORMED_AUTHORITY); // '('
if(m.start(4)>-1) {
// dotted-quad ip match: don't reverse
append(builder,s,m.start(4),m.end(4));
} else {
// other hostname match: do reverse
int hostSegEnd = m.end(5);
int hostStart = m.start(5);
for(int i = m.end(5)-1; i>=hostStart; i--) {
if(s.charAt(i-1)!=DOT && i > hostStart) {
continue;
}
append(builder,s,i,hostSegEnd); // rev host segment
builder.append(TRANSFORMED_HOST_DELIM); // ','
hostSegEnd = i-1;
}
}
append(builder,s,m.start(6),m.end(6)); // :port
append(builder,s,m.start(3),m.end(3)); // at
append(builder,s,m.start(2),m.end(2)); // userinfo
builder.append(END_TRANSFORMED_AUTHORITY); // ')'
append(builder,s,m.start(7),m.end(7)); // path
if (!preserveCase) {
for(int i = 0; i < builder.length(); i++) {
builder.setCharAt(i,Character.toLowerCase(builder.charAt((i))));
}
}
TextUtils.recycleMatcher(m);
return builder.toString();
}
private static void append(StringBuffer b, CharSequence cs, int start,
int end) {
if (start < 0) {
return;
}
b.append(cs, start, end);
}
/**
* Given a plain URI or hostname/hostname+path, deduce an implied SURT
* prefix from it. Results may be unpredictable on strings that cannot
* be interpreted as URIs.
*
* UURI 'fixup' is applied to the URI that is built.
*
* @param u URI or almost-URI to consider
* @return implied SURT prefix form
*/
public static String prefixFromPlain(String u) {
u = fromPlain(u);
// truncate to implied prefix
u = SurtPrefixSet.asPrefix(u);
return u;
}
/**
* Given a plain URI or hostname/hostname+path, give its SURT form.
* Results may be unpredictable on strings that cannot
* be interpreted as URIs.
*
* UURI 'fixup' is applied to the URI before conversion to SURT
* form.
*
* @param u URI or almost-URI to consider
* @return implied SURT prefix form
*/
public static String fromPlain(String u) {
u = ArchiveUtils.addImpliedHttpIfNecessary(u);
boolean trailingSlash = u.endsWith("/");
// ensure all typical UURI cleanup (incl. IDN-punycoding) is done
try {
u = UURIFactory.getInstance(u).toString();
} catch (URIException e) {
e.printStackTrace();
// allow to continue with original string uri
}
// except: don't let UURI-fixup add a trailing slash
// if it wasn't already there (presence or absence of
// such slash has special meaning specifying implied
// SURT prefixes)
if(!trailingSlash && u.endsWith("/")) {
u = u.substring(0,u.length()-1);
}
// convert to full SURT
u = SURT.fromURI(u);
return u;
}
/**
* Allow class to be used as a command-line tool for converting
* URL lists (or naked host or host/path fragments implied
* to be HTTP URLs) to SURT form. Lines that cannot be converted
* are returned unchanged.
*
*
* Read from stdin or first file argument. Writes to stdout or
* second argument filename
*
* @param args cmd-line arguments
* @throws IOException
*/
public static void main(String[] args) throws IOException {
InputStream in = args.length > 0 ? new BufferedInputStream(
new FileInputStream(args[0])) : System.in;
PrintStream out = args.length > 1 ? new PrintStream(
new BufferedOutputStream(new FileOutputStream(args[1])))
: System.out;
BufferedReader br =
new BufferedReader(new InputStreamReader(in));
String line;
while((line = br.readLine())!=null) {
if(line.indexOf("#")>0) line=line.substring(0,line.indexOf("#"));
line = line.trim();
if(line.length()==0) continue;
line = ArchiveUtils.addImpliedHttpIfNecessary(line);
out.println(SURT.fromURI(line));
}
br.close();
out.close();
}
}
commons/src/main/java/org/archive/util/SurtPrefixSet.java
-360
View File
@@ -1,360 +0,0 @@
/*
* This file is part of the Heritrix web crawler (crawler.archive.org).
*
* Licensed to the Internet Archive (IA) by one or more individual
* contributors.
*
* The IA licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.archive.util;
import java.io.BufferedInputStream;
import java.io.BufferedOutputStream;
import java.io.BufferedReader;
import java.io.FileInputStream;
import java.io.FileOutputStream;
import java.io.FileWriter;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.io.PrintStream;
import java.io.Reader;
import java.util.Iterator;
import org.archive.net.UURI;
import org.archive.util.iterator.LineReadingIterator;
import org.archive.util.iterator.RegexLineIterator;
/**
* Specialized TreeSet for keeping a set of String prefixes.
*
* Redundant prefixes (those that are themselves prefixed
* by other set entries) are eliminated.
*
* @author gojomo
*/
public class SurtPrefixSet extends PrefixSet {
private static final long serialVersionUID = 2598365040524933110L;
private static final String SURT_PREFIX_DIRECTIVE = "+";
/**
* Read a set of SURT prefixes from a reader source; keep sorted and
* with redundant entries removed.
*
* @param r reader over file of SURT_format strings
* @throws IOException
*/
public void importFrom(Reader r) {
BufferedReader reader = new BufferedReader(r);
String s;
Iterator<String> iter =
new RegexLineIterator(
new LineReadingIterator(reader),
RegexLineIterator.COMMENT_LINE,
RegexLineIterator.NONWHITESPACE_ENTRY_TRAILING_COMMENT,
RegexLineIterator.ENTRY);
while (iter.hasNext()) {
s = (String) iter.next();
add(s.toLowerCase());
}
}
/**
* @param r Where to read from.
*/
public void importFromUris(Reader r) {
BufferedReader reader = new BufferedReader(r);
String s;
Iterator<String> iter =
new RegexLineIterator(
new LineReadingIterator(reader),
RegexLineIterator.COMMENT_LINE,
RegexLineIterator.NONWHITESPACE_ENTRY_TRAILING_COMMENT,
RegexLineIterator.ENTRY);
while (iter.hasNext()) {
s = (String) iter.next();
// s is a URI (or even fragmentary hostname), not a SURT
addFromPlain(s);
}
}
/**
* Import SURT prefixes from a reader with mixed URI and SURT prefix
* format.
*
* @param r the reader to import the prefixes from
* @param deduceFromSeeds true to also import SURT prefixes implied
* from normal URIs/hostname seeds
*/
public void importFromMixed(Reader r, boolean deduceFromSeeds) {
BufferedReader reader = new BufferedReader(r);
String s;
Iterator<String> iter =
new RegexLineIterator(
new LineReadingIterator(reader),
RegexLineIterator.COMMENT_LINE,
RegexLineIterator.NONWHITESPACE_ENTRY_TRAILING_COMMENT,
RegexLineIterator.ENTRY);
while (iter.hasNext()) {
s = (String) iter.next();
if(s.startsWith(SURT_PREFIX_DIRECTIVE)) {
considerAsAddDirective(s.substring(SURT_PREFIX_DIRECTIVE.length()));
continue;
} else {
if(deduceFromSeeds) {
// also deducing 'implied' SURT prefixes
// from normal URIs/hostname seeds
addFromPlain(s);
}
}
}
}
/**
* Interpret the given SURT/URI/host as a directive, returning true
* if it was meaningful.
*
* @param suri potential directive
* @return boolean true if applied as directive, false otherwise
*/
public boolean considerAsAddDirective(String suri) {
String u = suri.trim();
if(u.length()==0) {
// empty string: consider a mistake
return false;
}
if(u.indexOf("(")>0) {
// formal SURT prefix; toLowerCase just in case
add(u.toLowerCase());
} else {
// hostname/normal form URI from which
// to deduce SURT prefix
addFromPlain(u);
}
return true;
}
/**
* Given a plain URI or hostname, deduce an implied SURT prefix from
* it and add to active prefixes.
*
* @param u String of URI or hostname
*/
public void addFromPlain(String u) {
u = prefixFromPlainForceHttp(u);
add(u);
}
/**
* Given a plain URI or hostname/hostname+path, deduce an implied SURT
* prefix from it. Results may be unpredictable on strings that cannot
* be interpreted as URIs.
*
* UURI 'fixup' is applied to the URI that is built.
*
* HTTPS URIs are changed to HTTP as a convenience for the usual
* preferred common-treatment.
*
* @param u URI or almost-URI to consider
* @return implied SURT prefix form
*/
public static String prefixFromPlainForceHttp(String u) {
u = SURT.prefixFromPlain(u);
u = coerceFromHttpsForComparison(u);
return u;
}
/**
* For SURT comparisons -- prefixes or candidates being checked against
* those prefixes -- we treat https URIs as if they were http.
*
* @param u string to coerce if it has https scheme
* @return string converted to http scheme, or original if not necessary
*/
private static String coerceFromHttpsForComparison(String u) {
if (u.startsWith("https://")) {
u = "http" + u.substring("https".length());
}
return u;
}
/**
* Utility method for truncating a SURT that came from a
* full URI (as a seed, for example) into a prefix
* for determining inclusion.
*
* This involves:
* <pre>
* (1) removing the last path component, if any
* (anything after the last '/', if there are
* at least 3 '/'s)
* (2) removing a trailing ')', if present, opening
* the possibility of proper subdomains. (This
* means that the presence or absence of a
* trailing '/' after a hostname in a seed list
* is significant for the how the SURT prefix is
* created, even though it is not signficant for
* the URI's treatment as a seed.)
* </pre>
*
* @param s String to work on.
* @return As prefix.
*/
public static String asPrefix(String s) {
// Strip last path-segment, if more than 3 slashes
s = s.replaceAll("^(.*//.*/)[^/]*","$1");
// Strip trailing ")", if present and NO path (no 3rd slash).
if (!s.endsWith("/")) {
s = s.replaceAll("^(.*)\\)","$1");
}
return s;
}
/**
* Calculate the SURT form URI to use as a candidate against prefixes
* from the given Object (CandidateURI or UURI)
*
* @param object CandidateURI or UURI
* @return SURT form of URI for evaluation, or null if unavailable
*/
public static String getCandidateSurt(UURI u) {
if (u == null) {
return null;
}
String candidateSurt = u.getSurtForm();
// also want to treat https as http
candidateSurt = coerceFromHttpsForComparison(candidateSurt);
return candidateSurt;
}
/**
* @param fw
* @throws IOException
*/
public void exportTo(FileWriter fw) throws IOException {
Iterator<String> iter = this.iterator();
while(iter.hasNext()) {
fw.write((String)iter.next() + "\n");
}
}
/**
* Changes all prefixes so that they enforce an exact host. For
* prefixes that already include a ')', this means discarding
* anything after ')' (path info). For prefixes that don't include
* a ')' -- domain prefixes open to subdomains -- add the closing
* ')' (or ",)").
*/
public void convertAllPrefixesToHosts() {
SurtPrefixSet iterCopy = (SurtPrefixSet) this.clone();
Iterator<String> iter = iterCopy.iterator();
while (iter.hasNext()) {
String prefix = (String) iter.next();
String convPrefix = convertPrefixToHost(prefix);
if(prefix!=convPrefix) {
// if returned value not unchanged, update set
this.remove(prefix);
this.add(convPrefix);
}
}
}
public static String convertPrefixToHost(String prefix) {
if(prefix.endsWith(")")) {
return prefix; // no change necessary
}
if(prefix.indexOf(')')<0) {
// open-ended domain prefix
if(!prefix.endsWith(",")) {
prefix += ",";
}
prefix += ")";
} else {
// prefix with excess path-info
prefix = prefix.substring(0,prefix.indexOf(')')+1);
}
return prefix;
}
/**
* Changes all prefixes so that they only enforce a general
* domain (allowing subdomains).For prefixes that don't include
* a ')', no change is necessary. For others, truncate everything
* from the ')' onward. Additionally, truncate off "www," if it
* appears.
*/
public void convertAllPrefixesToDomains() {
SurtPrefixSet iterCopy = (SurtPrefixSet) this.clone();
Iterator<String> iter = iterCopy.iterator();
while (iter.hasNext()) {
String prefix = (String) iter.next();
String convPrefix = convertPrefixToDomain(prefix);
if(prefix!=convPrefix) {
// if returned value not unchanged, update set
this.remove(prefix);
this.add(convPrefix);
}
}
}
public static String convertPrefixToDomain(String prefix) {
if(prefix.indexOf(')')>=0) {
prefix = prefix.substring(0,prefix.indexOf(')'));
}
// strip 'www,' when present
if(prefix.endsWith("www,")) {
prefix = prefix.substring(0,prefix.length()-4);
}
return prefix;
}
/**
* Allow class to be used as a command-line tool for converting
* URL lists (or naked host or host/path fragments implied
* to be HTTP URLs) to implied SURT prefix form.
*
* Read from stdin or first file argument. Writes to stdout.
*
* @param args cmd-line arguments: may include input file
* @throws IOException
*/
public static void main(String[] args) throws IOException {
InputStream in = args.length > 0 ? new BufferedInputStream(
new FileInputStream(args[0])) : System.in;
PrintStream out = args.length > 1 ? new PrintStream(
new BufferedOutputStream(new FileOutputStream(args[1])))
: System.out;
BufferedReader br =
new BufferedReader(new InputStreamReader(in));
String line;
while((line = br.readLine())!=null) {
if(line.indexOf("#")>0) line=line.substring(0,line.indexOf("#"));
line = line.trim();
if(line.length()==0) continue;
out.println(prefixFromPlainForceHttp(line));
}
br.close();
out.close();
}
}
commons/src/main/java/org/archive/util/TextUtils.java
-308
View File
@@ -1,308 +0,0 @@
/*
* This file is part of the Heritrix web crawler (crawler.archive.org).
*
* Licensed to the Internet Archive (IA) by one or more individual
* contributors.
*
* The IA licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.archive.util;
import java.io.BufferedReader;
import java.io.IOException;
import java.io.PrintWriter;
import java.io.StringReader;
import java.io.StringWriter;
import java.io.UnsupportedEncodingException;
import java.io.Writer;
import java.net.URLDecoder;
import java.net.URLEncoder;
import java.util.HashMap;
import java.util.Map;
import java.util.concurrent.ConcurrentMap;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import org.apache.commons.lang.StringEscapeUtils;
import com.google.common.base.Function;
import com.google.common.collect.MapMaker;
public class TextUtils {
private static final String FIRSTWORD = "^([^\\s]*).*$";
/** thread-local cached matchers, by string key */
private static final ThreadLocal<Map<String,Matcher>> TL_MATCHER_MAP
= new ThreadLocal<Map<String,Matcher>>() {
protected Map<String,Matcher> initialValue() {
return new HashMap<String,Matcher>(50);
}
};
/** global soft-cache of Patterns, by string key */
private static final ConcurrentMap<String, Pattern> PATTERNS = new MapMaker()
.concurrencyLevel(16)
.softValues()
.makeComputingMap(new Function<String, Pattern>() {
public Pattern apply(String regex) {
return Pattern.compile(regex);
}
});
/**
* Get a matcher object for a precompiled regex pattern.
*
* This method tries to reuse Matcher objects for efficiency.
* It can hold for recycling one Matcher per pattern per thread.
*
* Matchers retrieved should be returned for reuse via the
* recycleMatcher() method, but no errors will occur if they
* are not.
*
* This method is a hotspot frequently accessed.
*
* @param pattern the string pattern to use
* @param input the character sequence the matcher should be using
* @return a matcher object loaded with the submitted character sequence
*/
public static Matcher getMatcher(String pattern, CharSequence input) {
if (pattern == null) {
throw new IllegalArgumentException("String 'pattern' must not be null");
}
input = new InterruptibleCharSequence(input);
final Map<String,Matcher> matchers = TL_MATCHER_MAP.get();
Matcher m = (Matcher)matchers.get(pattern);
if(m == null) {
m = PATTERNS.get(pattern).matcher(input);
} else {
matchers.put(pattern,null);
m.reset(input);
}
return m;
}
public static void recycleMatcher(Matcher m) {
// while cached, eliminate reference to potentially-large prior 'input'
m.reset("");
final Map<String,Matcher> matchers = TL_MATCHER_MAP.get();
matchers.put(m.pattern().pattern(),m);
}
/**
* Utility method using a precompiled pattern instead of using the
* replaceAll method of the String class. This method will also be reusing
* Matcher objects.
*
* @see java.util.regex.Pattern
* @param pattern precompiled Pattern to match against
* @param input the character sequence to check
* @param replacement the String to substitute every match with
* @return the String with all the matches substituted
*/
public static String replaceAll(
String pattern, CharSequence input, String replacement) {
input = new InterruptibleCharSequence(input);
Matcher m = getMatcher(pattern, input);
String res = m.replaceAll(replacement);
recycleMatcher(m);
return res;
}
/**
* Utility method using a precompiled pattern instead of using the
* replaceFirst method of the String class. This method will also be reusing
* Matcher objects.
*
* @see java.util.regex.Pattern
* @param pattern precompiled Pattern to match against
* @param input the character sequence to check
* @param replacement the String to substitute the first match with
* @return the String with the first match substituted
*/
public static String replaceFirst(
String pattern, CharSequence input, String replacement) {
input = new InterruptibleCharSequence(input);
Matcher m = getMatcher(pattern, input);
String res = m.replaceFirst(replacement);
recycleMatcher(m);
return res;
}
/**
* Utility method using a precompiled pattern instead of using the matches
* method of the String class. This method will also be reusing Matcher
* objects.
*
* @see java.util.regex.Pattern
* @param pattern precompiled Pattern to match against
* @param input the character sequence to check
* @return true if character sequence matches
*/
public static boolean matches(String pattern, CharSequence input) {
input = new InterruptibleCharSequence(input);
Matcher m = getMatcher(pattern, input);
boolean res = m.matches();
recycleMatcher(m);
return res;
}
/**
* Utility method using a precompiled pattern instead of using the split
* method of the String class.
*
* @see java.util.regex.Pattern
* @param pattern precompiled Pattern to split by
* @param input the character sequence to split
* @return array of Strings split by pattern
*/
public static String[] split(String pattern, CharSequence input) {
input = new InterruptibleCharSequence(input);
Matcher m = getMatcher(pattern,input);
String[] retVal = m.pattern().split(input);
recycleMatcher(m);
return retVal;
}
/**
* @param s String to find first word in (Words are delimited by
* whitespace).
* @return First word in the passed string else null if no word found.
*/
public static String getFirstWord(String s) {
Matcher m = getMatcher(FIRSTWORD, s);
String retVal = (m != null && m.matches())? m.group(1): null;
recycleMatcher(m);
return retVal;
}
/**
* Escapes a string so that it can be passed as an argument to a javscript
* in a JSP page. This method takes a string and returns the same string
* with any single quote escaped by prepending the character with a
* backslash. Linebreaks are also replaced with '\n'. Also,
* less-than signs and ampersands are replaced with HTML entities.
*
* @param s The string to escape
* @return The same string escaped.
*/
public static String escapeForHTMLJavascript(String s) {
return escapeForHTML(StringEscapeUtils.escapeJavaScript(s));
}
/**
* Escapes a string so that it can be placed inside XML/HTML attribute.
* Replaces ampersand, less-than, greater-than, single-quote, and
* double-quote with escaped versions.
* @param s The string to escape
* @return The same string escaped.
*/
public static String escapeForMarkupAttribute(String s) {
return StringEscapeUtils.escapeXml(s);
}
/**
* Minimally escapes a string so that it can be placed inside XML/HTML
* attribute.
* Escapes lt and amp.
* @param s The string to escape
* @return The same string escaped.
*/
public static String escapeForHTML(String s) {
// TODO: do this in a single pass instead of creating 5 junk strings
String escaped = s.replaceAll("&","&amp;");
return escaped.replaceAll("<","&lt;");
}
/**
* Utility method for writing a (potentially large) String to a JspWriter,
* escaping it for HTML display, without constructing another large String
* of the whole content.
* @param s String to write
* @param out destination JspWriter
* @throws IOException
*/
public static void writeEscapedForHTML(String s, Writer w)
throws IOException {
PrintWriter out = new PrintWriter(w);
BufferedReader reader = new BufferedReader(new StringReader(s));
String line;
while((line=reader.readLine()) != null){
out.println(StringEscapeUtils.escapeHtml(line));
}
}
/**
* Replaces HTML Entity Encodings.
* @param cs The CharSequence to remove html codes from
* @return the same CharSequence or an escaped String.
*/
public static CharSequence unescapeHtml(final CharSequence cs) {
if (cs == null) {
return cs;
}
return StringEscapeUtils.unescapeHtml(cs.toString());
}
/**
* @param message Message to put at top of the string returned. May be
* null.
* @param e Exception to write into a string.
* @return Return formatted string made of passed message and stack trace
* of passed exception.
*/
public static String exceptionToString(String message, Throwable e) {
StringWriter sw = new StringWriter();
if (message == null || message.length() == 0) {
sw.write(message);
sw.write("\n");
}
e.printStackTrace(new PrintWriter(sw));
return sw.toString();
}
/**
* Exception- and warning-free URL-escaping utility method.
*
* @param s String to escape
* @return URL-escaped string
*/
@SuppressWarnings("deprecation")
public static String urlEscape(String s) {
try {
return URLEncoder.encode(s,"UTF8");
} catch (UnsupportedEncodingException e) {
// should be impossible; all JVMs must support UTF8
// but have a fallback just in case
return URLEncoder.encode(s);
}
}
/**
* Exception- and warning-free URL-unescaping utility method.
*
* @param s String do unescape
* @return URL-unescaped String
*/
@SuppressWarnings("deprecation")
public static String urlUnescape(String s) {
try {
return URLDecoder.decode(s, "UTF8");
} catch (UnsupportedEncodingException e) {
// should be impossible; all JVMs must support UTF8
// but have a fallback just in case
return URLDecoder.decode(s);
}
}
}
commons/src/main/java/org/archive/util/UriUtils.java
+1 -1
View File
@@ -33,8 +33,8 @@ import java.util.regex.Matcher;
import org.apache.commons.codec.DecoderException;
import org.apache.commons.httpclient.URIException;
import org.archive.net.LaxURLCodec;
import org.archive.net.UURI;
import org.archive.url.LaxURLCodec;
/**
commons/src/main/java/org/archive/util/iterator/LineReadingIterator.java
-59
View File
@@ -1,59 +0,0 @@
/*
* This file is part of the Heritrix web crawler (crawler.archive.org).
*
* Licensed to the Internet Archive (IA) by one or more individual
* contributors.
*
* The IA licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.archive.util.iterator;
import java.io.BufferedReader;
import java.io.IOException;
import java.util.logging.Logger;
/**
* Utility class providing an Iterator interface over line-oriented
* text input, as a thin wrapper over a BufferedReader.
*
* @author gojomo
*/
public class LineReadingIterator extends LookaheadIterator<String> {
private static final Logger logger =
Logger.getLogger(LineReadingIterator.class.getName());
protected BufferedReader reader = null;
public LineReadingIterator(BufferedReader r) {
reader = r;
}
/**
* Loads next line into lookahead spot
*
* @return whether any item was loaded into next field
*/
protected boolean lookahead() {
try {
next = this.reader.readLine();
if(next == null) {
// TODO: make this close-on-exhaust optional?
reader.close();
}
return (next!=null);
} catch (IOException e) {
logger.warning(e.toString());
return false;
}
}
}
commons/src/main/java/org/archive/util/iterator/LookaheadIterator.java
-73
View File
@@ -1,73 +0,0 @@
/*
* This file is part of the Heritrix web crawler (crawler.archive.org).
*
* Licensed to the Internet Archive (IA) by one or more individual
* contributors.
*
* The IA licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.archive.util.iterator;
import java.util.Iterator;
import java.util.NoSuchElementException;
/**
* Superclass for Iterators which must probe ahead to know if
* a 'next' exists, and thus have a cached next between a call
* to hasNext() and next().
*
* @author gojomo
*
*/
public abstract class LookaheadIterator<T> implements Iterator<T> {
protected T next;
/**
* Test whether any items remain; loads next item into
* holding 'next' field.
*
* @see java.util.Iterator#hasNext()
*/
public boolean hasNext() {
return (this.next != null)? true: lookahead();
}
/**
* Caches the next item if available.
*
* @return true if there was a next item to cache, false otherwise
*/
protected abstract boolean lookahead();
/**
* Return the next item.
*
* @see java.util.Iterator#next()
*/
public T next() {
if (!hasNext()) {
throw new NoSuchElementException();
}
// 'next' is guaranteed non-null by a hasNext() which returned true
T returnObj = this.next;
this.next = null;
return returnObj;
}
/* (non-Javadoc)
* @see java.util.Iterator#remove()
*/
public void remove() {
throw new UnsupportedOperationException();
}
}
commons/src/main/java/org/archive/util/iterator/RegexLineIterator.java
-91
View File
@@ -1,91 +0,0 @@
/*
* This file is part of the Heritrix web crawler (crawler.archive.org).
*
* Licensed to the Internet Archive (IA) by one or more individual
* contributors.
*
* The IA licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.archive.util.iterator;
import java.util.Iterator;
import java.util.logging.Logger;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
/**
* Utility class providing an Iterator interface over line-oriented
* text input. By providing regexs indicating lines to ignore
* (such as pure whitespace or comments), lines to consider input, and
* what to return from the input lines (such as a whitespace-trimmed
* non-whitespace token with optional trailing comment), this can
* be configured to handle a number of formats.
*
* The public static members provide pattern configurations that will
* be helpful in a wide variety of contexts.
*
* @author gojomo
*/
public class RegexLineIterator
extends TransformingIteratorWrapper<String,String> {
private static final Logger logger =
Logger.getLogger(RegexLineIterator.class.getName());
public static final String COMMENT_LINE = "\\s*(#.*)?";
public static final String NONWHITESPACE_ENTRY_TRAILING_COMMENT =
"^[\\s\ufeff]*(\\S+)\\s*(#.*)?$";
public static final String TRIMMED_ENTRY_TRAILING_COMMENT =
"^\\s*([^#]+?)\\s*(#.*)?$";
public static final String ENTRY = "$1";
protected Matcher ignoreLine = null;
protected Matcher extractLine = null;
protected String outputTemplate = null;
public RegexLineIterator(Iterator<String> inner, String ignore,
String extract, String replace) {
this.inner = inner;
ignoreLine = Pattern.compile(ignore).matcher("");
extractLine = Pattern.compile(extract).matcher("");
outputTemplate = replace;
}
/**
* Loads next item into lookahead spot, if available. Skips
* lines matching ignoreLine; extracts desired portion of
* lines matching extractLine; informationally reports any
* lines matching neither.
*
* @return whether any item was loaded into next field
*/
protected String transform(String line) {
ignoreLine.reset(line);
if(ignoreLine.matches()) {
return null;
}
extractLine.reset(line);
if(extractLine.matches()) {
StringBuffer output = new StringBuffer();
// TODO: consider if a loop that find()s all is more
// generally useful here
extractLine.appendReplacement(output,outputTemplate);
return output.toString();
}
// no match; possibly error
logger.warning("line not extracted nor no-op: "+line);
return null;
}
}
commons/src/main/java/org/archive/util/iterator/TransformingIteratorWrapper.java
-65
View File
@@ -1,65 +0,0 @@
/*
* This file is part of the Heritrix web crawler (crawler.archive.org).
*
* Licensed to the Internet Archive (IA) by one or more individual
* contributors.
*
* The IA licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.archive.util.iterator;
import java.util.Iterator;
/**
* Superclass for Iterators which transform and/or filter results
* from a wrapped Iterator. Because transform() has the option of
* discarding an item from the inner Iterator (by returning null),
* this is a kind of LookaheadIterator.
*
* @author gojomo
*/
public abstract class TransformingIteratorWrapper<Original,Transformed>
extends LookaheadIterator<Transformed> {
protected Iterator<Original> inner;
/* (non-Javadoc)
* @see org.archive.util.iterator.LookaheadIterator#lookahead()
*/
protected boolean lookahead() {
assert next == null : "looking ahead when next is already loaded";
while(inner.hasNext()) {
next = transform(inner.next());
if(next!=null) {
return true;
}
}
noteExhausted();
return false;
}
/**
* Any cleanup to occur when hasNext() is about to return false
*/
protected void noteExhausted() {
// by default, do nothing
}
/**
* @param object Object to transform.
* @return Transfomed object.
*/
protected abstract Transformed transform(Original object);
}
commons/src/main/java/org/archive/util/zip/OpenJDK7GZIPInputStream.java
-295
View File
@@ -1,295 +0,0 @@
/*
* Copyright (c) 1996, 2010, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
* under the terms of the GNU General Public License version 2 only, as
* published by the Free Software Foundation. Oracle designates this
* particular file as subject to the "Classpath" exception as provided
* by Oracle in the LICENSE file that accompanied this code.
*
* This code is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
* version 2 for more details (a copy is included in the LICENSE file that
* accompanied this code).
*
* You should have received a copy of the GNU General Public License version
* 2 along with this work; if not, write to the Free Software Foundation,
* Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
*
* Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
* or visit www.oracle.com if you need additional information or have any
* questions.
*/
// ALL INTERNETARCHIVE CHANGES INCLUDE A COMMENT STARTING "// IA "
package org.archive.util.zip;
import java.io.SequenceInputStream;
import java.io.ByteArrayInputStream;
import java.io.InputStream;
import java.io.IOException;
import java.io.EOFException;
import java.util.zip.CRC32; // IA ADDED IMPORT
import java.util.zip.CheckedInputStream; // IA ADDED IMPORT
import java.util.zip.Inflater; // IA ADDED IMPORT
import java.util.zip.ZipException; // IA ADDED IMPORT
/**
* This class implements a stream filter for reading compressed data in
* the GZIP file format.
*
* @see InflaterInputStream
* @author David Connelly
*
*/
public
class OpenJDK7GZIPInputStream extends OpenJDK7InflaterInputStream { // IA RENAMINGS
/**
* CRC-32 for uncompressed data.
*/
protected CRC32 crc = new CRC32();
/**
* Indicates end of input stream.
*/
protected boolean eos;
private boolean closed = false;
/**
* Check to make sure that this stream has not been closed
*/
private void ensureOpen() throws IOException {
if (closed) {
throw new IOException("Stream closed");
}
}
/**
* Creates a new input stream with the specified buffer size.
* @param in the input stream
* @param size the input buffer size
*
* @exception ZipException if a GZIP format error has occurred or the
* compression method used is unsupported
* @exception IOException if an I/O error has occurred
* @exception IllegalArgumentException if size is <= 0
*/
public OpenJDK7GZIPInputStream(InputStream in, int size) throws IOException {
super(in, new Inflater(true), size);
usesDefaultInflater = true;
readHeader(in);
}
/**
* Creates a new input stream with a default buffer size.
* @param in the input stream
*
* @exception ZipException if a GZIP format error has occurred or the
* compression method used is unsupported
* @exception IOException if an I/O error has occurred
*/
public OpenJDK7GZIPInputStream(InputStream in) throws IOException {
this(in, 512);
}
/**
* Reads uncompressed data into an array of bytes. If <code>len</code> is not
* zero, the method will block until some input can be decompressed; otherwise,
* no bytes are read and <code>0</code> is returned.
* @param buf the buffer into which the data is read
* @param off the start offset in the destination array <code>b</code>
* @param len the maximum number of bytes read
* @return the actual number of bytes read, or -1 if the end of the
* compressed input stream is reached
*
* @exception NullPointerException If <code>buf</code> is <code>null</code>.
* @exception IndexOutOfBoundsException If <code>off</code> is negative,
* <code>len</code> is negative, or <code>len</code> is greater than
* <code>buf.length - off</code>
* @exception ZipException if the compressed input data is corrupt.
* @exception IOException if an I/O error has occurred.
*
*/
public int read(byte[] buf, int off, int len) throws IOException {
ensureOpen();
if (eos) {
return -1;
}
int n = super.read(buf, off, len);
if (n == -1) {
if (readTrailer())
eos = true;
else
return this.read(buf, off, len);
} else {
crc.update(buf, off, n);
}
return n;
}
/**
* Closes this input stream and releases any system resources associated
* with the stream.
* @exception IOException if an I/O error has occurred
*/
public void close() throws IOException {
if (!closed) {
super.close();
eos = true;
closed = true;
}
}
/**
* GZIP header magic number.
*/
public final static int GZIP_MAGIC = 0x8b1f;
/*
* File header flags.
*/
protected final static int FTEXT = 1; // Extra text // IA VISIBILITY CHANGE FOR SUBCLASS USE
protected final static int FHCRC = 2; // Header CRC // IA VISIBILITY CHANGE FOR SUBCLASS USE
protected final static int FEXTRA = 4; // Extra field // IA VISIBILITY CHANGE FOR SUBCLASS USE
protected final static int FNAME = 8; // File name // IA VISIBILITY CHANGE FOR SUBCLASS USE
protected final static int FCOMMENT = 16; // File comment // IA VISIBILITY CHANGE FOR SUBCLASS USE
/*
* Reads GZIP member header and returns the total byte number
* of this member header.
*/
protected int readHeader(InputStream this_in) throws IOException { // IA VISIBILITY CHANGE FOR OVERRIDING
CheckedInputStream in = new CheckedInputStream(this_in, crc);
crc.reset();
// Check header magic
if (readUShort(in) != GZIP_MAGIC) {
throw new ZipException("Not in GZIP format");
}
// Check compression method
if (readUByte(in) != 8) {
throw new ZipException("Unsupported compression method");
}
// Read flags
int flg = readUByte(in);
// Skip MTIME, XFL, and OS fields
skipBytes(in, 6);
int n = 2 + 2 + 6;
// Skip optional extra field
if ((flg & FEXTRA) == FEXTRA) {
int m = readUShort(in);
skipBytes(in, m);
n += m + 2;
}
// Skip optional file name
if ((flg & FNAME) == FNAME) {
do {
n++;
} while (readUByte(in) != 0);
}
// Skip optional file comment
if ((flg & FCOMMENT) == FCOMMENT) {
do {
n++;
} while (readUByte(in) != 0);
}
// Check optional header CRC
if ((flg & FHCRC) == FHCRC) {
int v = (int)crc.getValue() & 0xffff;
if (readUShort(in) != v) {
throw new ZipException("Corrupt GZIP header");
}
n += 2;
}
crc.reset();
return n;
}
/*
* Reads GZIP member trailer and returns true if the eos
* reached, false if there are more (concatenated gzip
* data set)
*/
protected boolean readTrailer() throws IOException { // IA CHANGE VISIBILITY FOR OVERRIDING
InputStream in = this.in;
int n = inf.getRemaining();
if (n > 0) {
in = new SequenceInputStream(
new ByteArrayInputStream(buf, len - n, n), in);
}
// Uses left-to-right evaluation order
if ((readUInt(in) != crc.getValue()) ||
// rfc1952; ISIZE is the input size modulo 2^32
(readUInt(in) != (inf.getBytesWritten() & 0xffffffffL)))
throw new ZipException("Corrupt GZIP trailer");
// If there are more bytes available in "in" or
// the leftover in the "inf" is > 26 bytes:
// this.trailer(8) + next.header.min(10) + next.trailer(8)
// try concatenated case
if (this.in.available() > 0 || n > 26) {
int m = 8; // this.trailer
try {
m += readHeader(in); // next.header
} catch (IOException ze) {
return true; // ignore any malformed, do nothing
}
inf.reset();
if (n > m)
inf.setInput(buf, len - n + m, n - m);
return false;
}
return true;
}
/*
* Reads unsigned integer in Intel byte order.
*/
protected long readUInt(InputStream in) throws IOException { // IA VISIBILITY CHANGE FOR SUBCLASS USE
long s = readUShort(in);
return ((long)readUShort(in) << 16) | s;
}
/*
* Reads unsigned short in Intel byte order.
*/
protected int readUShort(InputStream in) throws IOException { // IA VISIBILITY CHANGE FOR SUBCLASS USE
int b = readUByte(in);
return ((int)readUByte(in) << 8) | b;
}
/*
* Reads unsigned byte.
*/
protected int readUByte(InputStream in) throws IOException { // IA VISIBILITY CHANGE FOR SUBCLASS USE
int b = in.read();
if (b == -1) {
throw new EOFException();
}
if (b < -1 || b > 255) {
// Report on this.in, not argument in; see read{Header, Trailer}.
throw new IOException(this.in.getClass().getName()
+ ".read() returned value out of range -1..255: " + b);
}
return b;
}
private byte[] tmpbuf = new byte[128];
/*
* Skips bytes of input data blocking until all bytes are skipped.
* Does not assume that the input stream is capable of seeking.
*/
protected void skipBytes(InputStream in, int n) throws IOException { // IA VISIBILITY CHANGE FOR SUBCLASS USE
while (n > 0) {
int len = in.read(tmpbuf, 0, n < tmpbuf.length ? n : tmpbuf.length);
if (len == -1) {
throw new EOFException();
}
n -= len;
}
}
}
commons/src/main/java/org/archive/util/zip/OpenJDK7InflaterInputStream.java
-291
View File
@@ -1,291 +0,0 @@
/*
* Copyright (c) 1996, 2006, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
* under the terms of the GNU General Public License version 2 only, as
* published by the Free Software Foundation. Oracle designates this
* particular file as subject to the "Classpath" exception as provided
* by Oracle in the LICENSE file that accompanied this code.
*
* This code is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
* version 2 for more details (a copy is included in the LICENSE file that
* accompanied this code).
*
* You should have received a copy of the GNU General Public License version
* 2 along with this work; if not, write to the Free Software Foundation,
* Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
*
* Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
* or visit www.oracle.com if you need additional information or have any
* questions.
*/
package org.archive.util.zip;
import java.io.FilterInputStream;
import java.io.InputStream;
import java.io.IOException;
import java.io.EOFException;
import java.util.zip.DataFormatException; // IA ADDED IMPORT
import java.util.zip.Inflater; // IA ADDED IMPORT
import java.util.zip.ZipException; // IA ADDED IMPORT
/**
* This class implements a stream filter for uncompressing data in the
* "deflate" compression format. It is also used as the basis for other
* decompression filters, such as GZIPInputStream.
*
* @see Inflater
* @author David Connelly
*/
public
class OpenJDK7InflaterInputStream extends FilterInputStream { // IA RENAMED CLASS
/**
* Decompressor for this stream.
*/
protected Inflater inf;
/**
* Input buffer for decompression.
*/
protected byte[] buf;
/**
* Length of input buffer.
*/
protected int len;
private boolean closed = false;
// this flag is set to true after EOF has reached
private boolean reachEOF = false;
/**
* Check to make sure that this stream has not been closed
*/
private void ensureOpen() throws IOException {
if (closed) {
throw new IOException("Stream closed");
}
}
/**
* Creates a new input stream with the specified decompressor and
* buffer size.
* @param in the input stream
* @param inf the decompressor ("inflater")
* @param size the input buffer size
* @exception IllegalArgumentException if size is <= 0
*/
public OpenJDK7InflaterInputStream(InputStream in, Inflater inf, int size) {
super(in);
if (in == null || inf == null) {
throw new NullPointerException();
} else if (size <= 0) {
throw new IllegalArgumentException("buffer size <= 0");
}
this.inf = inf;
buf = new byte[size];
}
/**
* Creates a new input stream with the specified decompressor and a
* default buffer size.
* @param in the input stream
* @param inf the decompressor ("inflater")
*/
public OpenJDK7InflaterInputStream(InputStream in, Inflater inf) {
this(in, inf, 512);
}
protected boolean usesDefaultInflater = false;
/**
* Creates a new input stream with a default decompressor and buffer size.
* @param in the input stream
*/
public OpenJDK7InflaterInputStream(InputStream in) {
this(in, new Inflater());
usesDefaultInflater = true;
}
private byte[] singleByteBuf = new byte[1];
/**
* Reads a byte of uncompressed data. This method will block until
* enough input is available for decompression.
* @return the byte read, or -1 if end of compressed input is reached
* @exception IOException if an I/O error has occurred
*/
public int read() throws IOException {
ensureOpen();
return read(singleByteBuf, 0, 1) == -1 ? -1 : singleByteBuf[0] & 0xff;
}
/**
* Reads uncompressed data into an array of bytes. If <code>len</code> is not
* zero, the method will block until some input can be decompressed; otherwise,
* no bytes are read and <code>0</code> is returned.
* @param b the buffer into which the data is read
* @param off the start offset in the destination array <code>b</code>
* @param len the maximum number of bytes read
* @return the actual number of bytes read, or -1 if the end of the
* compressed input is reached or a preset dictionary is needed
* @exception NullPointerException If <code>b</code> is <code>null</code>.
* @exception IndexOutOfBoundsException If <code>off</code> is negative,
* <code>len</code> is negative, or <code>len</code> is greater than
* <code>b.length - off</code>
* @exception ZipException if a ZIP format error has occurred
* @exception IOException if an I/O error has occurred
*/
public int read(byte[] b, int off, int len) throws IOException {
ensureOpen();
if (b == null) {
throw new NullPointerException();
} else if (off < 0 || len < 0 || len > b.length - off) {
throw new IndexOutOfBoundsException();
} else if (len == 0) {
return 0;
}
try {
int n;
while ((n = inf.inflate(b, off, len)) == 0) {
if (inf.finished() || inf.needsDictionary()) {
reachEOF = true;
return -1;
}
if (inf.needsInput()) {
fill();
}
}
return n;
} catch (DataFormatException e) {
String s = e.getMessage();
throw new ZipException(s != null ? s : "Invalid ZLIB data format");
}
}
/**
* Returns 0 after EOF has been reached, otherwise always return 1.
* <p>
* Programs should not count on this method to return the actual number
* of bytes that could be read without blocking.
*
* @return 1 before EOF and 0 after EOF.
* @exception IOException if an I/O error occurs.
*
*/
public int available() throws IOException {
ensureOpen();
if (reachEOF) {
return 0;
} else {
return 1;
}
}
private byte[] b = new byte[512];
/**
* Skips specified number of bytes of uncompressed data.
* @param n the number of bytes to skip
* @return the actual number of bytes skipped.
* @exception IOException if an I/O error has occurred
* @exception IllegalArgumentException if n < 0
*/
public long skip(long n) throws IOException {
if (n < 0) {
throw new IllegalArgumentException("negative skip length");
}
ensureOpen();
int max = (int)Math.min(n, Integer.MAX_VALUE);
int total = 0;
while (total < max) {
int len = max - total;
if (len > b.length) {
len = b.length;
}
len = read(b, 0, len);
if (len == -1) {
reachEOF = true;
break;
}
total += len;
}
return total;
}
/**
* Closes this input stream and releases any system resources associated
* with the stream.
* @exception IOException if an I/O error has occurred
*/
public void close() throws IOException {
if (!closed) {
if (usesDefaultInflater)
inf.end();
in.close();
closed = true;
}
}
/**
* Fills input buffer with more data to decompress.
* @exception IOException if an I/O error has occurred
*/
protected void fill() throws IOException {
ensureOpen();
len = in.read(buf, 0, buf.length);
if (len == -1) {
throw new EOFException("Unexpected end of ZLIB input stream");
}
inf.setInput(buf, 0, len);
}
/**
* Tests if this input stream supports the <code>mark</code> and
* <code>reset</code> methods. The <code>markSupported</code>
* method of <code>InflaterInputStream</code> returns
* <code>false</code>.
*
* @return a <code>boolean</code> indicating if this stream type supports
* the <code>mark</code> and <code>reset</code> methods.
* @see java.io.InputStream#mark(int)
* @see java.io.InputStream#reset()
*/
public boolean markSupported() {
return false;
}
/**
* Marks the current position in this input stream.
*
* <p> The <code>mark</code> method of <code>InflaterInputStream</code>
* does nothing.
*
* @param readlimit the maximum limit of bytes that can be read before
* the mark position becomes invalid.
* @see java.io.InputStream#reset()
*/
public synchronized void mark(int readlimit) {
}
/**
* Repositions this stream to the position at the time the
* <code>mark</code> method was last called on this input stream.
*
* <p> The method <code>reset</code> for class
* <code>InflaterInputStream</code> does nothing except throw an
* <code>IOException</code>.
*
* @exception IOException if this method is invoked.
* @see java.io.InputStream#mark(int)
* @see java.io.IOException
*/
public synchronized void reset() throws IOException {
throw new IOException("mark/reset not supported");
}
}
commons/src/main/resources/org/archive/util/tlds-alpha-by-domain.txt
-280
View File
@@ -1,280 +0,0 @@
# Version 2010041900, Last Updated Mon Apr 19 14:07:01 2010 UTC
AC
AD
AE
AERO
AF
AG
AI
AL
AM
AN
AO
AQ
AR
ARPA
AS
ASIA
AT
AU
AW
AX
AZ
BA
BB
BD
BE
BF
BG
BH
BI
BIZ
BJ
BM
BN
BO
BR
BS
BT
BV
BW
BY
BZ
CA
CAT
CC
CD
CF
CG
CH
CI
CK
CL
CM
CN
CO
COM
COOP
CR
CU
CV
CX
CY
CZ
DE
DJ
DK
DM
DO
DZ
EC
EDU
EE
EG
ER
ES
ET
EU
FI
FJ
FK
FM
FO
FR
GA
GB
GD
GE
GF
GG
GH
GI
GL
GM
GN
GOV
GP
GQ
GR
GS
GT
GU
GW
GY
HK
HM
HN
HR
HT
HU
ID
IE
IL
IM
IN
INFO
INT
IO
IQ
IR
IS
IT
JE
JM
JO
JOBS
JP
KE
KG
KH
KI
KM
KN
KP
KR
KW
KY
KZ
LA
LB
LC
LI
LK
LR
LS
LT
LU
LV
LY
MA
MC
MD
ME
MG
MH
MIL
MK
ML
MM
MN
MO
MOBI
MP
MQ
MR
MS
MT
MU
MUSEUM
MV
MW
MX
MY
MZ
NA
NAME
NC
NE
NET
NF
NG
NI
NL
NO
NP
NR
NU
NZ
OM
ORG
PA
PE
PF
PG
PH
PK
PL
PM
PN
PR
PRO
PS
PT
PW
PY
QA
RE
RO
RS
RU
RW
SA
SB
SC
SD
SE
SG
SH
SI
SJ
SK
SL
SM
SN
SO
SR
ST
SU
SV
SY
SZ
TC
TD
TEL
TF
TG
TH
TJ
TK
TL
TM
TN
TO
TP
TR
TRAVEL
TT
TV
TW
TZ
UA
UG
UK
US
UY
UZ
VA
VC
VE
VG
VI
VN
VU
WF
WS
XN--0ZWM56D
XN--11B5BS3A9AJ6G
XN--80AKHBYKNJ4F
XN--9T4B11YI5A
XN--DEBA0AD
XN--G6W251D
XN--HGBK6AJ7F53BBA
XN--HLCJ6AYA9ESC7A
XN--JXALPDLP
XN--KGBECHTV
XN--ZCKZAH
YE
YT
ZA
ZM
ZW
commons/src/test/java/org/archive/io/GZIPMembersInputStreamTest.java
-230
View File
@@ -1,230 +0,0 @@
/*
* This file is part of the Heritrix web crawler (crawler.archive.org).
*
* Licensed to the Internet Archive (IA) by one or more individual
* contributors.
*
* The IA licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.archive.io;
import java.io.ByteArrayInputStream;
import java.io.IOException;
import java.util.Iterator;
import java.util.Random;
import junit.framework.TestCase;
import org.apache.commons.io.IOUtils;
import org.archive.util.ArchiveUtils;
import com.google.common.io.NullOutputStream;
import com.google.common.primitives.Bytes;
/**
* Tests for GZIPMembersInputStream
* @contributor gojomo
* @version $ $
*/
public class GZIPMembersInputStreamTest extends TestCase {
byte[] noise1k_gz;
byte[] noise32k_gz;
byte[] a_gz;
byte[] hello_gz;
byte[] allfour_gz;
byte[] sixsmall_gz;
{
Random rand = new Random(1);
try {
byte[] buf = new byte[1024];
rand.nextBytes(buf);
noise1k_gz = ArchiveUtils.gzip(buf);
buf = new byte[32*1024];
rand.nextBytes(buf);
noise32k_gz = ArchiveUtils.gzip(buf);
a_gz = ArchiveUtils.gzip("a".getBytes("ASCII"));
hello_gz = ArchiveUtils.gzip("hello".getBytes("ASCII"));
allfour_gz = Bytes.concat(noise1k_gz,noise32k_gz,a_gz,hello_gz);
sixsmall_gz = Bytes.concat(a_gz,hello_gz,a_gz,hello_gz,a_gz,hello_gz);
} catch (IOException e) {
// should not happen
}
}
public static void main(String [] args) {
junit.textui.TestRunner.run(GZIPMembersInputStreamTest.class);
}
public void testFullReadAllFour() throws IOException {
GZIPMembersInputStream gzin =
new GZIPMembersInputStream(new ByteArrayInputStream(allfour_gz));
int count = IOUtils.copy(gzin, new NullOutputStream());
assertEquals("wrong length uncompressed data", 1024+(32*1024)+1+5, count);
}
public void testFullReadSixSmall() throws IOException {
GZIPMembersInputStream gzin =
new GZIPMembersInputStream(new ByteArrayInputStream(sixsmall_gz));
int count = IOUtils.copy(gzin, new NullOutputStream());
assertEquals("wrong length uncompressed data", 1+5+1+5+1+5, count);
}
public void testReadPerMemberAllFour() throws IOException {
GZIPMembersInputStream gzin =
new GZIPMembersInputStream(new ByteArrayInputStream(allfour_gz));
gzin.setEofEachMember(true);
int count0 = IOUtils.copy(gzin, new NullOutputStream());
assertEquals("wrong 1k member count", 1024, count0);
assertEquals("wrong member number", 0, gzin.getMemberNumber());
assertEquals("wrong member0 start", 0, gzin.getCurrentMemberStart());
assertEquals("wrong member0 end", noise1k_gz.length, gzin.getCurrentMemberEnd());
gzin.nextMember();
int count1 = IOUtils.copy(gzin, new NullOutputStream());
assertEquals("wrong 32k member count", (32*1024), count1);
assertEquals("wrong member number", 1, gzin.getMemberNumber());
assertEquals("wrong member1 start", noise1k_gz.length, gzin.getCurrentMemberStart());
assertEquals("wrong member1 end", noise1k_gz.length+noise32k_gz.length, gzin.getCurrentMemberEnd());
gzin.nextMember();
int count2 = IOUtils.copy(gzin, new NullOutputStream());
assertEquals("wrong 1-byte member count", 1, count2);
assertEquals("wrong member number", 2, gzin.getMemberNumber());
assertEquals("wrong member2 start", noise1k_gz.length+noise32k_gz.length, gzin.getCurrentMemberStart());
assertEquals("wrong member2 end", noise1k_gz.length+noise32k_gz.length+a_gz.length, gzin.getCurrentMemberEnd());
gzin.nextMember();
int count3 = IOUtils.copy(gzin, new NullOutputStream());
assertEquals("wrong 5-byte member count", 5, count3);
assertEquals("wrong member number", 3, gzin.getMemberNumber());
assertEquals("wrong member3 start", noise1k_gz.length+noise32k_gz.length+a_gz.length, gzin.getCurrentMemberStart());
assertEquals("wrong member3 end", noise1k_gz.length+noise32k_gz.length+a_gz.length+hello_gz.length, gzin.getCurrentMemberEnd());
gzin.nextMember();
int countEnd = IOUtils.copy(gzin, new NullOutputStream());
assertEquals("wrong eof count", 0, countEnd);
}
public void testReadPerMemberSixSmall() throws IOException {
GZIPMembersInputStream gzin =
new GZIPMembersInputStream(new ByteArrayInputStream(sixsmall_gz));
gzin.setEofEachMember(true);
for(int i = 0; i < 3; i++) {
int count2 = IOUtils.copy(gzin, new NullOutputStream());
assertEquals("wrong 1-byte member count", 1, count2);
gzin.nextMember();
int count3 = IOUtils.copy(gzin, new NullOutputStream());
assertEquals("wrong 5-byte member count", 5, count3);
gzin.nextMember();
}
int countEnd = IOUtils.copy(gzin, new NullOutputStream());
assertEquals("wrong eof count", 0, countEnd);
}
public void testByteReadPerMember() throws IOException {
GZIPMembersInputStream gzin =
new GZIPMembersInputStream(new ByteArrayInputStream(allfour_gz));
gzin.setEofEachMember(true);
int count0 = 0;
while(gzin.read()>-1) count0++;
assertEquals("wrong 1k member count", 1024, count0);
assertEquals("wrong member number", 0, gzin.getMemberNumber());
assertEquals("wrong member0 start", 0, gzin.getCurrentMemberStart());
assertEquals("wrong member0 end", noise1k_gz.length, gzin.getCurrentMemberEnd());
gzin.nextMember();
int count1 = 0;
while(gzin.read()>-1) count1++;
assertEquals("wrong 32k member count", (32*1024), count1);
assertEquals("wrong member number", 1, gzin.getMemberNumber());
assertEquals("wrong member1 start", noise1k_gz.length, gzin.getCurrentMemberStart());
assertEquals("wrong member1 end", noise1k_gz.length+noise32k_gz.length, gzin.getCurrentMemberEnd());
gzin.nextMember();
int count2 = 0;
while(gzin.read()>-1) count2++;
assertEquals("wrong 1-byte member count", 1, count2);
assertEquals("wrong member number", 2, gzin.getMemberNumber());
assertEquals("wrong member2 start", noise1k_gz.length+noise32k_gz.length, gzin.getCurrentMemberStart());
assertEquals("wrong member2 end", noise1k_gz.length+noise32k_gz.length+a_gz.length, gzin.getCurrentMemberEnd());
gzin.nextMember();
int count3 = 0;
while(gzin.read()>-1) count3++;
assertEquals("wrong 5-byte member count", 5, count3);
assertEquals("wrong member number", 3, gzin.getMemberNumber());
assertEquals("wrong member3 start", noise1k_gz.length+noise32k_gz.length+a_gz.length, gzin.getCurrentMemberStart());
assertEquals("wrong member3 end", noise1k_gz.length+noise32k_gz.length+a_gz.length+hello_gz.length, gzin.getCurrentMemberEnd());
gzin.nextMember();
int countEnd = 0;
while(gzin.read()>-1) countEnd++;
assertEquals("wrong eof count", 0, countEnd);
}
public void testMemberSeek() throws IOException {
GZIPMembersInputStream gzin =
new GZIPMembersInputStream(new ByteArrayInputStream(allfour_gz));
gzin.setEofEachMember(true);
gzin.compressedSeek(noise1k_gz.length+noise32k_gz.length);
int count2 = IOUtils.copy(gzin, new NullOutputStream());
assertEquals("wrong 1-byte member count", 1, count2);
// assertEquals("wrong Member number", 2, gzin.getMemberNumber());
assertEquals("wrong Member2 start", noise1k_gz.length+noise32k_gz.length, gzin.getCurrentMemberStart());
assertEquals("wrong Member2 end", noise1k_gz.length+noise32k_gz.length+a_gz.length, gzin.getCurrentMemberEnd());
gzin.nextMember();
int count3 = IOUtils.copy(gzin, new NullOutputStream());
assertEquals("wrong 5-byte member count", 5, count3);
// assertEquals("wrong Member number", 3, gzin.getMemberNumber());
assertEquals("wrong Member3 start", noise1k_gz.length+noise32k_gz.length+a_gz.length, gzin.getCurrentMemberStart());
assertEquals("wrong Member3 end", noise1k_gz.length+noise32k_gz.length+a_gz.length+hello_gz.length, gzin.getCurrentMemberEnd());
gzin.nextMember();
int countEnd = IOUtils.copy(gzin, new NullOutputStream());
assertEquals("wrong eof count", 0, countEnd);
}
@SuppressWarnings("deprecation")
public void testMemberIterator() throws IOException {
GZIPMembersInputStream gzin =
new GZIPMembersInputStream(new ByteArrayInputStream(allfour_gz));
Iterator<GZIPMembersInputStream> iter = gzin.memberIterator();
assertTrue(iter.hasNext());
GZIPMembersInputStream gzMember0 = iter.next();
int count0 = IOUtils.copy(gzMember0, new NullOutputStream());
assertEquals("wrong 1k member count", 1024, count0);
assertEquals("wrong member number", 0, gzin.getMemberNumber());
assertEquals("wrong member0 start", 0, gzin.getCurrentMemberStart());
assertEquals("wrong member0 end", noise1k_gz.length, gzin.getCurrentMemberEnd());
assertTrue(iter.hasNext());
GZIPMembersInputStream gzMember1 = iter.next();
int count1 = IOUtils.copy(gzMember1, new NullOutputStream());
assertEquals("wrong 32k member count", (32*1024), count1);
assertEquals("wrong member number", 1, gzin.getMemberNumber());
assertEquals("wrong member1 start", noise1k_gz.length, gzin.getCurrentMemberStart());
assertEquals("wrong member1 end", noise1k_gz.length+noise32k_gz.length, gzin.getCurrentMemberEnd());
assertTrue(iter.hasNext());
GZIPMembersInputStream gzMember2 = iter.next();
int count2 = IOUtils.copy(gzMember2, new NullOutputStream());
assertEquals("wrong 1-byte member count", 1, count2);
assertEquals("wrong member number", 2, gzin.getMemberNumber());
assertEquals("wrong member2 start", noise1k_gz.length+noise32k_gz.length, gzin.getCurrentMemberStart());
assertEquals("wrong member2 end", noise1k_gz.length+noise32k_gz.length+a_gz.length, gzin.getCurrentMemberEnd());
assertTrue(iter.hasNext());
GZIPMembersInputStream gzMember3 = iter.next();
int count3 = IOUtils.copy(gzMember3, new NullOutputStream());
assertEquals("wrong 5-byte member count", 5, count3);
assertEquals("wrong member number", 3, gzin.getMemberNumber());
assertEquals("wrong member3 start", noise1k_gz.length+noise32k_gz.length+a_gz.length, gzin.getCurrentMemberStart());
assertEquals("wrong member3 end", noise1k_gz.length+noise32k_gz.length+a_gz.length+hello_gz.length, gzin.getCurrentMemberEnd());
assertFalse(iter.hasNext());
}
}
commons/src/test/java/org/archive/net/UURIFactoryTest.java
-1200
View File
File diff suppressed because it is too large Load Diff
commons/src/test/java/org/archive/net/UURITest.java
-55
View File
@@ -1,55 +0,0 @@
/*
* This file is part of the Heritrix web crawler (crawler.archive.org).
*
* Licensed to the Internet Archive (IA) by one or more individual
* contributors.
*
* The IA licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.archive.net;
import java.net.URISyntaxException;
import org.apache.commons.httpclient.URIException;
import junit.framework.TestCase;
public class UURITest extends TestCase {
public void testHasScheme() {
assertTrue(UURI.hasScheme("http://www.archive.org"));
assertTrue(UURI.hasScheme("http:"));
assertFalse(UURI.hasScheme("ht/tp://www.archive.org"));
assertFalse(UURI.hasScheme("/tmp"));
}
public void testGetFileName() throws URISyntaxException {
final String filename = "x.arc.gz";
assertEquals(filename,
UURI.parseFilename("/tmp/one.two/" + filename));
assertEquals(filename,
UURI.parseFilename("http://archive.org/tmp/one.two/" +
filename));
assertEquals(filename,
UURI.parseFilename("rsync://archive.org/tmp/one.two/" +
filename));
}
public void testSchemalessRelative() throws URIException {
UURI base = new UURI("http://www.archive.org/a", true, "UTF-8");
UURI relative = new UURI("//www.facebook.com/?href=http://www.archive.org/a", true, "UTF-8");
assertEquals(null, relative.getScheme());
assertEquals("www.facebook.com", relative.getAuthority());
UURI test = new UURI(base, relative);
assertEquals("http://www.facebook.com/?href=http://www.archive.org/a", test.toString());
}
}
commons/src/test/java/org/archive/util/ArchiveUtilsTest.java
-421
View File
@@ -1,421 +0,0 @@
/*
* This file is part of the Heritrix web crawler (crawler.archive.org).
*
* Licensed to the Internet Archive (IA) by one or more individual
* contributors.
*
* The IA licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.archive.util;
import java.text.ParseException;
import java.util.Date;
import java.util.HashSet;
import java.util.concurrent.Semaphore;
import java.util.concurrent.atomic.AtomicInteger;
import junit.framework.Test;
import junit.framework.TestCase;
import junit.framework.TestSuite;
/**
* JUnit test suite for ArchiveUtils
*
* @author <a href="mailto:me@jamesc.net">James Casey</a>
* @version $Id$
*/
public class ArchiveUtilsTest extends TestCase {
/**
* Create a new ArchiveUtilsTest object
*
* @param testName the name of the test
*/
public ArchiveUtilsTest(final String testName) {
super(testName);
}
/**
* run all the tests for ArchiveUtilsTest
*
* @param argv the command line arguments
*/
public static void main(String argv[]) {
junit.textui.TestRunner.run(suite());
}
/**
* return the suite of tests for ArchiveUtilsTest
*
* @return the suite of test
*/
public static Test suite() {
return new TestSuite(ArchiveUtilsTest.class);
}
/** check the getXXDigitDate() methods produce valid dates*/
public void testGetXXDigitDate() {
// TODO - we only really test the date lengths here. How to test
// other stuff well ?
final String date12 = ArchiveUtils.get12DigitDate();
assertEquals("12 digits", 12, date12.length());
final String date14 = ArchiveUtils.get14DigitDate();
assertEquals("14 digits", 14, date14.length());
final String date17 = ArchiveUtils.get17DigitDate();
assertEquals("17 digits", 17, date17.length());
// now parse, and check they're all within 1 minute
try {
final long long12 = ArchiveUtils.parse12DigitDate(date12).getTime();
long long14 = ArchiveUtils.parse14DigitDate(date14).getTime();
long long17 = ArchiveUtils.parse17DigitDate(date17).getTime();
assertClose("12 and 14 close", long12, long14, 600000);
assertClose("12 and 17 close", long12, long17, 600000);
assertClose("14 and 17 close", long14, long17, 600000);
} catch (ParseException e) {
fail("Could not parse a date : " + e.getMessage());
}
}
/** check that getXXDigitDate(long) does the right thing */
public void testGetXXDigitDateLong() {
final long now = System.currentTimeMillis();
final String date12 = ArchiveUtils.get12DigitDate(now);
assertEquals("12 digits", 12, date12.length());
final String date14 = ArchiveUtils.get14DigitDate(now);
assertEquals("14 digits", 14, date14.length());
assertEquals("first twelve digits same as date12", date12, date14.substring(0, 12));
final String date17 = ArchiveUtils.get17DigitDate(now);
assertEquals("17 digits", 17, date17.length());
assertEquals("first twelve digits same as date12", date12, date17.substring(0, 12));
assertEquals("first fourteen digits same as date14", date14, date17.substring(0, 14));
}
/**
* Check that parseXXDigitDate() works
*
* @throws ParseException
*/
public void testParseXXDigitDate() throws ParseException {
// given a date, check it get resolved properly
// It's 02 Jan 2004, 12:40:02.111
final String date = "20040102124002111";
try {
final long long12 = ArchiveUtils.parse12DigitDate(date.substring(0, 12)).getTime();
final long long14 = ArchiveUtils.parse14DigitDate(date.substring(0, 14)).getTime();
final long long17 = ArchiveUtils.parse17DigitDate(date).getTime();
assertClose("12 and 14 close", long12, long14, 600000);
assertClose("12 and 17 close", long12, long17, 600000);
assertClose("14 and 17 close", long14, long17, 600000);
} catch (ParseException e) {
fail("Could not parse a date : " + e.getMessage());
}
}
public void testTooShortParseDigitDate() throws ParseException {
String d = "X";
boolean b = false;
try {
ArchiveUtils.getDate(d);
} catch (ParseException e) {
b = true;
}
assertTrue(b);
Date date = ArchiveUtils.getDate("1999");
assertTrue(date.getTime() == 915148800000L);
b = false;
try {
ArchiveUtils.getDate("19991");
} catch (ParseException e) {
b = true;
}
assertTrue(b);
ArchiveUtils.getDate("19990101");
ArchiveUtils.getDate("1999010101");
ArchiveUtils.getDate("19990101010101");
ArchiveUtils.getDate("1960");
}
/** check that parse12DigitDate doesn't accept a bad date */
public void testBad12Date() {
// now try a badly formed dates
assertBad12DigitDate("a-stringy-digit-date");
assertBad12DigitDate("20031201"); // too short
}
/**
* check that parse14DigitDate doesn't accept a bad date
*/
public void testBad14Date() {
// now try a badly formed dates
assertBad14DigitDate("a-stringy-digit-date");
assertBad14DigitDate("20031201"); // too short
assertBad14DigitDate("200401021240"); // 12 digit
}
/**
* check that parse12DigitDate doesn't accept a bad date
*/
public void testBad17Date() {
// now try a badly formed dates
assertBad17DigitDate("a-stringy-digit-date");
assertBad17DigitDate("20031201"); // too short
assertBad17DigitDate("200401021240"); // 12 digit
assertBad17DigitDate("20040102124002"); // 14 digit
}
/** check that padTo(String) works */
public void testPadToString() {
assertEquals("pad to one (smaller)", "foo", ArchiveUtils.padTo("foo", 1));
assertEquals("pad to 0 (no sense)", "foo", ArchiveUtils.padTo("foo", 0));
assertEquals("pad to neg (nonsense)", "foo", ArchiveUtils.padTo("foo", 0));
assertEquals("pad to 4", " foo", ArchiveUtils.padTo("foo", 4));
assertEquals("pad to 10", " foo", ArchiveUtils.padTo("foo", 10));
}
/**
* check that padTo(int) works
*/
public void testPadToInt() {
assertEquals("pad to one (smaller)", "123", ArchiveUtils.padTo(123, 1));
assertEquals("pad to 0 (no sense)", "123", ArchiveUtils.padTo(123, 0));
assertEquals("pad to neg (nonsense)", "123", ArchiveUtils.padTo(123, 0));
assertEquals("pad to 4", " 123", ArchiveUtils.padTo(123, 4));
assertEquals("pad to 10", " 123", ArchiveUtils.padTo(123, 10));
assertEquals("pad -123 to 10", " -123", ArchiveUtils.padTo(-123, 10));
}
/** check that byteArrayEquals() works */
public void testByteArrayEquals() {
// foo == foo2, foo != bar, foo != bar2
byte[] foo = new byte[10], bar = new byte[20];
byte[] foo2 = new byte[10], bar2 = new byte[10];
for (byte i = 0; i < 10 ; ++i) {
foo[i] = foo2[i] = bar[i] = i;
bar2[i] = (byte)(01 + i);
}
assertTrue("two nulls", ArchiveUtils.byteArrayEquals(null, null));
assertFalse("lhs null", ArchiveUtils.byteArrayEquals(null, foo));
assertFalse("rhs null", ArchiveUtils.byteArrayEquals(foo, null));
// now check with same length, with same (foo2) and different (bar2)
// contents
assertFalse("different lengths", ArchiveUtils.byteArrayEquals(foo, bar));
assertTrue("same to itself", ArchiveUtils.byteArrayEquals(foo, foo));
assertTrue("same contents", ArchiveUtils.byteArrayEquals(foo, foo2));
assertFalse("different contents", ArchiveUtils.byteArrayEquals(foo, bar2));
}
/** test doubleToString() */
public void testDoubleToString(){
double test = 12.345;
assertTrue(
"cecking zero precision",
ArchiveUtils.doubleToString(test, 0).equals("12"));
assertTrue(
"cecking 2 character precision",
ArchiveUtils.doubleToString(test, 2).equals("12.34"));
assertTrue(
"cecking precision higher then the double has",
ArchiveUtils.doubleToString(test, 65).equals("12.345"));
}
public void testFormatBytesForDisplayPrecise(){
assertEquals("formating negative number", "0 B", ArchiveUtils
.formatBytesForDisplay(-1));
assertEquals("0 bytes", "0 B", ArchiveUtils
.formatBytesForDisplay(0));
assertEquals("1023 bytes", "1,023 B", ArchiveUtils
.formatBytesForDisplay(1023));
assertEquals("1025 bytes", "1.0 KiB", ArchiveUtils
.formatBytesForDisplay(1025));
// expected display values taken from Google calculator
assertEquals("10,000 bytes", "9.8 KiB",
ArchiveUtils.formatBytesForDisplay(10000));
assertEquals("1,000,000 bytes", "977 KiB",
ArchiveUtils.formatBytesForDisplay(1000000));
assertEquals("100,000,000 bytes", "95 MiB",
ArchiveUtils.formatBytesForDisplay(100000000));
assertEquals("100,000,000,000 bytes", "93 GiB",
ArchiveUtils.formatBytesForDisplay(100000000000L));
assertEquals("100,000,000,000,000 bytes", "91 TiB",
ArchiveUtils.formatBytesForDisplay(100000000000000L));
assertEquals("100,000,000,000,000,000 bytes", "90,949 TiB",
ArchiveUtils.formatBytesForDisplay(100000000000000000L));
}
/*
* helper methods
*/
/** check that this is a bad date, and <code>fail()</code> if so.
*
* @param date the 12digit date to check
*/
private void assertBad12DigitDate(final String date) {
try {
ArchiveUtils.parse12DigitDate(date);
} catch (ParseException e) {
return;
}
fail("Expected exception on parse of : " + date);
}
/**
* check that this is a bad date, and <code>fail()</code> if so.
*
* @param date the 14digit date to check
*/
private void assertBad14DigitDate(final String date) {
try {
ArchiveUtils.parse14DigitDate(date);
} catch (ParseException e) {
return;
}
fail("Expected exception on parse of : " + date);
}
/**
* check that this is a bad date, and <code>fail()</code> if so.
*
* @param date the 17digit date to check
*/
private void assertBad17DigitDate(final String date) {
try {
ArchiveUtils.parse17DigitDate(date);
} catch (ParseException e) {
return;
}
fail("Expected exception on parse of : " + date);
}
/** check that two longs are within a given <code>delta</code> */
private void assertClose(String desc, long date1, long date2, long delta) {
assertTrue(desc, date1 == date2 ||
(date1 < date2 && date2 < (date1 + delta)) ||
(date2 < date1 && date1 < (date2 + delta)));
}
public void testArrayToLong() {
testOneArrayToLong(-1);
testOneArrayToLong(1);
testOneArrayToLong(1000);
testOneArrayToLong(Integer.MAX_VALUE);
}
private void testOneArrayToLong(final long testValue) {
byte [] a = new byte[8];
ArchiveUtils.longIntoByteArray(testValue, a, 0);
final long l = ArchiveUtils.byteArrayIntoLong(a, 0);
assertEquals(testValue, l);
}
public void testSecondsSinceEpochCalculation() throws ParseException {
assertEquals(ArchiveUtils.secondsSinceEpoch("20010909014640"),
"1000000000");
assertEquals(ArchiveUtils.secondsSinceEpoch("20010909014639"),
"0999999999");
assertEquals(ArchiveUtils.secondsSinceEpoch("19700101"),
"0000000000");
assertEquals(ArchiveUtils.secondsSinceEpoch("2005"), "1104537600");
assertEquals(ArchiveUtils.secondsSinceEpoch("200501"), "1104537600");
assertEquals(ArchiveUtils.secondsSinceEpoch("20050101"), "1104537600");
assertEquals(ArchiveUtils.secondsSinceEpoch("2005010100"),
"1104537600");
boolean eThrown = false;
try {
ArchiveUtils.secondsSinceEpoch("20050");
} catch (IllegalArgumentException e) {
eThrown = true;
}
assertTrue(eThrown);
}
public static void testZeroPadInteger() {
assertEquals(ArchiveUtils.zeroPadInteger(1), "0000000001");
assertEquals(ArchiveUtils.zeroPadInteger(1000000000), "1000000000");
}
/**
* Test stable behavior of date formatting under heavy concurrency.
*
* @throws InterruptedException
*/
public static void testDateFormatConcurrency() throws InterruptedException {
final int COUNT = 1000;
Thread [] ts = new Thread[COUNT];
final Semaphore allDone = new Semaphore(-COUNT+1);
final AtomicInteger failures = new AtomicInteger(0);
for (int i = 0; i < COUNT; i++) {
Thread t = new Thread() {
public void run() {
long n = System.currentTimeMillis();
final String d = ArchiveUtils.get17DigitDate(n);
for (int i = 0; i < 1000; i++) {
try {
sleep(10);
} catch (InterruptedException e) {
// TODO Auto-generated catch block
e.printStackTrace();
}
String d2 = ArchiveUtils.get17DigitDate(n);
if(!d.equals(d2)) {
failures.incrementAndGet();
break;
}
}
allDone.release();
}
};
ts[i] = t;
ts[i].setName(Integer.toString(i));
ts[i].start();
while(!ts[i].isAlive()) /* Wait for thread to spin up*/;
}
allDone.acquire(); // wait for all threads to finish
assertEquals(failures.get()+" format mismatches",0,failures.get());
}
public void testIsTld() {
assertTrue("TLD test problem", ArchiveUtils.isTld("com"));
assertTrue("TLD test problem", ArchiveUtils.isTld("COM"));
}
public void testUnique17() {
HashSet<String> uniqueTimestamps = new HashSet<String>();
for(int i = 0; i<10; i++) {
assertTrue("timestamp17 repeated",uniqueTimestamps.add(ArchiveUtils.getUnique17DigitDate()));
}
}
public void testUnique14() {
HashSet<String> uniqueTimestamps = new HashSet<String>();
for(int i = 0; i<10; i++) {
assertTrue("timestamp14 repeated",uniqueTimestamps.add(ArchiveUtils.getUnique14DigitDate()));
}
}
}
engine/src/main/java/org/archive/crawler/framework/CrawlController.java
+3 -2
View File
@@ -43,6 +43,7 @@ import org.archive.modules.net.ServerCache;
import org.archive.modules.seeds.SeedModule;
import org.archive.spring.ConfigPath;
import org.archive.util.ArchiveUtils;
import org.archive.util.ReportUtils;
import org.springframework.beans.BeansException;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Value;
@@ -615,7 +616,7 @@ implements Serializable,
}
public String getToeThreadReportShort() {
return (toePool == null) ? "" : ArchiveUtils.shortReportLine(toePool);
return (toePool == null) ? "" : ReportUtils.shortReportLine(toePool);
}
public Map<String,Object> getToeThreadReportShortData() {
@@ -624,7 +625,7 @@ implements Serializable,
}
public String getFrontierReportShort() {
return ArchiveUtils.shortReportLine(getFrontier());
return ReportUtils.shortReportLine(getFrontier());
}
/**
engine/src/main/java/org/archive/crawler/framework/ToeThread.java
+2 -1
View File
@@ -44,6 +44,7 @@ import org.archive.util.ArchiveUtils;
import org.archive.util.DevUtils;
import org.archive.util.ProgressStatisticsReporter;
import org.archive.util.Recorder;
import org.archive.util.ReportUtils;
import org.archive.util.Reporter;
import com.sleepycat.util.RuntimeExceptionWrapper;
@@ -573,7 +574,7 @@ implements Reporter, ProgressStatisticsReporter,
}
public String shortReportLine() {
return ArchiveUtils.shortReportLine(this);
return ReportUtils.shortReportLine(this);
}
public void progressStatisticsLine(PrintWriter writer) {
engine/src/main/java/org/archive/crawler/frontier/AbstractFrontier.java
+2 -1
View File
@@ -69,6 +69,7 @@ import org.archive.modules.seeds.SeedModule;
import org.archive.spring.HasKeyedProperties;
import org.archive.spring.KeyedProperties;
import org.archive.util.ArchiveUtils;
import org.archive.util.ReportUtils;
import org.archive.util.iterator.LineReadingIterator;
import org.archive.util.iterator.RegexLineIterator;
import org.json.JSONException;
@@ -1128,7 +1129,7 @@ public abstract class AbstractFrontier
// Reporter implementation
//
public String shortReportLine() {
return ArchiveUtils.shortReportLine(this);
return ReportUtils.shortReportLine(this);
}
@Override
engine/src/main/java/org/archive/crawler/frontier/WorkQueue.java
+2 -1
View File
@@ -39,6 +39,7 @@ import org.archive.modules.fetcher.FetchStats.Stage;
import org.archive.util.ArchiveUtils;
import org.archive.util.IdentityCacheable;
import org.archive.util.ObjectIdentityCache;
import org.archive.util.ReportUtils;
import org.archive.util.Reporter;
/**
@@ -541,7 +542,7 @@ public abstract class WorkQueue implements Frontier.FrontierGroup,
}
public String shortReportLine() {
return ArchiveUtils.shortReportLine(this);
return ReportUtils.shortReportLine(this);
}
/**
engine/src/main/java/org/archive/crawler/frontier/precedence/PrecedenceProvider.java
+2 -1
View File
@@ -27,6 +27,7 @@ import org.archive.modules.CrawlURI;
import org.archive.modules.fetcher.FetchStats;
import org.archive.modules.fetcher.FetchStats.Stage;
import org.archive.util.ArchiveUtils;
import org.archive.util.ReportUtils;
import org.archive.util.Reporter;
/**
@@ -62,7 +63,7 @@ FetchStats.CollectsFetchStats, Serializable {
}
public String shortReportLine() {
return ArchiveUtils.shortReportLine(this);
return ReportUtils.shortReportLine(this);
}
@Override
modules/src/main/java/org/archive/modules/CrawlURI.java
+2 -2
View File
@@ -90,9 +90,9 @@ import org.archive.net.UURI;
import org.archive.net.UURIFactory;
import org.archive.spring.OverlayContext;
import org.archive.spring.OverlayMapsSource;
import org.archive.util.ArchiveUtils;
import org.archive.util.Base32;
import org.archive.util.Recorder;
import org.archive.util.ReportUtils;
import org.archive.util.Reporter;
import org.json.JSONException;
import org.json.JSONObject;
@@ -1400,7 +1400,7 @@ implements Reporter, Serializable, OverlayContext {
//
public String shortReportLine() {
return ArchiveUtils.shortReportLine(this);
return ReportUtils.shortReportLine(this);
}
@Override
modules/src/main/java/org/archive/modules/extractor/ExtractorURI.java
+1 -1
View File
@@ -27,7 +27,7 @@ import java.util.regex.Matcher;
import org.apache.commons.codec.DecoderException;
import org.apache.commons.httpclient.URIException;
import org.archive.modules.CrawlURI;
import org.archive.net.LaxURLCodec;
import org.archive.url.LaxURLCodec;
import org.archive.net.UURI;
import org.archive.net.UURIFactory;
import org.archive.util.TextUtils;
modules/src/main/java/org/archive/modules/fetcher/FetchStats.java
+2 -1
View File
@@ -27,6 +27,7 @@ import org.apache.commons.httpclient.HttpStatus;
import org.archive.modules.CrawlURI;
import org.archive.modules.deciderules.recrawl.IdenticalDigestDecideRule;
import org.archive.util.ArchiveUtils;
import org.archive.util.ReportUtils;
import org.archive.util.Reporter;
/**
@@ -204,7 +205,7 @@ public class FetchStats implements Serializable, FetchStatusCodes, Reporter {
}
public String shortReportLine() {
return ArchiveUtils.shortReportLine(this);
return ReportUtils.shortReportLine(this);
}
@Override