lee/heritrix3
1
0
Fork 0
mirror of https://github.com/internetarchive/heritrix3.git synced 2026-09-23 06:05:46 +00:00
Code Issues Packages Projects Releases Wiki Activity

Cleanup/consolidation/expansion of *Utils classes

Browse Source 
* IoUtils.java
    remove class that confusingly-overlaps with commons IOUtils; move remaining methods to ArchiveUtils or archive's FileUtils
* ArchiveUtils.java, FileUtils.java
    receive relocated methods; eliminated deprecated or unused methods
* JSONUtils.java
    new class to collect common JSON actions
* Iteratorable.java
    wrap Iterator as Iterable for foreach usage
* (many)
    update to use alternate utils methods
This commit is contained in:
gojomo
2009-11-19 22:39:53 +00:00
parent 300ce15a06
commit e42e2ed7f6
21 changed files with 314 additions and 601 deletions
Download Patch File Download Diff File Expand all files Collapse all files
commons/src/main/java/org/archive/io/ArchiveReaderFactory.java
+2 -3
View File
@@ -33,7 +33,7 @@ import org.archive.io.warc.WARCReaderFactory;
import org.archive.net.UURI;
import org.archive.net.md5.Md5URLConnection;
import org.archive.net.rsync.RsyncURLConnection;
import org.archive.util.IoUtils;
import org.archive.util.FileUtils;
/**
@@ -273,8 +273,7 @@ public class ArchiveReaderFactory implements ArchiveFileConstants {
addUserAgent((HttpURLConnection)connection);
connection.connect();
try {
IoUtils.readFullyToFile(connection.getInputStream(), localFile,
new byte[16 * 1024]);
FileUtils.readFullyToFile(connection.getInputStream(), localFile);
} catch (IOException ioe) {
localFile.delete();
throw ioe;
commons/src/main/java/org/archive/io/WriterPoolMember.java
+2 -2
View File
@@ -35,7 +35,7 @@ import java.util.logging.Logger;
import java.util.zip.GZIPOutputStream;
import org.archive.util.ArchiveUtils;
import org.archive.util.IoUtils;
import org.archive.util.FileUtils;
import org.archive.util.TimestampSerialno;
@@ -269,7 +269,7 @@ public abstract class WriterPoolMember implements ArchiveFileConstants {
}
try {
IoUtils.ensureWriteableDirectory(d);
FileUtils.ensureWriteableDirectory(d);
} catch(IOException e) {
logger.warning("Directory " + d.getPath() + " is not" +
" writeable or cannot be created: " + e.getMessage());
commons/src/main/java/org/archive/util/ArchiveUtils.java
+75 -51
View File
@@ -19,25 +19,30 @@
package org.archive.util;
import java.io.BufferedInputStream;
import java.io.BufferedReader;
import java.io.Closeable;
import java.io.EOFException;
import java.io.File;
import java.io.FileInputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.io.PrintWriter;
import java.io.StringWriter;
import java.net.URL;
import java.net.URLConnection;
import java.text.NumberFormat;
import java.text.ParseException;
import java.text.SimpleDateFormat;
import java.util.Arrays;
import java.util.Calendar;
import java.util.Date;
import java.util.GregorianCalendar;
import java.util.HashSet;
import java.util.Locale;
import java.util.Map;
import java.util.Set;
import java.util.TimeZone;
import java.util.zip.GZIPInputStream;
/**
@@ -117,7 +122,7 @@ public class ArchiveUtils {
/**
* Utility function for creating arc-style date stamps
* in the format yyyMMddHHmmss.
* in the format yyyyMMddHHmmss.
* Date stamps are in the UTC time zone
* @return the date stamp
*/
@@ -127,7 +132,7 @@ public class ArchiveUtils {
/**
* Utility function for creating arc-style date stamps
* in the format yyyMMddHHmm.
* in the format yyyyMMddHHmm.
* Date stamps are in the UTC time zone
* @return the date stamp
*/
@@ -348,41 +353,6 @@ public class ArchiveUtils {
return TIMESTAMP12.get().parse(date);
}
/**
* Convert 17-digit date format timestamps (as found in crawl.log, for
* example) into a GregorianCalendar object. + * Useful so you can convert
* into milliseconds-since-epoch. Note: it is possible to compute
* milliseconds-since-epoch + * using {@link #parse17DigitDate}.UTC(), but
* that method is deprecated in favor of using Calendar.getTimeInMillis(). + *
* <p/>I probably should have dug into all the utility methods in
* DateFormat.java to parse the timestamp, but this was + * easier. If
* someone wants to fix this to use those methods, please have at it! <p/>
* Mike Schwartz, schwartz at CodeOnTheRoad dot com.
*
* @param timestamp17String
* @return Calendar set to <code>timestamp17String</code>.
*/
public static Calendar timestamp17ToCalendar(String timestamp17String) {
GregorianCalendar calendar = new GregorianCalendar();
int year = Integer.parseInt(timestamp17String.substring(0, 4));
int dayOfMonth = Integer.parseInt(timestamp17String.substring(6, 8));
// Month is 0-based
int month = Integer.parseInt(timestamp17String.substring(4, 6)) - 1;
int hourOfDay = Integer.parseInt(timestamp17String.substring(8, 10));
int minute = Integer.parseInt(timestamp17String.substring(10, 12));
int second = Integer.parseInt(timestamp17String.substring(12, 14));
int milliseconds = Integer
.parseInt(timestamp17String.substring(14, 17));
calendar.set(Calendar.YEAR, year);
calendar.set(Calendar.MONTH, month);
calendar.set(Calendar.DAY_OF_MONTH, dayOfMonth);
calendar.set(Calendar.HOUR_OF_DAY, hourOfDay);
calendar.set(Calendar.MINUTE, minute);
calendar.set(Calendar.SECOND, second);
calendar.set(Calendar.MILLISECOND, milliseconds);
return calendar;
}
/**
* @param timestamp A 14-digit timestamp or the suffix for a 14-digit
* timestamp: E.g. '20010909014640' or '20010101' or '1970'.
@@ -539,15 +509,10 @@ public class ArchiveUtils {
* Takes a byte size and formats it for display with 'friendly' units.
* <p>
* This involves converting it to the largest unit
* (of B, KB, MB, GB, TB) for which the amount will be > 1.
* (of B, KiB, MiB, GiB, TiB) for which the amount will be > 1.
* <p>
* Additionally, at least 2 significant digits are always displayed.
* <p>
* Displays as bytes (B): 0-1023
* Displays as kilobytes (KB): 1024 - 2097151 (~2Mb)
* Displays as megabytes (MB): 2097152 - 4294967295 (~4Gb)
* Displays as gigabytes (GB): 4294967296 - infinity
* <p>
* Negative numbers will be returned as '0 B'.
*
* @param amount the amount of bytes
@@ -566,8 +531,7 @@ public class ArchiveUtils {
unitPowerOf1024++;
}
// TODO: get didactic, make these KiB, MiB, GiB, TiB
final String[] units = { " B", " KB", " MB", " GB", " TB" };
final String[] units = { " B", " KiB", " MiB", " GiB", " TiB" };
// ensure at least 2 significant digits (#.#) for small displayValues
int fractionDigits = (displayAmount < 10) ? 1 : 0;
@@ -756,8 +720,8 @@ public class ArchiveUtils {
} else if (obj instanceof Map) {
return prettyString((Map) obj);
} else {
return "<"+obj+">";
}
return "<"+obj+">";
}
}
/**
@@ -806,6 +770,7 @@ public class ArchiveUtils {
return builder.toString();
}
private static String loadVersion() {
InputStream input = ArchiveUtils.class.getResourceAsStream(
"/org/archive/util/version.txt");
@@ -821,7 +786,7 @@ public class ArchiveUtils {
} catch (IOException e) {
return e.getMessage();
} finally {
IoUtils.close(br);
closeQuietly(br);
}
version = version.trim();
@@ -842,7 +807,7 @@ public class ArchiveUtils {
} catch (IOException e) {
return version;
} finally {
IoUtils.close(br);
closeQuietly(br);
}
if (timestamp.startsWith("timestamp=")) {
@@ -928,6 +893,65 @@ public class ArchiveUtils {
throw new InterruptedException("interrupt detected");
}
}
/**
* Read stream into buf until EOF or buf full.
*
* @param input
* @param buf
* @throws IOException
*/
public static void readFully(InputStream input, byte[] buf)
throws IOException {
int max = buf.length;
int ofs = 0;
while (ofs < max) {
int l = input.read(buf, ofs, max - ofs);
if (l == 0) {
throw new EOFException();
}
ofs += l;
}
}
/** suffix to recognize gzipped files */
public static final String GZIP_SUFFIX = ".gz";
/**
* Get a BufferedReader on the crawler journal given
*
* TODO: move to a general utils class
*
* @param source File journal
* @return journal buffered reader.
* @throws IOException
*/
public static BufferedReader getBufferedReader(File source) throws IOException {
InputStream is = new BufferedInputStream(new FileInputStream(source));
boolean isGzipped = source.getName().toLowerCase().
endsWith(GZIP_SUFFIX);
if(isGzipped) {
is = new GZIPInputStream(is);
}
return new BufferedReader(new InputStreamReader(is));
}
/**
* Get a BufferedReader on the crawler journal given.
*
* @param source URL journal
* @return journal buffered reader.
* @throws IOException
*/
public static BufferedReader getBufferedReader(URL source) throws IOException {
URLConnection conn = source.openConnection();
boolean isGzipped = conn.getContentType() != null && conn.getContentType().equalsIgnoreCase("application/x-gzip")
|| conn.getContentEncoding() != null && conn.getContentEncoding().equalsIgnoreCase("gzip");
InputStream uis = conn.getInputStream();
return new BufferedReader(isGzipped?
new InputStreamReader(new GZIPInputStream(uis)):
new InputStreamReader(uis));
}
// public static long doubleMurmur(byte[] data) {
// int first = MurmurHash.hash(data, 7);
commons/src/main/java/org/archive/util/FileUtils.java
+116 -76
View File
@@ -18,6 +18,9 @@
*/
package org.archive.util;
import it.unimi.dsi.fastutil.io.FastBufferedOutputStream;
import java.io.BufferedInputStream;
import java.io.ByteArrayInputStream;
import java.io.File;
import java.io.FileFilter;
@@ -25,6 +28,8 @@ import java.io.FileInputStream;
import java.io.FileNotFoundException;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
import java.nio.channels.ClosedByInterruptException;
import java.nio.channels.FileChannel;
import java.util.Iterator;
@@ -55,18 +60,6 @@ public class FileUtils {
private FileUtils() {
super();
}
/** Recursively copy all files from one directory to another.
*
* @param src file or directory to copy from.
* @param dest file or directory to copy to.
* @throws IOException
* @deprecated use org.apache.commons.io.FileUtils.copyDirectory()
*/
public static void copyFiles(File src, File dest)
throws IOException {
org.apache.commons.io.FileUtils.copyDirectory(src, dest);
}
/**
* Copy the src file to the destination. Deletes any preexisting
@@ -83,26 +76,6 @@ public class FileUtils {
return copyFile(src, dest, -1, true);
}
/**
* Copy the src file to the destination.
*
* @param src
* @param dest
* @param overwrite If target file already exits, and this parameter is
* true, overwrite target file (We do this by first deleting the target
* file before we begin the copy).
* @return True if the extent was greater than actual bytes copied.
* @throws FileNotFoundException
* @throws IOException
* @deprecated use org.apache.commons.io.FileUtils.co
*/
public static boolean copyFile(final File srcFile, final File destFile,
final boolean overwrite)
throws FileNotFoundException, IOException {
org.apache.commons.io.FileUtils.copyFile(srcFile, destFile);
return false;
}
/**
* Copy up to extent bytes of the source file to the destination.
* Deletes any preexisting file at destination.
@@ -237,40 +210,6 @@ public class FileUtils {
}
}
/** Deletes all files and subdirectories under dir.
* @param dir
* @return true if all deletions were successful. If a deletion fails, the
* method stops attempting to delete and returns false.
* @deprecated use org.apache.commons.io.FileUtils.deleteDirectory()
*/
public static boolean deleteDir(File dir) {
if (dir.isDirectory()) {
String[] children = dir.list();
for (int i=0; i<children.length; i++) {
boolean success = deleteDir(new File(dir, children[i]));
if (!success) {
return false;
}
}
}
// The directory is now empty so delete it
return dir.delete();
}
/**
* Utility method to read an entire file as a String.
*
* @param file
* @return File as String.
* @throws IOException
* @deprecated use org.apache.commons.io.FileUtils.readFileToString()
*/
public static String readFileAsString(File file) throws IOException {
return org.apache.commons.io.FileUtils.readFileToString(file);
}
/**
* Get a list of all files in directory that have passed prefix.
*
@@ -403,7 +342,7 @@ public class FileUtils {
p.load(finp);
return p;
} finally {
IoUtils.close(finp);
ArchiveUtils.closeQuietly(finp);
}
}
@@ -418,17 +357,24 @@ public class FileUtils {
try {
p.store(fos,"");
} finally {
IoUtils.close(fos);
ArchiveUtils.closeQuietly(fos);
}
}
public static void moveAsideIfExists(File file) throws IOException {
if(file.exists()) {
String newName =
file.getCanonicalPath() + "."
+ ArchiveUtils.get14DigitDate(file.lastModified());
file.renameTo(new File(newName));
// TODO: comment
public static boolean moveAsideIfExists(File file) throws IOException {
if(!file.exists()) {
return true;
}
String newName =
file.getCanonicalPath() + "."
+ ArchiveUtils.get14DigitDate(file.lastModified());
boolean retVal = file.renameTo(new File(newName));
if(!retVal) {
LOGGER.warning("unable to move aside: "+file+" to "+newName);
}
return retVal;
}
/**
@@ -484,7 +430,7 @@ public class FileUtils {
FileInputStream fis = new FileInputStream(file);
fis.getChannel().position(startPosition);
byte[] buf = new byte[bufferSize];
IoUtils.readFully(fis, buf);
ArchiveUtils.readFully(fis, buf);
IOUtils.closeQuietly(fis);
// find all line starts fully in buffer
@@ -657,5 +603,99 @@ public class FileUtils {
LOGGER.severe(">50 pending Files to delete even after gc/finalization");
}
}
static LinkedList<File> pendingDeletes = new LinkedList<File>();
static LinkedList<File> pendingDeletes = new LinkedList<File>();
/**
* Read the entire stream to EOF into the passed file.
* Closes <code>is</code> when done or if an exception.
* @param is Stream to read.
* @param toFile File to write to.
* @throws IOException
*/
public static void readFullyToFile(InputStream is, File toFile)
throws IOException {
byte[] buffer = new byte[16 * 1024];
long totalcount = -1;
OutputStream os = new FastBufferedOutputStream(new FileOutputStream(
toFile));
InputStream localIs = (is instanceof BufferedInputStream) ? is
: new BufferedInputStream(is);
try {
for (int count = -1; (count = localIs
.read(buffer, 0, buffer.length)) != -1; totalcount += count) {
os.write(buffer, 0, count);
}
} finally {
os.close();
if (localIs != null) {
localIs.close();
}
}
}
/**
* Ensure writeable directory.
*
* If doesn't exist, we attempt creation.
*
* @param dir Directory to test for exitence and is writeable.
*
* @return The passed <code>dir</code>.
*
* @exception IOException If passed directory does not exist and is not
* createable, or directory is not writeable or is not a directory.
*/
public static File ensureWriteableDirectory(String dir)
throws IOException {
return FileUtils.ensureWriteableDirectory(new File(dir));
}
/**
* Ensure writeable directories.
*
* If doesn't exist, we attempt creation.
*
* @param dirs List of Files to test.
*
* @return The passed <code>dirs</code>.
*
* @exception IOException If passed directory does not exist and is not
* createable, or directory is not writeable or is not a directory.
*/
public static List<File> ensureWriteableDirectory(List<File> dirs)
throws IOException {
for (Iterator<File> i = dirs.iterator(); i.hasNext();) {
FileUtils.ensureWriteableDirectory(i.next());
}
return dirs;
}
/**
* Ensure writeable directory.
*
* If doesn't exist, we attempt creation.
*
* @param dir Directory to test for exitence and is writeable.
*
* @return The passed <code>dir</code>.
*
* @exception IOException If passed directory does not exist and is not
* createable, or directory is not writeable or is not a directory.
*/
public static File ensureWriteableDirectory(File dir)
throws IOException {
if (!dir.exists()) {
dir.mkdirs();
} else {
if (!dir.canWrite()) {
throw new IOException("Dir " + dir.getAbsolutePath() +
" not writeable.");
} else if (!dir.isDirectory()) {
throw new IOException("Dir " + dir.getAbsolutePath() +
" is not a directory.");
}
}
return dir;
}
}
commons/src/main/java/org/archive/util/IoUtils.java
-375
View File
@@ -1,375 +0,0 @@
/*
* This file is part of the Heritrix web crawler (crawler.archive.org).
*
* Licensed to the Internet Archive (IA) by one or more individual
* contributors.
*
* The IA licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.archive.util;
import it.unimi.dsi.fastutil.io.FastBufferedOutputStream;
import java.io.BufferedInputStream;
import java.io.BufferedOutputStream;
import java.io.BufferedReader;
import java.io.Closeable;
import java.io.EOFException;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.io.ObjectInputStream;
import java.io.ObjectOutputStream;
import java.io.OutputStream;
import java.io.Reader;
import java.io.Serializable;
import java.net.URL;
import java.net.URLConnection;
import java.nio.charset.Charset;
import java.util.Iterator;
import java.util.List;
import java.util.logging.Level;
import java.util.logging.Logger;
import java.util.zip.GZIPInputStream;
import org.apache.commons.io.IOUtils;
import org.apache.commons.lang.SerializationUtils;
/**
* I/O Utility methods.
* @author stack
* @version $Date$, $Revision$
*/
public class IoUtils {
protected static Logger logger =
Logger.getLogger(IoUtils.class.getName());
/**
* @param file File to operate on.
* @return Path suitable for use getting resources off the CLASSPATH
* (CLASSPATH resources always use '/' as path separator, even on
* windows).
*/
public static String getClasspathPath(File file) {
String path = file.getPath();
if (File.separatorChar != '/') {
// OK. We're probably on a windows system. Strip
// drive if its present and convert '\' to '/'.
path = path.replace(File.separatorChar, '/');
int index = path.indexOf(':');
if (index > 0 && index < 3) {
path = path.substring(index + 1);
}
}
return path;
}
/**
* Ensure writeable directory.
*
* If doesn't exist, we attempt creation.
*
* @param dir Directory to test for exitence and is writeable.
*
* @return The passed <code>dir</code>.
*
* @exception IOException If passed directory does not exist and is not
* createable, or directory is not writeable or is not a directory.
*/
public static File ensureWriteableDirectory(String dir)
throws IOException {
return ensureWriteableDirectory(new File(dir));
}
/**
* Ensure writeable directories.
*
* If doesn't exist, we attempt creation.
*
* @param dirs List of Files to test.
*
* @return The passed <code>dirs</code>.
*
* @exception IOException If passed directory does not exist and is not
* createable, or directory is not writeable or is not a directory.
*/
public static List<File> ensureWriteableDirectory(List<File> dirs)
throws IOException {
for (Iterator<File> i = dirs.iterator(); i.hasNext();) {
ensureWriteableDirectory(i.next());
}
return dirs;
}
/**
* Ensure writeable directory.
*
* If doesn't exist, we attempt creation.
*
* @param dir Directory to test for exitence and is writeable.
*
* @return The passed <code>dir</code>.
*
* @exception IOException If passed directory does not exist and is not
* createable, or directory is not writeable or is not a directory.
*/
public static File ensureWriteableDirectory(File dir)
throws IOException {
if (!dir.exists()) {
dir.mkdirs();
} else {
if (!dir.canWrite()) {
throw new IOException("Dir " + dir.getAbsolutePath() +
" not writeable.");
} else if (!dir.isDirectory()) {
throw new IOException("Dir " + dir.getAbsolutePath() +
" is not a directory.");
}
}
return dir;
}
/**
* Read the entire stream to EOF, returning what's read as a String.
*
* @param inputStream
* @return String of the whole inputStream's contents
* @throws IOException
* @deprecated use org.apache.commons.io.IOUtils.toString()
*/
public static String readFullyAsString(InputStream inputStream)
throws IOException {
return IOUtils.toString(inputStream);
}
/**
* @deprecated use org.apache.commons.io.IOUtils.toString()
*/
public static String readFullyAsString(Reader r) throws IOException {
return IOUtils.toString(r);
}
/**
* Read the entire stream to EOF into the passed file.
* @param is
* @param toFile File to read into .
* @throws IOException
* @throws IOException
*/
public static void readFullyToFile(InputStream is,
File toFile) throws IOException {
readFullyToFile(is, toFile, new byte[4096]);
}
/**
* Read the entire stream to EOF into the passed file.
* Closes <code>is</code> when done or if an exception.
* @param is Stream to read.
* @param toFile File to read into .
* @param buffer Buffer to use reading.
* @return Count of bytes read.
* @throws IOException
*/
public static long readFullyToFile(final InputStream is, final File toFile,
final byte [] buffer)
throws IOException {
long totalcount = -1;
OutputStream os =
new FastBufferedOutputStream(new FileOutputStream(toFile));
InputStream localIs = (is instanceof BufferedInputStream)?
is: new BufferedInputStream(is);
try {
for (int count = -1;
(count = localIs.read(buffer, 0, buffer.length)) != -1;
totalcount += count) {
os.write(buffer, 0, count);
}
} finally {
os.close();
if (localIs != null) {
localIs.close();
}
}
return totalcount;
}
/**
* Wrap generic Throwable as a checked IOException
* @param e wrapped exception
* @return IOException
*/
public static IOException wrapAsIOException(Throwable e) {
IOException ioe = new IOException(e.toString());
ioe.initCause(e);
return ioe;
}
public static void readFully(InputStream input, byte[] buf)
throws IOException {
int max = buf.length;
int ofs = 0;
while (ofs < max) {
int l = input.read(buf, ofs, max - ofs);
if (l == 0) {
throw new EOFException();
}
ofs += l;
}
}
public static void close(Closeable c) {
if (c == null) {
return;
}
try {
c.close();
} catch (IOException e) {
}
}
/**
* Return the maximum number of bytes per character in the named
* encoding, or 0 if encoding is invalid or unsupported.
*
* @param encoding Encoding to consider. For now, should be java
* canonical name for the encoding.
*
* @return True if multibyte encoding.
*/
public static float encodingMaxBytesPerChar(String encoding) {
boolean isMultibyte = false;
final Charset cs;
try {
if (encoding != null && encoding.length() > 0) {
cs = Charset.forName(encoding);
if(cs.canEncode()) {
return cs.newEncoder().maxBytesPerChar();
} else {
logger.info("Encoding not fully supported: " + encoding
+ ". Defaulting to single byte.");
}
}
} catch (IllegalArgumentException e) {
// Unsupported encoding
logger.log(Level.INFO,"Illegal encoding name: " + encoding,e);
}
logger.fine("Encoding " + encoding + " is multibyte: "
+ ((isMultibyte) ? Boolean.TRUE : Boolean.FALSE));
// default: return 0
return 0;
}
/**
* Utility method to serialize an object to the given File.
*
* @param object Object to serialize
* @param file File to receive serialized copy
* @throws IOException
*/
public static void serializeToFile(Object object, File file) throws IOException {
ObjectOutputStream oos = new ObjectOutputStream(new BufferedOutputStream(new FileOutputStream(file)));
oos.writeObject(object);
oos.close();
}
/**
* Utility method to deserialize an Object from given File.
*
* @param file File source
* @return deserialized Object
* @throws IOException
*/
public static Object deserializeFromFile(File file) throws IOException {
ObjectInputStream ois = new ObjectInputStream(new BufferedInputStream(new FileInputStream(file)));
Object object;
try {
object = ois.readObject();
} catch (ClassNotFoundException e) {
// TODO Auto-generated catch block
throw new RuntimeException(e);
}
ois.close();
return object;
}
/**
* Utility method to serialize Object to byte[].
*
* @param object Object to be serialized
* @return byte[] serialized form
* @deprecated use SerializationUtils.serialize
*/
public static byte[] serializeToByteArray(Object object) {
return SerializationUtils.serialize((Serializable)object);
}
/**
* Utility method to deserialize Object from byte[].
*
* @param in byte[] source
* @return Object deserialized
* @deprecated use SerializationUtils.deserialize
*/
public static Object deserializeFromByteArray(byte[] in) {
return SerializationUtils.deserialize(in);
}
/** suffix to recognize gzipped files */
public static final String GZIP_SUFFIX = ".gz";
/**
* Get a BufferedReader on the crawler journal given
*
* TODO: move to a general utils class
*
* @param source File journal
* @return journal buffered reader.
* @throws IOException
*/
public static BufferedReader getBufferedReader(File source) throws IOException {
InputStream is = new BufferedInputStream(new FileInputStream(source));
boolean isGzipped = source.getName().toLowerCase().
endsWith(GZIP_SUFFIX);
if(isGzipped) {
is = new GZIPInputStream(is);
}
return new BufferedReader(new InputStreamReader(is));
}
/**
* Get a BufferedReader on the crawler journal given.
*
* @param source URL journal
* @return journal buffered reader.
* @throws IOException
*/
public static BufferedReader getBufferedReader(URL source) throws IOException {
URLConnection conn = source.openConnection();
boolean isGzipped = conn.getContentType() != null && conn.getContentType().equalsIgnoreCase("application/x-gzip")
|| conn.getContentEncoding() != null && conn.getContentEncoding().equalsIgnoreCase("gzip");
InputStream uis = conn.getInputStream();
return new BufferedReader(isGzipped?
new InputStreamReader(new GZIPInputStream(uis)):
new InputStreamReader(uis));
}
}
commons/src/main/java/org/archive/util/Iteratorable.java
+40
View File
@@ -0,0 +1,40 @@
/*
* This file is part of the Heritrix web crawler (crawler.archive.org).
*
* Licensed to the Internet Archive (IA) by one or more individual
* contributors.
*
* The IA licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.archive.util;
import java.util.Iterator;
/**
* Make an Iterator usable as an Iterable (and thus enable new-style
* for-each loops).
*
*/
public class Iteratorable<K> implements Iterable<K> {
Iterator<K> iterator;
public Iteratorable(Iterator<K> iterator) {
this.iterator = iterator;
}
public Iterator<K> iterator() {
return iterator;
}
}
commons/src/main/java/org/archive/util/JSONUtils.java
+47
View File
@@ -0,0 +1,47 @@
/*
* This file is part of the Heritrix web crawler (crawler.archive.org).
*
* Licensed to the Internet Archive (IA) by one or more individual
* contributors.
*
* The IA licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.archive.util;
import java.util.Map;
import java.util.concurrent.atomic.AtomicLong;
import org.json.JSONException;
import org.json.JSONObject;
/**
* Utilities for working with JSON/JSONObjects.
*
*/
public class JSONUtils {
@SuppressWarnings("unchecked")
public static void putAllLongs(Map<String,Long> targetMap, JSONObject sourceJson) throws JSONException {
for(String k : new Iteratorable<String>(sourceJson.keys())) {
targetMap.put(k, sourceJson.getLong(k));
}
}
@SuppressWarnings("unchecked")
public static void putAllAtomicLongs(Map<String,AtomicLong> targetMap, JSONObject sourceJson) throws JSONException {
for(String k : new Iteratorable<String>(sourceJson.keys())) {
targetMap.put(k, new AtomicLong(sourceJson.getLong(k)));
}
}
}
commons/src/main/java/org/archive/util/ms/DefaultBlockFileSystem.java
+3 -3
View File
@@ -24,7 +24,7 @@ import java.nio.ByteOrder;
import java.util.Map;
import org.archive.io.SeekInputStream;
import org.archive.util.IoUtils;
import org.archive.util.ArchiveUtils;
import org.archive.util.LRU;
@@ -156,7 +156,7 @@ public class DefaultBlockFileSystem implements BlockFileSystem {
throws IOException {
this.input = input;
byte[] temp = new byte[BLOCK_SIZE];
IoUtils.readFully(input, temp);
ArchiveUtils.readFully(input, temp);
this.header = new HeaderBlock(ByteBuffer.wrap(temp));
this.cache = new LRU<Integer,ByteBuffer>(batCacheSize);
}
@@ -268,7 +268,7 @@ public class DefaultBlockFileSystem implements BlockFileSystem {
byte[] buf = new byte[BLOCK_SIZE];
input.position((block + 1) * BLOCK_SIZE);
IoUtils.readFully(input, buf);
ArchiveUtils.readFully(input, buf);
r = ByteBuffer.wrap(buf);
r.order(ByteOrder.LITTLE_ENDIAN);
commons/src/main/java/org/archive/util/ms/DefaultEntry.java
+2 -2
View File
@@ -24,7 +24,7 @@ import java.nio.ByteOrder;
import java.util.ArrayList;
import java.util.List;
import org.archive.util.IoUtils;
import org.archive.util.ArchiveUtils;
import org.archive.io.SeekInputStream;
class DefaultEntry implements Entry {
@@ -47,7 +47,7 @@ class DefaultEntry implements Entry {
// FIXME: Read directly from the stream
this.origin = origin;
byte[] temp = new byte[128];
IoUtils.readFully(input, temp);
ArchiveUtils.readFully(input, temp);
ByteBuffer buf = ByteBuffer.wrap(temp);
buf.order(ByteOrder.LITTLE_ENDIAN);
buf.position(0);
commons/src/test/java/org/archive/util/ArchiveUtilsTest.java
+7 -7
View File
@@ -248,20 +248,20 @@ public class ArchiveUtilsTest extends TestCase {
.formatBytesForDisplay(0));
assertEquals("1023 bytes", "1,023 B", ArchiveUtils
.formatBytesForDisplay(1023));
assertEquals("1025 bytes", "1.0 KB", ArchiveUtils
assertEquals("1025 bytes", "1.0 KiB", ArchiveUtils
.formatBytesForDisplay(1025));
// expected display values taken from Google calculator
assertEquals("10,000 bytes", "9.8 KB",
assertEquals("10,000 bytes", "9.8 KiB",
ArchiveUtils.formatBytesForDisplay(10000));
assertEquals("1,000,000 bytes", "977 KB",
assertEquals("1,000,000 bytes", "977 KiB",
ArchiveUtils.formatBytesForDisplay(1000000));
assertEquals("100,000,000 bytes", "95 MB",
assertEquals("100,000,000 bytes", "95 MiB",
ArchiveUtils.formatBytesForDisplay(100000000));
assertEquals("100,000,000,000 bytes", "93 GB",
assertEquals("100,000,000,000 bytes", "93 GiB",
ArchiveUtils.formatBytesForDisplay(100000000000L));
assertEquals("100,000,000,000,000 bytes", "91 TB",
assertEquals("100,000,000,000,000 bytes", "91 TiB",
ArchiveUtils.formatBytesForDisplay(100000000000000L));
assertEquals("100,000,000,000,000,000 bytes", "90,949 TB",
assertEquals("100,000,000,000,000,000 bytes", "90,949 TiB",
ArchiveUtils.formatBytesForDisplay(100000000000000000L));
}
commons/src/test/java/org/archive/util/FileUtilsTest.java
-11
View File
@@ -119,17 +119,6 @@ public class FileUtilsTest extends TmpDirTestCase {
org.apache.commons.io.FileUtils.deleteQuietly(nakedLastLineWindows);
}
@SuppressWarnings("deprecation")
public void testCopyFiles() throws IOException {
FileUtils.copyFiles(this.srcDirFile, this.tgtDirFile);
File [] srcFiles = this.srcDirFile.listFiles();
for (int i = 0; i < srcFiles.length; i++) {
File tgt = new File(this.tgtDirFile, srcFiles[i].getName());
assertTrue("Tgt doesn't exist " + tgt.getAbsolutePath(),
tgt.exists());
}
}
public void testCopyFile() {
// Test exception copying nonexistent file.
commons/src/test/java/org/archive/util/IoUtilsTest.java
-45
View File
@@ -1,45 +0,0 @@
/*
* This file is part of the Heritrix web crawler (crawler.archive.org).
*
* Licensed to the Internet Archive (IA) by one or more individual
* contributors.
*
* The IA licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.archive.util;
import java.io.File;
import junit.framework.TestCase;
/**
* @author stack
* @version $Date$, $Revision$
*/
public class IoUtilsTest extends TestCase {
public void testGetClasspathPath() {
final String absUnixPath = "/one/two/three";
File f = new File(absUnixPath);
assertTrue("Path is wrong abs " + IoUtils.getClasspathPath(f),
IoUtils.getClasspathPath(f).equals(absUnixPath));
final String relUnixPath = "one/two/three";
f = new File(relUnixPath);
assertTrue("Path is wrong rel " + IoUtils.getClasspathPath(f),
IoUtils.getClasspathPath(f).equals(relUnixPath));
final String nameUnixPath = "three";
f = new File(nameUnixPath);
assertTrue("Path is wrong name " + IoUtils.getClasspathPath(f),
IoUtils.getClasspathPath(f).equals(nameUnixPath));
}
}
engine/src/main/java/org/archive/crawler/frontier/BdbWorkQueue.java
+3 -5
View File
@@ -26,7 +26,6 @@ import java.util.logging.Logger;
import org.archive.modules.CrawlURI;
import org.archive.util.ArchiveUtils;
import org.archive.util.IoUtils;
import com.sleepycat.je.DatabaseEntry;
import com.sleepycat.je.DatabaseException;
@@ -73,7 +72,7 @@ implements Serializable {
return queues.deleteMatchingFromQueue(match, classKey,
new DatabaseEntry(origin));
} catch (DatabaseException e) {
throw IoUtils.wrapAsIOException(e);
throw new IOException(e);
}
}
@@ -84,8 +83,7 @@ implements Serializable {
.getWorkQueues();
queues.delete(peekItem);
} catch (DatabaseException e) {
e.printStackTrace();
throw IoUtils.wrapAsIOException(e);
throw new IOException(e);
}
}
@@ -146,7 +144,7 @@ implements Serializable {
curi.toString());
}
} catch (DatabaseException e) {
throw IoUtils.wrapAsIOException(e);
throw new IOException(e);
}
}
engine/src/main/java/org/archive/crawler/frontier/FrontierJournal.java
+3 -3
View File
@@ -32,7 +32,7 @@ import org.archive.crawler.framework.Frontier;
import org.archive.io.CrawlerJournal;
import org.archive.modules.CrawlURI;
import org.archive.modules.deciderules.DecideRule;
import org.archive.util.IoUtils;
import org.archive.util.ArchiveUtils;
import org.json.JSONObject;
/**
@@ -191,7 +191,7 @@ public class FrontierJournal extends CrawlerJournal implements Checkpointable {
DecideRule scope = (scopeIncludes) ? frontier.getScope() : null;
FrontierJournal newJournal = frontier.getFrontierJournal();
BufferedReader br = IoUtils.getBufferedReader(source);
BufferedReader br = ArchiveUtils.getBufferedReader(source);
String read;
int lines = 0;
try {
@@ -268,7 +268,7 @@ public class FrontierJournal extends CrawlerJournal implements Checkpointable {
try {
// Scan log for all 'F+' lines: if not alreadyIncluded, schedule for
// visitation
br = IoUtils.getBufferedReader(source);
br = ArchiveUtils.getBufferedReader(source);
try {
while ((read = br.readLine())!=null) {
qLines++;
engine/src/main/java/org/archive/crawler/frontier/precedence/PrecedenceLoader.java
+2 -2
View File
@@ -30,7 +30,7 @@ import java.util.Iterator;
import java.util.Map;
import org.archive.modules.recrawl.PersistProcessor;
import org.archive.util.IoUtils;
import org.archive.util.ArchiveUtils;
import org.archive.util.bdbje.EnhancedEnvironment;
import org.archive.util.iterator.LineReadingIterator;
@@ -119,7 +119,7 @@ public class PrecedenceLoader {
if(source.isFile()) {
// scan log, writing to database
BufferedReader br = IoUtils.getBufferedReader(source);
BufferedReader br = ArchiveUtils.getBufferedReader(source);
Iterator iter = new LineReadingIterator(br);
while(iter.hasNext()) {
String line = (String) iter.next();
engine/src/main/java/org/archive/crawler/util/RecoveryLogMapper.java
+2 -2
View File
@@ -41,7 +41,7 @@
package org.archive.crawler.util;
import org.archive.crawler.frontier.FrontierJournal;
import org.archive.util.IoUtils;
import org.archive.util.ArchiveUtils;
import java.io.File;
import java.io.LineNumberReader;
@@ -118,7 +118,7 @@ public class RecoveryLogMapper {
protected void load(String recoverLogFileName)
throws java.io.FileNotFoundException, java.io.IOException,
SeedUrlNotFoundException {
LineNumberReader reader = new LineNumberReader(IoUtils
LineNumberReader reader = new LineNumberReader(ArchiveUtils
.getBufferedReader(new File(recoverLogFileName)));
String curLine = null;
while ((curLine = reader.readLine()) != null) {
engine/src/test/java/org/archive/crawler/framework/CrawlControllerTest.java
+2 -5
View File
@@ -29,7 +29,7 @@ import org.archive.bdb.BdbModule;
import org.archive.modules.net.BdbServerCache;
import org.archive.spring.ConfigPath;
import org.archive.state.ModuleTestBase;
import org.archive.util.IoUtils;
import org.archive.util.ArchiveUtils;
/**
*
@@ -55,7 +55,7 @@ public class CrawlControllerTest extends ModuleTestBase {
fileWriter.write("http://www.pandemoniummovie.com");
fileWriter.close();
} finally {
IoUtils.close(fileWriter);
ArchiveUtils.closeQuietly(fileWriter);
}
File state = new File(tmp, "state");
@@ -69,11 +69,8 @@ public class CrawlControllerTest extends ModuleTestBase {
// def.set(bdb, BdbModule.DIR, state.getAbsolutePath());
bdb.start();
String cp = checkpoints.getAbsolutePath();
CrawlController controller = new CrawlController();
controller.setServerCache(new BdbServerCache());
controller.setCheckpointsDir(new ConfigPath("test",cp));
controller.start();
return controller;
}
engine/src/test/java/org/archive/crawler/selftest/Precedence1SelfTest.java
+2 -2
View File
@@ -24,7 +24,7 @@ import java.io.File;
import java.io.FileReader;
import org.archive.crawler.frontier.precedence.BaseUriPrecedencePolicy;
import org.archive.util.IoUtils;
import org.archive.util.ArchiveUtils;
/**
* Tests that operators can create precedence groups for URIs, and that URIs
@@ -158,7 +158,7 @@ public class Precedence1SelfTest extends SelfTestBase {
crawled = crawled + s + "\n";
}
} finally {
IoUtils.close(br);
ArchiveUtils.closeQuietly(br);
}
assertEquals(EXPECTED, crawled);
modules/src/main/java/org/archive/modules/fetcher/AbstractCookieStorage.java
+2 -2
View File
@@ -35,7 +35,7 @@ import java.util.logging.Logger;
import org.apache.commons.httpclient.Cookie;
import org.archive.spring.ConfigPath;
import org.archive.util.IoUtils;
import org.archive.util.ArchiveUtils;
import org.springframework.context.Lifecycle;
/**
@@ -154,7 +154,7 @@ public abstract class AbstractCookieStorage
} catch (IOException e) {
LOGGER.log(Level.WARNING,e.getMessage(), e);
} finally {
IoUtils.close(raf);
ArchiveUtils.closeQuietly(raf);
}
}
modules/src/main/java/org/archive/modules/recrawl/PersistProcessor.java
+2 -3
View File
@@ -41,7 +41,6 @@ import org.archive.modules.CrawlURI;
import org.archive.modules.Processor;
import org.archive.util.ArchiveUtils;
import org.archive.util.FileUtils;
import org.archive.util.IoUtils;
import org.archive.util.OneLineSimpleLogger;
import org.archive.util.SURT;
import org.archive.util.bdbje.EnhancedEnvironment;
@@ -301,10 +300,10 @@ public abstract class PersistProcessor extends Processor {
} else {
BufferedReader persistLogReader = null;
if (sourceFile.isFile()) {
persistLogReader = IoUtils.getBufferedReader(sourceFile);
persistLogReader = ArchiveUtils.getBufferedReader(sourceFile);
} else {
URL sourceUrl = new URL(sourcePath);
persistLogReader = IoUtils.getBufferedReader(sourceUrl);
persistLogReader = ArchiveUtils.getBufferedReader(sourceUrl);
}
count = populatePersistEnvFromLog(persistLogReader, historyMap);
}
modules/src/main/java/org/archive/modules/writer/MirrorWriterProcessor.java
+2 -2
View File
@@ -50,7 +50,7 @@ import org.archive.io.ReplayInputStream;
import org.archive.modules.CrawlURI;
import org.archive.modules.Processor;
import org.archive.net.UURI;
import org.archive.util.IoUtils;
import org.archive.util.FileUtils;
/**
Processor module that writes the results of successful fetches to
@@ -399,7 +399,7 @@ public class MirrorWriterProcessor extends Processor {
destFile = new File(baseDir + File.separator + mps);
File parent = destFile.getParentFile();
if (null != parent) {
IoUtils.ensureWriteableDirectory(parent);
FileUtils.ensureWriteableDirectory(parent);
}
} else {
URIToFileReturn r = null; // Return from uriToFile().