timestamp17String.
- */
- public static Calendar timestamp17ToCalendar(String timestamp17String) {
- GregorianCalendar calendar = new GregorianCalendar();
- int year = Integer.parseInt(timestamp17String.substring(0, 4));
- int dayOfMonth = Integer.parseInt(timestamp17String.substring(6, 8));
- // Month is 0-based
- int month = Integer.parseInt(timestamp17String.substring(4, 6)) - 1;
- int hourOfDay = Integer.parseInt(timestamp17String.substring(8, 10));
- int minute = Integer.parseInt(timestamp17String.substring(10, 12));
- int second = Integer.parseInt(timestamp17String.substring(12, 14));
- int milliseconds = Integer
- .parseInt(timestamp17String.substring(14, 17));
- calendar.set(Calendar.YEAR, year);
- calendar.set(Calendar.MONTH, month);
- calendar.set(Calendar.DAY_OF_MONTH, dayOfMonth);
- calendar.set(Calendar.HOUR_OF_DAY, hourOfDay);
- calendar.set(Calendar.MINUTE, minute);
- calendar.set(Calendar.SECOND, second);
- calendar.set(Calendar.MILLISECOND, milliseconds);
- return calendar;
- }
-
/**
* @param timestamp A 14-digit timestamp or the suffix for a 14-digit
* timestamp: E.g. '20010909014640' or '20010101' or '1970'.
@@ -539,15 +509,10 @@ public class ArchiveUtils {
* Takes a byte size and formats it for display with 'friendly' units.
* * This involves converting it to the largest unit - * (of B, KB, MB, GB, TB) for which the amount will be > 1. + * (of B, KiB, MiB, GiB, TiB) for which the amount will be > 1. *
* Additionally, at least 2 significant digits are always displayed. *
- * Displays as bytes (B): 0-1023 - * Displays as kilobytes (KB): 1024 - 2097151 (~2Mb) - * Displays as megabytes (MB): 2097152 - 4294967295 (~4Gb) - * Displays as gigabytes (GB): 4294967296 - infinity - *
* Negative numbers will be returned as '0 B'.
*
* @param amount the amount of bytes
@@ -566,8 +531,7 @@ public class ArchiveUtils {
unitPowerOf1024++;
}
- // TODO: get didactic, make these KiB, MiB, GiB, TiB
- final String[] units = { " B", " KB", " MB", " GB", " TB" };
+ final String[] units = { " B", " KiB", " MiB", " GiB", " TiB" };
// ensure at least 2 significant digits (#.#) for small displayValues
int fractionDigits = (displayAmount < 10) ? 1 : 0;
@@ -756,8 +720,8 @@ public class ArchiveUtils {
} else if (obj instanceof Map) {
return prettyString((Map) obj);
} else {
- return "<"+obj+">";
- }
+ return "<"+obj+">";
+ }
}
/**
@@ -806,6 +770,7 @@ public class ArchiveUtils {
return builder.toString();
}
+
private static String loadVersion() {
InputStream input = ArchiveUtils.class.getResourceAsStream(
"/org/archive/util/version.txt");
@@ -821,7 +786,7 @@ public class ArchiveUtils {
} catch (IOException e) {
return e.getMessage();
} finally {
- IoUtils.close(br);
+ closeQuietly(br);
}
version = version.trim();
@@ -842,7 +807,7 @@ public class ArchiveUtils {
} catch (IOException e) {
return version;
} finally {
- IoUtils.close(br);
+ closeQuietly(br);
}
if (timestamp.startsWith("timestamp=")) {
@@ -928,6 +893,65 @@ public class ArchiveUtils {
throw new InterruptedException("interrupt detected");
}
}
+
+ /**
+ * Read stream into buf until EOF or buf full.
+ *
+ * @param input
+ * @param buf
+ * @throws IOException
+ */
+ public static void readFully(InputStream input, byte[] buf)
+ throws IOException {
+ int max = buf.length;
+ int ofs = 0;
+ while (ofs < max) {
+ int l = input.read(buf, ofs, max - ofs);
+ if (l == 0) {
+ throw new EOFException();
+ }
+ ofs += l;
+ }
+ }
+
+ /** suffix to recognize gzipped files */
+ public static final String GZIP_SUFFIX = ".gz";
+
+ /**
+ * Get a BufferedReader on the crawler journal given
+ *
+ * TODO: move to a general utils class
+ *
+ * @param source File journal
+ * @return journal buffered reader.
+ * @throws IOException
+ */
+ public static BufferedReader getBufferedReader(File source) throws IOException {
+ InputStream is = new BufferedInputStream(new FileInputStream(source));
+ boolean isGzipped = source.getName().toLowerCase().
+ endsWith(GZIP_SUFFIX);
+ if(isGzipped) {
+ is = new GZIPInputStream(is);
+ }
+ return new BufferedReader(new InputStreamReader(is));
+ }
+
+ /**
+ * Get a BufferedReader on the crawler journal given.
+ *
+ * @param source URL journal
+ * @return journal buffered reader.
+ * @throws IOException
+ */
+ public static BufferedReader getBufferedReader(URL source) throws IOException {
+ URLConnection conn = source.openConnection();
+ boolean isGzipped = conn.getContentType() != null && conn.getContentType().equalsIgnoreCase("application/x-gzip")
+ || conn.getContentEncoding() != null && conn.getContentEncoding().equalsIgnoreCase("gzip");
+ InputStream uis = conn.getInputStream();
+ return new BufferedReader(isGzipped?
+ new InputStreamReader(new GZIPInputStream(uis)):
+ new InputStreamReader(uis));
+ }
// public static long doubleMurmur(byte[] data) {
// int first = MurmurHash.hash(data, 7);
diff --git a/commons/src/main/java/org/archive/util/FileUtils.java b/commons/src/main/java/org/archive/util/FileUtils.java
index fb422073..ba1a89f7 100644
--- a/commons/src/main/java/org/archive/util/FileUtils.java
+++ b/commons/src/main/java/org/archive/util/FileUtils.java
@@ -18,6 +18,9 @@
*/
package org.archive.util;
+import it.unimi.dsi.fastutil.io.FastBufferedOutputStream;
+
+import java.io.BufferedInputStream;
import java.io.ByteArrayInputStream;
import java.io.File;
import java.io.FileFilter;
@@ -25,6 +28,8 @@ import java.io.FileInputStream;
import java.io.FileNotFoundException;
import java.io.FileOutputStream;
import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
import java.nio.channels.ClosedByInterruptException;
import java.nio.channels.FileChannel;
import java.util.Iterator;
@@ -55,18 +60,6 @@ public class FileUtils {
private FileUtils() {
super();
}
-
- /** Recursively copy all files from one directory to another.
- *
- * @param src file or directory to copy from.
- * @param dest file or directory to copy to.
- * @throws IOException
- * @deprecated use org.apache.commons.io.FileUtils.copyDirectory()
- */
- public static void copyFiles(File src, File dest)
- throws IOException {
- org.apache.commons.io.FileUtils.copyDirectory(src, dest);
- }
/**
* Copy the src file to the destination. Deletes any preexisting
@@ -83,26 +76,6 @@ public class FileUtils {
return copyFile(src, dest, -1, true);
}
- /**
- * Copy the src file to the destination.
- *
- * @param src
- * @param dest
- * @param overwrite If target file already exits, and this parameter is
- * true, overwrite target file (We do this by first deleting the target
- * file before we begin the copy).
- * @return True if the extent was greater than actual bytes copied.
- * @throws FileNotFoundException
- * @throws IOException
- * @deprecated use org.apache.commons.io.FileUtils.co
- */
- public static boolean copyFile(final File srcFile, final File destFile,
- final boolean overwrite)
- throws FileNotFoundException, IOException {
- org.apache.commons.io.FileUtils.copyFile(srcFile, destFile);
- return false;
- }
-
/**
* Copy up to extent bytes of the source file to the destination.
* Deletes any preexisting file at destination.
@@ -237,40 +210,6 @@ public class FileUtils {
}
}
- /** Deletes all files and subdirectories under dir.
- * @param dir
- * @return true if all deletions were successful. If a deletion fails, the
- * method stops attempting to delete and returns false.
- * @deprecated use org.apache.commons.io.FileUtils.deleteDirectory()
- */
- public static boolean deleteDir(File dir) {
- if (dir.isDirectory()) {
- String[] children = dir.list();
- for (int i=0; iis when done or if an exception.
+ * @param is Stream to read.
+ * @param toFile File to write to.
+ * @throws IOException
+ */
+ public static void readFullyToFile(InputStream is, File toFile)
+ throws IOException {
+ byte[] buffer = new byte[16 * 1024];
+ long totalcount = -1;
+ OutputStream os = new FastBufferedOutputStream(new FileOutputStream(
+ toFile));
+ InputStream localIs = (is instanceof BufferedInputStream) ? is
+ : new BufferedInputStream(is);
+ try {
+ for (int count = -1; (count = localIs
+ .read(buffer, 0, buffer.length)) != -1; totalcount += count) {
+ os.write(buffer, 0, count);
+ }
+ } finally {
+ os.close();
+ if (localIs != null) {
+ localIs.close();
+ }
+ }
+ }
+
+ /**
+ * Ensure writeable directory.
+ *
+ * If doesn't exist, we attempt creation.
+ *
+ * @param dir Directory to test for exitence and is writeable.
+ *
+ * @return The passed dir.
+ *
+ * @exception IOException If passed directory does not exist and is not
+ * createable, or directory is not writeable or is not a directory.
+ */
+ public static File ensureWriteableDirectory(String dir)
+ throws IOException {
+ return FileUtils.ensureWriteableDirectory(new File(dir));
+ }
+
+ /**
+ * Ensure writeable directories.
+ *
+ * If doesn't exist, we attempt creation.
+ *
+ * @param dirs List of Files to test.
+ *
+ * @return The passed dirs.
+ *
+ * @exception IOException If passed directory does not exist and is not
+ * createable, or directory is not writeable or is not a directory.
+ */
+ public static Listdir.
+ *
+ * @exception IOException If passed directory does not exist and is not
+ * createable, or directory is not writeable or is not a directory.
+ */
+ public static File ensureWriteableDirectory(File dir)
+ throws IOException {
+ if (!dir.exists()) {
+ dir.mkdirs();
+ } else {
+ if (!dir.canWrite()) {
+ throw new IOException("Dir " + dir.getAbsolutePath() +
+ " not writeable.");
+ } else if (!dir.isDirectory()) {
+ throw new IOException("Dir " + dir.getAbsolutePath() +
+ " is not a directory.");
+ }
+ }
+
+ return dir;
+ }
}
\ No newline at end of file
diff --git a/commons/src/main/java/org/archive/util/IoUtils.java b/commons/src/main/java/org/archive/util/IoUtils.java
deleted file mode 100644
index e0048d5d..00000000
--- a/commons/src/main/java/org/archive/util/IoUtils.java
+++ /dev/null
@@ -1,375 +0,0 @@
-/*
- * This file is part of the Heritrix web crawler (crawler.archive.org).
- *
- * Licensed to the Internet Archive (IA) by one or more individual
- * contributors.
- *
- * The IA licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.archive.util;
-
-import it.unimi.dsi.fastutil.io.FastBufferedOutputStream;
-
-import java.io.BufferedInputStream;
-import java.io.BufferedOutputStream;
-import java.io.BufferedReader;
-import java.io.Closeable;
-import java.io.EOFException;
-import java.io.File;
-import java.io.FileInputStream;
-import java.io.FileOutputStream;
-import java.io.IOException;
-import java.io.InputStream;
-import java.io.InputStreamReader;
-import java.io.ObjectInputStream;
-import java.io.ObjectOutputStream;
-import java.io.OutputStream;
-import java.io.Reader;
-import java.io.Serializable;
-import java.net.URL;
-import java.net.URLConnection;
-import java.nio.charset.Charset;
-import java.util.Iterator;
-import java.util.List;
-import java.util.logging.Level;
-import java.util.logging.Logger;
-import java.util.zip.GZIPInputStream;
-
-import org.apache.commons.io.IOUtils;
-import org.apache.commons.lang.SerializationUtils;
-
-/**
- * I/O Utility methods.
- * @author stack
- * @version $Date$, $Revision$
- */
-public class IoUtils {
- protected static Logger logger =
- Logger.getLogger(IoUtils.class.getName());
-
- /**
- * @param file File to operate on.
- * @return Path suitable for use getting resources off the CLASSPATH
- * (CLASSPATH resources always use '/' as path separator, even on
- * windows).
- */
- public static String getClasspathPath(File file) {
- String path = file.getPath();
- if (File.separatorChar != '/') {
- // OK. We're probably on a windows system. Strip
- // drive if its present and convert '\' to '/'.
- path = path.replace(File.separatorChar, '/');
- int index = path.indexOf(':');
- if (index > 0 && index < 3) {
- path = path.substring(index + 1);
- }
- }
- return path;
- }
-
- /**
- * Ensure writeable directory.
- *
- * If doesn't exist, we attempt creation.
- *
- * @param dir Directory to test for exitence and is writeable.
- *
- * @return The passed dir.
- *
- * @exception IOException If passed directory does not exist and is not
- * createable, or directory is not writeable or is not a directory.
- */
- public static File ensureWriteableDirectory(String dir)
- throws IOException {
- return ensureWriteableDirectory(new File(dir));
- }
-
- /**
- * Ensure writeable directories.
- *
- * If doesn't exist, we attempt creation.
- *
- * @param dirs List of Files to test.
- *
- * @return The passed dirs.
- *
- * @exception IOException If passed directory does not exist and is not
- * createable, or directory is not writeable or is not a directory.
- */
- public static Listdir.
- *
- * @exception IOException If passed directory does not exist and is not
- * createable, or directory is not writeable or is not a directory.
- */
- public static File ensureWriteableDirectory(File dir)
- throws IOException {
- if (!dir.exists()) {
- dir.mkdirs();
- } else {
- if (!dir.canWrite()) {
- throw new IOException("Dir " + dir.getAbsolutePath() +
- " not writeable.");
- } else if (!dir.isDirectory()) {
- throw new IOException("Dir " + dir.getAbsolutePath() +
- " is not a directory.");
- }
- }
-
- return dir;
- }
-
- /**
- * Read the entire stream to EOF, returning what's read as a String.
- *
- * @param inputStream
- * @return String of the whole inputStream's contents
- * @throws IOException
- * @deprecated use org.apache.commons.io.IOUtils.toString()
- */
- public static String readFullyAsString(InputStream inputStream)
- throws IOException {
- return IOUtils.toString(inputStream);
- }
-
-
- /**
- * @deprecated use org.apache.commons.io.IOUtils.toString()
- */
- public static String readFullyAsString(Reader r) throws IOException {
- return IOUtils.toString(r);
- }
-
- /**
- * Read the entire stream to EOF into the passed file.
- * @param is
- * @param toFile File to read into .
- * @throws IOException
- * @throws IOException
- */
- public static void readFullyToFile(InputStream is,
- File toFile) throws IOException {
- readFullyToFile(is, toFile, new byte[4096]);
- }
-
- /**
- * Read the entire stream to EOF into the passed file.
- * Closes is when done or if an exception.
- * @param is Stream to read.
- * @param toFile File to read into .
- * @param buffer Buffer to use reading.
- * @return Count of bytes read.
- * @throws IOException
- */
- public static long readFullyToFile(final InputStream is, final File toFile,
- final byte [] buffer)
- throws IOException {
- long totalcount = -1;
- OutputStream os =
- new FastBufferedOutputStream(new FileOutputStream(toFile));
- InputStream localIs = (is instanceof BufferedInputStream)?
- is: new BufferedInputStream(is);
- try {
- for (int count = -1;
- (count = localIs.read(buffer, 0, buffer.length)) != -1;
- totalcount += count) {
- os.write(buffer, 0, count);
- }
- } finally {
- os.close();
- if (localIs != null) {
- localIs.close();
- }
- }
- return totalcount;
- }
-
- /**
- * Wrap generic Throwable as a checked IOException
- * @param e wrapped exception
- * @return IOException
- */
- public static IOException wrapAsIOException(Throwable e) {
- IOException ioe = new IOException(e.toString());
- ioe.initCause(e);
- return ioe;
- }
-
-
- public static void readFully(InputStream input, byte[] buf)
- throws IOException {
- int max = buf.length;
- int ofs = 0;
- while (ofs < max) {
- int l = input.read(buf, ofs, max - ofs);
- if (l == 0) {
- throw new EOFException();
- }
- ofs += l;
- }
- }
-
-
- public static void close(Closeable c) {
- if (c == null) {
- return;
- }
- try {
- c.close();
- } catch (IOException e) {
-
- }
- }
-
- /**
- * Return the maximum number of bytes per character in the named
- * encoding, or 0 if encoding is invalid or unsupported.
- *
- * @param encoding Encoding to consider. For now, should be java
- * canonical name for the encoding.
- *
- * @return True if multibyte encoding.
- */
- public static float encodingMaxBytesPerChar(String encoding) {
- boolean isMultibyte = false;
- final Charset cs;
- try {
- if (encoding != null && encoding.length() > 0) {
- cs = Charset.forName(encoding);
- if(cs.canEncode()) {
- return cs.newEncoder().maxBytesPerChar();
- } else {
- logger.info("Encoding not fully supported: " + encoding
- + ". Defaulting to single byte.");
- }
- }
- } catch (IllegalArgumentException e) {
- // Unsupported encoding
- logger.log(Level.INFO,"Illegal encoding name: " + encoding,e);
- }
-
- logger.fine("Encoding " + encoding + " is multibyte: "
- + ((isMultibyte) ? Boolean.TRUE : Boolean.FALSE));
- // default: return 0
- return 0;
- }
-
- /**
- * Utility method to serialize an object to the given File.
- *
- * @param object Object to serialize
- * @param file File to receive serialized copy
- * @throws IOException
- */
- public static void serializeToFile(Object object, File file) throws IOException {
- ObjectOutputStream oos = new ObjectOutputStream(new BufferedOutputStream(new FileOutputStream(file)));
- oos.writeObject(object);
- oos.close();
- }
-
- /**
- * Utility method to deserialize an Object from given File.
- *
- * @param file File source
- * @return deserialized Object
- * @throws IOException
- */
- public static Object deserializeFromFile(File file) throws IOException {
- ObjectInputStream ois = new ObjectInputStream(new BufferedInputStream(new FileInputStream(file)));
- Object object;
- try {
- object = ois.readObject();
- } catch (ClassNotFoundException e) {
- // TODO Auto-generated catch block
- throw new RuntimeException(e);
- }
- ois.close();
- return object;
- }
-
- /**
- * Utility method to serialize Object to byte[].
- *
- * @param object Object to be serialized
- * @return byte[] serialized form
- * @deprecated use SerializationUtils.serialize
- */
- public static byte[] serializeToByteArray(Object object) {
- return SerializationUtils.serialize((Serializable)object);
- }
-
- /**
- * Utility method to deserialize Object from byte[].
- *
- * @param in byte[] source
- * @return Object deserialized
- * @deprecated use SerializationUtils.deserialize
- */
- public static Object deserializeFromByteArray(byte[] in) {
- return SerializationUtils.deserialize(in);
- }
-
- /** suffix to recognize gzipped files */
- public static final String GZIP_SUFFIX = ".gz";
- /**
- * Get a BufferedReader on the crawler journal given
- *
- * TODO: move to a general utils class
- *
- * @param source File journal
- * @return journal buffered reader.
- * @throws IOException
- */
- public static BufferedReader getBufferedReader(File source) throws IOException {
- InputStream is = new BufferedInputStream(new FileInputStream(source));
- boolean isGzipped = source.getName().toLowerCase().
- endsWith(GZIP_SUFFIX);
- if(isGzipped) {
- is = new GZIPInputStream(is);
- }
- return new BufferedReader(new InputStreamReader(is));
- }
-
- /**
- * Get a BufferedReader on the crawler journal given.
- *
- * @param source URL journal
- * @return journal buffered reader.
- * @throws IOException
- */
- public static BufferedReader getBufferedReader(URL source) throws IOException {
- URLConnection conn = source.openConnection();
- boolean isGzipped = conn.getContentType() != null && conn.getContentType().equalsIgnoreCase("application/x-gzip")
- || conn.getContentEncoding() != null && conn.getContentEncoding().equalsIgnoreCase("gzip");
- InputStream uis = conn.getInputStream();
- return new BufferedReader(isGzipped?
- new InputStreamReader(new GZIPInputStream(uis)):
- new InputStreamReader(uis));
-
- }
-}
diff --git a/commons/src/main/java/org/archive/util/Iteratorable.java b/commons/src/main/java/org/archive/util/Iteratorable.java
new file mode 100644
index 00000000..3bd77689
--- /dev/null
+++ b/commons/src/main/java/org/archive/util/Iteratorable.java
@@ -0,0 +1,40 @@
+/*
+ * This file is part of the Heritrix web crawler (crawler.archive.org).
+ *
+ * Licensed to the Internet Archive (IA) by one or more individual
+ * contributors.
+ *
+ * The IA licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.archive.util;
+
+import java.util.Iterator;
+
+/**
+ * Make an Iterator usable as an Iterable (and thus enable new-style
+ * for-each loops).
+ *
+ */
+public class Iteratorable