/* * This file is part of the Heritrix web crawler (crawler.archive.org). * * Licensed to the Internet Archive (IA) by one or more individual * contributors. * * The IA licenses this file to You under the Apache License, Version 2.0 * (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.archive.util; import it.unimi.dsi.fastutil.io.FastBufferedOutputStream; import java.io.BufferedInputStream; import java.io.BufferedOutputStream; import java.io.BufferedReader; import java.io.Closeable; import java.io.EOFException; import java.io.File; import java.io.FileInputStream; import java.io.FileOutputStream; import java.io.IOException; import java.io.InputStream; import java.io.InputStreamReader; import java.io.ObjectInputStream; import java.io.ObjectOutputStream; import java.io.OutputStream; import java.io.Reader; import java.io.Serializable; import java.net.URL; import java.net.URLConnection; import java.nio.charset.Charset; import java.util.Iterator; import java.util.List; import java.util.logging.Level; import java.util.logging.Logger; import java.util.zip.GZIPInputStream; import org.apache.commons.io.IOUtils; import org.apache.commons.lang.SerializationUtils; /** * I/O Utility methods. * @author stack * @version $Date$, $Revision$ */ public class IoUtils { protected static Logger logger = Logger.getLogger(IoUtils.class.getName()); /** * @param file File to operate on. * @return Path suitable for use getting resources off the CLASSPATH * (CLASSPATH resources always use '/' as path separator, even on * windows). */ public static String getClasspathPath(File file) { String path = file.getPath(); if (File.separatorChar != '/') { // OK. We're probably on a windows system. Strip // drive if its present and convert '\' to '/'. path = path.replace(File.separatorChar, '/'); int index = path.indexOf(':'); if (index > 0 && index < 3) { path = path.substring(index + 1); } } return path; } /** * Ensure writeable directory. * * If doesn't exist, we attempt creation. * * @param dir Directory to test for exitence and is writeable. * * @return The passed dir. * * @exception IOException If passed directory does not exist and is not * createable, or directory is not writeable or is not a directory. */ public static File ensureWriteableDirectory(String dir) throws IOException { return ensureWriteableDirectory(new File(dir)); } /** * Ensure writeable directories. * * If doesn't exist, we attempt creation. * * @param dirs List of Files to test. * * @return The passed dirs. * * @exception IOException If passed directory does not exist and is not * createable, or directory is not writeable or is not a directory. */ public static List ensureWriteableDirectory(List dirs) throws IOException { for (Iterator i = dirs.iterator(); i.hasNext();) { ensureWriteableDirectory(i.next()); } return dirs; } /** * Ensure writeable directory. * * If doesn't exist, we attempt creation. * * @param dir Directory to test for exitence and is writeable. * * @return The passed dir. * * @exception IOException If passed directory does not exist and is not * createable, or directory is not writeable or is not a directory. */ public static File ensureWriteableDirectory(File dir) throws IOException { if (!dir.exists()) { dir.mkdirs(); } else { if (!dir.canWrite()) { throw new IOException("Dir " + dir.getAbsolutePath() + " not writeable."); } else if (!dir.isDirectory()) { throw new IOException("Dir " + dir.getAbsolutePath() + " is not a directory."); } } return dir; } /** * Read the entire stream to EOF, returning what's read as a String. * * @param inputStream * @return String of the whole inputStream's contents * @throws IOException * @deprecated use org.apache.commons.io.IOUtils.toString() */ public static String readFullyAsString(InputStream inputStream) throws IOException { return IOUtils.toString(inputStream); } /** * @deprecated use org.apache.commons.io.IOUtils.toString() */ public static String readFullyAsString(Reader r) throws IOException { return IOUtils.toString(r); } /** * Read the entire stream to EOF into the passed file. * @param is * @param toFile File to read into . * @throws IOException * @throws IOException */ public static void readFullyToFile(InputStream is, File toFile) throws IOException { readFullyToFile(is, toFile, new byte[4096]); } /** * Read the entire stream to EOF into the passed file. * Closes is when done or if an exception. * @param is Stream to read. * @param toFile File to read into . * @param buffer Buffer to use reading. * @return Count of bytes read. * @throws IOException */ public static long readFullyToFile(final InputStream is, final File toFile, final byte [] buffer) throws IOException { long totalcount = -1; OutputStream os = new FastBufferedOutputStream(new FileOutputStream(toFile)); InputStream localIs = (is instanceof BufferedInputStream)? is: new BufferedInputStream(is); try { for (int count = -1; (count = localIs.read(buffer, 0, buffer.length)) != -1; totalcount += count) { os.write(buffer, 0, count); } } finally { os.close(); if (localIs != null) { localIs.close(); } } return totalcount; } /** * Wrap generic Throwable as a checked IOException * @param e wrapped exception * @return IOException */ public static IOException wrapAsIOException(Throwable e) { IOException ioe = new IOException(e.toString()); ioe.initCause(e); return ioe; } public static void readFully(InputStream input, byte[] buf) throws IOException { int max = buf.length; int ofs = 0; while (ofs < max) { int l = input.read(buf, ofs, max - ofs); if (l == 0) { throw new EOFException(); } ofs += l; } } public static void close(Closeable c) { if (c == null) { return; } try { c.close(); } catch (IOException e) { } } /** * Return the maximum number of bytes per character in the named * encoding, or 0 if encoding is invalid or unsupported. * * @param encoding Encoding to consider. For now, should be java * canonical name for the encoding. * * @return True if multibyte encoding. */ public static float encodingMaxBytesPerChar(String encoding) { boolean isMultibyte = false; final Charset cs; try { if (encoding != null && encoding.length() > 0) { cs = Charset.forName(encoding); if(cs.canEncode()) { return cs.newEncoder().maxBytesPerChar(); } else { logger.info("Encoding not fully supported: " + encoding + ". Defaulting to single byte."); } } } catch (IllegalArgumentException e) { // Unsupported encoding logger.log(Level.INFO,"Illegal encoding name: " + encoding,e); } logger.fine("Encoding " + encoding + " is multibyte: " + ((isMultibyte) ? Boolean.TRUE : Boolean.FALSE)); // default: return 0 return 0; } /** * Utility method to serialize an object to the given File. * * @param object Object to serialize * @param file File to receive serialized copy * @throws IOException */ public static void serializeToFile(Object object, File file) throws IOException { ObjectOutputStream oos = new ObjectOutputStream(new BufferedOutputStream(new FileOutputStream(file))); oos.writeObject(object); oos.close(); } /** * Utility method to deserialize an Object from given File. * * @param file File source * @return deserialized Object * @throws IOException */ public static Object deserializeFromFile(File file) throws IOException { ObjectInputStream ois = new ObjectInputStream(new BufferedInputStream(new FileInputStream(file))); Object object; try { object = ois.readObject(); } catch (ClassNotFoundException e) { // TODO Auto-generated catch block throw new RuntimeException(e); } ois.close(); return object; } /** * Utility method to serialize Object to byte[]. * * @param object Object to be serialized * @return byte[] serialized form * @deprecated use SerializationUtils.serialize */ public static byte[] serializeToByteArray(Object object) { return SerializationUtils.serialize((Serializable)object); } /** * Utility method to deserialize Object from byte[]. * * @param in byte[] source * @return Object deserialized * @deprecated use SerializationUtils.deserialize */ public static Object deserializeFromByteArray(byte[] in) { return SerializationUtils.deserialize(in); } /** suffix to recognize gzipped files */ public static final String GZIP_SUFFIX = ".gz"; /** * Get a BufferedReader on the crawler journal given * * TODO: move to a general utils class * * @param source File journal * @return journal buffered reader. * @throws IOException */ public static BufferedReader getBufferedReader(File source) throws IOException { InputStream is = new BufferedInputStream(new FileInputStream(source)); boolean isGzipped = source.getName().toLowerCase(). endsWith(GZIP_SUFFIX); if(isGzipped) { is = new GZIPInputStream(is); } return new BufferedReader(new InputStreamReader(is)); } /** * Get a BufferedReader on the crawler journal given. * * @param source URL journal * @return journal buffered reader. * @throws IOException */ public static BufferedReader getBufferedReader(URL source) throws IOException { URLConnection conn = source.openConnection(); boolean isGzipped = conn.getContentType() != null && conn.getContentType().equalsIgnoreCase("application/x-gzip") || conn.getContentEncoding() != null && conn.getContentEncoding().equalsIgnoreCase("gzip"); InputStream uis = conn.getInputStream(); return new BufferedReader(isGzipped? new InputStreamReader(new GZIPInputStream(uis)): new InputStreamReader(uis)); } }