diff --git a/commons/src/main/java/org/archive/io/ReplayCharSequence.java b/commons/src/main/java/org/archive/io/ReplayCharSequence.java index f9d8c83f..38d975eb 100644 --- a/commons/src/main/java/org/archive/io/ReplayCharSequence.java +++ b/commons/src/main/java/org/archive/io/ReplayCharSequence.java @@ -22,6 +22,7 @@ */ package org.archive.io; +import java.io.Closeable; import java.io.IOException; @@ -34,7 +35,7 @@ import java.io.IOException; * @author stack * @version $Revision$, $Date$ */ -public interface ReplayCharSequence extends CharSequence { +public interface ReplayCharSequence extends CharSequence, Closeable { /** * Call this method when done so implementation has chance to clean up diff --git a/commons/src/main/java/org/archive/util/ArchiveUtils.java b/commons/src/main/java/org/archive/util/ArchiveUtils.java index 7a3f949c..eb7f1c4e 100644 --- a/commons/src/main/java/org/archive/util/ArchiveUtils.java +++ b/commons/src/main/java/org/archive/util/ArchiveUtils.java @@ -20,6 +20,7 @@ package org.archive.util; import java.io.BufferedReader; +import java.io.Closeable; import java.io.IOException; import java.io.InputStream; import java.io.InputStreamReader; @@ -895,5 +896,15 @@ public class ArchiveUtils { public static boolean isTld(String dom) { return TLDS.contains(dom.toUpperCase()); } + + public static void closeQuietly(Closeable input) { + try { + if (input != null) { + input.close(); + } + } catch (IOException ioe) { + // ignore + } + } } diff --git a/modules/src/main/java/org/archive/modules/extractor/ExtractorCSS.java b/modules/src/main/java/org/archive/modules/extractor/ExtractorCSS.java index 70921d42..5c750dab 100644 --- a/modules/src/main/java/org/archive/modules/extractor/ExtractorCSS.java +++ b/modules/src/main/java/org/archive/modules/extractor/ExtractorCSS.java @@ -1,31 +1,25 @@ /* - * ExtractorCSS + * This file is part of the Heritrix web crawler (crawler.archive.org). * - * $Id$ + * Licensed to the Internet Archive (IA) by one or more individual + * contributors. * - * Created on Jan 6, 2004 + * The IA licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at * - * Copyright (C) 2004 Internet Archive. + * http://www.apache.org/licenses/LICENSE-2.0 * - * This file is part of the Heritrix web crawler (crawler.archive.org). - * - * Heritrix is free software; you can redistribute it and/or modify - * it under the terms of the GNU Lesser Public License as published by - * the Free Software Foundation; either version 2.1 of the License, or - * any later version. - * - * Heritrix is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Lesser Public License for more details. - * - * You should have received a copy of the GNU Lesser Public License - * along with Heritrix; if not, write to the Free Software - * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. */ package org.archive.modules.extractor; import java.io.IOException; +import java.util.logging.Level; import java.util.logging.Logger; import java.util.regex.Matcher; @@ -33,6 +27,7 @@ import org.apache.commons.httpclient.URIException; import org.archive.io.ReplayCharSequence; import org.archive.modules.ProcessorURI; import org.archive.net.UURI; +import org.archive.util.ArchiveUtils; import org.archive.util.DevUtils; import org.archive.util.TextUtils; @@ -108,32 +103,16 @@ public class ExtractorCSS extends ContentExtractor { ReplayCharSequence cs = null; try { cs = curi.getRecorder().getReplayCharSequence(); - } catch (IOException e) { - logger.severe("Failed getting ReplayCharSequence: " + e.getMessage()); - } - if (cs == null) { - logger.warning("Failed getting ReplayCharSequence: " + - curi.toString()); - return false; - } - - // We have a ReplayCharSequence open. Wrap all in finally so we - // for sure close it before we leave. - try { this.numberOfLinksExtracted += processStyleCode(this, curi, cs); // Set flag to indicate that link extraction is completed. return true; + } catch (IOException e) { + logger.log(Level.WARNING, "Problem with ReplayCharSequence: " + e.getMessage(), e); } finally { - if (cs != null) { - try { - cs.close(); - } catch (IOException ioe) { - logger.warning(TextUtils.exceptionToString( - "Failed close of ReplayCharSequence.", ioe)); - } - } + ArchiveUtils.closeQuietly(cs); } + return false; } public static long processStyleCode(Extractor ext, diff --git a/modules/src/main/java/org/archive/modules/extractor/ExtractorDOC.java b/modules/src/main/java/org/archive/modules/extractor/ExtractorDOC.java index 374f10a1..8591fb40 100644 --- a/modules/src/main/java/org/archive/modules/extractor/ExtractorDOC.java +++ b/modules/src/main/java/org/archive/modules/extractor/ExtractorDOC.java @@ -27,6 +27,7 @@ import java.util.regex.Matcher; import java.util.regex.Pattern; import org.apache.commons.httpclient.URIException; +import org.apache.commons.io.IOUtils; import org.archive.io.ReplayInputStream; import org.archive.io.SeekReader; import org.archive.io.SeekReaderCharSequence; @@ -96,11 +97,7 @@ public class ExtractorDOC extends ContentExtractor { curi.getNonFatalFailures().add(e); return false; } finally { - try { - documentStream.close(); - } catch (IOException ignored) { - - } + IOUtils.closeQuietly(documentStream); } CharSequence cs = new SeekReaderCharSequence(docReader, 0); diff --git a/modules/src/main/java/org/archive/modules/extractor/ExtractorHTML.java b/modules/src/main/java/org/archive/modules/extractor/ExtractorHTML.java index fbdb045c..2ef2e9b9 100644 --- a/modules/src/main/java/org/archive/modules/extractor/ExtractorHTML.java +++ b/modules/src/main/java/org/archive/modules/extractor/ExtractorHTML.java @@ -34,6 +34,7 @@ import org.archive.modules.net.RobotsHonoringPolicy; import org.archive.net.UURI; import org.archive.net.UURIFactory; import org.springframework.beans.factory.InitializingBean; +import org.archive.util.ArchiveUtils; import org.archive.util.DevUtils; import org.archive.util.TextUtils; import org.springframework.beans.factory.annotation.Autowired; @@ -600,41 +601,21 @@ public class ExtractorHTML extends ContentExtractor implements InitializingBean public boolean innerExtract(ProcessorURI curi) { this.numberOfCURIsHandled++; - ReplayCharSequence cs = null; - try { cs = curi.getRecorder().getReplayCharSequence(); + // Extract all links from the charsequence + extract(curi, cs); + // Set flag to indicate that link extraction is completed. + return true; } catch (IOException e) { curi.getNonFatalFailures().add(e); - //addLocalizedError(e, - // "Failed get of replay char sequence " + curi.toString() + - // " " + e.getMessage()); logger.log(Level.SEVERE,"Failed get of replay char sequence in " + Thread.currentThread().getName(), e); - } - - if (cs == null) { - return false; - } - - // We have a ReplayCharSequence open. Wrap all in finally so we - // for sure close it before we leave. - try { - // Extract all links from the charsequence - extract(curi, cs); - // Set flag to indicate that link extraction is completed. - return true; } finally { - if (cs != null) { - try { - cs.close(); - } catch (IOException ioe) { - logger.warning(TextUtils.exceptionToString( - "Failed close of ReplayCharSequence.", ioe)); - } - } + ArchiveUtils.closeQuietly(cs); } + return false; } /** diff --git a/modules/src/main/java/org/archive/modules/extractor/ExtractorJS.java b/modules/src/main/java/org/archive/modules/extractor/ExtractorJS.java index 193a67d1..abd670b3 100644 --- a/modules/src/main/java/org/archive/modules/extractor/ExtractorJS.java +++ b/modules/src/main/java/org/archive/modules/extractor/ExtractorJS.java @@ -139,21 +139,9 @@ public class ExtractorJS extends ContentExtractor { @Override protected boolean innerExtract(ProcessorURI curi) { this.numberOfCURIsHandled++; - ReplayCharSequence cs = null; try { cs = curi.getRecorder().getReplayCharSequence(); - } catch (IOException e) { - curi.getNonFatalFailures().add(e); - } - - if (cs == null) { - LOGGER.warning("Failed getting ReplayCharSequence: " + - curi.toString()); - return false; - } - - try { try { numberOfLinksExtracted += considerStrings(this, curi, cs, true); @@ -162,17 +150,12 @@ public class ExtractorJS extends ContentExtractor { } // Set flag to indicate that link extraction is completed. return true; + } catch (IOException e) { + curi.getNonFatalFailures().add(e); } finally { - // Done w/ the ReplayCharSequence. Close it. - if (cs != null) { - try { - cs.close(); - } catch (IOException ioe) { - LOGGER.warning(TextUtils.exceptionToString( - "Failed close of ReplayCharSequence.", ioe)); - } - } + ArchiveUtils.closeQuietly(cs); } + return false; } public static long considerStrings(Extractor ext, diff --git a/modules/src/main/java/org/archive/modules/extractor/ExtractorXML.java b/modules/src/main/java/org/archive/modules/extractor/ExtractorXML.java index 6d9df612..c32e041a 100644 --- a/modules/src/main/java/org/archive/modules/extractor/ExtractorXML.java +++ b/modules/src/main/java/org/archive/modules/extractor/ExtractorXML.java @@ -35,6 +35,7 @@ import java.util.regex.Pattern; import org.apache.commons.httpclient.URIException; import org.archive.io.ReplayCharSequence; import org.archive.modules.ProcessorURI; +import org.archive.util.ArchiveUtils; import org.archive.util.TextUtils; /** @@ -93,29 +94,15 @@ public class ExtractorXML extends ContentExtractor { ReplayCharSequence cs = null; try { cs = curi.getRecorder().getReplayCharSequence(); - } catch (IOException e) { - logger.severe("Failed getting ReplayCharSequence: " + e.getMessage()); - } - if (cs == null) { - logger.severe("Failed getting ReplayCharSequence: " + - curi.toString()); - return false; - } - try { this.linksExtracted.addAndGet(processXml(this, curi, cs)); - // Set flag to indicate that link extraction is completed. return true; + } catch (IOException e) { + logger.severe("Failed getting ReplayCharSequence: " + e.getMessage()); } finally { - if (cs != null) { - try { - cs.close(); - } catch (IOException ioe) { - logger.warning(TextUtils.exceptionToString( - "Failed close of ReplayCharSequence.", ioe)); - } - } + ArchiveUtils.closeQuietly(cs); } + return false; } public static long processXml(Extractor ext, diff --git a/modules/src/main/java/org/archive/modules/extractor/HTTPContentDigest.java b/modules/src/main/java/org/archive/modules/extractor/HTTPContentDigest.java index 7d5b56cc..2cf6fded 100644 --- a/modules/src/main/java/org/archive/modules/extractor/HTTPContentDigest.java +++ b/modules/src/main/java/org/archive/modules/extractor/HTTPContentDigest.java @@ -1,41 +1,33 @@ -/* HTTPContentDigest - * - * $Id$ - * - * Created on 5.1.2005 +/* + * This file is part of the Heritrix web crawler (crawler.archive.org). * - * Copyright (C) 2005 Kristinn Sigur?sson - * - * This file is part of the Heritrix web crawler (crawler.archive.org). - * - * Heritrix is free software; you can redistribute it and/or modify - * it under the terms of the GNU Lesser Public License as published by - * the Free Software Foundation; either version 2.1 of the License, or - * any later version. - * - * Heritrix is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Lesser Public License for more details. - * - * You should have received a copy of the GNU Lesser Public License - * along with Heritrix; if not, write to the Free Software - * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + * Licensed to the Internet Archive (IA) by one or more individual + * contributors. + * + * The IA licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. */ package org.archive.modules.extractor; -import java.io.IOException; import java.security.MessageDigest; import java.security.NoSuchAlgorithmException; import java.util.logging.Logger; import java.util.regex.Matcher; import java.util.regex.Pattern; - import org.archive.io.ReplayCharSequence; import org.archive.modules.Processor; import org.archive.modules.ProcessorURI; -import org.archive.util.TextUtils; +import org.archive.util.ArchiveUtils; /** * A processor for calculating custum HTTP content digests in place of the @@ -128,64 +120,42 @@ public class HTTPContentDigest extends Processor { // Get a replay of the document character seq. ReplayCharSequence cs = null; - try { cs = curi.getRecorder().getReplayCharSequence(); + // Create a MessageDigest + MessageDigest digest = null; + try { + digest = MessageDigest.getInstance(SHA1); + } catch (NoSuchAlgorithmException e1) { + e1.printStackTrace(); + return; + } + + digest.reset(); + + String s = null; + + if (regexpr != null) { + s = cs.toString(); + } else { + // Process the document + Matcher m = regexpr.matcher(cs); + s = m.replaceAll(" "); + } + digest.update(s.getBytes()); + // Get the new digest value + byte[] newDigestValue = digest.digest(); + // Save new digest value + curi.setContentDigest(SHA1, newDigestValue); + } catch (Exception e) { curi.getNonFatalFailures().add(e); logger.warning("Failed get of replay char sequence " + curi.toString() + " " + e.getMessage() + " " + Thread.currentThread().getName()); return; // Can't proceed if this happens. - } - - // Create a MessageDigest - MessageDigest digest = null; - - // We have a ReplayCharSequence open. Wrap all in finally so we - // for sure close it before we leave. - try { - try { - digest = MessageDigest.getInstance(SHA1); - } catch (NoSuchAlgorithmException e1) { - e1.printStackTrace(); - return; - } - - digest.reset(); - - String s = null; - - if (regexpr != null) { - s = cs.toString(); - } else { - // Process the document - Matcher m = regexpr.matcher(cs); - s = m.replaceAll(" "); - } - digest.update(s.getBytes()); - - // Get the new digest value - byte[] newDigestValue = digest.digest(); - - // Log if needed. -// if (logger.isLoggable(Level.FINEST)) { -// logger.finest("Recalculated content digest for " -// + curi.toString() + " old: " -// + Base32.encode((byte[]) curi.getContentDigest()) -// + ", new: " + Base32.encode(newDigestValue)); -// } - // Save new digest value - curi.setContentDigest(SHA1, newDigestValue); } finally { - if (cs != null) { - try { - cs.close(); - } catch (IOException ioe) { - logger.warning(TextUtils.exceptionToString( - "Failed close of ReplayCharSequence.", ioe)); - } - } + ArchiveUtils.closeQuietly(cs); } } } \ No newline at end of file diff --git a/modules/src/main/java/org/archive/modules/net/CrawlServer.java b/modules/src/main/java/org/archive/modules/net/CrawlServer.java index b706833f..c543aab6 100644 --- a/modules/src/main/java/org/archive/modules/net/CrawlServer.java +++ b/modules/src/main/java/org/archive/modules/net/CrawlServer.java @@ -33,6 +33,7 @@ import java.util.Set; import java.util.zip.Checksum; import org.apache.commons.httpclient.URIException; +import org.apache.commons.io.IOUtils; import static org.archive.modules.ProcessorURI.FetchType.*; @@ -205,9 +206,7 @@ public class CrawlServer implements Serializable, FetchStats.HasFetchStats { reader, honoringPolicy); validRobots = true; } finally { - if (contentBodyStream != null) { - contentBodyStream.close(); - } + IOUtils.closeQuietly(contentBodyStream); } } catch (IOException e) { robots = RobotsExclusionPolicy.ALLOWALL; diff --git a/modules/src/main/java/org/archive/modules/writer/MirrorWriterProcessor.java b/modules/src/main/java/org/archive/modules/writer/MirrorWriterProcessor.java index 2d65f1f5..420504a9 100644 --- a/modules/src/main/java/org/archive/modules/writer/MirrorWriterProcessor.java +++ b/modules/src/main/java/org/archive/modules/writer/MirrorWriterProcessor.java @@ -44,6 +44,7 @@ import java.util.regex.Pattern; import javax.management.AttributeNotFoundException; +import org.apache.commons.io.IOUtils; import org.archive.io.RecordingInputStream; import org.archive.io.ReplayInputStream; import org.archive.modules.Processor; @@ -725,14 +726,17 @@ public class MirrorWriterProcessor extends Processor { */ private void writeToPath(RecordingInputStream recis, File dest) throws IOException { - ReplayInputStream replayis = recis.getContentReplayInputStream(); File tf = new File (dest.getPath() + "N"); - FileOutputStream fos = new FileOutputStream(tf); + ReplayInputStream replayis = null; + FileOutputStream fos = null; try { + replayis = recis.getContentReplayInputStream(); + fos = new FileOutputStream(tf); + replayis.readFullyTo(fos); } finally { - fos.close(); - replayis.close(); + IOUtils.closeQuietly(replayis); + IOUtils.closeQuietly(fos); } if (!tf.renameTo(dest)) { throw new IOException("Can not rename " + tf.getAbsolutePath() diff --git a/modules/src/main/java/org/archive/modules/writer/WARCWriterProcessor.java b/modules/src/main/java/org/archive/modules/writer/WARCWriterProcessor.java index 71ab27da..c002052d 100644 --- a/modules/src/main/java/org/archive/modules/writer/WARCWriterProcessor.java +++ b/modules/src/main/java/org/archive/modules/writer/WARCWriterProcessor.java @@ -64,6 +64,7 @@ import java.util.logging.Logger; import org.apache.commons.httpclient.Header; import org.apache.commons.httpclient.HttpMethodBase; import org.apache.commons.httpclient.HttpStatus; +import org.apache.commons.io.IOUtils; import org.apache.commons.lang.StringUtils; import org.archive.io.ReplayInputStream; import org.archive.io.WriterPoolMember; @@ -323,16 +324,14 @@ public class WARCWriterProcessor extends WriterPoolProcessor { final ANVLRecord namedFields) throws IOException { final URI uid = qualifyRecordID(baseid, TYPE, REQUEST); - ReplayInputStream ris = - curi.getRecorder().getRecordedOutput().getReplayInputStream(); + ReplayInputStream + ris = curi.getRecorder().getRecordedOutput().getReplayInputStream(); try { w.writeRequestRecord(curi.toString(), timestamp, mimetype, uid, namedFields, ris, curi.getRecorder().getRecordedOutput().getSize()); } finally { - if (ris != null) { - ris.close(); - } + IOUtils.closeQuietly(ris); } return uid; } @@ -349,9 +348,7 @@ public class WARCWriterProcessor extends WriterPoolProcessor { namedFields, ris, curi.getRecorder().getRecordedInput().getSize()); } finally { - if (ris != null) { - ris.close(); - } + IOUtils.closeQuietly(ris); } return baseid; } @@ -361,16 +358,13 @@ public class WARCWriterProcessor extends WriterPoolProcessor { final URI baseid, final ProcessorURI curi, final ANVLRecord namedFields) throws IOException { - ReplayInputStream ris = - curi.getRecorder().getRecordedInput().getReplayInputStream(); + ReplayInputStream ris = curi.getRecorder().getRecordedInput().getReplayInputStream(); try { w.writeResourceRecord(curi.toString(), timestamp, mimetype, baseid, namedFields, ris, curi.getRecorder().getRecordedInput().getSize()); } finally { - if (ris != null) { - ris.close(); - } + IOUtils.closeQuietly(ris); } return baseid; } @@ -395,9 +389,7 @@ public class WARCWriterProcessor extends WriterPoolProcessor { w.writeRevisitRecord(curi.toString(), timestamp, mimetype, baseid, namedFields, ris, revisedLength); } finally { - if (ris != null) { - ris.close(); - } + IOUtils.closeQuietly(ris); } curi.getAnnotations().add("warcRevisit:digest"); return baseid; @@ -427,9 +419,7 @@ public class WARCWriterProcessor extends WriterPoolProcessor { w.writeRevisitRecord(curi.toString(), timestamp, null, baseid, namedFields, ris, 0); } finally { - if (ris != null) { - ris.close(); - } + IOUtils.closeQuietly(ris); } curi.getAnnotations().add("warcRevisit:notModified"); return baseid;