diff --git a/commons/src/main/java/org/archive/io/GenericReplayCharSequence.java b/commons/src/main/java/org/archive/io/GenericReplayCharSequence.java index c5219c3c..d61d729b 100644 --- a/commons/src/main/java/org/archive/io/GenericReplayCharSequence.java +++ b/commons/src/main/java/org/archive/io/GenericReplayCharSequence.java @@ -128,6 +128,8 @@ public class GenericReplayCharSequence implements ReplayCharSequence { */ private CharBuffer prefixBuffer = null; + private boolean isOpen = true; + /** * Constructor. * @@ -353,7 +355,15 @@ public class GenericReplayCharSequence implements ReplayCharSequence { } } + + @Override + public boolean isOpen() { + return this.isOpen; + } + public void close() throws IOException { + this.isOpen = false; + logger.fine("closing"); if (this.backingFileChannel != null && this.backingFileChannel.isOpen()) { diff --git a/commons/src/main/java/org/archive/io/InMemoryReplayCharSequence.java b/commons/src/main/java/org/archive/io/InMemoryReplayCharSequence.java index c3e3440e..41f80509 100644 --- a/commons/src/main/java/org/archive/io/InMemoryReplayCharSequence.java +++ b/commons/src/main/java/org/archive/io/InMemoryReplayCharSequence.java @@ -157,4 +157,9 @@ public class InMemoryReplayCharSequence implements ReplayCharSequence { public CharacterCodingException getCodingException() { return codingException; } + + @Override + public boolean isOpen() { + return this.charBuffer != null; + } } diff --git a/commons/src/main/java/org/archive/io/ReplayCharSequence.java b/commons/src/main/java/org/archive/io/ReplayCharSequence.java index bab1f58f..f5f15176 100644 --- a/commons/src/main/java/org/archive/io/ReplayCharSequence.java +++ b/commons/src/main/java/org/archive/io/ReplayCharSequence.java @@ -61,4 +61,9 @@ public interface ReplayCharSequence extends CharSequence, Closeable { * @return CharacterCodingException */ public CharacterCodingException getCodingException(); + + /** + * @return false if {@link #close()} has been called + */ + public boolean isOpen(); } diff --git a/commons/src/main/java/org/archive/util/Recorder.java b/commons/src/main/java/org/archive/util/Recorder.java index d10b1cb5..6247ba83 100644 --- a/commons/src/main/java/org/archive/util/Recorder.java +++ b/commons/src/main/java/org/archive/util/Recorder.java @@ -75,6 +75,8 @@ public class Recorder { */ private String characterEncoding = null; + private ReplayCharSequence replayCharSequence; + /** * Create an HttpRecorder. @@ -283,15 +285,20 @@ public class Recorder { } /** - * @return A ReplayCharSequence. Call close on the RCS when done w/ it. - * Will return indeterminate results if the underlying recording streams - * have not been closed first. - * @throws IOException + * @return A ReplayCharSequence. Caller may call + * {@link ReplayCharSequence#close()} when finished. However, in + * heritrix, the ReplayCharSequence is closed automatically when url + * processing has finished; in that context it's preferable not + * to close, so that processors can reuse the same instance. * @throws IOException + * @see {@link #endReplays()} */ public ReplayCharSequence getReplayCharSequence() throws IOException { - return getRecordedInput(). - getReplayCharSequence(this.characterEncoding); + if (replayCharSequence == null || !replayCharSequence.isOpen()) { + replayCharSequence = getRecordedInput().getReplayCharSequence(this.characterEncoding); + } + + return replayCharSequence; } /** @@ -335,4 +342,9 @@ public class Recorder { is.close(); return rec; } + + public void endReplays() { + ArchiveUtils.closeQuietly(replayCharSequence); + replayCharSequence = null; + } } diff --git a/engine/src/main/java/org/archive/crawler/framework/ToeThread.java b/engine/src/main/java/org/archive/crawler/framework/ToeThread.java index b079a200..52833bfe 100644 --- a/engine/src/main/java/org/archive/crawler/framework/ToeThread.java +++ b/engine/src/main/java/org/archive/crawler/framework/ToeThread.java @@ -175,6 +175,7 @@ implements MultiReporter, ProgressStatisticsReporter, // OutOfMemory and any others seriousError(err); } finally { + httpRecorder.endReplays(); KeyedProperties.clearOverridesFrom(curi); } diff --git a/engine/src/test/java/org/archive/crawler/selftest/KeyWordProcessor.java b/engine/src/test/java/org/archive/crawler/selftest/KeyWordProcessor.java index 63f99756..90b0fdfd 100644 --- a/engine/src/test/java/org/archive/crawler/selftest/KeyWordProcessor.java +++ b/engine/src/test/java/org/archive/crawler/selftest/KeyWordProcessor.java @@ -92,7 +92,6 @@ public class KeyWordProcessor extends Processor { if(!viaUri.getData().containsKey("keywordHit")) { ReplayCharSequence seq = viaUri.getRecorder().getReplayCharSequence(); viaUri.getData().put("keywordHit", getPattern().matcher(seq).find()); - seq.close(); } boolean keywordHit = (Boolean) viaUri.getData().get("keywordHit"); int precedence = keywordHit ? getFoundPrecedence() : getNotFoundPrecedence(); diff --git a/modules/src/main/java/org/archive/modules/extractor/ExtractorCSS.java b/modules/src/main/java/org/archive/modules/extractor/ExtractorCSS.java index 7364153a..59eed505 100644 --- a/modules/src/main/java/org/archive/modules/extractor/ExtractorCSS.java +++ b/modules/src/main/java/org/archive/modules/extractor/ExtractorCSS.java @@ -99,17 +99,14 @@ public class ExtractorCSS extends ContentExtractor { * @param curi Crawl URI to process. */ public boolean innerExtract(CrawlURI curi) { - ReplayCharSequence cs = null; try { - cs = curi.getRecorder().getReplayCharSequence(); + ReplayCharSequence cs = curi.getRecorder().getReplayCharSequence(); numberOfLinksExtracted.addAndGet( processStyleCode(this, curi, cs)); // Set flag to indicate that link extraction is completed. return true; } catch (IOException e) { logger.log(Level.WARNING, "Problem with ReplayCharSequence: " + e.getMessage(), e); - } finally { - ArchiveUtils.closeQuietly(cs); } return false; } diff --git a/modules/src/main/java/org/archive/modules/extractor/ExtractorHTML.java b/modules/src/main/java/org/archive/modules/extractor/ExtractorHTML.java index feb44f0d..1c2492d0 100644 --- a/modules/src/main/java/org/archive/modules/extractor/ExtractorHTML.java +++ b/modules/src/main/java/org/archive/modules/extractor/ExtractorHTML.java @@ -35,7 +35,6 @@ import org.archive.modules.CrawlURI; import org.archive.modules.net.RobotsPolicy; import org.archive.net.UURI; import org.archive.net.UURIFactory; -import org.archive.util.ArchiveUtils; import org.archive.util.DevUtils; import org.archive.util.TextUtils; import org.archive.util.UriUtils; @@ -672,8 +671,6 @@ public class ExtractorHTML extends ContentExtractor implements InitializingBean curi.getNonFatalFailures().add(e); logger.log(Level.SEVERE,"Failed get of replay char sequence in " + Thread.currentThread().getName(), e); - } finally { - ArchiveUtils.closeQuietly(cs); } return false; } diff --git a/modules/src/main/java/org/archive/modules/extractor/ExtractorXML.java b/modules/src/main/java/org/archive/modules/extractor/ExtractorXML.java index 2481c8d0..9894338f 100644 --- a/modules/src/main/java/org/archive/modules/extractor/ExtractorXML.java +++ b/modules/src/main/java/org/archive/modules/extractor/ExtractorXML.java @@ -27,7 +27,6 @@ import org.apache.commons.httpclient.URIException; import org.apache.commons.lang.StringEscapeUtils; import org.archive.io.ReplayCharSequence; import org.archive.modules.CrawlURI; -import org.archive.util.ArchiveUtils; /** * A simple extractor which finds HTTP URIs inside XML/RSS files, @@ -87,8 +86,6 @@ public class ExtractorXML extends ContentExtractor { return true; } catch (IOException e) { logger.severe("Failed getting ReplayCharSequence: " + e.getMessage()); - } finally { - ArchiveUtils.closeQuietly(cs); } return false; } diff --git a/modules/src/main/java/org/archive/modules/extractor/HTTPContentDigest.java b/modules/src/main/java/org/archive/modules/extractor/HTTPContentDigest.java index 8c2ac053..1b2166a9 100644 --- a/modules/src/main/java/org/archive/modules/extractor/HTTPContentDigest.java +++ b/modules/src/main/java/org/archive/modules/extractor/HTTPContentDigest.java @@ -27,7 +27,6 @@ import org.apache.commons.lang.StringUtils; import org.archive.io.ReplayCharSequence; import org.archive.modules.CrawlURI; import org.archive.modules.Processor; -import org.archive.util.ArchiveUtils; import org.archive.util.TextUtils; /** @@ -156,8 +155,6 @@ public class HTTPContentDigest extends Processor { curi.toString() + " " + e.getMessage() + " " + Thread.currentThread().getName()); return; // Can't proceed if this happens. - } finally { - ArchiveUtils.closeQuietly(cs); } } } \ No newline at end of file