From 9d2b293d4c6b64bf0145be8eb317b6cfab47dc1d Mon Sep 17 00:00:00 2001 From: gojomo Date: Mon, 20 Jun 2011 20:32:14 +0000 Subject: [PATCH] [HER-1904] H3: OOME after a few hours of previously non-problematic crawl * Recorder use min(raw-buffer size, actual recorded byte size) for char buffer size * TextUtils on recycleMatcher, dereference previous 'input' --- .../main/java/org/archive/util/Recorder.java | 17 ++++++++++++++++- .../main/java/org/archive/util/TextUtils.java | 2 ++ 2 files changed, 18 insertions(+), 1 deletion(-) diff --git a/commons/src/main/java/org/archive/util/Recorder.java b/commons/src/main/java/org/archive/util/Recorder.java index 96f197f9..4875a698 100644 --- a/commons/src/main/java/org/archive/util/Recorder.java +++ b/commons/src/main/java/org/archive/util/Recorder.java @@ -382,13 +382,28 @@ public class Recorder { InputStream ris = getContentReplayInputStream(); ReplayCharSequence rcs = new GenericReplayCharSequence( ris, - this.getRecordedInput().getRecordedBufferLength()/2, + calcRecommendedCharBufferSize(this.getRecordedInput()), this.backingFileBasename + RECORDING_OUTPUT_STREAM_SUFFIX, requestedCharset); ris.close(); return rcs; } + /** + * Calculate a recommended size for an in-memory decoded-character buffer + * of this content. We seek a size that is itself no larger (in 2-byte chars) + * than the memory already used by the RecordingInputStream's internal raw + * byte buffer, and also no larger than likely necessary. So, we take the + * minimum of the actual recorded byte size and the RecordingInputStream's + * max buffer size. + * + * @param inStream + * @return int length for in-memory decoded-character buffer + */ + static protected int calcRecommendedCharBufferSize(RecordingInputStream inStream) { + return Math.min(inStream.getRecordedBufferLength()/2,(int)inStream.getSize()); + } + /** * Get a raw replay of all recorded data (including, for example, HTTP * protocol headers) diff --git a/commons/src/main/java/org/archive/util/TextUtils.java b/commons/src/main/java/org/archive/util/TextUtils.java index dc17b1c4..883e8cde 100644 --- a/commons/src/main/java/org/archive/util/TextUtils.java +++ b/commons/src/main/java/org/archive/util/TextUtils.java @@ -78,6 +78,8 @@ public class TextUtils { } public static void recycleMatcher(Matcher m) { + // while cached, eliminate reference to potentially-large prior 'input' + m.reset(""); final Map matchers = TL_MATCHER_MAP.get(); matchers.put(m.pattern().pattern(),m); }