mirror of
https://github.com/internetarchive/heritrix3.git
synced 2026-09-25 15:16:24 +00:00
[HER-1904] H3: OOME after a few hours of previously non-problematic crawl
* Recorder
use min(raw-buffer size, actual recorded byte size) for char buffer size
* TextUtils
on recycleMatcher, dereference previous 'input'
This commit is contained in:
@@ -382,13 +382,28 @@ public class Recorder {
|
||||
InputStream ris = getContentReplayInputStream();
|
||||
ReplayCharSequence rcs = new GenericReplayCharSequence(
|
||||
ris,
|
||||
this.getRecordedInput().getRecordedBufferLength()/2,
|
||||
calcRecommendedCharBufferSize(this.getRecordedInput()),
|
||||
this.backingFileBasename + RECORDING_OUTPUT_STREAM_SUFFIX,
|
||||
requestedCharset);
|
||||
ris.close();
|
||||
return rcs;
|
||||
}
|
||||
|
||||
/**
|
||||
* Calculate a recommended size for an in-memory decoded-character buffer
|
||||
* of this content. We seek a size that is itself no larger (in 2-byte chars)
|
||||
* than the memory already used by the RecordingInputStream's internal raw
|
||||
* byte buffer, and also no larger than likely necessary. So, we take the
|
||||
* minimum of the actual recorded byte size and the RecordingInputStream's
|
||||
* max buffer size.
|
||||
*
|
||||
* @param inStream
|
||||
* @return int length for in-memory decoded-character buffer
|
||||
*/
|
||||
static protected int calcRecommendedCharBufferSize(RecordingInputStream inStream) {
|
||||
return Math.min(inStream.getRecordedBufferLength()/2,(int)inStream.getSize());
|
||||
}
|
||||
|
||||
/**
|
||||
* Get a raw replay of all recorded data (including, for example, HTTP
|
||||
* protocol headers)
|
||||
|
||||
@@ -78,6 +78,8 @@ public class TextUtils {
|
||||
}
|
||||
|
||||
public static void recycleMatcher(Matcher m) {
|
||||
// while cached, eliminate reference to potentially-large prior 'input'
|
||||
m.reset("");
|
||||
final Map<String,Matcher> matchers = TL_MATCHER_MAP.get();
|
||||
matchers.put(m.pattern().pattern(),m);
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user