From 00d1c46d60cff0abd504b215f876181da7514298 Mon Sep 17 00:00:00 2001 From: Adam Miller Date: Thu, 30 Jul 2020 23:19:43 +0000 Subject: [PATCH 1/2] Ensure Replay Input Stream and File Channels are closed after writing --- .../modules/writer/WARCWriterChainProcessor.java | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/modules/src/main/java/org/archive/modules/writer/WARCWriterChainProcessor.java b/modules/src/main/java/org/archive/modules/writer/WARCWriterChainProcessor.java index 4fc45558..b25d9ed7 100644 --- a/modules/src/main/java/org/archive/modules/writer/WARCWriterChainProcessor.java +++ b/modules/src/main/java/org/archive/modules/writer/WARCWriterChainProcessor.java @@ -1,5 +1,6 @@ package org.archive.modules.writer; +import java.io.InputStream; import java.io.IOException; import java.net.URI; import java.util.Arrays; @@ -7,6 +8,7 @@ import java.util.List; import java.util.logging.Level; import java.util.logging.Logger; +import org.apache.commons.io.IOUtils; import org.archive.io.warc.WARCRecordInfo; import org.archive.io.warc.WARCWriter; import org.archive.modules.CrawlURI; @@ -159,6 +161,17 @@ public class WARCWriterChainProcessor extends BaseWARCWriterProcessor implements WARCRecordInfo record = recordBuilder.buildRecord(curi, concurrentTo); if (record != null) { writer.writeRecord(record); + InputStream is = null; + try { + is = record.getContentStream(); + is.close(); + } + catch (Exception e){ + logger.log(Level.WARNING, "problem closing youtube-dl temp file " + e); + } + finally { + IOUtils.closeQuietly(record.getContentStream()); //but for real, close this time + } if (concurrentTo == null) { concurrentTo = record.getRecordId(); } From 52a8f345ed2686b0ac7de7acebaf4e2871a84b35 Mon Sep 17 00:00:00 2001 From: Adam Miller Date: Thu, 30 Jul 2020 23:30:08 +0000 Subject: [PATCH 2/2] Fixing up logging and comments --- .../org/archive/modules/writer/WARCWriterChainProcessor.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/modules/src/main/java/org/archive/modules/writer/WARCWriterChainProcessor.java b/modules/src/main/java/org/archive/modules/writer/WARCWriterChainProcessor.java index b25d9ed7..bab6dbc3 100644 --- a/modules/src/main/java/org/archive/modules/writer/WARCWriterChainProcessor.java +++ b/modules/src/main/java/org/archive/modules/writer/WARCWriterChainProcessor.java @@ -167,10 +167,10 @@ public class WARCWriterChainProcessor extends BaseWARCWriterProcessor implements is.close(); } catch (Exception e){ - logger.log(Level.WARNING, "problem closing youtube-dl temp file " + e); + logger.log(Level.WARNING, "problem closing Warc Record Content Stream " + e); } finally { - IOUtils.closeQuietly(record.getContentStream()); //but for real, close this time + IOUtils.closeQuietly(record.getContentStream()); //Closing one way or the other seems to leave some file handles open. Calling close() and using closeQuietly() handles both FileStreams and FileChannels } if (concurrentTo == null) { concurrentTo = record.getRecordId();