diff --git a/contrib/src/main/java/org/archive/modules/extractor/ExtractorYoutubeDL.java b/contrib/src/main/java/org/archive/modules/extractor/ExtractorYoutubeDL.java index 92217d45..a1f2df97 100644 --- a/contrib/src/main/java/org/archive/modules/extractor/ExtractorYoutubeDL.java +++ b/contrib/src/main/java/org/archive/modules/extractor/ExtractorYoutubeDL.java @@ -23,7 +23,6 @@ import java.io.IOException; import java.io.InputStreamReader; import java.io.Reader; import java.util.ArrayList; -import java.util.Collections; import java.util.List; import java.util.concurrent.Callable; import java.util.concurrent.ExecutionException; @@ -116,31 +115,6 @@ public class ExtractorYoutubeDL extends Extractor implements Lifecycle { } } - // see https://github.com/internetarchive/heritrix3/pull/257/files#r279990349 - protected String readToEndInThread(Reader reader) throws IOException { - ExecutorService threadPool = Executors.newSingleThreadExecutor(); - Future future = threadPool.submit(new Callable() { - @Override - public String call() throws IOException { - return readToEnd(reader); - } - }); - - try { - return future.get(); - } catch (InterruptedException e) { - throw new IOException(e); // :shrug: - } catch (ExecutionException e) { - if (e.getCause() instanceof IOException) { - throw (IOException) e.getCause(); - } else { - throw new IOException(e); - } - } finally { - threadPool.shutdown(); - } - } - /** * - If {@code uri} is annotated "youtube-dl" and is a 3xx (redirect), * find the redirect among the outlinks and add the "youtube-dl" @@ -276,13 +250,52 @@ public class ExtractorYoutubeDL extends Extractor implements Lifecycle { } } + static protected class ProcessOutput { + public String stdout; + public String stderr; + } + + // read stdout in this thread, stderr in separate thread + // see https://github.com/internetarchive/heritrix3/pull/257/files#r279990349 + protected ProcessOutput readOutput(Process proc) throws IOException { + ProcessOutput output = new ProcessOutput(); + + Reader err = new InputStreamReader(proc.getErrorStream(), "UTF-8"); + InputStreamReader out = new InputStreamReader(proc.getInputStream(), "UTF-8"); + ExecutorService threadPool = Executors.newSingleThreadExecutor(); + + Future future = threadPool.submit(new Callable() { + @Override + public String call() throws IOException { + return readToEnd(err); + } + }); + + output.stdout = readToEnd(out); + + try { + output.stderr = future.get(); + } catch (InterruptedException e) { + throw new IOException(e); // :shrug: + } catch (ExecutionException e) { + if (e.getCause() instanceof IOException) { + throw (IOException) e.getCause(); + } else { + throw new IOException(e); + } + } finally { + threadPool.shutdown(); + } + + return output; + } + /** * * @param uri * @return list of json blobs returned by {@code youtube-dl --dump-json}, or * empty list if no videos found, or failure */ - @SuppressWarnings("unchecked") protected List runYoutubeDL(CrawlURI uri) { /* * --format=best @@ -300,21 +313,17 @@ public class ExtractorYoutubeDL extends Extractor implements Lifecycle { proc = pb.start(); } catch (IOException e) { logger.log(Level.WARNING, "youtube-dl failed " + pb.command(), e); - return (List) Collections.EMPTY_LIST; + return null; } - String stdout = null; - String stderr = null; + ProcessOutput output; try { - stderr = readToEndInThread( - new InputStreamReader(proc.getErrorStream(), "UTF-8")); - stdout = readToEnd( - new InputStreamReader(proc.getInputStream(), "UTF-8")); + output = readOutput(proc); } catch (IOException e) { logger.log(Level.WARNING, "problem reading output from youtube-dl " + pb.command(), e); - return (List) Collections.EMPTY_LIST; + return null; } try { @@ -327,16 +336,16 @@ public class ExtractorYoutubeDL extends Extractor implements Lifecycle { */ logger.fine("youtube-dl exited with status " + proc.waitFor() + " " + pb.command() - + "\n=== stdout ===\n" + stdout - + "\n=== stderr ===\n" + stderr); - return (List) Collections.EMPTY_LIST; + + "\n=== stdout ===\n" + output.stdout + + "\n=== stderr ===\n" + output.stderr); + return null; } } catch (InterruptedException e) { proc.destroyForcibly(); } - ArrayList ydlJsons = new ArrayList(); - JsonStreamParser parser = new JsonStreamParser(stdout); + List ydlJsons = new ArrayList(); + JsonStreamParser parser = new JsonStreamParser(output.stdout); try { while (parser.hasNext()) { ydlJsons.add((JsonObject) parser.next()); @@ -346,10 +355,10 @@ public class ExtractorYoutubeDL extends Extractor implements Lifecycle { // manifests as a JsonIOException logger.log(Level.FINE, "problem parsing json from youtube-dl " + pb.command() - + "\n=== stdout ===\n" + stdout - + "\n=== stderr ===\n" + stderr, + + "\n=== stdout ===\n" + output.stdout + + "\n=== stderr ===\n" + output.stderr, e); - return (List) Collections.EMPTY_LIST; + return null; } return ydlJsons;