- * Also implements {@link WARCRecordBuilder} to write youtube-dl json to the + * Also implements {@link WARCRecordBuilder} to write yt-dlp json to the * warc. * *
@@ -81,7 +82,7 @@ import com.google.gson.stream.JsonToken; * fetch chain, and to the end of the warc writer chain. * *
- * Keeps a log of containing pages and media captured as a result of youtube-dl + * Keeps a log of containing pages and media captured as a result of yt-dlp * extraction. The format of the log is as follows: * *
@@ -140,7 +141,7 @@ public class ExtractorYoutubeDL extends Extractor
localTemp = openNewTempFile();
tempfile.set(localTemp);
}
- logger.info("Getting youtube-dl temp file ");
+ logger.info("Getting yt-dlp temp file ");
return localTemp;
}
protected boolean isOpen(RandomAccessFile f) {
@@ -149,12 +150,12 @@ public class ExtractorYoutubeDL extends Extractor
return true;
}
catch (IOException e) {
- logger.info("youtube-dl temp file is not open");
+ logger.info("yt-dlp temp file is not open");
return false ;
}
}
protected RandomAccessFile openNewTempFile() {
- logger.info("Opening New youtube-dl temp file ");
+ logger.info("Opening New yt-dlp temp file ");
File t;
try {
t = File.createTempFile("ydl", ".json");
@@ -194,6 +195,18 @@ public class ExtractorYoutubeDL extends Extractor
kp.put("logMetadataRecord",logMetadataRecord);
}
+ {
+ setProcessArguments(Arrays.asList("yt-dlp", "--ignore-config",
+ "--simulate", "--dump-single-json", "-S vcodec:h264,res:720,acodec:aac",
+ "--no-cache-dir", "--no-playlist", "--playlist-end=" + MAX_VIDEOS_PER_PAGE ));
+ }
+ public List getProcessArguments() {
+ return (List) kp.get("processArguments");
+ }
+ public void setProcessArguments(List processArguments) {
+ kp.put("processArguments", processArguments);
+ }
+
@Override
public void start() {
if (!isRunning) {
@@ -229,7 +242,7 @@ public class ExtractorYoutubeDL extends Extractor
* - If {@code uri} is annotated "youtube-dl" and is an actual video
* download, log a line to ExtractorYoutubeDL.log
*
- * - If {@link #shouldExtract(CrawlURI)}, do youtube-dl extraction.
+ * - If {@link #shouldExtract(CrawlURI)}, do yt-dlp extraction.
*/
@Override
protected void extract(CrawlURI uri) {
@@ -400,7 +413,7 @@ public class ExtractorYoutubeDL extends Extractor
}
/**
- * Streams through youtube-dl json output. Sticks video urls in
+ * Streams through yt-dlp json output. Sticks video urls in
* results.videoUrls, web page urls in
* results.pageUrls, and saves the json in anonymous temp file
* results.jsonFile.
@@ -468,17 +481,16 @@ public class ExtractorYoutubeDL extends Extractor
* the best audio with best acodec no better than aac and
* with the smallest dimension no larger than 720.
*/
- ProcessBuilder pb = new ProcessBuilder("yt-dlp", "--ignore-config",
- "--simulate", "--dump-single-json", "-S vcodec:h264,res:720,acodec:aac",
- "--no-cache-dir", "--no-playlist",
- "--playlist-end=" + MAX_VIDEOS_PER_PAGE, uri.toString());
+ ArrayList processArguments = new ArrayList(getProcessArguments());
+ processArguments.add(uri.toString());
+ ProcessBuilder pb = new ProcessBuilder(processArguments);
logger.info("running: " + String.join(" ", pb.command()));
Process proc = null;
try {
proc = pb.start();
} catch (IOException e) {
- logger.log(Level.WARNING, "youtube-dl failed " + pb.command(), e);
+ logger.log(Level.WARNING, "yt-dlp failed " + pb.command(), e);
return null;
}
@@ -506,7 +518,7 @@ public class ExtractorYoutubeDL extends Extractor
try {
// this happens when there was no json output, which means no videos
// were found, totally normal
- logger.log(Level.FINE, "problem parsing json from youtube-dl " + pb.command() + " " + future.get());
+ logger.log(Level.FINE, "problem parsing json from yt-dlp " + pb.command() + " " + future.get());
} catch (InterruptedException e1) {
throw new IOException(e1);
} catch (ExecutionException e1) {
@@ -515,7 +527,7 @@ public class ExtractorYoutubeDL extends Extractor
}
} catch (IOException e) {
logger.log(Level.WARNING,
- "problem reading output from youtube-dl " + pb.command(),
+ "problem reading output from yt-dlp " + pb.command(),
e);
return null;
} finally {
@@ -523,7 +535,7 @@ public class ExtractorYoutubeDL extends Extractor
// the process should already have completed
proc.waitFor(1, TimeUnit.SECONDS);
} catch (InterruptedException e) {
- logger.warning("youtube-dl still running? killing it");
+ logger.warning("yt-dlp still running? killing it");
proc.destroyForcibly();
}
threadPool.shutdown();
@@ -540,13 +552,13 @@ public class ExtractorYoutubeDL extends Extractor
return true;
}
- // Otherwise, check if we want to run youtube-dl on the url.
+ // Otherwise, check if we want to run yt-dlp on the url.
return shouldExtract(uri);
}
/**
- * Returns {@code true} if we should run youtube-dl on this url. We run
- * youtube-dl on html 200s that are not too huge.
+ * Returns {@code true} if we should run yt-dlp on this url. We run
+ * yt-dlp on html 200s that are not too huge.
*/
protected boolean shouldExtract(CrawlURI uri) {
if (uri.getFetchStatus() != 200) {