lee/heritrix3
1
0
Fork 0
mirror of https://github.com/internetarchive/heritrix3.git synced 2026-09-25 23:25:44 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #644 from adam-miller/adam/parameterize-yt-dlp-process-command

Browse Source 
feat: Parameterize yt-dlp process command arguments
This commit is contained in:
Alex Osborne
2025-03-18 11:39:53 +09:00
committed by GitHub
parent cc1a668abd 710662fcd0
commit ace2ab80e8
1 changed files with 30 additions and 18 deletions
Download Patch File Download Diff File Expand all files Collapse all files
contrib/src/main/java/org/archive/modules/extractor/ExtractorYoutubeDL.java
+30 -18
View File
@@ -34,6 +34,7 @@ import java.io.UnsupportedEncodingException;
import java.net.URI;
import java.nio.channels.Channels;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
import java.util.concurrent.Callable;
import java.util.concurrent.ExecutionException;
@@ -67,7 +68,7 @@ import com.google.gson.stream.JsonToken;
* Extracts links to media by running yt-dlp in a subprocess. Runs only on html.
*
* <p>
* Also implements {@link WARCRecordBuilder} to write youtube-dl json to the
* Also implements {@link WARCRecordBuilder} to write yt-dlp json to the
* warc.
*
* <p>
@@ -81,7 +82,7 @@ import com.google.gson.stream.JsonToken;
* fetch chain, and to the end of the warc writer chain.
*
* <p>
* Keeps a log of containing pages and media captured as a result of youtube-dl
* Keeps a log of containing pages and media captured as a result of yt-dlp
* extraction. The format of the log is as follows:
*
* <pre>
@@ -140,7 +141,7 @@ public class ExtractorYoutubeDL extends Extractor
localTemp = openNewTempFile();
tempfile.set(localTemp);
}
logger.info("Getting youtube-dl temp file ");
logger.info("Getting yt-dlp temp file ");
return localTemp;
}
protected boolean isOpen(RandomAccessFile f) {
@@ -149,12 +150,12 @@ public class ExtractorYoutubeDL extends Extractor
return true;
}
catch (IOException e) {
logger.info("youtube-dl temp file is not open");
logger.info("yt-dlp temp file is not open");
return false ;
}
}
protected RandomAccessFile openNewTempFile() {
logger.info("Opening New youtube-dl temp file ");
logger.info("Opening New yt-dlp temp file ");
File t;
try {
t = File.createTempFile("ydl", ".json");
@@ -194,6 +195,18 @@ public class ExtractorYoutubeDL extends Extractor
kp.put("logMetadataRecord",logMetadataRecord);
}
{
setProcessArguments(Arrays.asList("yt-dlp", "--ignore-config",
"--simulate", "--dump-single-json", "-S vcodec:h264,res:720,acodec:aac",
"--no-cache-dir", "--no-playlist", "--playlist-end=" + MAX_VIDEOS_PER_PAGE ));
}
public List<String> getProcessArguments() {
return (List<String>) kp.get("processArguments");
}
public void setProcessArguments(List<String> processArguments) {
kp.put("processArguments", processArguments);
}
@Override
public void start() {
if (!isRunning) {
@@ -229,7 +242,7 @@ public class ExtractorYoutubeDL extends Extractor
* - If {@code uri} is annotated "youtube-dl" and is an actual video
* download, log a line to ExtractorYoutubeDL.log
*
* - If {@link #shouldExtract(CrawlURI)}, do youtube-dl extraction.
* - If {@link #shouldExtract(CrawlURI)}, do yt-dlp extraction.
*/
@Override
protected void extract(CrawlURI uri) {
@@ -400,7 +413,7 @@ public class ExtractorYoutubeDL extends Extractor
}
/**
* Streams through youtube-dl json output. Sticks video urls in
* Streams through yt-dlp json output. Sticks video urls in
* <code>results.videoUrls</code>, web page urls in
* <code>results.pageUrls</code>, and saves the json in anonymous temp file
* <code>results.jsonFile</code>.
@@ -468,17 +481,16 @@ public class ExtractorYoutubeDL extends Extractor
* the best audio with best acodec no better than aac and
* with the smallest dimension no larger than 720.
*/
ProcessBuilder pb = new ProcessBuilder("yt-dlp", "--ignore-config",
"--simulate", "--dump-single-json", "-S vcodec:h264,res:720,acodec:aac",
"--no-cache-dir", "--no-playlist",
"--playlist-end=" + MAX_VIDEOS_PER_PAGE, uri.toString());
ArrayList<String> processArguments = new ArrayList<String>(getProcessArguments());
processArguments.add(uri.toString());
ProcessBuilder pb = new ProcessBuilder(processArguments);
logger.info("running: " + String.join(" ", pb.command()));
Process proc = null;
try {
proc = pb.start();
} catch (IOException e) {
logger.log(Level.WARNING, "youtube-dl failed " + pb.command(), e);
logger.log(Level.WARNING, "yt-dlp failed " + pb.command(), e);
return null;
}
@@ -506,7 +518,7 @@ public class ExtractorYoutubeDL extends Extractor
try {
// this happens when there was no json output, which means no videos
// were found, totally normal
logger.log(Level.FINE, "problem parsing json from youtube-dl " + pb.command() + " " + future.get());
logger.log(Level.FINE, "problem parsing json from yt-dlp " + pb.command() + " " + future.get());
} catch (InterruptedException e1) {
throw new IOException(e1);
} catch (ExecutionException e1) {
@@ -515,7 +527,7 @@ public class ExtractorYoutubeDL extends Extractor
}
} catch (IOException e) {
logger.log(Level.WARNING,
"problem reading output from youtube-dl " + pb.command(),
"problem reading output from yt-dlp " + pb.command(),
e);
return null;
} finally {
@@ -523,7 +535,7 @@ public class ExtractorYoutubeDL extends Extractor
// the process should already have completed
proc.waitFor(1, TimeUnit.SECONDS);
} catch (InterruptedException e) {
logger.warning("youtube-dl still running? killing it");
logger.warning("yt-dlp still running? killing it");
proc.destroyForcibly();
}
threadPool.shutdown();
@@ -540,13 +552,13 @@ public class ExtractorYoutubeDL extends Extractor
return true;
}
// Otherwise, check if we want to run youtube-dl on the url.
// Otherwise, check if we want to run yt-dlp on the url.
return shouldExtract(uri);
}
/**
* Returns {@code true} if we should run youtube-dl on this url. We run
* youtube-dl on html 200s that are not too huge.
* Returns {@code true} if we should run yt-dlp on this url. We run
* yt-dlp on html 200s that are not too huge.
*/
protected boolean shouldExtract(CrawlURI uri) {
if (uri.getFetchStatus() != 200) {