From d50158c0124ae99207f8a21bc7e87f6c49c72521 Mon Sep 17 00:00:00 2001 From: Barbara Miller Date: Wed, 6 Sep 2023 11:25:12 -0700 Subject: [PATCH 1/6] Update ExtractorYoutubeDL.java --- .../archive/modules/extractor/ExtractorYoutubeDL.java | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/contrib/src/main/java/org/archive/modules/extractor/ExtractorYoutubeDL.java b/contrib/src/main/java/org/archive/modules/extractor/ExtractorYoutubeDL.java index 4848aa44..91aa96ad 100644 --- a/contrib/src/main/java/org/archive/modules/extractor/ExtractorYoutubeDL.java +++ b/contrib/src/main/java/org/archive/modules/extractor/ExtractorYoutubeDL.java @@ -436,14 +436,13 @@ public class ExtractorYoutubeDL extends Extractor */ protected YoutubeDLResults runYoutubeDL(CrawlURI uri) { /* - * --format=best - * - * best: Select the best quality format represented by a single file - * with video and audio. - * https://github.com/ytdl-org/youtube-dl/blob/master/README.md#format-selection + * updated for yt-dlp v.2023.07.06 + * Download the best video with best codec no better than h264 and + * with height no larger than 576... + * https://github.com/yt-dlp/yt-dlp#format-selection-examples */ ProcessBuilder pb = new ProcessBuilder("youtube-dl", "--ignore-config", - "--simulate", "--dump-single-json", "--format=best[height <=? 576]", + "--simulate", "--dump-single-json", "-S codec:h264,height:576", "--no-cache-dir", "--no-playlist", "--playlist-end=" + MAX_VIDEOS_PER_PAGE, uri.toString()); logger.info("running: " + String.join(" ", pb.command())); From 1c07e43ade91b4586d00b5c978b596e72b4f5801 Mon Sep 17 00:00:00 2001 From: Alex Osborne Date: Tue, 19 Sep 2023 11:08:56 +0900 Subject: [PATCH 2/6] Update groovy to 4.0.15 for JDK 20 support --- modules/pom.xml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/modules/pom.xml b/modules/pom.xml index 32613751..a6755776 100644 --- a/modules/pom.xml +++ b/modules/pom.xml @@ -30,12 +30,12 @@ compile - org.codehaus.groovy + org.apache.groovy groovy-jsr223 ${groovy.version} - org.codehaus.groovy + org.apache.groovy groovy-templates ${groovy.version} @@ -115,6 +115,6 @@ UTF-8 - 3.0.13 + 4.0.15 From 1a27551e60cff9ba1f0eab7e8a0a83e5ea00b0e0 Mon Sep 17 00:00:00 2001 From: Alex Osborne Date: Tue, 19 Sep 2023 11:10:28 +0900 Subject: [PATCH 3/6] Update CI tests from JDK 19 to 20 --- .github/workflows/maven.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/maven.yml b/.github/workflows/maven.yml index 3a4d7e32..5742cbdb 100644 --- a/.github/workflows/maven.yml +++ b/.github/workflows/maven.yml @@ -10,7 +10,7 @@ jobs: build: strategy: matrix: - jdk: [8, 11, 17, 19] + jdk: [8, 11, 17, 20] runs-on: ubuntu-latest timeout-minutes: 30 From bcfed0c1f7b638a3386f2e48351bcaa802c410a1 Mon Sep 17 00:00:00 2001 From: Barbara Miller Date: Fri, 29 Sep 2023 15:41:13 -0700 Subject: [PATCH 4/6] yt-dlp (not youtube-dl) --- .../org/archive/modules/extractor/ExtractorYoutubeDL.java | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/contrib/src/main/java/org/archive/modules/extractor/ExtractorYoutubeDL.java b/contrib/src/main/java/org/archive/modules/extractor/ExtractorYoutubeDL.java index 91aa96ad..3753ebe5 100644 --- a/contrib/src/main/java/org/archive/modules/extractor/ExtractorYoutubeDL.java +++ b/contrib/src/main/java/org/archive/modules/extractor/ExtractorYoutubeDL.java @@ -63,8 +63,7 @@ import com.google.gson.stream.JsonReader; import com.google.gson.stream.JsonToken; /** - * Extracts links to media by running youtube-dl in a subprocess. Runs only on - * html. + * Extracts links to media by running yt-dlp in a subprocess. Runs only on html. * *

* Also implements {@link WARCRecordBuilder} to write youtube-dl json to the @@ -441,7 +440,7 @@ public class ExtractorYoutubeDL extends Extractor * with height no larger than 576... * https://github.com/yt-dlp/yt-dlp#format-selection-examples */ - ProcessBuilder pb = new ProcessBuilder("youtube-dl", "--ignore-config", + ProcessBuilder pb = new ProcessBuilder("yt-dlp", "--ignore-config", "--simulate", "--dump-single-json", "-S codec:h264,height:576", "--no-cache-dir", "--no-playlist", "--playlist-end=" + MAX_VIDEOS_PER_PAGE, uri.toString()); From d3ca691f8d111d9f9bd1f4ec9eecbbcf02942679 Mon Sep 17 00:00:00 2001 From: Alex Osborne Date: Wed, 11 Oct 2023 11:11:39 +0900 Subject: [PATCH 5/6] Update CI tests from JDK 20 to 21 --- .github/workflows/maven.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/maven.yml b/.github/workflows/maven.yml index 5742cbdb..15241faf 100644 --- a/.github/workflows/maven.yml +++ b/.github/workflows/maven.yml @@ -10,7 +10,7 @@ jobs: build: strategy: matrix: - jdk: [8, 11, 17, 20] + jdk: [8, 11, 17, 21] runs-on: ubuntu-latest timeout-minutes: 30 From e4c9eb210ad03d592cab67cce0d5e57cd556e3c6 Mon Sep 17 00:00:00 2001 From: Barbara Miller Date: Tue, 7 Nov 2023 14:09:13 -0800 Subject: [PATCH 6/6] more updates for newer yt-dlp --- .../archive/modules/extractor/ExtractorYoutubeDL.java | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/contrib/src/main/java/org/archive/modules/extractor/ExtractorYoutubeDL.java b/contrib/src/main/java/org/archive/modules/extractor/ExtractorYoutubeDL.java index 3753ebe5..ce17fb09 100644 --- a/contrib/src/main/java/org/archive/modules/extractor/ExtractorYoutubeDL.java +++ b/contrib/src/main/java/org/archive/modules/extractor/ExtractorYoutubeDL.java @@ -435,13 +435,14 @@ public class ExtractorYoutubeDL extends Extractor */ protected YoutubeDLResults runYoutubeDL(CrawlURI uri) { /* - * updated for yt-dlp v.2023.07.06 - * Download the best video with best codec no better than h264 and - * with height no larger than 576... * https://github.com/yt-dlp/yt-dlp#format-selection-examples + * updated for yt-dlp v.2023.07.06 and higher + * Download the best video with best vcodec no better than h264 and + * the best audio with best acodec no better than aac and + * with neither height nor width larger than 576. */ ProcessBuilder pb = new ProcessBuilder("yt-dlp", "--ignore-config", - "--simulate", "--dump-single-json", "-S codec:h264,height:576", + "--simulate", "--dump-single-json", "-S res:576,hcodec:h264,acodec:aac", "--no-cache-dir", "--no-playlist", "--playlist-end=" + MAX_VIDEOS_PER_PAGE, uri.toString()); logger.info("running: " + String.join(" ", pb.command()));