From 75b33663f98f89aefad33bd6921761eea9a3d6d7 Mon Sep 17 00:00:00 2001
From: Noah Levitt <nlevitt@archive.org>
Date: Mon, 6 May 2019 15:35:47 -0700
Subject: [PATCH] also annotate and log containing pages

---
 .../modules/extractor/ExtractorYoutubeDL.java | 47 ++++++++++++++-----
 1 file changed, 35 insertions(+), 12 deletions(-)

diff --git a/contrib/src/main/java/org/archive/modules/extractor/ExtractorYoutubeDL.java b/contrib/src/main/java/org/archive/modules/extractor/ExtractorYoutubeDL.java
index 49d11fce..92217d45 100644
--- a/contrib/src/main/java/org/archive/modules/extractor/ExtractorYoutubeDL.java
+++ b/contrib/src/main/java/org/archive/modules/extractor/ExtractorYoutubeDL.java
@@ -53,18 +53,22 @@ import com.google.gson.JsonStreamParser;
  * html.
  *
  * <p>
- * Keeps a log of media captured as a result of youtube-dl extraction. The
- * format of the log is as follows:
+ * Keeps a log of containing pages and media captured as a result of youtube-dl
+ * extraction. The format of the log is as follows:
  *
  * <pre>[timestamp] [media-http-status] [media-length] [media-mimetype] [media-digest] [media-timestamp] [media-url] [annotation] [containing-page-digest] [containing-page-timestamp] [containing-page-url] [seed-url]</pre>
  *
  * <p>
- * The annotation field looks like {@code "youtube-dl:1/3"}. In this example,
- * "3" is the number of media urls youtube-dl discovered on the page, and "1" is
- * the index of this media within the page. The intention is to use this for
- * playback. The rest of the fields included in the log were also chosen to
- * support creation of an index of media by containing page, to be used for
- * playback.
+ * For containing pages, all of the {@code media-*} fields have the value
+ * {@code "-"}, and the annotation field looks like {@code "youtube-dl:3"},
+ * meaning that ExtractorYoutubeDL extracted 3 media links from the page.
+ *
+ * <p>
+ * For media, the annotation field looks like {@code "youtube-dl:1/3"}, meaning
+ * this is the first of three media links extracted from the containing page.
+ * The intention is to use this for playback. The rest of the fields included in
+ * the log were also chosen to support creation of an index of media by
+ * containing page, to be used for playback.
  *
  * @author nlevitt
  */
@@ -160,11 +164,16 @@ public class ExtractorYoutubeDL extends Extractor implements Lifecycle {
             }
         } else {
             List<JsonObject> ydlJsons = runYoutubeDL(uri);
-            for (JsonObject json: ydlJsons) {
-                if (json.get("url") != null) {
-                    String videoUrl = json.get("url").getAsString();
-                    addVideoOutlink(uri, json, videoUrl);
+            if (ydlJsons != null && !ydlJsons.isEmpty()) {
+                for (JsonObject json: ydlJsons) {
+                    if (json.get("url") != null) {
+                        String videoUrl = json.get("url").getAsString();
+                        addVideoOutlink(uri, json, videoUrl);
+                    }
                 }
+                String annotation = "youtube-dl:" + ydlJsons.size();
+                uri.getAnnotations().add(annotation);
+                logContainingPage(uri, annotation);
             }
         }
     }
@@ -239,6 +248,20 @@ public class ExtractorYoutubeDL extends Extractor implements Lifecycle {
                 + " " + seed);
     }
 
+    protected void logContainingPage(CrawlURI uri, String annotation) {
+        String seed = uri.containsDataKey(CoreAttributeConstants.A_SOURCE_TAG)
+                ? uri.getSourceTag()
+                : "-";
+
+        ydlLogger.info(
+                "- - - - - -"
+                + " " + annotation
+                + " " + uri.getContentDigestSchemeString()
+                + " " + ArchiveUtils.get17DigitDate(uri.getFetchBeginTime())
+                + " " + uri
+                + " " + seed);
+    }
+
     protected void doRedirectInheritance(CrawlURI uri, String ydlAnnotation) {
         for (CrawlURI link: uri.getOutLinks()) {
             if ("R".equals(link.getLastHop())) {