From 91ef23eddfa3c452c441c5e0ce0923c3aae97283 Mon Sep 17 00:00:00 2001
From: Alex Osborne <aosborne@nla.gov.au>
Date: Fri, 30 Jul 2021 14:24:10 +0900
Subject: [PATCH] ExtractorPDFContext, ExtractorYoutubeDL: use addOutlink()
 helper method

---
 .../extractor/ExtractorPDFContent.java        |  9 +---
 .../modules/extractor/ExtractorYoutubeDL.java | 42 +++++++------------
 .../archive/modules/extractor/Extractor.java  |  9 ++--
 3 files changed, 20 insertions(+), 40 deletions(-)

diff --git a/contrib/src/main/java/org/archive/modules/extractor/ExtractorPDFContent.java b/contrib/src/main/java/org/archive/modules/extractor/ExtractorPDFContent.java
index 7092de5d..b09bde3a 100644
--- a/contrib/src/main/java/org/archive/modules/extractor/ExtractorPDFContent.java
+++ b/contrib/src/main/java/org/archive/modules/extractor/ExtractorPDFContent.java
@@ -137,14 +137,7 @@ public class ExtractorPDFContent extends ContentExtractor {
         }
 
         for (String uri: uris) {
-            try {
-                LinkContext lc = LinkContext.NAVLINK_MISC;
-                Hop hop = Hop.NAVLINK;
-                CrawlURI out = curi.createCrawlURI(uri, lc, hop);
-                curi.getOutLinks().add(out);
-            } catch (URIException e1) {
-                logUriError(e1, curi.getUURI(), uri);
-            }
+            addOutlink(curi, uri, LinkContext.NAVLINK_MISC, Hop.NAVLINK);
         }
         
         numberOfLinksExtracted.addAndGet(uris.size());
diff --git a/contrib/src/main/java/org/archive/modules/extractor/ExtractorYoutubeDL.java b/contrib/src/main/java/org/archive/modules/extractor/ExtractorYoutubeDL.java
index 98af6012..4848aa44 100644
--- a/contrib/src/main/java/org/archive/modules/extractor/ExtractorYoutubeDL.java
+++ b/contrib/src/main/java/org/archive/modules/extractor/ExtractorYoutubeDL.java
@@ -227,14 +227,7 @@ public class ExtractorYoutubeDL extends Extractor
             }
 
             for (String pageUrl: results.pageUrls) {
-                try {
-                    UURI dest = UURIFactory.getInstance(uri.getUURI(), pageUrl);
-                    CrawlURI link = uri.createCrawlURI(dest, LinkContext.NAVLINK_MISC,
-                            Hop.NAVLINK);
-                    uri.getOutLinks().add(link);
-                } catch (URIException e1) {
-                    logUriError(e1, uri.getUURI(), pageUrl);
-                }
+                addOutlink(uri, pageUrl, LinkContext.NAVLINK_MISC, Hop.NAVLINK);
             }
 
             if (results.videoUrls.size() > 0) {
@@ -246,26 +239,21 @@ public class ExtractorYoutubeDL extends Extractor
     }
 
     protected void addVideoOutlink(CrawlURI uri, String videoUrl, int playlistIndex, int nEntries) {
-        try {
-            UURI dest = UURIFactory.getInstance(uri.getUURI(), videoUrl);
-            CrawlURI link = uri.createCrawlURI(dest, LinkContext.EMBED_MISC,
-                    Hop.EMBED);
-
-            // annotation
-            String annotation = "youtube-dl:" + (playlistIndex + 1) + "/" + nEntries;
-            link.getAnnotations().add(annotation);
-
-            // save info unambiguously identifying containing page capture
-            link.getData().put(YDL_CONTAINING_PAGE_URI, uri.toString());
-            link.getData().put(YDL_CONTAINING_PAGE_TIMESTAMP,
-                    ArchiveUtils.get17DigitDate(uri.getFetchBeginTime()));
-            link.getData().put(YDL_CONTAINING_PAGE_DIGEST,
-                    uri.getContentDigestSchemeString());
-
-            uri.getOutLinks().add(link);
-        } catch (URIException e) {
-            logUriError(e, uri.getUURI(), videoUrl);
+        CrawlURI link = addOutlink(uri, videoUrl, LinkContext.EMBED_MISC, Hop.EMBED);
+        if (link == null) {
+            return;
         }
+
+        // annotation
+        String annotation = "youtube-dl:" + (playlistIndex + 1) + "/" + nEntries;
+        link.getAnnotations().add(annotation);
+
+        // save info unambiguously identifying containing page capture
+        link.getData().put(YDL_CONTAINING_PAGE_URI, uri.toString());
+        link.getData().put(YDL_CONTAINING_PAGE_TIMESTAMP,
+                ArchiveUtils.get17DigitDate(uri.getFetchBeginTime()));
+        link.getData().put(YDL_CONTAINING_PAGE_DIGEST,
+                uri.getContentDigestSchemeString());
     }
 
     protected String findYdlAnnotation(CrawlURI uri) {
diff --git a/modules/src/main/java/org/archive/modules/extractor/Extractor.java b/modules/src/main/java/org/archive/modules/extractor/Extractor.java
index e7fe8b6a..b33df2cc 100644
--- a/modules/src/main/java/org/archive/modules/extractor/Extractor.java
+++ b/modules/src/main/java/org/archive/modules/extractor/Extractor.java
@@ -134,19 +134,18 @@ public abstract class Extractor extends Processor {
 
     /**
      * Create and add a 'Link' to the CrawlURI with given URI/context/hop-type
-     * @param curi
-     * @param uri
-     * @param context
-     * @param hop
+     * @return the new outlink or null if it was not valid
      */
-    protected void addOutlink(CrawlURI curi, String uri, LinkContext context,
+    protected CrawlURI addOutlink(CrawlURI curi, String uri, LinkContext context,
             Hop hop) {
         try {
             UURI dest = UURIFactory.getInstance(curi.getUURI(), uri);
             CrawlURI link = curi.createCrawlURI(dest, context, hop);
             curi.getOutLinks().add(link);
+            return link;
         } catch (URIException e) {
             logUriError(e, curi.getUURI(), uri);
+            return null;
         }
     }