From cf57532dbdafdfa4c5f0d091c4370751db1d57f2 Mon Sep 17 00:00:00 2001 From: Barbara Miller Date: Thu, 23 Apr 2020 17:35:01 -0700 Subject: [PATCH] try again --- .../archive/modules/extractor/ExtractorYoutubeDL.java | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/contrib/src/main/java/org/archive/modules/extractor/ExtractorYoutubeDL.java b/contrib/src/main/java/org/archive/modules/extractor/ExtractorYoutubeDL.java index b68e9e7d..fb868029 100644 --- a/contrib/src/main/java/org/archive/modules/extractor/ExtractorYoutubeDL.java +++ b/contrib/src/main/java/org/archive/modules/extractor/ExtractorYoutubeDL.java @@ -33,6 +33,7 @@ import java.io.UnsupportedEncodingException; import java.net.URI; import java.nio.channels.Channels; import java.util.ArrayList; +import java.util.HashMap; import java.util.List; import java.util.concurrent.Callable; import java.util.concurrent.ExecutionException; @@ -112,6 +113,9 @@ public class ExtractorYoutubeDL extends Extractor protected static final int MAX_VIDEOS_PER_PAGE = 1000; + // for shouldExtract + protected HashMap seedsYDLd = new HashMap(); + protected transient Logger ydlLogger = null; // unnamed toethread-local temporary file @@ -508,9 +512,11 @@ public class ExtractorYoutubeDL extends Extractor } // don't check seeds twice, e.g., when processed again post-umbra - if (uri.via == null) { - if (results.pageUrls && results.pageUrls.contains(uri.toString())) { + if (uri.getVia() == null) { + if (seedsYDLd.get(uri.toString())) { return false; + } else { + seedsYDLd.put(uri.toString(), true); } }