mirror of
https://github.com/internetarchive/heritrix3.git
synced 2026-09-21 21:26:16 +00:00
try again
This commit is contained in:
@@ -33,6 +33,7 @@ import java.io.UnsupportedEncodingException;
|
||||
import java.net.URI;
|
||||
import java.nio.channels.Channels;
|
||||
import java.util.ArrayList;
|
||||
import java.util.HashMap;
|
||||
import java.util.List;
|
||||
import java.util.concurrent.Callable;
|
||||
import java.util.concurrent.ExecutionException;
|
||||
@@ -112,6 +113,9 @@ public class ExtractorYoutubeDL extends Extractor
|
||||
|
||||
protected static final int MAX_VIDEOS_PER_PAGE = 1000;
|
||||
|
||||
// for shouldExtract
|
||||
protected HashMap<String, Boolean> seedsYDLd = new HashMap<String, Boolean>();
|
||||
|
||||
protected transient Logger ydlLogger = null;
|
||||
|
||||
// unnamed toethread-local temporary file
|
||||
@@ -508,9 +512,11 @@ public class ExtractorYoutubeDL extends Extractor
|
||||
}
|
||||
|
||||
// don't check seeds twice, e.g., when processed again post-umbra
|
||||
if (uri.via == null) {
|
||||
if (results.pageUrls && results.pageUrls.contains(uri.toString())) {
|
||||
if (uri.getVia() == null) {
|
||||
if (seedsYDLd.get(uri.toString())) {
|
||||
return false;
|
||||
} else {
|
||||
seedsYDLd.put(uri.toString(), true);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
Reference in New Issue
Block a user