mirror of
https://github.com/internetarchive/heritrix3.git
synced 2026-09-21 21:26:16 +00:00
don't youtube-dl seeds twice
This commit is contained in:
@@ -507,6 +507,13 @@ public class ExtractorYoutubeDL extends Extractor
|
||||
return false;
|
||||
}
|
||||
|
||||
// don't check seeds twice, e.g., when processed again post-umbra
|
||||
if (uri.via == null) {
|
||||
if (results.pageUrls && results.pageUrls.contains(uri.toString()))
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
String mime = uri.getContentType().toLowerCase();
|
||||
if (mime.startsWith("text/html")
|
||||
|| mime.startsWith("application/xhtml")
|
||||
|
||||
Reference in New Issue
Block a user