lee/heritrix3
1
0
Fork 0
mirror of https://github.com/internetarchive/heritrix3.git synced 2026-09-22 21:55:54 +00:00
Code Issues Packages Projects Releases Wiki Activity

ExtractorPDFContext, ExtractorYoutubeDL: use addOutlink() helper method

Browse Source
This commit is contained in:
Alex Osborne
2021-07-30 14:25:34 +09:00
parent d2d9d4f157
commit 91ef23eddf
3 changed files with 20 additions and 40 deletions
Download Patch File Download Diff File Expand all files Collapse all files
contrib/src/main/java/org/archive/modules/extractor/ExtractorPDFContent.java
+1 -8
View File
@@ -137,14 +137,7 @@ public class ExtractorPDFContent extends ContentExtractor {
}
for (String uri: uris) {
try {
LinkContext lc = LinkContext.NAVLINK_MISC;
Hop hop = Hop.NAVLINK;
CrawlURI out = curi.createCrawlURI(uri, lc, hop);
curi.getOutLinks().add(out);
} catch (URIException e1) {
logUriError(e1, curi.getUURI(), uri);
}
addOutlink(curi, uri, LinkContext.NAVLINK_MISC, Hop.NAVLINK);
}
numberOfLinksExtracted.addAndGet(uris.size());
contrib/src/main/java/org/archive/modules/extractor/ExtractorYoutubeDL.java
+15 -27
View File
@@ -227,14 +227,7 @@ public class ExtractorYoutubeDL extends Extractor
}
for (String pageUrl: results.pageUrls) {
try {
UURI dest = UURIFactory.getInstance(uri.getUURI(), pageUrl);
CrawlURI link = uri.createCrawlURI(dest, LinkContext.NAVLINK_MISC,
Hop.NAVLINK);
uri.getOutLinks().add(link);
} catch (URIException e1) {
logUriError(e1, uri.getUURI(), pageUrl);
}
addOutlink(uri, pageUrl, LinkContext.NAVLINK_MISC, Hop.NAVLINK);
}
if (results.videoUrls.size() > 0) {
@@ -246,26 +239,21 @@ public class ExtractorYoutubeDL extends Extractor
}
protected void addVideoOutlink(CrawlURI uri, String videoUrl, int playlistIndex, int nEntries) {
try {
UURI dest = UURIFactory.getInstance(uri.getUURI(), videoUrl);
CrawlURI link = uri.createCrawlURI(dest, LinkContext.EMBED_MISC,
Hop.EMBED);
// annotation
String annotation = "youtube-dl:" + (playlistIndex + 1) + "/" + nEntries;
link.getAnnotations().add(annotation);
// save info unambiguously identifying containing page capture
link.getData().put(YDL_CONTAINING_PAGE_URI, uri.toString());
link.getData().put(YDL_CONTAINING_PAGE_TIMESTAMP,
ArchiveUtils.get17DigitDate(uri.getFetchBeginTime()));
link.getData().put(YDL_CONTAINING_PAGE_DIGEST,
uri.getContentDigestSchemeString());
uri.getOutLinks().add(link);
} catch (URIException e) {
logUriError(e, uri.getUURI(), videoUrl);
CrawlURI link = addOutlink(uri, videoUrl, LinkContext.EMBED_MISC, Hop.EMBED);
if (link == null) {
return;
}
// annotation
String annotation = "youtube-dl:" + (playlistIndex + 1) + "/" + nEntries;
link.getAnnotations().add(annotation);
// save info unambiguously identifying containing page capture
link.getData().put(YDL_CONTAINING_PAGE_URI, uri.toString());
link.getData().put(YDL_CONTAINING_PAGE_TIMESTAMP,
ArchiveUtils.get17DigitDate(uri.getFetchBeginTime()));
link.getData().put(YDL_CONTAINING_PAGE_DIGEST,
uri.getContentDigestSchemeString());
}
protected String findYdlAnnotation(CrawlURI uri) {
modules/src/main/java/org/archive/modules/extractor/Extractor.java
+4 -5
View File
@@ -134,19 +134,18 @@ public abstract class Extractor extends Processor {
/**
* Create and add a 'Link' to the CrawlURI with given URI/context/hop-type
* @param curi
* @param uri
* @param context
* @param hop
* @return the new outlink or null if it was not valid
*/
protected void addOutlink(CrawlURI curi, String uri, LinkContext context,
protected CrawlURI addOutlink(CrawlURI curi, String uri, LinkContext context,
Hop hop) {
try {
UURI dest = UURIFactory.getInstance(curi.getUURI(), uri);
CrawlURI link = curi.createCrawlURI(dest, context, hop);
curi.getOutLinks().add(link);
return link;
} catch (URIException e) {
logUriError(e, curi.getUURI(), uri);
return null;
}
}