diff --git a/modules/src/main/java/org/archive/modules/extractor/Extractor.java b/modules/src/main/java/org/archive/modules/extractor/Extractor.java index 7a7f4f1c..71018ee3 100644 --- a/modules/src/main/java/org/archive/modules/extractor/Extractor.java +++ b/modules/src/main/java/org/archive/modules/extractor/Extractor.java @@ -125,6 +125,24 @@ public abstract class Extractor extends Processor { */ protected abstract void extract(CrawlURI uri); + + /** + * Create and add a 'Link' to the CrawlURI with given URI/context/hop-type + * @param curi + * @param uri + * @param context + * @param hop + */ + protected void addOutlink(CrawlURI curi, String uri, LinkContext context, + Hop hop) { + try { + UURI dest = UURIFactory.getInstance(curi.getUURI(), uri); + Link link = new Link(curi.getUURI(), dest, context, hop); + curi.getOutLinks().add(link); + } catch (URIException e) { + logUriError(e, curi.getUURI(), uri); + } + } public void logUriError(URIException e, UURI uuri, CharSequence l) { diff --git a/modules/src/main/java/org/archive/modules/extractor/ExtractorHTTP.java b/modules/src/main/java/org/archive/modules/extractor/ExtractorHTTP.java index 8f7d1ee2..f53b0a21 100644 --- a/modules/src/main/java/org/archive/modules/extractor/ExtractorHTTP.java +++ b/modules/src/main/java/org/archive/modules/extractor/ExtractorHTTP.java @@ -52,8 +52,12 @@ public class ExtractorHTTP extends Extractor { @Override protected void extract(CrawlURI curi) { HttpMethod method = curi.getHttpMethod(); + // discover headers if present addHeaderLink(curi, method.getResponseHeader("Location")); addHeaderLink(curi, method.getResponseHeader("Content-Location")); + + // try /favicon.ico for every HTTP(S) URI + addOutlink(curi, "/favicon.ico", LinkContext.EMBED_MISC, Hop.EMBED); } protected void addHeaderLink(CrawlURI curi, Header loc) {