diff --git a/modules/src/main/java/org/archive/modules/extractor/ExtractorHTTP.java b/modules/src/main/java/org/archive/modules/extractor/ExtractorHTTP.java index 0e7fd78a..ab6f0bf6 100644 --- a/modules/src/main/java/org/archive/modules/extractor/ExtractorHTTP.java +++ b/modules/src/main/java/org/archive/modules/extractor/ExtractorHTTP.java @@ -62,8 +62,7 @@ public class ExtractorHTTP extends Extractor { protected void extract(CrawlURI curi) { // discover headers if present addHeaderLink(curi, "Location"); - addHeaderLink(curi, "Content-Location"); - + addContentLocationHeaderLink(curi, "Content-Location"); addRefreshHeaderLink(curi, "Refresh"); // try /favicon.ico for every HTTP(S) URI @@ -92,6 +91,19 @@ public class ExtractorHTTP extends Extractor { } } + protected void addContentLocationHeaderLink(CrawlURI curi, String headerKey) { + String headerValue = curi.getHttpResponseHeader(headerKey); + if (headerValue != null) { + try { + UURI dest = UURIFactory.getInstance(curi.getUURI(), headerValue); + addOutlink(curi, dest.toString(), LinkContext.INFERRED_MISC, Hop.INFERRED); + numberOfLinksExtracted.incrementAndGet(); + } catch (URIException e) { + logUriError(e, curi.getUURI(), headerValue); + } + } + } + protected void addHeaderLink(CrawlURI curi, String headerName, String url) { try { UURI dest = UURIFactory.getInstance(curi.getUURI(), url);