lee/heritrix3
1
0
Fork 0
mirror of https://github.com/internetarchive/heritrix3.git synced 2026-09-22 21:55:54 +00:00
Code Issues Packages Projects Releases Wiki Activity

Make Content-Location header url INFERRED not REFFER hop type since Content-Location is not for redirection (#151)

Browse Source
This commit is contained in:
Hunter
2016-04-13 15:19:21 -07:00
committed by Noah Levitt
parent 124907341f
commit a02a32cdfe
1 changed files with 14 additions and 2 deletions
Download Patch File Download Diff File Expand all files Collapse all files
modules/src/main/java/org/archive/modules/extractor/ExtractorHTTP.java
+14 -2
View File
@@ -62,8 +62,7 @@ public class ExtractorHTTP extends Extractor {
protected void extract(CrawlURI curi) {
// discover headers if present
addHeaderLink(curi, "Location");
addHeaderLink(curi, "Content-Location");
addContentLocationHeaderLink(curi, "Content-Location");
addRefreshHeaderLink(curi, "Refresh");
// try /favicon.ico for every HTTP(S) URI
@@ -92,6 +91,19 @@ public class ExtractorHTTP extends Extractor {
}
}
protected void addContentLocationHeaderLink(CrawlURI curi, String headerKey) {
String headerValue = curi.getHttpResponseHeader(headerKey);
if (headerValue != null) {
try {
UURI dest = UURIFactory.getInstance(curi.getUURI(), headerValue);
addOutlink(curi, dest.toString(), LinkContext.INFERRED_MISC, Hop.INFERRED);
numberOfLinksExtracted.incrementAndGet();
} catch (URIException e) {
logUriError(e, curi.getUURI(), headerValue);
}
}
}
protected void addHeaderLink(CrawlURI curi, String headerName, String url) {
try {
UURI dest = UURIFactory.getInstance(curi.getUURI(), url);