lee/heritrix3
1
0
Fork 0
mirror of https://github.com/internetarchive/heritrix3.git synced 2026-09-25 07:05:49 +00:00
Code Issues Packages Projects Releases Wiki Activity

HER-1895 support HTTP Refresh header

Browse Source
This commit is contained in:
Noah Levitt
2013-08-26 18:33:02 -07:00
parent d1bb1a4637
commit 4abfcb01b0
1 changed files with 30 additions and 8 deletions
Download Patch File Download Diff File Expand all files Collapse all files
modules/src/main/java/org/archive/modules/extractor/ExtractorHTTP.java
+30 -8
View File
@@ -19,6 +19,8 @@
package org.archive.modules.extractor;
import java.util.logging.Logger;
import org.apache.commons.httpclient.Header;
import org.apache.commons.httpclient.HttpMethod;
import org.apache.commons.httpclient.URIException;
@@ -36,6 +38,9 @@ public class ExtractorHTTP extends Extractor {
@SuppressWarnings("unused")
private static final long serialVersionUID = 3L;
private static Logger logger =
Logger.getLogger(ExtractorHTTP.class.getName());
public ExtractorHTTP() {
}
@@ -67,6 +72,8 @@ public class ExtractorHTTP extends Extractor {
addHeaderLink(curi, method.getResponseHeader("Location"));
addHeaderLink(curi, method.getResponseHeader("Content-Location"));
addRefreshHeaderLink(curi, method.getResponseHeader("Refresh"));
// try /favicon.ico for every HTTP(S) URI
addOutlink(curi, "/favicon.ico", LinkContext.INFERRED_MISC, Hop.INFERRED);
if(getInferRootPage()) {
@@ -74,21 +81,36 @@ public class ExtractorHTTP extends Extractor {
}
}
protected void addHeaderLink(CrawlURI curi, Header loc) {
if (loc == null) {
// If null, return without adding anything.
protected void addRefreshHeaderLink(CrawlURI curi, Header refreshHeader) {
if (refreshHeader == null) {
return;
}
// TODO: consider possibility of multiple headers
// parsing logic copied from ExtractorHTML meta-refresh handling
int urlIndex = refreshHeader.getValue().indexOf("=") + 1;
if (urlIndex > 0) {
String refreshUri = refreshHeader.getValue().substring(urlIndex);
addHeaderLink(curi, refreshHeader.getName(), refreshUri);
} else {
logger.warning("failed to parse refresh header \"" + refreshHeader.toExternalForm() + "\" from " + curi);
}
}
protected void addHeaderLink(CrawlURI curi, Header loc) {
if (loc != null) {
addHeaderLink(curi, loc.getName(), loc.getValue());
}
}
protected void addHeaderLink(CrawlURI curi, String headerName, String url) {
try {
UURI dest = UURIFactory.getInstance(curi.getUURI(), loc.getValue());
LinkContext lc = HTMLLinkContext.get(loc.getName()+":");
UURI dest = UURIFactory.getInstance(curi.getUURI(), url);
LinkContext lc = HTMLLinkContext.get(headerName+":");
Link link = new Link(curi.getUURI(), dest, lc, Hop.REFER);
curi.getOutLinks().add(link);
numberOfLinksExtracted.incrementAndGet();
} catch (URIException e) {
logUriError(e, curi.getUURI(), loc.getValue());
logUriError(e, curi.getUURI(), url);
}
}
}