lee/heritrix3
1
0
Fork 0
mirror of https://github.com/internetarchive/heritrix3.git synced 2026-08-21 22:16:50 +00:00
Code Issues Packages Projects Releases Wiki Activity

treat content as html and extract links if it looks like html, even if mime type belies that

Browse Source
This commit is contained in:
Noah Levitt
2014-07-17 21:09:51 -07:00
parent f101bba509
commit 4b75dfa16f
1 changed files with 13 additions and 5 deletions
Download Patch File Download Diff File Expand all files Collapse all files
modules/src/main/java/org/archive/modules/extractor/ExtractorHTML.java
+13 -5
View File
@@ -694,16 +694,24 @@ public class ExtractorHTML extends ContentExtractor implements InitializingBean
// assume it's okay to extract
}
}
String mime = uri.getContentType().toLowerCase();
return mime.startsWith("text/html")
if (mime.startsWith("text/html")
|| mime.startsWith("application/xhtml")
|| mime.startsWith("text/vnd.wap.wml")
|| mime.startsWith("application/vnd.wap.wml")
|| mime.startsWith("application/vnd.wap.xhtml");
|| mime.startsWith("application/vnd.wap.xhtml")) {
return true;
}
String contentPrefixLC = uri.getRecorder().getContentReplayPrefixString(1000).toLowerCase();
if (contentPrefixLC.contains("<html") || contentPrefixLC.contains("<!doctype html")) {
return true;
}
return false;
}
public boolean innerExtract(CrawlURI curi) {
if (!curi.containsContentTypeCharsetDeclaration()) {
String contentPrefix = curi.getRecorder().getContentReplayPrefixString(1000);