lee/heritrix3
1
0
Fork 0
mirror of https://github.com/internetarchive/heritrix3.git synced 2026-09-26 15:46:31 +00:00
Code Issues Packages Projects Releases Wiki Activity

[HER-741] Make extractors interrogate for charset

Browse Source 
* ExtractorHTML
    fix reflexive probe to look as many characters in (1000) as initially
This commit is contained in:
gojomo
2011-07-14 06:53:56 +00:00
parent be12d1e8f3
commit 2e10d07da2
1 changed files with 1 additions and 1 deletions
Download Patch File Download Diff File Expand all files Collapse all files
modules/src/main/java/org/archive/modules/extractor/ExtractorHTML.java
+1 -1
View File
@@ -672,7 +672,7 @@ public class ExtractorHTML extends ContentExtractor implements InitializingBean
String contentPrefix = curi.getRecorder().getContentReplayPrefixString(1000);
Charset contentDeclaredEncoding = getContentDeclaredCharset(curi,contentPrefix);
if(!curi.getRecorder().getCharset().equals(contentDeclaredEncoding) && contentDeclaredEncoding!=null) {
String newContentPrefix = curi.getRecorder().getContentReplayPrefixString(50,contentDeclaredEncoding);
String newContentPrefix = curi.getRecorder().getContentReplayPrefixString(1000,contentDeclaredEncoding);
Charset reflexiveCharset = getContentDeclaredCharset(curi, newContentPrefix);
if(contentDeclaredEncoding.equals(reflexiveCharset)) {
// content-declared charset is self-consistent; use