lee/heritrix3
1
0
Fork 0
mirror of https://github.com/internetarchive/heritrix3.git synced 2026-09-12 16:55:52 +00:00
Code Issues Packages Projects Releases Wiki Activity

also handle case where Charset.forName() throws IllegalCharsetNameException, as opposed to UnsupportedCharsetException

Browse Source
This commit is contained in:
Noah Levitt
2014-01-28 16:36:51 -08:00
parent 161b359979
commit 31aea014d9
3 changed files with 21 additions and 7 deletions
Download Patch File Download Diff File Expand all files Collapse all files
modules/src/main/java/org/archive/modules/fetcher/FetchHTTP.java
+2 -2
View File
@@ -29,7 +29,6 @@ import static org.archive.modules.recrawl.RecrawlAttributeConstants.A_REFERENCE_
import java.io.IOException;
import java.nio.charset.Charset;
import java.nio.charset.UnsupportedCharsetException;
import java.security.MessageDigest;
import java.util.Arrays;
import java.util.HashMap;
@@ -584,7 +583,8 @@ public class FetchHTTP extends Processor implements Lifecycle {
if (charset != null) {
rec.setCharset(charset);
}
} catch (UnsupportedCharsetException e) {
} catch (IllegalArgumentException e) {
// exception could be UnsupportedCharsetException or IllegalCharsetNameException
String unsatisfiableCharset;
try {
unsatisfiableCharset = response.getFirstHeader("content-type").getElements()[0].getParameterByName("charset").getValue();
modules/src/test/java/org/archive/modules/fetcher/FetchHTTPTest.java
+9 -2
View File
@@ -184,8 +184,15 @@ public class FetchHTTPTest extends ProcessorTestBase {
response.setStatus(HttpServletResponse.SC_OK);
response.getOutputStream().write(CP1251_PAYLOAD);
((Request)request).setHandled(true);
} else if (target.equals("/bad-charset")) {
response.setContentType("text/plain;charset=BAD-CHARSET");
} else if (target.equals("/unsupported-charset")) {
response.setContentType("text/plain;charset=UNSUPPORTED-CHARSET");
response.setDateHeader("Last-Modified", 0);
response.setHeader("ETag", ETAG_TEST_VALUE);
response.setStatus(HttpServletResponse.SC_OK);
response.getOutputStream().write(DEFAULT_PAYLOAD_STRING.getBytes("US-ASCII"));
((Request)request).setHandled(true);
} else if (target.equals("/invalid-charset")) {
response.setContentType("text/plain;charset=%%INVALID-CHARSET%%");
response.setDateHeader("Last-Modified", 0);
response.setHeader("ETag", ETAG_TEST_VALUE);
response.setStatus(HttpServletResponse.SC_OK);
modules/src/test/java/org/archive/modules/fetcher/FetchHTTPTests.java
+10 -3
View File
@@ -806,10 +806,17 @@ public class FetchHTTPTests extends ProcessorTestBase {
+ "\u0438\u043E\u0432.\n",
curi.getRecorder().getContentReplayCharSequence().toString());
curi = makeCrawlURI("http://localhost:7777/bad-charset");
curi = makeCrawlURI("http://localhost:7777/unsupported-charset");
fetcher().process(curi);
assertEquals("text/plain;charset=BAD-CHARSET", curi.getHttpResponseHeader("content-type"));
assertTrue(curi.getAnnotations().contains("unsatisfiableCharsetInHeader:BAD-CHARSET"));
assertEquals("text/plain;charset=UNSUPPORTED-CHARSET", curi.getHttpResponseHeader("content-type"));
assertTrue(curi.getAnnotations().contains("unsatisfiableCharsetInHeader:UNSUPPORTED-CHARSET"));
assertEquals(Charset.forName("latin1"), curi.getRecorder().getCharset()); // default fallback
runDefaultChecks(curi, "requestLine", "contentType");
curi = makeCrawlURI("http://localhost:7777/invalid-charset");
fetcher().process(curi);
assertEquals("text/plain;charset=%%INVALID-CHARSET%%", curi.getHttpResponseHeader("content-type"));
assertTrue(curi.getAnnotations().contains("unsatisfiableCharsetInHeader:%%INVALID-CHARSET%%"));
assertEquals(Charset.forName("latin1"), curi.getRecorder().getCharset()); // default fallback
runDefaultChecks(curi, "requestLine", "contentType");
}