From 31aea014d9bb2ed406bc74507775b04b35e89d1f Mon Sep 17 00:00:00 2001 From: Noah Levitt Date: Tue, 28 Jan 2014 16:36:51 -0800 Subject: [PATCH] also handle case where Charset.forName() throws IllegalCharsetNameException, as opposed to UnsupportedCharsetException --- .../java/org/archive/modules/fetcher/FetchHTTP.java | 4 ++-- .../org/archive/modules/fetcher/FetchHTTPTest.java | 11 +++++++++-- .../org/archive/modules/fetcher/FetchHTTPTests.java | 13 ++++++++++--- 3 files changed, 21 insertions(+), 7 deletions(-) diff --git a/modules/src/main/java/org/archive/modules/fetcher/FetchHTTP.java b/modules/src/main/java/org/archive/modules/fetcher/FetchHTTP.java index d9aab989..1f78a7e9 100644 --- a/modules/src/main/java/org/archive/modules/fetcher/FetchHTTP.java +++ b/modules/src/main/java/org/archive/modules/fetcher/FetchHTTP.java @@ -29,7 +29,6 @@ import static org.archive.modules.recrawl.RecrawlAttributeConstants.A_REFERENCE_ import java.io.IOException; import java.nio.charset.Charset; -import java.nio.charset.UnsupportedCharsetException; import java.security.MessageDigest; import java.util.Arrays; import java.util.HashMap; @@ -584,7 +583,8 @@ public class FetchHTTP extends Processor implements Lifecycle { if (charset != null) { rec.setCharset(charset); } - } catch (UnsupportedCharsetException e) { + } catch (IllegalArgumentException e) { + // exception could be UnsupportedCharsetException or IllegalCharsetNameException String unsatisfiableCharset; try { unsatisfiableCharset = response.getFirstHeader("content-type").getElements()[0].getParameterByName("charset").getValue(); diff --git a/modules/src/test/java/org/archive/modules/fetcher/FetchHTTPTest.java b/modules/src/test/java/org/archive/modules/fetcher/FetchHTTPTest.java index 0ddc034d..76154bb8 100644 --- a/modules/src/test/java/org/archive/modules/fetcher/FetchHTTPTest.java +++ b/modules/src/test/java/org/archive/modules/fetcher/FetchHTTPTest.java @@ -184,8 +184,15 @@ public class FetchHTTPTest extends ProcessorTestBase { response.setStatus(HttpServletResponse.SC_OK); response.getOutputStream().write(CP1251_PAYLOAD); ((Request)request).setHandled(true); - } else if (target.equals("/bad-charset")) { - response.setContentType("text/plain;charset=BAD-CHARSET"); + } else if (target.equals("/unsupported-charset")) { + response.setContentType("text/plain;charset=UNSUPPORTED-CHARSET"); + response.setDateHeader("Last-Modified", 0); + response.setHeader("ETag", ETAG_TEST_VALUE); + response.setStatus(HttpServletResponse.SC_OK); + response.getOutputStream().write(DEFAULT_PAYLOAD_STRING.getBytes("US-ASCII")); + ((Request)request).setHandled(true); + } else if (target.equals("/invalid-charset")) { + response.setContentType("text/plain;charset=%%INVALID-CHARSET%%"); response.setDateHeader("Last-Modified", 0); response.setHeader("ETag", ETAG_TEST_VALUE); response.setStatus(HttpServletResponse.SC_OK); diff --git a/modules/src/test/java/org/archive/modules/fetcher/FetchHTTPTests.java b/modules/src/test/java/org/archive/modules/fetcher/FetchHTTPTests.java index 70bc8717..1078bdb8 100644 --- a/modules/src/test/java/org/archive/modules/fetcher/FetchHTTPTests.java +++ b/modules/src/test/java/org/archive/modules/fetcher/FetchHTTPTests.java @@ -806,10 +806,17 @@ public class FetchHTTPTests extends ProcessorTestBase { + "\u0438\u043E\u0432.\n", curi.getRecorder().getContentReplayCharSequence().toString()); - curi = makeCrawlURI("http://localhost:7777/bad-charset"); + curi = makeCrawlURI("http://localhost:7777/unsupported-charset"); fetcher().process(curi); - assertEquals("text/plain;charset=BAD-CHARSET", curi.getHttpResponseHeader("content-type")); - assertTrue(curi.getAnnotations().contains("unsatisfiableCharsetInHeader:BAD-CHARSET")); + assertEquals("text/plain;charset=UNSUPPORTED-CHARSET", curi.getHttpResponseHeader("content-type")); + assertTrue(curi.getAnnotations().contains("unsatisfiableCharsetInHeader:UNSUPPORTED-CHARSET")); + assertEquals(Charset.forName("latin1"), curi.getRecorder().getCharset()); // default fallback + runDefaultChecks(curi, "requestLine", "contentType"); + + curi = makeCrawlURI("http://localhost:7777/invalid-charset"); + fetcher().process(curi); + assertEquals("text/plain;charset=%%INVALID-CHARSET%%", curi.getHttpResponseHeader("content-type")); + assertTrue(curi.getAnnotations().contains("unsatisfiableCharsetInHeader:%%INVALID-CHARSET%%")); assertEquals(Charset.forName("latin1"), curi.getRecorder().getCharset()); // default fallback runDefaultChecks(curi, "requestLine", "contentType"); }