diff --git a/modules/src/main/java/org/archive/modules/fetcher/FetchHTTP.java b/modules/src/main/java/org/archive/modules/fetcher/FetchHTTP.java index 16d124a2..8b839b3b 100644 --- a/modules/src/main/java/org/archive/modules/fetcher/FetchHTTP.java +++ b/modules/src/main/java/org/archive/modules/fetcher/FetchHTTP.java @@ -852,10 +852,10 @@ public class FetchHTTP extends Processor implements Lifecycle { hcChallengeHeaders = authStrategy.getChallenges(null, response, null); } catch (MalformedChallengeException e) { logger.fine("Failed challenge parse: " + e.getMessage()); + hcChallengeHeaders = new HashMap(); } - if (hcChallengeHeaders == null || hcChallengeHeaders.size() <= 0) { - logger.fine("Failed to get auth challenge headers for " + curi); - return null; + if (hcChallengeHeaders.size() < 1) { + logger.warning("Failed to extract auth challenge headers for uri with response status 401: " + curi); } // reorganize in non-library-specific way diff --git a/modules/src/test/java/org/archive/modules/fetcher/FetchHTTPTest.java b/modules/src/test/java/org/archive/modules/fetcher/FetchHTTPTest.java index 3fdbf03b..077a10bb 100644 --- a/modules/src/test/java/org/archive/modules/fetcher/FetchHTTPTest.java +++ b/modules/src/test/java/org/archive/modules/fetcher/FetchHTTPTest.java @@ -135,6 +135,13 @@ public class FetchHTTPTest extends ProcessorTestBase { response.setStatus(HttpServletResponse.SC_OK); response.getOutputStream().write(DEFAULT_GZIPPED_PAYLOAD); ((Request)request).setHandled(true); + } else if (target.equals("/401-no-challenge")) { + response.setStatus(401); + response.setContentType("text/plain;charset=US-ASCII"); + response.setDateHeader("Last-Modified", 0); + response.setHeader("ETag", ETAG_TEST_VALUE); + response.getOutputStream().write(DEFAULT_PAYLOAD_STRING.getBytes("US-ASCII")); + ((Request)request).setHandled(true); } else { response.setContentType("text/plain;charset=US-ASCII"); response.setDateHeader("Last-Modified", 0); diff --git a/modules/src/test/java/org/archive/modules/fetcher/FetchHTTPTests.java b/modules/src/test/java/org/archive/modules/fetcher/FetchHTTPTests.java index e961c703..4f2c44d6 100644 --- a/modules/src/test/java/org/archive/modules/fetcher/FetchHTTPTests.java +++ b/modules/src/test/java/org/archive/modules/fetcher/FetchHTTPTests.java @@ -142,7 +142,9 @@ public class FetchHTTPTests extends ProcessorTestBase { assertEquals(Charset.forName("US-ASCII"), curi.getRecorder().getCharset()); assertTrue(curi.getCredentials().isEmpty()); assertTrue(curi.getFetchDuration() >= 0); - assertTrue(curi.getFetchStatus() == 200); + if (!exclusions.contains("fetchStatus")) { + assertTrue(curi.getFetchStatus() == 200); + } assertTrue(curi.getFetchType() == FetchType.HTTP_GET); // check message body, i.e. "raw, possibly chunked-transfer-encoded message contents not including the leading headers" @@ -322,6 +324,13 @@ public class FetchHTTPTests extends ProcessorTestBase { runDefaultChecks(curi, "requestLine", "hostHeader"); } + public void test401NoChallenge() throws URIException, IOException, InterruptedException { + CrawlURI curi = makeCrawlURI("http://localhost:7777/401-no-challenge"); + fetcher().process(curi); + assertEquals(401, curi.getFetchStatus()); + runDefaultChecks(curi, "requestLine", "fetchStatus"); + } + protected void checkSetCookieURI() throws URIException, IOException, InterruptedException, UnsupportedEncodingException { CrawlURI curi = makeCrawlURI("http://localhost:7777/set-cookie");