lee/heritrix3
1
0
Fork 0
mirror of https://github.com/internetarchive/heritrix3.git synced 2026-08-26 08:26:53 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #34 from nlevitt/401-no-challenge

Browse Source 
fix NullPointerException in case of 401 with no auth challenge (includes...
This commit is contained in:
Adam Miller
2014-01-27 11:52:51 -08:00
parent b1a28358d7 26a5dfebdc
commit 21f04cdf79
3 changed files with 20 additions and 4 deletions
Download Patch File Download Diff File Expand all files Collapse all files
modules/src/main/java/org/archive/modules/fetcher/FetchHTTP.java
+3 -3
View File
@@ -852,10 +852,10 @@ public class FetchHTTP extends Processor implements Lifecycle {
hcChallengeHeaders = authStrategy.getChallenges(null, response, null);
} catch (MalformedChallengeException e) {
logger.fine("Failed challenge parse: " + e.getMessage());
hcChallengeHeaders = new HashMap<String, Header>();
}
if (hcChallengeHeaders == null || hcChallengeHeaders.size() <= 0) {
logger.fine("Failed to get auth challenge headers for " + curi);
return null;
if (hcChallengeHeaders.size() < 1) {
logger.warning("Failed to extract auth challenge headers for uri with response status 401: " + curi);
}
// reorganize in non-library-specific way
modules/src/test/java/org/archive/modules/fetcher/FetchHTTPTest.java
+7
View File
@@ -135,6 +135,13 @@ public class FetchHTTPTest extends ProcessorTestBase {
response.setStatus(HttpServletResponse.SC_OK);
response.getOutputStream().write(DEFAULT_GZIPPED_PAYLOAD);
((Request)request).setHandled(true);
} else if (target.equals("/401-no-challenge")) {
response.setStatus(401);
response.setContentType("text/plain;charset=US-ASCII");
response.setDateHeader("Last-Modified", 0);
response.setHeader("ETag", ETAG_TEST_VALUE);
response.getOutputStream().write(DEFAULT_PAYLOAD_STRING.getBytes("US-ASCII"));
((Request)request).setHandled(true);
} else {
response.setContentType("text/plain;charset=US-ASCII");
response.setDateHeader("Last-Modified", 0);
modules/src/test/java/org/archive/modules/fetcher/FetchHTTPTests.java
+10 -1
View File
@@ -142,7 +142,9 @@ public class FetchHTTPTests extends ProcessorTestBase {
assertEquals(Charset.forName("US-ASCII"), curi.getRecorder().getCharset());
assertTrue(curi.getCredentials().isEmpty());
assertTrue(curi.getFetchDuration() >= 0);
assertTrue(curi.getFetchStatus() == 200);
if (!exclusions.contains("fetchStatus")) {
assertTrue(curi.getFetchStatus() == 200);
}
assertTrue(curi.getFetchType() == FetchType.HTTP_GET);
// check message body, i.e. "raw, possibly chunked-transfer-encoded message contents not including the leading headers"
@@ -322,6 +324,13 @@ public class FetchHTTPTests extends ProcessorTestBase {
runDefaultChecks(curi, "requestLine", "hostHeader");
}
public void test401NoChallenge() throws URIException, IOException, InterruptedException {
CrawlURI curi = makeCrawlURI("http://localhost:7777/401-no-challenge");
fetcher().process(curi);
assertEquals(401, curi.getFetchStatus());
runDefaultChecks(curi, "requestLine", "fetchStatus");
}
protected void checkSetCookieURI() throws URIException, IOException,
InterruptedException, UnsupportedEncodingException {
CrawlURI curi = makeCrawlURI("http://localhost:7777/set-cookie");