diff --git a/modules/src/main/java/org/archive/modules/CoreAttributeConstants.java b/modules/src/main/java/org/archive/modules/CoreAttributeConstants.java index 0e5c3cc5..a42d2655 100644 --- a/modules/src/main/java/org/archive/modules/CoreAttributeConstants.java +++ b/modules/src/main/java/org/archive/modules/CoreAttributeConstants.java @@ -116,4 +116,6 @@ public interface CoreAttributeConstants { public static final String A_WHOIS_SERVER_IP = "whois-server-ip"; public static final String A_HTTP_RESPONSE_HEADERS = "http-response-headers"; + + public static final String A_HTTP_AUTH_CHALLENGES = "http-auth-challenges"; } diff --git a/modules/src/main/java/org/archive/modules/CrawlURI.java b/modules/src/main/java/org/archive/modules/CrawlURI.java index d7ceeb46..b5d009db 100644 --- a/modules/src/main/java/org/archive/modules/CrawlURI.java +++ b/modules/src/main/java/org/archive/modules/CrawlURI.java @@ -26,6 +26,7 @@ import static org.archive.modules.CoreAttributeConstants.A_FETCH_COMPLETED_TIME; import static org.archive.modules.CoreAttributeConstants.A_FORCE_RETIRE; import static org.archive.modules.CoreAttributeConstants.A_HERITABLE_KEYS; import static org.archive.modules.CoreAttributeConstants.A_HTML_BASE; +import static org.archive.modules.CoreAttributeConstants.A_HTTP_AUTH_CHALLENGES; import static org.archive.modules.CoreAttributeConstants.A_HTTP_RESPONSE_HEADERS; import static org.archive.modules.CoreAttributeConstants.A_NONFATAL_ERRORS; import static org.archive.modules.CoreAttributeConstants.A_PREREQUISITE_URI; @@ -1877,4 +1878,13 @@ implements Reporter, Serializable, OverlayContext { } httpResponseHeaders.put(key.toLowerCase(), value); } + + @SuppressWarnings("unchecked") + public Map getHttpAuthChallenges() { + return (Map) getData().get(A_HTTP_AUTH_CHALLENGES); + } + + public void setHttpAuthChallenges(Map httpAuthChallenges) { + getData().put(A_HTTP_AUTH_CHALLENGES, httpAuthChallenges); + } } diff --git a/modules/src/main/java/org/archive/modules/credential/CommonsHttpCredentialUtil.java b/modules/src/main/java/org/archive/modules/credential/CommonsHttpCredentialUtil.java index 7012634e..c360ef10 100644 --- a/modules/src/main/java/org/archive/modules/credential/CommonsHttpCredentialUtil.java +++ b/modules/src/main/java/org/archive/modules/credential/CommonsHttpCredentialUtil.java @@ -30,8 +30,12 @@ import org.apache.commons.httpclient.HttpMethodBase; import org.apache.commons.httpclient.NameValuePair; import org.apache.commons.httpclient.URIException; import org.apache.commons.httpclient.UsernamePasswordCredentials; +import org.apache.commons.httpclient.auth.AuthChallengeProcessor; import org.apache.commons.httpclient.auth.AuthPolicy; +import org.apache.commons.httpclient.auth.AuthScheme; import org.apache.commons.httpclient.auth.AuthScope; +import org.apache.commons.httpclient.auth.AuthenticationException; +import org.apache.commons.httpclient.auth.MalformedChallengeException; import org.apache.commons.httpclient.methods.GetMethod; import org.apache.commons.httpclient.methods.PostMethod; import org.apache.commons.lang.StringUtils; @@ -42,9 +46,9 @@ public class CommonsHttpCredentialUtil { private static Logger logger = Logger.getLogger(CommonsHttpCredentialUtil.class.getName()); public static boolean populate(CrawlURI curi, HttpClient http, - HttpMethod method, Credential cred) { + HttpMethod method, Credential cred, Map httpAuthChallenges) { if (cred instanceof HttpAuthenticationCredential) { - return populate(curi, http, method, (HttpAuthenticationCredential) cred); + return populate(curi, http, method, (HttpAuthenticationCredential) cred, httpAuthChallenges); } else if (cred instanceof HtmlFormCredential) { return populate(curi, http, method, (HtmlFormCredential) cred); } else { @@ -99,8 +103,18 @@ public class CommonsHttpCredentialUtil { } public static boolean populate(CrawlURI curi, HttpClient http, - HttpMethod method, HttpAuthenticationCredential cred) { + HttpMethod method, HttpAuthenticationCredential cred, Map httpAuthChallenges) { boolean result = false; + + AuthChallengeProcessor authChallengeProcessor = new AuthChallengeProcessor(http.getParams()); + try { + AuthScheme authScheme = authChallengeProcessor.processChallenge(method.getHostAuthState(), httpAuthChallenges); + method.getHostAuthState().setAuthScheme(authScheme); + } catch (MalformedChallengeException e) { + return result; + } catch (AuthenticationException e) { + return result; + } // Always add the credential to HttpState. Doing this because no way of // removing the credential once added AND there is a bug in the diff --git a/modules/src/main/java/org/archive/modules/fetcher/FetchHTTP.java b/modules/src/main/java/org/archive/modules/fetcher/FetchHTTP.java index d80eea02..9b000e97 100644 --- a/modules/src/main/java/org/archive/modules/fetcher/FetchHTTP.java +++ b/modules/src/main/java/org/archive/modules/fetcher/FetchHTTP.java @@ -71,10 +71,9 @@ import org.apache.commons.httpclient.HttpVersion; import org.apache.commons.httpclient.NTCredentials; import org.apache.commons.httpclient.URIException; import org.apache.commons.httpclient.auth.AuthChallengeParser; +import org.apache.commons.httpclient.auth.AuthPolicy; import org.apache.commons.httpclient.auth.AuthScheme; import org.apache.commons.httpclient.auth.AuthScope; -import org.apache.commons.httpclient.auth.BasicScheme; -import org.apache.commons.httpclient.auth.DigestScheme; import org.apache.commons.httpclient.auth.MalformedChallengeException; import org.apache.commons.httpclient.cookie.CookiePolicy; import org.apache.commons.httpclient.params.HttpClientParams; @@ -1153,7 +1152,7 @@ public class FetchHTTP extends AbstractFetchHTTP implements Lifecycle { if (server.hasCredentials()) { for (Credential cred : server.getCredentials()) { if (cred.isEveryTime()) { - CommonsHttpCredentialUtil.populate(curi, this.http, method, cred); + CommonsHttpCredentialUtil.populate(curi, this.http, method, cred, server.getHttpAuthChallenges()); } } } @@ -1164,7 +1163,7 @@ public class FetchHTTP extends AbstractFetchHTTP implements Lifecycle { // by the handle401 method if its a rfc2617 or it'll have been set into // the curi by the preconditionenforcer as this login uri came through. for (Credential c: curi.getCredentials()) { - if (CommonsHttpCredentialUtil.populate(curi, this.http, method, c)) { + if (CommonsHttpCredentialUtil.populate(curi, this.http, method, c, curi.getHttpAuthChallenges())) { result = true; } } @@ -1193,6 +1192,7 @@ public class FetchHTTP extends AbstractFetchHTTP implements Lifecycle { CrawlServer cs = serverCache.getServerFor(cd); if (cs != null) { cs.addCredential(c); + cs.setHttpAuthChallenges(curi.getHttpAuthChallenges()); } } } @@ -1282,6 +1282,8 @@ public class FetchHTTP extends AbstractFetchHTTP implements Lifecycle { @SuppressWarnings("unchecked") Map parsedChallenges = AuthChallengeParser.parseChallenges(headers); authschemes = parsedChallenges; + + curi.setHttpAuthChallenges(authschemes); } catch (MalformedChallengeException e) { logger.fine("Failed challenge parse: " + e.getMessage()); } @@ -1303,12 +1305,8 @@ public class FetchHTTP extends AbstractFetchHTTP implements Lifecycle { + Arrays.toString(headers)); continue; } - AuthScheme authscheme = null; - if (key.equals("basic")) { - authscheme = new BasicScheme(); - } else if (key.equals("digest")) { - authscheme = new DigestScheme(); - } else { + AuthScheme authscheme = AuthPolicy.getAuthScheme(key); + if (authscheme == null) { logger.fine("Unsupported scheme: " + key); continue; } @@ -1458,7 +1456,7 @@ public class FetchHTTP extends AbstractFetchHTTP implements Lifecycle { // Set client to be version 1.0. hcp.setVersion(HttpVersion.HTTP_1_0); // We handle 401s, so when we do auth, we want it preemptive. - hcp.setAuthenticationPreemptive(true); + // hcp.setAuthenticationPreemptive(true); // configureHttpCookies(defaults); diff --git a/modules/src/main/java/org/archive/modules/net/CrawlServer.java b/modules/src/main/java/org/archive/modules/net/CrawlServer.java index f2330b1d..b42f7b31 100644 --- a/modules/src/main/java/org/archive/modules/net/CrawlServer.java +++ b/modules/src/main/java/org/archive/modules/net/CrawlServer.java @@ -28,6 +28,7 @@ import java.io.InputStream; import java.io.InputStreamReader; import java.io.Serializable; import java.util.HashSet; +import java.util.Map; import java.util.Set; import java.util.logging.Level; import java.util.logging.Logger; @@ -348,5 +349,13 @@ public class CrawlServer implements Serializable, FetchStats.HasFetchStats, Iden @Override public void setIdentityCache(ObjectIdentityCache cache) { this.cache = cache; - } + } + + transient private Map httpAuthChallenges; + public Map getHttpAuthChallenges() { + return httpAuthChallenges; + } + public void setHttpAuthChallenges(Map httpAuthChallenges) { + this.httpAuthChallenges = httpAuthChallenges; + } } diff --git a/modules/src/test/java/org/archive/modules/fetcher/FetchHTTPTestBase.java b/modules/src/test/java/org/archive/modules/fetcher/FetchHTTPTestBase.java index cd665e2d..3050542d 100644 --- a/modules/src/test/java/org/archive/modules/fetcher/FetchHTTPTestBase.java +++ b/modules/src/test/java/org/archive/modules/fetcher/FetchHTTPTestBase.java @@ -411,6 +411,7 @@ public abstract class FetchHTTPTestBase extends ProcessorTestBase { // check that we got the expected response and the fetcher did its thing assertEquals(401, curi.getFetchStatus()); assertTrue(curi.getCredentials().contains(basicAuthCredential)); + assertTrue(curi.getHttpAuthChallenges() != null && curi.getHttpAuthChallenges().containsKey("basic")); // fetch again with the credentials getFetcher().process(curi); @@ -448,16 +449,18 @@ public abstract class FetchHTTPTestBase extends ProcessorTestBase { // check that we got the expected response and the fetcher did its thing assertEquals(401, curi.getFetchStatus()); assertTrue(curi.getCredentials().contains(digestAuthCred)); + assertTrue(curi.getHttpAuthChallenges() != null && curi.getHttpAuthChallenges().containsKey("digest")); // stick a basic auth 401 in there to check it doesn't mess with the digest auth we're working on CrawlURI interferingUri = makeCrawlURI("http://localhost:7777/auth/basic"); getFetcher().process(interferingUri); assertEquals(401, interferingUri.getFetchStatus()); + logger.info('\n' + httpRequestString(interferingUri) + "\n\n" + rawResponseString(interferingUri)); // fetch original again with the credentials getFetcher().process(curi); String httpRequestString = httpRequestString(curi); - // logger.info('\n' + httpRequestString + contentString(curi)); + logger.info('\n' + httpRequestString + "\n\n" + rawResponseString(interferingUri)); assertTrue(httpRequestString.contains("Authorization: Digest")); // otherwise should be a normal 200 response runDefaultChecks(curi, new HashSet(Arrays.asList("requestLine", "hostHeader"))); @@ -468,7 +471,7 @@ public abstract class FetchHTTPTestBase extends ProcessorTestBase { httpRequestString = httpRequestString(curi); assertTrue(httpRequestString.contains("Authorization: Digest")); // otherwise should be a normal 200 response - runDefaultChecks(curi, new HashSet(Arrays.asList("requestLine"))); + runDefaultChecks(curi, new HashSet(Arrays.asList("requestLine", "hostHeader"))); } // server for form auth is at localhost:7779 @@ -496,7 +499,7 @@ public abstract class FetchHTTPTestBase extends ProcessorTestBase { // login (if not we get a NPE within jetty) curi = makeCrawlURI("http://localhost:7779/auth/1"); getFetcher().process(curi); - logger.info('\n' + httpRequestString(curi) + rawResponseString(curi)); + logger.info('\n' + httpRequestString(curi) + "\n\n" + rawResponseString(curi)); assertEquals(302, curi.getFetchStatus()); assertTrue(curi.getHttpResponseHeader("Location").startsWith("http://localhost:7779/login.html"));