diff --git a/modules/src/main/java/org/archive/modules/CoreAttributeConstants.java b/modules/src/main/java/org/archive/modules/CoreAttributeConstants.java index 8f6d389e..32932957 100644 --- a/modules/src/main/java/org/archive/modules/CoreAttributeConstants.java +++ b/modules/src/main/java/org/archive/modules/CoreAttributeConstants.java @@ -114,4 +114,6 @@ public interface CoreAttributeConstants { public static final String A_FTP_FETCH_STATUS = "ftp-fetch-status"; public static final String A_WHOIS_SERVER_IP = "whois-server-ip"; + + public static final String A_HTTP_AUTH_CHALLENGES = "http-auth-challenges"; } diff --git a/modules/src/main/java/org/archive/modules/CrawlURI.java b/modules/src/main/java/org/archive/modules/CrawlURI.java index 5377cce1..66fb5fb1 100644 --- a/modules/src/main/java/org/archive/modules/CrawlURI.java +++ b/modules/src/main/java/org/archive/modules/CrawlURI.java @@ -26,6 +26,7 @@ import static org.archive.modules.CoreAttributeConstants.A_FETCH_COMPLETED_TIME; import static org.archive.modules.CoreAttributeConstants.A_FORCE_RETIRE; import static org.archive.modules.CoreAttributeConstants.A_HERITABLE_KEYS; import static org.archive.modules.CoreAttributeConstants.A_HTML_BASE; +import static org.archive.modules.CoreAttributeConstants.A_HTTP_AUTH_CHALLENGES; import static org.archive.modules.CoreAttributeConstants.A_NONFATAL_ERRORS; import static org.archive.modules.CoreAttributeConstants.A_PREREQUISITE_URI; import static org.archive.modules.CoreAttributeConstants.A_SOURCE_TAG; @@ -249,7 +250,7 @@ implements Reporter, Serializable, OverlayContext { */ private static final Collection persistentKeys = new CopyOnWriteArrayList( - new String [] {A_CREDENTIALS_KEY}); + new String [] {A_CREDENTIALS_KEY, A_HTTP_AUTH_CHALLENGES}); /** maximum length for pathFromSeed/hopsPath; longer truncated with leading counter **/ private static final int MAX_HOPS_DISPLAYED = 50; @@ -1874,4 +1875,12 @@ implements Reporter, Serializable, OverlayContext { return getContentType().matches("(?i).*charset=.*"); } + @SuppressWarnings("unchecked") + public Map getHttpAuthChallenges() { + return (Map) getData().get(A_HTTP_AUTH_CHALLENGES); + } + + public void setHttpAuthChallenges(Map httpAuthChallenges) { + getData().put(A_HTTP_AUTH_CHALLENGES, httpAuthChallenges); + } } diff --git a/modules/src/main/java/org/archive/modules/credential/Credential.java b/modules/src/main/java/org/archive/modules/credential/Credential.java index 03b7895c..ca232942 100644 --- a/modules/src/main/java/org/archive/modules/credential/Credential.java +++ b/modules/src/main/java/org/archive/modules/credential/Credential.java @@ -20,6 +20,7 @@ package org.archive.modules.credential; import java.io.Serializable; import java.util.Iterator; +import java.util.Map; import java.util.logging.Logger; import javax.management.AttributeNotFoundException; @@ -167,10 +168,11 @@ public abstract class Credential implements Serializable { * @param curi CrawlURI to as for context. * @param http Instance of httpclient. * @param method Method to populate. + * @param httpAuthChallenges * @return True if added a credentials. */ public abstract boolean populate(CrawlURI curi, HttpClient http, - HttpMethod method); + HttpMethod method, Map httpAuthChallenges); /** * @param curi CrawlURI to look at. diff --git a/modules/src/main/java/org/archive/modules/credential/HtmlFormCredential.java b/modules/src/main/java/org/archive/modules/credential/HtmlFormCredential.java index 258f1820..f58099ad 100644 --- a/modules/src/main/java/org/archive/modules/credential/HtmlFormCredential.java +++ b/modules/src/main/java/org/archive/modules/credential/HtmlFormCredential.java @@ -133,7 +133,9 @@ public class HtmlFormCredential extends Credential { return false; } - public boolean populate(CrawlURI curi, HttpClient http, HttpMethod method) { + @Override + public boolean populate(CrawlURI curi, HttpClient http, HttpMethod method, + Map httpAuthChallenges) { // http is not used boolean result = false; Map formItems = getFormItems(); diff --git a/modules/src/main/java/org/archive/modules/credential/HttpAuthenticationCredential.java b/modules/src/main/java/org/archive/modules/credential/HttpAuthenticationCredential.java index 7035e1c9..bb161795 100644 --- a/modules/src/main/java/org/archive/modules/credential/HttpAuthenticationCredential.java +++ b/modules/src/main/java/org/archive/modules/credential/HttpAuthenticationCredential.java @@ -21,6 +21,7 @@ package org.archive.modules.credential; import java.util.Arrays; import java.util.Iterator; +import java.util.Map; import java.util.Set; import java.util.logging.Logger; @@ -28,8 +29,12 @@ import org.apache.commons.httpclient.HttpClient; import org.apache.commons.httpclient.HttpMethod; import org.apache.commons.httpclient.URIException; import org.apache.commons.httpclient.UsernamePasswordCredentials; +import org.apache.commons.httpclient.auth.AuthChallengeProcessor; import org.apache.commons.httpclient.auth.AuthPolicy; +import org.apache.commons.httpclient.auth.AuthScheme; import org.apache.commons.httpclient.auth.AuthScope; +import org.apache.commons.httpclient.auth.AuthenticationException; +import org.apache.commons.httpclient.auth.MalformedChallengeException; import org.archive.modules.CrawlURI; /** @@ -106,9 +111,22 @@ public class HttpAuthenticationCredential extends Credential { return true; } - public boolean populate(CrawlURI curi, HttpClient http, HttpMethod method) { + @Override + public boolean populate(CrawlURI curi, HttpClient http, HttpMethod method, + Map httpAuthChallenges) { boolean result = false; + AuthChallengeProcessor authChallengeProcessor = new AuthChallengeProcessor( + http.getParams()); + try { + AuthScheme authScheme = authChallengeProcessor.processChallenge( + method.getHostAuthState(), httpAuthChallenges); + method.getHostAuthState().setAuthScheme(authScheme); + } catch (MalformedChallengeException e) { + return result; + } catch (AuthenticationException e) { + return result; + } // Always add the credential to HttpState. Doing this because no way of // removing the credential once added AND there is a bug in the diff --git a/modules/src/main/java/org/archive/modules/fetcher/FetchHTTP.java b/modules/src/main/java/org/archive/modules/fetcher/FetchHTTP.java index 1c020759..e646ba49 100644 --- a/modules/src/main/java/org/archive/modules/fetcher/FetchHTTP.java +++ b/modules/src/main/java/org/archive/modules/fetcher/FetchHTTP.java @@ -71,10 +71,9 @@ import org.apache.commons.httpclient.HttpVersion; import org.apache.commons.httpclient.NTCredentials; import org.apache.commons.httpclient.URIException; import org.apache.commons.httpclient.auth.AuthChallengeParser; +import org.apache.commons.httpclient.auth.AuthPolicy; import org.apache.commons.httpclient.auth.AuthScheme; import org.apache.commons.httpclient.auth.AuthScope; -import org.apache.commons.httpclient.auth.BasicScheme; -import org.apache.commons.httpclient.auth.DigestScheme; import org.apache.commons.httpclient.auth.MalformedChallengeException; import org.apache.commons.httpclient.cookie.CookiePolicy; import org.apache.commons.httpclient.params.HttpClientParams; @@ -1146,7 +1145,7 @@ public class FetchHTTP extends Processor implements Lifecycle { if (server.hasCredentials()) { for (Credential cred : server.getCredentials()) { if (cred.isEveryTime()) { - cred.populate(curi, this.http, method); + cred.populate(curi, this.http, method, server.getHttpAuthChallenges()); } } } @@ -1157,7 +1156,7 @@ public class FetchHTTP extends Processor implements Lifecycle { // by the handle401 method if its a rfc2617 or it'll have been set into // the curi by the preconditionenforcer as this login uri came through. for (Credential c: curi.getCredentials()) { - if (c.populate(curi, this.http, method)) { + if (c.populate(curi, this.http, method, curi.getHttpAuthChallenges())) { result = true; } } @@ -1186,6 +1185,7 @@ public class FetchHTTP extends Processor implements Lifecycle { CrawlServer cs = serverCache.getServerFor(cd); if (cs != null) { cs.addCredential(c); + cs.setHttpAuthChallenges(curi.getHttpAuthChallenges()); } } } @@ -1270,38 +1270,39 @@ public class FetchHTTP extends Processor implements Lifecycle { return null; } - Map authschemes = null; + Map authChallenges = null; try { @SuppressWarnings("unchecked") Map parsedChallenges = AuthChallengeParser.parseChallenges(headers); - authschemes = parsedChallenges; + authChallenges = parsedChallenges; + + // remember WWW-Authenticate headers for later use + curi.setHttpAuthChallenges(authChallenges); } catch (MalformedChallengeException e) { logger.fine("Failed challenge parse: " + e.getMessage()); } - if (authschemes == null || authschemes.size() <= 0) { + if (authChallenges == null || authChallenges.size() <= 0) { logger.fine("We got a 401 and WWW-Authenticate challenge" + " but failed parse of the header " + curi.toString()); return null; } + // XXX there's a lot of overlap below with AuthChallengeProcessor.processChallenge() + AuthScheme result = null; // Use the first auth found. - for (Iterator i = authschemes.keySet().iterator(); result == null + for (Iterator i = authChallenges.keySet().iterator(); result == null && i.hasNext();) { String key = (String) i.next(); - String challenge = (String) authschemes.get(key); + String challenge = (String) authChallenges.get(key); if (key == null || key.length() <= 0 || challenge == null || challenge.length() <= 0) { logger.warning("Empty scheme: " + curi.toString() + ": " + Arrays.toString(headers)); continue; } - AuthScheme authscheme = null; - if (key.equals("basic")) { - authscheme = new BasicScheme(); - } else if (key.equals("digest")) { - authscheme = new DigestScheme(); - } else { + AuthScheme authscheme = AuthPolicy.getAuthScheme(key); + if (authscheme == null) { logger.fine("Unsupported scheme: " + key); continue; } @@ -1450,8 +1451,6 @@ public class FetchHTTP extends Processor implements Lifecycle { hcp.setSoTimeout(timeout); // Set client to be version 1.0. hcp.setVersion(HttpVersion.HTTP_1_0); - // We handle 401s, so when we do auth, we want it preemptive. - hcp.setAuthenticationPreemptive(true); // configureHttpCookies(defaults); diff --git a/modules/src/main/java/org/archive/modules/net/CrawlServer.java b/modules/src/main/java/org/archive/modules/net/CrawlServer.java index f2330b1d..5b589ef2 100644 --- a/modules/src/main/java/org/archive/modules/net/CrawlServer.java +++ b/modules/src/main/java/org/archive/modules/net/CrawlServer.java @@ -28,6 +28,7 @@ import java.io.InputStream; import java.io.InputStreamReader; import java.io.Serializable; import java.util.HashSet; +import java.util.Map; import java.util.Set; import java.util.logging.Level; import java.util.logging.Logger; @@ -348,5 +349,13 @@ public class CrawlServer implements Serializable, FetchStats.HasFetchStats, Iden @Override public void setIdentityCache(ObjectIdentityCache cache) { this.cache = cache; - } + } + + transient private Map httpAuthChallenges; + public Map getHttpAuthChallenges() { + return httpAuthChallenges; + } + public void setHttpAuthChallenges(Map httpAuthChallenges) { + this.httpAuthChallenges = httpAuthChallenges; + } }