From 0144df024e44db1295c7f84f7637590cf1ffe769 Mon Sep 17 00:00:00 2001 From: Noah Levitt Date: Thu, 16 Aug 2012 12:17:01 -0700 Subject: [PATCH] Real fix for HER-2016 FetchHTTP tries every url twice without credentials before sending credentials: Remember WWW-Authenticate headers, aka auth challenges, with CrawlURI, and after successful auth, with the CrawlServer. It's necessary to do this to know what Authorization header to send without getting another 401 first. * CoreAttributeConstants.java new constant A_HTTP_AUTH_CHALLENGES key for CrawlURI.getData() for http auth challenges * CrawlURI.java add A_HTTP_AUTH_CHALLENGES to keys to remember across URI processing; convenience getter and setter for auth challenges * Credential.java, HtmlFormCredential.java, HttpAuthenticationCredential.java add httpAuthChallenges argument to populate() * HttpAuthenticationCredential.java populate() - process httpAuthChallenges to set up auth state of http client to send the right auth header * CrawlServer.java cache httpAuthChallenges * FetchHTTP.java getAuthScheme() - cache auth challenges with CrawlURI configureHttp() - do not set authentication preemptive - this is handled by the configuration done in HttpAuthenticationCredential.populate() promoteCredentials() - cache auth challenges with the CrawlServer --- .../modules/CoreAttributeConstants.java | 2 ++ .../java/org/archive/modules/CrawlURI.java | 11 ++++++- .../modules/credential/Credential.java | 4 ++- .../credential/HtmlFormCredential.java | 4 ++- .../HttpAuthenticationCredential.java | 20 ++++++++++- .../archive/modules/fetcher/FetchHTTP.java | 33 +++++++++---------- .../org/archive/modules/net/CrawlServer.java | 11 ++++++- 7 files changed, 63 insertions(+), 22 deletions(-) diff --git a/modules/src/main/java/org/archive/modules/CoreAttributeConstants.java b/modules/src/main/java/org/archive/modules/CoreAttributeConstants.java index 8f6d389e..32932957 100644 --- a/modules/src/main/java/org/archive/modules/CoreAttributeConstants.java +++ b/modules/src/main/java/org/archive/modules/CoreAttributeConstants.java @@ -114,4 +114,6 @@ public interface CoreAttributeConstants { public static final String A_FTP_FETCH_STATUS = "ftp-fetch-status"; public static final String A_WHOIS_SERVER_IP = "whois-server-ip"; + + public static final String A_HTTP_AUTH_CHALLENGES = "http-auth-challenges"; } diff --git a/modules/src/main/java/org/archive/modules/CrawlURI.java b/modules/src/main/java/org/archive/modules/CrawlURI.java index 5377cce1..66fb5fb1 100644 --- a/modules/src/main/java/org/archive/modules/CrawlURI.java +++ b/modules/src/main/java/org/archive/modules/CrawlURI.java @@ -26,6 +26,7 @@ import static org.archive.modules.CoreAttributeConstants.A_FETCH_COMPLETED_TIME; import static org.archive.modules.CoreAttributeConstants.A_FORCE_RETIRE; import static org.archive.modules.CoreAttributeConstants.A_HERITABLE_KEYS; import static org.archive.modules.CoreAttributeConstants.A_HTML_BASE; +import static org.archive.modules.CoreAttributeConstants.A_HTTP_AUTH_CHALLENGES; import static org.archive.modules.CoreAttributeConstants.A_NONFATAL_ERRORS; import static org.archive.modules.CoreAttributeConstants.A_PREREQUISITE_URI; import static org.archive.modules.CoreAttributeConstants.A_SOURCE_TAG; @@ -249,7 +250,7 @@ implements Reporter, Serializable, OverlayContext { */ private static final Collection persistentKeys = new CopyOnWriteArrayList( - new String [] {A_CREDENTIALS_KEY}); + new String [] {A_CREDENTIALS_KEY, A_HTTP_AUTH_CHALLENGES}); /** maximum length for pathFromSeed/hopsPath; longer truncated with leading counter **/ private static final int MAX_HOPS_DISPLAYED = 50; @@ -1874,4 +1875,12 @@ implements Reporter, Serializable, OverlayContext { return getContentType().matches("(?i).*charset=.*"); } + @SuppressWarnings("unchecked") + public Map getHttpAuthChallenges() { + return (Map) getData().get(A_HTTP_AUTH_CHALLENGES); + } + + public void setHttpAuthChallenges(Map httpAuthChallenges) { + getData().put(A_HTTP_AUTH_CHALLENGES, httpAuthChallenges); + } } diff --git a/modules/src/main/java/org/archive/modules/credential/Credential.java b/modules/src/main/java/org/archive/modules/credential/Credential.java index 03b7895c..ca232942 100644 --- a/modules/src/main/java/org/archive/modules/credential/Credential.java +++ b/modules/src/main/java/org/archive/modules/credential/Credential.java @@ -20,6 +20,7 @@ package org.archive.modules.credential; import java.io.Serializable; import java.util.Iterator; +import java.util.Map; import java.util.logging.Logger; import javax.management.AttributeNotFoundException; @@ -167,10 +168,11 @@ public abstract class Credential implements Serializable { * @param curi CrawlURI to as for context. * @param http Instance of httpclient. * @param method Method to populate. + * @param httpAuthChallenges * @return True if added a credentials. */ public abstract boolean populate(CrawlURI curi, HttpClient http, - HttpMethod method); + HttpMethod method, Map httpAuthChallenges); /** * @param curi CrawlURI to look at. diff --git a/modules/src/main/java/org/archive/modules/credential/HtmlFormCredential.java b/modules/src/main/java/org/archive/modules/credential/HtmlFormCredential.java index 258f1820..f58099ad 100644 --- a/modules/src/main/java/org/archive/modules/credential/HtmlFormCredential.java +++ b/modules/src/main/java/org/archive/modules/credential/HtmlFormCredential.java @@ -133,7 +133,9 @@ public class HtmlFormCredential extends Credential { return false; } - public boolean populate(CrawlURI curi, HttpClient http, HttpMethod method) { + @Override + public boolean populate(CrawlURI curi, HttpClient http, HttpMethod method, + Map httpAuthChallenges) { // http is not used boolean result = false; Map formItems = getFormItems(); diff --git a/modules/src/main/java/org/archive/modules/credential/HttpAuthenticationCredential.java b/modules/src/main/java/org/archive/modules/credential/HttpAuthenticationCredential.java index 7035e1c9..bb161795 100644 --- a/modules/src/main/java/org/archive/modules/credential/HttpAuthenticationCredential.java +++ b/modules/src/main/java/org/archive/modules/credential/HttpAuthenticationCredential.java @@ -21,6 +21,7 @@ package org.archive.modules.credential; import java.util.Arrays; import java.util.Iterator; +import java.util.Map; import java.util.Set; import java.util.logging.Logger; @@ -28,8 +29,12 @@ import org.apache.commons.httpclient.HttpClient; import org.apache.commons.httpclient.HttpMethod; import org.apache.commons.httpclient.URIException; import org.apache.commons.httpclient.UsernamePasswordCredentials; +import org.apache.commons.httpclient.auth.AuthChallengeProcessor; import org.apache.commons.httpclient.auth.AuthPolicy; +import org.apache.commons.httpclient.auth.AuthScheme; import org.apache.commons.httpclient.auth.AuthScope; +import org.apache.commons.httpclient.auth.AuthenticationException; +import org.apache.commons.httpclient.auth.MalformedChallengeException; import org.archive.modules.CrawlURI; /** @@ -106,9 +111,22 @@ public class HttpAuthenticationCredential extends Credential { return true; } - public boolean populate(CrawlURI curi, HttpClient http, HttpMethod method) { + @Override + public boolean populate(CrawlURI curi, HttpClient http, HttpMethod method, + Map httpAuthChallenges) { boolean result = false; + AuthChallengeProcessor authChallengeProcessor = new AuthChallengeProcessor( + http.getParams()); + try { + AuthScheme authScheme = authChallengeProcessor.processChallenge( + method.getHostAuthState(), httpAuthChallenges); + method.getHostAuthState().setAuthScheme(authScheme); + } catch (MalformedChallengeException e) { + return result; + } catch (AuthenticationException e) { + return result; + } // Always add the credential to HttpState. Doing this because no way of // removing the credential once added AND there is a bug in the diff --git a/modules/src/main/java/org/archive/modules/fetcher/FetchHTTP.java b/modules/src/main/java/org/archive/modules/fetcher/FetchHTTP.java index 1c020759..e646ba49 100644 --- a/modules/src/main/java/org/archive/modules/fetcher/FetchHTTP.java +++ b/modules/src/main/java/org/archive/modules/fetcher/FetchHTTP.java @@ -71,10 +71,9 @@ import org.apache.commons.httpclient.HttpVersion; import org.apache.commons.httpclient.NTCredentials; import org.apache.commons.httpclient.URIException; import org.apache.commons.httpclient.auth.AuthChallengeParser; +import org.apache.commons.httpclient.auth.AuthPolicy; import org.apache.commons.httpclient.auth.AuthScheme; import org.apache.commons.httpclient.auth.AuthScope; -import org.apache.commons.httpclient.auth.BasicScheme; -import org.apache.commons.httpclient.auth.DigestScheme; import org.apache.commons.httpclient.auth.MalformedChallengeException; import org.apache.commons.httpclient.cookie.CookiePolicy; import org.apache.commons.httpclient.params.HttpClientParams; @@ -1146,7 +1145,7 @@ public class FetchHTTP extends Processor implements Lifecycle { if (server.hasCredentials()) { for (Credential cred : server.getCredentials()) { if (cred.isEveryTime()) { - cred.populate(curi, this.http, method); + cred.populate(curi, this.http, method, server.getHttpAuthChallenges()); } } } @@ -1157,7 +1156,7 @@ public class FetchHTTP extends Processor implements Lifecycle { // by the handle401 method if its a rfc2617 or it'll have been set into // the curi by the preconditionenforcer as this login uri came through. for (Credential c: curi.getCredentials()) { - if (c.populate(curi, this.http, method)) { + if (c.populate(curi, this.http, method, curi.getHttpAuthChallenges())) { result = true; } } @@ -1186,6 +1185,7 @@ public class FetchHTTP extends Processor implements Lifecycle { CrawlServer cs = serverCache.getServerFor(cd); if (cs != null) { cs.addCredential(c); + cs.setHttpAuthChallenges(curi.getHttpAuthChallenges()); } } } @@ -1270,38 +1270,39 @@ public class FetchHTTP extends Processor implements Lifecycle { return null; } - Map authschemes = null; + Map authChallenges = null; try { @SuppressWarnings("unchecked") Map parsedChallenges = AuthChallengeParser.parseChallenges(headers); - authschemes = parsedChallenges; + authChallenges = parsedChallenges; + + // remember WWW-Authenticate headers for later use + curi.setHttpAuthChallenges(authChallenges); } catch (MalformedChallengeException e) { logger.fine("Failed challenge parse: " + e.getMessage()); } - if (authschemes == null || authschemes.size() <= 0) { + if (authChallenges == null || authChallenges.size() <= 0) { logger.fine("We got a 401 and WWW-Authenticate challenge" + " but failed parse of the header " + curi.toString()); return null; } + // XXX there's a lot of overlap below with AuthChallengeProcessor.processChallenge() + AuthScheme result = null; // Use the first auth found. - for (Iterator i = authschemes.keySet().iterator(); result == null + for (Iterator i = authChallenges.keySet().iterator(); result == null && i.hasNext();) { String key = (String) i.next(); - String challenge = (String) authschemes.get(key); + String challenge = (String) authChallenges.get(key); if (key == null || key.length() <= 0 || challenge == null || challenge.length() <= 0) { logger.warning("Empty scheme: " + curi.toString() + ": " + Arrays.toString(headers)); continue; } - AuthScheme authscheme = null; - if (key.equals("basic")) { - authscheme = new BasicScheme(); - } else if (key.equals("digest")) { - authscheme = new DigestScheme(); - } else { + AuthScheme authscheme = AuthPolicy.getAuthScheme(key); + if (authscheme == null) { logger.fine("Unsupported scheme: " + key); continue; } @@ -1450,8 +1451,6 @@ public class FetchHTTP extends Processor implements Lifecycle { hcp.setSoTimeout(timeout); // Set client to be version 1.0. hcp.setVersion(HttpVersion.HTTP_1_0); - // We handle 401s, so when we do auth, we want it preemptive. - hcp.setAuthenticationPreemptive(true); // configureHttpCookies(defaults); diff --git a/modules/src/main/java/org/archive/modules/net/CrawlServer.java b/modules/src/main/java/org/archive/modules/net/CrawlServer.java index f2330b1d..5b589ef2 100644 --- a/modules/src/main/java/org/archive/modules/net/CrawlServer.java +++ b/modules/src/main/java/org/archive/modules/net/CrawlServer.java @@ -28,6 +28,7 @@ import java.io.InputStream; import java.io.InputStreamReader; import java.io.Serializable; import java.util.HashSet; +import java.util.Map; import java.util.Set; import java.util.logging.Level; import java.util.logging.Logger; @@ -348,5 +349,13 @@ public class CrawlServer implements Serializable, FetchStats.HasFetchStats, Iden @Override public void setIdentityCache(ObjectIdentityCache cache) { this.cache = cache; - } + } + + transient private Map httpAuthChallenges; + public Map getHttpAuthChallenges() { + return httpAuthChallenges; + } + public void setHttpAuthChallenges(Map httpAuthChallenges) { + this.httpAuthChallenges = httpAuthChallenges; + } }