Real fix for HER-2016 FetchHTTP tries every url twice without credentials before sending credentials: Remember WWW-Authenticate headers, aka auth challenges, with CrawlURI, and after successful auth, with the CrawlServer. It's necessary to do this to know what Authorization header to send without getting another 401 first.

* CoreAttributeConstants.java
    new constant A_HTTP_AUTH_CHALLENGES key for CrawlURI.getData() for http auth challenges
* CrawlURI.java
    add A_HTTP_AUTH_CHALLENGES to keys to remember across URI processing; convenience getter and setter for auth challenges
* Credential.java, HtmlFormCredential.java, HttpAuthenticationCredential.java
    add httpAuthChallenges argument to populate()
* HttpAuthenticationCredential.java
    populate() - process httpAuthChallenges to set up auth state of http client to send the right auth header
* CrawlServer.java
    cache httpAuthChallenges
* FetchHTTP.java
    getAuthScheme() - cache auth challenges with CrawlURI
    configureHttp() - do not set authentication preemptive - this is handled by the configuration done in HttpAuthenticationCredential.populate()
    promoteCredentials() - cache auth challenges with the CrawlServer

modules/src/main/java/org/archive/modules/CoreAttributeConstants.java

@@ -114,4 +114,6 @@ public interface CoreAttributeConstants {
     public static final String A_FTP_FETCH_STATUS = "ftp-fetch-status";
 
     public static final String A_WHOIS_SERVER_IP = "whois-server-ip";
+    
+    public static final String A_HTTP_AUTH_CHALLENGES = "http-auth-challenges";
 }

modules/src/main/java/org/archive/modules/CrawlURI.java

@@ -26,6 +26,7 @@ import static org.archive.modules.CoreAttributeConstants.A_FETCH_COMPLETED_TIME;
 import static org.archive.modules.CoreAttributeConstants.A_FORCE_RETIRE;
 import static org.archive.modules.CoreAttributeConstants.A_HERITABLE_KEYS;
 import static org.archive.modules.CoreAttributeConstants.A_HTML_BASE;
+import static org.archive.modules.CoreAttributeConstants.A_HTTP_AUTH_CHALLENGES;
 import static org.archive.modules.CoreAttributeConstants.A_NONFATAL_ERRORS;
 import static org.archive.modules.CoreAttributeConstants.A_PREREQUISITE_URI;
 import static org.archive.modules.CoreAttributeConstants.A_SOURCE_TAG;
@@ -249,7 +250,7 @@ implements Reporter, Serializable, OverlayContext {
      */
     private static final Collection<String> persistentKeys
      = new CopyOnWriteArrayList<String>(
-            new String [] {A_CREDENTIALS_KEY});
+            new String [] {A_CREDENTIALS_KEY, A_HTTP_AUTH_CHALLENGES});
 
     /** maximum length for pathFromSeed/hopsPath; longer truncated with leading counter **/ 
     private static final int MAX_HOPS_DISPLAYED = 50;
@@ -1874,4 +1875,12 @@ implements Reporter, Serializable, OverlayContext {
         return getContentType().matches("(?i).*charset=.*");
     }
 
+    @SuppressWarnings("unchecked")
+    public Map<String,String> getHttpAuthChallenges() {
+        return (Map<String, String>) getData().get(A_HTTP_AUTH_CHALLENGES);
+    }
+
+    public void setHttpAuthChallenges(Map<String, String> httpAuthChallenges) {
+        getData().put(A_HTTP_AUTH_CHALLENGES, httpAuthChallenges);
+    }
 }

modules/src/main/java/org/archive/modules/credential/Credential.java

@@ -20,6 +20,7 @@ package org.archive.modules.credential;
 
 import java.io.Serializable;
 import java.util.Iterator;
+import java.util.Map;
 import java.util.logging.Logger;
 
 import javax.management.AttributeNotFoundException;
@@ -167,10 +168,11 @@ public abstract class Credential implements Serializable {
      * @param curi CrawlURI to as for context.
      * @param http Instance of httpclient.
      * @param method Method to populate.
+     * @param httpAuthChallenges 
      * @return True if added a credentials.
      */
     public abstract boolean populate(CrawlURI curi, HttpClient http,
-        HttpMethod method);
+        HttpMethod method, Map<String, String> httpAuthChallenges);
 
     /**
      * @param curi CrawlURI to look at.

modules/src/main/java/org/archive/modules/credential/HtmlFormCredential.java

@@ -133,7 +133,9 @@ public class HtmlFormCredential extends Credential {
         return false;
     }
 
-    public boolean populate(CrawlURI curi, HttpClient http, HttpMethod method) {
+    @Override
+    public boolean populate(CrawlURI curi, HttpClient http, HttpMethod method,
+            Map<String, String> httpAuthChallenges) {
         // http is not used
         boolean result = false;
         Map<String,String> formItems = getFormItems();

modules/src/main/java/org/archive/modules/credential/HttpAuthenticationCredential.java

@@ -21,6 +21,7 @@ package org.archive.modules.credential;
 
 import java.util.Arrays;
 import java.util.Iterator;
+import java.util.Map;
 import java.util.Set;
 import java.util.logging.Logger;
 
@@ -28,8 +29,12 @@ import org.apache.commons.httpclient.HttpClient;
 import org.apache.commons.httpclient.HttpMethod;
 import org.apache.commons.httpclient.URIException;
 import org.apache.commons.httpclient.UsernamePasswordCredentials;
+import org.apache.commons.httpclient.auth.AuthChallengeProcessor;
 import org.apache.commons.httpclient.auth.AuthPolicy;
+import org.apache.commons.httpclient.auth.AuthScheme;
 import org.apache.commons.httpclient.auth.AuthScope;
+import org.apache.commons.httpclient.auth.AuthenticationException;
+import org.apache.commons.httpclient.auth.MalformedChallengeException;
 import org.archive.modules.CrawlURI;
 
 /**
@@ -106,9 +111,22 @@ public class HttpAuthenticationCredential extends Credential {
         return true;
     }
 
-    public boolean populate(CrawlURI curi, HttpClient http, HttpMethod method) {
+    @Override
+    public boolean populate(CrawlURI curi, HttpClient http, HttpMethod method,
+            Map<String, String> httpAuthChallenges) {
         boolean result = false;
 
+        AuthChallengeProcessor authChallengeProcessor = new AuthChallengeProcessor(
+                http.getParams());
+        try {
+            AuthScheme authScheme = authChallengeProcessor.processChallenge(
+                    method.getHostAuthState(), httpAuthChallenges);
+            method.getHostAuthState().setAuthScheme(authScheme);
+        } catch (MalformedChallengeException e) {
+            return result;
+        } catch (AuthenticationException e) {
+            return result;
+        }
 
         // Always add the credential to HttpState. Doing this because no way of
         // removing the credential once added AND there is a bug in the

modules/src/main/java/org/archive/modules/fetcher/FetchHTTP.java

@@ -71,10 +71,9 @@ import org.apache.commons.httpclient.HttpVersion;
 import org.apache.commons.httpclient.NTCredentials;
 import org.apache.commons.httpclient.URIException;
 import org.apache.commons.httpclient.auth.AuthChallengeParser;
+import org.apache.commons.httpclient.auth.AuthPolicy;
 import org.apache.commons.httpclient.auth.AuthScheme;
 import org.apache.commons.httpclient.auth.AuthScope;
-import org.apache.commons.httpclient.auth.BasicScheme;
-import org.apache.commons.httpclient.auth.DigestScheme;
 import org.apache.commons.httpclient.auth.MalformedChallengeException;
 import org.apache.commons.httpclient.cookie.CookiePolicy;
 import org.apache.commons.httpclient.params.HttpClientParams;
@@ -1146,7 +1145,7 @@ public class FetchHTTP extends Processor implements Lifecycle {
         if (server.hasCredentials()) {
             for (Credential cred : server.getCredentials()) {
                 if (cred.isEveryTime()) {
-                    cred.populate(curi, this.http, method);
+                    cred.populate(curi, this.http, method, server.getHttpAuthChallenges());
                 }
             }
         }
@@ -1157,7 +1156,7 @@ public class FetchHTTP extends Processor implements Lifecycle {
         // by the handle401 method if its a rfc2617 or it'll have been set into
         // the curi by the preconditionenforcer as this login uri came through.
         for (Credential c: curi.getCredentials()) {
-            if (c.populate(curi, this.http, method)) {
+            if (c.populate(curi, this.http, method, curi.getHttpAuthChallenges())) {
                 result = true;
             }
         }
@@ -1186,6 +1185,7 @@ public class FetchHTTP extends Processor implements Lifecycle {
                 CrawlServer cs = serverCache.getServerFor(cd);
                 if (cs != null) {
                     cs.addCredential(c);
+                    cs.setHttpAuthChallenges(curi.getHttpAuthChallenges());
                 }
             }
         }
@@ -1270,38 +1270,39 @@ public class FetchHTTP extends Processor implements Lifecycle {
             return null;
         }
 
-        Map<String, String> authschemes = null;
+        Map<String, String> authChallenges = null;
         try {
             @SuppressWarnings("unchecked")
             Map<String, String> parsedChallenges = AuthChallengeParser.parseChallenges(headers);
-            authschemes = parsedChallenges;
+            authChallenges = parsedChallenges;
+            
+            // remember WWW-Authenticate headers for later use 
+            curi.setHttpAuthChallenges(authChallenges);
         } catch (MalformedChallengeException e) {
             logger.fine("Failed challenge parse: " + e.getMessage());
         }
-        if (authschemes == null || authschemes.size() <= 0) {
+        if (authChallenges == null || authChallenges.size() <= 0) {
             logger.fine("We got a 401 and WWW-Authenticate challenge"
                     + " but failed parse of the header " + curi.toString());
             return null;
         }
 
+        // XXX there's a lot of overlap below with AuthChallengeProcessor.processChallenge()
+        
         AuthScheme result = null;
         // Use the first auth found.
-        for (Iterator<String> i = authschemes.keySet().iterator(); result == null
+        for (Iterator<String> i = authChallenges.keySet().iterator(); result == null
                 && i.hasNext();) {
             String key = (String) i.next();
-            String challenge = (String) authschemes.get(key);
+            String challenge = (String) authChallenges.get(key);
             if (key == null || key.length() <= 0 || challenge == null
                     || challenge.length() <= 0) {
                 logger.warning("Empty scheme: " + curi.toString() + ": "
                         + Arrays.toString(headers));
                 continue;
             }
-            AuthScheme authscheme = null;
-            if (key.equals("basic")) {
-                authscheme = new BasicScheme();
-            } else if (key.equals("digest")) {
-                authscheme = new DigestScheme();
-            } else {
+            AuthScheme authscheme = AuthPolicy.getAuthScheme(key);
+            if (authscheme == null) {
                 logger.fine("Unsupported scheme: " + key);
                 continue;
             }
@@ -1450,8 +1451,6 @@ public class FetchHTTP extends Processor implements Lifecycle {
         hcp.setSoTimeout(timeout);
         // Set client to be version 1.0.
         hcp.setVersion(HttpVersion.HTTP_1_0);
-        // We handle 401s, so when we do auth, we want it preemptive.
-        hcp.setAuthenticationPreemptive(true);
 
         // configureHttpCookies(defaults);
 

modules/src/main/java/org/archive/modules/net/CrawlServer.java

@@ -28,6 +28,7 @@ import java.io.InputStream;
 import java.io.InputStreamReader;
 import java.io.Serializable;
 import java.util.HashSet;
+import java.util.Map;
 import java.util.Set;
 import java.util.logging.Level;
 import java.util.logging.Logger;
@@ -348,5 +349,13 @@ public class CrawlServer implements Serializable, FetchStats.HasFetchStats, Iden
     @Override
     public void setIdentityCache(ObjectIdentityCache<?> cache) {
         this.cache = cache; 
-    } 
+    }
+    
+    transient private Map<String,String> httpAuthChallenges;
+    public Map<String,String> getHttpAuthChallenges() {
+        return httpAuthChallenges;
+    }
+    public void setHttpAuthChallenges(Map<String, String> httpAuthChallenges) {
+        this.httpAuthChallenges = httpAuthChallenges;
+    }
 }