lee/heritrix3
1
0
Fork 0
mirror of https://github.com/internetarchive/heritrix3.git synced 2026-09-26 15:46:31 +00:00
Code Issues Packages Projects Releases Wiki Activity

remember www-authenticate headers to make http auth work as intended, without lots of extra 401s

Browse Source
This commit is contained in:
Noah Levitt
2012-08-15 22:11:34 -07:00
parent 5272b0b8c8
commit 883ca25925
6 changed files with 54 additions and 18 deletions
Download Patch File Download Diff File Expand all files Collapse all files
modules/src/main/java/org/archive/modules/CoreAttributeConstants.java
+2
View File
@@ -116,4 +116,6 @@ public interface CoreAttributeConstants {
public static final String A_WHOIS_SERVER_IP = "whois-server-ip";
public static final String A_HTTP_RESPONSE_HEADERS = "http-response-headers";
public static final String A_HTTP_AUTH_CHALLENGES = "http-auth-challenges";
}
modules/src/main/java/org/archive/modules/CrawlURI.java
+10
View File
@@ -26,6 +26,7 @@ import static org.archive.modules.CoreAttributeConstants.A_FETCH_COMPLETED_TIME;
import static org.archive.modules.CoreAttributeConstants.A_FORCE_RETIRE;
import static org.archive.modules.CoreAttributeConstants.A_HERITABLE_KEYS;
import static org.archive.modules.CoreAttributeConstants.A_HTML_BASE;
import static org.archive.modules.CoreAttributeConstants.A_HTTP_AUTH_CHALLENGES;
import static org.archive.modules.CoreAttributeConstants.A_HTTP_RESPONSE_HEADERS;
import static org.archive.modules.CoreAttributeConstants.A_NONFATAL_ERRORS;
import static org.archive.modules.CoreAttributeConstants.A_PREREQUISITE_URI;
@@ -1877,4 +1878,13 @@ implements Reporter, Serializable, OverlayContext {
}
httpResponseHeaders.put(key.toLowerCase(), value);
}
@SuppressWarnings("unchecked")
public Map<String,String> getHttpAuthChallenges() {
return (Map<String, String>) getData().get(A_HTTP_AUTH_CHALLENGES);
}
public void setHttpAuthChallenges(Map<String, String> httpAuthChallenges) {
getData().put(A_HTTP_AUTH_CHALLENGES, httpAuthChallenges);
}
}
modules/src/main/java/org/archive/modules/credential/CommonsHttpCredentialUtil.java
+17 -3
View File
@@ -30,8 +30,12 @@ import org.apache.commons.httpclient.HttpMethodBase;
import org.apache.commons.httpclient.NameValuePair;
import org.apache.commons.httpclient.URIException;
import org.apache.commons.httpclient.UsernamePasswordCredentials;
import org.apache.commons.httpclient.auth.AuthChallengeProcessor;
import org.apache.commons.httpclient.auth.AuthPolicy;
import org.apache.commons.httpclient.auth.AuthScheme;
import org.apache.commons.httpclient.auth.AuthScope;
import org.apache.commons.httpclient.auth.AuthenticationException;
import org.apache.commons.httpclient.auth.MalformedChallengeException;
import org.apache.commons.httpclient.methods.GetMethod;
import org.apache.commons.httpclient.methods.PostMethod;
import org.apache.commons.lang.StringUtils;
@@ -42,9 +46,9 @@ public class CommonsHttpCredentialUtil {
private static Logger logger = Logger.getLogger(CommonsHttpCredentialUtil.class.getName());
public static boolean populate(CrawlURI curi, HttpClient http,
HttpMethod method, Credential cred) {
HttpMethod method, Credential cred, Map<String, String> httpAuthChallenges) {
if (cred instanceof HttpAuthenticationCredential) {
return populate(curi, http, method, (HttpAuthenticationCredential) cred);
return populate(curi, http, method, (HttpAuthenticationCredential) cred, httpAuthChallenges);
} else if (cred instanceof HtmlFormCredential) {
return populate(curi, http, method, (HtmlFormCredential) cred);
} else {
@@ -99,8 +103,18 @@ public class CommonsHttpCredentialUtil {
}
public static boolean populate(CrawlURI curi, HttpClient http,
HttpMethod method, HttpAuthenticationCredential cred) {
HttpMethod method, HttpAuthenticationCredential cred, Map<String,String> httpAuthChallenges) {
boolean result = false;
AuthChallengeProcessor authChallengeProcessor = new AuthChallengeProcessor(http.getParams());
try {
AuthScheme authScheme = authChallengeProcessor.processChallenge(method.getHostAuthState(), httpAuthChallenges);
method.getHostAuthState().setAuthScheme(authScheme);
} catch (MalformedChallengeException e) {
return result;
} catch (AuthenticationException e) {
return result;
}
// Always add the credential to HttpState. Doing this because no way of
// removing the credential once added AND there is a bug in the
modules/src/main/java/org/archive/modules/fetcher/FetchHTTP.java
+9 -11
View File
@@ -71,10 +71,9 @@ import org.apache.commons.httpclient.HttpVersion;
import org.apache.commons.httpclient.NTCredentials;
import org.apache.commons.httpclient.URIException;
import org.apache.commons.httpclient.auth.AuthChallengeParser;
import org.apache.commons.httpclient.auth.AuthPolicy;
import org.apache.commons.httpclient.auth.AuthScheme;
import org.apache.commons.httpclient.auth.AuthScope;
import org.apache.commons.httpclient.auth.BasicScheme;
import org.apache.commons.httpclient.auth.DigestScheme;
import org.apache.commons.httpclient.auth.MalformedChallengeException;
import org.apache.commons.httpclient.cookie.CookiePolicy;
import org.apache.commons.httpclient.params.HttpClientParams;
@@ -1153,7 +1152,7 @@ public class FetchHTTP extends AbstractFetchHTTP implements Lifecycle {
if (server.hasCredentials()) {
for (Credential cred : server.getCredentials()) {
if (cred.isEveryTime()) {
CommonsHttpCredentialUtil.populate(curi, this.http, method, cred);
CommonsHttpCredentialUtil.populate(curi, this.http, method, cred, server.getHttpAuthChallenges());
}
}
}
@@ -1164,7 +1163,7 @@ public class FetchHTTP extends AbstractFetchHTTP implements Lifecycle {
// by the handle401 method if its a rfc2617 or it'll have been set into
// the curi by the preconditionenforcer as this login uri came through.
for (Credential c: curi.getCredentials()) {
if (CommonsHttpCredentialUtil.populate(curi, this.http, method, c)) {
if (CommonsHttpCredentialUtil.populate(curi, this.http, method, c, curi.getHttpAuthChallenges())) {
result = true;
}
}
@@ -1193,6 +1192,7 @@ public class FetchHTTP extends AbstractFetchHTTP implements Lifecycle {
CrawlServer cs = serverCache.getServerFor(cd);
if (cs != null) {
cs.addCredential(c);
cs.setHttpAuthChallenges(curi.getHttpAuthChallenges());
}
}
}
@@ -1282,6 +1282,8 @@ public class FetchHTTP extends AbstractFetchHTTP implements Lifecycle {
@SuppressWarnings("unchecked")
Map<String, String> parsedChallenges = AuthChallengeParser.parseChallenges(headers);
authschemes = parsedChallenges;
curi.setHttpAuthChallenges(authschemes);
} catch (MalformedChallengeException e) {
logger.fine("Failed challenge parse: " + e.getMessage());
}
@@ -1303,12 +1305,8 @@ public class FetchHTTP extends AbstractFetchHTTP implements Lifecycle {
+ Arrays.toString(headers));
continue;
}
AuthScheme authscheme = null;
if (key.equals("basic")) {
authscheme = new BasicScheme();
} else if (key.equals("digest")) {
authscheme = new DigestScheme();
} else {
AuthScheme authscheme = AuthPolicy.getAuthScheme(key);
if (authscheme == null) {
logger.fine("Unsupported scheme: " + key);
continue;
}
@@ -1458,7 +1456,7 @@ public class FetchHTTP extends AbstractFetchHTTP implements Lifecycle {
// Set client to be version 1.0.
hcp.setVersion(HttpVersion.HTTP_1_0);
// We handle 401s, so when we do auth, we want it preemptive.
hcp.setAuthenticationPreemptive(true);
// hcp.setAuthenticationPreemptive(true);
// configureHttpCookies(defaults);
modules/src/main/java/org/archive/modules/net/CrawlServer.java
+10 -1
View File
@@ -28,6 +28,7 @@ import java.io.InputStream;
import java.io.InputStreamReader;
import java.io.Serializable;
import java.util.HashSet;
import java.util.Map;
import java.util.Set;
import java.util.logging.Level;
import java.util.logging.Logger;
@@ -348,5 +349,13 @@ public class CrawlServer implements Serializable, FetchStats.HasFetchStats, Iden
@Override
public void setIdentityCache(ObjectIdentityCache<?> cache) {
this.cache = cache;
}
}
transient private Map<String,String> httpAuthChallenges;
public Map<String,String> getHttpAuthChallenges() {
return httpAuthChallenges;
}
public void setHttpAuthChallenges(Map<String, String> httpAuthChallenges) {
this.httpAuthChallenges = httpAuthChallenges;
}
}
modules/src/test/java/org/archive/modules/fetcher/FetchHTTPTestBase.java
+6 -3
View File
@@ -411,6 +411,7 @@ public abstract class FetchHTTPTestBase extends ProcessorTestBase {
// check that we got the expected response and the fetcher did its thing
assertEquals(401, curi.getFetchStatus());
assertTrue(curi.getCredentials().contains(basicAuthCredential));
assertTrue(curi.getHttpAuthChallenges() != null && curi.getHttpAuthChallenges().containsKey("basic"));
// fetch again with the credentials
getFetcher().process(curi);
@@ -448,16 +449,18 @@ public abstract class FetchHTTPTestBase extends ProcessorTestBase {
// check that we got the expected response and the fetcher did its thing
assertEquals(401, curi.getFetchStatus());
assertTrue(curi.getCredentials().contains(digestAuthCred));
assertTrue(curi.getHttpAuthChallenges() != null && curi.getHttpAuthChallenges().containsKey("digest"));
// stick a basic auth 401 in there to check it doesn't mess with the digest auth we're working on
CrawlURI interferingUri = makeCrawlURI("http://localhost:7777/auth/basic");
getFetcher().process(interferingUri);
assertEquals(401, interferingUri.getFetchStatus());
logger.info('\n' + httpRequestString(interferingUri) + "\n\n" + rawResponseString(interferingUri));
// fetch original again with the credentials
getFetcher().process(curi);
String httpRequestString = httpRequestString(curi);
// logger.info('\n' + httpRequestString + contentString(curi));
logger.info('\n' + httpRequestString + "\n\n" + rawResponseString(interferingUri));
assertTrue(httpRequestString.contains("Authorization: Digest"));
// otherwise should be a normal 200 response
runDefaultChecks(curi, new HashSet<String>(Arrays.asList("requestLine", "hostHeader")));
@@ -468,7 +471,7 @@ public abstract class FetchHTTPTestBase extends ProcessorTestBase {
httpRequestString = httpRequestString(curi);
assertTrue(httpRequestString.contains("Authorization: Digest"));
// otherwise should be a normal 200 response
runDefaultChecks(curi, new HashSet<String>(Arrays.asList("requestLine")));
runDefaultChecks(curi, new HashSet<String>(Arrays.asList("requestLine", "hostHeader")));
}
// server for form auth is at localhost:7779
@@ -496,7 +499,7 @@ public abstract class FetchHTTPTestBase extends ProcessorTestBase {
// login (if not we get a NPE within jetty)
curi = makeCrawlURI("http://localhost:7779/auth/1");
getFetcher().process(curi);
logger.info('\n' + httpRequestString(curi) + rawResponseString(curi));
logger.info('\n' + httpRequestString(curi) + "\n\n" + rawResponseString(curi));
assertEquals(302, curi.getFetchStatus());
assertTrue(curi.getHttpResponseHeader("Location").startsWith("http://localhost:7779/login.html"));