lee/heritrix3
1
0
Fork 0
mirror of https://github.com/internetarchive/heritrix3.git synced 2026-09-23 06:05:46 +00:00
Code Issues Packages Projects Releases Wiki Activity

tell CrawlURI to remember auth challenges across URI processings; couple other tweaks

Browse Source
This commit is contained in:
Noah Levitt
2012-08-16 12:00:17 -07:00
parent 883ca25925
commit 201e941b69
3 changed files with 10 additions and 14 deletions
Download Patch File Download Diff File Expand all files Collapse all files
modules/src/main/java/org/archive/modules/CrawlURI.java
+1 -1
View File
@@ -245,7 +245,7 @@ implements Reporter, Serializable, OverlayContext {
*/
private static final Collection<String> persistentKeys
= new CopyOnWriteArrayList<String>(
new String [] {A_CREDENTIALS_KEY});
new String [] {A_CREDENTIALS_KEY, A_HTTP_AUTH_CHALLENGES});
/** maximum length for pathFromSeed/hopsPath; longer truncated with leading counter **/
private static final int MAX_HOPS_DISPLAYED = 50;
modules/src/main/java/org/archive/modules/credential/CommonsHttpCredentialUtil.java
-5
View File
@@ -19,7 +19,6 @@
package org.archive.modules.credential;
import java.util.Arrays;
import java.util.Iterator;
import java.util.Map;
import java.util.logging.Logger;
@@ -31,7 +30,6 @@ import org.apache.commons.httpclient.NameValuePair;
import org.apache.commons.httpclient.URIException;
import org.apache.commons.httpclient.UsernamePasswordCredentials;
import org.apache.commons.httpclient.auth.AuthChallengeProcessor;
import org.apache.commons.httpclient.auth.AuthPolicy;
import org.apache.commons.httpclient.auth.AuthScheme;
import org.apache.commons.httpclient.auth.AuthScope;
import org.apache.commons.httpclient.auth.AuthenticationException;
@@ -137,9 +135,6 @@ public class CommonsHttpCredentialUtil {
logger.fine("Credentials for realm " + cred.getRealm() +
" for CrawlURI " + curi.toString() + " added to request");
http.getParams().setParameter(AuthPolicy.AUTH_SCHEME_PRIORITY,
Arrays.asList(AuthPolicy.DIGEST, AuthPolicy.BASIC));
result = true;
} catch (URIException e) {
logger.severe("Failed to parse host from " + curi + ": " +
modules/src/main/java/org/archive/modules/fetcher/FetchHTTP.java
+9 -8
View File
@@ -1277,28 +1277,31 @@ public class FetchHTTP extends AbstractFetchHTTP implements Lifecycle {
return null;
}
Map<String, String> authschemes = null;
Map<String, String> authChallenges = null;
try {
@SuppressWarnings("unchecked")
Map<String, String> parsedChallenges = AuthChallengeParser.parseChallenges(headers);
authschemes = parsedChallenges;
authChallenges = parsedChallenges;
curi.setHttpAuthChallenges(authschemes);
// remember WWW-Authenticate headers for later use
curi.setHttpAuthChallenges(authChallenges);
} catch (MalformedChallengeException e) {
logger.fine("Failed challenge parse: " + e.getMessage());
}
if (authschemes == null || authschemes.size() <= 0) {
if (authChallenges == null || authChallenges.size() <= 0) {
logger.fine("We got a 401 and WWW-Authenticate challenge"
+ " but failed parse of the header " + curi.toString());
return null;
}
// XXX there's a lot of overlap below with AuthChallengeProcessor.processChallenge()
AuthScheme result = null;
// Use the first auth found.
for (Iterator<String> i = authschemes.keySet().iterator(); result == null
for (Iterator<String> i = authChallenges.keySet().iterator(); result == null
&& i.hasNext();) {
String key = (String) i.next();
String challenge = (String) authschemes.get(key);
String challenge = (String) authChallenges.get(key);
if (key == null || key.length() <= 0 || challenge == null
|| challenge.length() <= 0) {
logger.warning("Empty scheme: " + curi.toString() + ": "
@@ -1455,8 +1458,6 @@ public class FetchHTTP extends AbstractFetchHTTP implements Lifecycle {
hcp.setSoTimeout(timeout);
// Set client to be version 1.0.
hcp.setVersion(HttpVersion.HTTP_1_0);
// We handle 401s, so when we do auth, we want it preemptive.
// hcp.setAuthenticationPreemptive(true);
// configureHttpCookies(defaults);