lee/heritrix3
1
0
Fork 0
mirror of https://github.com/internetarchive/heritrix3.git synced 2026-09-23 06:05:46 +00:00
Code Issues Packages Projects Releases Wiki Activity

testHttpProxyAuth() passes

Browse Source
This commit is contained in:
Noah Levitt
2012-12-29 21:05:37 -08:00
parent 6e8d4efe35
commit 2a6a2aa6d4
3 changed files with 29 additions and 28 deletions
Download Patch File Download Diff File Expand all files Collapse all files
modules/src/main/java/org/archive/modules/fetcher/FetchHTTP.java
+5 -3
View File
@@ -64,6 +64,7 @@ import org.apache.http.config.RegistryBuilder;
import org.apache.http.entity.ContentType;
import org.apache.http.impl.auth.BasicSchemeFactory;
import org.apache.http.impl.auth.DigestSchemeFactory;
import org.apache.http.impl.client.ProxyAuthenticationStrategy;
import org.apache.http.impl.client.TargetAuthenticationStrategy;
import org.apache.http.message.BasicHeader;
import org.archive.httpclient.ConfigurableX509TrustManager;
@@ -718,8 +719,8 @@ public class FetchHTTP extends Processor implements Lifecycle {
handle401(response, curi);
} else if (response.getStatusLine().getStatusCode() == HttpStatus.SC_PROXY_AUTHENTICATION_REQUIRED) {
// 407 - remember Proxy-Authenticate headers for later use
// kp.put("proxyAuthChallenges",
// extractChallenges(response, curi, httpClient().getProxyAuthenticationStrategy()));
kp.put("proxyAuthChallenges",
extractChallenges(response, curi, ProxyAuthenticationStrategy.INSTANCE));
}
if (rec.getRecordedInput().isOpen()) {
@@ -843,7 +844,8 @@ public class FetchHTTP extends Processor implements Lifecycle {
*
* @return Map<authSchemeName -> challenge header value>
*/
protected Map<String,String> extractChallenges(HttpResponse response, final CrawlURI curi, AuthenticationStrategy authStrategy) {
protected Map<String, String> extractChallenges(HttpResponse response,
final CrawlURI curi, AuthenticationStrategy authStrategy) {
Map<String, Header> hcChallengeHeaders = null;
try {
hcChallengeHeaders = authStrategy.getChallenges(null, response, null);
modules/src/main/java/org/archive/modules/fetcher/FetchHTTPRequest.java
+19 -23
View File
@@ -30,6 +30,7 @@ import java.nio.charset.CharsetEncoder;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
import java.util.concurrent.TimeUnit;
import java.util.logging.Logger;
@@ -136,6 +137,7 @@ public class FetchHTTPRequest {
protected AbortableHttpRequestBase request;
protected HttpHost targetHost;
protected boolean addedCredentials;
protected HttpHost proxyHost;
public FetchHTTPRequest(FetchHTTP fetcher, CrawlURI curi) throws URIException {
this.fetcher = fetcher;
@@ -150,7 +152,6 @@ public class FetchHTTPRequest {
String proxyHostname = (String) fetcher.getAttributeEither(curi, "httpProxyHost");
Integer proxyPort = (Integer) fetcher.getAttributeEither(curi, "httpProxyPort");
HttpHost proxyHost = null;
String requestLineUri;
if (StringUtils.isNotEmpty(proxyHostname) && proxyPort != null) {
proxyHost = new HttpHost(proxyHostname, proxyPort);
@@ -179,7 +180,8 @@ public class FetchHTTPRequest {
configureRequestHeaders();
configureRequest();
this.addedCredentials = populateTargetCredentials();
this.addedCredentials = populateTargetCredential();
populateHttpProxyCredential();
}
protected void configureRequestHeaders() {
@@ -213,9 +215,9 @@ public class FetchHTTPRequest {
}
if (!curi.isPrerequisite()) {
setConditionalGetHeader(fetcher.getSendIfModifiedSince(),
maybeAddConditionalGetHeader(fetcher.getSendIfModifiedSince(),
A_LAST_MODIFIED_HEADER, "If-Modified-Since");
setConditionalGetHeader(fetcher.getSendIfNoneMatch(),
maybeAddConditionalGetHeader(fetcher.getSendIfNoneMatch(),
A_ETAG_HEADER, "If-None-Match");
}
@@ -236,13 +238,13 @@ public class FetchHTTPRequest {
}
/**
* Set the given conditional-GET header, if the setting is enabled and
* Add the given conditional-GET header, if the setting is enabled and
* a suitable value is available in the URI history.
* @param setting true/false enablement setting name to consult
* @param sourceHeader header to consult in URI history
* @param targetHeader header to set if possible
*/
protected void setConditionalGetHeader(boolean conditional,
protected void maybeAddConditionalGetHeader(boolean conditional,
String sourceHeader, String targetHeader) {
if (conditional) {
try {
@@ -314,7 +316,7 @@ public class FetchHTTPRequest {
* CrawlServer so they are available for all subsequent CrawlURIs on this
* server.
*/
protected boolean populateTargetCredentials() {
protected boolean populateTargetCredential() {
// First look at the server avatars. Add any that are to be volunteered
// on every request (e.g. RFC2617 credentials). Every time creds will
// return true when we call 'isEveryTime().
@@ -359,22 +361,16 @@ public class FetchHTTPRequest {
}
protected void populateHttpProxyCredential() {
// HttpClientContext context = httpClientContext;
//
// // this should have been set earlier
// HttpHost proxyHost = ConnRouteParams.getDefaultProxy(request.getParams());
//
// String user = (String) fetcher.getAttributeEither(curi, "httpProxyUser");
// String password = (String) fetcher.getAttributeEither(curi, "httpProxyPassword");
//
// if (proxyHost != null && kp.get("proxyAuthChallenges") != null && StringUtils.isNotEmpty(user)) {
//
// @SuppressWarnings("unchecked")
// Map<String,String> challenges = (Map<String, String>) kp.get("proxyAuthChallenges");
//
// AuthScheme authScheme = chooseAuthScheme(challenges, HttpHeaders.PROXY_AUTHENTICATE);
// populateHttpCredential(proxyHost, context, authScheme, user, password);
// }
String user = (String) fetcher.getAttributeEither(curi, "httpProxyUser");
String password = (String) fetcher.getAttributeEither(curi, "httpProxyPassword");
@SuppressWarnings("unchecked")
Map<String,String> challenges = (Map<String, String>) fetcher.getKeyedProperties().get("proxyAuthChallenges");
if (proxyHost != null && challenges != null && StringUtils.isNotEmpty(user)) {
AuthScheme authScheme = fetcher.chooseAuthScheme(challenges, HttpHeaders.PROXY_AUTHENTICATE);
populateHttpCredential(proxyHost, authScheme, user, password);
}
}
protected boolean populateHtmlFormCredential(HtmlFormCredential cred) {
modules/src/test/java/org/archive/modules/fetcher/FetchHTTPTests.java
+5 -2
View File
@@ -273,6 +273,7 @@ public class FetchHTTPTests extends ProcessorTestBase {
// check that we got the expected response and the fetcher did its thing
assertEquals(401, curi.getFetchStatus());
assertEquals("Basic realm=\"basic-auth-realm\"", curi.getHttpResponseHeader("WWW-Authenticate"));
assertTrue(curi.getCredentials().contains(basicAuthCredential));
assertTrue(curi.getHttpAuthChallenges() != null && curi.getHttpAuthChallenges().containsKey("basic"));
@@ -539,19 +540,21 @@ public class FetchHTTPTests extends ProcessorTestBase {
CrawlURI curi = makeCrawlURI("http://localhost:7777/");
fetcher().process(curi);
// logger.info('\n' + httpRequestString(curi) + "\n\n" + rawResponseString(curi));
logger.info('\n' + httpRequestString(curi) + "\n\n" + rawResponseString(curi));
String requestString = httpRequestString(curi);
assertTrue(requestString.startsWith("GET http://localhost:7777/ HTTP/1.0\r\n"));
assertTrue(requestString.contains("Proxy-Connection: close\r\n"));
assertNull(proxiedRequestRememberer.getLastProxiedRequest()); // request didn't make it this far
assertNotNull(curi.getHttpResponseHeader("Proxy-Authenticate"));
assertEquals(407, curi.getFetchStatus());
// fetch original again now that credentials should be populated
proxiedRequestRememberer.clear();
curi = makeCrawlURI("http://localhost:7777/");
fetcher().process(curi);
// logger.info('\n' + httpRequestString(curi) + "\n\n" + rawResponseString(curi));
logger.info('\n' + httpRequestString(curi) + "\n\n" + rawResponseString(curi));
requestString = httpRequestString(curi);
assertTrue(requestString.startsWith("GET http://localhost:7777/ HTTP/1.0\r\n"));