diff --git a/modules/src/main/java/org/archive/modules/fetcher/FetchHTTP.java b/modules/src/main/java/org/archive/modules/fetcher/FetchHTTP.java index a0d53151..126a81ee 100644 --- a/modules/src/main/java/org/archive/modules/fetcher/FetchHTTP.java +++ b/modules/src/main/java/org/archive/modules/fetcher/FetchHTTP.java @@ -64,6 +64,7 @@ import org.apache.http.config.RegistryBuilder; import org.apache.http.entity.ContentType; import org.apache.http.impl.auth.BasicSchemeFactory; import org.apache.http.impl.auth.DigestSchemeFactory; +import org.apache.http.impl.client.ProxyAuthenticationStrategy; import org.apache.http.impl.client.TargetAuthenticationStrategy; import org.apache.http.message.BasicHeader; import org.archive.httpclient.ConfigurableX509TrustManager; @@ -718,8 +719,8 @@ public class FetchHTTP extends Processor implements Lifecycle { handle401(response, curi); } else if (response.getStatusLine().getStatusCode() == HttpStatus.SC_PROXY_AUTHENTICATION_REQUIRED) { // 407 - remember Proxy-Authenticate headers for later use -// kp.put("proxyAuthChallenges", -// extractChallenges(response, curi, httpClient().getProxyAuthenticationStrategy())); + kp.put("proxyAuthChallenges", + extractChallenges(response, curi, ProxyAuthenticationStrategy.INSTANCE)); } if (rec.getRecordedInput().isOpen()) { @@ -843,7 +844,8 @@ public class FetchHTTP extends Processor implements Lifecycle { * * @return Map challenge header value> */ - protected Map extractChallenges(HttpResponse response, final CrawlURI curi, AuthenticationStrategy authStrategy) { + protected Map extractChallenges(HttpResponse response, + final CrawlURI curi, AuthenticationStrategy authStrategy) { Map hcChallengeHeaders = null; try { hcChallengeHeaders = authStrategy.getChallenges(null, response, null); diff --git a/modules/src/main/java/org/archive/modules/fetcher/FetchHTTPRequest.java b/modules/src/main/java/org/archive/modules/fetcher/FetchHTTPRequest.java index 7bb3c5da..7d6cebb9 100644 --- a/modules/src/main/java/org/archive/modules/fetcher/FetchHTTPRequest.java +++ b/modules/src/main/java/org/archive/modules/fetcher/FetchHTTPRequest.java @@ -30,6 +30,7 @@ import java.nio.charset.CharsetEncoder; import java.util.ArrayList; import java.util.HashMap; import java.util.List; +import java.util.Map; import java.util.Map.Entry; import java.util.concurrent.TimeUnit; import java.util.logging.Logger; @@ -136,6 +137,7 @@ public class FetchHTTPRequest { protected AbortableHttpRequestBase request; protected HttpHost targetHost; protected boolean addedCredentials; + protected HttpHost proxyHost; public FetchHTTPRequest(FetchHTTP fetcher, CrawlURI curi) throws URIException { this.fetcher = fetcher; @@ -150,7 +152,6 @@ public class FetchHTTPRequest { String proxyHostname = (String) fetcher.getAttributeEither(curi, "httpProxyHost"); Integer proxyPort = (Integer) fetcher.getAttributeEither(curi, "httpProxyPort"); - HttpHost proxyHost = null; String requestLineUri; if (StringUtils.isNotEmpty(proxyHostname) && proxyPort != null) { proxyHost = new HttpHost(proxyHostname, proxyPort); @@ -179,7 +180,8 @@ public class FetchHTTPRequest { configureRequestHeaders(); configureRequest(); - this.addedCredentials = populateTargetCredentials(); + this.addedCredentials = populateTargetCredential(); + populateHttpProxyCredential(); } protected void configureRequestHeaders() { @@ -213,9 +215,9 @@ public class FetchHTTPRequest { } if (!curi.isPrerequisite()) { - setConditionalGetHeader(fetcher.getSendIfModifiedSince(), + maybeAddConditionalGetHeader(fetcher.getSendIfModifiedSince(), A_LAST_MODIFIED_HEADER, "If-Modified-Since"); - setConditionalGetHeader(fetcher.getSendIfNoneMatch(), + maybeAddConditionalGetHeader(fetcher.getSendIfNoneMatch(), A_ETAG_HEADER, "If-None-Match"); } @@ -236,13 +238,13 @@ public class FetchHTTPRequest { } /** - * Set the given conditional-GET header, if the setting is enabled and + * Add the given conditional-GET header, if the setting is enabled and * a suitable value is available in the URI history. * @param setting true/false enablement setting name to consult * @param sourceHeader header to consult in URI history * @param targetHeader header to set if possible */ - protected void setConditionalGetHeader(boolean conditional, + protected void maybeAddConditionalGetHeader(boolean conditional, String sourceHeader, String targetHeader) { if (conditional) { try { @@ -314,7 +316,7 @@ public class FetchHTTPRequest { * CrawlServer so they are available for all subsequent CrawlURIs on this * server. */ - protected boolean populateTargetCredentials() { + protected boolean populateTargetCredential() { // First look at the server avatars. Add any that are to be volunteered // on every request (e.g. RFC2617 credentials). Every time creds will // return true when we call 'isEveryTime(). @@ -359,22 +361,16 @@ public class FetchHTTPRequest { } protected void populateHttpProxyCredential() { -// HttpClientContext context = httpClientContext; -// -// // this should have been set earlier -// HttpHost proxyHost = ConnRouteParams.getDefaultProxy(request.getParams()); -// -// String user = (String) fetcher.getAttributeEither(curi, "httpProxyUser"); -// String password = (String) fetcher.getAttributeEither(curi, "httpProxyPassword"); -// -// if (proxyHost != null && kp.get("proxyAuthChallenges") != null && StringUtils.isNotEmpty(user)) { -// -// @SuppressWarnings("unchecked") -// Map challenges = (Map) kp.get("proxyAuthChallenges"); -// -// AuthScheme authScheme = chooseAuthScheme(challenges, HttpHeaders.PROXY_AUTHENTICATE); -// populateHttpCredential(proxyHost, context, authScheme, user, password); -// } + String user = (String) fetcher.getAttributeEither(curi, "httpProxyUser"); + String password = (String) fetcher.getAttributeEither(curi, "httpProxyPassword"); + + @SuppressWarnings("unchecked") + Map challenges = (Map) fetcher.getKeyedProperties().get("proxyAuthChallenges"); + + if (proxyHost != null && challenges != null && StringUtils.isNotEmpty(user)) { + AuthScheme authScheme = fetcher.chooseAuthScheme(challenges, HttpHeaders.PROXY_AUTHENTICATE); + populateHttpCredential(proxyHost, authScheme, user, password); + } } protected boolean populateHtmlFormCredential(HtmlFormCredential cred) { diff --git a/modules/src/test/java/org/archive/modules/fetcher/FetchHTTPTests.java b/modules/src/test/java/org/archive/modules/fetcher/FetchHTTPTests.java index 4cbbb588..05345111 100644 --- a/modules/src/test/java/org/archive/modules/fetcher/FetchHTTPTests.java +++ b/modules/src/test/java/org/archive/modules/fetcher/FetchHTTPTests.java @@ -273,6 +273,7 @@ public class FetchHTTPTests extends ProcessorTestBase { // check that we got the expected response and the fetcher did its thing assertEquals(401, curi.getFetchStatus()); + assertEquals("Basic realm=\"basic-auth-realm\"", curi.getHttpResponseHeader("WWW-Authenticate")); assertTrue(curi.getCredentials().contains(basicAuthCredential)); assertTrue(curi.getHttpAuthChallenges() != null && curi.getHttpAuthChallenges().containsKey("basic")); @@ -539,19 +540,21 @@ public class FetchHTTPTests extends ProcessorTestBase { CrawlURI curi = makeCrawlURI("http://localhost:7777/"); fetcher().process(curi); - // logger.info('\n' + httpRequestString(curi) + "\n\n" + rawResponseString(curi)); + logger.info('\n' + httpRequestString(curi) + "\n\n" + rawResponseString(curi)); String requestString = httpRequestString(curi); assertTrue(requestString.startsWith("GET http://localhost:7777/ HTTP/1.0\r\n")); assertTrue(requestString.contains("Proxy-Connection: close\r\n")); + assertNull(proxiedRequestRememberer.getLastProxiedRequest()); // request didn't make it this far + assertNotNull(curi.getHttpResponseHeader("Proxy-Authenticate")); assertEquals(407, curi.getFetchStatus()); // fetch original again now that credentials should be populated proxiedRequestRememberer.clear(); curi = makeCrawlURI("http://localhost:7777/"); fetcher().process(curi); - // logger.info('\n' + httpRequestString(curi) + "\n\n" + rawResponseString(curi)); + logger.info('\n' + httpRequestString(curi) + "\n\n" + rawResponseString(curi)); requestString = httpRequestString(curi); assertTrue(requestString.startsWith("GET http://localhost:7777/ HTTP/1.0\r\n"));