lee/heritrix3
1
0
Fork 0
mirror of https://github.com/internetarchive/heritrix3.git synced 2026-09-26 23:55:50 +00:00
Code Issues Packages Projects Releases Wiki Activity

fetchhttp2 support for proxy auth, somewhat elaborately done, sharing principles and code with http "target" auth

Browse Source
This commit is contained in:
Noah Levitt
2012-08-29 17:41:50 -07:00
parent 0e02aa3523
commit 731d382ae0
5 changed files with 150 additions and 73 deletions
Download Patch File Download Diff File Expand all files Collapse all files
modules/pom.xml
+6
View File
@@ -91,6 +91,12 @@
<version>1.6.6</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>commons-codec</groupId>
<artifactId>commons-codec</artifactId>
<version>1.6</version>
<scope>test</scope>
</dependency>
</dependencies>
<build>
<plugins>
modules/src/main/java/org/archive/modules/fetcher/FetchHTTP2.java
+88 -67
View File
@@ -58,6 +58,7 @@ import org.apache.http.auth.AuthScope;
import org.apache.http.auth.MalformedChallengeException;
import org.apache.http.auth.UsernamePasswordCredentials;
import org.apache.http.client.AuthCache;
import org.apache.http.client.AuthenticationStrategy;
import org.apache.http.client.ClientProtocolException;
import org.apache.http.client.entity.UrlEncodedFormEntity;
import org.apache.http.client.methods.HttpGet;
@@ -453,7 +454,8 @@ public class FetchHTTP2 extends AbstractFetchHTTP implements Lifecycle {
// Populate credentials. Set config so auth. is not automatic.
BasicHttpContext contextForAuth = new BasicHttpContext();
boolean addedCredentials = populateCredentials(curi, request, targetHost, contextForAuth);
boolean addedCredentials = populateTargetCredentials(curi, request, targetHost, contextForAuth);
populateHttpProxyCredential(curi, request, contextForAuth);
HttpResponse response = null;
try {
@@ -471,7 +473,7 @@ public class FetchHTTP2 extends AbstractFetchHTTP implements Lifecycle {
long softMax = -1l;
Header h = response.getLastHeader("content-length");
if (h != null) {
Long.parseLong(h.getValue());
softMax = Long.parseLong(h.getValue());
}
try {
if (!request.isAborted()) {
@@ -507,8 +509,6 @@ public class FetchHTTP2 extends AbstractFetchHTTP implements Lifecycle {
setCharacterEncoding(curi, rec, response);
setSizes(curi, rec);
setOtherCodings(curi, rec, response);
httpClient().getCredentialsProvider().clear();
}
if (digestContent) {
@@ -532,6 +532,10 @@ public class FetchHTTP2 extends AbstractFetchHTTP implements Lifecycle {
} else if (response.getStatusLine().getStatusCode() == HttpStatus.SC_UNAUTHORIZED) {
// 401 is not 'success'.
handle401(response, curi);
} else if (response.getStatusLine().getStatusCode() == HttpStatus.SC_PROXY_AUTHENTICATION_REQUIRED) {
// 407 - remember Proxy-Authenticate headers for later use
kp.put("proxyAuthChallenges",
extractChallenges(response, curi, httpClient().getProxyAuthenticationStrategy()));
}
if (rec.getRecordedInput().isOpen()) {
@@ -546,6 +550,25 @@ public class FetchHTTP2 extends AbstractFetchHTTP implements Lifecycle {
}
}
protected void populateHttpProxyCredential(CrawlURI curi,
HttpRequestBase request, BasicHttpContext context) {
// this should have been set earlier
HttpHost proxyHost = ConnRouteParams.getDefaultProxy(request.getParams());
String user = (String) getAttributeEither(curi, "httpProxyUser");
String password = (String) getAttributeEither(curi, "httpProxyPassword");
if (proxyHost != null && kp.get("proxyAuthChallenges") != null && StringUtils.isNotEmpty(user)) {
@SuppressWarnings("unchecked")
Map<String,String> challenges = (Map<String, String>) kp.get("proxyAuthChallenges");
AuthScheme authScheme = chooseAuthScheme(challenges, HttpHeaders.PROXY_AUTHENTICATE);
populateHttpCredential(proxyHost, context, authScheme, user, password);
}
}
// clear out any state that could conceivably be left over from a fetch
protected void resetHttpClient() {
httpClient().getCredentialsProvider().clear();
@@ -571,23 +594,20 @@ public class FetchHTTP2 extends AbstractFetchHTTP implements Lifecycle {
return true;
}
protected boolean populateHttpAuthCredential(HttpHost targetHost,
HttpContext context, HttpAuthenticationCredential cred,
Map<String, String> httpAuthChallenges) {
AuthScheme authScheme = chooseAuthScheme(httpAuthChallenges);
AuthCache authCache = new BasicAuthCache();
authCache.put(targetHost, authScheme);
// http auth credential, either for proxy or target host
protected void populateHttpCredential(HttpHost host, HttpContext context, AuthScheme authScheme, String user, String password) {
UsernamePasswordCredentials credentials = new UsernamePasswordCredentials(user, password);
AuthCache authCache = (AuthCache) context.getAttribute(ClientContext.AUTH_CACHE);
if (authCache == null) {
authCache = new BasicAuthCache();
context.setAttribute(ClientContext.AUTH_CACHE, authCache);
}
authCache.put(host, authScheme);
context.setAttribute(ClientContext.AUTH_CACHE, authCache);
UsernamePasswordCredentials credentials = new UsernamePasswordCredentials(cred.getLogin(), cred.getPassword());
AuthScope authscope = new AuthScope(targetHost, cred.getRealm(), authScheme.getSchemeName());
httpClient().getCredentialsProvider().setCredentials(authscope,
credentials);
return true;
httpClient().getCredentialsProvider().setCredentials(new AuthScope(host), credentials);
}
/**
* Add credentials if any to passed <code>method</code>.
*
@@ -611,7 +631,7 @@ public class FetchHTTP2 extends AbstractFetchHTTP implements Lifecycle {
* CrawlServer so they are available for all subsequent CrawlURIs on this
* server.
*/
protected boolean populateCredentials(CrawlURI curi,
protected boolean populateTargetCredentials(CrawlURI curi,
HttpRequestBase request, HttpHost targetHost, HttpContext context) {
// First look at the server avatars. Add any that are to be volunteered
// on every request (e.g. RFC2617 credentials). Every time creds will
@@ -624,13 +644,14 @@ public class FetchHTTP2 extends AbstractFetchHTTP implements Lifecycle {
}
CrawlServer server = serverCache.getServerFor(serverKey);
if (server.hasCredentials()) {
for (Credential cred : server.getCredentials()) {
if (cred.isEveryTime()) {
if (cred instanceof HttpAuthenticationCredential) {
populateHttpAuthCredential(targetHost, context, (HttpAuthenticationCredential) cred,
server.getHttpAuthChallenges());
for (Credential c: server.getCredentials()) {
if (c.isEveryTime()) {
if (c instanceof HttpAuthenticationCredential) {
HttpAuthenticationCredential cred = (HttpAuthenticationCredential) c;
AuthScheme authScheme = chooseAuthScheme(server.getHttpAuthChallenges(), HttpHeaders.WWW_AUTHENTICATE);
populateHttpCredential(targetHost, context, authScheme, cred.getLogin(), cred.getPassword());
} else {
populateHtmlFormCredential(curi, request, (HtmlFormCredential) cred);
populateHtmlFormCredential(curi, request, (HtmlFormCredential) c);
}
}
}
@@ -643,8 +664,10 @@ public class FetchHTTP2 extends AbstractFetchHTTP implements Lifecycle {
// the curi by the preconditionenforcer as this login uri came through.
for (Credential c: curi.getCredentials()) {
if (c instanceof HttpAuthenticationCredential) {
result = populateHttpAuthCredential(targetHost, context, (HttpAuthenticationCredential) c,
curi.getHttpAuthChallenges());
HttpAuthenticationCredential cred = (HttpAuthenticationCredential) c;
AuthScheme authScheme = chooseAuthScheme(curi.getHttpAuthChallenges(), HttpHeaders.WWW_AUTHENTICATE);
populateHttpCredential(targetHost, context, authScheme, cred.getLogin(), cred.getPassword());
result = true;
} else {
result = populateHtmlFormCredential(curi, request, (HtmlFormCredential) c);
}
@@ -691,7 +714,12 @@ public class FetchHTTP2 extends AbstractFetchHTTP implements Lifecycle {
* CrawlURI that got a 401.
*/
protected void handle401(HttpResponse response, final CrawlURI curi) {
AuthScheme authscheme = chooseAuthScheme(response, curi);
Map<String, String> challenges = extractChallenges(response, curi, httpClient().getTargetAuthenticationStrategy());
AuthScheme authscheme = chooseAuthScheme(challenges, HttpHeaders.WWW_AUTHENTICATE);
// remember WWW-Authenticate headers for later use
curi.setHttpAuthChallenges(challenges);
if (authscheme == null) {
return;
}
@@ -743,50 +771,45 @@ public class FetchHTTP2 extends AbstractFetchHTTP implements Lifecycle {
}
/**
* @param response
* @param response
* @param method
* Method that got a 401.
* Method that got a 401 or 407.
* @param curi
* CrawlURI that got a 401.
* @return Returns first wholesome authscheme found else null.
* CrawlURI that got a 401 or 407.
* @param authStrategy
* Either ProxyAuthenticationStrategy or
* TargetAuthenticationStrategy. Determines whether
* Proxy-Authenticate or WWW-Authenticate header is consulted.
*
* @return Map<authSchemeName -> challenge header value>
*/
protected AuthScheme chooseAuthScheme(HttpResponse response, final CrawlURI curi) {
Header[] headers = response.getHeaders(HttpHeaders.WWW_AUTHENTICATE);
if (headers == null || headers.length <= 0) {
logger.fine("We got a 401 but no WWW-Authenticate challenge: "
+ curi.toString());
return null;
}
Map<String, Header> wwwAuthHeaders = null;
protected Map<String,String> extractChallenges(HttpResponse response, final CrawlURI curi, AuthenticationStrategy authStrategy) {
Map<String, Header> hcChallengeHeaders = null;
try {
wwwAuthHeaders = httpClient().getTargetAuthenticationStrategy().getChallenges(null, response, null);
hcChallengeHeaders = authStrategy.getChallenges(null, response, null);
} catch (MalformedChallengeException e) {
logger.fine("Failed challenge parse: " + e.getMessage());
}
if (wwwAuthHeaders == null || wwwAuthHeaders.size() <= 0) {
logger.fine("We got a 401 and WWW-Authenticate challenge"
+ " but failed parse of the header " + curi.toString());
if (hcChallengeHeaders == null || hcChallengeHeaders.size() <= 0) {
logger.fine("Failed to get auth challenge headers for " + curi);
return null;
}
// cache for later use in non-library specific way
// reorganize in non-library-specific way
Map<String,String> challenges = new HashMap<String, String>();
for (Entry<String, Header> challenge: wwwAuthHeaders.entrySet()) {
for (Entry<String, Header> challenge: hcChallengeHeaders.entrySet()) {
challenges.put(challenge.getKey(), challenge.getValue().getValue());
}
// remember WWW-Authenticate headers for later use
curi.setHttpAuthChallenges(challenges);
return chooseAuthScheme(challenges);
return challenges;
}
protected AuthScheme chooseAuthScheme(Map<String, String> challenges) {
protected AuthScheme chooseAuthScheme(Map<String, String> challenges, String challengeHeaderKey) {
HashSet<String> authSchemesLeftToTry = new HashSet<String>(challenges.keySet());
for (String authSchemeName: new String[]{"digest","basic"}) {
if (authSchemesLeftToTry.remove(authSchemeName)) {
AuthScheme authscheme = httpClient().getAuthSchemes().getAuthScheme(authSchemeName, null);
BasicHeader challenge = new BasicHeader(HttpHeaders.WWW_AUTHENTICATE, challenges.get(authSchemeName));
BasicHeader challenge = new BasicHeader(challengeHeaderKey, challenges.get(authSchemeName));
try {
authscheme.processChallenge(challenge);
@@ -892,17 +915,15 @@ public class FetchHTTP2 extends AbstractFetchHTTP implements Lifecycle {
protected void configureProxy(CrawlURI curi, HttpRequestBase request) {
String host = (String) getAttributeEither(curi, "httpProxyHost");
int port = (Integer) getAttributeEither(curi, "httpProxyPort");
String user = (String) getAttributeEither(curi, "httpProxyUser");
String password = (String) getAttributeEither(curi, "httpProxyPassword");
Integer port = (Integer) getAttributeEither(curi, "httpProxyPort");
if (StringUtils.isNotEmpty(host)) {
if (StringUtils.isNotEmpty(host) && port != null) {
HttpHost proxyHost = new HttpHost(host, port);
ConnRouteParams.setDefaultProxy(request.getParams(), proxyHost);
if (StringUtils.isNotEmpty(user) || StringUtils.isNotEmpty(password)) {
UsernamePasswordCredentials credentials = new UsernamePasswordCredentials(user, password);
httpClient().getCredentialsProvider().setCredentials(new AuthScope(proxyHost), credentials);
}
// Without this, httpcomponents adds "Proxy-Connection: Keep-Alive".
// Not sure if that would cause actual problems.
request.addHeader("Proxy-Connection", "close");
}
}
@@ -912,7 +933,7 @@ public class FetchHTTP2 extends AbstractFetchHTTP implements Lifecycle {
* settings, in given HostConfiguration
* @param request
*/
private void configureBindAddress(CrawlURI curi, HttpRequestBase request) {
protected void configureBindAddress(CrawlURI curi, HttpRequestBase request) {
String addressString = (String) getAttributeEither(curi, HTTP_BIND_ADDRESS);
if (StringUtils.isNotEmpty(addressString)) {
try {
@@ -968,11 +989,13 @@ public class FetchHTTP2 extends AbstractFetchHTTP implements Lifecycle {
protected RecordingHttpClient httpClient() {
if (httpClient == null) {
httpClient = new RecordingHttpClient(getServerCache());
// XXX should this in the constructor? in configureRequest()? somewhere else?
// some http client config
HttpClientParams.setRedirecting(httpClient.getParams(), false);
httpClient.setCookieStore(getCookieStore());
if (getCookieStore() != null) {
httpClient.setCookieStore(getCookieStore());
}
}
return httpClient;
@@ -1075,8 +1098,6 @@ public class FetchHTTP2 extends AbstractFetchHTTP implements Lifecycle {
}
super.start();
// configureHttp();
if (getCookieStore() != null) {
getCookieStore().start();
}
modules/src/test/java/org/archive/modules/fetcher/FetchHTTPTestBase.java
+54 -4
View File
@@ -52,6 +52,7 @@ import org.jboss.netty.handler.codec.http.HttpRequest;
import org.littleshoot.proxy.DefaultHttpProxyServer;
import org.littleshoot.proxy.HttpFilter;
import org.littleshoot.proxy.HttpRequestFilter;
import org.littleshoot.proxy.ProxyAuthorizationHandler;
import org.mortbay.jetty.NCSARequestLog;
import org.mortbay.jetty.Request;
import org.mortbay.jetty.Response;
@@ -636,6 +637,10 @@ public abstract class FetchHTTPTestBase extends ProcessorTestBase {
public void filter(HttpRequest httpRequest) {
lastProxiedRequest = httpRequest;
}
public void clear() {
lastProxiedRequest = null;
}
}
public void testHttpProxy() throws Exception {
@@ -657,10 +662,7 @@ public abstract class FetchHTTPTestBase extends ProcessorTestBase {
assertTrue(requestString.startsWith("GET http://localhost:7777/ HTTP/1.0\r\n"));
assertNotNull(curi.getHttpResponseHeader("Via"));
// XXX commons-httpclient sends "Proxy-Connection: close" by
// default. But httpcomponents sends "Proxy-Connection: Keep-Alive"
// by default. Is this something to worry about?
assertTrue(requestString.contains("Proxy-Connection: "));
assertTrue(requestString.contains("Proxy-Connection: close\r\n"));
// check that our little proxy server really handled a request
assertNotNull(proxiedRequestRememberer.getLastProxiedRequest());
@@ -670,4 +672,52 @@ public abstract class FetchHTTPTestBase extends ProcessorTestBase {
httpProxyServer.stop();
}
}
public void testHttpProxyAuth() throws Exception {
ensureHttpServers();
ProxiedRequestRememberer proxiedRequestRememberer = new ProxiedRequestRememberer();
DefaultHttpProxyServer httpProxyServer = new DefaultHttpProxyServer(7877, proxiedRequestRememberer, new HashMap<String, HttpFilter>());
httpProxyServer.addProxyAuthenticationHandler(new ProxyAuthorizationHandler() {
@Override
public boolean authenticate(String userName, String password) {
logger.info("username=" + userName + " password=" + password);
return "http-proxy-user".equals(userName) && "http-proxy-password".equals(password);
}
});
httpProxyServer.start(true, false);
try {
getFetcher().setHttpProxyHost("localhost");
getFetcher().setHttpProxyPort(7877);
getFetcher().setHttpProxyUser("http-proxy-user");
getFetcher().setHttpProxyPassword("http-proxy-password");
CrawlURI curi = makeCrawlURI("http://localhost:7777/");
getFetcher().process(curi);
logger.info('\n' + httpRequestString(curi) + "\n\n" + rawResponseString(curi));
String requestString = httpRequestString(curi);
assertTrue(requestString.startsWith("GET http://localhost:7777/ HTTP/1.0\r\n"));
assertTrue(requestString.contains("Proxy-Connection: close\r\n"));
assertNull(proxiedRequestRememberer.getLastProxiedRequest()); // request didn't make it this far
assertEquals(407, curi.getFetchStatus());
// fetch original again now that credentials should be populated
proxiedRequestRememberer.clear();
curi = makeCrawlURI("http://localhost:7777/");
getFetcher().process(curi);
logger.info('\n' + httpRequestString(curi) + "\n\n" + rawResponseString(curi));
requestString = httpRequestString(curi);
assertTrue(requestString.startsWith("GET http://localhost:7777/ HTTP/1.0\r\n"));
assertTrue(requestString.contains("Proxy-Connection: close\r\n"));
assertNotNull(curi.getHttpResponseHeader("Via"));
assertNotNull(proxiedRequestRememberer.getLastProxiedRequest());
runDefaultChecks(curi, "requestLine");
} finally {
httpProxyServer.stop();
}
}
}