lee/heritrix3
1
0
Fork 0
mirror of https://github.com/internetarchive/heritrix3.git synced 2026-09-27 08:05:46 +00:00
Code Issues Packages Projects Releases Wiki Activity

DnsResolver using ServerCache (should add a test for this)

Browse Source
This commit is contained in:
Noah Levitt
2012-12-30 01:00:07 -08:00
parent d1c41d624f
commit dac492f2c6
1 changed files with 43 additions and 4 deletions
Download Patch File Download Diff File Expand all files Collapse all files
modules/src/main/java/org/archive/modules/fetcher/FetchHTTPRequest.java
+43 -4
View File
@@ -61,6 +61,8 @@ import org.apache.http.client.protocol.HttpClientContext;
import org.apache.http.config.MessageConstraints;
import org.apache.http.config.Registry;
import org.apache.http.config.RegistryBuilder;
import org.apache.http.conn.DnsResolver;
import org.apache.http.conn.HttpClientConnectionManager;
import org.apache.http.conn.SocketClientConnection;
import org.apache.http.conn.socket.ConnectionSocketFactory;
import org.apache.http.conn.socket.PlainSocketFactory;
@@ -86,9 +88,14 @@ import org.archive.modules.credential.Credential;
import org.archive.modules.credential.HtmlFormCredential;
import org.archive.modules.credential.HttpAuthenticationCredential;
import org.archive.modules.extractor.LinkContext;
import org.archive.modules.net.CrawlHost;
import org.archive.modules.net.CrawlServer;
import org.archive.modules.net.ServerCache;
import org.archive.util.Recorder;
/**
* @contributor nlevitt
*/
public class FetchHTTPRequest {
protected static class RecordingSocketClientConnection extends SocketClientConnectionImpl {
@@ -127,6 +134,33 @@ public class FetchHTTPRequest {
}
}
}
/**
* Implementation of {@link DnsResolver} that uses the server cache which is
* normally expected to have been populated by FetchDNS.
*/
public static class ServerCacheResolver implements DnsResolver {
protected static Logger logger = Logger.getLogger(DnsResolver.class.getName());
protected ServerCache serverCache;
public ServerCacheResolver(ServerCache serverCache) {
this.serverCache = serverCache;
}
@Override
public InetAddress[] resolve(String host) throws UnknownHostException {
CrawlHost crawlHost = this.serverCache.getHostFor(host);
if (crawlHost != null) {
InetAddress ip = crawlHost.getIP();
if (ip != null) {
return new InetAddress[] {ip};
}
}
logger.info("host \"" + host + "\" is not in serverCache, allowing java to resolve it");
return new InetAddress[] {InetAddress.getByName(host)};
}
}
private static Logger logger = Logger.getLogger(FetchHTTPRequest.class.getName());
@@ -421,6 +455,11 @@ public class FetchHTTPRequest {
httpClientBuilder.setCookieStore(fetcher.getCookieStore());
HttpClientConnectionManager connManager = makeConnectionManager();
httpClientBuilder.setConnectionManager(connManager);
}
protected HttpClientConnectionManager makeConnectionManager() {
Registry<ConnectionSocketFactory> socketFactoryRegistry = RegistryBuilder.<ConnectionSocketFactory>create()
.register("http", PlainSocketFactory.getSocketFactory())
.register("https", new SSLSocketFactory(fetcher.sslContext(), new AllowAllHostnameVerifier()))
@@ -438,11 +477,11 @@ public class FetchHTTPRequest {
}
};
PoolingHttpClientConnectionManager connManager = new PoolingHttpClientConnectionManager(
socketFactoryRegistry, connFactory, -1, TimeUnit.MILLISECONDS);
DnsResolver dnsResolver = new ServerCacheResolver(fetcher.getServerCache());
httpClientBuilder.setConnectionManager(connManager);
}
return new PoolingHttpClientConnectionManager(socketFactoryRegistry,
connFactory, null, dnsResolver, -1, TimeUnit.MILLISECONDS);
}
protected void initHttpClientBuilder() {
httpClientBuilder = HttpClientBuilder.create();