lee/heritrix3
1
0
Fork 0
mirror of https://github.com/internetarchive/heritrix3.git synced 2026-09-23 06:05:46 +00:00
Code Issues Packages Projects Releases Wiki Activity

Fix socket timeout

Browse Source
This commit is contained in:
Noah Levitt
2012-12-30 19:05:55 -08:00
parent dac492f2c6
commit cb725bb38e
2 changed files with 19 additions and 11 deletions
Download Patch File Download Diff File Expand all files Collapse all files
modules/src/main/java/org/archive/modules/fetcher/FetchHTTP.java
+2 -7
View File
@@ -640,13 +640,8 @@ public class FetchHTTP extends Processor implements Lifecycle {
return;
}
// set hardMax on bytes (if set by operator)
long hardMax = getMaxLengthBytes();
// set overall timeout (if set by operator)
long timeoutMs = 1000 * getTimeoutSeconds();
// Get max fetch rate (bytes/ms). It comes in in KB/sec
long maxRateKBps = getMaxFetchKBSec();
rec.getRecordedInput().setLimits(hardMax, timeoutMs, maxRateKBps);
rec.getRecordedInput().setLimits(getMaxLengthBytes(),
1000l * (long) getTimeoutSeconds(), (long) getMaxFetchKBSec());
HttpResponse response = null;
try {
modules/src/main/java/org/archive/modules/fetcher/FetchHTTPRequest.java
+17 -4
View File
@@ -61,6 +61,7 @@ import org.apache.http.client.protocol.HttpClientContext;
import org.apache.http.config.MessageConstraints;
import org.apache.http.config.Registry;
import org.apache.http.config.RegistryBuilder;
import org.apache.http.config.SocketConfig;
import org.apache.http.conn.DnsResolver;
import org.apache.http.conn.HttpClientConnectionManager;
import org.apache.http.conn.SocketClientConnection;
@@ -311,6 +312,12 @@ public class FetchHTTPRequest {
configBuilder.setConnectionRequestTimeout(fetcher.getSoTimeoutMs());
configBuilder.setConnectTimeout(fetcher.getSoTimeoutMs());
/*
* XXX This socket timeout seems to be ignored. The one on the
* socketConfig on the PoolingHttpClientConnectionManager in the
* HttpClientBuilder is respected.
*/
configBuilder.setSocketTimeout(fetcher.getSoTimeoutMs());
// local bind address
@@ -455,11 +462,11 @@ public class FetchHTTPRequest {
httpClientBuilder.setCookieStore(fetcher.getCookieStore());
HttpClientConnectionManager connManager = makeConnectionManager();
HttpClientConnectionManager connManager = buildConnectionManager();
httpClientBuilder.setConnectionManager(connManager);
}
protected HttpClientConnectionManager makeConnectionManager() {
protected HttpClientConnectionManager buildConnectionManager() {
Registry<ConnectionSocketFactory> socketFactoryRegistry = RegistryBuilder.<ConnectionSocketFactory>create()
.register("http", PlainSocketFactory.getSocketFactory())
.register("https", new SSLSocketFactory(fetcher.sslContext(), new AllowAllHostnameVerifier()))
@@ -479,9 +486,15 @@ public class FetchHTTPRequest {
DnsResolver dnsResolver = new ServerCacheResolver(fetcher.getServerCache());
return new PoolingHttpClientConnectionManager(socketFactoryRegistry,
PoolingHttpClientConnectionManager connMan = new PoolingHttpClientConnectionManager(socketFactoryRegistry,
connFactory, null, dnsResolver, -1, TimeUnit.MILLISECONDS);
}
SocketConfig.Builder socketConfigBuilder = SocketConfig.custom();
socketConfigBuilder.setSoTimeout(fetcher.getSoTimeoutMs());
connMan.setDefaultSocketConfig(socketConfigBuilder.build());
return connMan;
}
protected void initHttpClientBuilder() {
httpClientBuilder = HttpClientBuilder.create();