lee/heritrix3
1
0
Fork 0
mirror of https://github.com/internetarchive/heritrix3.git synced 2026-09-24 22:55:45 +00:00
Code Issues Packages Projects Releases Wiki Activity

socket/connection timeout

Browse Source
This commit is contained in:
Noah Levitt
2012-09-06 12:39:44 -07:00
parent d11338d620
commit 948d641b14
2 changed files with 35 additions and 4 deletions
Download Patch File Download Diff File Expand all files Collapse all files
engine/src/test/java/org/archive/modules/fetcher/FetchHTTPTestBase.java
+28 -2
View File
@@ -19,6 +19,7 @@
package org.archive.modules.fetcher;
import java.io.IOException;
import java.io.InterruptedIOException;
import java.io.UnsupportedEncodingException;
import java.nio.charset.Charset;
import java.util.Arrays;
@@ -852,8 +853,33 @@ public abstract class FetchHTTPTestBase extends ProcessorTestBase {
getFetcher().setTimeoutSeconds(2);
getFetcher().process(curi);
logger.info('\n' + httpRequestString(curi) + "\n\n" + rawResponseString(curi));
// logger.info('\n' + httpRequestString(curi) + "\n\n" + rawResponseString(curi));
assertTrue(curi.getAnnotations().contains("timeTrunc"));
assertTrue(curi.getFetchDuration() > 2000 && curi.getFetchDuration() < 2200);
assertTrue(curi.getFetchDuration() >= 2000 && curi.getFetchDuration() < 2200);
}
// see http://stackoverflow.com/questions/100841/artificially-create-a-connection-timeout-error
public void testConnectionTimeout() throws Exception {
CrawlURI curi = makeCrawlURI("http://10.255.255.1/");
getFetcher().setSoTimeoutMs(300);
long start = System.currentTimeMillis();
getFetcher().process(curi);
long elapsed = System.currentTimeMillis() - start;
assertTrue(elapsed >= 300 && elapsed < 400);
// Httpcomponents throws org.apache.http.conn.ConnectTimeoutException,
// commons-httpclient throws java.net.SocketTimeoutException. Both are
// instances of InterruptedIOException
assertEquals(1, curi.getNonFatalFailures().size());
assertTrue(curi.getNonFatalFailures().toArray()[0] instanceof InterruptedIOException);
assertTrue(curi.getNonFatalFailures().toArray()[0].toString().matches("(?i).*connect.*timed out.*"));
assertEquals(FetchStatusCodes.S_CONNECT_FAILED, curi.getFetchStatus());
assertEquals(0, curi.getFetchCompletedTime());
}
// XXX testSocketTimeout() (the other kind) - how to simulate?
}
modules/src/main/java/org/archive/modules/fetcher/FetchHTTP2.java
+7 -2
View File
@@ -75,6 +75,7 @@ import org.apache.http.entity.ContentType;
import org.apache.http.impl.client.BasicAuthCache;
import org.apache.http.message.BasicHeader;
import org.apache.http.message.BasicNameValuePair;
import org.apache.http.params.HttpConnectionParams;
import org.apache.http.params.HttpProtocolParams;
import org.apache.http.protocol.BasicHttpContext;
import org.apache.http.protocol.HTTP;
@@ -1099,6 +1100,9 @@ public class FetchHTTP2 extends AbstractFetchHTTP implements Lifecycle {
A_ETAG_HEADER, "If-None-Match");
}
HttpConnectionParams.setConnectionTimeout(request.getParams(), getSoTimeoutMs());
HttpConnectionParams.setSoTimeout(request.getParams(), getSoTimeoutMs());
// TODO: What happens if below method adds a header already
// added above: e.g. Connection, Range, or Referer?
configureAcceptHeaders(request);
@@ -1297,9 +1301,10 @@ public class FetchHTTP2 extends AbstractFetchHTTP implements Lifecycle {
*/
protected void cleanup(final CrawlURI curi, final Exception exception,
final String message, final int status) {
logger.log(Level.WARNING, message, exception);
if (logger.isLoggable(Level.FINE)) {
logger.fine(message + ": " + exception);
}
// message ignored!
curi.getNonFatalFailures().add(exception);
curi.setFetchStatus(status);
curi.getRecorder().close();