From 1fe4781cf5f3788bd0b5e90dc66697019d7f8358 Mon Sep 17 00:00:00 2001 From: Noah Levitt Date: Sun, 2 Sep 2012 01:37:00 -0700 Subject: [PATCH] axLengthBytes, sendRange --- .../modules/fetcher/FetchHTTPTestBase.java | 29 ++++++++++++++----- .../modules/fetcher/AbstractFetchHTTP.java | 2 ++ .../archive/modules/fetcher/FetchHTTP.java | 4 +-- .../archive/modules/fetcher/FetchHTTP2.java | 25 ++++++++++++++++ 4 files changed, 51 insertions(+), 9 deletions(-) diff --git a/engine/src/test/java/org/archive/modules/fetcher/FetchHTTPTestBase.java b/engine/src/test/java/org/archive/modules/fetcher/FetchHTTPTestBase.java index bac01988..c60b11d8 100644 --- a/engine/src/test/java/org/archive/modules/fetcher/FetchHTTPTestBase.java +++ b/engine/src/test/java/org/archive/modules/fetcher/FetchHTTPTestBase.java @@ -691,7 +691,7 @@ public abstract class FetchHTTPTestBase extends ProcessorTestBase { httpProxyServer.addProxyAuthenticationHandler(new ProxyAuthorizationHandler() { @Override public boolean authenticate(String userName, String password) { - logger.info("username=" + userName + " password=" + password); + // logger.info("username=" + userName + " password=" + password); return "http-proxy-user".equals(userName) && "http-proxy-password".equals(password); } }); @@ -705,7 +705,7 @@ public abstract class FetchHTTPTestBase extends ProcessorTestBase { CrawlURI curi = makeCrawlURI("http://localhost:7777/"); getFetcher().process(curi); - logger.info('\n' + httpRequestString(curi) + "\n\n" + rawResponseString(curi)); + // logger.info('\n' + httpRequestString(curi) + "\n\n" + rawResponseString(curi)); String requestString = httpRequestString(curi); assertTrue(requestString.startsWith("GET http://localhost:7777/ HTTP/1.0\r\n")); @@ -717,7 +717,7 @@ public abstract class FetchHTTPTestBase extends ProcessorTestBase { proxiedRequestRememberer.clear(); curi = makeCrawlURI("http://localhost:7777/"); getFetcher().process(curi); - logger.info('\n' + httpRequestString(curi) + "\n\n" + rawResponseString(curi)); + // logger.info('\n' + httpRequestString(curi) + "\n\n" + rawResponseString(curi)); requestString = httpRequestString(curi); assertTrue(requestString.startsWith("GET http://localhost:7777/ HTTP/1.0\r\n")); @@ -732,14 +732,29 @@ public abstract class FetchHTTPTestBase extends ProcessorTestBase { public void testMaxFetchKBSec() throws Exception { ensureHttpServers(); - CrawlURI curi = makeCrawlURI("http://localhost:7777/200k"); getFetcher().setMaxFetchKBSec(100); - getFetcher().process(curi); - assertEquals(200000, curi.getContentLength()); - assertTrue(curi.getFetchDuration() > 1800 && curi.getFetchDuration() < 2200); } + + public void testMaxLengthBytes() throws Exception { + ensureHttpServers(); + CrawlURI curi = makeCrawlURI("http://localhost:7777/200k"); + getFetcher().setMaxLengthBytes(50000); + getFetcher().process(curi); + assertEquals(50001, curi.getRecordedSize()); + } + + public void testSendRange() throws Exception { + ensureHttpServers(); + CrawlURI curi = makeCrawlURI("http://localhost:7777/200k"); + getFetcher().setMaxLengthBytes(50000); + getFetcher().setSendRange(true); + getFetcher().process(curi); + logger.info("\n" + httpRequestString(curi)); + assertTrue(httpRequestString(curi).contains("Range: bytes=0-49999\r\n")); + // assertEquals(50000, curi.getRecordedSize()); + } } diff --git a/modules/src/main/java/org/archive/modules/fetcher/AbstractFetchHTTP.java b/modules/src/main/java/org/archive/modules/fetcher/AbstractFetchHTTP.java index aa80fc90..7695aab2 100644 --- a/modules/src/main/java/org/archive/modules/fetcher/AbstractFetchHTTP.java +++ b/modules/src/main/java/org/archive/modules/fetcher/AbstractFetchHTTP.java @@ -36,4 +36,6 @@ abstract public class AbstractFetchHTTP extends Processor { abstract public void setHttpProxyUser(String user); abstract public void setHttpProxyPassword(String password); abstract public void setMaxFetchKBSec(int maxFetchKBSec); + abstract public void setMaxLengthBytes(long maxLengthBytes); + abstract public void setSendRange(boolean sendRange); } diff --git a/modules/src/main/java/org/archive/modules/fetcher/FetchHTTP.java b/modules/src/main/java/org/archive/modules/fetcher/FetchHTTP.java index 31993210..7c27181d 100644 --- a/modules/src/main/java/org/archive/modules/fetcher/FetchHTTP.java +++ b/modules/src/main/java/org/archive/modules/fetcher/FetchHTTP.java @@ -219,8 +219,8 @@ public class FetchHTTP extends AbstractFetchHTTP implements Lifecycle { public long getMaxLengthBytes() { return (Long) kp.get("maxLengthBytes"); } - public void setMaxLengthBytes(long timeout) { - kp.put("maxLengthBytes",timeout); + public void setMaxLengthBytes(long maxLengthBytes) { + kp.put("maxLengthBytes",maxLengthBytes); } /** diff --git a/modules/src/main/java/org/archive/modules/fetcher/FetchHTTP2.java b/modules/src/main/java/org/archive/modules/fetcher/FetchHTTP2.java index 895b2714..f75d2555 100644 --- a/modules/src/main/java/org/archive/modules/fetcher/FetchHTTP2.java +++ b/modules/src/main/java/org/archive/modules/fetcher/FetchHTTP2.java @@ -394,6 +394,26 @@ public class FetchHTTP2 extends AbstractFetchHTTP implements Lifecycle { kp.put("maxLengthBytes",timeout); } + /** + * Send 'Range' header when a limit ({@link #MAX_LENGTH_BYTES}) on + * document size. + *

+ * Be polite to the HTTP servers and send the 'Range' header, stating that + * you are only interested in the first n bytes. Only pertinent if + * {@link #MAX_LENGTH_BYTES} > 0. Sending the 'Range' header results in a + * '206 Partial Content' status response, which is better than just cutting + * the response mid-download. On rare occasion, sending 'Range' will + * generate '416 Request Range Not Satisfiable' response. + */ + { + setSendRange(false); + } + public boolean getSendRange() { + return (Boolean) kp.get("sendRange"); + } + public void setSendRange(boolean sendRange) { + kp.put("sendRange",sendRange); + } protected static final Header HEADER_SEND_CONNECTION_CLOSE = new BasicHeader( HTTP.CONN_DIRECTIVE, HTTP.CONN_CLOSE); @@ -958,6 +978,11 @@ public class FetchHTTP2 extends AbstractFetchHTTP implements Lifecycle { request.setHeader("From", from); } + if (getMaxLengthBytes() > 0 && getSendRange()) { + request.setHeader(RANGE, RANGE_PREFIX.concat(Long + .toString(getMaxLengthBytes() - 1))); + } + if (getSendConnectionClose()) { request.setHeader(HEADER_SEND_CONNECTION_CLOSE); }