From b803ea0ec03b4e9d2b6a7bdaead8ccfc4ae760f9 Mon Sep 17 00:00:00 2001 From: Noah Levitt Date: Mon, 27 Jan 2014 16:57:37 -0800 Subject: [PATCH] whoops! send escaped path+query on http request line; had been sending raw spaces (includes unit test) --- .../archive/modules/fetcher/FetchHTTPRequest.java | 2 +- .../org/archive/modules/fetcher/FetchHTTPTests.java | 12 ++++++++++++ 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/modules/src/main/java/org/archive/modules/fetcher/FetchHTTPRequest.java b/modules/src/main/java/org/archive/modules/fetcher/FetchHTTPRequest.java index b5318067..b227583e 100644 --- a/modules/src/main/java/org/archive/modules/fetcher/FetchHTTPRequest.java +++ b/modules/src/main/java/org/archive/modules/fetcher/FetchHTTPRequest.java @@ -167,7 +167,7 @@ class FetchHTTPRequest { this.requestConfigBuilder.setProxy(this.proxyHost); requestLineUri = curi.getUURI().toString(); } else { - requestLineUri = curi.getUURI().getPathQuery(); + requestLineUri = curi.getUURI().getEscapedPathQuery(); } if (curi.getFetchType() == FetchType.HTTP_POST) { diff --git a/modules/src/test/java/org/archive/modules/fetcher/FetchHTTPTests.java b/modules/src/test/java/org/archive/modules/fetcher/FetchHTTPTests.java index 4f2c44d6..53edba3b 100644 --- a/modules/src/test/java/org/archive/modules/fetcher/FetchHTTPTests.java +++ b/modules/src/test/java/org/archive/modules/fetcher/FetchHTTPTests.java @@ -763,6 +763,18 @@ public class FetchHTTPTests extends ProcessorTestBase { assertTrue(httpRequestString(curi).startsWith("GET /??blahblah HTTP/1.0\r\n")); runDefaultChecks(curi, "requestLine"); } + + public void testUrlWithSpaces() throws Exception { + CrawlURI curi = makeCrawlURI("http://localhost:7777/url with spaces"); + fetcher().process(curi); + assertTrue(httpRequestString(curi).startsWith("GET /url%20with%20spaces HTTP/1.0\r\n")); + runDefaultChecks(curi, "requestLine"); + + curi = makeCrawlURI("http://localhost:7777/url%20with%20spaces"); + fetcher().process(curi); + assertTrue(httpRequestString(curi).startsWith("GET /url%20with%20spaces HTTP/1.0\r\n")); + runDefaultChecks(curi, "requestLine"); + } @Override protected FetchHTTP makeModule() throws IOException {