lee/heritrix3
1
0
Fork 0
mirror of https://github.com/internetarchive/heritrix3.git synced 2026-08-22 14:36:37 +00:00
Code Issues Packages Projects Releases Wiki Activity

whoops! send escaped path+query on http request line; had been sending raw spaces (includes unit test)

Browse Source
This commit is contained in:
Noah Levitt
2014-01-27 16:57:37 -08:00
parent 21f04cdf79
commit b803ea0ec0
2 changed files with 13 additions and 1 deletions
Download Patch File Download Diff File Expand all files Collapse all files
modules/src/main/java/org/archive/modules/fetcher/FetchHTTPRequest.java
+1 -1
View File
@@ -167,7 +167,7 @@ class FetchHTTPRequest {
this.requestConfigBuilder.setProxy(this.proxyHost);
requestLineUri = curi.getUURI().toString();
} else {
requestLineUri = curi.getUURI().getPathQuery();
requestLineUri = curi.getUURI().getEscapedPathQuery();
}
if (curi.getFetchType() == FetchType.HTTP_POST) {
modules/src/test/java/org/archive/modules/fetcher/FetchHTTPTests.java
+12
View File
@@ -763,6 +763,18 @@ public class FetchHTTPTests extends ProcessorTestBase {
assertTrue(httpRequestString(curi).startsWith("GET /??blahblah HTTP/1.0\r\n"));
runDefaultChecks(curi, "requestLine");
}
public void testUrlWithSpaces() throws Exception {
CrawlURI curi = makeCrawlURI("http://localhost:7777/url with spaces");
fetcher().process(curi);
assertTrue(httpRequestString(curi).startsWith("GET /url%20with%20spaces HTTP/1.0\r\n"));
runDefaultChecks(curi, "requestLine");
curi = makeCrawlURI("http://localhost:7777/url%20with%20spaces");
fetcher().process(curi);
assertTrue(httpRequestString(curi).startsWith("GET /url%20with%20spaces HTTP/1.0\r\n"));
runDefaultChecks(curi, "requestLine");
}
@Override
protected FetchHTTP makeModule() throws IOException {