From 9073d12f962a48b5ec0c670523b2fb17394d68af Mon Sep 17 00:00:00 2001 From: Noah Levitt Date: Thu, 19 Jul 2012 12:36:05 -0700 Subject: [PATCH 1/4] Revert "HER-2015 credentials cached to server never used" This reverts commit 37a8504fda0e02c52738bcb8418e520eccaa1f38. --- .../main/java/org/archive/modules/fetcher/FetchHTTP.java | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/modules/src/main/java/org/archive/modules/fetcher/FetchHTTP.java b/modules/src/main/java/org/archive/modules/fetcher/FetchHTTP.java index 1a1e7576..f0580160 100644 --- a/modules/src/main/java/org/archive/modules/fetcher/FetchHTTP.java +++ b/modules/src/main/java/org/archive/modules/fetcher/FetchHTTP.java @@ -1143,20 +1143,17 @@ public class FetchHTTP extends Processor implements Lifecycle { } catch (URIException e) { return false; } - - boolean result = false; - CrawlServer server = serverCache.getServerFor(serverKey); if (server.hasCredentials()) { for (Credential cred : server.getCredentials()) { if (cred.isEveryTime()) { - if (cred.populate(curi, this.http, method)) { - result = true; - } + cred.populate(curi, this.http, method); } } } + boolean result = false; + // Now look in the curi. The Curi will have credentials loaded either // by the handle401 method if its a rfc2617 or it'll have been set into // the curi by the preconditionenforcer as this login uri came through. From 6f4053cab701d40aeb1fdcf8c8fea7004d5116a5 Mon Sep 17 00:00:00 2001 From: Noah Levitt Date: Thu, 19 Jul 2012 13:15:27 -0700 Subject: [PATCH 2/4] HER-2016 FetchHTTP tries every url twice without credentials before sending credentials * FetchHTTP.java configureHttp() - enable preemptive authentication parameter innerProcess() - do not call method.setDoAuthentication() (it's for automatic 401 handling which is not what we want) --- .../src/main/java/org/archive/modules/fetcher/FetchHTTP.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/modules/src/main/java/org/archive/modules/fetcher/FetchHTTP.java b/modules/src/main/java/org/archive/modules/fetcher/FetchHTTP.java index f0580160..1c020759 100644 --- a/modules/src/main/java/org/archive/modules/fetcher/FetchHTTP.java +++ b/modules/src/main/java/org/archive/modules/fetcher/FetchHTTP.java @@ -633,7 +633,6 @@ public class FetchHTTP extends Processor implements Lifecycle { if (http.getState().getProxyCredentials(new AuthScope(getProxyHost(), getProxyPort())) != null) { addedCredentials = true; } - method.setDoAuthentication(addedCredentials); // set hardMax on bytes (if set by operator) long hardMax = getMaxLengthBytes(); @@ -1451,6 +1450,8 @@ public class FetchHTTP extends Processor implements Lifecycle { hcp.setSoTimeout(timeout); // Set client to be version 1.0. hcp.setVersion(HttpVersion.HTTP_1_0); + // We handle 401s, so when we do auth, we want it preemptive. + hcp.setAuthenticationPreemptive(true); // configureHttpCookies(defaults); From 8fc44b626c052d42058830ebfe32c2518a181dd6 Mon Sep 17 00:00:00 2001 From: Kenji Nagahashi Date: Wed, 20 Jun 2012 18:27:47 -0700 Subject: [PATCH 3/4] added a test case for catching HER-2013 --- .../src/test/java/org/archive/net/UURIFactoryTest.java | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/commons/src/test/java/org/archive/net/UURIFactoryTest.java b/commons/src/test/java/org/archive/net/UURIFactoryTest.java index a708535f..b0827352 100644 --- a/commons/src/test/java/org/archive/net/UURIFactoryTest.java +++ b/commons/src/test/java/org/archive/net/UURIFactoryTest.java @@ -373,6 +373,16 @@ public class UURIFactoryTest extends TestCase { uuriTgt.toString().equals(uuri.toString())); } + public void testSchemelessRelative() throws URIException { + UURI base = UURIFactory.getInstance("http://www.itsnicethat.com/articles/laura-hobson"); + UURI test1 = UURIFactory.getInstance(base, "//www.facebook.com/plugins/like.php"); + assertEquals("schemaless relative 1", "http://www.facebook.com/plugins/like.php", test1.toString()); + // reported by Erin Staniland + UURI test2 = UURIFactory.getInstance(base, "//www.facebook.com/plugins/like.php?href=http://www.itsnicethat.com/articles/laura-hobson"); + assertEquals("schemeless relative 2", "http://www.facebook.com/plugins/like.php?href=http://www.itsnicethat.com/articles/laura-hobson", + test2.toString()); + } + /** * Test that an empty uuri does the right thing -- that we get back the * base. From 3078eac4ab5b1f80e985067565581f8b7e386822 Mon Sep 17 00:00:00 2001 From: Kenji Nagahashi Date: Thu, 19 Jul 2012 14:11:48 -0700 Subject: [PATCH 4/4] test case for scheme-less relative URL --- commons/src/test/java/org/archive/net/UURITest.java | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/commons/src/test/java/org/archive/net/UURITest.java b/commons/src/test/java/org/archive/net/UURITest.java index 887d2da6..8e5084a5 100644 --- a/commons/src/test/java/org/archive/net/UURITest.java +++ b/commons/src/test/java/org/archive/net/UURITest.java @@ -20,6 +20,8 @@ package org.archive.net; import java.net.URISyntaxException; +import org.apache.commons.httpclient.URIException; + import junit.framework.TestCase; public class UURITest extends TestCase { @@ -41,4 +43,13 @@ public class UURITest extends TestCase { UURI.parseFilename("rsync://archive.org/tmp/one.two/" + filename)); } + + public void testSchemalessRelative() throws URIException { + UURI base = new UURI("http://www.archive.org/a", true, "UTF-8"); + UURI relative = new UURI("//www.facebook.com/?href=http://www.archive.org/a", true, "UTF-8"); + assertEquals(null, relative.getScheme()); + assertEquals("www.facebook.com", relative.getAuthority()); + UURI test = new UURI(base, relative); + assertEquals("http://www.facebook.com/?href=http://www.archive.org/a", test.toString()); + } }