diff --git a/commons/src/test/java/org/archive/net/UURIFactoryTest.java b/commons/src/test/java/org/archive/net/UURIFactoryTest.java index a708535f..b0827352 100644 --- a/commons/src/test/java/org/archive/net/UURIFactoryTest.java +++ b/commons/src/test/java/org/archive/net/UURIFactoryTest.java @@ -373,6 +373,16 @@ public class UURIFactoryTest extends TestCase { uuriTgt.toString().equals(uuri.toString())); } + public void testSchemelessRelative() throws URIException { + UURI base = UURIFactory.getInstance("http://www.itsnicethat.com/articles/laura-hobson"); + UURI test1 = UURIFactory.getInstance(base, "//www.facebook.com/plugins/like.php"); + assertEquals("schemaless relative 1", "http://www.facebook.com/plugins/like.php", test1.toString()); + // reported by Erin Staniland + UURI test2 = UURIFactory.getInstance(base, "//www.facebook.com/plugins/like.php?href=http://www.itsnicethat.com/articles/laura-hobson"); + assertEquals("schemeless relative 2", "http://www.facebook.com/plugins/like.php?href=http://www.itsnicethat.com/articles/laura-hobson", + test2.toString()); + } + /** * Test that an empty uuri does the right thing -- that we get back the * base. diff --git a/commons/src/test/java/org/archive/net/UURITest.java b/commons/src/test/java/org/archive/net/UURITest.java index 887d2da6..8e5084a5 100644 --- a/commons/src/test/java/org/archive/net/UURITest.java +++ b/commons/src/test/java/org/archive/net/UURITest.java @@ -20,6 +20,8 @@ package org.archive.net; import java.net.URISyntaxException; +import org.apache.commons.httpclient.URIException; + import junit.framework.TestCase; public class UURITest extends TestCase { @@ -41,4 +43,13 @@ public class UURITest extends TestCase { UURI.parseFilename("rsync://archive.org/tmp/one.two/" + filename)); } + + public void testSchemalessRelative() throws URIException { + UURI base = new UURI("http://www.archive.org/a", true, "UTF-8"); + UURI relative = new UURI("//www.facebook.com/?href=http://www.archive.org/a", true, "UTF-8"); + assertEquals(null, relative.getScheme()); + assertEquals("www.facebook.com", relative.getAuthority()); + UURI test = new UURI(base, relative); + assertEquals("http://www.facebook.com/?href=http://www.archive.org/a", test.toString()); + } } diff --git a/modules/src/main/java/org/archive/modules/fetcher/FetchHTTP.java b/modules/src/main/java/org/archive/modules/fetcher/FetchHTTP.java index 7683132f..1bf48eb9 100644 --- a/modules/src/main/java/org/archive/modules/fetcher/FetchHTTP.java +++ b/modules/src/main/java/org/archive/modules/fetcher/FetchHTTP.java @@ -633,7 +633,6 @@ public class FetchHTTP extends AbstractFetchHTTP implements Lifecycle { if (http.getState().getProxyCredentials(new AuthScope(getProxyHost(), getProxyPort())) != null) { addedCredentials = true; } - method.setDoAuthentication(addedCredentials); // set hardMax on bytes (if set by operator) long hardMax = getMaxLengthBytes(); @@ -1147,20 +1146,17 @@ public class FetchHTTP extends AbstractFetchHTTP implements Lifecycle { } catch (URIException e) { return false; } - - boolean result = false; - CrawlServer server = serverCache.getServerFor(serverKey); if (server.hasCredentials()) { for (Credential cred : server.getCredentials()) { if (cred.isEveryTime()) { - if (CommonsHttpCredentialUtil.populate(curi, this.http, method, cred)) { - result = true; - } + CommonsHttpCredentialUtil.populate(curi, this.http, method, cred); } } } + boolean result = false; + // Now look in the curi. The Curi will have credentials loaded either // by the handle401 method if its a rfc2617 or it'll have been set into // the curi by the preconditionenforcer as this login uri came through. @@ -1458,6 +1454,8 @@ public class FetchHTTP extends AbstractFetchHTTP implements Lifecycle { hcp.setSoTimeout(timeout); // Set client to be version 1.0. hcp.setVersion(HttpVersion.HTTP_1_0); + // We handle 401s, so when we do auth, we want it preemptive. + hcp.setAuthenticationPreemptive(true); // configureHttpCookies(defaults);