Merge branch 'master' into fetchhttp-hc

Conflicts:
	modules/src/main/java/org/archive/modules/fetcher/FetchHTTP.java

commons/src/test/java/org/archive/net/UURIFactoryTest.java

@@ -373,6 +373,16 @@ public class UURIFactoryTest extends TestCase {
 				uuriTgt.toString().equals(uuri.toString()));
 	}
 	
+	public void testSchemelessRelative() throws URIException {
+	    UURI base = UURIFactory.getInstance("http://www.itsnicethat.com/articles/laura-hobson");
+	    UURI test1 = UURIFactory.getInstance(base, "//www.facebook.com/plugins/like.php");
+	    assertEquals("schemaless relative 1", "http://www.facebook.com/plugins/like.php", test1.toString());
+	    // reported by Erin Staniland
+	    UURI test2 = UURIFactory.getInstance(base, "//www.facebook.com/plugins/like.php?href=http://www.itsnicethat.com/articles/laura-hobson");
+	    assertEquals("schemeless relative 2", "http://www.facebook.com/plugins/like.php?href=http://www.itsnicethat.com/articles/laura-hobson",
+	            test2.toString());
+	}
+	
 	/**
 	 * Test that an empty uuri does the right thing -- that we get back the
 	 * base.

commons/src/test/java/org/archive/net/UURITest.java

@@ -20,6 +20,8 @@ package org.archive.net;
 
 import java.net.URISyntaxException;
 
+import org.apache.commons.httpclient.URIException;
+
 import junit.framework.TestCase;
 
 public class UURITest extends TestCase {
@@ -41,4 +43,13 @@ public class UURITest extends TestCase {
             UURI.parseFilename("rsync://archive.org/tmp/one.two/" +
                     filename)); 
     }
+    
+    public void testSchemalessRelative() throws URIException {
+        UURI base = new UURI("http://www.archive.org/a", true, "UTF-8");
+        UURI relative = new UURI("//www.facebook.com/?href=http://www.archive.org/a", true, "UTF-8");
+        assertEquals(null, relative.getScheme());
+        assertEquals("www.facebook.com", relative.getAuthority());
+        UURI test = new UURI(base, relative);
+        assertEquals("http://www.facebook.com/?href=http://www.archive.org/a", test.toString());
+    }
 }

modules/src/main/java/org/archive/modules/fetcher/FetchHTTP.java

@@ -633,7 +633,6 @@ public class FetchHTTP extends AbstractFetchHTTP implements Lifecycle {
         if (http.getState().getProxyCredentials(new AuthScope(getProxyHost(), getProxyPort())) != null) {
             addedCredentials = true;
         }
-        method.setDoAuthentication(addedCredentials);
 
         // set hardMax on bytes (if set by operator)
         long hardMax = getMaxLengthBytes();
@@ -1147,20 +1146,17 @@ public class FetchHTTP extends AbstractFetchHTTP implements Lifecycle {
         } catch (URIException e) {
             return false;
         }
-        
-        boolean result = false;
-        
         CrawlServer server = serverCache.getServerFor(serverKey);
         if (server.hasCredentials()) {
             for (Credential cred : server.getCredentials()) {
                 if (cred.isEveryTime()) {
-                    if (CommonsHttpCredentialUtil.populate(curi, this.http, method, cred)) {
-                        result = true;
-                    }
+			CommonsHttpCredentialUtil.populate(curi, this.http, method, cred);
                 }
             }
         }
 
+        boolean result = false;
+
         // Now look in the curi. The Curi will have credentials loaded either
         // by the handle401 method if its a rfc2617 or it'll have been set into
         // the curi by the preconditionenforcer as this login uri came through.
@@ -1458,6 +1454,8 @@ public class FetchHTTP extends AbstractFetchHTTP implements Lifecycle {
         hcp.setSoTimeout(timeout);
         // Set client to be version 1.0.
         hcp.setVersion(HttpVersion.HTTP_1_0);
+        // We handle 401s, so when we do auth, we want it preemptive.
+        hcp.setAuthenticationPreemptive(true);
 
         // configureHttpCookies(defaults);