Merge branch 'master' into fetchhttp-hc

Conflicts:
	modules/src/main/java/org/archive/modules/CoreAttributeConstants.java
	modules/src/main/java/org/archive/modules/CrawlURI.java
	modules/src/main/java/org/archive/modules/credential/Credential.java
	modules/src/main/java/org/archive/modules/credential/HtmlFormCredential.java
	modules/src/main/java/org/archive/modules/credential/HttpAuthenticationCredential.java
	modules/src/main/java/org/archive/modules/extractor/ExtractorHTTP.java
	modules/src/main/java/org/archive/modules/fetcher/FetchHTTP.java
	modules/src/main/java/org/archive/modules/net/CrawlServer.java

commons/src/main/java/org/archive/util/UriUtils.java

@@ -280,6 +280,13 @@ public class UriUtils {
             return true;
         }
         
+        if (TextUtils.matches("^.*[^:]//.*$", candidate)) {
+            if (LOGGER.isLoggable(Level.FINE)) {
+                LOGGER.fine("rejected: contains '//' (but not '://'): " + candidate);
+            }
+            return true;
+        }
+        
         // look for things that look like hostnames and not filenames?
         // look for too many dots but make sure we take into account that url may have hostname?
 

commons/src/test/java/org/archive/util/UriUtilsTest.java

@@ -210,6 +210,14 @@ public class UriUtilsTest extends TestCase {
         tryAll(startsOrEndsWithPlusFalsePositives, false);
     }
     
+    protected static final String[] doubleSlashFalsePositives = new String[] {
+        ".//*",
+        "http://example.com/monkey//foo/whatever"
+    };
+    public void testDoubleSlashFalsePositives() {
+        tryAll(startsOrEndsWithPlusFalsePositives, false);
+    }
+
     /**
      * Test that all supplied candidates give the expected result, for each of 
      * the 'legacy' (H1) likely-URI-tests

modules/src/main/java/org/archive/modules/CrawlURI.java

@@ -277,7 +277,7 @@ implements Reporter, Serializable, OverlayContext {
             UURIFactory.getInstance(args[2].toString()):
             null;
         LinkContext viaContext = (args.length > 3 && args[2].length()>1) ?
-                new HTMLLinkContext(args[3].toString()): null;
+                HTMLLinkContext.get(args[3].toString()): null;
         CrawlURI caUri = new CrawlURI(u, pathFromSeed, via, viaContext);
         return caUri;
     }

modules/src/main/java/org/archive/modules/extractor/ExtractorHTML.java

@@ -641,7 +641,7 @@ public class ExtractorHTML extends ContentExtractor implements InitializingBean
             // the underlying ReplayCharSequence and the link its about
             // to become a part of is expected to outlive the current
             // ReplayCharSequence.
-            HTMLLinkContext hc = new HTMLLinkContext(context.toString());
+            HTMLLinkContext hc = HTMLLinkContext.get(context.toString());
             int max = getExtractorParameters().getMaxOutlinks();
             Link.addRelativeToBase(curi, max, uri.toString(), hc, hop);
         } catch (URIException e) {

modules/src/main/java/org/archive/modules/extractor/ExtractorHTTP.java

@@ -80,7 +80,7 @@ public class ExtractorHTTP extends Extractor {
         // TODO: consider possibility of multiple headers
         try {
             UURI dest = UURIFactory.getInstance(curi.getUURI(), headerValue);
-            LinkContext lc = new HTMLLinkContext(headerKey + ":"); 
+            LinkContext lc = HTMLLinkContext.get(headerKey + ":");
             Link link = new Link(curi.getUURI(), dest, lc, Hop.REFER);
             curi.getOutLinks().add(link);
             numberOfLinksExtracted.incrementAndGet();

modules/src/main/java/org/archive/modules/extractor/HTMLLinkContext.java

@@ -30,26 +30,63 @@ public class HTMLLinkContext extends LinkContext {
 
     
     final public static HTMLLinkContext META = new HTMLLinkContext("meta");
+    final public static HTMLLinkContext A_HREF = new HTMLLinkContext("a", "href");
+    final public static HTMLLinkContext IMG_SRC = new HTMLLinkContext("img", "src");
+    final public static HTMLLinkContext SCRIPT_SRC = new HTMLLinkContext("script", "src");
+    final public static HTMLLinkContext META_HREF = new HTMLLinkContext("meta", "href");
     
     
     /**
      * The HTML path to the URL.
      */
-    private String path;
-    
+    private final String path;
     
+    /**
+     * return an instance of HTMLLinkContext for attribute {@code attr} in
+     * element {@code el}. returns pre-allocated shared instance for common case,
+     * or new instance for others.
+     * @param el element name
+     * @param attr attribute name
+     * @return instance of HTMLLinkContext
+     */
+    public static HTMLLinkContext get(CharSequence el, CharSequence attr) {
+        if (attr.equals("href") || attr.equals("HREF")) {
+            if (el.equals("a") || el.equals("A")) return A_HREF;
+            if (el.equals("meta") || el.equals("META")) return META_HREF;
+        } else if (attr.equals("src") || attr.equals("SRC")) {
+            if (el.equals("img") || attr.equals("IMG")) return IMG_SRC;
+            if (el.equals("script") || attr.equals("SCRIPT")) return SCRIPT_SRC;
+        }
+        return new HTMLLinkContext(el, attr);
+    }
+    /**
+     * return an instance of HTMLLinkContext for path {@code path}.
+     * returns pre-allocated shared instance for common case, or new instance for others.
+     * <p>TODO: most code calling this method builds path by concatenating element name
+     * and attribute name. consider changing such code to call {@link #get(CharSequence, CharSequence)}
+     * instead.</p> 
+     * @param path element and attribute in XLink-like path notation
+     * @return instance of HTMLLinkContext
+     */
+    public static HTMLLinkContext get(String path) {
+        if (path.equalsIgnoreCase("a/@href")) return A_HREF;
+        if (path.equalsIgnoreCase("meta/@href")) return META_HREF;
+        if (path.equalsIgnoreCase("img/@src")) return IMG_SRC;
+        if (path.equalsIgnoreCase("script/@src")) return SCRIPT_SRC;
+        return new HTMLLinkContext(path);
+    }
     /**
      * Constructor.
      * 
      * @param path   an XPath-like context, eg "A\@HREF"
      */
-    public HTMLLinkContext(String path) {
+    protected HTMLLinkContext(String path) {
         // FIXME: Verify that path really is XPath-like
         this.path = path;
     }
 
     
-    public HTMLLinkContext(CharSequence element, CharSequence attribute) {
+    protected HTMLLinkContext(CharSequence element, CharSequence attribute) {
         if (attribute == null) {
             this.path = "";
         } else {

modules/src/test/java/org/archive/modules/extractor/ExtractorHTMLTest.java

@@ -126,10 +126,10 @@ public class ExtractorHTMLTest extends StringExtractorTestBase {
     
     private static LinkContext determineContext(String s) {
         if (s.endsWith(" A")) {
-            return new HTMLLinkContext("a/@href");
+            return HTMLLinkContext.get("a/@href");
         }
         if (s.endsWith(" IMG")) {
-            return new HTMLLinkContext("img/@src");
+            return HTMLLinkContext.get("img/@src");
         }
         return LinkContext.NAVLINK_MISC;
     }