Fix for [HER-1268] ExtractorHTML not able to extract some links (due to them violating RFC2396)

* ExtractorHTMLTest.java, UURIFactoryTest.java
    test that relative-URIs with late-position colons aren't interpreted as absolute URIs with long, illegal schemes
* UURIFactory.java
    update RFC2396REGEX to require legal scheme
* LaxURI.java
    allow illegal candidate scheme to just be interpreted as something else

commons/src/main/java/org/archive/net/LaxURI.java

@@ -217,6 +217,8 @@ public class LaxURI extends URI {
     /**
      * IA OVERRIDDEN IN LaxURI TO INCLUDE FIX FOR 
      * http://issues.apache.org/jira/browse/HTTPCLIENT-588
+     * AND
+     * http://webteam.archive.org/jira/browse/HER-1268
      * 
      * In order to avoid any possilbity of conflict with non-ASCII characters,
      * Parse a URI reference as a <code>String</code> with the character
@@ -321,10 +323,12 @@ public class LaxURI extends URI {
             char[] target = tmp.substring(0, at).toLowerCase().toCharArray();
             if (validate(target, scheme)) {
                 _scheme = target;
+                from = ++at;
             } else {
-                throw new URIException("incorrect scheme");
+                // IA CHANGE:
+                // do nothing; allow interpretation as URI with 
+                // later colon in other syntactical component
             }
-            from = ++at;
         }
 
         /*

commons/src/main/java/org/archive/net/UURIFactory.java

@@ -123,15 +123,16 @@ public class UURIFactory extends URI {
      * </pre>
      *
      * -- 
-     * <p>Below differs from the rfc regex in that it has java escaping of
-     * regex characters and we allow a URI made of a fragment only (Added extra
+     * <p>Below differs from the rfc regex in that... 
+     * (1) it has java escaping of regex characters 
+     * (2) we allow a URI made of a fragment only (Added extra
      * group so indexing is off by one after scheme).
+     * (3) scheme is limited to legal scheme characters 
      */
     final public static Pattern RFC2396REGEX = Pattern.compile(
-        "^(([^:/?#]+):)?((//([^/?#]*))?([^?#]*)(\\?([^#]*))?)?(#(.*))?");
-    //    12            34  5          6       7   8          9 A
-    //              2 1             54        6          87 3      A9
-    // 1: scheme
+        "^(([a-zA-Z][a-zA-Z\\+\\-\\.]*):)?((//([^/?#]*))?([^?#]*)(\\?([^#]*))?)?(#(.*))?");
+    //    12                             34  5          6       7   8          9 A
+    //                                2 1             54        6          87 3      A9    // 1: scheme
     // 2: scheme:
     // 3: //authority/path
     // 4: //authority

commons/src/test/java/org/archive/net/UURIFactoryTest.java

@@ -778,6 +778,25 @@ public class UURIFactoryTest extends TestCase {
                 "http://www.example.com/path/:foo");
     }
     
+    /**
+     * Ensure that relative URIs with colons in late positions 
+     * aren't mistakenly interpreted as absolute URIs with long, 
+     * illegal schemes. 
+     * 
+     * @throws URIException
+     */
+    public void testLateColon() throws URIException {
+        UURI base = UURIFactory.getInstance("http://www.example.com/path/page");
+        UURI uuri1 = UURIFactory.getInstance(base,"example.html;jsessionid=deadbeef:deadbeed?parameter=this:value");
+        assertEquals("derelativize lateColon",
+                uuri1.getURI(),
+                "http://www.example.com/path/example.html;jsessionid=deadbeef:deadbeed?parameter=this:value");
+        UURI uuri2 = UURIFactory.getInstance(base,"example.html?parameter=this:value");
+        assertEquals("derelativize lateColon",
+                uuri2.getURI(),
+                "http://www.example.com/path/example.html?parameter=this:value");
+    }
+    
     /**
      * Ensure that stray trailing '%' characters do not prevent
      * UURI instances from being created, and are reasonably 

modules/src/test/java/org/archive/modules/extractor/ExtractorHTMLTest.java

@@ -204,6 +204,40 @@ public class ExtractorHTMLTest extends StringExtractorTestBase {
         assertTrue("link extracted despite meta robots",links.length==0);
     }
     
+    /**
+     * Test that relative URIs with late colons aren't misinterpreted
+     * as absolute URIs with long, illegal scheme components. 
+     * 
+     * See http://webteam.archive.org/jira/browse/HER-1268
+     * 
+     * @throws URIException
+     */
+    public void testBadRelativeLinks() throws URIException {
+        DefaultProcessorURI curi = new DefaultProcessorURI(UURIFactory
+                .getInstance("http://www.example.com"), null);
+        CharSequence cs = "<a href=\"example.html;jsessionid=deadbeef:deadbeed?parameter=this:value\"/>"
+                + "<a href=\"example.html?parameter=this:value\"/>";
+        ExtractorHTML extractor = (ExtractorHTML)makeExtractor();
+        extractor.extract(curi, cs);
+
+        assertTrue(CollectionUtils.exists(curi.getOutLinks(), new Predicate() {
+            public boolean evaluate(Object object) {
+                return ((Link) object)
+                        .getDestination()
+                        .toString()
+                        .indexOf(
+                                "/example.html;jsessionid=deadbeef:deadbeed?parameter=this:value") >= 0;
+            }
+        }));
+
+        assertTrue(CollectionUtils.exists(curi.getOutLinks(), new Predicate() {
+            public boolean evaluate(Object object) {
+                return ((Link) object).getDestination().toString().indexOf(
+                        "/example.html?parameter=this:value") >= 0;
+            }
+        }));
+    }
+    
     /**
      * Test if scheme is maintained by speculative hops onto exact 
      * same host