String with the character
@@ -321,10 +323,12 @@ public class LaxURI extends URI {
char[] target = tmp.substring(0, at).toLowerCase().toCharArray();
if (validate(target, scheme)) {
_scheme = target;
+ from = ++at;
} else {
- throw new URIException("incorrect scheme");
+ // IA CHANGE:
+ // do nothing; allow interpretation as URI with
+ // later colon in other syntactical component
}
- from = ++at;
}
/*
diff --git a/commons/src/main/java/org/archive/net/UURIFactory.java b/commons/src/main/java/org/archive/net/UURIFactory.java
index 0339bef3..f7021a12 100644
--- a/commons/src/main/java/org/archive/net/UURIFactory.java
+++ b/commons/src/main/java/org/archive/net/UURIFactory.java
@@ -123,15 +123,16 @@ public class UURIFactory extends URI {
*
*
* --
- * Below differs from the rfc regex in that it has java escaping of - * regex characters and we allow a URI made of a fragment only (Added extra + *
Below differs from the rfc regex in that... + * (1) it has java escaping of regex characters + * (2) we allow a URI made of a fragment only (Added extra * group so indexing is off by one after scheme). + * (3) scheme is limited to legal scheme characters */ final public static Pattern RFC2396REGEX = Pattern.compile( - "^(([^:/?#]+):)?((//([^/?#]*))?([^?#]*)(\\?([^#]*))?)?(#(.*))?"); - // 12 34 5 6 7 8 9 A - // 2 1 54 6 87 3 A9 - // 1: scheme + "^(([a-zA-Z][a-zA-Z\\+\\-\\.]*):)?((//([^/?#]*))?([^?#]*)(\\?([^#]*))?)?(#(.*))?"); + // 12 34 5 6 7 8 9 A + // 2 1 54 6 87 3 A9 // 1: scheme // 2: scheme: // 3: //authority/path // 4: //authority diff --git a/commons/src/test/java/org/archive/net/UURIFactoryTest.java b/commons/src/test/java/org/archive/net/UURIFactoryTest.java index dffe2021..a83b0821 100644 --- a/commons/src/test/java/org/archive/net/UURIFactoryTest.java +++ b/commons/src/test/java/org/archive/net/UURIFactoryTest.java @@ -778,6 +778,25 @@ public class UURIFactoryTest extends TestCase { "http://www.example.com/path/:foo"); } + /** + * Ensure that relative URIs with colons in late positions + * aren't mistakenly interpreted as absolute URIs with long, + * illegal schemes. + * + * @throws URIException + */ + public void testLateColon() throws URIException { + UURI base = UURIFactory.getInstance("http://www.example.com/path/page"); + UURI uuri1 = UURIFactory.getInstance(base,"example.html;jsessionid=deadbeef:deadbeed?parameter=this:value"); + assertEquals("derelativize lateColon", + uuri1.getURI(), + "http://www.example.com/path/example.html;jsessionid=deadbeef:deadbeed?parameter=this:value"); + UURI uuri2 = UURIFactory.getInstance(base,"example.html?parameter=this:value"); + assertEquals("derelativize lateColon", + uuri2.getURI(), + "http://www.example.com/path/example.html?parameter=this:value"); + } + /** * Ensure that stray trailing '%' characters do not prevent * UURI instances from being created, and are reasonably diff --git a/modules/src/test/java/org/archive/modules/extractor/ExtractorHTMLTest.java b/modules/src/test/java/org/archive/modules/extractor/ExtractorHTMLTest.java index 580c2ff3..66fa8ab9 100644 --- a/modules/src/test/java/org/archive/modules/extractor/ExtractorHTMLTest.java +++ b/modules/src/test/java/org/archive/modules/extractor/ExtractorHTMLTest.java @@ -204,6 +204,40 @@ public class ExtractorHTMLTest extends StringExtractorTestBase { assertTrue("link extracted despite meta robots",links.length==0); } + /** + * Test that relative URIs with late colons aren't misinterpreted + * as absolute URIs with long, illegal scheme components. + * + * See http://webteam.archive.org/jira/browse/HER-1268 + * + * @throws URIException + */ + public void testBadRelativeLinks() throws URIException { + DefaultProcessorURI curi = new DefaultProcessorURI(UURIFactory + .getInstance("http://www.example.com"), null); + CharSequence cs = "" + + ""; + ExtractorHTML extractor = (ExtractorHTML)makeExtractor(); + extractor.extract(curi, cs); + + assertTrue(CollectionUtils.exists(curi.getOutLinks(), new Predicate() { + public boolean evaluate(Object object) { + return ((Link) object) + .getDestination() + .toString() + .indexOf( + "/example.html;jsessionid=deadbeef:deadbeed?parameter=this:value") >= 0; + } + })); + + assertTrue(CollectionUtils.exists(curi.getOutLinks(), new Predicate() { + public boolean evaluate(Object object) { + return ((Link) object).getDestination().toString().indexOf( + "/example.html?parameter=this:value") >= 0; + } + })); + } + /** * Test if scheme is maintained by speculative hops onto exact * same host