[HER-1480] URIs logged (as -6 or -7) that should never be scheduled

* LaxURI.java
    increase tolerance with regard to path-segments
* UURIFactory.java
    allow digits in scheme regex
    discard old supported-schemes code
* UURIFactoryTest.java
    tests for roundtrip serialization of problematic hostlike-scheme and bars-in-path
* CrawlerLoggerModule.java
    discard special-casing of URI scheme errors
* SchemeNotInSetDecideRule.java
    rule to apply to URIs of unknown schemes; by default REJECTs those not usually handled by Heritrix
* **/profile-crawler-beans.cxml
    add REJECT SchemeNotInSetDecideRule
This commit is contained in:
gojomo committed 2010-09-03 00:24:13 +00:00
1 parent 0347269785
commit bc698c09fc
7 files changed
+118 -63

No files matched your search

@@ -1123,6 +1123,24 @@ public class UURIFactoryTest extends TestCase {
// uuri2 = UURIFactory.getInstance(uuri.toCustomString());
// assertEquals("Not equal", uuri.toString(), uuri2.toString());
}
/**
* A UURI's string representation should be same after a
* toCustomString-getInstance roundtrip.
*
* @throws URIException
*/
public final void testHostnamePortRoundtrip() throws URIException {
UURI base = UURIFactory.
getInstance("http://www.example.com/path?query#anchor");
UURI test = UURIFactory.getInstance(base,"boom1.hostname.com:9999");
System.out.println("scheme:"+test.getScheme());
System.out.println(test.toCustomString());
UURI roundtrip = UURIFactory.getInstance(test.toCustomString());
assertEquals("Not equal", test.toString(), roundtrip.toString());
}
/**
* Test bad port throws URIException not NumberFormatException
*/
@@ -1135,4 +1153,17 @@ public class UURIFactoryTest extends TestCase {
// expected
}
}
/**
* Bars ('|') in path-segments aren't encoded by FF, preferred by some
* RESTful-URI-ideas guides, so should work without error.
*
* @throws URIException
*/
public void testBarsInRelativePath() throws URIException {
UURI base = UURIFactory.getInstance("http://www.example.com");
String relative = "foo/bar|baz|yorple";
base.resolve(relative);
UURIFactory.getInstance(base,relative);
}
}