mirror of
https://github.com/internetarchive/heritrix3.git
synced 2026-10-08 05:11:20 +00:00
[HER-1480] URIs logged (as -6 or -7) that should never be scheduled
* LaxURI.java
increase tolerance with regard to path-segments
* UURIFactory.java
allow digits in scheme regex
discard old supported-schemes code
* UURIFactoryTest.java
tests for roundtrip serialization of problematic hostlike-scheme and bars-in-path
* CrawlerLoggerModule.java
discard special-casing of URI scheme errors
* SchemeNotInSetDecideRule.java
rule to apply to URIs of unknown schemes; by default REJECTs those not usually handled by Heritrix
* **/profile-crawler-beans.cxml
add REJECT SchemeNotInSetDecideRule
This commit is contained in:
1 parent
0347269785
commit
bc698c09fc
7 files changed
+118
-63
No files matched your search
@@ -1123,6 +1123,24 @@ public class UURIFactoryTest extends TestCase {
|
||||
// uuri2 = UURIFactory.getInstance(uuri.toCustomString());
|
||||
// assertEquals("Not equal", uuri.toString(), uuri2.toString());
|
||||
}
|
||||
|
||||
/**
|
||||
* A UURI's string representation should be same after a
|
||||
* toCustomString-getInstance roundtrip.
|
||||
*
|
||||
* @throws URIException
|
||||
*/
|
||||
public final void testHostnamePortRoundtrip() throws URIException {
|
||||
UURI base = UURIFactory.
|
||||
getInstance("http://www.example.com/path?query#anchor");
|
||||
UURI test = UURIFactory.getInstance(base,"boom1.hostname.com:9999");
|
||||
System.out.println("scheme:"+test.getScheme());
|
||||
System.out.println(test.toCustomString());
|
||||
UURI roundtrip = UURIFactory.getInstance(test.toCustomString());
|
||||
assertEquals("Not equal", test.toString(), roundtrip.toString());
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* Test bad port throws URIException not NumberFormatException
|
||||
*/
|
||||
@@ -1135,4 +1153,17 @@ public class UURIFactoryTest extends TestCase {
|
||||
// expected
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Bars ('|') in path-segments aren't encoded by FF, preferred by some
|
||||
* RESTful-URI-ideas guides, so should work without error.
|
||||
*
|
||||
* @throws URIException
|
||||
*/
|
||||
public void testBarsInRelativePath() throws URIException {
|
||||
UURI base = UURIFactory.getInstance("http://www.example.com");
|
||||
String relative = "foo/bar|baz|yorple";
|
||||
base.resolve(relative);
|
||||
UURIFactory.getInstance(base,relative);
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user