lee/heritrix3
1
0
Fork 0
mirror of https://github.com/internetarchive/heritrix3.git synced 2026-09-24 14:45:56 +00:00
Code Issues Packages Projects Releases Wiki Activity

Fix for [HER-1268] ExtractorHTML not able to extract some links (due to them violating RFC2396)

Browse Source 
* ExtractorHTMLTest.java, UURIFactoryTest.java
    test that relative-URIs with late-position colons aren't interpreted as absolute URIs with long, illegal schemes
* UURIFactory.java
    update RFC2396REGEX to require legal scheme
* LaxURI.java
    allow illegal candidate scheme to just be interpreted as something else
This commit is contained in:
gojomo
2009-05-12 22:22:11 +00:00
parent 252cf63863
commit 2dec5dfd1a
4 changed files with 66 additions and 8 deletions
Download Patch File Download Diff File Expand all files Collapse all files
modules/src/test/java/org/archive/modules/extractor/ExtractorHTMLTest.java
+34
View File
@@ -204,6 +204,40 @@ public class ExtractorHTMLTest extends StringExtractorTestBase {
assertTrue("link extracted despite meta robots",links.length==0);
}
/**
* Test that relative URIs with late colons aren't misinterpreted
* as absolute URIs with long, illegal scheme components.
*
* See http://webteam.archive.org/jira/browse/HER-1268
*
* @throws URIException
*/
public void testBadRelativeLinks() throws URIException {
DefaultProcessorURI curi = new DefaultProcessorURI(UURIFactory
.getInstance("http://www.example.com"), null);
CharSequence cs = "<a href=\"example.html;jsessionid=deadbeef:deadbeed?parameter=this:value\"/>"
+ "<a href=\"example.html?parameter=this:value\"/>";
ExtractorHTML extractor = (ExtractorHTML)makeExtractor();
extractor.extract(curi, cs);
assertTrue(CollectionUtils.exists(curi.getOutLinks(), new Predicate() {
public boolean evaluate(Object object) {
return ((Link) object)
.getDestination()
.toString()
.indexOf(
"/example.html;jsessionid=deadbeef:deadbeed?parameter=this:value") >= 0;
}
}));
assertTrue(CollectionUtils.exists(curi.getOutLinks(), new Predicate() {
public boolean evaluate(Object object) {
return ((Link) object).getDestination().toString().indexOf(
"/example.html?parameter=this:value") >= 0;
}
}));
}
/**
* Test if scheme is maintained by speculative hops onto exact
* same host