lee/heritrix3
1
0
Fork 0
mirror of https://github.com/internetarchive/heritrix3.git synced 2026-08-03 05:11:11 +00:00
Code Issues Packages Projects Releases Wiki Activity

new method BdbUriUniqFilter.forgetSchemeHost(String schemeHost), and unit test

Browse Source
This commit is contained in:
Noah Levitt
2013-07-25 18:51:26 -07:00
parent b6b3ae6970
commit b9726c8d4f
2 changed files with 72 additions and 4 deletions
Download Patch File Download Diff File Expand all files Collapse all files
engine/src/test/java/org/archive/crawler/util/BdbUriUniqFilterTest.java
+36
View File
@@ -185,6 +185,42 @@ implements UriUniqFilter.CrawlUriReceiver {
assertEquals("Didn't forget", 0, this.filter.count());
}
public void testForgetSchemeHost() throws URIException {
long countBefore = this.filter.count();
for (String uri: new String[] {
"http://forgetme.com/",
"http://forgetme.com/foo",
"hTtP://fOrGeTmE.cOm/bar",
"http://forgetme.com:80/toot/spuh",
"http://forgetme.com:90/toot/spuh",
"https://forgetme.com/baz",
}) {
CrawlURI curi = new CrawlURI(UURIFactory.getInstance(uri));
System.out.println(uri + " => " + curi);
this.filter.add(curi.getUURI().toCustomString(), curi);
}
assertEquals(countBefore + 6, this.filter.count());
BdbUriUniqFilter bdbFilter = (BdbUriUniqFilter) filter;
assertFalse(bdbFilter.setAdd("http://forgetme.com/foo"));
bdbFilter.forgetSchemeHost("http://forgetme.com");
assertEquals(countBefore + 2, this.filter.count());
assertTrue(bdbFilter.setAdd("http://forgetme.com/foo"));
assertFalse(bdbFilter.setAdd("http://forgetme.com/foo"));
assertTrue(bdbFilter.setRemove("http://forgetme.com/foo"));
assertFalse(bdbFilter.setRemove("http://forgetme.com/foo"));
bdbFilter.forgetSchemeHost("https://forgetme.com/extra-stuff-ignored");
assertEquals(countBefore + 1, this.filter.count());
bdbFilter.forgetSchemeHost("http://forgetme.com:90/");
assertEquals(countBefore, this.filter.count());
}
// TODO: Add testForget when non-empty
public void receive(CrawlURI item) {