From a06c14174080d8b8084ebeebe03ef8bc6e27a04d Mon Sep 17 00:00:00 2001 From: Noah Levitt Date: Wed, 21 Aug 2013 11:31:05 -0700 Subject: [PATCH] rename BdbUriUniqFilter.forgetSchemeHost() to forgetAllSchemeAuthorityMatching(), improve performance --- .../crawler/framework/CheckpointService.java | 1 - .../crawler/util/BdbUriUniqFilter.java | 44 +++++++++++++------ .../crawler/util/BdbUriUniqFilterTest.java | 8 ++-- 3 files changed, 34 insertions(+), 19 deletions(-) diff --git a/engine/src/main/java/org/archive/crawler/framework/CheckpointService.java b/engine/src/main/java/org/archive/crawler/framework/CheckpointService.java index 0e939bca..564c0916 100644 --- a/engine/src/main/java/org/archive/crawler/framework/CheckpointService.java +++ b/engine/src/main/java/org/archive/crawler/framework/CheckpointService.java @@ -40,7 +40,6 @@ import org.archive.spring.ConfigPath; import org.archive.spring.ConfigPathConfigurer; import org.archive.spring.HasValidator; import org.springframework.beans.BeansException; -import org.springframework.beans.factory.BeanFactoryUtils; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.context.ApplicationContext; import org.springframework.context.ApplicationContextAware; diff --git a/engine/src/main/java/org/archive/crawler/util/BdbUriUniqFilter.java b/engine/src/main/java/org/archive/crawler/util/BdbUriUniqFilter.java index dfff305a..e3355c81 100644 --- a/engine/src/main/java/org/archive/crawler/util/BdbUriUniqFilter.java +++ b/engine/src/main/java/org/archive/crawler/util/BdbUriUniqFilter.java @@ -303,17 +303,17 @@ implements Lifecycle, Checkpointable, BeanNameAware, DisposableBean { */ public static long createKey(CharSequence uri) { String url = uri.toString(); - long schemeHostKeyPart = calcSchemeHostKeyPart(url); - return schemeHostKeyPart | (FPGenerator.std40.fp(url) >>> 24); + long schemeAuthorityKeyPart = calcSchemeAuthorityKeyBytes(url); + return schemeAuthorityKeyPart | (FPGenerator.std40.fp(url) >>> 24); } - protected static long calcSchemeHostKeyPart(String url) { + protected static long calcSchemeAuthorityKeyBytes(String url) { int index = url.indexOf(COLON_SLASH_SLASH); if (index > 0) { index = url.indexOf('/', index + COLON_SLASH_SLASH.length()); } - CharSequence hostPlusScheme = (index == -1)? url: url.subSequence(0, index); - return FPGenerator.std24.fp(hostPlusScheme); + CharSequence schemeAuthority = (index == -1)? url: url.subSequence(0, index); + return FPGenerator.std24.fp(schemeAuthority); } protected boolean setAdd(CharSequence uri) { @@ -400,7 +400,7 @@ implements Lifecycle, Checkpointable, BeanNameAware, DisposableBean { } /** - * Forget all entries that match the scheme+host+port of the given key, so + * Forget all entries that match the scheme+host+port of the given url, so * that they can be crawled again if discovered again. Expensive operation. * *

@@ -408,26 +408,42 @@ implements Lifecycle, Checkpointable, BeanNameAware, DisposableBean { * grouping of urls that is feasible to forget in bulk. See * {@link #createKey(CharSequence)} * - * @param schemeHost + *

+ * WARNING: Value collisions in this 24-bit schemeAuthority part are going + * to be fairly common, by 'birthday problem' over 50% likely to show up + * with as few as 2^12 unique schemeAuthority strings. So the forgetting may + * forget other hosts. + * + * @param url + * whose scheme+host+port should be forgotten (remainder of url + * is ignored) */ - public void forgetSchemeHost(String schemeHost) { - long schemeHostKeyPart = calcSchemeHostKeyPart(schemeHost); + public void forgetAllSchemeAuthorityMatching(String url) { + long schemeAuthorityKeyBytes = calcSchemeAuthorityKeyBytes(url); DatabaseEntry key = new DatabaseEntry(); DatabaseEntry value = new DatabaseEntry(); + + LongBinding.longToEntry(schemeAuthorityKeyBytes, key); Cursor cursor = alreadySeen.openCursor(null, null); long forgottenCount = 0l; - while (cursor.getNext(key, value, null) == OperationStatus.SUCCESS) { + + for (OperationStatus status = cursor.getSearchKeyRange(key, value, null); + status == OperationStatus.SUCCESS; + status = cursor.getNext(key, value, null)) { + long alreadySeenKey = LongBinding.entryToLong(key); - // System.out.printf("schemeHostKeyPart=%017x alreadySeenKey=%017x\n", schemeHostKeyPart, alreadySeenKey); - if ((alreadySeenKey & 0xffffff0000000000l) == schemeHostKeyPart) { + if ((alreadySeenKey & 0xffffff0000000000l) == schemeAuthorityKeyBytes) { cursor.delete(); - count.decrementAndGet(); forgottenCount++; + } else { + break; } } + cursor.close(); - logger.info("forgot " + forgottenCount + " urls from scheme+host+port " + schemeHost + " (" + count.get() + " urls left)"); + long newCount = count.addAndGet(-forgottenCount); + logger.info("forgot " + forgottenCount + " urls from scheme+authority of url " + url + " (leaving " + newCount + " urls from other scheme+authorities)"); } } //EOC \ No newline at end of file diff --git a/engine/src/test/java/org/archive/crawler/util/BdbUriUniqFilterTest.java b/engine/src/test/java/org/archive/crawler/util/BdbUriUniqFilterTest.java index c2b92849..bb11825e 100644 --- a/engine/src/test/java/org/archive/crawler/util/BdbUriUniqFilterTest.java +++ b/engine/src/test/java/org/archive/crawler/util/BdbUriUniqFilterTest.java @@ -185,7 +185,7 @@ implements UriUniqFilter.CrawlUriReceiver { assertEquals("Didn't forget", 0, this.filter.count()); } - public void testForgetSchemeHost() throws URIException { + public void testForgetAllSchemeAuthorityMatching() throws URIException { long countBefore = this.filter.count(); for (String uri: new String[] { @@ -205,7 +205,7 @@ implements UriUniqFilter.CrawlUriReceiver { BdbUriUniqFilter bdbFilter = (BdbUriUniqFilter) filter; assertFalse(bdbFilter.setAdd("http://forgetme.com/foo")); - bdbFilter.forgetSchemeHost("http://forgetme.com"); + bdbFilter.forgetAllSchemeAuthorityMatching("http://forgetme.com"); assertEquals(countBefore + 2, this.filter.count()); assertTrue(bdbFilter.setAdd("http://forgetme.com/foo")); @@ -213,10 +213,10 @@ implements UriUniqFilter.CrawlUriReceiver { assertTrue(bdbFilter.setRemove("http://forgetme.com/foo")); assertFalse(bdbFilter.setRemove("http://forgetme.com/foo")); - bdbFilter.forgetSchemeHost("https://forgetme.com/extra-stuff-ignored"); + bdbFilter.forgetAllSchemeAuthorityMatching("https://forgetme.com/extra-stuff-ignored"); assertEquals(countBefore + 1, this.filter.count()); - bdbFilter.forgetSchemeHost("http://forgetme.com:90/"); + bdbFilter.forgetAllSchemeAuthorityMatching("http://forgetme.com:90/"); assertEquals(countBefore, this.filter.count()); }