lee/heritrix3
1
0
Fork 0
mirror of https://github.com/internetarchive/heritrix3.git synced 2026-09-23 14:15:47 +00:00
Code Issues Packages Projects Releases Wiki Activity

rename BdbUriUniqFilter.forgetSchemeHost() to forgetAllSchemeAuthorityMatching(), improve performance

Browse Source
This commit is contained in:
Noah Levitt
2013-08-21 11:31:05 -07:00
parent a6958951df
commit a06c141740
3 changed files with 34 additions and 19 deletions
Download Patch File Download Diff File Expand all files Collapse all files
engine/src/main/java/org/archive/crawler/framework/CheckpointService.java
-1
View File
@@ -40,7 +40,6 @@ import org.archive.spring.ConfigPath;
import org.archive.spring.ConfigPathConfigurer;
import org.archive.spring.HasValidator;
import org.springframework.beans.BeansException;
import org.springframework.beans.factory.BeanFactoryUtils;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.context.ApplicationContext;
import org.springframework.context.ApplicationContextAware;
engine/src/main/java/org/archive/crawler/util/BdbUriUniqFilter.java
+30 -14
View File
@@ -303,17 +303,17 @@ implements Lifecycle, Checkpointable, BeanNameAware, DisposableBean {
*/
public static long createKey(CharSequence uri) {
String url = uri.toString();
long schemeHostKeyPart = calcSchemeHostKeyPart(url);
return schemeHostKeyPart | (FPGenerator.std40.fp(url) >>> 24);
long schemeAuthorityKeyPart = calcSchemeAuthorityKeyBytes(url);
return schemeAuthorityKeyPart | (FPGenerator.std40.fp(url) >>> 24);
}
protected static long calcSchemeHostKeyPart(String url) {
protected static long calcSchemeAuthorityKeyBytes(String url) {
int index = url.indexOf(COLON_SLASH_SLASH);
if (index > 0) {
index = url.indexOf('/', index + COLON_SLASH_SLASH.length());
}
CharSequence hostPlusScheme = (index == -1)? url: url.subSequence(0, index);
return FPGenerator.std24.fp(hostPlusScheme);
CharSequence schemeAuthority = (index == -1)? url: url.subSequence(0, index);
return FPGenerator.std24.fp(schemeAuthority);
}
protected boolean setAdd(CharSequence uri) {
@@ -400,7 +400,7 @@ implements Lifecycle, Checkpointable, BeanNameAware, DisposableBean {
}
/**
* Forget all entries that match the scheme+host+port of the given key, so
* Forget all entries that match the scheme+host+port of the given url, so
* that they can be crawled again if discovered again. Expensive operation.
*
* <p>
@@ -408,26 +408,42 @@ implements Lifecycle, Checkpointable, BeanNameAware, DisposableBean {
* grouping of urls that is feasible to forget in bulk. See
* {@link #createKey(CharSequence)}
*
* @param schemeHost
* <p>
* WARNING: Value collisions in this 24-bit schemeAuthority part are going
* to be fairly common, by 'birthday problem' over 50% likely to show up
* with as few as 2^12 unique schemeAuthority strings. So the forgetting may
* forget other hosts.
*
* @param url
* whose scheme+host+port should be forgotten (remainder of url
* is ignored)
*/
public void forgetSchemeHost(String schemeHost) {
long schemeHostKeyPart = calcSchemeHostKeyPart(schemeHost);
public void forgetAllSchemeAuthorityMatching(String url) {
long schemeAuthorityKeyBytes = calcSchemeAuthorityKeyBytes(url);
DatabaseEntry key = new DatabaseEntry();
DatabaseEntry value = new DatabaseEntry();
LongBinding.longToEntry(schemeAuthorityKeyBytes, key);
Cursor cursor = alreadySeen.openCursor(null, null);
long forgottenCount = 0l;
while (cursor.getNext(key, value, null) == OperationStatus.SUCCESS) {
for (OperationStatus status = cursor.getSearchKeyRange(key, value, null);
status == OperationStatus.SUCCESS;
status = cursor.getNext(key, value, null)) {
long alreadySeenKey = LongBinding.entryToLong(key);
// System.out.printf("schemeHostKeyPart=%017x alreadySeenKey=%017x\n", schemeHostKeyPart, alreadySeenKey);
if ((alreadySeenKey & 0xffffff0000000000l) == schemeHostKeyPart) {
if ((alreadySeenKey & 0xffffff0000000000l) == schemeAuthorityKeyBytes) {
cursor.delete();
count.decrementAndGet();
forgottenCount++;
} else {
break;
}
}
cursor.close();
logger.info("forgot " + forgottenCount + " urls from scheme+host+port " + schemeHost + " (" + count.get() + " urls left)");
long newCount = count.addAndGet(-forgottenCount);
logger.info("forgot " + forgottenCount + " urls from scheme+authority of url " + url + " (leaving " + newCount + " urls from other scheme+authorities)");
}
} //EOC
engine/src/test/java/org/archive/crawler/util/BdbUriUniqFilterTest.java
+4 -4
View File
@@ -185,7 +185,7 @@ implements UriUniqFilter.CrawlUriReceiver {
assertEquals("Didn't forget", 0, this.filter.count());
}
public void testForgetSchemeHost() throws URIException {
public void testForgetAllSchemeAuthorityMatching() throws URIException {
long countBefore = this.filter.count();
for (String uri: new String[] {
@@ -205,7 +205,7 @@ implements UriUniqFilter.CrawlUriReceiver {
BdbUriUniqFilter bdbFilter = (BdbUriUniqFilter) filter;
assertFalse(bdbFilter.setAdd("http://forgetme.com/foo"));
bdbFilter.forgetSchemeHost("http://forgetme.com");
bdbFilter.forgetAllSchemeAuthorityMatching("http://forgetme.com");
assertEquals(countBefore + 2, this.filter.count());
assertTrue(bdbFilter.setAdd("http://forgetme.com/foo"));
@@ -213,10 +213,10 @@ implements UriUniqFilter.CrawlUriReceiver {
assertTrue(bdbFilter.setRemove("http://forgetme.com/foo"));
assertFalse(bdbFilter.setRemove("http://forgetme.com/foo"));
bdbFilter.forgetSchemeHost("https://forgetme.com/extra-stuff-ignored");
bdbFilter.forgetAllSchemeAuthorityMatching("https://forgetme.com/extra-stuff-ignored");
assertEquals(countBefore + 1, this.filter.count());
bdbFilter.forgetSchemeHost("http://forgetme.com:90/");
bdbFilter.forgetAllSchemeAuthorityMatching("http://forgetme.com:90/");
assertEquals(countBefore, this.filter.count());
}