From b9726c8d4f5df4355657114730b43e113ccc81fd Mon Sep 17 00:00:00 2001
From: Noah Levitt <nlevitt@archive.org>
Date: Thu, 25 Jul 2013 18:51:26 -0700
Subject: [PATCH] new method BdbUriUniqFilter.forgetSchemeHost(String
 schemeHost), and unit test

---
 .../crawler/util/BdbUriUniqFilter.java        | 40 +++++++++++++++++--
 .../crawler/util/BdbUriUniqFilterTest.java    | 36 +++++++++++++++++
 2 files changed, 72 insertions(+), 4 deletions(-)

diff --git a/engine/src/main/java/org/archive/crawler/util/BdbUriUniqFilter.java b/engine/src/main/java/org/archive/crawler/util/BdbUriUniqFilter.java
index 5fdea362..fdc5f8f8 100644
--- a/engine/src/main/java/org/archive/crawler/util/BdbUriUniqFilter.java
+++ b/engine/src/main/java/org/archive/crawler/util/BdbUriUniqFilter.java
@@ -38,6 +38,7 @@ import org.springframework.context.Lifecycle;
 import st.ata.util.FPGenerator;
 
 import com.sleepycat.bind.tuple.LongBinding;
+import com.sleepycat.je.Cursor;
 import com.sleepycat.je.Database;
 import com.sleepycat.je.DatabaseEntry;
 import com.sleepycat.je.DatabaseException;
@@ -302,17 +303,19 @@ implements Lifecycle, Checkpointable, BeanNameAware, DisposableBean {
      */
     public static long createKey(CharSequence uri) {
         String url = uri.toString();
+        long schemeHostKeyPart = calcSchemeHostKeyPart(url);
+        return schemeHostKeyPart | (FPGenerator.std40.fp(url) >>> 24);
+    }
+
+    protected static long calcSchemeHostKeyPart(String url) {
         int index = url.indexOf(COLON_SLASH_SLASH);
         if (index > 0) {
             index = url.indexOf('/', index + COLON_SLASH_SLASH.length());
         }
         CharSequence hostPlusScheme = (index == -1)? url: url.subSequence(0, index);
-        long tmp = FPGenerator.std24.fp(hostPlusScheme);
-        return tmp | (FPGenerator.std40.fp(url) >>> 24);
+        return FPGenerator.std24.fp(hostPlusScheme);
     }
 
-
-
     protected boolean setAdd(CharSequence uri) {
         DatabaseEntry key = new DatabaseEntry();
         LongBinding.longToEntry(createKey(uri), key);
@@ -395,4 +398,33 @@ implements Lifecycle, Checkpointable, BeanNameAware, DisposableBean {
     public void setRecoveryCheckpoint(Checkpoint recoveryCheckpoint) {
         this.recoveryCheckpoint = recoveryCheckpoint;
     }
+
+    /**
+     * Forget all entries that match the scheme+host+port of the given key, so
+     * that they can be crawled again if discovered again. Expensive operation.
+     * 
+     * <p>
+     * Because of the way keys are calculated, scheme+host+port is the only
+     * grouping of urls that is feasible to forget in bulk. See
+     * {@link #createKey(CharSequence)}
+     * 
+     * @param schemeHost
+     */
+    public void forgetSchemeHost(String schemeHost) {
+        long schemeHostKeyPart = calcSchemeHostKeyPart(schemeHost);
+        
+        DatabaseEntry key = new DatabaseEntry();
+        DatabaseEntry value = new DatabaseEntry();
+        Cursor cursor = alreadySeen.openCursor(null, null);
+        while (cursor.getNext(key, value, null) == OperationStatus.SUCCESS) {
+            long alreadySeenKey = LongBinding.entryToLong(key);
+            System.out.printf("schemeHostKeyPart=%017x alreadySeenKey=%017x\n", schemeHostKeyPart, alreadySeenKey);
+            if ((alreadySeenKey & 0xffffff0000000000l) == schemeHostKeyPart) {
+                cursor.delete();
+                count.decrementAndGet();
+            }
+        }
+        cursor.close(); 
+    }
+    
 } //EOC
\ No newline at end of file
diff --git a/engine/src/test/java/org/archive/crawler/util/BdbUriUniqFilterTest.java b/engine/src/test/java/org/archive/crawler/util/BdbUriUniqFilterTest.java
index 18f84e0f..0161aeaa 100644
--- a/engine/src/test/java/org/archive/crawler/util/BdbUriUniqFilterTest.java
+++ b/engine/src/test/java/org/archive/crawler/util/BdbUriUniqFilterTest.java
@@ -185,6 +185,42 @@ implements UriUniqFilter.CrawlUriReceiver {
         assertEquals("Didn't forget", 0, this.filter.count());
     }
     
+    public void testForgetSchemeHost() throws URIException {
+        long countBefore = this.filter.count();
+        
+        for (String uri: new String[] {
+                "http://forgetme.com/",
+                "http://forgetme.com/foo",
+                "hTtP://fOrGeTmE.cOm/bar",
+                "http://forgetme.com:80/toot/spuh",
+                "http://forgetme.com:90/toot/spuh",
+                "https://forgetme.com/baz",
+        }) {
+            CrawlURI curi = new CrawlURI(UURIFactory.getInstance(uri));
+            System.out.println(uri + " => " + curi);
+            this.filter.add(curi.getUURI().toCustomString(), curi);
+        }
+
+        assertEquals(countBefore + 6, this.filter.count());
+
+        BdbUriUniqFilter bdbFilter = (BdbUriUniqFilter) filter;
+        assertFalse(bdbFilter.setAdd("http://forgetme.com/foo"));
+
+        bdbFilter.forgetSchemeHost("http://forgetme.com");
+        assertEquals(countBefore + 2, this.filter.count());
+
+        assertTrue(bdbFilter.setAdd("http://forgetme.com/foo"));
+        assertFalse(bdbFilter.setAdd("http://forgetme.com/foo"));
+        assertTrue(bdbFilter.setRemove("http://forgetme.com/foo"));
+        assertFalse(bdbFilter.setRemove("http://forgetme.com/foo"));
+
+        bdbFilter.forgetSchemeHost("https://forgetme.com/extra-stuff-ignored");
+        assertEquals(countBefore + 1, this.filter.count());
+
+        bdbFilter.forgetSchemeHost("http://forgetme.com:90/");
+        assertEquals(countBefore, this.filter.count());
+    }
+    
     // TODO: Add testForget when non-empty
     
 	public void receive(CrawlURI item) {