From f4f3b9be3860ef1318c40b54758720a15ee707f6 Mon Sep 17 00:00:00 2001
From: Noah Levitt <nlevitt@archive.org>
Date: Fri, 26 Jul 2013 17:50:36 -0700
Subject: [PATCH] add logging to BdbUriUniqFilter.forgetSchemeHost(), clean up
 some javadocs

---
 .../org/archive/crawler/util/BdbUriUniqFilter.java   |  7 +++++--
 .../archive/crawler/util/BdbUriUniqFilterTest.java   |  1 -
 .../archive/modules/forms/FormLoginProcessor.java    | 12 ++++++------
 3 files changed, 11 insertions(+), 9 deletions(-)

diff --git a/engine/src/main/java/org/archive/crawler/util/BdbUriUniqFilter.java b/engine/src/main/java/org/archive/crawler/util/BdbUriUniqFilter.java
index fdc5f8f8..dfff305a 100644
--- a/engine/src/main/java/org/archive/crawler/util/BdbUriUniqFilter.java
+++ b/engine/src/main/java/org/archive/crawler/util/BdbUriUniqFilter.java
@@ -416,15 +416,18 @@ implements Lifecycle, Checkpointable, BeanNameAware, DisposableBean {
         DatabaseEntry key = new DatabaseEntry();
         DatabaseEntry value = new DatabaseEntry();
         Cursor cursor = alreadySeen.openCursor(null, null);
+        long forgottenCount = 0l;
         while (cursor.getNext(key, value, null) == OperationStatus.SUCCESS) {
             long alreadySeenKey = LongBinding.entryToLong(key);
-            System.out.printf("schemeHostKeyPart=%017x alreadySeenKey=%017x\n", schemeHostKeyPart, alreadySeenKey);
+            // System.out.printf("schemeHostKeyPart=%017x alreadySeenKey=%017x\n", schemeHostKeyPart, alreadySeenKey);
             if ((alreadySeenKey & 0xffffff0000000000l) == schemeHostKeyPart) {
                 cursor.delete();
                 count.decrementAndGet();
+                forgottenCount++;
             }
         }
-        cursor.close(); 
+        cursor.close();
+        logger.info("forgot " + forgottenCount + " urls from scheme+host+port " + schemeHost + " (" + count.get() + " urls left)");
     }
     
 } //EOC
\ No newline at end of file
diff --git a/engine/src/test/java/org/archive/crawler/util/BdbUriUniqFilterTest.java b/engine/src/test/java/org/archive/crawler/util/BdbUriUniqFilterTest.java
index 0161aeaa..c2b92849 100644
--- a/engine/src/test/java/org/archive/crawler/util/BdbUriUniqFilterTest.java
+++ b/engine/src/test/java/org/archive/crawler/util/BdbUriUniqFilterTest.java
@@ -197,7 +197,6 @@ implements UriUniqFilter.CrawlUriReceiver {
                 "https://forgetme.com/baz",
         }) {
             CrawlURI curi = new CrawlURI(UURIFactory.getInstance(uri));
-            System.out.println(uri + " => " + curi);
             this.filter.add(curi.getUURI().toCustomString(), curi);
         }
 
diff --git a/modules/src/main/java/org/archive/modules/forms/FormLoginProcessor.java b/modules/src/main/java/org/archive/modules/forms/FormLoginProcessor.java
index e9f1be04..94716960 100644
--- a/modules/src/main/java/org/archive/modules/forms/FormLoginProcessor.java
+++ b/modules/src/main/java/org/archive/modules/forms/FormLoginProcessor.java
@@ -47,22 +47,22 @@ import com.google.common.collect.MapMaker;
 
 /**
  * A step, post-ExtractorHTMLForms, where a followup CrawlURI to 
- * attempt a form-=submission may be synthesized. 
+ * attempt a form submission may be synthesized. 
  * 
- * If an HTMLForm was previously discovered (by ExtractorHTMLForms),
+ * <p>If an HTMLForm was previously discovered (by ExtractorHTMLForms),
  * and that form appears to be a login form, and at the very least
  * the loginUsername setting is non-empty, and the current 
  * (NOT 'action') URI fits under a configured SURT prefix, then a
  * submission CrawlURI will be composed.
  * 
- * This submission CrawlURI will be added to the current URI's
+ * <p>This submission CrawlURI will be added to the current URI's
  * outCandidates, and prefilled with settings for a POST and 
  * input values that are a merging of: (a) original discovered 
  * in-page values; (b) the 'loginUsername' into the first plausible
  * text/email-type input field; (c) the 'loginPassword' into the 
  * first password-type input field. 
  * 
- * Typically the settings 'applicableSurtPrefix', 'loginUsername', 
+ * <p>Typically the settings 'applicableSurtPrefix', 'loginUsername', 
  * and 'loginPassword' would be set in an overlay sheet and only
  * applied to one or more sites (by SURT prefix), rather than 
  * set globally.  An example minimal set of beans to add to CXML
@@ -95,12 +95,12 @@ import com.google.common.collect.MapMaker;
  * }
  * </pre>
  * 
- * (Remember: https URIs are always collapsed to http form before
+ * <p>(Remember: https URIs are always collapsed to http form before
  * overlay-surt-prefix comparisons, so surtPrefixes in the above
  * association should always be in http form, even if the actual 
  * target URIs are https.)
  * 
- * Finally, while there is not yet support for testing if the 
+ * <p>Finally, while there is not yet support for testing if the 
  * submitted CrawlURI succeeded, this processor keeps track of 
  * a count of FORMS seen that are eligible for attempts, and 
  * attempts made (for now, just once), per 'formTrackingDomain'