From a7a2b3cf7edf98e806509dddaf2b09704f1f1806 Mon Sep 17 00:00:00 2001
From: Noah Levitt <nlevitt@archive.org>
Date: Fri, 24 Aug 2012 14:03:45 -0700
Subject: [PATCH] Make speculative link extractor ignore urls with two
 consecutive slashes (not preceded by a colon as in http://). We've seen the
 double slashes lead to crawler traps. * UriUtils.java    
 isLikelyFalsePositive() - consider likely false positive if it contains '//'
 (but not '://') * UriUtilsTest.java     testDoubleSlashFalsePositives()

---
 commons/src/main/java/org/archive/util/UriUtils.java     | 7 +++++++
 commons/src/test/java/org/archive/util/UriUtilsTest.java | 8 ++++++++
 2 files changed, 15 insertions(+)

diff --git a/commons/src/main/java/org/archive/util/UriUtils.java b/commons/src/main/java/org/archive/util/UriUtils.java
index 8e01850b..7d546327 100644
--- a/commons/src/main/java/org/archive/util/UriUtils.java
+++ b/commons/src/main/java/org/archive/util/UriUtils.java
@@ -280,6 +280,13 @@ public class UriUtils {
             return true;
         }
         
+        if (TextUtils.matches("^.*[^:]//.*$", candidate)) {
+            if (LOGGER.isLoggable(Level.FINE)) {
+                LOGGER.fine("rejected: contains '//' (but not '://'): " + candidate);
+            }
+            return true;
+        }
+        
         // look for things that look like hostnames and not filenames?
         // look for too many dots but make sure we take into account that url may have hostname?
 
diff --git a/commons/src/test/java/org/archive/util/UriUtilsTest.java b/commons/src/test/java/org/archive/util/UriUtilsTest.java
index 3db3d6c0..ba990550 100644
--- a/commons/src/test/java/org/archive/util/UriUtilsTest.java
+++ b/commons/src/test/java/org/archive/util/UriUtilsTest.java
@@ -210,6 +210,14 @@ public class UriUtilsTest extends TestCase {
         tryAll(startsOrEndsWithPlusFalsePositives, false);
     }
     
+    protected static final String[] doubleSlashFalsePositives = new String[] {
+        ".//*",
+        "http://example.com/monkey//foo/whatever"
+    };
+    public void testDoubleSlashFalsePositives() {
+        tryAll(startsOrEndsWithPlusFalsePositives, false);
+    }
+
     /**
      * Test that all supplied candidates give the expected result, for each of 
      * the 'legacy' (H1) likely-URI-tests