Make speculative link extractor ignore urls with two consecutive slashes (not preceded by a colon as in http://). We've seen the double slashes lead to crawler traps.

* UriUtils.java
    isLikelyFalsePositive() - consider likely false positive if it contains '//' (but not '://')
* UriUtilsTest.java
    testDoubleSlashFalsePositives()
This commit is contained in:
Noah Levitt
2012-08-24 14:03:45 -07:00
parent f072dfc83c
commit a7a2b3cf7e
2 changed files with 15 additions and 0 deletions
@@ -280,6 +280,13 @@ public class UriUtils {
return true;
}
if (TextUtils.matches("^.*[^:]//.*$", candidate)) {
if (LOGGER.isLoggable(Level.FINE)) {
LOGGER.fine("rejected: contains '//' (but not '://'): " + candidate);
}
return true;
}
// look for things that look like hostnames and not filenames?
// look for too many dots but make sure we take into account that url may have hostname?
@@ -210,6 +210,14 @@ public class UriUtilsTest extends TestCase {
tryAll(startsOrEndsWithPlusFalsePositives, false);
}
protected static final String[] doubleSlashFalsePositives = new String[] {
".//*",
"http://example.com/monkey//foo/whatever"
};
public void testDoubleSlashFalsePositives() {
tryAll(startsOrEndsWithPlusFalsePositives, false);
}
/**
* Test that all supplied candidates give the expected result, for each of
* the 'legacy' (H1) likely-URI-tests