mirror of
https://github.com/internetarchive/heritrix3.git
synced 2026-09-23 22:26:01 +00:00
Make speculative link extractor ignore urls with two consecutive slashes (not preceded by a colon as in http://). We've seen the double slashes lead to crawler traps.
* UriUtils.java
isLikelyFalsePositive() - consider likely false positive if it contains '//' (but not '://')
* UriUtilsTest.java
testDoubleSlashFalsePositives()
This commit is contained in:
@@ -280,6 +280,13 @@ public class UriUtils {
|
||||
return true;
|
||||
}
|
||||
|
||||
if (TextUtils.matches("^.*[^:]//.*$", candidate)) {
|
||||
if (LOGGER.isLoggable(Level.FINE)) {
|
||||
LOGGER.fine("rejected: contains '//' (but not '://'): " + candidate);
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
// look for things that look like hostnames and not filenames?
|
||||
// look for too many dots but make sure we take into account that url may have hostname?
|
||||
|
||||
|
||||
@@ -210,6 +210,14 @@ public class UriUtilsTest extends TestCase {
|
||||
tryAll(startsOrEndsWithPlusFalsePositives, false);
|
||||
}
|
||||
|
||||
protected static final String[] doubleSlashFalsePositives = new String[] {
|
||||
".//*",
|
||||
"http://example.com/monkey//foo/whatever"
|
||||
};
|
||||
public void testDoubleSlashFalsePositives() {
|
||||
tryAll(startsOrEndsWithPlusFalsePositives, false);
|
||||
}
|
||||
|
||||
/**
|
||||
* Test that all supplied candidates give the expected result, for each of
|
||||
* the 'legacy' (H1) likely-URI-tests
|
||||
|
||||
Reference in New Issue
Block a user