lee/heritrix3
1
0 Fork 0
mirror of https://github.com/internetarchive/heritrix3.git synced 2026-10-06 20:31:23 +00:00
Code Issues Packages Projects Releases Wiki Activity

* UriUtils.java

Browse Source 
some improvements to likely-uri detection based on crawl log analysis
This commit is contained in:
Noah Levitt committed 2012-01-20 11:23:45 -08:00
1 parent 20502f54d5
commit 9dbe673bc3
1 file changed
+12 -6
Download Patch File Download Diff File Expand all files Collapse all files
commons/src/main/java/org/archive/util/UriUtils.java
+12 -6
View File
@@ -88,8 +88,7 @@ public class UriUtils {
// naive likely-uri test:
// no whitespace or '<' or '>'
// at least one '.' or '/';
// not ending with '.'
static final String NAIVE_LIKELY_URI_PATTERN = "[^<>\\s]*[\\./][^<>\\s]*(?<!\\.)";
static final String NAIVE_LIKELY_URI_PATTERN = "[^<>\\s]*[\\./][^<>\\s]*";
public static boolean isPossibleUri(CharSequence candidate) {
return TextUtils.matches(NAIVE_LIKELY_URI_PATTERN, candidate);
@@ -253,7 +252,7 @@ public class UriUtils {
}
}
if (TextUtils.matches("\\d+\\.\\d+", candidate)) {
if (TextUtils.matches("\\d+(?:\\.\\d+)*", candidate)) {
if (LOGGER.isLoggable(Level.FINE)) {
LOGGER.fine("rejected: looks like a decimal number: " + candidate);
}
@@ -267,10 +266,17 @@ public class UriUtils {
return true;
}
// this happens often because of string concatenation in javascript
if (TextUtils.matches("^[+].*|.*[+]$", candidate)) {
// starting or ending with + particularly common because of string concatenation in javascript
if (TextUtils.matches("^[,;+:].*|.*[.,;+:]$", candidate)) {
if (LOGGER.isLoggable(Level.FINE)) {
LOGGER.fine("rejected: starts or ends with a '+': " + candidate);
LOGGER.fine("rejected: starts or ends with an unusual starting or ending character: " + candidate);
}
return true;
}
if (TextUtils.matches("^\\.\\.?[^/].*", candidate)) {
if (LOGGER.isLoggable(Level.FINE)) {
LOGGER.fine("rejected: starts with '.' (but not './' or '../'): " + candidate);
}
return true;
}