lee/heritrix3
1
0
Fork 0
mirror of https://github.com/internetarchive/heritrix3.git synced 2026-09-14 09:45:52 +00:00
Code Issues Packages Projects Releases Wiki Activity

uses of UriUtils.isLikelyUri() in Extractor{HTML,SWF,XML} with UriUtils.isVeryLikelyUri() to reap the benefits of HER-1523 improvements (should address archive-it issue ARI-3492)

Browse Source
This commit is contained in:
Noah Levitt
2013-08-09 18:17:11 -07:00
parent 8c64cba9ae
commit 705a375daf
4 changed files with 8 additions and 4 deletions
Download Patch File Download Diff File Expand all files Collapse all files
commons/src/main/java/org/archive/util/UriUtils.java
+5 -1
View File
@@ -97,6 +97,10 @@ public class UriUtils {
return TextUtils.matches(NAIVE_LIKELY_URI_PATTERN, candidate);
}
/**
* @deprecated produces too many false positives,
* {@link #isVeryLikelyUri(CharSequence)} is preferred
*/
public static boolean isLikelyUri(CharSequence candidate) {
return isPossibleUri(candidate) && !isLikelyFalsePositive(candidate);
}
@@ -384,7 +388,7 @@ public class UriUtils {
+ "(?:\\?(?:"+ QNV + ")(?:&(?:" + QNV + "))*)?" // may have a ?query=string
+ "(?:#[\\w-]+)?"; // may have a #fragment
public static boolean isVeryLikelyUri(String candidate) {
public static boolean isVeryLikelyUri(CharSequence candidate) {
// must have a . or /
if (!TextUtils.matches(NAIVE_LIKELY_URI_PATTERN, candidate)) {
return false;
modules/src/main/java/org/archive/modules/extractor/ExtractorHTML.java
+1 -1
View File
@@ -606,7 +606,7 @@ public class ExtractorHTML extends ContentExtractor implements InitializingBean
*/
protected void considerIfLikelyUri(CrawlURI curi, CharSequence candidate,
CharSequence valueContext, Hop hop) {
if(UriUtils.isLikelyUri(candidate)) {
if(UriUtils.isVeryLikelyUri(candidate)) {
addLinkFromString(curi,candidate,valueContext,hop);
}
}
modules/src/main/java/org/archive/modules/extractor/ExtractorSWF.java
+1 -1
View File
@@ -341,7 +341,7 @@ public class ExtractorSWF extends ContentExtractor {
}
public void considerStringAsUri(String str) throws IOException {
if (UriUtils.isLikelyUri(str)) {
if (UriUtils.isVeryLikelyUri(str)) {
int max = ext.getExtractorParameters().getMaxOutlinks();
Link.addRelativeToVia(curi, max, str,
LinkContext.SPECULATIVE_MISC, Hop.SPECULATIVE);
modules/src/main/java/org/archive/modules/extractor/ExtractorXML.java
+1 -1
View File
@@ -152,7 +152,7 @@ public class ExtractorXML extends ContentExtractor {
Matcher matcher = XML_URI_EXTRACTOR.matcher(cs);
while (matcher.find()) {
String xmlUri = StringEscapeUtils.unescapeXml(matcher.group(1));
if (UriUtils.isLikelyUri(xmlUri)) {
if (UriUtils.isVeryLikelyUri(xmlUri)) {
foundLinks++;
try {
// treat as speculative, as whether context really