From 705a375daf4840635449d7061d12e20489b4787f Mon Sep 17 00:00:00 2001
From: Noah Levitt <nlevitt@archive.org>
Date: Fri, 9 Aug 2013 18:17:11 -0700
Subject: [PATCH] uses of UriUtils.isLikelyUri() in Extractor{HTML,SWF,XML}
 with UriUtils.isVeryLikelyUri() to reap the benefits of HER-1523 improvements
 (should address archive-it issue ARI-3492)

---
 commons/src/main/java/org/archive/util/UriUtils.java        | 6 +++++-
 .../java/org/archive/modules/extractor/ExtractorHTML.java   | 2 +-
 .../java/org/archive/modules/extractor/ExtractorSWF.java    | 2 +-
 .../java/org/archive/modules/extractor/ExtractorXML.java    | 2 +-
 4 files changed, 8 insertions(+), 4 deletions(-)

diff --git a/commons/src/main/java/org/archive/util/UriUtils.java b/commons/src/main/java/org/archive/util/UriUtils.java
index 5dfd2327..87ae4841 100644
--- a/commons/src/main/java/org/archive/util/UriUtils.java
+++ b/commons/src/main/java/org/archive/util/UriUtils.java
@@ -97,6 +97,10 @@ public class UriUtils {
         return TextUtils.matches(NAIVE_LIKELY_URI_PATTERN, candidate);
     }
     
+    /**
+     * @deprecated produces too many false positives,
+     *             {@link #isVeryLikelyUri(CharSequence)} is preferred
+     */
     public static boolean isLikelyUri(CharSequence candidate) {
         return isPossibleUri(candidate) && !isLikelyFalsePositive(candidate);
     }
@@ -384,7 +388,7 @@ public class UriUtils {
             + "(?:\\?(?:"+ QNV + ")(?:&(?:" + QNV + "))*)?" // may have a ?query=string
             + "(?:#[\\w-]+)?";                              // may have a #fragment
 
-    public static boolean isVeryLikelyUri(String candidate) {
+    public static boolean isVeryLikelyUri(CharSequence candidate) {
         // must have a . or /
         if (!TextUtils.matches(NAIVE_LIKELY_URI_PATTERN, candidate)) {
             return false;
diff --git a/modules/src/main/java/org/archive/modules/extractor/ExtractorHTML.java b/modules/src/main/java/org/archive/modules/extractor/ExtractorHTML.java
index 0c93cd38..58459184 100644
--- a/modules/src/main/java/org/archive/modules/extractor/ExtractorHTML.java
+++ b/modules/src/main/java/org/archive/modules/extractor/ExtractorHTML.java
@@ -606,7 +606,7 @@ public class ExtractorHTML extends ContentExtractor implements InitializingBean
      */
     protected void considerIfLikelyUri(CrawlURI curi, CharSequence candidate, 
             CharSequence valueContext, Hop hop) {
-        if(UriUtils.isLikelyUri(candidate)) {
+        if(UriUtils.isVeryLikelyUri(candidate)) {
             addLinkFromString(curi,candidate,valueContext,hop);
         }
     }
diff --git a/modules/src/main/java/org/archive/modules/extractor/ExtractorSWF.java b/modules/src/main/java/org/archive/modules/extractor/ExtractorSWF.java
index 70ab4a57..cdfe7360 100644
--- a/modules/src/main/java/org/archive/modules/extractor/ExtractorSWF.java
+++ b/modules/src/main/java/org/archive/modules/extractor/ExtractorSWF.java
@@ -341,7 +341,7 @@ public class ExtractorSWF extends ContentExtractor {
         }
 
         public void considerStringAsUri(String str) throws IOException {
-            if (UriUtils.isLikelyUri(str)) {
+            if (UriUtils.isVeryLikelyUri(str)) {
                 int max = ext.getExtractorParameters().getMaxOutlinks();
                 Link.addRelativeToVia(curi, max, str, 
                         LinkContext.SPECULATIVE_MISC, Hop.SPECULATIVE);
diff --git a/modules/src/main/java/org/archive/modules/extractor/ExtractorXML.java b/modules/src/main/java/org/archive/modules/extractor/ExtractorXML.java
index 903abfbd..dbf97d59 100644
--- a/modules/src/main/java/org/archive/modules/extractor/ExtractorXML.java
+++ b/modules/src/main/java/org/archive/modules/extractor/ExtractorXML.java
@@ -152,7 +152,7 @@ public class ExtractorXML extends ContentExtractor {
         Matcher matcher = XML_URI_EXTRACTOR.matcher(cs);
         while (matcher.find()) {
             String xmlUri = StringEscapeUtils.unescapeXml(matcher.group(1));
-            if (UriUtils.isLikelyUri(xmlUri)) {
+            if (UriUtils.isVeryLikelyUri(xmlUri)) {
                 foundLinks++;
                 try {
                     // treat as speculative, as whether context really