From 438431264799f4ef4ad434c42b646bdae5d3b7e4 Mon Sep 17 00:00:00 2001 From: Kenji Nagahashi Date: Tue, 25 Jan 2011 14:06:43 -0800 Subject: [PATCH 1/4] reuse pre-allocated shared HTMLLinkContext instance. --- .../java/org/archive/modules/CrawlURI.java | 2 +- .../modules/extractor/ExtractorHTML.java | 2 +- .../modules/extractor/ExtractorHTTP.java | 2 +- .../modules/extractor/HTMLLinkContext.java | 45 +++++++++++++++++-- .../modules/extractor/ExtractorHTMLTest.java | 4 +- 5 files changed, 46 insertions(+), 9 deletions(-) diff --git a/modules/src/main/java/org/archive/modules/CrawlURI.java b/modules/src/main/java/org/archive/modules/CrawlURI.java index e1846bb6..82e805c7 100644 --- a/modules/src/main/java/org/archive/modules/CrawlURI.java +++ b/modules/src/main/java/org/archive/modules/CrawlURI.java @@ -295,7 +295,7 @@ implements MultiReporter, Serializable, OverlayContext { UURIFactory.getInstance(args[2].toString()): null; LinkContext viaContext = (args.length > 3 && args[2].length()>1) ? - new HTMLLinkContext(args[3].toString()): null; + HTMLLinkContext.get(args[3].toString()): null; CrawlURI caUri = new CrawlURI(u, pathFromSeed, via, viaContext); return caUri; } diff --git a/modules/src/main/java/org/archive/modules/extractor/ExtractorHTML.java b/modules/src/main/java/org/archive/modules/extractor/ExtractorHTML.java index 9122cf18..2c2c04b3 100644 --- a/modules/src/main/java/org/archive/modules/extractor/ExtractorHTML.java +++ b/modules/src/main/java/org/archive/modules/extractor/ExtractorHTML.java @@ -617,7 +617,7 @@ public class ExtractorHTML extends ContentExtractor implements InitializingBean // the underlying ReplayCharSequence and the link its about // to become a part of is expected to outlive the current // ReplayCharSequence. - HTMLLinkContext hc = new HTMLLinkContext(context.toString()); + HTMLLinkContext hc = HTMLLinkContext.get(context.toString()); int max = getExtractorParameters().getMaxOutlinks(); Link.addRelativeToBase(curi, max, uri.toString(), hc, hop); } catch (URIException e) { diff --git a/modules/src/main/java/org/archive/modules/extractor/ExtractorHTTP.java b/modules/src/main/java/org/archive/modules/extractor/ExtractorHTTP.java index 2b321543..3c3828d2 100644 --- a/modules/src/main/java/org/archive/modules/extractor/ExtractorHTTP.java +++ b/modules/src/main/java/org/archive/modules/extractor/ExtractorHTTP.java @@ -81,7 +81,7 @@ public class ExtractorHTTP extends Extractor { // TODO: consider possibility of multiple headers try { UURI dest = UURIFactory.getInstance(curi.getUURI(), loc.getValue()); - LinkContext lc = new HTMLLinkContext(loc.getName()+":"); + LinkContext lc = HTMLLinkContext.get(loc.getName()+":"); Link link = new Link(curi.getUURI(), dest, lc, Hop.REFER); curi.getOutLinks().add(link); numberOfLinksExtracted.incrementAndGet(); diff --git a/modules/src/main/java/org/archive/modules/extractor/HTMLLinkContext.java b/modules/src/main/java/org/archive/modules/extractor/HTMLLinkContext.java index b5eebdab..f44cef74 100644 --- a/modules/src/main/java/org/archive/modules/extractor/HTMLLinkContext.java +++ b/modules/src/main/java/org/archive/modules/extractor/HTMLLinkContext.java @@ -30,26 +30,63 @@ public class HTMLLinkContext extends LinkContext { final public static HTMLLinkContext META = new HTMLLinkContext("meta"); + final public static HTMLLinkContext A_HREF = new HTMLLinkContext("a", "href"); + final public static HTMLLinkContext IMG_SRC = new HTMLLinkContext("img", "src"); + final public static HTMLLinkContext SCRIPT_SRC = new HTMLLinkContext("script", "src"); + final public static HTMLLinkContext META_HREF = new HTMLLinkContext("meta", "href"); /** * The HTML path to the URL. */ - private String path; - + private final String path; + /** + * return an instance of HTMLLinkContext for attribute {@code attr} in + * element {@code el}. returns pre-allocated shared instance for common case, + * or new instance for others. + * @param el element name + * @param attr attribute name + * @return instance of HTMLLinkContext + */ + public static HTMLLinkContext get(CharSequence el, CharSequence attr) { + if (attr.equals("href") || attr.equals("HREF")) { + if (el.equals("a") || el.equals("A")) return A_HREF; + if (el.equals("meta") || el.equals("META")) return META_HREF; + } else if (attr.equals("src") || attr.equals("SRC")) { + if (el.equals("img") || attr.equals("IMG")) return IMG_SRC; + if (el.equals("script") || attr.equals("SCRIPT")) return SCRIPT_SRC; + } + return new HTMLLinkContext(el, attr); + } + /** + * return an instance of HTMLLinkContext for path {@code path}. + * returns pre-allocated shared instance for common case, or new instance for others. + *

TODO: most code calling this method builds path by concatenating element name + * and attribute name. consider changing such code to call {@link #get(CharSequence, CharSequence)} + * instead.

+ * @param path element and attribute in XLink-like path notation + * @return instance of HTMLLinkContext + */ + public static HTMLLinkContext get(String path) { + if (path.equalsIgnoreCase("a/@href")) return A_HREF; + if (path.equalsIgnoreCase("meta/@href")) return META_HREF; + if (path.equalsIgnoreCase("img/@src")) return IMG_SRC; + if (path.equalsIgnoreCase("script/@src")) return SCRIPT_SRC; + return new HTMLLinkContext(path); + } /** * Constructor. * * @param path an XPath-like context, eg "A\@HREF" */ - public HTMLLinkContext(String path) { + protected HTMLLinkContext(String path) { // FIXME: Verify that path really is XPath-like this.path = path; } - public HTMLLinkContext(CharSequence element, CharSequence attribute) { + protected HTMLLinkContext(CharSequence element, CharSequence attribute) { if (attribute == null) { this.path = ""; } else { diff --git a/modules/src/test/java/org/archive/modules/extractor/ExtractorHTMLTest.java b/modules/src/test/java/org/archive/modules/extractor/ExtractorHTMLTest.java index fae65d52..0599ba74 100644 --- a/modules/src/test/java/org/archive/modules/extractor/ExtractorHTMLTest.java +++ b/modules/src/test/java/org/archive/modules/extractor/ExtractorHTMLTest.java @@ -125,10 +125,10 @@ public class ExtractorHTMLTest extends StringExtractorTestBase { private static LinkContext determineContext(String s) { if (s.endsWith(" A")) { - return new HTMLLinkContext("a/@href"); + return HTMLLinkContext.get("a/@href"); } if (s.endsWith(" IMG")) { - return new HTMLLinkContext("img/@src"); + return HTMLLinkContext.get("img/@src"); } return LinkContext.NAVLINK_MISC; } From 0144df024e44db1295c7f84f7637590cf1ffe769 Mon Sep 17 00:00:00 2001 From: Noah Levitt Date: Thu, 16 Aug 2012 12:17:01 -0700 Subject: [PATCH 2/4] Real fix for HER-2016 FetchHTTP tries every url twice without credentials before sending credentials: Remember WWW-Authenticate headers, aka auth challenges, with CrawlURI, and after successful auth, with the CrawlServer. It's necessary to do this to know what Authorization header to send without getting another 401 first. * CoreAttributeConstants.java new constant A_HTTP_AUTH_CHALLENGES key for CrawlURI.getData() for http auth challenges * CrawlURI.java add A_HTTP_AUTH_CHALLENGES to keys to remember across URI processing; convenience getter and setter for auth challenges * Credential.java, HtmlFormCredential.java, HttpAuthenticationCredential.java add httpAuthChallenges argument to populate() * HttpAuthenticationCredential.java populate() - process httpAuthChallenges to set up auth state of http client to send the right auth header * CrawlServer.java cache httpAuthChallenges * FetchHTTP.java getAuthScheme() - cache auth challenges with CrawlURI configureHttp() - do not set authentication preemptive - this is handled by the configuration done in HttpAuthenticationCredential.populate() promoteCredentials() - cache auth challenges with the CrawlServer --- .../modules/CoreAttributeConstants.java | 2 ++ .../java/org/archive/modules/CrawlURI.java | 11 ++++++- .../modules/credential/Credential.java | 4 ++- .../credential/HtmlFormCredential.java | 4 ++- .../HttpAuthenticationCredential.java | 20 ++++++++++- .../archive/modules/fetcher/FetchHTTP.java | 33 +++++++++---------- .../org/archive/modules/net/CrawlServer.java | 11 ++++++- 7 files changed, 63 insertions(+), 22 deletions(-) diff --git a/modules/src/main/java/org/archive/modules/CoreAttributeConstants.java b/modules/src/main/java/org/archive/modules/CoreAttributeConstants.java index 8f6d389e..32932957 100644 --- a/modules/src/main/java/org/archive/modules/CoreAttributeConstants.java +++ b/modules/src/main/java/org/archive/modules/CoreAttributeConstants.java @@ -114,4 +114,6 @@ public interface CoreAttributeConstants { public static final String A_FTP_FETCH_STATUS = "ftp-fetch-status"; public static final String A_WHOIS_SERVER_IP = "whois-server-ip"; + + public static final String A_HTTP_AUTH_CHALLENGES = "http-auth-challenges"; } diff --git a/modules/src/main/java/org/archive/modules/CrawlURI.java b/modules/src/main/java/org/archive/modules/CrawlURI.java index 5377cce1..66fb5fb1 100644 --- a/modules/src/main/java/org/archive/modules/CrawlURI.java +++ b/modules/src/main/java/org/archive/modules/CrawlURI.java @@ -26,6 +26,7 @@ import static org.archive.modules.CoreAttributeConstants.A_FETCH_COMPLETED_TIME; import static org.archive.modules.CoreAttributeConstants.A_FORCE_RETIRE; import static org.archive.modules.CoreAttributeConstants.A_HERITABLE_KEYS; import static org.archive.modules.CoreAttributeConstants.A_HTML_BASE; +import static org.archive.modules.CoreAttributeConstants.A_HTTP_AUTH_CHALLENGES; import static org.archive.modules.CoreAttributeConstants.A_NONFATAL_ERRORS; import static org.archive.modules.CoreAttributeConstants.A_PREREQUISITE_URI; import static org.archive.modules.CoreAttributeConstants.A_SOURCE_TAG; @@ -249,7 +250,7 @@ implements Reporter, Serializable, OverlayContext { */ private static final Collection persistentKeys = new CopyOnWriteArrayList( - new String [] {A_CREDENTIALS_KEY}); + new String [] {A_CREDENTIALS_KEY, A_HTTP_AUTH_CHALLENGES}); /** maximum length for pathFromSeed/hopsPath; longer truncated with leading counter **/ private static final int MAX_HOPS_DISPLAYED = 50; @@ -1874,4 +1875,12 @@ implements Reporter, Serializable, OverlayContext { return getContentType().matches("(?i).*charset=.*"); } + @SuppressWarnings("unchecked") + public Map getHttpAuthChallenges() { + return (Map) getData().get(A_HTTP_AUTH_CHALLENGES); + } + + public void setHttpAuthChallenges(Map httpAuthChallenges) { + getData().put(A_HTTP_AUTH_CHALLENGES, httpAuthChallenges); + } } diff --git a/modules/src/main/java/org/archive/modules/credential/Credential.java b/modules/src/main/java/org/archive/modules/credential/Credential.java index 03b7895c..ca232942 100644 --- a/modules/src/main/java/org/archive/modules/credential/Credential.java +++ b/modules/src/main/java/org/archive/modules/credential/Credential.java @@ -20,6 +20,7 @@ package org.archive.modules.credential; import java.io.Serializable; import java.util.Iterator; +import java.util.Map; import java.util.logging.Logger; import javax.management.AttributeNotFoundException; @@ -167,10 +168,11 @@ public abstract class Credential implements Serializable { * @param curi CrawlURI to as for context. * @param http Instance of httpclient. * @param method Method to populate. + * @param httpAuthChallenges * @return True if added a credentials. */ public abstract boolean populate(CrawlURI curi, HttpClient http, - HttpMethod method); + HttpMethod method, Map httpAuthChallenges); /** * @param curi CrawlURI to look at. diff --git a/modules/src/main/java/org/archive/modules/credential/HtmlFormCredential.java b/modules/src/main/java/org/archive/modules/credential/HtmlFormCredential.java index 258f1820..f58099ad 100644 --- a/modules/src/main/java/org/archive/modules/credential/HtmlFormCredential.java +++ b/modules/src/main/java/org/archive/modules/credential/HtmlFormCredential.java @@ -133,7 +133,9 @@ public class HtmlFormCredential extends Credential { return false; } - public boolean populate(CrawlURI curi, HttpClient http, HttpMethod method) { + @Override + public boolean populate(CrawlURI curi, HttpClient http, HttpMethod method, + Map httpAuthChallenges) { // http is not used boolean result = false; Map formItems = getFormItems(); diff --git a/modules/src/main/java/org/archive/modules/credential/HttpAuthenticationCredential.java b/modules/src/main/java/org/archive/modules/credential/HttpAuthenticationCredential.java index 7035e1c9..bb161795 100644 --- a/modules/src/main/java/org/archive/modules/credential/HttpAuthenticationCredential.java +++ b/modules/src/main/java/org/archive/modules/credential/HttpAuthenticationCredential.java @@ -21,6 +21,7 @@ package org.archive.modules.credential; import java.util.Arrays; import java.util.Iterator; +import java.util.Map; import java.util.Set; import java.util.logging.Logger; @@ -28,8 +29,12 @@ import org.apache.commons.httpclient.HttpClient; import org.apache.commons.httpclient.HttpMethod; import org.apache.commons.httpclient.URIException; import org.apache.commons.httpclient.UsernamePasswordCredentials; +import org.apache.commons.httpclient.auth.AuthChallengeProcessor; import org.apache.commons.httpclient.auth.AuthPolicy; +import org.apache.commons.httpclient.auth.AuthScheme; import org.apache.commons.httpclient.auth.AuthScope; +import org.apache.commons.httpclient.auth.AuthenticationException; +import org.apache.commons.httpclient.auth.MalformedChallengeException; import org.archive.modules.CrawlURI; /** @@ -106,9 +111,22 @@ public class HttpAuthenticationCredential extends Credential { return true; } - public boolean populate(CrawlURI curi, HttpClient http, HttpMethod method) { + @Override + public boolean populate(CrawlURI curi, HttpClient http, HttpMethod method, + Map httpAuthChallenges) { boolean result = false; + AuthChallengeProcessor authChallengeProcessor = new AuthChallengeProcessor( + http.getParams()); + try { + AuthScheme authScheme = authChallengeProcessor.processChallenge( + method.getHostAuthState(), httpAuthChallenges); + method.getHostAuthState().setAuthScheme(authScheme); + } catch (MalformedChallengeException e) { + return result; + } catch (AuthenticationException e) { + return result; + } // Always add the credential to HttpState. Doing this because no way of // removing the credential once added AND there is a bug in the diff --git a/modules/src/main/java/org/archive/modules/fetcher/FetchHTTP.java b/modules/src/main/java/org/archive/modules/fetcher/FetchHTTP.java index 1c020759..e646ba49 100644 --- a/modules/src/main/java/org/archive/modules/fetcher/FetchHTTP.java +++ b/modules/src/main/java/org/archive/modules/fetcher/FetchHTTP.java @@ -71,10 +71,9 @@ import org.apache.commons.httpclient.HttpVersion; import org.apache.commons.httpclient.NTCredentials; import org.apache.commons.httpclient.URIException; import org.apache.commons.httpclient.auth.AuthChallengeParser; +import org.apache.commons.httpclient.auth.AuthPolicy; import org.apache.commons.httpclient.auth.AuthScheme; import org.apache.commons.httpclient.auth.AuthScope; -import org.apache.commons.httpclient.auth.BasicScheme; -import org.apache.commons.httpclient.auth.DigestScheme; import org.apache.commons.httpclient.auth.MalformedChallengeException; import org.apache.commons.httpclient.cookie.CookiePolicy; import org.apache.commons.httpclient.params.HttpClientParams; @@ -1146,7 +1145,7 @@ public class FetchHTTP extends Processor implements Lifecycle { if (server.hasCredentials()) { for (Credential cred : server.getCredentials()) { if (cred.isEveryTime()) { - cred.populate(curi, this.http, method); + cred.populate(curi, this.http, method, server.getHttpAuthChallenges()); } } } @@ -1157,7 +1156,7 @@ public class FetchHTTP extends Processor implements Lifecycle { // by the handle401 method if its a rfc2617 or it'll have been set into // the curi by the preconditionenforcer as this login uri came through. for (Credential c: curi.getCredentials()) { - if (c.populate(curi, this.http, method)) { + if (c.populate(curi, this.http, method, curi.getHttpAuthChallenges())) { result = true; } } @@ -1186,6 +1185,7 @@ public class FetchHTTP extends Processor implements Lifecycle { CrawlServer cs = serverCache.getServerFor(cd); if (cs != null) { cs.addCredential(c); + cs.setHttpAuthChallenges(curi.getHttpAuthChallenges()); } } } @@ -1270,38 +1270,39 @@ public class FetchHTTP extends Processor implements Lifecycle { return null; } - Map authschemes = null; + Map authChallenges = null; try { @SuppressWarnings("unchecked") Map parsedChallenges = AuthChallengeParser.parseChallenges(headers); - authschemes = parsedChallenges; + authChallenges = parsedChallenges; + + // remember WWW-Authenticate headers for later use + curi.setHttpAuthChallenges(authChallenges); } catch (MalformedChallengeException e) { logger.fine("Failed challenge parse: " + e.getMessage()); } - if (authschemes == null || authschemes.size() <= 0) { + if (authChallenges == null || authChallenges.size() <= 0) { logger.fine("We got a 401 and WWW-Authenticate challenge" + " but failed parse of the header " + curi.toString()); return null; } + // XXX there's a lot of overlap below with AuthChallengeProcessor.processChallenge() + AuthScheme result = null; // Use the first auth found. - for (Iterator i = authschemes.keySet().iterator(); result == null + for (Iterator i = authChallenges.keySet().iterator(); result == null && i.hasNext();) { String key = (String) i.next(); - String challenge = (String) authschemes.get(key); + String challenge = (String) authChallenges.get(key); if (key == null || key.length() <= 0 || challenge == null || challenge.length() <= 0) { logger.warning("Empty scheme: " + curi.toString() + ": " + Arrays.toString(headers)); continue; } - AuthScheme authscheme = null; - if (key.equals("basic")) { - authscheme = new BasicScheme(); - } else if (key.equals("digest")) { - authscheme = new DigestScheme(); - } else { + AuthScheme authscheme = AuthPolicy.getAuthScheme(key); + if (authscheme == null) { logger.fine("Unsupported scheme: " + key); continue; } @@ -1450,8 +1451,6 @@ public class FetchHTTP extends Processor implements Lifecycle { hcp.setSoTimeout(timeout); // Set client to be version 1.0. hcp.setVersion(HttpVersion.HTTP_1_0); - // We handle 401s, so when we do auth, we want it preemptive. - hcp.setAuthenticationPreemptive(true); // configureHttpCookies(defaults); diff --git a/modules/src/main/java/org/archive/modules/net/CrawlServer.java b/modules/src/main/java/org/archive/modules/net/CrawlServer.java index f2330b1d..5b589ef2 100644 --- a/modules/src/main/java/org/archive/modules/net/CrawlServer.java +++ b/modules/src/main/java/org/archive/modules/net/CrawlServer.java @@ -28,6 +28,7 @@ import java.io.InputStream; import java.io.InputStreamReader; import java.io.Serializable; import java.util.HashSet; +import java.util.Map; import java.util.Set; import java.util.logging.Level; import java.util.logging.Logger; @@ -348,5 +349,13 @@ public class CrawlServer implements Serializable, FetchStats.HasFetchStats, Iden @Override public void setIdentityCache(ObjectIdentityCache cache) { this.cache = cache; - } + } + + transient private Map httpAuthChallenges; + public Map getHttpAuthChallenges() { + return httpAuthChallenges; + } + public void setHttpAuthChallenges(Map httpAuthChallenges) { + this.httpAuthChallenges = httpAuthChallenges; + } } From f072dfc83c74c6d9090dc495fea3fc486a6bae9d Mon Sep 17 00:00:00 2001 From: Noah Levitt Date: Thu, 16 Aug 2012 12:28:19 -0700 Subject: [PATCH 3/4] * Credential.java, HtmlFormCredential.java, HttpAuthenticationCredential.java bump serialVersionUID because of changed method signature on populate() --- .../main/java/org/archive/modules/credential/Credential.java | 2 +- .../java/org/archive/modules/credential/HtmlFormCredential.java | 2 +- .../modules/credential/HttpAuthenticationCredential.java | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/modules/src/main/java/org/archive/modules/credential/Credential.java b/modules/src/main/java/org/archive/modules/credential/Credential.java index ca232942..2f23d630 100644 --- a/modules/src/main/java/org/archive/modules/credential/Credential.java +++ b/modules/src/main/java/org/archive/modules/credential/Credential.java @@ -46,7 +46,7 @@ import org.archive.modules.net.ServerCache; */ public abstract class Credential implements Serializable { - private static final long serialVersionUID = 1L; + private static final long serialVersionUID = 2L; private static final Logger logger = Logger.getLogger(Credential.class.getName()); diff --git a/modules/src/main/java/org/archive/modules/credential/HtmlFormCredential.java b/modules/src/main/java/org/archive/modules/credential/HtmlFormCredential.java index f58099ad..2163b073 100644 --- a/modules/src/main/java/org/archive/modules/credential/HtmlFormCredential.java +++ b/modules/src/main/java/org/archive/modules/credential/HtmlFormCredential.java @@ -43,7 +43,7 @@ import org.archive.net.UURIFactory; * @version $Revision$, $Date$ */ public class HtmlFormCredential extends Credential { - private static final long serialVersionUID = -3L; + private static final long serialVersionUID = -4L; private static final Logger logger = Logger.getLogger(HtmlFormCredential.class.getName()); diff --git a/modules/src/main/java/org/archive/modules/credential/HttpAuthenticationCredential.java b/modules/src/main/java/org/archive/modules/credential/HttpAuthenticationCredential.java index bb161795..75aa8932 100644 --- a/modules/src/main/java/org/archive/modules/credential/HttpAuthenticationCredential.java +++ b/modules/src/main/java/org/archive/modules/credential/HttpAuthenticationCredential.java @@ -46,7 +46,7 @@ import org.archive.modules.CrawlURI; * @version $Revision$, $Date$ */ public class HttpAuthenticationCredential extends Credential { - private static final long serialVersionUID = 3L; + private static final long serialVersionUID = 4L; private static Logger logger = Logger.getLogger(HttpAuthenticationCredential.class.getName()); From a7a2b3cf7edf98e806509dddaf2b09704f1f1806 Mon Sep 17 00:00:00 2001 From: Noah Levitt Date: Fri, 24 Aug 2012 14:03:45 -0700 Subject: [PATCH 4/4] Make speculative link extractor ignore urls with two consecutive slashes (not preceded by a colon as in http://). We've seen the double slashes lead to crawler traps. * UriUtils.java isLikelyFalsePositive() - consider likely false positive if it contains '//' (but not '://') * UriUtilsTest.java testDoubleSlashFalsePositives() --- commons/src/main/java/org/archive/util/UriUtils.java | 7 +++++++ commons/src/test/java/org/archive/util/UriUtilsTest.java | 8 ++++++++ 2 files changed, 15 insertions(+) diff --git a/commons/src/main/java/org/archive/util/UriUtils.java b/commons/src/main/java/org/archive/util/UriUtils.java index 8e01850b..7d546327 100644 --- a/commons/src/main/java/org/archive/util/UriUtils.java +++ b/commons/src/main/java/org/archive/util/UriUtils.java @@ -280,6 +280,13 @@ public class UriUtils { return true; } + if (TextUtils.matches("^.*[^:]//.*$", candidate)) { + if (LOGGER.isLoggable(Level.FINE)) { + LOGGER.fine("rejected: contains '//' (but not '://'): " + candidate); + } + return true; + } + // look for things that look like hostnames and not filenames? // look for too many dots but make sure we take into account that url may have hostname? diff --git a/commons/src/test/java/org/archive/util/UriUtilsTest.java b/commons/src/test/java/org/archive/util/UriUtilsTest.java index 3db3d6c0..ba990550 100644 --- a/commons/src/test/java/org/archive/util/UriUtilsTest.java +++ b/commons/src/test/java/org/archive/util/UriUtilsTest.java @@ -210,6 +210,14 @@ public class UriUtilsTest extends TestCase { tryAll(startsOrEndsWithPlusFalsePositives, false); } + protected static final String[] doubleSlashFalsePositives = new String[] { + ".//*", + "http://example.com/monkey//foo/whatever" + }; + public void testDoubleSlashFalsePositives() { + tryAll(startsOrEndsWithPlusFalsePositives, false); + } + /** * Test that all supplied candidates give the expected result, for each of * the 'legacy' (H1) likely-URI-tests