diff --git a/commons/src/main/java/org/archive/util/UriUtils.java b/commons/src/main/java/org/archive/util/UriUtils.java index 8e01850b..7d546327 100644 --- a/commons/src/main/java/org/archive/util/UriUtils.java +++ b/commons/src/main/java/org/archive/util/UriUtils.java @@ -280,6 +280,13 @@ public class UriUtils { return true; } + if (TextUtils.matches("^.*[^:]//.*$", candidate)) { + if (LOGGER.isLoggable(Level.FINE)) { + LOGGER.fine("rejected: contains '//' (but not '://'): " + candidate); + } + return true; + } + // look for things that look like hostnames and not filenames? // look for too many dots but make sure we take into account that url may have hostname? diff --git a/commons/src/test/java/org/archive/util/UriUtilsTest.java b/commons/src/test/java/org/archive/util/UriUtilsTest.java index 3db3d6c0..ba990550 100644 --- a/commons/src/test/java/org/archive/util/UriUtilsTest.java +++ b/commons/src/test/java/org/archive/util/UriUtilsTest.java @@ -210,6 +210,14 @@ public class UriUtilsTest extends TestCase { tryAll(startsOrEndsWithPlusFalsePositives, false); } + protected static final String[] doubleSlashFalsePositives = new String[] { + ".//*", + "http://example.com/monkey//foo/whatever" + }; + public void testDoubleSlashFalsePositives() { + tryAll(startsOrEndsWithPlusFalsePositives, false); + } + /** * Test that all supplied candidates give the expected result, for each of * the 'legacy' (H1) likely-URI-tests diff --git a/modules/src/main/java/org/archive/modules/CrawlURI.java b/modules/src/main/java/org/archive/modules/CrawlURI.java index f52f8096..66bfe7b7 100644 --- a/modules/src/main/java/org/archive/modules/CrawlURI.java +++ b/modules/src/main/java/org/archive/modules/CrawlURI.java @@ -277,7 +277,7 @@ implements Reporter, Serializable, OverlayContext { UURIFactory.getInstance(args[2].toString()): null; LinkContext viaContext = (args.length > 3 && args[2].length()>1) ? - new HTMLLinkContext(args[3].toString()): null; + HTMLLinkContext.get(args[3].toString()): null; CrawlURI caUri = new CrawlURI(u, pathFromSeed, via, viaContext); return caUri; } diff --git a/modules/src/main/java/org/archive/modules/extractor/ExtractorHTML.java b/modules/src/main/java/org/archive/modules/extractor/ExtractorHTML.java index 8722882d..af499aa9 100644 --- a/modules/src/main/java/org/archive/modules/extractor/ExtractorHTML.java +++ b/modules/src/main/java/org/archive/modules/extractor/ExtractorHTML.java @@ -641,7 +641,7 @@ public class ExtractorHTML extends ContentExtractor implements InitializingBean // the underlying ReplayCharSequence and the link its about // to become a part of is expected to outlive the current // ReplayCharSequence. - HTMLLinkContext hc = new HTMLLinkContext(context.toString()); + HTMLLinkContext hc = HTMLLinkContext.get(context.toString()); int max = getExtractorParameters().getMaxOutlinks(); Link.addRelativeToBase(curi, max, uri.toString(), hc, hop); } catch (URIException e) { diff --git a/modules/src/main/java/org/archive/modules/extractor/ExtractorHTTP.java b/modules/src/main/java/org/archive/modules/extractor/ExtractorHTTP.java index b38d5b4b..17c62310 100644 --- a/modules/src/main/java/org/archive/modules/extractor/ExtractorHTTP.java +++ b/modules/src/main/java/org/archive/modules/extractor/ExtractorHTTP.java @@ -80,7 +80,7 @@ public class ExtractorHTTP extends Extractor { // TODO: consider possibility of multiple headers try { UURI dest = UURIFactory.getInstance(curi.getUURI(), headerValue); - LinkContext lc = new HTMLLinkContext(headerKey + ":"); + LinkContext lc = HTMLLinkContext.get(headerKey + ":"); Link link = new Link(curi.getUURI(), dest, lc, Hop.REFER); curi.getOutLinks().add(link); numberOfLinksExtracted.incrementAndGet(); diff --git a/modules/src/main/java/org/archive/modules/extractor/HTMLLinkContext.java b/modules/src/main/java/org/archive/modules/extractor/HTMLLinkContext.java index b5eebdab..f44cef74 100644 --- a/modules/src/main/java/org/archive/modules/extractor/HTMLLinkContext.java +++ b/modules/src/main/java/org/archive/modules/extractor/HTMLLinkContext.java @@ -30,26 +30,63 @@ public class HTMLLinkContext extends LinkContext { final public static HTMLLinkContext META = new HTMLLinkContext("meta"); + final public static HTMLLinkContext A_HREF = new HTMLLinkContext("a", "href"); + final public static HTMLLinkContext IMG_SRC = new HTMLLinkContext("img", "src"); + final public static HTMLLinkContext SCRIPT_SRC = new HTMLLinkContext("script", "src"); + final public static HTMLLinkContext META_HREF = new HTMLLinkContext("meta", "href"); /** * The HTML path to the URL. */ - private String path; - + private final String path; + /** + * return an instance of HTMLLinkContext for attribute {@code attr} in + * element {@code el}. returns pre-allocated shared instance for common case, + * or new instance for others. + * @param el element name + * @param attr attribute name + * @return instance of HTMLLinkContext + */ + public static HTMLLinkContext get(CharSequence el, CharSequence attr) { + if (attr.equals("href") || attr.equals("HREF")) { + if (el.equals("a") || el.equals("A")) return A_HREF; + if (el.equals("meta") || el.equals("META")) return META_HREF; + } else if (attr.equals("src") || attr.equals("SRC")) { + if (el.equals("img") || attr.equals("IMG")) return IMG_SRC; + if (el.equals("script") || attr.equals("SCRIPT")) return SCRIPT_SRC; + } + return new HTMLLinkContext(el, attr); + } + /** + * return an instance of HTMLLinkContext for path {@code path}. + * returns pre-allocated shared instance for common case, or new instance for others. + *

TODO: most code calling this method builds path by concatenating element name + * and attribute name. consider changing such code to call {@link #get(CharSequence, CharSequence)} + * instead.

+ * @param path element and attribute in XLink-like path notation + * @return instance of HTMLLinkContext + */ + public static HTMLLinkContext get(String path) { + if (path.equalsIgnoreCase("a/@href")) return A_HREF; + if (path.equalsIgnoreCase("meta/@href")) return META_HREF; + if (path.equalsIgnoreCase("img/@src")) return IMG_SRC; + if (path.equalsIgnoreCase("script/@src")) return SCRIPT_SRC; + return new HTMLLinkContext(path); + } /** * Constructor. * * @param path an XPath-like context, eg "A\@HREF" */ - public HTMLLinkContext(String path) { + protected HTMLLinkContext(String path) { // FIXME: Verify that path really is XPath-like this.path = path; } - public HTMLLinkContext(CharSequence element, CharSequence attribute) { + protected HTMLLinkContext(CharSequence element, CharSequence attribute) { if (attribute == null) { this.path = ""; } else { diff --git a/modules/src/test/java/org/archive/modules/extractor/ExtractorHTMLTest.java b/modules/src/test/java/org/archive/modules/extractor/ExtractorHTMLTest.java index c17c4cb8..7ba6d705 100644 --- a/modules/src/test/java/org/archive/modules/extractor/ExtractorHTMLTest.java +++ b/modules/src/test/java/org/archive/modules/extractor/ExtractorHTMLTest.java @@ -126,10 +126,10 @@ public class ExtractorHTMLTest extends StringExtractorTestBase { private static LinkContext determineContext(String s) { if (s.endsWith(" A")) { - return new HTMLLinkContext("a/@href"); + return HTMLLinkContext.get("a/@href"); } if (s.endsWith(" IMG")) { - return new HTMLLinkContext("img/@src"); + return HTMLLinkContext.get("img/@src"); } return LinkContext.NAVLINK_MISC; }