Files
heritrix3/commons/src/main/java/org/archive/util/UriUtilsTest.java
T
nlevitt c5de0048d1 "Fix" for HER-1951 ExtractorJS produces many false positives
* UriUtils.java
    isLikelyFalsePositive() - check for unusual characters, likely mimetypes, and a couple of other common traps
* UriUtilsTest.java
    add some tests for this code
2011-09-30 00:48:00 +00:00

249 lines
8.6 KiB
Java
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
/* UriUtilsTest
*
* $Id: ArchiveUtilsTest.java 5052 2007-04-10 02:26:52Z gojomo $
*
* Copyright (C) 2010 Internet Archive.
*
* This file is part of the Heritrix web crawler (crawler.archive.org).
*
* Heritrix is free software; you can redistribute it and/or modify
* it under the terms of the GNU Lesser Public License as published by
* the Free Software Foundation; either version 2.1 of the License, or
* any later version.
*
* Heritrix is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Lesser Public License for more details.
*
* You should have received a copy of the GNU Lesser Public License
* along with Heritrix; if not, write to the Free Software
* Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
*/
package org.archive.util;
import junit.framework.Test;
import junit.framework.TestCase;
import junit.framework.TestSuite;
/**
* JUnit test suite for UriUtils.
*
* Several of the tests for the 'legacy' (H1 through at least 1.14.4)
* heuristics are disabled by renaming, because those heuristics have known
* failures; however, until more experience with the new heuristics is
* collected, H1 still uses them for consistency.
*
* @contributor gojomo
* @version $Id: ArchiveUtilsTest.java 5052 2007-04-10 02:26:52Z gojomo $
*/
public class UriUtilsTest extends TestCase {
public UriUtilsTest(final String testName) {
super(testName);
}
/**
* run all the tests for ArchiveUtilsTest
*
* @param argv
* the command line arguments
*/
public static void main(String argv[]) {
junit.textui.TestRunner.run(suite());
}
public static Test suite() {
return new TestSuite(UriUtilsTest.class);
}
/** image URIs that should be considered likely URIs **/
static String[] urisRelativeImages = {
"photo.jpg",
"./photo.jpg",
"../photo.jpg",
"images/photo.jpg",
"../../images/photo.jpg" };
/** check that plausible relative image URIs return true with legacy tests */
public void xestLegacySimpleImageRelatives() {
legacyTryAll(urisRelativeImages, true);
}
/** check that plausible relative image URIs return true with new tests */
public void testNewSimpleImageRelatives() {
tryAll(urisRelativeImages,true);
}
/** absolute URIs that should be considered likely URIs **/
static String[] urisAbsolute = {
"http://example.com",
"http://example.com/", "http://www.example.com",
"http://www.example.com/", "http://www.example.com/about",
"http://www.example.com/about/",
"http://www.example.com/about/index.html", "https://example.com",
"https://example.com/", "https://www.example.com",
"https://www.example.com/", "https://www.example.com/about",
"https://www.example.com/about/",
"https://www.example.com/about/index.html",
"ftp://example.com/public/report.pdf",
"http://a.example.com/combiner/c?js=analytics/sOmni.js,analytics/analytics.js,analytics/zf.js,analytics/externalnielsen.js",
"http://l.example.com/jn/util/anysize/74*74c-86400,http%3A%2F%2Fl.example.com%2Fa%2Fi%2Fus%2Fshine%2Fmoreon%2F74.upallnight.jpg",
// TODO: other schemes? mailto?
};
/** check that absolute URIs return true with legacy tests */
public void testLegacyAbsolutes() {
legacyTryAll(urisAbsolute,true);
}
/** check that absolute URIs return true with new tests */
public void testAbsolutes() {
tryAll(urisAbsolute,true);
}
protected static String[] urisRelative = new String[] {
"default.asp?type=1",
"\\/add\\/page?.crumb=O2.eArRHJUUWRkVHN6L0Y.&frompg=p1",
"/wiki/Ficheiro:Wikiversity-logo.svg",
"cssp!gelui-1/overlay",
"/wiki/%E0%B4%B8%E0%B4%B9%E0%B4%BE%E0%B4%AF%E0%B4%82:To_Read_in_Malayalam",
"/wiki/Wikiversity:Why_create_an_account%3F",
";s.channel=d.channel?d.channel:"
};
public void testRelatives() {
tryAll(urisRelative, true);
}
/** path-absolute images URIs that should be considered likely URIs **/
static String[] urisPathAbsoluteImages = {
"/photo.jpg",
"/images/photo.jpg",
};
/** check that path-absolute image URIs return true with legacy tests*/
public void testLegacySimpleImagePathAbsolutes() {
legacyTryAll(urisPathAbsoluteImages, true);
}
/** check that path-absolute image URIs return true with new tests*/
public void testSimpleImagePathAbsolutes() {
tryAll(urisPathAbsoluteImages, true);
}
/** URI-like strings risking false positives that should NOT be likely URIs **/
static String[] notUrisNaiveFalsePositives = {
"0.99",
"3.14157",
"text/javascript"
};
/** check that typical false-positives of the naive test are not deemed URIs */
public void xestLegacyNaiveFalsePositives() {
legacyTryAll(notUrisNaiveFalsePositives, false);
}
/** check that typical false-positives of the naive test are not deemed URIs */
public void testNaiveFalsePositives() {
tryAll(notUrisNaiveFalsePositives, false);
}
/** strings that should not be considered likely URIs **/
static String[] notUrisNaive = {
"foo bar",
"<script>foo=bar</script>",
"item\t$0.99\tred",
};
/** check that strings that fail naive test are not deemed URIs legacy tests*/
public void testLegacyNaiveNotUris() {
legacyTryAll(notUrisNaive, false);
}
/** check that strings that fail naive test are not deemed URIs new tests*/
public void testNaiveNotUris() {
tryAll(notUrisNaive, false);
}
protected static final String[] unusualCharacterFalsePositives = new String[] {
"),f=document.getElementsByTagName(",
"window.location.href='/'",
"location='http://example.com/blah/'",
"http://example.com/intent/user?screen_name='+p.user+'",
").append(",
"[\\x3cb\\x3e−\\x3c/b\\x3e]",
"http://demo.example.net/panama.php?cgroup=ron728x90&pid=\"+pid+\"&uid=\"+uid+\"&rid=\"+rid+\"&kw=10&cx=10&bh=10",
};
public void testUnusualCharacterFalsePositives() {
tryAll(unusualCharacterFalsePositives, false);
}
protected static final String[] mimetypesFalsePositives = new String[] {
"text/javascript",
"text/css",
"application/x-shockwave-flash",
"text/javaScript",
"text/html",
"application/x-www-form-urlencoded",
"text/xml",
"text/plain",
"application/x-mplayer2",
"application/json",
"image/jpeg",
"image/x-icon",
"audio/mpeg",
"image/gif",
"audio/ogg",
"video/quicktime",
"audio/x-pn-realaudio-plugin",
};
public void testMimetypesFalsePositives() {
tryAll(mimetypesFalsePositives, false);
}
protected static final String[] startsOrEndsWithPlusFalsePositives = new String[] {
"+resp.result+",
";overlay.style.width=viewport_dimensions.width+",
"+_ti;bb.src=",
};
public void testStartsOrEndsWithPlusFalsePositives() {
tryAll(startsOrEndsWithPlusFalsePositives, false);
}
/**
* Test that all supplied candidates give the expected result, for each of
* the 'legacy' (H1) likely-URI-tests
*
* @param candidates String[] to test
* @param expected desired answer
*/
protected void legacyTryAll(String[] candidates, boolean expected) {
for (String candidate : candidates) {
assertEquals("javascript context: " + candidate,
expected,
UriUtils.isLikelyUriJavascriptContextLegacy(candidate));
assertEquals("html context: " + candidate,
expected,
UriUtils.isLikelyUriHtmlContextLegacy(candidate));
}
}
/**
* Test that all supplied candidates give the expected results, for
* the 'new' heuristics now in this class.
* @param candidates String[] to test
* @param expected desired answer
*/
protected void tryAll(String[] candidates, boolean expected) {
for (String candidate : candidates) {
assertEquals("new: " + candidate,
expected,
UriUtils.isLikelyUri(candidate));
assertEquals("html context: " + candidate,
expected,
UriUtils.isLikelyUri(candidate));
}
}
}