lee/heritrix3
1
0
Fork 0
mirror of https://github.com/internetarchive/heritrix3.git synced 2026-09-19 04:05:45 +00:00
Code Issues Packages Projects Releases Wiki Activity

moving org.archive.net.PublicSuffixes to ia-web-commons

Browse Source
This commit is contained in:
Noah Levitt
2013-12-05 16:13:30 -08:00
parent 552794dc89
commit 98d0901531
3 changed files with 0 additions and 7601 deletions
Download Patch File Download Diff File Expand all files Collapse all files
commons/src/main/java/org/archive/net/PublicSuffixes.java
-363
View File
@@ -1,363 +0,0 @@
/*
* This file is part of the Heritrix web crawler (crawler.archive.org).
*
* Licensed to the Internet Archive (IA) by one or more individual
* contributors.
*
* The IA licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.archive.net;
import java.io.BufferedReader;
import java.io.BufferedWriter;
import java.io.FileInputStream;
import java.io.FileWriter;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.io.OutputStreamWriter;
import java.io.PrintWriter;
import java.io.UnsupportedEncodingException;
import java.util.ArrayList;
import java.util.List;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import org.apache.commons.io.IOUtils;
import org.archive.util.TextUtils;
/**
* Utility class for making use of the information about 'public suffixes' at
* http://publicsuffix.org.
*
* The public suffix list (once known as 'effective TLDs') was motivated by the
* need to decide on which broader domains a subdomain was allowed to set
* cookies. For example, a server at 'www.example.com' can set cookies for
* 'www.example.com' or 'example.com' but not 'com'. 'www.example.co.uk' can set
* cookies for 'www.example.co.uk' or 'example.co.uk' but not 'co.uk' or 'uk'.
* The number of rules for all top-level-domains and 2nd- or 3rd- level domains
* has become quite long; essentially the broadest domain a subdomain may assign
* to is the one that was sold/registered to a specific name registrant.
*
* This concept should be useful in other contexts, too. Grouping URIs (or
* queues of URIs to crawl) together with others sharing the same registered
* suffix may be useful for applying the same rules to all, such as assigning
* them to the same queue or crawler in a multi- machine setup.
*
* As of Heritrix3, we prefer the term 'Assignment Level Domain' (ALD)
* for such domains, by analogy to 'Top Level Domain' (TLD) or '2nd Level
* Domain' (2LD), etc.
*
* @author Gojomo
*
* this version of PublicSuffixes uses suffix-tree data structure for generating less
* redundant regular expression. It may be even possible to write a light-weight,
* thread-safe matcher based on this class.
* @author Kenji Nagahashi
*/
public class PublicSuffixes {
protected static Pattern topmostAssignedSurtPrefixPattern;
protected static String topmostAssignedSurtPrefixRegex;
/**
* prefix tree node. each Node represents sequence of letters (prefix)
* and alternative sequences following it (list of Node's). Nodes in
* {@code branches} are sorted for skip list like lookup and for generating
* effective regular expression (see {@link #compareTo(Node)} and {@link #compareTo(char).)
*
* as is intended for internal use only, there's no access methods. procedures for updating
* prefix tree with new input are defined within this class ({@link #addBranch(CharSequence)}).
*
* terminal node could be represented in two different form: 1) Node with zero branches,
* or 2) Node with zero-length {@code cs}. So, root node must be initialized with empty (not null)
* {@code branches} unless empty string matches the overall pattern.
* {@code cs} must not be null except for root node.
*/
public static class Node implements Comparable<Node> {
protected CharSequence cs;
protected List<Node> branches;
public Node() {
this("", null);
}
protected Node(CharSequence cs) {
this(cs, null);
}
protected Node(CharSequence cs, List<Node> branches) {
this.cs = cs;
this.branches = branches;
}
public void addBranch(CharSequence s) {
if (branches == null) {
branches = new ArrayList<Node>();
branches.add(new Node("", null));
}
for (int i = 0; i < branches.size(); i++) {
Node alt = branches.get(i);
if (alt.add(s)) return;
if (alt.compareTo(s.charAt(0)) > 0) {
Node alt1 = new Node(s, null);
branches.add(i, alt1);
return;
}
}
Node alt2 = new Node(s, null);
branches.add(alt2);
}
public boolean add(CharSequence s) {
int l = Math.min(s.length(), cs.length());
int i = 0;
while (i < l && s.charAt(i) == cs.charAt(i))
i++;
// zero-length match holds only when both cs and s are empty.
if (i == 0) return cs.length() == 0 && s.length() == 0;
if (i < cs.length()) {
CharSequence cs0 = cs.subSequence(0, i);
CharSequence cs1 = cs.subSequence(i, cs.length());
CharSequence cs2 = s.subSequence(i, s.length());
cs = cs0;
Node alt1 = new Node(cs1, branches);
(branches = new ArrayList<Node>()).add(alt1);
addBranch(cs2);
} else {
assert i == cs.length();
addBranch(s.subSequence(i, s.length()));
}
return true;
}
public int compareTo(Node other) {
if (other.cs == null || other.cs.length() == 0)
return (cs == null || cs.length() == 0) ? 0 : -1;
return compareTo(other.cs.charAt(0));
}
public int compareTo(char oc) {
if (cs == null || cs.length() == 0) return 1;
// '!' and '*' must come after ordinary letters, in this order, for regexp
// to work as intended.
char c = cs.charAt(0);
if (c == oc) return 0;
if (c == '!') return oc == '*' ? -1 : 1;
if (c == '*') return 1;
if (oc == '*' || oc == '!') return -1;
return Character.valueOf(c).compareTo(oc);
// for generating the same regexp as previous version.
//return Character.valueOf(oc).compareTo(c);
}
}
/**
* Utility method for dumping a regex String, based on a published public
* suffix list, which matches any SURT-form hostname up through the broadest
* 'private' (assigned/sold) domain-segment. That is, for any of the
* SURT-form hostnames...
*
* com,example, com,example,www, com,example,california,www
*
* ...the regex will match 'com,example,'.
*
* @param args
* @throws IOException
*/
public static void main(String args[]) throws IOException {
InputStream is;
if (args.length == 0 || "=".equals(args[0])) {
// use bundled list
is = PublicSuffixes.class.getClassLoader().getResourceAsStream(
"effective_tld_names.dat");
} else {
is = new FileInputStream(args[0]);
}
BufferedReader reader = new BufferedReader(new InputStreamReader(is, "UTF-8"));
String regex = getTopmostAssignedSurtPrefixRegex(reader);
IOUtils.closeQuietly(is);
boolean needsClose = false;
BufferedWriter writer;
if (args.length >= 2) {
// write to specified file
writer = new BufferedWriter(new FileWriter(args[1]));
needsClose = true;
} else {
// write to stdout
writer = new BufferedWriter(new OutputStreamWriter(System.out));
}
writer.append(regex);
writer.flush();
if (needsClose) {
writer.close();
}
}
/**
* Reads a file of the format promulgated by publicsuffix.org, ignoring
* comments and '!' exceptions/notations, converting domain segments to
* SURT-ordering. Leaves glob-style '*' wildcarding in place. Returns root
* node of SURT-ordered prefix tree.
*
* @param reader
* @return root of prefix tree node.
* @throws IOException
*/
protected static Node readPublishedFileToSurtTrie(BufferedReader reader) throws IOException {
// initializing with empty Alt list prevents empty pattern from being
// created for the first addBranch()
Node alt = new Node(null, new ArrayList<Node>());
String line;
while ((line = reader.readLine()) != null) {
// discard whitespace, empty lines, comments, exceptions
line = line.trim();
if (line.length() == 0 || line.startsWith("//")) continue;
// discard utf8 notation after entry
line = line.split("\\s+")[0];
// TODO: maybe we don't need to create lower-cased String
line = line.toLowerCase();
// SURT-order domain segments
String[] segs = line.split("\\.");
StringBuilder sb = new StringBuilder();
for (int i = segs.length - 1; i >= 0; i--) {
if (segs[i].length() == 0) continue;
sb.append(segs[i]).append(',');
}
alt.addBranch(sb.toString());
}
return alt;
}
/**
* utility function for dumping prefix tree structure. intended for debug use.
* @param alt root of prefix tree.
* @param lv indent level. 0 for root (no indent).
* @param out writer to send output to.
*/
public static void dump(Node alt, int lv, PrintWriter out) {
for (int i = 0; i < lv; i++)
out.print(" ");
out.println(alt.cs != null ? ('"'+alt.cs.toString()+'"') : "(null)");
if (alt.branches != null) {
for (Node br : alt.branches) {
dump(br, lv + 1, out);
}
}
}
/**
* bulids regular expression from prefix-tree {@code alt} into buffer {@code sb}.
* @param alt prefix tree root.
* @param sb StringBuffer to store regular expression.
*/
protected static void buildRegex(Node alt, StringBuilder sb) {
String close = null;
if (alt.cs != null) {
// actually '!' always be the first character, because it is
// always used along with '*'.
for (int i = 0; i < alt.cs.length(); i++) {
char c = alt.cs.charAt(i);
if (c == '!') {
if (close != null)
throw new RuntimeException("more than one '!'");
sb.append("(?=");
close = ")";
} else if (c == '*') {
sb.append("[-\\w]+");
} else {
sb.append(c);
}
}
}
if (alt.branches != null) {
// alt.branches.size() should always be > 1
if (alt.branches.size() > 1) {
sb.append("(?:");
}
String sep = "";
for (Node alt1 : alt.branches) {
sb.append(sep); sep = "|";
buildRegex(alt1, sb);
}
if (alt.branches.size() > 1) {
sb.append(")");
}
}
if (close != null)
sb.append(close);
}
/**
* Converts SURT-ordered list of public prefixes into a Java regex which
* matches the public-portion "plus one" segment, giving the domain on which
* cookies can be set or other policy grouping should occur. Also adds to
* regex a fallback matcher that for any new/unknown TLDs assumes the
* second-level domain is assignable. (Eg: 'zzz,example,').
*
* @param list
* @return
*/
private static String surtPrefixRegexFromTrie(Node trie) {
StringBuilder regex = new StringBuilder();
regex.append("(?ix)^\n");
trie.addBranch("*,"); // for new/unknown TLDs
buildRegex(trie, regex);
regex.append("\n([-\\w]+,)");
return regex.toString();
}
public static synchronized Pattern getTopmostAssignedSurtPrefixPattern() {
if (topmostAssignedSurtPrefixPattern == null) {
topmostAssignedSurtPrefixPattern = Pattern
.compile(getTopmostAssignedSurtPrefixRegex());
}
return topmostAssignedSurtPrefixPattern;
}
public static synchronized String getTopmostAssignedSurtPrefixRegex() {
if (topmostAssignedSurtPrefixRegex == null) {
// use bundled list
try {
BufferedReader reader = new BufferedReader(new InputStreamReader(
PublicSuffixes.class.getClassLoader().getResourceAsStream(
"effective_tld_names.dat"), "UTF-8"));
topmostAssignedSurtPrefixRegex = getTopmostAssignedSurtPrefixRegex(reader);
IOUtils.closeQuietly(reader);
} catch (UnsupportedEncodingException ex) {
// should never happen
throw new RuntimeException(ex);
}
}
return topmostAssignedSurtPrefixRegex;
}
public static String getTopmostAssignedSurtPrefixRegex(BufferedReader reader) {
try {
Node trie = readPublishedFileToSurtTrie(reader);
return surtPrefixRegexFromTrie(trie);
} catch (IOException e) {
throw new RuntimeException(e);
}
}
/**
* Truncate SURT to its topmost assigned domain segment; that is,
* the public suffix plus one segment, but as a SURT-ordered prefix.
*
* if the pattern doesn't match, the passed-in SURT is returned.
*
* @param surt SURT to truncate
* @return truncated-to-topmost-assigned SURT prefix
*/
public static String reduceSurtToAssignmentLevel(String surt) {
Matcher matcher = TextUtils.getMatcher(
getTopmostAssignedSurtPrefixRegex(), surt);
if (matcher.find()) {
surt = matcher.group();
}
TextUtils.recycleMatcher(matcher);
return surt;
}
}
commons/src/main/resources/effective_tld_names.dat
-7045
View File
File diff suppressed because it is too large Load Diff
commons/src/test/java/org/archive/net/PublicSuffixesTest.java
-193
View File
@@ -1,193 +0,0 @@
/*
* This file is part of the Heritrix web crawler (crawler.archive.org).
*
* Licensed to the Internet Archive (IA) by one or more individual
* contributors.
*
* The IA licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.archive.net;
import java.io.PrintWriter;
import java.io.StringWriter;
import java.util.ArrayList;
import java.util.regex.Matcher;
import junit.framework.TestCase;
import org.archive.net.PublicSuffixes.Node;
/**
* Test cases for PublicSuffixes utility. Confirm expected matches/nonmatches
* from constructed regex.
*
* @author gojomo
*/
public class PublicSuffixesTest extends TestCase {
// test of low level implementation
public void testCompare() {
Node n = new Node("hoge");
assertTrue(n.compareTo('a') > 0);
assertEquals(-1, n.compareTo('*'));
assertEquals(-1, n.compareTo('!'));
assertEquals(-1, n.compareTo(new Node("*,")));
assertEquals(-1, n.compareTo(new Node("!muga,")));
assertEquals(-1, n.compareTo(new Node("")));
n = new Node("*,");
assertEquals(1, n.compareTo('a'));
assertEquals(0, n.compareTo('*'));
assertEquals(1, n.compareTo('!'));
assertEquals(0, n.compareTo(new Node("*,")));
assertEquals(1, n.compareTo(new Node("!muga,")));
assertEquals(-1, n.compareTo(new Node("")));
n = new Node("!hoge");
assertEquals(1, n.compareTo('a'));
assertEquals(-1, n.compareTo('*'));
assertEquals(0, n.compareTo('!'));
assertEquals(-1, n.compareTo(new Node("*,")));
assertEquals(0, n.compareTo(new Node("!muga,")));
assertEquals(-1, n.compareTo(new Node("")));
n = new Node("");
assertEquals(1, n.compareTo('a'));
assertEquals(1, n.compareTo('*'));
assertEquals(1, n.compareTo('!'));
assertEquals(0, n.compareTo(new Node("")));
}
protected String dump(Node alt) {
StringWriter w = new StringWriter();
PublicSuffixes.dump(alt, 0, new PrintWriter(w));
return w.toString();
}
public void testTrie1() {
Node alt = new Node(null, new ArrayList<Node>());
alt.addBranch("ac,");
// specifically, should not have empty string as match.
assertEquals("(null)\n" +
" \"ac,\"\n", dump(alt));
alt.addBranch("ac,com,");
assertEquals("(null)\n" +
" \"ac,\"\n" +
" \"com,\"\n" +
" \"\"\n", dump(alt));
alt.addBranch("ac,edu,");
assertEquals("(null)\n" +
" \"ac,\"\n" +
" \"com,\"\n" +
" \"edu,\"\n" +
" \"\"\n", dump(alt));
}
public void testTrie2() {
Node alt = new Node(null, new ArrayList<Node>());
alt.addBranch("ac,");
alt.addBranch("*,");
assertEquals("(null)\n" +
" \"ac,\"\n" +
" \"*,\"\n", dump(alt));
}
public void testTrie3() {
Node alt = new Node(null, new ArrayList<Node>());
alt.addBranch("ac,");
alt.addBranch("ac,!hoge,");
alt.addBranch("ac,*,");
// exception goes first.
assertEquals("(null)\n" +
" \"ac,\"\n" +
" \"!hoge,\"\n" +
" \"*,\"\n" +
" \"\"\n", dump(alt));
}
// test of higher-level functionality
Matcher m = PublicSuffixes.getTopmostAssignedSurtPrefixPattern()
.matcher("");
public void testBasics() {
matchPrefix("com,example,www,", "com,example,");
matchPrefix("com,example,", "com,example,");
matchPrefix("org,archive,www,", "org,archive,");
matchPrefix("org,archive,", "org,archive,");
matchPrefix("fr,yahoo,www,", "fr,yahoo,");
matchPrefix("fr,yahoo,", "fr,yahoo,");
matchPrefix("au,com,foobar,www,", "au,com,foobar,");
matchPrefix("au,com,foobar,", "au,com,foobar,");
matchPrefix("uk,co,virgin,www,", "uk,co,virgin,");
matchPrefix("uk,co,virgin,", "uk,co,virgin,");
matchPrefix("au,com,example,www,", "au,com,example,");
matchPrefix("au,com,example,", "au,com,example,");
matchPrefix("jp,yokohama,public,assigned,www,",
"jp,yokohama,public,assigned,");
matchPrefix("jp,yokohama,public,assigned,", "jp,yokohama,public,assigned,");
}
public void testDomainWithDash() {
matchPrefix("de,bad-site,www", "de,bad-site,");
}
public void testDomainWithNumbers() {
matchPrefix("de,archive4u,www", "de,archive4u,");
}
public void testIPV4() {
assertEquals("unexpected reduction",
"1.2.3.4",
PublicSuffixes.reduceSurtToAssignmentLevel("1.2.3.4"));
}
public void testIPV6() {
assertEquals("unexpected reduction",
"[2001:0db8:85a3:08d3:1319:8a2e:0370:7344]",
PublicSuffixes.reduceSurtToAssignmentLevel(
"[2001:0db8:85a3:08d3:1319:8a2e:0370:7344]"));
}
public void testExceptions() {
matchPrefix("uk,bl,www,", "uk,bl,");
matchPrefix("uk,bl,", "uk,bl,");
matchPrefix("jp,tokyo,city,subdomain,", "jp,tokyo,city,");
matchPrefix("jp,tokyo,city,", "jp,tokyo,city,");
}
public void testFakeTLD() {
// we assume any new/unknonwn TLD should be assumed as 2-level;
// this is preferable for our grouping purpose but might not be
// for a cookie-assigning browser (original purpose of publicsuffixlist)
matchPrefix("zzz,example,www,", "zzz,example,");
}
public void testUnsegmentedHostname() {
m.reset("example");
assertFalse("unexpected match found in 'example'", m.find());
}
public void testTopmostAssignedCaching() {
assertSame("topmostAssignedSurtPrefixPattern not cached",PublicSuffixes.getTopmostAssignedSurtPrefixPattern(),PublicSuffixes.getTopmostAssignedSurtPrefixPattern());
assertSame("topmostAssignedSurtPrefixRegex not cached",PublicSuffixes.getTopmostAssignedSurtPrefixRegex(),PublicSuffixes.getTopmostAssignedSurtPrefixRegex());
}
// TODO: test UTF domains?
protected void matchPrefix(String surtDomain, String expectedAssignedPrefix) {
m.reset(surtDomain);
assertTrue("expected match not found in '" + surtDomain, m.find());
assertEquals("expected match not found", expectedAssignedPrefix, m
.group());
}
}