From c7b7ee18420de21c5deee15a7216e035b0d65e59 Mon Sep 17 00:00:00 2001 From: Alex Osborne Date: Fri, 6 Jun 2025 18:55:21 +0900 Subject: [PATCH] Support * and $ wildcards in robots.txt --- CHANGELOG.md | 3 + README.md | 2 - docs/configuring-jobs.rst | 7 +- .../archive/modules/net/RobotsDirectives.java | 184 +++++++++++++----- .../archive/modules/net/RobotstxtTest.java | 67 +++++++ 5 files changed, 208 insertions(+), 55 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index e0d6c72b..ffc0d262 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,6 +10,9 @@ with the `--web-auth basic` command-line option. This is useful when running Heritrix behind a reverse proxy that adds external authentication. +- **Robots.txt wildcards:** The `*` and `$` wildcard rules from RFC 9309 are now supported. + [#656](https://github.com/internetarchive/heritrix3/pull/656) + #### Fixes - **Code editor:** The configuration editor and script console were upgraded to CodeMirror 6. This resolves some browser diff --git a/README.md b/README.md index 7d1b1cb3..542279cb 100644 --- a/README.md +++ b/README.md @@ -13,8 +13,6 @@ Heritrix is the Internet Archive's open-source, extensible, web-scale, archival- Heritrix is designed to respect the [`robots.txt`](http://www.robotstxt.org/robotstxt.html) exclusion directives† and [META nofollow tags](http://www.robotstxt.org/meta.html). Please consider the load your crawl will place on seed sites and set politeness policies accordingly. Also, always identify your crawl with contact information in the `User-Agent` so sites that may be adversely affected by your crawl can contact you or adapt their server behavior accordingly. -† The newer wildcard extension to robots.txt is [not yet](https://github.com/internetarchive/heritrix3/issues/250) supported. - ## Documentation - [Getting Started](https://heritrix.readthedocs.io/en/latest/getting-started.html) diff --git a/docs/configuring-jobs.rst b/docs/configuring-jobs.rst index ce3ccf09..08cfa567 100644 --- a/docs/configuring-jobs.rst +++ b/docs/configuring-jobs.rst @@ -94,16 +94,17 @@ ignore .. note:: - Heritrix currently only supports wildcards (*) at the end of paths in robots.txt rules. + Heritrix supports RFC 9309 path wildcards (*, $) in robots.txt rules. The only supported value for robots meta tags is "nofollow" which will cause the HTML extractor to stop processing - and ignore all links (including embeds like images and stylesheets). Heritrix does not support "rel=nofollow" on - individual links. + and ignore all links (including embeds like images and stylesheets). .. code-block:: html + Obeying "rel=nofollow" on individual links is configured separately as ``obeyRelNoFollow`` on ``ExtractorHTML``. + Crawl Scope ----------- diff --git a/modules/src/main/java/org/archive/modules/net/RobotsDirectives.java b/modules/src/main/java/org/archive/modules/net/RobotsDirectives.java index 3e393b21..0e54da4b 100644 --- a/modules/src/main/java/org/archive/modules/net/RobotsDirectives.java +++ b/modules/src/main/java/org/archive/modules/net/RobotsDirectives.java @@ -16,75 +16,159 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.archive.modules.net; - +package org.archive.modules.net; + import java.io.Serializable; +import java.util.*; import java.util.concurrent.ConcurrentSkipListSet; import org.archive.bdb.AutoKryo; - -/** - * Represents the directives that apply to a user-agent (or set of - * user-agents) - */ -public class RobotsDirectives implements Serializable { - private static final long serialVersionUID = 5386542759286155383L; - - protected ConcurrentSkipListSet disallows = new ConcurrentSkipListSet(); - protected ConcurrentSkipListSet allows = new ConcurrentSkipListSet(); - protected float crawlDelay = -1; - public transient boolean hasDirectives = false; - - public boolean allows(String path) { - return !(longestPrefixLength(disallows, path) > longestPrefixLength(allows, path)); - } - + +/** + * Represents the directives that apply to a user-agent (or set of + * user-agents) + */ +public class RobotsDirectives implements Serializable { + private static final long serialVersionUID = 5386542759286155384L; + + protected PatternSet disallows = new PatternSet(); + protected PatternSet allows = new PatternSet(); + protected float crawlDelay = -1; + public transient boolean hasDirectives = false; + /** - * @param prefixSet - * @param str - * @return length of longest entry in {@code prefixSet} that prefixes {@code str}, or zero - * if no entry prefixes {@code str} + * A set of robots.txt path patterns. Patterns without wildcards are stored in a NavigableSet for faster matching. */ - protected int longestPrefixLength(ConcurrentSkipListSet prefixSet, - String str) { - String possiblePrefix = prefixSet.floor(str); - if (possiblePrefix != null && str.startsWith(possiblePrefix)) { - return possiblePrefix.length(); - } else { - return 0; + protected static class PatternSet { + private final NavigableSet prefixes = new ConcurrentSkipListSet<>(); + private final Set wildcards = new HashSet<>(); + + public void add(String pattern) { + if (pattern.endsWith("$") || pattern.contains("*")) { + wildcards.add(new WildcardPattern(pattern)); + } else { + prefixes.add(pattern); + } + } + + /** + * Returns the length of the longest pattern matching the given path, or zero if no patterns match. + */ + private int longestMatch(String path) { + int longestMatch = longestPrefixLength(path); + for (WildcardPattern pattern : wildcards) { + if (pattern.length > longestMatch && pattern.matches(path)) { + longestMatch = pattern.length; + } + } + return longestMatch; + } + + /** + * @return length of longest entry in {@code prefixes} that prefixes {@code str}, or zero + * if no entry prefixes {@code str} + */ + private int longestPrefixLength(String str) { + String possiblePrefix = prefixes.floor(str); + if (possiblePrefix != null && str.startsWith(possiblePrefix)) { + return possiblePrefix.length(); + } else { + return 0; + } + } + + } + + protected static class WildcardPattern { + private final String[] segments; + private final int length; + private final boolean anchored; + + public WildcardPattern(String pattern) { + this.length = pattern.length(); + if (pattern.endsWith("$")) { + pattern = pattern.substring(0, pattern.length() - 1); + anchored = !pattern.endsWith("*"); // *$ is effectively unanchored + } else { + anchored = false; + } + segments = pattern.split("\\*", -1); + } + + public boolean matches(String path) { + int position = 0; + if (!segments[0].isEmpty()) { + if (!path.startsWith(segments[0])) { + return false; + } + position = segments[0].length(); + } + + for (int i = 1; i < segments.length; i++) { + String segment = segments[i]; + if (segment.isEmpty()) continue; + int match = path.indexOf(segment, position); + if (match < 0) return false; + position = match + segment.length(); + } + + if (anchored) { + return position == path.length(); + } + + return true; + } + + @Override + public boolean equals(Object object) { + if (object == null || getClass() != object.getClass()) return false; + WildcardPattern that = (WildcardPattern) object; + return length == that.length && anchored == that.anchored && Objects.deepEquals(segments, that.segments); + } + + @Override + public int hashCode() { + return Objects.hash(length, Arrays.hashCode(segments), anchored); } } - public void addDisallow(String path) { - hasDirectives = true; - if(path.length()==0) { - // ignore empty-string disallows - // (they really mean allow, when alone) + public boolean allows(String path) { + return disallows.longestMatch(path) <= allows.longestMatch(path); + } + + public void addDisallow(String path) { + hasDirectives = true; + if(path.length()==0) { + // ignore empty-string disallows + // (they really mean allow, when alone) return; - } - disallows.add(path); - } - - public void addAllow(String path) { + } + disallows.add(path); + } + + public void addAllow(String path) { hasDirectives = true; - allows.add(path); - } - - public void setCrawlDelay(float i) { + allows.add(path); + } + + public void setCrawlDelay(float i) { hasDirectives = true; - crawlDelay=i; - } - - public float getCrawlDelay() { - return crawlDelay; + crawlDelay=i; + } + + public float getCrawlDelay() { + return crawlDelay; } // Kryo support public static void autoregisterTo(AutoKryo kryo) { kryo.register(RobotsDirectives.class); + kryo.register(PatternSet.class); + kryo.register(WildcardPattern.class); + kryo.register(HashSet.class); kryo.useReferencesFor(RobotsDirectives.class); kryo.autoregister(ConcurrentSkipListSet.class); // now used instead of PrefixSet in RobotsDirectives } - + } diff --git a/modules/src/test/java/org/archive/modules/net/RobotstxtTest.java b/modules/src/test/java/org/archive/modules/net/RobotstxtTest.java index f4099d26..7fc7811d 100644 --- a/modules/src/test/java/org/archive/modules/net/RobotstxtTest.java +++ b/modules/src/test/java/org/archive/modules/net/RobotstxtTest.java @@ -259,4 +259,71 @@ public class RobotstxtTest { } new Robotstxt(new StringReader(builder.toString())); } + + @Test + public void testWildcards() throws IOException { + Robotstxt rt = new Robotstxt(new StringReader(""" + User-Agent: * + Disallow: *.gif$ + Disallow: /example/ + Allow: /publications/ + """)); + assertTrue(rt.getDirectivesFor("x").allows("/")); + assertFalse(rt.getDirectivesFor("x").allows("/example/blocked")); + assertFalse(rt.getDirectivesFor("x").allows("/image.gif")); + assertTrue(rt.getDirectivesFor("x").allows("/image.gif?size=large")); + assertTrue(rt.getDirectivesFor("x").allows("/publications/image.gif")); + + rt = new Robotstxt(new StringReader(""" + User-Agent: * + allow: /a/*/c + disallow: /a/b/c + disallow: /a/bb/c + """)); + assertTrue(rt.getDirectivesFor("x").allows("/a/b/c")); + assertFalse(rt.getDirectivesFor("x").allows("/a/bb/c")); + + rt = new Robotstxt(new StringReader(""" + User-Agent: * + allow: /a + disallow: /a + """)); + assertTrue(rt.getDirectivesFor("x").allows("/a/b")); + + rt = new Robotstxt(new StringReader(""" + User-Agent: * + allow: /$ + Disallow: / + """)); + assertTrue(rt.getDirectivesFor("x").allows("/")); + assertFalse(rt.getDirectivesFor("x").allows("/a")); + assertFalse(rt.getDirectivesFor("x").allows("//")); + + rt = new Robotstxt(new StringReader(""" + User-Agent: * + Disallow: /foo*/bar + """)); + assertFalse(rt.getDirectivesFor("x").allows("/foo/bar")); + assertFalse(rt.getDirectivesFor("x").allows("/fooooo/bar")); + assertTrue(rt.getDirectivesFor("x").allows("/fox/bar")); + + rt = new Robotstxt(new StringReader(""" + User-Agent: * + Disallow: /foo$/ + Allow: /foo$ + """)); + assertFalse(rt.getDirectivesFor("x").allows("/foo$/")); + assertFalse(rt.getDirectivesFor("x").allows("/foo$/bar")); + assertTrue(rt.getDirectivesFor("x").allows("/foo$")); + assertTrue(rt.getDirectivesFor("x").allows("/foo")); + + rt = new Robotstxt(new StringReader(""" + User-Agent: * + Disallow: /*$ + Allow: /* + """)); + assertFalse(rt.getDirectivesFor("x").allows("/")); + assertFalse(rt.getDirectivesFor("x").allows("/a")); + assertFalse(rt.getDirectivesFor("x").allows("//")); + } }