Merge pull request #656 from internetarchive/robotstxt-wildcard

Support * and $ wildcards in robots.txt

CHANGELOG.md

@@ -10,6 +10,9 @@
   with the `--web-auth basic` command-line option. This is useful when running Heritrix behind a reverse proxy that
   adds external authentication.
 
+- **Robots.txt wildcards:** The `*` and `$` wildcard rules from RFC 9309 are now supported.
+  [#656](https://github.com/internetarchive/heritrix3/pull/656)
+
 #### Fixes
 
 - **Code editor:** The configuration editor and script console were upgraded to CodeMirror 6. This resolves some browser

README.md

@@ -13,8 +13,6 @@ Heritrix is the Internet Archive's open-source, extensible, web-scale, archival-
 Heritrix is designed to respect the [`robots.txt`](http://www.robotstxt.org/robotstxt.html) exclusion directives<sup>†</sup> and [META nofollow tags](http://www.robotstxt.org/meta.html). Please consider the
 load your crawl will place on seed sites and set politeness policies accordingly. Also, always identify your crawl with contact information in the `User-Agent` so sites that may be adversely affected by your crawl can contact you or adapt their server behavior accordingly.
 
-<sup>†</sup> The newer wildcard extension to robots.txt is [not yet](https://github.com/internetarchive/heritrix3/issues/250) supported.
-
 ## Documentation
 
 - [Getting Started](https://heritrix.readthedocs.io/en/latest/getting-started.html)

docs/configuring-jobs.rst

@@ -94,16 +94,17 @@ ignore
 
 .. note::
 
-   Heritrix currently only supports wildcards (*) at the end of paths in robots.txt rules.
+   Heritrix supports RFC 9309 path wildcards (*, $) in robots.txt rules.
 
    The only supported value for robots meta tags is "nofollow" which will cause the HTML extractor to stop processing
-   and ignore all links (including embeds like images and stylesheets). Heritrix does not support "rel=nofollow" on
-   individual links.
+   and ignore all links (including embeds like images and stylesheets).
 
    .. code-block:: html
 
        <meta name="robots" content="nofollow"/>
 
+   Obeying "rel=nofollow" on individual links is configured separately as ``obeyRelNoFollow`` on ``ExtractorHTML``.
+
 Crawl Scope
 -----------
 

modules/src/main/java/org/archive/modules/net/RobotsDirectives.java

@@ -16,75 +16,159 @@
  *  See the License for the specific language governing permissions and
  *  limitations under the License.
  */
-package org.archive.modules.net;
-
+package org.archive.modules.net;
+
 import java.io.Serializable;
+import java.util.*;
 import java.util.concurrent.ConcurrentSkipListSet;
 
 import org.archive.bdb.AutoKryo;
 
-
-/**
- * Represents the directives that apply to a user-agent (or set of
- * user-agents)
- */
-public class RobotsDirectives implements Serializable {
-    private static final long serialVersionUID = 5386542759286155383L;
-    
-    protected ConcurrentSkipListSet<String> disallows = new ConcurrentSkipListSet<String>();
-    protected ConcurrentSkipListSet<String> allows = new ConcurrentSkipListSet<String>();
-    protected float crawlDelay = -1; 
-    public transient boolean hasDirectives = false;
-
-    public boolean allows(String path) {
-        return !(longestPrefixLength(disallows, path) > longestPrefixLength(allows, path));
-    }
-
+
+/**
+ * Represents the directives that apply to a user-agent (or set of
+ * user-agents)
+ */
+public class RobotsDirectives implements Serializable {
+    private static final long serialVersionUID = 5386542759286155384L;
+    
+    protected PatternSet disallows = new PatternSet();
+    protected PatternSet allows = new PatternSet();
+    protected float crawlDelay = -1;
+    public transient boolean hasDirectives = false;
+
     /**
-     * @param prefixSet
-     * @param str
-     * @return length of longest entry in {@code prefixSet} that prefixes {@code str}, or zero
-     *         if no entry prefixes {@code str}
+     * A set of robots.txt path patterns. Patterns without wildcards are stored in a NavigableSet for faster matching.
      */
-    protected int longestPrefixLength(ConcurrentSkipListSet<String> prefixSet,
-            String str) {
-        String possiblePrefix = prefixSet.floor(str);
-        if (possiblePrefix != null && str.startsWith(possiblePrefix)) {
-            return possiblePrefix.length();
-        } else {
-            return 0;
+    protected static class PatternSet {
+        private final NavigableSet<String> prefixes = new ConcurrentSkipListSet<>();
+        private final Set<WildcardPattern> wildcards = new HashSet<>();
+
+        public void add(String pattern) {
+            if (pattern.endsWith("$") || pattern.contains("*")) {
+                wildcards.add(new WildcardPattern(pattern));
+            } else {
+                prefixes.add(pattern);
+            }
+        }
+
+        /**
+         * Returns the length of the longest pattern matching the given path, or zero if no patterns match.
+         */
+        private int longestMatch(String path) {
+            int longestMatch = longestPrefixLength(path);
+            for (WildcardPattern pattern : wildcards) {
+                if (pattern.length > longestMatch && pattern.matches(path)) {
+                    longestMatch = pattern.length;
+                }
+            }
+            return longestMatch;
+        }
+
+        /**
+         * @return length of longest entry in {@code prefixes} that prefixes {@code str}, or zero
+         *         if no entry prefixes {@code str}
+         */
+        private int longestPrefixLength(String str) {
+            String possiblePrefix = prefixes.floor(str);
+            if (possiblePrefix != null && str.startsWith(possiblePrefix)) {
+                return possiblePrefix.length();
+            } else {
+                return 0;
+            }
+        }
+
+    }
+
+    protected static class WildcardPattern {
+        private final String[] segments;
+        private final int length;
+        private final boolean anchored;
+
+        public WildcardPattern(String pattern) {
+            this.length = pattern.length();
+            if (pattern.endsWith("$")) {
+                pattern = pattern.substring(0, pattern.length() - 1);
+                anchored = !pattern.endsWith("*"); // *$ is effectively unanchored
+            } else {
+                anchored = false;
+            }
+            segments = pattern.split("\\*", -1);
+        }
+
+        public boolean matches(String path) {
+            int position = 0;
+            if (!segments[0].isEmpty()) {
+                if (!path.startsWith(segments[0])) {
+                    return false;
+                }
+                position = segments[0].length();
+            }
+
+            for (int i = 1; i < segments.length; i++) {
+                String segment = segments[i];
+                if (segment.isEmpty()) continue;
+                int match = path.indexOf(segment, position);
+                if (match < 0) return false;
+                position = match + segment.length();
+            }
+
+            if (anchored) {
+                return position == path.length();
+            }
+
+            return true;
+        }
+
+        @Override
+        public boolean equals(Object object) {
+            if (object == null || getClass() != object.getClass()) return false;
+            WildcardPattern that = (WildcardPattern) object;
+            return length == that.length && anchored == that.anchored && Objects.deepEquals(segments, that.segments);
+        }
+
+        @Override
+        public int hashCode() {
+            return Objects.hash(length, Arrays.hashCode(segments), anchored);
         }
     }
 
-    public void addDisallow(String path) {
-        hasDirectives = true;
-        if(path.length()==0) {
-            // ignore empty-string disallows 
-            // (they really mean allow, when alone)
+    public boolean allows(String path) {
+        return disallows.longestMatch(path) <= allows.longestMatch(path);
+    }
+
+    public void addDisallow(String path) {
+        hasDirectives = true;
+        if(path.length()==0) {
+            // ignore empty-string disallows 
+            // (they really mean allow, when alone)
             return;
-        }
-        disallows.add(path);
-    }
-
-    public void addAllow(String path) {
+        }
+        disallows.add(path);
+    }
+
+    public void addAllow(String path) {
         hasDirectives = true;
-        allows.add(path);
-    }
-
-    public void setCrawlDelay(float i) {
+        allows.add(path);
+    }
+
+    public void setCrawlDelay(float i) {
         hasDirectives = true;
-        crawlDelay=i;
-    }
-
-    public float getCrawlDelay() {
-        return crawlDelay;
+        crawlDelay=i;
+    }
+
+    public float getCrawlDelay() {
+        return crawlDelay;
     }
     
     // Kryo support
     public static void autoregisterTo(AutoKryo kryo) {
         kryo.register(RobotsDirectives.class);
+        kryo.register(PatternSet.class);
+        kryo.register(WildcardPattern.class);
+        kryo.register(HashSet.class);
         kryo.useReferencesFor(RobotsDirectives.class);
         kryo.autoregister(ConcurrentSkipListSet.class); // now used instead of PrefixSet in RobotsDirectives
     }
-
+
 }

modules/src/test/java/org/archive/modules/net/RobotstxtTest.java

@@ -259,4 +259,71 @@ public class RobotstxtTest {
         }
         new Robotstxt(new StringReader(builder.toString()));
     }
+
+    @Test
+    public void testWildcards() throws IOException {
+        Robotstxt rt = new Robotstxt(new StringReader("""
+                User-Agent: *
+                Disallow: *.gif$
+                Disallow: /example/
+                Allow: /publications/
+                """));
+        assertTrue(rt.getDirectivesFor("x").allows("/"));
+        assertFalse(rt.getDirectivesFor("x").allows("/example/blocked"));
+        assertFalse(rt.getDirectivesFor("x").allows("/image.gif"));
+        assertTrue(rt.getDirectivesFor("x").allows("/image.gif?size=large"));
+        assertTrue(rt.getDirectivesFor("x").allows("/publications/image.gif"));
+
+        rt = new Robotstxt(new StringReader("""
+                User-Agent: *
+                allow: /a/*/c
+                disallow: /a/b/c
+                disallow: /a/bb/c
+                """));
+        assertTrue(rt.getDirectivesFor("x").allows("/a/b/c"));
+        assertFalse(rt.getDirectivesFor("x").allows("/a/bb/c"));
+
+        rt = new Robotstxt(new StringReader("""
+                User-Agent: *
+                allow: /a
+                disallow: /a
+                """));
+        assertTrue(rt.getDirectivesFor("x").allows("/a/b"));
+
+        rt = new Robotstxt(new StringReader("""
+                User-Agent: *
+                allow: /$
+                Disallow: /
+                """));
+        assertTrue(rt.getDirectivesFor("x").allows("/"));
+        assertFalse(rt.getDirectivesFor("x").allows("/a"));
+        assertFalse(rt.getDirectivesFor("x").allows("//"));
+
+        rt = new Robotstxt(new StringReader("""
+                User-Agent: *
+                Disallow: /foo*/bar
+                """));
+        assertFalse(rt.getDirectivesFor("x").allows("/foo/bar"));
+        assertFalse(rt.getDirectivesFor("x").allows("/fooooo/bar"));
+        assertTrue(rt.getDirectivesFor("x").allows("/fox/bar"));
+
+        rt = new Robotstxt(new StringReader("""
+                User-Agent: *
+                Disallow: /foo$/
+                Allow: /foo$
+                """));
+        assertFalse(rt.getDirectivesFor("x").allows("/foo$/"));
+        assertFalse(rt.getDirectivesFor("x").allows("/foo$/bar"));
+        assertTrue(rt.getDirectivesFor("x").allows("/foo$"));
+        assertTrue(rt.getDirectivesFor("x").allows("/foo"));
+
+        rt = new Robotstxt(new StringReader("""
+                User-Agent: *
+                Disallow: /*$
+                Allow: /*
+                """));
+        assertFalse(rt.getDirectivesFor("x").allows("/"));
+        assertFalse(rt.getDirectivesFor("x").allows("/a"));
+        assertFalse(rt.getDirectivesFor("x").allows("//"));
+    }
 }