diff --git a/CHANGELOG.md b/CHANGELOG.md
index e0d6c72b..ffc0d262 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -10,6 +10,9 @@
with the `--web-auth basic` command-line option. This is useful when running Heritrix behind a reverse proxy that
adds external authentication.
+- **Robots.txt wildcards:** The `*` and `$` wildcard rules from RFC 9309 are now supported.
+ [#656](https://github.com/internetarchive/heritrix3/pull/656)
+
#### Fixes
- **Code editor:** The configuration editor and script console were upgraded to CodeMirror 6. This resolves some browser
diff --git a/README.md b/README.md
index 7d1b1cb3..542279cb 100644
--- a/README.md
+++ b/README.md
@@ -13,8 +13,6 @@ Heritrix is the Internet Archive's open-source, extensible, web-scale, archival-
Heritrix is designed to respect the [`robots.txt`](http://www.robotstxt.org/robotstxt.html) exclusion directives† and [META nofollow tags](http://www.robotstxt.org/meta.html). Please consider the
load your crawl will place on seed sites and set politeness policies accordingly. Also, always identify your crawl with contact information in the `User-Agent` so sites that may be adversely affected by your crawl can contact you or adapt their server behavior accordingly.
-† The newer wildcard extension to robots.txt is [not yet](https://github.com/internetarchive/heritrix3/issues/250) supported.
-
## Documentation
- [Getting Started](https://heritrix.readthedocs.io/en/latest/getting-started.html)
diff --git a/docs/configuring-jobs.rst b/docs/configuring-jobs.rst
index ce3ccf09..08cfa567 100644
--- a/docs/configuring-jobs.rst
+++ b/docs/configuring-jobs.rst
@@ -94,16 +94,17 @@ ignore
.. note::
- Heritrix currently only supports wildcards (*) at the end of paths in robots.txt rules.
+ Heritrix supports RFC 9309 path wildcards (*, $) in robots.txt rules.
The only supported value for robots meta tags is "nofollow" which will cause the HTML extractor to stop processing
- and ignore all links (including embeds like images and stylesheets). Heritrix does not support "rel=nofollow" on
- individual links.
+ and ignore all links (including embeds like images and stylesheets).
.. code-block:: html
+ Obeying "rel=nofollow" on individual links is configured separately as ``obeyRelNoFollow`` on ``ExtractorHTML``.
+
Crawl Scope
-----------
diff --git a/modules/src/main/java/org/archive/modules/net/RobotsDirectives.java b/modules/src/main/java/org/archive/modules/net/RobotsDirectives.java
index 3e393b21..0e54da4b 100644
--- a/modules/src/main/java/org/archive/modules/net/RobotsDirectives.java
+++ b/modules/src/main/java/org/archive/modules/net/RobotsDirectives.java
@@ -16,75 +16,159 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package org.archive.modules.net;
-
+package org.archive.modules.net;
+
import java.io.Serializable;
+import java.util.*;
import java.util.concurrent.ConcurrentSkipListSet;
import org.archive.bdb.AutoKryo;
-
-/**
- * Represents the directives that apply to a user-agent (or set of
- * user-agents)
- */
-public class RobotsDirectives implements Serializable {
- private static final long serialVersionUID = 5386542759286155383L;
-
- protected ConcurrentSkipListSet disallows = new ConcurrentSkipListSet();
- protected ConcurrentSkipListSet allows = new ConcurrentSkipListSet();
- protected float crawlDelay = -1;
- public transient boolean hasDirectives = false;
-
- public boolean allows(String path) {
- return !(longestPrefixLength(disallows, path) > longestPrefixLength(allows, path));
- }
-
+
+/**
+ * Represents the directives that apply to a user-agent (or set of
+ * user-agents)
+ */
+public class RobotsDirectives implements Serializable {
+ private static final long serialVersionUID = 5386542759286155384L;
+
+ protected PatternSet disallows = new PatternSet();
+ protected PatternSet allows = new PatternSet();
+ protected float crawlDelay = -1;
+ public transient boolean hasDirectives = false;
+
/**
- * @param prefixSet
- * @param str
- * @return length of longest entry in {@code prefixSet} that prefixes {@code str}, or zero
- * if no entry prefixes {@code str}
+ * A set of robots.txt path patterns. Patterns without wildcards are stored in a NavigableSet for faster matching.
*/
- protected int longestPrefixLength(ConcurrentSkipListSet prefixSet,
- String str) {
- String possiblePrefix = prefixSet.floor(str);
- if (possiblePrefix != null && str.startsWith(possiblePrefix)) {
- return possiblePrefix.length();
- } else {
- return 0;
+ protected static class PatternSet {
+ private final NavigableSet prefixes = new ConcurrentSkipListSet<>();
+ private final Set wildcards = new HashSet<>();
+
+ public void add(String pattern) {
+ if (pattern.endsWith("$") || pattern.contains("*")) {
+ wildcards.add(new WildcardPattern(pattern));
+ } else {
+ prefixes.add(pattern);
+ }
+ }
+
+ /**
+ * Returns the length of the longest pattern matching the given path, or zero if no patterns match.
+ */
+ private int longestMatch(String path) {
+ int longestMatch = longestPrefixLength(path);
+ for (WildcardPattern pattern : wildcards) {
+ if (pattern.length > longestMatch && pattern.matches(path)) {
+ longestMatch = pattern.length;
+ }
+ }
+ return longestMatch;
+ }
+
+ /**
+ * @return length of longest entry in {@code prefixes} that prefixes {@code str}, or zero
+ * if no entry prefixes {@code str}
+ */
+ private int longestPrefixLength(String str) {
+ String possiblePrefix = prefixes.floor(str);
+ if (possiblePrefix != null && str.startsWith(possiblePrefix)) {
+ return possiblePrefix.length();
+ } else {
+ return 0;
+ }
+ }
+
+ }
+
+ protected static class WildcardPattern {
+ private final String[] segments;
+ private final int length;
+ private final boolean anchored;
+
+ public WildcardPattern(String pattern) {
+ this.length = pattern.length();
+ if (pattern.endsWith("$")) {
+ pattern = pattern.substring(0, pattern.length() - 1);
+ anchored = !pattern.endsWith("*"); // *$ is effectively unanchored
+ } else {
+ anchored = false;
+ }
+ segments = pattern.split("\\*", -1);
+ }
+
+ public boolean matches(String path) {
+ int position = 0;
+ if (!segments[0].isEmpty()) {
+ if (!path.startsWith(segments[0])) {
+ return false;
+ }
+ position = segments[0].length();
+ }
+
+ for (int i = 1; i < segments.length; i++) {
+ String segment = segments[i];
+ if (segment.isEmpty()) continue;
+ int match = path.indexOf(segment, position);
+ if (match < 0) return false;
+ position = match + segment.length();
+ }
+
+ if (anchored) {
+ return position == path.length();
+ }
+
+ return true;
+ }
+
+ @Override
+ public boolean equals(Object object) {
+ if (object == null || getClass() != object.getClass()) return false;
+ WildcardPattern that = (WildcardPattern) object;
+ return length == that.length && anchored == that.anchored && Objects.deepEquals(segments, that.segments);
+ }
+
+ @Override
+ public int hashCode() {
+ return Objects.hash(length, Arrays.hashCode(segments), anchored);
}
}
- public void addDisallow(String path) {
- hasDirectives = true;
- if(path.length()==0) {
- // ignore empty-string disallows
- // (they really mean allow, when alone)
+ public boolean allows(String path) {
+ return disallows.longestMatch(path) <= allows.longestMatch(path);
+ }
+
+ public void addDisallow(String path) {
+ hasDirectives = true;
+ if(path.length()==0) {
+ // ignore empty-string disallows
+ // (they really mean allow, when alone)
return;
- }
- disallows.add(path);
- }
-
- public void addAllow(String path) {
+ }
+ disallows.add(path);
+ }
+
+ public void addAllow(String path) {
hasDirectives = true;
- allows.add(path);
- }
-
- public void setCrawlDelay(float i) {
+ allows.add(path);
+ }
+
+ public void setCrawlDelay(float i) {
hasDirectives = true;
- crawlDelay=i;
- }
-
- public float getCrawlDelay() {
- return crawlDelay;
+ crawlDelay=i;
+ }
+
+ public float getCrawlDelay() {
+ return crawlDelay;
}
// Kryo support
public static void autoregisterTo(AutoKryo kryo) {
kryo.register(RobotsDirectives.class);
+ kryo.register(PatternSet.class);
+ kryo.register(WildcardPattern.class);
+ kryo.register(HashSet.class);
kryo.useReferencesFor(RobotsDirectives.class);
kryo.autoregister(ConcurrentSkipListSet.class); // now used instead of PrefixSet in RobotsDirectives
}
-
+
}
diff --git a/modules/src/test/java/org/archive/modules/net/RobotstxtTest.java b/modules/src/test/java/org/archive/modules/net/RobotstxtTest.java
index f4099d26..7fc7811d 100644
--- a/modules/src/test/java/org/archive/modules/net/RobotstxtTest.java
+++ b/modules/src/test/java/org/archive/modules/net/RobotstxtTest.java
@@ -259,4 +259,71 @@ public class RobotstxtTest {
}
new Robotstxt(new StringReader(builder.toString()));
}
+
+ @Test
+ public void testWildcards() throws IOException {
+ Robotstxt rt = new Robotstxt(new StringReader("""
+ User-Agent: *
+ Disallow: *.gif$
+ Disallow: /example/
+ Allow: /publications/
+ """));
+ assertTrue(rt.getDirectivesFor("x").allows("/"));
+ assertFalse(rt.getDirectivesFor("x").allows("/example/blocked"));
+ assertFalse(rt.getDirectivesFor("x").allows("/image.gif"));
+ assertTrue(rt.getDirectivesFor("x").allows("/image.gif?size=large"));
+ assertTrue(rt.getDirectivesFor("x").allows("/publications/image.gif"));
+
+ rt = new Robotstxt(new StringReader("""
+ User-Agent: *
+ allow: /a/*/c
+ disallow: /a/b/c
+ disallow: /a/bb/c
+ """));
+ assertTrue(rt.getDirectivesFor("x").allows("/a/b/c"));
+ assertFalse(rt.getDirectivesFor("x").allows("/a/bb/c"));
+
+ rt = new Robotstxt(new StringReader("""
+ User-Agent: *
+ allow: /a
+ disallow: /a
+ """));
+ assertTrue(rt.getDirectivesFor("x").allows("/a/b"));
+
+ rt = new Robotstxt(new StringReader("""
+ User-Agent: *
+ allow: /$
+ Disallow: /
+ """));
+ assertTrue(rt.getDirectivesFor("x").allows("/"));
+ assertFalse(rt.getDirectivesFor("x").allows("/a"));
+ assertFalse(rt.getDirectivesFor("x").allows("//"));
+
+ rt = new Robotstxt(new StringReader("""
+ User-Agent: *
+ Disallow: /foo*/bar
+ """));
+ assertFalse(rt.getDirectivesFor("x").allows("/foo/bar"));
+ assertFalse(rt.getDirectivesFor("x").allows("/fooooo/bar"));
+ assertTrue(rt.getDirectivesFor("x").allows("/fox/bar"));
+
+ rt = new Robotstxt(new StringReader("""
+ User-Agent: *
+ Disallow: /foo$/
+ Allow: /foo$
+ """));
+ assertFalse(rt.getDirectivesFor("x").allows("/foo$/"));
+ assertFalse(rt.getDirectivesFor("x").allows("/foo$/bar"));
+ assertTrue(rt.getDirectivesFor("x").allows("/foo$"));
+ assertTrue(rt.getDirectivesFor("x").allows("/foo"));
+
+ rt = new Robotstxt(new StringReader("""
+ User-Agent: *
+ Disallow: /*$
+ Allow: /*
+ """));
+ assertFalse(rt.getDirectivesFor("x").allows("/"));
+ assertFalse(rt.getDirectivesFor("x").allows("/a"));
+ assertFalse(rt.getDirectivesFor("x").allows("//"));
+ }
}