diff --git a/modules/src/main/java/org/archive/modules/net/RobotsPolicy.java b/modules/src/main/java/org/archive/modules/net/RobotsPolicy.java index 1bace024..d4a3a616 100644 --- a/modules/src/main/java/org/archive/modules/net/RobotsPolicy.java +++ b/modules/src/main/java/org/archive/modules/net/RobotsPolicy.java @@ -45,7 +45,7 @@ abstract public class RobotsPolicy { public String getPath(CrawlURI curi) { try { - return curi.getUURI().getPath(); + return curi.getUURI().getPathQuery(); // TODO rename method and change callers; add test } catch (URIException e) { // unlikely return "";