mirror of
https://github.com/internetarchive/heritrix3.git
synced 2026-09-25 15:16:24 +00:00
[HER-1976] H3: query-portion ignored when checking against robots.txt rules (using only path)
* RobotsPolicy.java: use UURI.getPathQuery -- like H1 does
This commit is contained in:
@@ -45,7 +45,7 @@ abstract public class RobotsPolicy {
|
||||
|
||||
public String getPath(CrawlURI curi) {
|
||||
try {
|
||||
return curi.getUURI().getPath();
|
||||
return curi.getUURI().getPathQuery(); // TODO rename method and change callers; add test
|
||||
} catch (URIException e) {
|
||||
// unlikely
|
||||
return "";
|
||||
|
||||
Reference in New Issue
Block a user