diff --git a/modules/src/main/java/org/archive/modules/net/CustomRobotsPolicy.java b/modules/src/main/java/org/archive/modules/net/CustomRobotsPolicy.java index 1796189e..988dd3bc 100644 --- a/modules/src/main/java/org/archive/modules/net/CustomRobotsPolicy.java +++ b/modules/src/main/java/org/archive/modules/net/CustomRobotsPolicy.java @@ -59,7 +59,7 @@ public class CustomRobotsPolicy extends RobotsPolicy { @Override public boolean allows(String userAgent, CrawlURI curi, Robotstxt robotstxt) { - return customRobotstxt.getDirectivesFor(userAgent).allows(getPath(curi)); + return customRobotstxt.getDirectivesFor(userAgent).allows(getPathQuery(curi)); } @Override diff --git a/modules/src/main/java/org/archive/modules/net/FirstNamedRobotsPolicy.java b/modules/src/main/java/org/archive/modules/net/FirstNamedRobotsPolicy.java index f9c20542..4d28ec4a 100644 --- a/modules/src/main/java/org/archive/modules/net/FirstNamedRobotsPolicy.java +++ b/modules/src/main/java/org/archive/modules/net/FirstNamedRobotsPolicy.java @@ -78,7 +78,7 @@ public class FirstNamedRobotsPolicy extends RobotsPolicy { public boolean allows(String userAgent, CrawlURI curi, Robotstxt robotstxt) { RobotsDirectives directives = robotstxt.getDirectivesFor(userAgent, false); if(directives!=null) { - return directives.allows(getPath(curi)); + return directives.allows(getPathQuery(curi)); } for(String candidate : candidateUserAgents) { @@ -87,10 +87,10 @@ public class FirstNamedRobotsPolicy extends RobotsPolicy { if(shouldMasquerade) { curi.setUserAgent(candidate); } - return directives.allows(getPath(curi)); + return directives.allows(getPathQuery(curi)); } } - return robotstxt.getDirectivesFor(userAgent).allows(getPath(curi)); + return robotstxt.getDirectivesFor(userAgent).allows(getPathQuery(curi)); } @Override diff --git a/modules/src/main/java/org/archive/modules/net/MostFavoredRobotsPolicy.java b/modules/src/main/java/org/archive/modules/net/MostFavoredRobotsPolicy.java index 5d4457ad..2d54d997 100644 --- a/modules/src/main/java/org/archive/modules/net/MostFavoredRobotsPolicy.java +++ b/modules/src/main/java/org/archive/modules/net/MostFavoredRobotsPolicy.java @@ -72,11 +72,11 @@ public class MostFavoredRobotsPolicy extends RobotsPolicy { @Override public boolean allows(String userAgent, CrawlURI curi, Robotstxt robotstxt) { - if (robotstxt.getDirectivesFor(userAgent).allows(getPath(curi))) { + if (robotstxt.getDirectivesFor(userAgent).allows(getPathQuery(curi))) { return true; } for(String candidate : candidateUserAgents) { - if (robotstxt.getDirectivesFor(candidate).allows(getPath(curi))) { + if (robotstxt.getDirectivesFor(candidate).allows(getPathQuery(curi))) { if(shouldMasquerade) { curi.setUserAgent(candidate); } diff --git a/modules/src/main/java/org/archive/modules/net/ObeyRobotsPolicy.java b/modules/src/main/java/org/archive/modules/net/ObeyRobotsPolicy.java index 60fb4e7a..52f962cd 100644 --- a/modules/src/main/java/org/archive/modules/net/ObeyRobotsPolicy.java +++ b/modules/src/main/java/org/archive/modules/net/ObeyRobotsPolicy.java @@ -30,7 +30,7 @@ public class ObeyRobotsPolicy extends RobotsPolicy { @Override public boolean allows(String userAgent, CrawlURI curi, Robotstxt robotstxt) { - return robotstxt.getDirectivesFor(userAgent).allows(getPath(curi)); + return robotstxt.getDirectivesFor(userAgent).allows(getPathQuery(curi)); } @Override diff --git a/modules/src/main/java/org/archive/modules/net/RobotsPolicy.java b/modules/src/main/java/org/archive/modules/net/RobotsPolicy.java index d4a3a616..742586ff 100644 --- a/modules/src/main/java/org/archive/modules/net/RobotsPolicy.java +++ b/modules/src/main/java/org/archive/modules/net/RobotsPolicy.java @@ -43,9 +43,9 @@ abstract public class RobotsPolicy { public abstract boolean obeyMetaRobotsNofollow(); - public String getPath(CrawlURI curi) { + public String getPathQuery(CrawlURI curi) { try { - return curi.getUURI().getPathQuery(); // TODO rename method and change callers; add test + return curi.getUURI().getPathQuery(); } catch (URIException e) { // unlikely return ""; diff --git a/modules/src/test/java/org/archive/modules/net/RobotsPolicyTest.java b/modules/src/test/java/org/archive/modules/net/RobotsPolicyTest.java new file mode 100644 index 00000000..d4e8c7ec --- /dev/null +++ b/modules/src/test/java/org/archive/modules/net/RobotsPolicyTest.java @@ -0,0 +1,50 @@ +/* + * This file is part of the Heritrix web crawler (crawler.archive.org). + * + * Licensed to the Internet Archive (IA) by one or more individual + * contributors. + * + * The IA licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.archive.modules.net; + +import java.io.IOException; + +import junit.framework.TestCase; + +import org.archive.modules.CrawlURI; +import org.archive.net.UURIFactory; + +public class RobotsPolicyTest extends TestCase { + + public void testClassicRobots() throws IOException { + Robotstxt rtxt = RobotstxtTest.sampleRobots1(); + RobotsPolicy policy = RobotsPolicy.STANDARD_POLICIES.get("classic"); + evalQueryString(policy, rtxt); + } + + + /** + * HER-1976: query-string disallow + * + * @param policy + * @param r + * @throws IOException + */ + public void evalQueryString(RobotsPolicy policy, Robotstxt rtxt) throws IOException { + CrawlURI qs = new CrawlURI(UURIFactory.getInstance("http://example.com/ok?butno=something")); + + assertFalse("ignoring query-string", policy.allows("Mozilla allowbot2 99.9", qs, rtxt)); + + } +} \ No newline at end of file diff --git a/modules/src/test/java/org/archive/modules/net/RobotstxtTest.java b/modules/src/test/java/org/archive/modules/net/RobotstxtTest.java index 5b95d956..c8083e96 100644 --- a/modules/src/test/java/org/archive/modules/net/RobotstxtTest.java +++ b/modules/src/test/java/org/archive/modules/net/RobotstxtTest.java @@ -56,7 +56,7 @@ public class RobotstxtTest extends TestCase { assertEquals(0, r.agentsToDirectives.size()); } - Robotstxt sampleRobots1() throws IOException { + static Robotstxt sampleRobots1() throws IOException { BufferedReader reader = new BufferedReader( new StringReader( "User-agent: *\n" + @@ -71,6 +71,7 @@ public class RobotstxtTest extends TestCase { "\n"+ "User-agent: allowbot2\n" + "Disallow: /foo\n" + + "Disallow: /ok?butno\n" + "Allow: /\n"+ "\n"+ "User-agent: delaybot\n" + @@ -93,6 +94,7 @@ public class RobotstxtTest extends TestCase { " Disallow: \n" + " User-agent: allowbot2\n" + " Disallow: /foo\n" + + " Disallow: /ok?butno\n" + " Allow: /\n"+ " User-agent: delaybot\n" + " Disallow: /\n" + @@ -116,19 +118,27 @@ public class RobotstxtTest extends TestCase { // bot allowed with empty disallows assertTrue(r.getDirectivesFor("Mozilla allowbot1 99.9").allows("/path")); assertTrue(r.getDirectivesFor("Mozilla allowbot1 99.9").allows("/")); + // bot allowed with explicit allow assertTrue(r.getDirectivesFor("Mozilla allowbot2 99.9").allows("/path")); assertTrue(r.getDirectivesFor("Mozilla allowbot2 99.9").allows("/")); + // bot denied with specific disallow overriding general allow assertFalse(r.getDirectivesFor("Mozilla allowbot2 99.9").allows("/foo")); + // HER-1976: query-string disallow + assertFalse("ignoring query-string", r.getDirectivesFor("Mozilla allowbot2 99.9").allows("/ok?butno=something")); + // bot denied with blanket deny assertFalse(r.getDirectivesFor("Mozilla denybot 99.9").allows("/path")); assertFalse(r.getDirectivesFor("Mozilla denybot 99.9").allows("/")); + // unnamed bot with mixed catchall allow/deny assertTrue(r.getDirectivesFor("Mozilla anonbot 99.9").allows("/path")); assertFalse(r.getDirectivesFor("Mozilla anonbot 99.9").allows("/cgi-bin/foo.pl")); + // no crawl-delay assertEquals(r.getDirectivesFor("Mozilla denybot 99.9").getCrawlDelay(),-1f); + // with crawl-delay assertEquals(r.getDirectivesFor("Mozilla delaybot 99.9").getCrawlDelay(),20f); }