[HER-1976] query-portion ignored when checking against robots.txt rules

(using only path)
(RobotsPolicyTest) added for case that failed pre-fix
(RobotsTxtTest) added case similar to problem (but was already OK at
RobotsTxt level)
(RobotsPolicy, *RobotsPolicy) rename method to reflect new behavior

modules/src/main/java/org/archive/modules/net/CustomRobotsPolicy.java

@@ -59,7 +59,7 @@ public class CustomRobotsPolicy extends RobotsPolicy {
     
     @Override
     public boolean allows(String userAgent, CrawlURI curi, Robotstxt robotstxt) {
-        return customRobotstxt.getDirectivesFor(userAgent).allows(getPath(curi));
+        return customRobotstxt.getDirectivesFor(userAgent).allows(getPathQuery(curi));
     }
 
     @Override

modules/src/main/java/org/archive/modules/net/FirstNamedRobotsPolicy.java

@@ -78,7 +78,7 @@ public class FirstNamedRobotsPolicy extends RobotsPolicy {
     public boolean allows(String userAgent, CrawlURI curi, Robotstxt robotstxt) {
         RobotsDirectives directives = robotstxt.getDirectivesFor(userAgent, false);
         if(directives!=null) {
-            return directives.allows(getPath(curi));
+            return directives.allows(getPathQuery(curi));
         }
         
         for(String candidate : candidateUserAgents) {
@@ -87,10 +87,10 @@ public class FirstNamedRobotsPolicy extends RobotsPolicy {
                 if(shouldMasquerade) {
                     curi.setUserAgent(candidate);
                 }
-                return directives.allows(getPath(curi));
+                return directives.allows(getPathQuery(curi));
             }
         }
-        return robotstxt.getDirectivesFor(userAgent).allows(getPath(curi));
+        return robotstxt.getDirectivesFor(userAgent).allows(getPathQuery(curi));
     }
     
     @Override

modules/src/main/java/org/archive/modules/net/MostFavoredRobotsPolicy.java

@@ -72,11 +72,11 @@ public class MostFavoredRobotsPolicy extends RobotsPolicy {
     
     @Override
     public boolean allows(String userAgent, CrawlURI curi, Robotstxt robotstxt) {
-        if (robotstxt.getDirectivesFor(userAgent).allows(getPath(curi))) {
+        if (robotstxt.getDirectivesFor(userAgent).allows(getPathQuery(curi))) {
             return true;
         }
         for(String candidate : candidateUserAgents) {
-            if (robotstxt.getDirectivesFor(candidate).allows(getPath(curi))) {
+            if (robotstxt.getDirectivesFor(candidate).allows(getPathQuery(curi))) {
                 if(shouldMasquerade) {
                     curi.setUserAgent(candidate);
                 }

modules/src/main/java/org/archive/modules/net/ObeyRobotsPolicy.java

@@ -30,7 +30,7 @@ public class ObeyRobotsPolicy extends RobotsPolicy {
     
     @Override
     public boolean allows(String userAgent, CrawlURI curi, Robotstxt robotstxt) {
-        return robotstxt.getDirectivesFor(userAgent).allows(getPath(curi));
+        return robotstxt.getDirectivesFor(userAgent).allows(getPathQuery(curi));
     }
 
     @Override

modules/src/main/java/org/archive/modules/net/RobotsPolicy.java

@@ -43,9 +43,9 @@ abstract public class RobotsPolicy {
     
     public abstract boolean obeyMetaRobotsNofollow();
     
-    public String getPath(CrawlURI curi) {
+    public String getPathQuery(CrawlURI curi) {
         try {
-            return curi.getUURI().getPathQuery(); // TODO rename method and change callers; add test
+            return curi.getUURI().getPathQuery();
         } catch (URIException e) {
             // unlikely
             return "";

modules/src/test/java/org/archive/modules/net/RobotsPolicyTest.java

@@ -0,0 +1,50 @@
+/*
+ *  This file is part of the Heritrix web crawler (crawler.archive.org).
+ *
+ *  Licensed to the Internet Archive (IA) by one or more individual 
+ *  contributors. 
+ *
+ *  The IA licenses this file to You under the Apache License, Version 2.0
+ *  (the "License"); you may not use this file except in compliance with
+ *  the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ */
+package org.archive.modules.net;
+
+import java.io.IOException;
+
+import junit.framework.TestCase;
+
+import org.archive.modules.CrawlURI;
+import org.archive.net.UURIFactory;
+
+public class RobotsPolicyTest extends TestCase {
+
+    public void testClassicRobots() throws IOException {
+        Robotstxt rtxt = RobotstxtTest.sampleRobots1();
+        RobotsPolicy policy = RobotsPolicy.STANDARD_POLICIES.get("classic");
+        evalQueryString(policy, rtxt); 
+    }
+    
+    
+    /**
+     * HER-1976: query-string disallow
+     * 
+     * @param policy
+     * @param r
+     * @throws IOException
+     */
+    public void evalQueryString(RobotsPolicy policy, Robotstxt rtxt) throws IOException {
+        CrawlURI qs = new CrawlURI(UURIFactory.getInstance("http://example.com/ok?butno=something"));
+
+        assertFalse("ignoring query-string", policy.allows("Mozilla allowbot2 99.9", qs, rtxt));
+        
+    }
+}

modules/src/test/java/org/archive/modules/net/RobotstxtTest.java

@@ -56,7 +56,7 @@ public class RobotstxtTest extends TestCase {
         assertEquals(0, r.agentsToDirectives.size());
     }
     
-    Robotstxt sampleRobots1() throws IOException {
+    static Robotstxt sampleRobots1() throws IOException {
         BufferedReader reader = new BufferedReader(
             new StringReader(
                 "User-agent: *\n" +
@@ -71,6 +71,7 @@ public class RobotstxtTest extends TestCase {
                 "\n"+
                 "User-agent: allowbot2\n" +
                 "Disallow: /foo\n" +
+                "Disallow: /ok?butno\n" +
                 "Allow: /\n"+
                 "\n"+
                 "User-agent: delaybot\n" +
@@ -93,6 +94,7 @@ public class RobotstxtTest extends TestCase {
                 "  Disallow: \n" +
                 " User-agent: allowbot2\n" +
                 " Disallow: /foo\n" +
+                " Disallow: /ok?butno\n" +
                 " Allow: /\n"+
                 " User-agent: delaybot\n" +
                 "  Disallow: /\n" +
@@ -116,19 +118,27 @@ public class RobotstxtTest extends TestCase {
         // bot allowed with empty disallows
         assertTrue(r.getDirectivesFor("Mozilla allowbot1 99.9").allows("/path"));
         assertTrue(r.getDirectivesFor("Mozilla allowbot1 99.9").allows("/"));
+        
         // bot allowed with explicit allow
         assertTrue(r.getDirectivesFor("Mozilla allowbot2 99.9").allows("/path"));
         assertTrue(r.getDirectivesFor("Mozilla allowbot2 99.9").allows("/"));
+        
         // bot denied with specific disallow overriding general allow
         assertFalse(r.getDirectivesFor("Mozilla allowbot2 99.9").allows("/foo"));
+        // HER-1976: query-string disallow
+        assertFalse("ignoring query-string", r.getDirectivesFor("Mozilla allowbot2 99.9").allows("/ok?butno=something"));
+        
         // bot denied with blanket deny
         assertFalse(r.getDirectivesFor("Mozilla denybot 99.9").allows("/path"));
         assertFalse(r.getDirectivesFor("Mozilla denybot 99.9").allows("/"));
+        
         // unnamed bot with mixed catchall allow/deny
         assertTrue(r.getDirectivesFor("Mozilla anonbot 99.9").allows("/path"));
         assertFalse(r.getDirectivesFor("Mozilla anonbot 99.9").allows("/cgi-bin/foo.pl"));
+        
         // no crawl-delay
         assertEquals(r.getDirectivesFor("Mozilla denybot 99.9").getCrawlDelay(),-1f);
+        
         // with crawl-delay 
         assertEquals(r.getDirectivesFor("Mozilla delaybot 99.9").getCrawlDelay(),20f);
     }