Fix HER-1880 robots.txt less specific Allow overrides more specific Disallow

* RobotsDirectives.java
    - use plain ConcurrentSkipListSet<String> instead of PrefixSet, to
      maintain complete list of prefixes, so we can know the longest match
    - allows(String) - return true if longest matching Allow prefix is longer
      than or equal to longest matching Disallow prefix
* RobotstxtTest.java
    flip expected result of test of generic Allow against specific Disallow

modules/src/main/java/org/archive/modules/net/RobotsDirectives.java

@@ -18,9 +18,8 @@
  */
 package org.archive.modules.net;
 
-import java.io.Serializable;
-
-import org.archive.util.PrefixSet;
+import java.io.Serializable;
+import java.util.concurrent.ConcurrentSkipListSet;
 
 /**
  * Represents the directives that apply to a user-agent (or set of
@@ -29,17 +28,30 @@ import org.archive.util.PrefixSet;
 public class RobotsDirectives implements Serializable {
     private static final long serialVersionUID = 5386542759286155383L;
     
-    PrefixSet disallows = new PrefixSet();
-    PrefixSet allows = new PrefixSet();
+    ConcurrentSkipListSet<String> disallows = new ConcurrentSkipListSet<String>();
+    ConcurrentSkipListSet<String> allows = new ConcurrentSkipListSet<String>();
     float crawlDelay = -1; 
 
     public boolean allows(String path) {
-        if(disallows.containsPrefixOf(path)) {
-            return allows.containsPrefixOf(path);
-        }
-        return true;
+        return !(longestPrefixLength(disallows, path) > longestPrefixLength(allows, path));
     }
 
+    /**
+     * @param prefixSet
+     * @param str
+     * @return length of longest entry in {@code prefixSet} that prefixes {@code str}, or zero
+     *         if no entry prefixes {@code str}
+     */
+    protected int longestPrefixLength(ConcurrentSkipListSet<String> prefixSet,
+            String str) {
+        String possiblePrefix = prefixSet.floor(str);
+        if (possiblePrefix != null && str.startsWith(possiblePrefix)) {
+            return possiblePrefix.length();
+        } else {
+            return 0;
+        }
+    }
+
     public void addDisallow(String path) {
         if(path.length()==0) {
             // ignore empty-string disallows 

modules/src/test/java/org/archive/modules/net/RobotstxtTest.java

@@ -116,7 +116,8 @@ public class RobotstxtTest extends TestCase {
         // bot allowed with explicit allow
         assertTrue(r.getDirectivesFor("Mozilla allowbot2 99.9").allows("/path"));
         assertTrue(r.getDirectivesFor("Mozilla allowbot2 99.9").allows("/"));
-        assertTrue(r.getDirectivesFor("Mozilla allowbot2 99.9").allows("/foo"));
+        // bot denied with specific disallow overriding general allow
+        assertFalse(r.getDirectivesFor("Mozilla allowbot2 99.9").allows("/foo"));
         // bot denied with blanket deny
         assertFalse(r.getDirectivesFor("Mozilla denybot 99.9").allows("/path"));
         assertFalse(r.getDirectivesFor("Mozilla denybot 99.9").allows("/"));