diff --git a/modules/src/main/java/org/archive/modules/net/RobotsDirectives.java b/modules/src/main/java/org/archive/modules/net/RobotsDirectives.java
index aefaccb1..3c265234 100644
--- a/modules/src/main/java/org/archive/modules/net/RobotsDirectives.java
+++ b/modules/src/main/java/org/archive/modules/net/RobotsDirectives.java
@@ -18,9 +18,8 @@
  */
 package org.archive.modules.net;
 
-import java.io.Serializable;
-
-import org.archive.util.PrefixSet;
+import java.io.Serializable;
+import java.util.concurrent.ConcurrentSkipListSet;
 
 /**
  * Represents the directives that apply to a user-agent (or set of
@@ -29,17 +28,30 @@ import org.archive.util.PrefixSet;
 public class RobotsDirectives implements Serializable {
     private static final long serialVersionUID = 5386542759286155383L;
     
-    PrefixSet disallows = new PrefixSet();
-    PrefixSet allows = new PrefixSet();
+    ConcurrentSkipListSet<String> disallows = new ConcurrentSkipListSet<String>();
+    ConcurrentSkipListSet<String> allows = new ConcurrentSkipListSet<String>();
     float crawlDelay = -1; 
 
     public boolean allows(String path) {
-        if(disallows.containsPrefixOf(path)) {
-            return allows.containsPrefixOf(path);
-        }
-        return true;
+        return !(longestPrefixLength(disallows, path) > longestPrefixLength(allows, path));
     }
 
+    /**
+     * @param prefixSet
+     * @param str
+     * @return length of longest entry in {@code prefixSet} that prefixes {@code str}, or zero
+     *         if no entry prefixes {@code str}
+     */
+    protected int longestPrefixLength(ConcurrentSkipListSet<String> prefixSet,
+            String str) {
+        String possiblePrefix = prefixSet.floor(str);
+        if (possiblePrefix != null && str.startsWith(possiblePrefix)) {
+            return possiblePrefix.length();
+        } else {
+            return 0;
+        }
+    }
+
     public void addDisallow(String path) {
         if(path.length()==0) {
             // ignore empty-string disallows 
diff --git a/modules/src/test/java/org/archive/modules/net/RobotstxtTest.java b/modules/src/test/java/org/archive/modules/net/RobotstxtTest.java
index fa3c5b7f..4e4ed719 100644
--- a/modules/src/test/java/org/archive/modules/net/RobotstxtTest.java
+++ b/modules/src/test/java/org/archive/modules/net/RobotstxtTest.java
@@ -116,7 +116,8 @@ public class RobotstxtTest extends TestCase {
         // bot allowed with explicit allow
         assertTrue(r.getDirectivesFor("Mozilla allowbot2 99.9").allows("/path"));
         assertTrue(r.getDirectivesFor("Mozilla allowbot2 99.9").allows("/"));
-        assertTrue(r.getDirectivesFor("Mozilla allowbot2 99.9").allows("/foo"));
+        // bot denied with specific disallow overriding general allow
+        assertFalse(r.getDirectivesFor("Mozilla allowbot2 99.9").allows("/foo"));
         // bot denied with blanket deny
         assertFalse(r.getDirectivesFor("Mozilla denybot 99.9").allows("/path"));
         assertFalse(r.getDirectivesFor("Mozilla denybot 99.9").allows("/"));