Add robotsTxtOnly robots policy

This policy obeys robots.txt but ignores meta nofollow. We're
increasingly frequently encountering sites which fail to be archived
properly due to usage of this meta tag but where the robots.txt file
contains reasonable rules. In many cases it seems to be unintentional,
and we've even had questions from webmasters asking why archiving was
failing when they were explicitly allowing us in robots.txt.

modules/src/main/java/org/archive/modules/net/RobotsPolicy.java

@@ -36,7 +36,8 @@ abstract public class RobotsPolicy {
         STANDARD_POLICIES.put("obey", ObeyRobotsPolicy.INSTANCE);
         // the obey policy has also historically been called 'classic'
         STANDARD_POLICIES.put("classic", ObeyRobotsPolicy.INSTANCE);
-        STANDARD_POLICIES.put("ignore", IgnoreRobotsPolicy.INSTANCE); 
+        STANDARD_POLICIES.put("ignore", IgnoreRobotsPolicy.INSTANCE);
+        STANDARD_POLICIES.put("robotsTxtOnly", RobotsTxtOnlyPolicy.INSTANCE);
     }
 
     public abstract boolean allows(String userAgent, CrawlURI curi, Robotstxt robotstxt);

modules/src/main/java/org/archive/modules/net/RobotsTxtOnlyPolicy.java

@@ -0,0 +1,31 @@
+/*
+ *  This file is part of the Heritrix web crawler (crawler.archive.org).
+ *
+ *  Licensed to the Internet Archive (IA) by one or more individual
+ *  contributors.
+ *
+ *  The IA licenses this file to You under the Apache License, Version 2.0
+ *  (the "License"); you may not use this file except in compliance with
+ *  the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ */
+package org.archive.modules.net;
+
+/**
+ * Policy to obey robots.txt but ignore meta nofollow.
+ */
+public class RobotsTxtOnlyPolicy extends ObeyRobotsPolicy {
+    public static RobotsPolicy INSTANCE = new RobotsTxtOnlyPolicy();
+
+    @Override
+    public boolean obeyMetaRobotsNofollow() {
+        return false;
+    }
+}