From 1209b2d42f46589cd1c2dca2d224499b4e491387 Mon Sep 17 00:00:00 2001
From: Alex Osborne <aosborne@nla.gov.au>
Date: Tue, 5 Jul 2022 15:35:48 +0900
Subject: [PATCH] Add robotsTxtOnly robots policy

This policy obeys robots.txt but ignores meta nofollow. We're
increasingly frequently encountering sites which fail to be archived
properly due to usage of this meta tag but where the robots.txt file
contains reasonable rules. In many cases it seems to be unintentional,
and we've even had questions from webmasters asking why archiving was
failing when they were explicitly allowing us in robots.txt.
---
 .../org/archive/modules/net/RobotsPolicy.java |  3 +-
 .../modules/net/RobotsTxtOnlyPolicy.java      | 31 +++++++++++++++++++
 2 files changed, 33 insertions(+), 1 deletion(-)
 create mode 100644 modules/src/main/java/org/archive/modules/net/RobotsTxtOnlyPolicy.java

diff --git a/modules/src/main/java/org/archive/modules/net/RobotsPolicy.java b/modules/src/main/java/org/archive/modules/net/RobotsPolicy.java
index 5703c880..0ee4faaa 100644
--- a/modules/src/main/java/org/archive/modules/net/RobotsPolicy.java
+++ b/modules/src/main/java/org/archive/modules/net/RobotsPolicy.java
@@ -36,7 +36,8 @@ abstract public class RobotsPolicy {
         STANDARD_POLICIES.put("obey", ObeyRobotsPolicy.INSTANCE);
         // the obey policy has also historically been called 'classic'
         STANDARD_POLICIES.put("classic", ObeyRobotsPolicy.INSTANCE);
-        STANDARD_POLICIES.put("ignore", IgnoreRobotsPolicy.INSTANCE); 
+        STANDARD_POLICIES.put("ignore", IgnoreRobotsPolicy.INSTANCE);
+        STANDARD_POLICIES.put("robotsTxtOnly", RobotsTxtOnlyPolicy.INSTANCE);
     }
 
     public abstract boolean allows(String userAgent, CrawlURI curi, Robotstxt robotstxt);
diff --git a/modules/src/main/java/org/archive/modules/net/RobotsTxtOnlyPolicy.java b/modules/src/main/java/org/archive/modules/net/RobotsTxtOnlyPolicy.java
new file mode 100644
index 00000000..65545f5a
--- /dev/null
+++ b/modules/src/main/java/org/archive/modules/net/RobotsTxtOnlyPolicy.java
@@ -0,0 +1,31 @@
+/*
+ *  This file is part of the Heritrix web crawler (crawler.archive.org).
+ *
+ *  Licensed to the Internet Archive (IA) by one or more individual
+ *  contributors.
+ *
+ *  The IA licenses this file to You under the Apache License, Version 2.0
+ *  (the "License"); you may not use this file except in compliance with
+ *  the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ */
+package org.archive.modules.net;
+
+/**
+ * Policy to obey robots.txt but ignore meta nofollow.
+ */
+public class RobotsTxtOnlyPolicy extends ObeyRobotsPolicy {
+    public static RobotsPolicy INSTANCE = new RobotsTxtOnlyPolicy();
+
+    @Override
+    public boolean obeyMetaRobotsNofollow() {
+        return false;
+    }
+}