diff --git a/modules/src/main/java/org/archive/modules/net/RobotsPolicy.java b/modules/src/main/java/org/archive/modules/net/RobotsPolicy.java index 5703c880..0ee4faaa 100644 --- a/modules/src/main/java/org/archive/modules/net/RobotsPolicy.java +++ b/modules/src/main/java/org/archive/modules/net/RobotsPolicy.java @@ -36,7 +36,8 @@ abstract public class RobotsPolicy { STANDARD_POLICIES.put("obey", ObeyRobotsPolicy.INSTANCE); // the obey policy has also historically been called 'classic' STANDARD_POLICIES.put("classic", ObeyRobotsPolicy.INSTANCE); - STANDARD_POLICIES.put("ignore", IgnoreRobotsPolicy.INSTANCE); + STANDARD_POLICIES.put("ignore", IgnoreRobotsPolicy.INSTANCE); + STANDARD_POLICIES.put("robotsTxtOnly", RobotsTxtOnlyPolicy.INSTANCE); } public abstract boolean allows(String userAgent, CrawlURI curi, Robotstxt robotstxt); diff --git a/modules/src/main/java/org/archive/modules/net/RobotsTxtOnlyPolicy.java b/modules/src/main/java/org/archive/modules/net/RobotsTxtOnlyPolicy.java new file mode 100644 index 00000000..65545f5a --- /dev/null +++ b/modules/src/main/java/org/archive/modules/net/RobotsTxtOnlyPolicy.java @@ -0,0 +1,31 @@ +/* + * This file is part of the Heritrix web crawler (crawler.archive.org). + * + * Licensed to the Internet Archive (IA) by one or more individual + * contributors. + * + * The IA licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.archive.modules.net; + +/** + * Policy to obey robots.txt but ignore meta nofollow. + */ +public class RobotsTxtOnlyPolicy extends ObeyRobotsPolicy { + public static RobotsPolicy INSTANCE = new RobotsTxtOnlyPolicy(); + + @Override + public boolean obeyMetaRobotsNofollow() { + return false; + } +}