diff --git a/modules/src/main/java/org/archive/modules/net/Robotstxt.java b/modules/src/main/java/org/archive/modules/net/Robotstxt.java index d1192654..07078b17 100644 --- a/modules/src/main/java/org/archive/modules/net/Robotstxt.java +++ b/modules/src/main/java/org/archive/modules/net/Robotstxt.java @@ -68,6 +68,8 @@ public class Robotstxt implements Serializable { reader.close(); reader = null; } else { + // remove any html markup + read = read.replaceAll("<[^>]+>",""); int commentIndex = read.indexOf("#"); if (commentIndex > -1) { // Strip trailing comment @@ -112,8 +114,13 @@ public class Robotstxt implements Serializable { // yet understand it, as sufficient to end a // grouping of User-Agent lines hasDirectivesYet = true; - current.setCrawlDelay( - Float.parseFloat(read.substring(12).trim())); + String val = read.substring(12).trim(); + val = val.split("[^\\d\\.]+")[0]; + try { + current.setCrawlDelay(Float.parseFloat(val)); + } catch (NumberFormatException nfe) { + // ignore + } continue; } if (read.matches("(?i)Allow:.*")) { diff --git a/modules/src/test/java/org/archive/modules/net/RobotstxtTest.java b/modules/src/test/java/org/archive/modules/net/RobotstxtTest.java index f2949b19..49d5033c 100644 --- a/modules/src/test/java/org/archive/modules/net/RobotstxtTest.java +++ b/modules/src/test/java/org/archive/modules/net/RobotstxtTest.java @@ -105,4 +105,32 @@ public class RobotstxtTest extends TestCase { // with crawl-delay assertEquals(r.getDirectivesFor("Mozilla delaybot 99.9").getCrawlDelay(),20f); } + + Robotstxt htmlMarkupRobots() throws IOException { + BufferedReader reader = new BufferedReader( + new StringReader( + "\n" + +"
\n" + +"