From 9558d94b5d6d67cb4d0357d0663a62b9d7ba4b46 Mon Sep 17 00:00:00 2001 From: gojomo Date: Wed, 13 May 2009 23:34:29 +0000 Subject: [PATCH] apply to h3 Fix for [HER-1532] robots.txt with html
markup causing NumberFormatException * RobotsTxtTest.java test based on ecnountered robots.txt with HTML markup but clear intent * RobotsTxt.java strip HTML markup from each line before parsing strip expected numbers in Crawl-Delay to first run of digits-and-decimal-points catch any NumberFormatException in Crawl-Delay number and ignore, continuing other parsing --- .../org/archive/modules/net/Robotstxt.java | 11 ++++++-- .../archive/modules/net/RobotstxtTest.java | 28 +++++++++++++++++++ 2 files changed, 37 insertions(+), 2 deletions(-) diff --git a/modules/src/main/java/org/archive/modules/net/Robotstxt.java b/modules/src/main/java/org/archive/modules/net/Robotstxt.java index d1192654..07078b17 100644 --- a/modules/src/main/java/org/archive/modules/net/Robotstxt.java +++ b/modules/src/main/java/org/archive/modules/net/Robotstxt.java @@ -68,6 +68,8 @@ public class Robotstxt implements Serializable { reader.close(); reader = null; } else { + // remove any html markup + read = read.replaceAll("<[^>]+>",""); int commentIndex = read.indexOf("#"); if (commentIndex > -1) { // Strip trailing comment @@ -112,8 +114,13 @@ public class Robotstxt implements Serializable { // yet understand it, as sufficient to end a // grouping of User-Agent lines hasDirectivesYet = true; - current.setCrawlDelay( - Float.parseFloat(read.substring(12).trim())); + String val = read.substring(12).trim(); + val = val.split("[^\\d\\.]+")[0]; + try { + current.setCrawlDelay(Float.parseFloat(val)); + } catch (NumberFormatException nfe) { + // ignore + } continue; } if (read.matches("(?i)Allow:.*")) { diff --git a/modules/src/test/java/org/archive/modules/net/RobotstxtTest.java b/modules/src/test/java/org/archive/modules/net/RobotstxtTest.java index f2949b19..49d5033c 100644 --- a/modules/src/test/java/org/archive/modules/net/RobotstxtTest.java +++ b/modules/src/test/java/org/archive/modules/net/RobotstxtTest.java @@ -105,4 +105,32 @@ public class RobotstxtTest extends TestCase { // with crawl-delay assertEquals(r.getDirectivesFor("Mozilla delaybot 99.9").getCrawlDelay(),20f); } + + Robotstxt htmlMarkupRobots() throws IOException { + BufferedReader reader = new BufferedReader( + new StringReader( + "\n" + +"\n" + +"/robots.txt\n" + +"\n" + +"\n" + +"User-agent: *
\n" + +"Disallow: /
\n" + +"Crawl-Delay: 30
\n" + +"\n" + +"\n" + +"\n" + )); + return new Robotstxt(reader); + } + + /** + * Test handling of a robots.txt with extraneous HTML markup + * @throws IOException + */ + public void testHtmlMarkupRobots() throws IOException { + Robotstxt r = htmlMarkupRobots(); + assertFalse(r.getDirectivesFor("anybot").allows("/index.html")); + assertEquals(30f,r.getDirectivesFor("anybot").getCrawlDelay()); + } }