apply to h3

Fix for [HER-1532] robots.txt with html <BR> markup causing NumberFormatException
* RobotsTxtTest.java
    test based on ecnountered robots.txt with HTML markup but clear intent
* RobotsTxt.java
    strip HTML markup from each line before parsing
    strip expected numbers in Crawl-Delay to first run of digits-and-decimal-points
    catch any NumberFormatException in Crawl-Delay number and ignore, continuing other parsing
This commit is contained in:
gojomo committed 2009-05-13 23:34:29 +00:00
1 parent 58486c5f18
commit 9558d94b5d
2 files changed
+37 -2

No files matched your search

@@ -105,4 +105,32 @@ public class RobotstxtTest extends TestCase {
// with crawl-delay
assertEquals(r.getDirectivesFor("Mozilla delaybot 99.9").getCrawlDelay(),20f);
}
Robotstxt htmlMarkupRobots() throws IOException {
BufferedReader reader = new BufferedReader(
new StringReader(
"<!DOCTYPE HTML PUBLIC \"-//W3C//DTD HTML 3.2 Final//EN\"><HTML>\n"
+"<HEAD>\n"
+"<TITLE>/robots.txt</TITLE>\n"
+"<HEAD>\n"
+"<BODY>\n"
+"User-agent: *<BR>\n"
+"Disallow: /<BR>\n"
+"Crawl-Delay: 30<BR>\n"
+"\n"
+"</BODY>\n"
+"</HTML>\n"
));
return new Robotstxt(reader);
}
/**
* Test handling of a robots.txt with extraneous HTML markup
* @throws IOException
*/
public void testHtmlMarkupRobots() throws IOException {
Robotstxt r = htmlMarkupRobots();
assertFalse(r.getDirectivesFor("anybot").allows("/index.html"));
assertEquals(30f,r.getDirectivesFor("anybot").getCrawlDelay());
}
}