mirror of
https://github.com/internetarchive/heritrix3.git
synced 2026-11-15 18:34:03 +00:00
apply to h3
Fix for [HER-1532] robots.txt with html <BR> markup causing NumberFormatException
* RobotsTxtTest.java
test based on ecnountered robots.txt with HTML markup but clear intent
* RobotsTxt.java
strip HTML markup from each line before parsing
strip expected numbers in Crawl-Delay to first run of digits-and-decimal-points
catch any NumberFormatException in Crawl-Delay number and ignore, continuing other parsing
This commit is contained in:
1 parent
58486c5f18
commit
9558d94b5d
2 files changed
+37
-2
No files matched your search
@@ -105,4 +105,32 @@ public class RobotstxtTest extends TestCase {
|
||||
// with crawl-delay
|
||||
assertEquals(r.getDirectivesFor("Mozilla delaybot 99.9").getCrawlDelay(),20f);
|
||||
}
|
||||
|
||||
Robotstxt htmlMarkupRobots() throws IOException {
|
||||
BufferedReader reader = new BufferedReader(
|
||||
new StringReader(
|
||||
"<!DOCTYPE HTML PUBLIC \"-//W3C//DTD HTML 3.2 Final//EN\"><HTML>\n"
|
||||
+"<HEAD>\n"
|
||||
+"<TITLE>/robots.txt</TITLE>\n"
|
||||
+"<HEAD>\n"
|
||||
+"<BODY>\n"
|
||||
+"User-agent: *<BR>\n"
|
||||
+"Disallow: /<BR>\n"
|
||||
+"Crawl-Delay: 30<BR>\n"
|
||||
+"\n"
|
||||
+"</BODY>\n"
|
||||
+"</HTML>\n"
|
||||
));
|
||||
return new Robotstxt(reader);
|
||||
}
|
||||
|
||||
/**
|
||||
* Test handling of a robots.txt with extraneous HTML markup
|
||||
* @throws IOException
|
||||
*/
|
||||
public void testHtmlMarkupRobots() throws IOException {
|
||||
Robotstxt r = htmlMarkupRobots();
|
||||
assertFalse(r.getDirectivesFor("anybot").allows("/index.html"));
|
||||
assertEquals(30f,r.getDirectivesFor("anybot").getCrawlDelay());
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user