mirror of
https://github.com/internetarchive/heritrix3.git
synced 2026-11-15 18:34:03 +00:00
Improvement for [HER-1] Support 'Crawl-Delay' and 'Allow' robots.txt directives
* AbstractFrontier.java
change boolean 'respect-crawl-delay' to integer 'respect-crawl-delay-up-to-secs'
Fix for [HER-1516] in datamodel.Robotstxt, Integer.parseInt generates NumberFormatException on floating point Crawl-delay strings
* RobotsDirectives.java, RobotsTxt.java, RobotsExclusionPolicy.java
parse, store, return crawlDelay as float
* RobotsTxtTest.java
compare value as float
This commit is contained in:
1 parent
2c7c07ba64
commit
a530af6e5c
5 files changed
+20
-14
No files matched your search
@@ -101,8 +101,8 @@ public class RobotstxtTest extends TestCase {
|
||||
assertTrue(r.getDirectivesFor("Mozilla anonbot 99.9").allows("/path"));
|
||||
assertFalse(r.getDirectivesFor("Mozilla anonbot 99.9").allows("/cgi-bin/foo.pl"));
|
||||
// no crawl-delay
|
||||
assertEquals(r.getDirectivesFor("Mozilla denybot 99.9").getCrawlDelay(),-1);
|
||||
assertEquals(r.getDirectivesFor("Mozilla denybot 99.9").getCrawlDelay(),-1f);
|
||||
// with crawl-delay
|
||||
assertEquals(r.getDirectivesFor("Mozilla delaybot 99.9").getCrawlDelay(),20);
|
||||
assertEquals(r.getDirectivesFor("Mozilla delaybot 99.9").getCrawlDelay(),20f);
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user