From a530af6e5c41716f9f914e91ed9348b9ac643fe5 Mon Sep 17 00:00:00 2001 From: gojomo Date: Wed, 13 May 2009 01:14:58 +0000 Subject: [PATCH] Improvement for [HER-1] Support 'Crawl-Delay' and 'Allow' robots.txt directives * AbstractFrontier.java change boolean 'respect-crawl-delay' to integer 'respect-crawl-delay-up-to-secs' Fix for [HER-1516] in datamodel.Robotstxt, Integer.parseInt generates NumberFormatException on floating point Crawl-delay strings * RobotsDirectives.java, RobotsTxt.java, RobotsExclusionPolicy.java parse, store, return crawlDelay as float * RobotsTxtTest.java compare value as float --- .../crawler/frontier/AbstractFrontier.java | 20 ++++++++++++------- .../archive/modules/net/RobotsDirectives.java | 6 +++--- .../modules/net/RobotsExclusionPolicy.java | 2 +- .../org/archive/modules/net/Robotstxt.java | 2 +- .../archive/modules/net/RobotstxtTest.java | 4 ++-- 5 files changed, 20 insertions(+), 14 deletions(-) diff --git a/engine/src/main/java/org/archive/crawler/frontier/AbstractFrontier.java b/engine/src/main/java/org/archive/crawler/frontier/AbstractFrontier.java index 30f68033..bb143e3d 100644 --- a/engine/src/main/java/org/archive/crawler/frontier/AbstractFrontier.java +++ b/engine/src/main/java/org/archive/crawler/frontier/AbstractFrontier.java @@ -222,13 +222,13 @@ public abstract class AbstractFrontier * robots.txt */ { - setRespectCrawlDelay(true); + setRespectCrawlDelayUpToSeconds(300); } - public boolean getRespectCrawlDelay() { - return (Boolean) kp.get("respectCrawlDelay"); + public int getRespectCrawlDelayUpToSeconds() { + return (Integer) kp.get("respectCrawlDelayUpToSeconds"); } - public void setRespectCrawlDelay(boolean respect) { - kp.put("respectCrawlDelay",respect); + public void setRespectCrawlDelayUpToSeconds(int respect) { + kp.put("respectCrawlDelayUpToSeconds",respect); } /** never wait more than this long, regardless of multiple */ @@ -1156,7 +1156,9 @@ public abstract class AbstractFrontier durationToWait = maxDelay; } - if (getRespectCrawlDelay()) { + long respectThreshold = getRespectCrawlDelayUpToSeconds() * 1000; + if (durationToWait respectThreshold) + ? respectThreshold + : crawlDelay; if (crawlDelay > durationToWait) { // wait at least the directive crawl-delay durationToWait = crawlDelay; diff --git a/modules/src/main/java/org/archive/modules/net/RobotsDirectives.java b/modules/src/main/java/org/archive/modules/net/RobotsDirectives.java index 5ecd3db1..a43f3373 100644 --- a/modules/src/main/java/org/archive/modules/net/RobotsDirectives.java +++ b/modules/src/main/java/org/archive/modules/net/RobotsDirectives.java @@ -37,7 +37,7 @@ public class RobotsDirectives implements Serializable { PrefixSet disallows = new PrefixSet(); PrefixSet allows = new PrefixSet(); - int crawlDelay = -1; + float crawlDelay = -1; public boolean allows(String path) { if(disallows.containsPrefixOf(path)) { @@ -59,11 +59,11 @@ public class RobotsDirectives implements Serializable { allows.add(path); } - public void setCrawlDelay(int i) { + public void setCrawlDelay(float i) { crawlDelay=i; } - public int getCrawlDelay() { + public float getCrawlDelay() { return crawlDelay; } } \ No newline at end of file diff --git a/modules/src/main/java/org/archive/modules/net/RobotsExclusionPolicy.java b/modules/src/main/java/org/archive/modules/net/RobotsExclusionPolicy.java index f740f08c..189de727 100644 --- a/modules/src/main/java/org/archive/modules/net/RobotsExclusionPolicy.java +++ b/modules/src/main/java/org/archive/modules/net/RobotsExclusionPolicy.java @@ -255,7 +255,7 @@ public class RobotsExclusionPolicy implements Serializable { * @param userAgent * @return int Crawl-Delay value, or -1 if non available */ - public int getCrawlDelay(String userAgent) { + public float getCrawlDelay(String userAgent) { if (robotstxt==null) { return -1; } diff --git a/modules/src/main/java/org/archive/modules/net/Robotstxt.java b/modules/src/main/java/org/archive/modules/net/Robotstxt.java index 04127b6a..d1192654 100644 --- a/modules/src/main/java/org/archive/modules/net/Robotstxt.java +++ b/modules/src/main/java/org/archive/modules/net/Robotstxt.java @@ -113,7 +113,7 @@ public class Robotstxt implements Serializable { // grouping of User-Agent lines hasDirectivesYet = true; current.setCrawlDelay( - Integer.parseInt(read.substring(12).trim())); + Float.parseFloat(read.substring(12).trim())); continue; } if (read.matches("(?i)Allow:.*")) { diff --git a/modules/src/test/java/org/archive/modules/net/RobotstxtTest.java b/modules/src/test/java/org/archive/modules/net/RobotstxtTest.java index 275bfe85..f2949b19 100644 --- a/modules/src/test/java/org/archive/modules/net/RobotstxtTest.java +++ b/modules/src/test/java/org/archive/modules/net/RobotstxtTest.java @@ -101,8 +101,8 @@ public class RobotstxtTest extends TestCase { assertTrue(r.getDirectivesFor("Mozilla anonbot 99.9").allows("/path")); assertFalse(r.getDirectivesFor("Mozilla anonbot 99.9").allows("/cgi-bin/foo.pl")); // no crawl-delay - assertEquals(r.getDirectivesFor("Mozilla denybot 99.9").getCrawlDelay(),-1); + assertEquals(r.getDirectivesFor("Mozilla denybot 99.9").getCrawlDelay(),-1f); // with crawl-delay - assertEquals(r.getDirectivesFor("Mozilla delaybot 99.9").getCrawlDelay(),20); + assertEquals(r.getDirectivesFor("Mozilla delaybot 99.9").getCrawlDelay(),20f); } }