diff --git a/modules/src/main/java/org/archive/modules/processor/BotBlockDetector.java b/modules/src/main/java/org/archive/modules/processor/BotBlockDetector.java index acb856a4..b19c1e41 100644 --- a/modules/src/main/java/org/archive/modules/processor/BotBlockDetector.java +++ b/modules/src/main/java/org/archive/modules/processor/BotBlockDetector.java @@ -82,6 +82,7 @@ public class BotBlockDetector extends Processor { if (detectAkamai(curi)) return "akamai"; if (detectAnubis(curi)) return "anubis"; if (detectCloudflare(curi)) return "cloudflare"; + if (detectDataDome(curi)) return "datadome"; if (detectIncapsula(curi)) return "incapsula"; return null; } @@ -114,6 +115,11 @@ public class BotBlockDetector extends Processor { return false; } + private static boolean detectDataDome(CrawlURI curi) { + return curi.getHttpResponseHeader("x-dd-b") != null && + "protected".equalsIgnoreCase(curi.getHttpResponseHeader("x-datadome")); + } + private static boolean detectIncapsula(CrawlURI curi) { return curi.getFetchStatus() == 404 && curi.getHttpResponseHeader("x-iinfo") != null && diff --git a/modules/src/test/java/org/archive/modules/processor/BotBlockDetectorTest.java b/modules/src/test/java/org/archive/modules/processor/BotBlockDetectorTest.java index cc45ecfc..9982b156 100644 --- a/modules/src/test/java/org/archive/modules/processor/BotBlockDetectorTest.java +++ b/modules/src/test/java/org/archive/modules/processor/BotBlockDetectorTest.java @@ -81,6 +81,18 @@ class BotBlockDetectorTest { assertBlocked(cloudflareBlock(), "cloudflare"); } + @Test + void detectsDatadomeBlock() throws Exception { + CrawlURI curi = curi(403); + // some sites use both Cloudflare and DataDome + curi.putHttpResponseHeader("server", "cloudflare"); + curi.putHttpResponseHeader("cf-ray", "1111111111111111-SJC"); + curi.putHttpResponseHeader("x-datadome", "protected"); + curi.putHttpResponseHeader("x-dd-b", "2"); + curi.putHttpResponseHeader("x-datadome-cid", "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA=="); + assertBlocked(curi, "datadome"); + } + @Test void detectsIncapsulaChallenge() throws Exception { CrawlURI curi = curi(404);