lee/heritrix3
1
0
Fork 0
mirror of https://github.com/internetarchive/heritrix3.git synced 2026-08-25 07:56:35 +00:00
Code Issues Packages Projects Releases Wiki Activity

BotBlockDetector: Add DataDome

Browse Source
This commit is contained in:
Alex Osborne
2026-08-19 10:49:55 +09:00
parent cec6cfbb64
commit 9973a9f2f0
2 changed files with 18 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files
modules/src/main/java/org/archive/modules/processor/BotBlockDetector.java
+6
View File
@@ -82,6 +82,7 @@ public class BotBlockDetector extends Processor {
if (detectAkamai(curi)) return "akamai";
if (detectAnubis(curi)) return "anubis";
if (detectCloudflare(curi)) return "cloudflare";
if (detectDataDome(curi)) return "datadome";
if (detectIncapsula(curi)) return "incapsula";
return null;
}
@@ -114,6 +115,11 @@ public class BotBlockDetector extends Processor {
return false;
}
private static boolean detectDataDome(CrawlURI curi) {
return curi.getHttpResponseHeader("x-dd-b") != null &&
"protected".equalsIgnoreCase(curi.getHttpResponseHeader("x-datadome"));
}
private static boolean detectIncapsula(CrawlURI curi) {
return curi.getFetchStatus() == 404 &&
curi.getHttpResponseHeader("x-iinfo") != null &&
modules/src/test/java/org/archive/modules/processor/BotBlockDetectorTest.java
+12
View File
@@ -81,6 +81,18 @@ class BotBlockDetectorTest {
assertBlocked(cloudflareBlock(), "cloudflare");
}
@Test
void detectsDatadomeBlock() throws Exception {
CrawlURI curi = curi(403);
// some sites use both Cloudflare and DataDome
curi.putHttpResponseHeader("server", "cloudflare");
curi.putHttpResponseHeader("cf-ray", "1111111111111111-SJC");
curi.putHttpResponseHeader("x-datadome", "protected");
curi.putHttpResponseHeader("x-dd-b", "2");
curi.putHttpResponseHeader("x-datadome-cid", "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA==");
assertBlocked(curi, "datadome");
}
@Test
void detectsIncapsulaChallenge() throws Exception {
CrawlURI curi = curi(404);