From cec6cfbb645538bfd8f989bf2b0c50749abd6063 Mon Sep 17 00:00:00 2001 From: Alex Osborne Date: Tue, 18 Aug 2026 16:02:07 +0900 Subject: [PATCH 1/3] Add BotBlockDetector Detects responses produced by common bot-blocking services and adds a "botblock:service" annotation. This is intended for reporting and troubleshooting but may also be useful as a signal to stop crawling or to skip writing WARC records. --- docs/bean-reference.rst | 18 ++ .../modules/processor/BotBlockDetector.java | 146 ++++++++++++++++ .../processor/BotBlockDetectorTest.java | 161 ++++++++++++++++++ 3 files changed, 325 insertions(+) create mode 100644 modules/src/main/java/org/archive/modules/processor/BotBlockDetector.java create mode 100644 modules/src/test/java/org/archive/modules/processor/BotBlockDetectorTest.java diff --git a/docs/bean-reference.rst b/docs/bean-reference.rst index 538ffa4b..666f71da 100644 --- a/docs/bean-reference.rst +++ b/docs/bean-reference.rst @@ -521,6 +521,24 @@ ScrollDownBehavior .. bean-doc:: org.archive.modules.behaviors.ScrollDownBehavior +Miscellaneous Processors +------------------------ + +BotBlockDetector +~~~~~~~~~~~~~~~~ + +.. bean-doc:: org.archive.modules.processor.BotBlockDetector + +HashCrawlMapper +~~~~~~~~~~~~~~~ + +.. bean-doc:: org.archive.crawler.processor.HashCrawlMapper + +LexicalCrawlMapper +~~~~~~~~~~~~~~~~~~ + +.. bean-doc:: org.archive.crawler.processor.LexicalCrawlMapper + Post-Processors --------------- diff --git a/modules/src/main/java/org/archive/modules/processor/BotBlockDetector.java b/modules/src/main/java/org/archive/modules/processor/BotBlockDetector.java new file mode 100644 index 00000000..acb856a4 --- /dev/null +++ b/modules/src/main/java/org/archive/modules/processor/BotBlockDetector.java @@ -0,0 +1,146 @@ +/* + * This file is part of the Heritrix web crawler (crawler.archive.org). + * + * Licensed to the Internet Archive (IA) by one or more individual + * contributors. + * + * The IA licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.archive.modules.processor; + +import java.util.Map; +import java.util.TreeMap; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentMap; +import java.util.concurrent.atomic.AtomicLong; + +import org.archive.modules.CrawlURI; +import org.archive.modules.Processor; +import org.archive.util.JSONUtils; +import org.json.JSONException; +import org.json.JSONObject; + +/** + * Detects responses produced by bot-blocking services and adds a "botblock:service" annotation. + *

+ * Normally added to the fetch chain before the extractors. + */ +public class BotBlockDetector extends Processor { + protected final ConcurrentMap counts = new ConcurrentHashMap<>(); + + public Map getCounts() { + return counts; + } + + @Override + protected boolean shouldProcess(CrawlURI curi) { + return curi.isHttpTransaction() && + curi.getFetchStatus() > 0; + } + + @Override + protected void innerProcess(CrawlURI curi) throws InterruptedException { + String detected = detect(curi); + if (detected != null && + curi.getAnnotations().add("botblock:" + detected)) { + counts.computeIfAbsent(detected, ignored -> new AtomicLong()).incrementAndGet(); + } + } + + @Override + protected JSONObject toCheckpointJson() throws JSONException { + JSONObject json = super.toCheckpointJson(); + json.put("counts", counts); + return json; + } + + @Override + protected void fromCheckpointJson(JSONObject json) throws JSONException { + super.fromCheckpointJson(json); + counts.clear(); + JSONObject counts = json.optJSONObject("counts"); + if (counts != null) JSONUtils.putAllAtomicLongs(this.counts, counts); + } + + @Override + public String report() { + return super.report() + " Blocked requests by service: " + + new TreeMap<>(counts) + "\n"; + } + + protected static String detect(CrawlURI curi) { + if (detectAkamai(curi)) return "akamai"; + if (detectAnubis(curi)) return "anubis"; + if (detectCloudflare(curi)) return "cloudflare"; + if (detectIncapsula(curi)) return "incapsula"; + return null; + } + + private static boolean detectAkamai(CrawlURI curi) { + return curi.getFetchStatus() == 403 && + "AkamaiGHost".equals(curi.getHttpResponseHeader("server")); + } + + private static boolean detectAnubis(CrawlURI curi) { + if (curi.getFetchStatus() == 307) { + String location = curi.getHttpResponseHeader("location"); + return location != null && location.contains("/.within.website/?redir="); + } else if (curi.getFetchStatus() == 200) { + String setCookie = curi.getHttpResponseHeader("set-cookie"); + return setCookie != null && setCookie.startsWith("techaro.lol-anubis-") && + bodyContainsHtml(curi, ""); + + assertBlocked(curi, "anubis"); + } + + @Test + void ordinaryResponseDoesNotMatch() throws Exception { + CrawlURI curi = curi(200); + + assertNotBlocked(curi); + } + + @Test + void detectsCloudflareChallenge() throws Exception { + CrawlURI curi = curi(403); + curi.putHttpResponseHeader("Server", "cloudflare"); + curi.putHttpResponseHeader("CF-Mitigated", "challenge"); + + assertBlocked(curi, "cloudflare"); + } + + @Test + void detectsCloudflareBlock() throws Exception { + assertBlocked(cloudflareBlock(), "cloudflare"); + } + + @Test + void detectsIncapsulaChallenge() throws Exception { + CrawlURI curi = curi(404); + curi.putHttpResponseHeader("x-iinfo", "foo"); + recordResponse(curi, + "Request unsuccessful. Incapsula incident ID: 123456"); + + assertBlocked(curi, "incapsula"); + } + + @Test + void unrelated403ResponseDoesNotMatch() throws Exception { + CrawlURI curi = curi(403); + curi.putHttpResponseHeader("Server", "example"); + + assertNotBlocked(curi); + } + + @Test + void checkpointRoundTripPreservesBlockedRequestCounts() throws Exception { + BotBlockDetector detector = new BotBlockDetector(); + detector.process(akamaiBlock()); + detector.process(cloudflareBlock()); + + BotBlockDetector restored = new BotBlockDetector(); + restored.fromCheckpointJson(detector.toCheckpointJson()); + + assertEquals(1, restored.getCounts().get("akamai").get()); + assertEquals(1, restored.getCounts().get("cloudflare").get()); + assertEquals(2, restored.getURICount()); + } + + private static void assertBlocked(CrawlURI curi, String service) + throws InterruptedException { + new BotBlockDetector().process(curi); + assertEquals(Set.of("botblock:" + service), curi.getAnnotations()); + } + + private static void assertNotBlocked(CrawlURI curi) + throws InterruptedException { + new BotBlockDetector().process(curi); + assertTrue(curi.getAnnotations().isEmpty()); + } + + private static CrawlURI akamaiBlock() throws Exception { + CrawlURI curi = curi(403); + curi.putHttpResponseHeader("Server", "AkamaiGHost"); + return curi; + } + + private CrawlURI cloudflareBlock() throws Exception { + CrawlURI curi = curi(403); + curi.putHttpResponseHeader("Server", "cloudflare"); + recordResponse(curi, "

Sorry, you have been blocked

"); + return curi; + } + + private static CrawlURI curi(int status) throws Exception { + CrawlURI curi = new CrawlURI(UURIFactory.getInstance("https://example.com/")); + curi.setFetchStatus(status); + curi.setFetchType(CrawlURI.FetchType.HTTP_GET); + curi.setContentType("text/html"); + return curi; + } + + private void recordResponse(CrawlURI curi, String body) throws Exception { + byte[] response = ("HTTP/1.1 " + curi.getFetchStatus() + " Test\r\n" + + "Content-Type: text/html\r\n" + + "Content-Length: " + body.getBytes(StandardCharsets.UTF_8).length + "\r\n" + + "\r\n" + + body).getBytes(StandardCharsets.UTF_8); + Recorder recorder = new Recorder(tempDir.toFile(), "bot-block-detector"); + curi.setRecorder(recorder); + recorder.inputWrap(new ByteArrayInputStream(response)); + recorder.getRecordedInput().readFully(); + recorder.close(); + } +} From 9973a9f2f0d8a7f414b5d907ce974bb5d7a18cd2 Mon Sep 17 00:00:00 2001 From: Alex Osborne Date: Wed, 19 Aug 2026 09:39:53 +0900 Subject: [PATCH 2/3] BotBlockDetector: Add DataDome --- .../archive/modules/processor/BotBlockDetector.java | 6 ++++++ .../modules/processor/BotBlockDetectorTest.java | 12 ++++++++++++ 2 files changed, 18 insertions(+) diff --git a/modules/src/main/java/org/archive/modules/processor/BotBlockDetector.java b/modules/src/main/java/org/archive/modules/processor/BotBlockDetector.java index acb856a4..b19c1e41 100644 --- a/modules/src/main/java/org/archive/modules/processor/BotBlockDetector.java +++ b/modules/src/main/java/org/archive/modules/processor/BotBlockDetector.java @@ -82,6 +82,7 @@ public class BotBlockDetector extends Processor { if (detectAkamai(curi)) return "akamai"; if (detectAnubis(curi)) return "anubis"; if (detectCloudflare(curi)) return "cloudflare"; + if (detectDataDome(curi)) return "datadome"; if (detectIncapsula(curi)) return "incapsula"; return null; } @@ -114,6 +115,11 @@ public class BotBlockDetector extends Processor { return false; } + private static boolean detectDataDome(CrawlURI curi) { + return curi.getHttpResponseHeader("x-dd-b") != null && + "protected".equalsIgnoreCase(curi.getHttpResponseHeader("x-datadome")); + } + private static boolean detectIncapsula(CrawlURI curi) { return curi.getFetchStatus() == 404 && curi.getHttpResponseHeader("x-iinfo") != null && diff --git a/modules/src/test/java/org/archive/modules/processor/BotBlockDetectorTest.java b/modules/src/test/java/org/archive/modules/processor/BotBlockDetectorTest.java index cc45ecfc..9982b156 100644 --- a/modules/src/test/java/org/archive/modules/processor/BotBlockDetectorTest.java +++ b/modules/src/test/java/org/archive/modules/processor/BotBlockDetectorTest.java @@ -81,6 +81,18 @@ class BotBlockDetectorTest { assertBlocked(cloudflareBlock(), "cloudflare"); } + @Test + void detectsDatadomeBlock() throws Exception { + CrawlURI curi = curi(403); + // some sites use both Cloudflare and DataDome + curi.putHttpResponseHeader("server", "cloudflare"); + curi.putHttpResponseHeader("cf-ray", "1111111111111111-SJC"); + curi.putHttpResponseHeader("x-datadome", "protected"); + curi.putHttpResponseHeader("x-dd-b", "2"); + curi.putHttpResponseHeader("x-datadome-cid", "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA=="); + assertBlocked(curi, "datadome"); + } + @Test void detectsIncapsulaChallenge() throws Exception { CrawlURI curi = curi(404); From cd4f910c57e6fc7571b6b067e827021058b0c468 Mon Sep 17 00:00:00 2001 From: Alex Osborne Date: Wed, 19 Aug 2026 09:54:26 +0900 Subject: [PATCH 3/3] BotBlockDetector: Look for cf-mitigated without server header Might catch unusual cases where something else is in front of Cloudflare. --- .../modules/processor/BotBlockDetector.java | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/modules/src/main/java/org/archive/modules/processor/BotBlockDetector.java b/modules/src/main/java/org/archive/modules/processor/BotBlockDetector.java index b19c1e41..d8ccc45f 100644 --- a/modules/src/main/java/org/archive/modules/processor/BotBlockDetector.java +++ b/modules/src/main/java/org/archive/modules/processor/BotBlockDetector.java @@ -105,14 +105,14 @@ public class BotBlockDetector extends Processor { } private static boolean detectCloudflare(CrawlURI curi) { - if ("cloudflare".equalsIgnoreCase(curi.getHttpResponseHeader("server"))) { - if ("challenge".equalsIgnoreCase(curi.getHttpResponseHeader("cf-mitigated"))) { - return true; - } - return curi.getFetchStatus() == 403 && - bodyContainsHtml(curi, "

Sorry, you have been blocked

"); + // challenge page + if ("challenge".equalsIgnoreCase(curi.getHttpResponseHeader("cf-mitigated"))) { + return true; } - return false; + // block page + return curi.getFetchStatus() == 403 && + "cloudflare".equalsIgnoreCase(curi.getHttpResponseHeader("server")) && + bodyContainsHtml(curi, "

Sorry, you have been blocked

"); } private static boolean detectDataDome(CrawlURI curi) {