mirror of
https://github.com/internetarchive/heritrix3.git
synced 2026-08-25 07:56:35 +00:00
Merge pull request #760 from internetarchive/ato/botblock-detector
Add BotBlockDetector
This commit is contained in:
Alex Osborne
@@ -521,6 +521,24 @@ ScrollDownBehavior
|
||||
|
||||
.. bean-doc:: org.archive.modules.behaviors.ScrollDownBehavior
|
||||
|
||||
Miscellaneous Processors
|
||||
------------------------
|
||||
|
||||
BotBlockDetector
|
||||
~~~~~~~~~~~~~~~~
|
||||
|
||||
.. bean-doc:: org.archive.modules.processor.BotBlockDetector
|
||||
|
||||
HashCrawlMapper
|
||||
~~~~~~~~~~~~~~~
|
||||
|
||||
.. bean-doc:: org.archive.crawler.processor.HashCrawlMapper
|
||||
|
||||
LexicalCrawlMapper
|
||||
~~~~~~~~~~~~~~~~~~
|
||||
|
||||
.. bean-doc:: org.archive.crawler.processor.LexicalCrawlMapper
|
||||
|
||||
Post-Processors
|
||||
---------------
|
||||
|
||||
|
||||
@@ -0,0 +1,152 @@
|
||||
/*
|
||||
* This file is part of the Heritrix web crawler (crawler.archive.org).
|
||||
*
|
||||
* Licensed to the Internet Archive (IA) by one or more individual
|
||||
* contributors.
|
||||
*
|
||||
* The IA licenses this file to You under the Apache License, Version 2.0
|
||||
* (the "License"); you may not use this file except in compliance with
|
||||
* the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
package org.archive.modules.processor;
|
||||
|
||||
import java.util.Map;
|
||||
import java.util.TreeMap;
|
||||
import java.util.concurrent.ConcurrentHashMap;
|
||||
import java.util.concurrent.ConcurrentMap;
|
||||
import java.util.concurrent.atomic.AtomicLong;
|
||||
|
||||
import org.archive.modules.CrawlURI;
|
||||
import org.archive.modules.Processor;
|
||||
import org.archive.util.JSONUtils;
|
||||
import org.json.JSONException;
|
||||
import org.json.JSONObject;
|
||||
|
||||
/**
|
||||
* Detects responses produced by bot-blocking services and adds a "botblock:service" annotation.
|
||||
* <p>
|
||||
* Normally added to the fetch chain before the extractors.
|
||||
*/
|
||||
public class BotBlockDetector extends Processor {
|
||||
protected final ConcurrentMap<String, AtomicLong> counts = new ConcurrentHashMap<>();
|
||||
|
||||
public Map<String, AtomicLong> getCounts() {
|
||||
return counts;
|
||||
}
|
||||
|
||||
@Override
|
||||
protected boolean shouldProcess(CrawlURI curi) {
|
||||
return curi.isHttpTransaction() &&
|
||||
curi.getFetchStatus() > 0;
|
||||
}
|
||||
|
||||
@Override
|
||||
protected void innerProcess(CrawlURI curi) throws InterruptedException {
|
||||
String detected = detect(curi);
|
||||
if (detected != null &&
|
||||
curi.getAnnotations().add("botblock:" + detected)) {
|
||||
counts.computeIfAbsent(detected, ignored -> new AtomicLong()).incrementAndGet();
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
protected JSONObject toCheckpointJson() throws JSONException {
|
||||
JSONObject json = super.toCheckpointJson();
|
||||
json.put("counts", counts);
|
||||
return json;
|
||||
}
|
||||
|
||||
@Override
|
||||
protected void fromCheckpointJson(JSONObject json) throws JSONException {
|
||||
super.fromCheckpointJson(json);
|
||||
counts.clear();
|
||||
JSONObject counts = json.optJSONObject("counts");
|
||||
if (counts != null) JSONUtils.putAllAtomicLongs(this.counts, counts);
|
||||
}
|
||||
|
||||
@Override
|
||||
public String report() {
|
||||
return super.report() + " Blocked requests by service: " +
|
||||
new TreeMap<>(counts) + "\n";
|
||||
}
|
||||
|
||||
protected static String detect(CrawlURI curi) {
|
||||
if (detectAkamai(curi)) return "akamai";
|
||||
if (detectAnubis(curi)) return "anubis";
|
||||
if (detectCloudflare(curi)) return "cloudflare";
|
||||
if (detectDataDome(curi)) return "datadome";
|
||||
if (detectIncapsula(curi)) return "incapsula";
|
||||
return null;
|
||||
}
|
||||
|
||||
private static boolean detectAkamai(CrawlURI curi) {
|
||||
return curi.getFetchStatus() == 403 &&
|
||||
"AkamaiGHost".equals(curi.getHttpResponseHeader("server"));
|
||||
}
|
||||
|
||||
private static boolean detectAnubis(CrawlURI curi) {
|
||||
if (curi.getFetchStatus() == 307) {
|
||||
String location = curi.getHttpResponseHeader("location");
|
||||
return location != null && location.contains("/.within.website/?redir=");
|
||||
} else if (curi.getFetchStatus() == 200) {
|
||||
String setCookie = curi.getHttpResponseHeader("set-cookie");
|
||||
return setCookie != null && setCookie.startsWith("techaro.lol-anubis-") &&
|
||||
bodyContainsHtml(curi, "<script id=\"anubis_challenge\"");
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
private static boolean detectCloudflare(CrawlURI curi) {
|
||||
// challenge page
|
||||
if ("challenge".equalsIgnoreCase(curi.getHttpResponseHeader("cf-mitigated"))) {
|
||||
return true;
|
||||
}
|
||||
// block page
|
||||
return curi.getFetchStatus() == 403 &&
|
||||
"cloudflare".equalsIgnoreCase(curi.getHttpResponseHeader("server")) &&
|
||||
bodyContainsHtml(curi, "<h1 data-translate=\"block_headline\">Sorry, you have been blocked</h1>");
|
||||
}
|
||||
|
||||
private static boolean detectDataDome(CrawlURI curi) {
|
||||
return curi.getHttpResponseHeader("x-dd-b") != null &&
|
||||
"protected".equalsIgnoreCase(curi.getHttpResponseHeader("x-datadome"));
|
||||
}
|
||||
|
||||
private static boolean detectIncapsula(CrawlURI curi) {
|
||||
return curi.getFetchStatus() == 404 &&
|
||||
curi.getHttpResponseHeader("x-iinfo") != null &&
|
||||
bodyContainsHtml(curi, "Incapsula incident ID:");
|
||||
}
|
||||
|
||||
private static boolean bodyContainsHtml(CrawlURI curi, String string) {
|
||||
return hasHtmlContentType(curi) &&
|
||||
curi.getRecorder() != null &&
|
||||
curi.getRecorder().getContentReplayPrefixString(8000).contains(string);
|
||||
}
|
||||
|
||||
private static boolean hasHtmlContentType(CrawlURI curi) {
|
||||
String type = curi.getContentType();
|
||||
return type == null ||
|
||||
matchesMediaType(type, "text/html") ||
|
||||
matchesMediaType(type, "application/xhtml+xml");
|
||||
}
|
||||
|
||||
private static boolean matchesMediaType(String value, String expected) {
|
||||
if (!value.regionMatches(true, 0, expected, 0, expected.length())) {
|
||||
return false;
|
||||
}
|
||||
int i = expected.length();
|
||||
while (i < value.length() && Character.isWhitespace(value.charAt(i))) {
|
||||
i++;
|
||||
}
|
||||
return i == value.length() || value.charAt(i) == ';';
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,173 @@
|
||||
/*
|
||||
* This file is part of the Heritrix web crawler (crawler.archive.org).
|
||||
*
|
||||
* Licensed to the Internet Archive (IA) by one or more individual
|
||||
* contributors.
|
||||
*
|
||||
* The IA licenses this file to You under the Apache License, Version 2.0
|
||||
* (the "License"); you may not use this file except in compliance with
|
||||
* the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
package org.archive.modules.processor;
|
||||
|
||||
import java.io.ByteArrayInputStream;
|
||||
import java.nio.charset.StandardCharsets;
|
||||
import java.nio.file.Path;
|
||||
import java.util.Set;
|
||||
|
||||
import org.archive.modules.CrawlURI;
|
||||
import org.archive.net.UURIFactory;
|
||||
import org.archive.util.Recorder;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import org.junit.jupiter.api.io.TempDir;
|
||||
|
||||
import static org.junit.jupiter.api.Assertions.*;
|
||||
|
||||
class BotBlockDetectorTest {
|
||||
|
||||
@TempDir
|
||||
Path tempDir;
|
||||
|
||||
@Test
|
||||
void detectsAkamaiBlock() throws Exception {
|
||||
assertBlocked(akamaiBlock(), "akamai");
|
||||
}
|
||||
|
||||
@Test
|
||||
void detectsAnubisRedirectChallenge() throws Exception {
|
||||
CrawlURI curi = curi(307);
|
||||
curi.putHttpResponseHeader("Location",
|
||||
"/.within.website/?redir=https%3A%2F%2Fexample.com%2F");
|
||||
|
||||
assertBlocked(curi, "anubis");
|
||||
}
|
||||
|
||||
@Test
|
||||
void detectsAnubisCookieChallenge() throws Exception {
|
||||
CrawlURI curi = curi(200);
|
||||
curi.putHttpResponseHeader("Set-Cookie",
|
||||
"techaro.lol-anubis-auth=token; Path=/");
|
||||
recordResponse(curi, "<script id=\"anubis_challenge\"></script>");
|
||||
|
||||
assertBlocked(curi, "anubis");
|
||||
}
|
||||
|
||||
@Test
|
||||
void ordinaryResponseDoesNotMatch() throws Exception {
|
||||
CrawlURI curi = curi(200);
|
||||
|
||||
assertNotBlocked(curi);
|
||||
}
|
||||
|
||||
@Test
|
||||
void detectsCloudflareChallenge() throws Exception {
|
||||
CrawlURI curi = curi(403);
|
||||
curi.putHttpResponseHeader("Server", "cloudflare");
|
||||
curi.putHttpResponseHeader("CF-Mitigated", "challenge");
|
||||
|
||||
assertBlocked(curi, "cloudflare");
|
||||
}
|
||||
|
||||
@Test
|
||||
void detectsCloudflareBlock() throws Exception {
|
||||
assertBlocked(cloudflareBlock(), "cloudflare");
|
||||
}
|
||||
|
||||
@Test
|
||||
void detectsDatadomeBlock() throws Exception {
|
||||
CrawlURI curi = curi(403);
|
||||
// some sites use both Cloudflare and DataDome
|
||||
curi.putHttpResponseHeader("server", "cloudflare");
|
||||
curi.putHttpResponseHeader("cf-ray", "1111111111111111-SJC");
|
||||
curi.putHttpResponseHeader("x-datadome", "protected");
|
||||
curi.putHttpResponseHeader("x-dd-b", "2");
|
||||
curi.putHttpResponseHeader("x-datadome-cid", "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA==");
|
||||
assertBlocked(curi, "datadome");
|
||||
}
|
||||
|
||||
@Test
|
||||
void detectsIncapsulaChallenge() throws Exception {
|
||||
CrawlURI curi = curi(404);
|
||||
curi.putHttpResponseHeader("x-iinfo", "foo");
|
||||
recordResponse(curi,
|
||||
"Request unsuccessful. Incapsula incident ID: 123456");
|
||||
|
||||
assertBlocked(curi, "incapsula");
|
||||
}
|
||||
|
||||
@Test
|
||||
void unrelated403ResponseDoesNotMatch() throws Exception {
|
||||
CrawlURI curi = curi(403);
|
||||
curi.putHttpResponseHeader("Server", "example");
|
||||
|
||||
assertNotBlocked(curi);
|
||||
}
|
||||
|
||||
@Test
|
||||
void checkpointRoundTripPreservesBlockedRequestCounts() throws Exception {
|
||||
BotBlockDetector detector = new BotBlockDetector();
|
||||
detector.process(akamaiBlock());
|
||||
detector.process(cloudflareBlock());
|
||||
|
||||
BotBlockDetector restored = new BotBlockDetector();
|
||||
restored.fromCheckpointJson(detector.toCheckpointJson());
|
||||
|
||||
assertEquals(1, restored.getCounts().get("akamai").get());
|
||||
assertEquals(1, restored.getCounts().get("cloudflare").get());
|
||||
assertEquals(2, restored.getURICount());
|
||||
}
|
||||
|
||||
private static void assertBlocked(CrawlURI curi, String service)
|
||||
throws InterruptedException {
|
||||
new BotBlockDetector().process(curi);
|
||||
assertEquals(Set.of("botblock:" + service), curi.getAnnotations());
|
||||
}
|
||||
|
||||
private static void assertNotBlocked(CrawlURI curi)
|
||||
throws InterruptedException {
|
||||
new BotBlockDetector().process(curi);
|
||||
assertTrue(curi.getAnnotations().isEmpty());
|
||||
}
|
||||
|
||||
private static CrawlURI akamaiBlock() throws Exception {
|
||||
CrawlURI curi = curi(403);
|
||||
curi.putHttpResponseHeader("Server", "AkamaiGHost");
|
||||
return curi;
|
||||
}
|
||||
|
||||
private CrawlURI cloudflareBlock() throws Exception {
|
||||
CrawlURI curi = curi(403);
|
||||
curi.putHttpResponseHeader("Server", "cloudflare");
|
||||
recordResponse(curi, "<h1 data-translate=\"block_headline\">Sorry, you have been blocked</h1>");
|
||||
return curi;
|
||||
}
|
||||
|
||||
private static CrawlURI curi(int status) throws Exception {
|
||||
CrawlURI curi = new CrawlURI(UURIFactory.getInstance("https://example.com/"));
|
||||
curi.setFetchStatus(status);
|
||||
curi.setFetchType(CrawlURI.FetchType.HTTP_GET);
|
||||
curi.setContentType("text/html");
|
||||
return curi;
|
||||
}
|
||||
|
||||
private void recordResponse(CrawlURI curi, String body) throws Exception {
|
||||
byte[] response = ("HTTP/1.1 " + curi.getFetchStatus() + " Test\r\n"
|
||||
+ "Content-Type: text/html\r\n"
|
||||
+ "Content-Length: " + body.getBytes(StandardCharsets.UTF_8).length + "\r\n"
|
||||
+ "\r\n"
|
||||
+ body).getBytes(StandardCharsets.UTF_8);
|
||||
Recorder recorder = new Recorder(tempDir.toFile(), "bot-block-detector");
|
||||
curi.setRecorder(recorder);
|
||||
recorder.inputWrap(new ByteArrayInputStream(response));
|
||||
recorder.getRecordedInput().readFully();
|
||||
recorder.close();
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user