From 28a70c4e2aad7599d644a4b4154b918951e2d87c Mon Sep 17 00:00:00 2001 From: dgtlmoon Date: Thu, 5 Mar 2026 10:59:06 +0100 Subject: [PATCH] Always replace/upgrade broken utf8 Re #3952 --- changedetectionio/processors/base.py | 8 ++++++++ changedetectionio/tests/test_encoding.py | 22 ++++++++++++++++++++++ 2 files changed, 30 insertions(+) diff --git a/changedetectionio/processors/base.py b/changedetectionio/processors/base.py index 9399ab765..021e1b90f 100644 --- a/changedetectionio/processors/base.py +++ b/changedetectionio/processors/base.py @@ -260,6 +260,14 @@ class difference_detection_processor(): # @todo .quit here could go on close object, so we can run JS if change-detected await self.fetcher.quit(watch=self.watch) + # Sanitize lone surrogates - these can appear when servers return malformed/mixed-encoding + # content that gets decoded into surrogate characters (e.g. \udcad). Without this, + # encode('utf-8') raises UnicodeEncodeError downstream in checksums, diffs, file writes, etc. + # Covers all fetchers (requests, playwright, puppeteer, selenium) in one place. + # See: https://github.com/dgtlmoon/changedetection.io/issues/3952 + if self.fetcher.content and isinstance(self.fetcher.content, str): + self.fetcher.content = self.fetcher.content.encode('utf-8', errors='replace').decode('utf-8') + # After init, call run_changedetection() which will do the actual change-detection def get_extra_watch_config(self, filename): diff --git a/changedetectionio/tests/test_encoding.py b/changedetectionio/tests/test_encoding.py index cbc1595b6..c298d4345 100644 --- a/changedetectionio/tests/test_encoding.py +++ b/changedetectionio/tests/test_encoding.py @@ -1,6 +1,7 @@ #!/usr/bin/env python3 # coding=utf-8 +import hashlib import time from flask import url_for from .util import live_server_setup, wait_for_all_checks, extract_UUID_from_client @@ -11,6 +12,27 @@ import os +def test_surrogate_characters_in_content_are_sanitized(): + """Lone surrogates can appear in requests' r.text when a server returns malformed/mixed-encoding + content. Without sanitization, encoding to UTF-8 raises UnicodeEncodeError. + See: https://github.com/dgtlmoon/changedetection.io/issues/3952 + """ + content_with_surrogate = 'Hello \udcad World' + + # Confirm the raw problem exists + with pytest.raises(UnicodeEncodeError): + content_with_surrogate.encode('utf-8') + + # Our fix: sanitize after fetcher.run() in processors/base.py call_browser() + sanitized = content_with_surrogate.encode('utf-8', errors='replace').decode('utf-8') + assert 'Hello' in sanitized + assert 'World' in sanitized + assert '\udcad' not in sanitized + + # Checksum computation (processors/base.py get_raw_document_checksum) must not crash + hashlib.md5(sanitized.encode('utf-8')).hexdigest() + + def set_html_response(datastore_path): test_return_data = """