From 1453119516f56c1bc6ca2d535adb20a51b0ccba1 Mon Sep 17 00:00:00 2001 From: dgtlmoon Date: Thu, 5 Mar 2026 11:31:03 +0100 Subject: [PATCH] More non standard encoding fixes --- .../content_fetchers/requests.py | 17 ++++++-- changedetectionio/processors/base.py | 5 +-- changedetectionio/tests/test_encoding.py | 42 +++++++++++++++++++ 3 files changed, 57 insertions(+), 7 deletions(-) diff --git a/changedetectionio/content_fetchers/requests.py b/changedetectionio/content_fetchers/requests.py index 87a6cf9ae..f9d5de083 100644 --- a/changedetectionio/content_fetchers/requests.py +++ b/changedetectionio/content_fetchers/requests.py @@ -148,10 +148,19 @@ class fetcher(Fetcher): # Default to UTF-8 for XML if no encoding found r.encoding = 'utf-8' else: - # For other content types, use chardet - encoding = chardet.detect(r.content)['encoding'] - if encoding: - r.encoding = encoding + # Try UTF-8 first - the vast majority of modern pages are UTF-8. + # chardet can misdetect UTF-8 content as UTF-7 or other encodings, + # which causes surrogates/mojibake and is also slow (scans entire body). + # See: https://github.com/dgtlmoon/changedetection.io/issues/3952 + try: + r.content.decode('utf-8') # try to decode, validation only + r.encoding = 'utf-8' # If it got this far, set it to utf-8 + except UnicodeDecodeError: + # Not valid UTF-8, fall back to chardet + encoding = chardet.detect(r.content)['encoding'] + logger.warning(f"URL: {url} Did not decode as UTF-8, got UnicodeDecodeError, guessed new encoding as '{encoding}' via chardet") + if encoding: + r.encoding = encoding self.headers = r.headers diff --git a/changedetectionio/processors/base.py b/changedetectionio/processors/base.py index 0258c0969..a84bd5bfa 100644 --- a/changedetectionio/processors/base.py +++ b/changedetectionio/processors/base.py @@ -266,9 +266,8 @@ class difference_detection_processor(): # Covers all fetchers (requests, playwright, puppeteer, selenium) in one place. # See: https://github.com/dgtlmoon/changedetection.io/issues/3952 - # DISABLED FOR NOW, TRY TO SEE IF THE TEST BREAKS ON GITHUB - # if self.fetcher.content and isinstance(self.fetcher.content, str): - # self.fetcher.content = self.fetcher.content.encode('utf-8', errors='replace').decode('utf-8') + if self.fetcher.content and isinstance(self.fetcher.content, str): + self.fetcher.content = self.fetcher.content.encode('utf-8', errors='replace').decode('utf-8') # After init, call run_changedetection() which will do the actual change-detection diff --git a/changedetectionio/tests/test_encoding.py b/changedetectionio/tests/test_encoding.py index c298d4345..d1008b6c0 100644 --- a/changedetectionio/tests/test_encoding.py +++ b/changedetectionio/tests/test_encoding.py @@ -33,6 +33,48 @@ def test_surrogate_characters_in_content_are_sanitized(): hashlib.md5(sanitized.encode('utf-8')).hexdigest() +def test_utf8_content_without_charset_header(client, live_server, datastore_path): + """Server returns UTF-8 content but no charset in Content-Type header. + chardet can misdetect such pages as UTF-7 (Python 3.14 then produces surrogates). + Our fix tries UTF-8 first before falling back to chardet. + See: https://github.com/dgtlmoon/changedetection.io/issues/3952 + """ + from .util import write_test_file_and_sync + # UTF-8 encoded content with non-ASCII chars - no charset will be in the header + html = '

Español

Français

日本語

' + write_test_file_and_sync(os.path.join(datastore_path, "endpoint-content.txt"), html.encode('utf-8'), mode='wb') + + test_url = url_for('test_endpoint', content_type="text/html", _external=True) + client.application.config.get('DATASTORE').add_watch(url=test_url) + client.get(url_for("ui.form_watch_checknow"), follow_redirects=True) + wait_for_all_checks(client) + + res = client.get(url_for("ui.ui_preview.preview_page", uuid="first"), follow_redirects=True) + # Should decode correctly as UTF-8, not produce mojibake (Español) or replacement chars + assert 'Español'.encode('utf-8') in res.data + assert 'Français'.encode('utf-8') in res.data + assert '日本語'.encode('utf-8') in res.data + + +def test_shiftjis_content_without_charset_header(client, live_server, datastore_path): + """Server returns Shift-JIS encoded content with no charset header. + UTF-8 decode will fail, so we fall back to chardet which should detect Shift-JIS. + """ + from .util import write_test_file_and_sync + japanese_text = '日本語のページ' + html = f'

{japanese_text}

' + write_test_file_and_sync(os.path.join(datastore_path, "endpoint-content.txt"), html.encode('shift_jis'), mode='wb') + + test_url = url_for('test_endpoint', content_type="text/html", _external=True) + client.application.config.get('DATASTORE').add_watch(url=test_url) + client.get(url_for("ui.form_watch_checknow"), follow_redirects=True) + wait_for_all_checks(client) + + res = client.get(url_for("ui.ui_preview.preview_page", uuid="first"), follow_redirects=True) + # chardet should detect Shift-JIS and decode correctly to Unicode + assert japanese_text.encode('utf-8') in res.data + + def set_html_response(datastore_path): test_return_data = """