No encoding in HTTP header -> Try to get it from the document -> use chardet last resort

changedetectionio/content_fetchers/requests.py

@@ -148,20 +148,19 @@ class fetcher(Fetcher):
                         # Default to UTF-8 for XML if no encoding found
                         r.encoding = 'utf-8'
                 else:
-                    # Try UTF-8 first - the vast majority of modern pages are UTF-8.
-                    # chardet can misdetect UTF-8 content as UTF-7 or other encodings,
-                    # which causes surrogates/mojibake and is also slow (scans entire body).
+                    # No charset in HTTP header - check for <meta charset=...> in the first 2kb.
+                    # This is more reliable than chardet which can misdetect encodings (e.g. UTF-8 as UTF-7).
+                    # Handles both HTML5 <meta charset="Shift-JIS"> and
+                    # HTML4 <meta http-equiv="Content-Type" content="text/html;charset=Shift-JIS">
                     # See: https://github.com/dgtlmoon/changedetection.io/issues/3952
-                    try:
-                        r.content.decode('utf-8') # try to decode, validation only
-                        original_encoding = r.encoding
-                        r.encoding = 'utf-8' # If it got this far, set it to utf-8
-                        if original_encoding != r.encoding:
-                            logger.info(f"URL: {url} content was re-encoded successfully from '{original_encoding}' to '{r.encoding}'")
-                    except UnicodeDecodeError:
-                        # Not valid UTF-8, fall back to chardetr
+                    meta_charset_match = re.search(rb'<meta[^>]+charset\s*=\s*["\']?\s*([^"\'\s;>]+)', r.content[:2000], re.IGNORECASE)
+                    if meta_charset_match:
+                        encoding = meta_charset_match.group(1).decode('ascii', errors='ignore')
+                        logger.info(f"URL: {url} No content-type encoding in HTTP headers - Using encoding '{encoding}' from HTML meta charset tag")
+                        r.encoding = encoding
+                    else:
                         encoding = chardet.detect(r.content)['encoding']
-                        logger.warning(f"URL: {url} Did not decode as utf-8, got UnicodeDecodeError, guessed new encoding as '{encoding}' via chardet")
+                        logger.warning(f"URL: {url} No charset in headers or meta tag, guessed encoding as '{encoding}' via chardet")
                         if encoding:
                             r.encoding = encoding
 

changedetectionio/processors/base.py

@@ -264,6 +264,7 @@ class difference_detection_processor():
         # content that gets decoded into surrogate characters (e.g. \udcad). Without this,
         # encode('utf-8') raises UnicodeEncodeError downstream in checksums, diffs, file writes, etc.
         # Covers all fetchers (requests, playwright, puppeteer, selenium) in one place.
+        # Also note: By this point we SHOULD know the original encoding so it can safely convert to utf-8 for the rest of the app.
         # See: https://github.com/dgtlmoon/changedetection.io/issues/3952
 
         if self.fetcher.content and isinstance(self.fetcher.content, str):

changedetectionio/tests/test_encoding.py

@@ -56,13 +56,14 @@ def test_utf8_content_without_charset_header(client, live_server, datastore_path
     assert '日本語'.encode('utf-8') in res.data
 
 
-def test_shiftjis_content_without_charset_header(client, live_server, datastore_path):
-    """Server returns Shift-JIS encoded content with no charset header.
-    UTF-8 decode will fail, so we fall back to chardet which should detect Shift-JIS.
+def test_shiftjis_with_meta_charset(client, live_server, datastore_path):
+    """Server returns Shift-JIS content with no charset in HTTP header, but the HTML
+    declares <meta charset="Shift-JIS">. We should use the meta tag, not chardet.
+    Real-world case: https://github.com/dgtlmoon/changedetection.io/issues/3952
     """
     from .util import write_test_file_and_sync
     japanese_text = '日本語のページ'
-    html = f'<html><body><p>{japanese_text}</p></body></html>'
+    html = f'<html><head><meta http-equiv="Content-Type" content="text/html;charset=Shift-JIS"></head><body><p>{japanese_text}</p></body></html>'
     write_test_file_and_sync(os.path.join(datastore_path, "endpoint-content.txt"), html.encode('shift_jis'), mode='wb')
 
     test_url = url_for('test_endpoint', content_type="text/html", _external=True)
@@ -71,7 +72,6 @@ def test_shiftjis_content_without_charset_header(client, live_server, datastore_
     wait_for_all_checks(client)
 
     res = client.get(url_for("ui.ui_preview.preview_page", uuid="first"), follow_redirects=True)
-    # chardet should detect Shift-JIS and decode correctly to Unicode
     assert japanese_text.encode('utf-8') in res.data