fix: sanitize lone unicode surrogates before filtering JSON (#4293)

Co-authored-by: snowyukitty <270071858+snowyukitty@users.noreply.github.com>

changedetectionio/html_tools.py

@@ -406,10 +406,45 @@ def extract_element(find='title', html_content=''):
 
     return element_text
 
+# Any surrogate still present after json.loads() is a lone one - the decoder already folds
+# well-formed pairs (😀 -> a single emoji character) before we see them. Lone ones
+# come from escapes like \uD800 in the source document, which travel as plain ASCII and so
+# pass straight through the fetched-content sanitizer in processors/base.py.
+_LONE_SURROGATE_RE = re.compile(r'[\ud800-\udfff]')
+
+def _has_lone_surrogate(value):
+    if isinstance(value, str):
+        return bool(_LONE_SURROGATE_RE.search(value))
+    if isinstance(value, dict):
+        return any(_has_lone_surrogate(k) or _has_lone_surrogate(v) for k, v in value.items())
+    if isinstance(value, list):
+        return any(_has_lone_surrogate(v) for v in value)
+    return False
+
+def _sanitize_lone_surrogates(value):
+    if isinstance(value, str):
+        return _LONE_SURROGATE_RE.sub('\ufffd', value)
+    if isinstance(value, dict):
+        return {_sanitize_lone_surrogates(k): _sanitize_lone_surrogates(v) for k, v in value.items()}
+    if isinstance(value, list):
+        return [_sanitize_lone_surrogates(v) for v in value]
+    return value
+
 #
 def _parse_json(json_data, json_filter):
     from jsonpath_ng.ext import parse
 
+    # Replace lone surrogates with U+FFFD, matching what processors/base.py already does for
+    # fetched content. Two separate failures otherwise, neither of which is confined to the
+    # offending value: jq re-serializes the whole document for its own C parser, which rejects
+    # a lone high surrogate, so every jq:/jqraw: filter on the page errors even when it points
+    # at an unrelated field; and on the json: path the surrogate reaches the caller intact and
+    # raises UnicodeEncodeError later in checksums and history writes.
+    # See: https://github.com/dgtlmoon/changedetection.io/issues/4273
+    if _has_lone_surrogate(json_data):
+        logger.warning("Lone unicode surrogate(s) in JSON document, replacing with U+FFFD before filtering")
+        json_data = _sanitize_lone_surrogates(json_data)
+
     if json_filter.startswith("json:"):
         jsonpath_expression = parse(json_filter.replace('json:', ''))
         match = jsonpath_expression.find(json_data)

changedetectionio/tests/test_jsonpath_jq_selector.py

@@ -121,6 +121,44 @@ and it can also be repeated
             html_tools.extract_json_as_string('COMPLETE GIBBERISH, NO JSON!', "jqraw:.id")
 
 
+def test_lone_surrogate_escapes_do_not_break_filters():
+    """A \\uD800-style escape in the watched JSON must not take the whole document down.
+
+    jq re-serializes the document for its own C parser, which rejects a lone high surrogate,
+    so every jq:/jqraw: filter on the page used to error even when pointing at an unrelated
+    field. On the json: path the surrogate instead reached the caller intact and blew up later
+    in checksums and history writes. Valid pairs must keep working untouched.
+    See: https://github.com/dgtlmoon/changedetection.io/issues/4273
+    """
+    from .. import html_tools
+
+    BS = chr(92)  # build the escapes at runtime so no quoting layer eats them
+    lone_high = '{"title": "Example ' + BS + 'uD800", "other": "untouched"}'
+    lone_low = '{"title": "Example ' + BS + 'uDC00", "other": "untouched"}'
+    valid_pair = '{"title": "smile ' + BS + 'uD83D' + BS + 'uDE00", "other": "untouched"}'
+
+    filters = ["json:$.%s", "jq:.%s", "jqraw:.%s"] if jq_support else ["json:$.%s"]
+
+    for content in (lone_high, lone_low):
+        for f in filters:
+            # An unrelated field must still be reachable
+            text = html_tools.extract_json_as_string(content, f % "other")
+            assert "untouched" in text
+
+            # And the offending field itself resolves, with the surrogate replaced
+            text = html_tools.extract_json_as_string(content, f % "title")
+            assert "Example" in text
+            assert not any(0xD800 <= ord(c) <= 0xDFFF for c in text)
+            # Whatever comes back has to survive the downstream checksum/history write
+            text.encode('utf-8')
+
+    # A well-formed surrogate pair is a normal emoji and must be preserved as-is
+    for f in filters:
+        text = html_tools.extract_json_as_string(valid_pair, f % "title")
+        assert '\U0001F600' in text
+        text.encode('utf-8')
+
+
 def test_unittest_inline_extract_body():
     content = """
     <html>