diff --git a/changedetectionio/html_tools.py b/changedetectionio/html_tools.py
index cdf4fa2a..8e3c392d 100644
--- a/changedetectionio/html_tools.py
+++ b/changedetectionio/html_tools.py
@@ -406,10 +406,45 @@ def extract_element(find='title', html_content=''):
return element_text
+# Any surrogate still present after json.loads() is a lone one - the decoder already folds
+# well-formed pairs (😀 -> a single emoji character) before we see them. Lone ones
+# come from escapes like \uD800 in the source document, which travel as plain ASCII and so
+# pass straight through the fetched-content sanitizer in processors/base.py.
+_LONE_SURROGATE_RE = re.compile(r'[\ud800-\udfff]')
+
+def _has_lone_surrogate(value):
+ if isinstance(value, str):
+ return bool(_LONE_SURROGATE_RE.search(value))
+ if isinstance(value, dict):
+ return any(_has_lone_surrogate(k) or _has_lone_surrogate(v) for k, v in value.items())
+ if isinstance(value, list):
+ return any(_has_lone_surrogate(v) for v in value)
+ return False
+
+def _sanitize_lone_surrogates(value):
+ if isinstance(value, str):
+ return _LONE_SURROGATE_RE.sub('\ufffd', value)
+ if isinstance(value, dict):
+ return {_sanitize_lone_surrogates(k): _sanitize_lone_surrogates(v) for k, v in value.items()}
+ if isinstance(value, list):
+ return [_sanitize_lone_surrogates(v) for v in value]
+ return value
+
#
def _parse_json(json_data, json_filter):
from jsonpath_ng.ext import parse
+ # Replace lone surrogates with U+FFFD, matching what processors/base.py already does for
+ # fetched content. Two separate failures otherwise, neither of which is confined to the
+ # offending value: jq re-serializes the whole document for its own C parser, which rejects
+ # a lone high surrogate, so every jq:/jqraw: filter on the page errors even when it points
+ # at an unrelated field; and on the json: path the surrogate reaches the caller intact and
+ # raises UnicodeEncodeError later in checksums and history writes.
+ # See: https://github.com/dgtlmoon/changedetection.io/issues/4273
+ if _has_lone_surrogate(json_data):
+ logger.warning("Lone unicode surrogate(s) in JSON document, replacing with U+FFFD before filtering")
+ json_data = _sanitize_lone_surrogates(json_data)
+
if json_filter.startswith("json:"):
jsonpath_expression = parse(json_filter.replace('json:', ''))
match = jsonpath_expression.find(json_data)
diff --git a/changedetectionio/tests/test_jsonpath_jq_selector.py b/changedetectionio/tests/test_jsonpath_jq_selector.py
index 5e60d4d2..d51400ee 100644
--- a/changedetectionio/tests/test_jsonpath_jq_selector.py
+++ b/changedetectionio/tests/test_jsonpath_jq_selector.py
@@ -121,6 +121,44 @@ and it can also be repeated
html_tools.extract_json_as_string('COMPLETE GIBBERISH, NO JSON!', "jqraw:.id")
+def test_lone_surrogate_escapes_do_not_break_filters():
+ """A \\uD800-style escape in the watched JSON must not take the whole document down.
+
+ jq re-serializes the document for its own C parser, which rejects a lone high surrogate,
+ so every jq:/jqraw: filter on the page used to error even when pointing at an unrelated
+ field. On the json: path the surrogate instead reached the caller intact and blew up later
+ in checksums and history writes. Valid pairs must keep working untouched.
+ See: https://github.com/dgtlmoon/changedetection.io/issues/4273
+ """
+ from .. import html_tools
+
+ BS = chr(92) # build the escapes at runtime so no quoting layer eats them
+ lone_high = '{"title": "Example ' + BS + 'uD800", "other": "untouched"}'
+ lone_low = '{"title": "Example ' + BS + 'uDC00", "other": "untouched"}'
+ valid_pair = '{"title": "smile ' + BS + 'uD83D' + BS + 'uDE00", "other": "untouched"}'
+
+ filters = ["json:$.%s", "jq:.%s", "jqraw:.%s"] if jq_support else ["json:$.%s"]
+
+ for content in (lone_high, lone_low):
+ for f in filters:
+ # An unrelated field must still be reachable
+ text = html_tools.extract_json_as_string(content, f % "other")
+ assert "untouched" in text
+
+ # And the offending field itself resolves, with the surrogate replaced
+ text = html_tools.extract_json_as_string(content, f % "title")
+ assert "Example" in text
+ assert not any(0xD800 <= ord(c) <= 0xDFFF for c in text)
+ # Whatever comes back has to survive the downstream checksum/history write
+ text.encode('utf-8')
+
+ # A well-formed surrogate pair is a normal emoji and must be preserved as-is
+ for f in filters:
+ text = html_tools.extract_json_as_string(valid_pair, f % "title")
+ assert '\U0001F600' in text
+ text.encode('utf-8')
+
+
def test_unittest_inline_extract_body():
content = """