Notifications - Escape only the diff variables before Jinja2 renders them into the template ( Stop breaking custom HTML for plaintext pages on HTML notifications) #4121 (#4123)

changedetectionio/notification/handler.py

@@ -382,6 +382,20 @@ def process_notification(n_object: NotificationContextData, datastore):
         n_object['llm_summary'] = _llm_change_summary or (n_object.get('_llm_result') or {}).get('summary', '')
         n_object['llm_intent'] = n_object.get('_llm_intent', '')
 
+    # Re #3529: diff content from text/plain pages may contain raw '<' chars that break HTML emails.
+    # Escape only the diff variables before Jinja2 renders them into the template, so the user's
+    # own HTML in the notification body (e.g. <a href="{{watch_url}}">) is never touched.
+    # Diff placemarkers (e.g. @removed_PLACEMARKER_OPEN) contain no HTML chars so they survive
+    # html_escape and are still replaced with <span> tags by apply_service_tweaks later.
+    watch_mime_type = n_object.get('watch_mime_type')
+    if (watch_mime_type and 'text/' in watch_mime_type.lower() and 'html' not in watch_mime_type.lower()
+            and 'html' in requested_output_format):
+        from markupsafe import escape as html_escape
+        _page_content_keys = {'raw_diff', 'current_snapshot', 'prev_snapshot', 'triggered_text'}
+        for key in [k for k in notification_parameters if k.startswith('diff') or k in _page_content_keys]:
+            if notification_parameters.get(key):
+                notification_parameters[key] = str(html_escape(str(notification_parameters[key])))
+
     with (apprise.LogCapture(level=apprise.logging.DEBUG) as logs):
         for url in n_object['notification_urls']:
 
@@ -399,13 +413,6 @@ def process_notification(n_object: NotificationContextData, datastore):
             logger.info(f">> Process Notification: AppRise start notifying '{url}'")
             url = jinja_render(template_str=url, **notification_parameters)
 
-            # If it's a plaintext document, and they want HTML type email/alerts, so it needs to be escaped
-            watch_mime_type = n_object.get('watch_mime_type')
-            if watch_mime_type and 'text/' in watch_mime_type.lower() and not 'html' in watch_mime_type.lower():
-                if 'html' in requested_output_format:
-                    from markupsafe import escape
-                    n_body = str(escape(n_body))
-
             if 'html' in requested_output_format:
                 # Since the n_body is always some kind of text from the 'diff' engine, attempt to preserve whitespaces that get sent to the HTML output
                 # But only where its more than 1 consecutive whitespace, otherwise "and this" becomes "and&nbsp;this" etc which is too much.

changedetectionio/tests/test_notification.py

@@ -635,3 +635,68 @@ def test_html_color_notifications(client, live_server, measure_memory_usage, dat
     _test_color_notifications(client, '{{diff}}',datastore_path=datastore_path)
     _test_color_notifications(client, '{{diff_full}}',datastore_path=datastore_path)
 
+
+def _test_custom_html_in_notification_body_not_escaped(client, datastore_path, content_type=None):
+    """
+    #4121 - Custom HTML in the notification body (e.g. <a href="{{watch_url}}">) must NOT be
+    HTML-escaped regardless of the watched page's content-type. Only raw diff content from
+    text/plain pages needs escaping (to prevent raw '<' chars breaking HTML email rendering).
+    """
+    set_original_response(datastore_path=datastore_path)
+
+    if os.path.isfile(os.path.join(datastore_path, "notification.txt")):
+        os.unlink(os.path.join(datastore_path, "notification.txt"))
+
+    test_notification_url = url_for('test_notification_endpoint', _external=True).replace('http://', 'post://')
+
+    kwargs = {'content_type': content_type} if content_type else {}
+    test_url = url_for('test_endpoint', _external=True, **kwargs)
+
+    res = client.post(
+        url_for("settings.settings_page"),
+        data={
+            "application-fetch_backend": "html_requests",
+            "application-minutes_between_check": 180,
+            "application-notification_body": '<a href="{{watch_url}}">Watch Link</a> had changes\n\n{{diff}}',
+            "application-notification_format": "htmlcolor",
+            "application-notification_urls": test_notification_url,
+            "application-notification_title": "Change detected",
+        },
+        follow_redirects=True
+    )
+    assert b'Settings updated' in res.data
+
+    res = client.post(
+        url_for("ui.ui_views.form_quick_watch_add"),
+        data={"url": test_url, "tags": ''},
+        follow_redirects=True
+    )
+    assert b"Watch added" in res.data
+
+    wait_for_all_checks(client)
+    set_modified_response(datastore_path=datastore_path)
+
+    res = client.get(url_for("ui.form_watch_checknow"), follow_redirects=True)
+    assert b'Queued 1 watch for rechecking.' in res.data
+
+    wait_for_all_checks(client)
+    wait_for_notification_endpoint_output(datastore_path=datastore_path)
+
+    with open(os.path.join(datastore_path, "notification.txt"), 'r') as f:
+        x = f.read()
+
+    assert '&lt;a href=' not in x, f"Custom HTML <a> tag was incorrectly escaped (content_type={content_type})"
+    assert '<a href=' in x, f"Custom HTML <a> tag not found unescaped (content_type={content_type})"
+    assert '<span' in x, f"Expected color <span> tags not found (content_type={content_type})"
+
+    client.get(url_for("ui.form_delete", uuid="all"), follow_redirects=True)
+
+
+def test_plaintext_watch_custom_html_in_notification_body_not_escaped(client, live_server, measure_memory_usage, datastore_path):
+    # text/plain: diff content may contain raw '<' chars — those must be escaped, but NOT the user's template HTML
+    _test_custom_html_in_notification_body_not_escaped(client, datastore_path, content_type="text/plain")
+    # text/html: HTML processor strips tags before diffing, no escaping needed, user's template HTML must be preserved
+    _test_custom_html_in_notification_body_not_escaped(client, datastore_path, content_type="text/html")
+    # no MIME type (None): same as HTML case, user's template HTML must be preserved
+    _test_custom_html_in_notification_body_not_escaped(client, datastore_path, content_type=None)
+