diff --git a/changedetectionio/__init__.py b/changedetectionio/__init__.py index 303a13ef..6b20940f 100644 --- a/changedetectionio/__init__.py +++ b/changedetectionio/__init__.py @@ -2,7 +2,7 @@ # Read more https://github.com/dgtlmoon/changedetection.io/wiki -__version__ = '0.50.35' +__version__ = '0.50.36' from changedetectionio.strtobool import strtobool from json.decoder import JSONDecodeError diff --git a/changedetectionio/notification/handler.py b/changedetectionio/notification/handler.py index 63f98b8f..1e80db8b 100644 --- a/changedetectionio/notification/handler.py +++ b/changedetectionio/notification/handler.py @@ -5,13 +5,15 @@ from apprise import NotifyFormat from loguru import logger from urllib.parse import urlparse from .apprise_plugin.assets import apprise_asset, APPRISE_AVATAR_URL -from .apprise_plugin.custom_handlers import SUPPORTED_HTTP_METHODS from .email_helpers import as_monospaced_html_email from ..diff import HTML_REMOVED_STYLE, REMOVED_PLACEMARKER_OPEN, REMOVED_PLACEMARKER_CLOSED, ADDED_PLACEMARKER_OPEN, HTML_ADDED_STYLE, \ ADDED_PLACEMARKER_CLOSED, CHANGED_INTO_PLACEMARKER_OPEN, CHANGED_INTO_PLACEMARKER_CLOSED, CHANGED_PLACEMARKER_OPEN, \ CHANGED_PLACEMARKER_CLOSED, HTML_CHANGED_STYLE, HTML_CHANGED_INTO_STYLE -from ..notification_service import NotificationContextData, CUSTOM_LINEBREAK_PLACEHOLDER +import re +from ..notification_service import NotificationContextData + +newline_re = re.compile(r'\r\n|\r|\n') def markup_text_links_to_html(body): @@ -127,6 +129,62 @@ def apply_standard_markdown_to_body(n_body): return n_body +def replace_placemarkers_in_text(text, url, requested_output_format): + """ + Replace diff placemarkers in text based on the URL service type and requested output format. + Used for both notification title and body to ensure consistent placeholder replacement. + + :param text: The text to process + :param url: The notification URL (to detect service type) + :param requested_output_format: The output format (html, htmlcolor, markdown, text, etc.) + :return: Processed text with placemarkers replaced + """ + if not text: + return text + + if url.startswith('tgram://'): + # Telegram only supports a limited subset of HTML + # Use strikethrough for removed content, bold for added content + text = text.replace(REMOVED_PLACEMARKER_OPEN, '') + text = text.replace(REMOVED_PLACEMARKER_CLOSED, '') + text = text.replace(ADDED_PLACEMARKER_OPEN, '') + text = text.replace(ADDED_PLACEMARKER_CLOSED, '') + # Handle changed/replaced lines (old → new) + text = text.replace(CHANGED_PLACEMARKER_OPEN, '') + text = text.replace(CHANGED_PLACEMARKER_CLOSED, '') + text = text.replace(CHANGED_INTO_PLACEMARKER_OPEN, '') + text = text.replace(CHANGED_INTO_PLACEMARKER_CLOSED, '') + elif (url.startswith('discord://') or url.startswith('https://discordapp.com/api/webhooks') + or url.startswith('https://discord.com/api')) and requested_output_format == 'html': + # Discord doesn't support HTML, use Discord markdown + text = apply_discord_markdown_to_body(n_body=text) + elif requested_output_format == 'htmlcolor': + # https://github.com/dgtlmoon/changedetection.io/issues/821#issuecomment-1241837050 + text = text.replace(REMOVED_PLACEMARKER_OPEN, f'') + text = text.replace(REMOVED_PLACEMARKER_CLOSED, f'') + text = text.replace(ADDED_PLACEMARKER_OPEN, f'') + text = text.replace(ADDED_PLACEMARKER_CLOSED, f'') + # Handle changed/replaced lines (old → new) + text = text.replace(CHANGED_PLACEMARKER_OPEN, f'') + text = text.replace(CHANGED_PLACEMARKER_CLOSED, f'') + text = text.replace(CHANGED_INTO_PLACEMARKER_OPEN, f'') + text = text.replace(CHANGED_INTO_PLACEMARKER_CLOSED, f'') + elif requested_output_format == 'markdown': + # Markdown to HTML - Apprise will convert this to HTML + text = apply_standard_markdown_to_body(n_body=text) + else: + # plaintext, html, and default - use simple text markers + text = text.replace(REMOVED_PLACEMARKER_OPEN, '(removed) ') + text = text.replace(REMOVED_PLACEMARKER_CLOSED, '') + text = text.replace(ADDED_PLACEMARKER_OPEN, '(added) ') + text = text.replace(ADDED_PLACEMARKER_CLOSED, '') + text = text.replace(CHANGED_PLACEMARKER_OPEN, f'(changed) ') + text = text.replace(CHANGED_PLACEMARKER_CLOSED, f'') + text = text.replace(CHANGED_INTO_PLACEMARKER_OPEN, f'(into) ') + text = text.replace(CHANGED_INTO_PLACEMARKER_CLOSED, f'') + + return text + def apply_service_tweaks(url, n_body, n_title, requested_output_format): # Re 323 - Limit discord length to their 2000 char limit total or it wont send. @@ -138,6 +196,12 @@ def apply_service_tweaks(url, n_body, n_title, requested_output_format): if not n_body or not n_body.strip(): return url, n_body, n_title + # Normalize URL scheme to lowercase to prevent case-sensitivity issues + # e.g., "Discord://webhook" -> "discord://webhook", "TGRAM://bot123" -> "tgram://bot123" + scheme_separator_pos = url.find('://') + if scheme_separator_pos > 0: + url = url[:scheme_separator_pos].lower() + url[scheme_separator_pos:] + # So if no avatar_url is specified, add one so it can be correctly calculated into the total payload parsed = urlparse(url) k = '?' if not parsed.query else '&' @@ -149,24 +213,22 @@ def apply_service_tweaks(url, n_body, n_title, requested_output_format): and not url.startswith('put'): url += k + f"avatar_url={APPRISE_AVATAR_URL}" + # Replace placemarkers in title first (this was the missing piece causing the bug) + # Titles are ALWAYS plain text across all notification services (Discord embeds, Slack attachments, + # email Subject headers, etc.), so we always use 'text' format for title placemarker replacement + # Looking over apprise library it seems that all plugins only expect plain-text. + n_title = replace_placemarkers_in_text(n_title, url, 'text') + if url.startswith('tgram://'): # Telegram only supports a limit subset of HTML, remove the '
' we place in. # re https://github.com/dgtlmoon/changedetection.io/issues/555 # @todo re-use an existing library we have already imported to strip all non-allowed tags n_body = n_body.replace('
', '\n') n_body = n_body.replace('
', '\n') - n_body = n_body.replace(CUSTOM_LINEBREAK_PLACEHOLDER, '\n') + n_body = newline_re.sub('\n', n_body) - # Use strikethrough for removed content, bold for added content - n_body = n_body.replace(REMOVED_PLACEMARKER_OPEN, '') - n_body = n_body.replace(REMOVED_PLACEMARKER_CLOSED, '') - n_body = n_body.replace(ADDED_PLACEMARKER_OPEN, '') - n_body = n_body.replace(ADDED_PLACEMARKER_CLOSED, '') - # Handle changed/replaced lines (old → new) - n_body = n_body.replace(CHANGED_PLACEMARKER_OPEN, '') - n_body = n_body.replace(CHANGED_PLACEMARKER_CLOSED, '') - n_body = n_body.replace(CHANGED_INTO_PLACEMARKER_OPEN, '') - n_body = n_body.replace(CHANGED_INTO_PLACEMARKER_CLOSED, '') + # Replace placemarkers for body + n_body = replace_placemarkers_in_text(n_body, url, requested_output_format) # real limit is 4096, but minus some for extra metadata payload_max_size = 3600 @@ -180,7 +242,7 @@ def apply_service_tweaks(url, n_body, n_title, requested_output_format): # Discord doesn't support HTML, replace
with newlines n_body = n_body.strip().replace('
', '\n') n_body = n_body.replace('
', '\n') - n_body = n_body.replace(CUSTOM_LINEBREAK_PLACEHOLDER, '\n') + n_body = newline_re.sub('\n', n_body) # Don't replace placeholders or truncate here - let the custom Discord plugin handle it # The plugin will use embeds (6000 char limit across all embeds) if placeholders are present, @@ -190,7 +252,7 @@ def apply_service_tweaks(url, n_body, n_title, requested_output_format): if requested_output_format == 'html': # No diff placeholders, use Discord markdown for any other formatting # Use Discord markdown: strikethrough for removed, bold for added - n_body = apply_discord_markdown_to_body(n_body=n_body) + n_body = replace_placemarkers_in_text(n_body, url, requested_output_format) # Apply 2000 char limit for plain content payload_max_size = 1700 @@ -201,40 +263,17 @@ def apply_service_tweaks(url, n_body, n_title, requested_output_format): # Is not discord/tgram and they want htmlcolor elif requested_output_format == 'htmlcolor': - # https://github.com/dgtlmoon/changedetection.io/issues/821#issuecomment-1241837050 - n_body = n_body.replace(REMOVED_PLACEMARKER_OPEN, f'') - n_body = n_body.replace(REMOVED_PLACEMARKER_CLOSED, f'') - n_body = n_body.replace(ADDED_PLACEMARKER_OPEN, f'') - n_body = n_body.replace(ADDED_PLACEMARKER_CLOSED, f'') - # Handle changed/replaced lines (old → new) - n_body = n_body.replace(CHANGED_PLACEMARKER_OPEN, f'') - n_body = n_body.replace(CHANGED_PLACEMARKER_CLOSED, f'') - n_body = n_body.replace(CHANGED_INTO_PLACEMARKER_OPEN, f'') - n_body = n_body.replace(CHANGED_INTO_PLACEMARKER_CLOSED, f'') - n_body = n_body.replace('\n', f'{CUSTOM_LINEBREAK_PLACEHOLDER}\n') + n_body = replace_placemarkers_in_text(n_body, url, requested_output_format) + n_body = newline_re.sub('
\n', n_body) elif requested_output_format == 'html': - n_body = n_body.replace(REMOVED_PLACEMARKER_OPEN, '(removed) ') - n_body = n_body.replace(REMOVED_PLACEMARKER_CLOSED, '') - n_body = n_body.replace(ADDED_PLACEMARKER_OPEN, '(added) ') - n_body = n_body.replace(ADDED_PLACEMARKER_CLOSED, '') - n_body = n_body.replace(CHANGED_PLACEMARKER_OPEN, f'(changed) ') - n_body = n_body.replace(CHANGED_PLACEMARKER_CLOSED, f'') - n_body = n_body.replace(CHANGED_INTO_PLACEMARKER_OPEN, f'(into) ') - n_body = n_body.replace(CHANGED_INTO_PLACEMARKER_CLOSED, f'') - n_body = n_body.replace('\n', f'{CUSTOM_LINEBREAK_PLACEHOLDER}\n') + n_body = replace_placemarkers_in_text(n_body, url, requested_output_format) + n_body = newline_re.sub('
\n', n_body) elif requested_output_format == 'markdown': # Markdown to HTML - Apprise will convert this to HTML - n_body = apply_standard_markdown_to_body(n_body=n_body) + n_body = replace_placemarkers_in_text(n_body, url, requested_output_format) else: #plaintext etc default - n_body = n_body.replace(REMOVED_PLACEMARKER_OPEN, '(removed) ') - n_body = n_body.replace(REMOVED_PLACEMARKER_CLOSED, '') - n_body = n_body.replace(ADDED_PLACEMARKER_OPEN, '(added) ') - n_body = n_body.replace(ADDED_PLACEMARKER_CLOSED, '') - n_body = n_body.replace(CHANGED_PLACEMARKER_OPEN, f'(changed) ') - n_body = n_body.replace(CHANGED_PLACEMARKER_CLOSED, f'') - n_body = n_body.replace(CHANGED_INTO_PLACEMARKER_OPEN, f'(into) ') - n_body = n_body.replace(CHANGED_INTO_PLACEMARKER_CLOSED, f'') + n_body = replace_placemarkers_in_text(n_body, url, requested_output_format) return url, n_body, n_title @@ -295,24 +334,18 @@ def process_notification(n_object: NotificationContextData, datastore): with (apprise.LogCapture(level=apprise.logging.DEBUG) as logs): for url in n_object['notification_urls']: - # Get the notification body from datastore n_body = jinja_render(template_str=n_object.get('notification_body', ''), **notification_parameters) + n_title = jinja_render(template_str=n_object.get('notification_title', ''), **notification_parameters) if n_object.get('markup_text_links_to_html_links'): n_body = markup_text_links_to_html(body=n_body) - n_title = jinja_render(template_str=n_object.get('notification_title', ''), **notification_parameters) - url = url.strip() - if url.startswith('#'): - logger.trace(f"Skipping commented out notification URL - {url}") + if not url or url.startswith('#'): + logger.debug(f"Skipping commented out or empty notification URL - '{url}'") continue - if not url: - logger.warning(f"Process Notification: skipping empty notification URL.") - continue - - logger.info(f">> Process Notification: AppRise notifying {url}") + logger.info(f">> Process Notification: AppRise start notifying '{url}'") url = jinja_render(template_str=url, **notification_parameters) # If it's a plaintext document, and they want HTML type email/alerts, so it needs to be escaped @@ -353,25 +386,18 @@ def process_notification(n_object: NotificationContextData, datastore): requested_output_format = NotifyFormat.HTML.value apprise_input_format = NotifyFormat.HTML.value # Changed from MARKDOWN to HTML - # Could have arrived at any stage, so we dont end up running .escape on it - if 'html' in requested_output_format: - n_body = n_body.replace(CUSTOM_LINEBREAK_PLACEHOLDER, '
\r\n') - else: - # texty types - n_body = n_body.replace(CUSTOM_LINEBREAK_PLACEHOLDER, '\r\n') - else: # ?format was IN the apprise URL, they are kind of on their own here, we will try our best if 'format=html' in url: - n_body = n_body.replace(CUSTOM_LINEBREAK_PLACEHOLDER, '
\r\n') + n_body = newline_re.sub('
\r\n', n_body) # This will also prevent apprise from doing conversion apprise_input_format = NotifyFormat.HTML.value requested_output_format = NotifyFormat.HTML.value elif 'format=text' in url: - n_body = n_body.replace(CUSTOM_LINEBREAK_PLACEHOLDER, '\r\n') apprise_input_format = NotifyFormat.TEXT.value requested_output_format = NotifyFormat.TEXT.value + sent_objs.append({'title': n_title, 'body': n_body, 'url': url}) diff --git a/changedetectionio/notification_service.py b/changedetectionio/notification_service.py index f144c513..d77fb551 100644 --- a/changedetectionio/notification_service.py +++ b/changedetectionio/notification_service.py @@ -129,11 +129,11 @@ class NotificationService: n_object.update({ 'current_snapshot': snapshot_contents, - 'diff': diff.render_diff(prev_snapshot, current_snapshot, line_feed_sep=CUSTOM_LINEBREAK_PLACEHOLDER), - 'diff_added': diff.render_diff(prev_snapshot, current_snapshot, include_removed=False, line_feed_sep=CUSTOM_LINEBREAK_PLACEHOLDER), - 'diff_full': diff.render_diff(prev_snapshot, current_snapshot, include_equal=True, line_feed_sep=CUSTOM_LINEBREAK_PLACEHOLDER), - 'diff_patch': diff.render_diff(prev_snapshot, current_snapshot, line_feed_sep=CUSTOM_LINEBREAK_PLACEHOLDER, patch_format=True), - 'diff_removed': diff.render_diff(prev_snapshot, current_snapshot, include_added=False, line_feed_sep=CUSTOM_LINEBREAK_PLACEHOLDER), + 'diff': diff.render_diff(prev_snapshot, current_snapshot), + 'diff_added': diff.render_diff(prev_snapshot, current_snapshot, include_removed=False), + 'diff_full': diff.render_diff(prev_snapshot, current_snapshot, include_equal=True), + 'diff_patch': diff.render_diff(prev_snapshot, current_snapshot, patch_format=True), + 'diff_removed': diff.render_diff(prev_snapshot, current_snapshot, include_added=False), 'screenshot': watch.get_screenshot() if watch and watch.get('notification_screenshot') else None, 'triggered_text': triggered_text, 'uuid': watch.get('uuid') if watch else None, @@ -141,6 +141,7 @@ class NotificationService: 'watch_uuid': watch.get('uuid') if watch else None, 'watch_mime_type': watch.get('content-type') }) + # The \n's in the content from the above will get converted to
etc depending on the notification format if watch: n_object.update(watch.extra_notification_token_values()) diff --git a/changedetectionio/tests/smtp/test_notification_smtp.py b/changedetectionio/tests/smtp/test_notification_smtp.py index 02482b9b..c391f524 100644 --- a/changedetectionio/tests/smtp/test_notification_smtp.py +++ b/changedetectionio/tests/smtp/test_notification_smtp.py @@ -3,8 +3,9 @@ from flask import url_for from email import message_from_string from email.policy import default as email_policy -from changedetectionio.diff import HTML_REMOVED_STYLE, HTML_ADDED_STYLE, HTML_CHANGED_STYLE -from changedetectionio.notification_service import NotificationContextData, CUSTOM_LINEBREAK_PLACEHOLDER +from changedetectionio.diff import HTML_REMOVED_STYLE, HTML_ADDED_STYLE, HTML_CHANGED_STYLE, REMOVED_PLACEMARKER_OPEN, \ + CHANGED_PLACEMARKER_OPEN, ADDED_PLACEMARKER_OPEN +from changedetectionio.notification_service import NotificationContextData from changedetectionio.tests.util import set_original_response, set_modified_response, set_more_modified_response, live_server_setup, \ wait_for_all_checks, \ set_longer_modified_response, delete_all_watches @@ -100,7 +101,6 @@ def test_check_notification_email_formats_default_HTML(client, live_server, meas text_content = text_part.get_content() assert '(added) So let\'s see what happens.\r\n' in text_content # The plaintext part assert 'fallback-body\r\n' in text_content # The plaintext part - assert CUSTOM_LINEBREAK_PLACEHOLDER not in text_content # Second part should be text/html html_part = parts[1] @@ -109,7 +109,6 @@ def test_check_notification_email_formats_default_HTML(client, live_server, meas assert 'some text
' in html_content # We converted \n from the notification body assert 'fallback-body
' in html_content # kept the original
assert '(added) So let\'s see what happens.
' in html_content # the html part - assert CUSTOM_LINEBREAK_PLACEHOLDER not in html_content delete_all_watches(client) @@ -124,8 +123,8 @@ def test_check_notification_plaintext_format(client, live_server, measure_memory res = client.post( url_for("settings.settings_page"), data={"application-notification_urls": notification_url, - "application-notification_title": "fallback-title " + default_notification_title, - "application-notification_body": "some text\n" + default_notification_body, + "application-notification_title": "fallback-title {{watch_title}} {{ diff_added.splitlines()[0] if diff_added else 'diff added didnt split' }} " + default_notification_title, + "application-notification_body": f"some text\n" + default_notification_body + f"\nMore output test\n{ALL_MARKUP_TOKENS}", "application-notification_format": 'text', "requests-time_between_check-minutes": 180, 'application-fetch_backend': "html_requests"}, @@ -148,9 +147,17 @@ def test_check_notification_plaintext_format(client, live_server, measure_memory msg_raw = get_last_message_from_smtp_server() assert len(msg_raw) >= 1 - + #time.sleep(60) # Parse the email properly using Python's email library msg = message_from_string(msg_raw, policy=email_policy) + # Subject/title got marked up + subject = msg['subject'] + # Subject should always be plaintext and never marked up to anything else + assert REMOVED_PLACEMARKER_OPEN not in subject + assert CHANGED_PLACEMARKER_OPEN not in subject + assert ADDED_PLACEMARKER_OPEN not in subject + assert 'diff added didnt split' not in subject + assert '(changed) Which is across' in subject # The email should be plain text only (not multipart) assert not msg.is_multipart() @@ -177,7 +184,7 @@ def test_check_notification_html_color_format(client, live_server, measure_memor res = client.post( url_for("settings.settings_page"), data={"application-notification_urls": notification_url, - "application-notification_title": "fallback-title " + default_notification_title, + "application-notification_title": "fallback-title {{watch_title}} - diff_added_lines_test : '{{ diff_added.splitlines()[0] if diff_added else 'diff added didnt split' }}' " + default_notification_title, "application-notification_body": f"some text\n{default_notification_body}\nMore output test\n{ALL_MARKUP_TOKENS}", "application-notification_format": 'htmlcolor', "requests-time_between_check-minutes": 180, @@ -211,6 +218,18 @@ def test_check_notification_html_color_format(client, live_server, measure_memor # Parse the email properly using Python's email library msg = message_from_string(msg_raw, policy=email_policy) + # Subject/title got marked up + subject = msg['subject'] + # Subject should always be plaintext and never marked up to anything else + assert REMOVED_PLACEMARKER_OPEN not in subject + assert CHANGED_PLACEMARKER_OPEN not in subject + assert ADDED_PLACEMARKER_OPEN not in subject + assert 'diff added didnt split' not in subject + assert '(changed) Which is across' in subject + assert 'head title' in subject + assert "span" not in subject + assert 'background-color' not in subject + # The email should have two bodies (multipart/alternative with text/plain and text/html) assert msg.is_multipart() @@ -249,7 +268,7 @@ def test_check_notification_markdown_format(client, live_server, measure_memory_ res = client.post( url_for("settings.settings_page"), data={"application-notification_urls": notification_url, - "application-notification_title": "fallback-title " + default_notification_title, + "application-notification_title": "fallback-title diff_added_lines_test : '{{ diff_added.splitlines()[0] if diff_added else 'diff added didnt split' }}' " + default_notification_title, "application-notification_body": "*header*\n\nsome text\n" + default_notification_body, "application-notification_format": 'markdown', "requests-time_between_check-minutes": 180, @@ -287,6 +306,14 @@ def test_check_notification_markdown_format(client, live_server, measure_memory_ # The email should have two bodies (multipart/alternative with text/plain and text/html) assert msg.is_multipart() assert msg.get_content_type() == 'multipart/alternative' + subject = msg['subject'] + # Subject should always be plaintext and never marked up to anything else + assert REMOVED_PLACEMARKER_OPEN not in subject + assert CHANGED_PLACEMARKER_OPEN not in subject + assert ADDED_PLACEMARKER_OPEN not in subject + assert 'diff added didnt split' not in subject + assert '(changed) Which is across' in subject + # Get the parts parts = list(msg.iter_parts()) @@ -305,7 +332,10 @@ def test_check_notification_markdown_format(client, live_server, measure_memory_ assert html_part.get_content_type() == 'text/html' html_content = html_part.get_content() assert '

header

' in html_content - assert 'So let\'s see what happens.
' in html_content # Additions are in markdown + assert 'So let\'s see what happens.
' in html_content # Additions are in markdown + # the '
' will come from apprises conversion, not from our code, we would rather use '
' correctly + # the '
' is actually a nice way to know if apprise done the conversion. + delete_all_watches(client) # Custom notification body with HTML, that is either sent as HTML or rendered to plaintext and sent @@ -752,7 +782,6 @@ def test_check_html_notification_with_apprise_format_is_html(client, live_server text_content = text_part.get_content() assert '(added) So let\'s see what happens.\r\n' in text_content # The plaintext part assert 'fallback-body\r\n' in text_content # The plaintext part - assert CUSTOM_LINEBREAK_PLACEHOLDER not in text_content # Second part should be text/html html_part = parts[1] @@ -761,5 +790,4 @@ def test_check_html_notification_with_apprise_format_is_html(client, live_server assert 'some text
' in html_content # We converted \n from the notification body assert 'fallback-body
' in html_content # kept the original
assert '(added) So let\'s see what happens.
' in html_content # the html part - assert CUSTOM_LINEBREAK_PLACEHOLDER not in html_content delete_all_watches(client) \ No newline at end of file