diff --git a/changedetectionio/notification/__init__.py b/changedetectionio/notification/__init__.py index 8f6bd81c..06ed830a 100644 --- a/changedetectionio/notification/__init__.py +++ b/changedetectionio/notification/__init__.py @@ -7,10 +7,10 @@ default_notification_title = 'ChangeDetection.io Notification - {{watch_url}}' # The values (markdown etc) are from apprise NotifyFormat, # But to avoid importing the whole heavy module just use the same strings here. valid_notification_formats = { - 'Text': 'text', - 'Markdown': 'markdown', + 'Plain Text': 'text', 'HTML': 'html', 'HTML Color': 'htmlcolor', + 'Markdown to HTML': 'markdown', # Used only for editing a watch (not for global) default_notification_format_for_watch: default_notification_format_for_watch } diff --git a/changedetectionio/notification/handler.py b/changedetectionio/notification/handler.py index d5c6edc3..948e4558 100644 --- a/changedetectionio/notification/handler.py +++ b/changedetectionio/notification/handler.py @@ -53,6 +53,7 @@ def notification_format_align_with_apprise(n_format : str): """ Correctly align changedetection's formats with apprise's formats Probably these are the same - but good to be sure. + These set the expected OUTPUT format type :param n_format: :return: """ @@ -71,12 +72,63 @@ def notification_format_align_with_apprise(n_format : str): return n_format + +def apply_service_tweaks(url, n_body, n_title): + # Re 323 - Limit discord length to their 2000 char limit total or it wont send. + # Because different notifications may require different pre-processing, run each sequentially :( + # 2000 bytes minus - + # 200 bytes for the overhead of the _entire_ json payload, 200 bytes for {tts, wait, content} etc headers + # Length of URL - Incase they specify a longer custom avatar_url + + # So if no avatar_url is specified, add one so it can be correctly calculated into the total payload + parsed = urlparse(url) + k = '?' if not parsed.query else '&' + if not 'avatar_url' in url \ + and not url.startswith('mail') \ + and not url.startswith('post') \ + and not url.startswith('get') \ + and not url.startswith('delete') \ + and not url.startswith('put'): + url += k + f"avatar_url={APPRISE_AVATAR_URL}" + + if url.startswith('tgram://'): + # Telegram only supports a limit subset of HTML, remove the '
' we place in. + # re https://github.com/dgtlmoon/changedetection.io/issues/555 + # @todo re-use an existing library we have already imported to strip all non-allowed tags + n_body = n_body.replace('
', '\n') + n_body = n_body.replace('
', '\n') + # real limit is 4096, but minus some for extra metadata + payload_max_size = 3600 + body_limit = max(0, payload_max_size - len(n_title)) + n_title = n_title[0:payload_max_size] + n_body = n_body[0:body_limit] + + elif url.startswith('discord://') or url.startswith('https://discordapp.com/api/webhooks') or url.startswith( + 'https://discord.com/api'): + # real limit is 2000, but minus some for extra metadata + payload_max_size = 1700 + body_limit = max(0, payload_max_size - len(n_title)) + n_title = n_title[0:payload_max_size] + n_body = n_body[0:body_limit] + + return url, n_body, n_title + + def process_notification(n_object: NotificationContextData, datastore): from changedetectionio.jinja2_custom import render as jinja_render from . import default_notification_format_for_watch, default_notification_format, valid_notification_formats # be sure its registered from .apprise_plugin.custom_handlers import apprise_http_custom_handler + # Create list of custom handler protocols (both http and https versions) + custom_handler_protocols = [f"{method}://" for method in SUPPORTED_HTTP_METHODS] + custom_handler_protocols += [f"{method}s://" for method in SUPPORTED_HTTP_METHODS] + + has_custom_handler = any( + url.startswith(tuple(custom_handler_protocols)) + for url in n_object['notification_urls'] + ) + if not isinstance(n_object, NotificationContextData): raise TypeError(f"Expected NotificationContextData, got {type(n_object)}") @@ -87,20 +139,25 @@ def process_notification(n_object: NotificationContextData, datastore): # Insert variables into the notification content notification_parameters = create_notification_parameters(n_object, datastore) - n_format = valid_notification_formats.get( + requested_output_format = valid_notification_formats.get( n_object.get('notification_format', default_notification_format), valid_notification_formats[default_notification_format], ) # If we arrived with 'System default' then look it up - if n_format == default_notification_format_for_watch and datastore.data['settings']['application'].get('notification_format') != default_notification_format_for_watch: + if requested_output_format == default_notification_format_for_watch and datastore.data['settings']['application'].get('notification_format') != default_notification_format_for_watch: # Initially text or whatever - n_format = datastore.data['settings']['application'].get('notification_format', valid_notification_formats[default_notification_format]).lower() + requested_output_format = datastore.data['settings']['application'].get('notification_format', valid_notification_formats[default_notification_format]).lower() - n_format = notification_format_align_with_apprise(n_format=n_format) + requested_output_format = notification_format_align_with_apprise(n_format=requested_output_format) logger.trace(f"Complete notification body including Jinja and placeholders calculated in {time.time() - now:.2f}s") + # If we have custom handlers, use invalid format to prevent conversion + # Otherwise use the proper format + if has_custom_handler: + input_format = 'raw-no-convert' + # https://github.com/caronc/apprise/wiki/Development_LogCapture # Anything higher than or equal to WARNING (which covers things like Connection errors) # raise it as an exception @@ -117,6 +174,8 @@ def process_notification(n_object: NotificationContextData, datastore): with apprise.LogCapture(level=apprise.logging.DEBUG) as logs: for url in n_object['notification_urls']: + parsed_url = urlparse(url) + prefix_add_to_url = '?' if not parsed_url.query else '&' # Get the notification body from datastore n_body = jinja_render(template_str=n_object.get('notification_body', ''), **notification_parameters) @@ -124,8 +183,32 @@ def process_notification(n_object: NotificationContextData, datastore): if n_object.get('markup_text_to_html'): n_body = markup_text_links_to_html(body=n_body) - if n_format == NotifyFormat.HTML.value: + # This actually means we request "Markdown to HTML" + if requested_output_format == NotifyFormat.MARKDOWN.value: + output_format = NotifyFormat.HTML.value + input_format = NotifyFormat.MARKDOWN.value + if not 'format=' in url.lower(): + url = f"{url}{prefix_add_to_url}format={output_format}" + + # Deviation from apprise. + # No conversion, its like they want to send raw HTML but we add linebreaks + elif requested_output_format == NotifyFormat.HTML.value: + # same in and out means apprise wont try to convert + input_format = output_format = NotifyFormat.HTML.value n_body = n_body.replace("\n", '
') + if not 'format=' in url.lower(): + url = f"{url}{prefix_add_to_url}format={output_format}" + + else: + # Nothing to be done, leave it as plaintext + # `body_format` Tell apprise what format the INPUT is in + # &format= in URL Tell apprise what format the OUTPUT should be in (it can convert between) + input_format = output_format = NotifyFormat.TEXT.value + if not 'format=' in url.lower(): + url = f"{url}{prefix_add_to_url}format={output_format}" + + if has_custom_handler: + input_format='raw-no-convert' n_title = jinja_render(template_str=n_object.get('notification_title', ''), **notification_parameters) @@ -141,88 +224,24 @@ def process_notification(n_object: NotificationContextData, datastore): logger.info(f">> Process Notification: AppRise notifying {url}") url = jinja_render(template_str=url, **notification_parameters) - # Re 323 - Limit discord length to their 2000 char limit total or it wont send. - # Because different notifications may require different pre-processing, run each sequentially :( - # 2000 bytes minus - - # 200 bytes for the overhead of the _entire_ json payload, 200 bytes for {tts, wait, content} etc headers - # Length of URL - Incase they specify a longer custom avatar_url - - # So if no avatar_url is specified, add one so it can be correctly calculated into the total payload - parsed = urlparse(url) - k = '?' if not parsed.query else '&' - if not 'avatar_url' in url \ - and not url.startswith('mail') \ - and not url.startswith('post') \ - and not url.startswith('get') \ - and not url.startswith('delete') \ - and not url.startswith('put'): - url += k + f"avatar_url={APPRISE_AVATAR_URL}" - - if url.startswith('tgram://'): - # Telegram only supports a limit subset of HTML, remove the '
' we place in. - # re https://github.com/dgtlmoon/changedetection.io/issues/555 - # @todo re-use an existing library we have already imported to strip all non-allowed tags - n_body = n_body.replace('
', '\n') - n_body = n_body.replace('
', '\n') - # real limit is 4096, but minus some for extra metadata - payload_max_size = 3600 - body_limit = max(0, payload_max_size - len(n_title)) - n_title = n_title[0:payload_max_size] - n_body = n_body[0:body_limit] - - elif url.startswith('discord://') or url.startswith('https://discordapp.com/api/webhooks') or url.startswith( - 'https://discord.com/api'): - # real limit is 2000, but minus some for extra metadata - payload_max_size = 1700 - body_limit = max(0, payload_max_size - len(n_title)) - n_title = n_title[0:payload_max_size] - n_body = n_body[0:body_limit] - - # Add format parameter to mailto URLs to ensure proper text/html handling - # https://github.com/caronc/apprise/issues/633#issuecomment-1191449321 - # Note: Custom handlers (post://, get://, etc.) don't need this as we handle them - # differently by passing an invalid body_format to prevent HTML conversion - if not 'format=' in url and url.startswith(('mailto', 'mailtos')): - parsed = urlparse(url) - prefix = '?' if not parsed.query else '&' - # Apprise format is already lowercase from notification_format_align_with_apprise() - url = f"{url}{prefix}format={n_format}" + (url, n_body, n_title) = apply_service_tweaks(url=url, n_body=n_body, n_title=n_title) apobj.add(url) sent_objs.append({'title': n_title, 'body': n_body, - 'url': url, - 'body_format': n_format}) - - # Blast off the notifications tht are set in .add() - # Check if we have any custom HTTP handlers (post://, get://, etc.) - # These handlers created with @notify decorator don't handle format conversion properly - # and will strip HTML if we pass a valid format. So we pass an invalid format string - # to prevent Apprise from converting HTML->TEXT - - # Create list of custom handler protocols (both http and https versions) - custom_handler_protocols = [f"{method}://" for method in SUPPORTED_HTTP_METHODS] - custom_handler_protocols += [f"{method}s://" for method in SUPPORTED_HTTP_METHODS] - - has_custom_handler = any( - url.startswith(tuple(custom_handler_protocols)) - for url in n_object['notification_urls'] - ) - - # If we have custom handlers, use invalid format to prevent conversion - # Otherwise use the proper format - notify_format = 'raw-no-convert' if has_custom_handler else n_format + 'url': url}) apobj.notify( title=n_title, body=n_body, - body_format=notify_format, + # `body_format` Tell apprise what format the INPUT is in + # &format= in URL Tell apprise what format the OUTPUT should be in (it can convert between) + body_format=input_format, # False is not an option for AppRise, must be type None attach=n_object.get('screenshot', None) ) - # Returns empty string if nothing found, multi-line string otherwise log_value = logs.getvalue() diff --git a/changedetectionio/tests/smtp/test_notification_smtp.py b/changedetectionio/tests/smtp/test_notification_smtp.py index 6f897e17..9c60433d 100644 --- a/changedetectionio/tests/smtp/test_notification_smtp.py +++ b/changedetectionio/tests/smtp/test_notification_smtp.py @@ -1,7 +1,4 @@ -import json -import os import time -import re from flask import url_for from email import message_from_string from email.policy import default as email_policy @@ -10,9 +7,9 @@ from changedetectionio.diff import REMOVED_STYLE, ADDED_STYLE from changedetectionio.tests.util import set_original_response, set_modified_response, set_more_modified_response, live_server_setup, \ wait_for_all_checks, \ set_longer_modified_response, delete_all_watches -from changedetectionio.tests.util import extract_UUID_from_client + import logging -import base64 + # NOTE - RELIES ON mailserver as hostname running, see github build recipes smtp_test_server = 'mailserver' @@ -124,7 +121,7 @@ def test_check_notification_plaintext_format(client, live_server, measure_memory data={"application-notification_urls": notification_url, "application-notification_title": "fallback-title " + default_notification_title, "application-notification_body": "some text\n" + default_notification_body, - "application-notification_format": 'Text', + "application-notification_format": 'Plain Text', "requests-time_between_check-minutes": 180, 'application-fetch_backend': "html_requests"}, follow_redirects=True @@ -134,18 +131,11 @@ def test_check_notification_plaintext_format(client, live_server, measure_memory # Add a watch and trigger a HTTP POST test_url = url_for('test_endpoint', _external=True) - res = client.post( - url_for("ui.ui_views.form_quick_watch_add"), - data={"url": test_url, "tags": 'nice one'}, - follow_redirects=True - ) - - assert b"Watch added" in res.data - - wait_for_all_checks(client) - set_longer_modified_response() + uuid = client.application.config.get('DATASTORE').add_watch(url=test_url) + client.get(url_for("ui.form_watch_checknow"), follow_redirects=True) time.sleep(2) + set_longer_modified_response() client.get(url_for("ui.form_watch_checknow"), follow_redirects=True) wait_for_all_checks(client) @@ -241,9 +231,76 @@ def test_check_notification_html_color_format(client, live_server, measure_memor assert 'some text
' in html_content delete_all_watches(client) +def test_check_notification_markdown_format(client, live_server, measure_memory_usage): + set_original_response() + + notification_url = f'mailto://changedetection@{smtp_test_server}:11025/?to=fff@home.com' + + ##################### + # Set this up for when we remove the notification from the watch, it should fallback with these details + res = client.post( + url_for("settings.settings_page"), + data={"application-notification_urls": notification_url, + "application-notification_title": "fallback-title " + default_notification_title, + "application-notification_body": "*header*\n\nsome text\n" + default_notification_body, + "application-notification_format": 'Markdown to HTML', + "requests-time_between_check-minutes": 180, + 'application-fetch_backend': "html_requests"}, + follow_redirects=True + ) + + assert b"Settings updated." in res.data + + # Add a watch and trigger a HTTP POST + test_url = url_for('test_endpoint', _external=True) + res = client.post( + url_for("ui.ui_views.form_quick_watch_add"), + data={"url": test_url, "tags": 'nice one'}, + follow_redirects=True + ) + + assert b"Watch added" in res.data + + wait_for_all_checks(client) + set_longer_modified_response() + time.sleep(2) + + client.get(url_for("ui.form_watch_checknow"), follow_redirects=True) + wait_for_all_checks(client) + + time.sleep(3) + + msg_raw = get_last_message_from_smtp_server() + assert len(msg_raw) >= 1 + + # Parse the email properly using Python's email library + msg = message_from_string(msg_raw, policy=email_policy) + + # The email should have two bodies (multipart/alternative with text/plain and text/html) + assert msg.is_multipart() + assert msg.get_content_type() == 'multipart/alternative' + + # Get the parts + parts = list(msg.iter_parts()) + assert len(parts) == 2 + + # First part should be text/plain (the auto-generated plaintext version) + text_part = parts[0] + assert text_part.get_content_type() == 'text/plain' + text_content = text_part.get_content() + assert '(added) So let\'s see what happens.\r\n' in text_content # The plaintext part + + + # Second part should be text/html and roughly converted from markdown to HTML + html_part = parts[1] + assert html_part.get_content_type() == 'text/html' + html_content = html_part.get_content() + assert '

header

' in html_content + assert '(added) So let\'s see what happens.