Notifications - Refactor/cleanup notification handling and rename 'Markdown' to "Markdown to HTML" to make more sense. (#3527) Re #3526 -

changedetectionio/notification/__init__.py

@@ -7,10 +7,10 @@ default_notification_title = 'ChangeDetection.io Notification - {{watch_url}}'
 # The values (markdown etc) are from apprise NotifyFormat,
 # But to avoid importing the whole heavy module just use the same strings here.
 valid_notification_formats = {
-    'Text': 'text',
-    'Markdown': 'markdown',
+    'Plain Text': 'text',
     'HTML': 'html',
     'HTML Color': 'htmlcolor',
+    'Markdown to HTML': 'markdown',
     # Used only for editing a watch (not for global)
     default_notification_format_for_watch: default_notification_format_for_watch
 }

changedetectionio/notification/handler.py

@@ -53,6 +53,7 @@ def notification_format_align_with_apprise(n_format : str):
     """
     Correctly align changedetection's formats with apprise's formats
     Probably these are the same - but good to be sure.
+    These set the expected OUTPUT format type
     :param n_format:
     :return:
     """
@@ -71,12 +72,63 @@ def notification_format_align_with_apprise(n_format : str):
 
     return n_format
 
+
+def apply_service_tweaks(url, n_body, n_title):
+    # Re 323 - Limit discord length to their 2000 char limit total or it wont send.
+    # Because different notifications may require different pre-processing, run each sequentially :(
+    # 2000 bytes minus -
+    #     200 bytes for the overhead of the _entire_ json payload, 200 bytes for {tts, wait, content} etc headers
+    #     Length of URL - Incase they specify a longer custom avatar_url
+
+    # So if no avatar_url is specified, add one so it can be correctly calculated into the total payload
+    parsed = urlparse(url)
+    k = '?' if not parsed.query else '&'
+    if not 'avatar_url' in url \
+            and not url.startswith('mail') \
+            and not url.startswith('post') \
+            and not url.startswith('get') \
+            and not url.startswith('delete') \
+            and not url.startswith('put'):
+        url += k + f"avatar_url={APPRISE_AVATAR_URL}"
+
+    if url.startswith('tgram://'):
+        # Telegram only supports a limit subset of HTML, remove the '<br>' we place in.
+        # re https://github.com/dgtlmoon/changedetection.io/issues/555
+        # @todo re-use an existing library we have already imported to strip all non-allowed tags
+        n_body = n_body.replace('<br>', '\n')
+        n_body = n_body.replace('</br>', '\n')
+        # real limit is 4096, but minus some for extra metadata
+        payload_max_size = 3600
+        body_limit = max(0, payload_max_size - len(n_title))
+        n_title = n_title[0:payload_max_size]
+        n_body = n_body[0:body_limit]
+
+    elif url.startswith('discord://') or url.startswith('https://discordapp.com/api/webhooks') or url.startswith(
+            'https://discord.com/api'):
+        # real limit is 2000, but minus some for extra metadata
+        payload_max_size = 1700
+        body_limit = max(0, payload_max_size - len(n_title))
+        n_title = n_title[0:payload_max_size]
+        n_body = n_body[0:body_limit]
+
+    return url, n_body, n_title
+
+
 def process_notification(n_object: NotificationContextData, datastore):
     from changedetectionio.jinja2_custom import render as jinja_render
     from . import default_notification_format_for_watch, default_notification_format, valid_notification_formats
     # be sure its registered
     from .apprise_plugin.custom_handlers import apprise_http_custom_handler
 
+    # Create list of custom handler protocols (both http and https versions)
+    custom_handler_protocols = [f"{method}://" for method in SUPPORTED_HTTP_METHODS]
+    custom_handler_protocols += [f"{method}s://" for method in SUPPORTED_HTTP_METHODS]
+
+    has_custom_handler = any(
+        url.startswith(tuple(custom_handler_protocols))
+        for url in n_object['notification_urls']
+    )
+
     if not isinstance(n_object, NotificationContextData):
         raise TypeError(f"Expected NotificationContextData, got {type(n_object)}")
 
@@ -87,20 +139,25 @@ def process_notification(n_object: NotificationContextData, datastore):
     # Insert variables into the notification content
     notification_parameters = create_notification_parameters(n_object, datastore)
 
-    n_format = valid_notification_formats.get(
+    requested_output_format = valid_notification_formats.get(
         n_object.get('notification_format', default_notification_format),
         valid_notification_formats[default_notification_format],
     )
 
     # If we arrived with 'System default' then look it up
-    if n_format == default_notification_format_for_watch and datastore.data['settings']['application'].get('notification_format') != default_notification_format_for_watch:
+    if requested_output_format == default_notification_format_for_watch and datastore.data['settings']['application'].get('notification_format') != default_notification_format_for_watch:
         # Initially text or whatever
-        n_format = datastore.data['settings']['application'].get('notification_format', valid_notification_formats[default_notification_format]).lower()
+        requested_output_format = datastore.data['settings']['application'].get('notification_format', valid_notification_formats[default_notification_format]).lower()
 
-    n_format = notification_format_align_with_apprise(n_format=n_format)
+    requested_output_format = notification_format_align_with_apprise(n_format=requested_output_format)
 
     logger.trace(f"Complete notification body including Jinja and placeholders calculated in  {time.time() - now:.2f}s")
 
+    # If we have custom handlers, use invalid format to prevent conversion
+    # Otherwise use the proper format
+    if has_custom_handler:
+        input_format = 'raw-no-convert'
+
     # https://github.com/caronc/apprise/wiki/Development_LogCapture
     # Anything higher than or equal to WARNING (which covers things like Connection errors)
     # raise it as an exception
@@ -117,6 +174,8 @@ def process_notification(n_object: NotificationContextData, datastore):
 
     with apprise.LogCapture(level=apprise.logging.DEBUG) as logs:
         for url in n_object['notification_urls']:
+            parsed_url = urlparse(url)
+            prefix_add_to_url = '?' if not parsed_url.query else '&'
 
             # Get the notification body from datastore
             n_body = jinja_render(template_str=n_object.get('notification_body', ''), **notification_parameters)
@@ -124,8 +183,32 @@ def process_notification(n_object: NotificationContextData, datastore):
             if n_object.get('markup_text_to_html'):
                 n_body = markup_text_links_to_html(body=n_body)
 
-            if n_format == NotifyFormat.HTML.value:
+            # This actually means we request "Markdown to HTML"
+            if requested_output_format == NotifyFormat.MARKDOWN.value:
+                output_format = NotifyFormat.HTML.value
+                input_format = NotifyFormat.MARKDOWN.value
+                if not 'format=' in url.lower():
+                    url = f"{url}{prefix_add_to_url}format={output_format}"
+
+            # Deviation from apprise.
+            # No conversion, its like they want to send raw HTML but we add linebreaks
+            elif requested_output_format == NotifyFormat.HTML.value:
+                # same in and out means apprise wont try to convert
+                input_format = output_format = NotifyFormat.HTML.value
                 n_body = n_body.replace("\n", '<br>')
+                if not 'format=' in url.lower():
+                    url = f"{url}{prefix_add_to_url}format={output_format}"
+
+            else:
+                # Nothing to be done, leave it as plaintext
+                # `body_format` Tell apprise what format the INPUT is in
+                # &format= in URL Tell apprise what format the OUTPUT should be in (it can convert between)
+                input_format = output_format = NotifyFormat.TEXT.value
+                if not 'format=' in url.lower():
+                    url = f"{url}{prefix_add_to_url}format={output_format}"
+
+            if has_custom_handler:
+                input_format='raw-no-convert'
 
             n_title = jinja_render(template_str=n_object.get('notification_title', ''), **notification_parameters)
 
@@ -141,88 +224,24 @@ def process_notification(n_object: NotificationContextData, datastore):
             logger.info(f">> Process Notification: AppRise notifying {url}")
             url = jinja_render(template_str=url, **notification_parameters)
 
-            # Re 323 - Limit discord length to their 2000 char limit total or it wont send.
-            # Because different notifications may require different pre-processing, run each sequentially :(
-            # 2000 bytes minus -
-            #     200 bytes for the overhead of the _entire_ json payload, 200 bytes for {tts, wait, content} etc headers
-            #     Length of URL - Incase they specify a longer custom avatar_url
-
-            # So if no avatar_url is specified, add one so it can be correctly calculated into the total payload
-            parsed = urlparse(url)
-            k = '?' if not parsed.query else '&'
-            if not 'avatar_url' in url \
-                    and not url.startswith('mail') \
-                    and not url.startswith('post') \
-                    and not url.startswith('get') \
-                    and not url.startswith('delete') \
-                    and not url.startswith('put'):
-                url += k + f"avatar_url={APPRISE_AVATAR_URL}"
-
-            if url.startswith('tgram://'):
-                # Telegram only supports a limit subset of HTML, remove the '<br>' we place in.
-                # re https://github.com/dgtlmoon/changedetection.io/issues/555
-                # @todo re-use an existing library we have already imported to strip all non-allowed tags
-                n_body = n_body.replace('<br>', '\n')
-                n_body = n_body.replace('</br>', '\n')
-                # real limit is 4096, but minus some for extra metadata
-                payload_max_size = 3600
-                body_limit = max(0, payload_max_size - len(n_title))
-                n_title = n_title[0:payload_max_size]
-                n_body = n_body[0:body_limit]
-
-            elif url.startswith('discord://') or url.startswith('https://discordapp.com/api/webhooks') or url.startswith(
-                    'https://discord.com/api'):
-                # real limit is 2000, but minus some for extra metadata
-                payload_max_size = 1700
-                body_limit = max(0, payload_max_size - len(n_title))
-                n_title = n_title[0:payload_max_size]
-                n_body = n_body[0:body_limit]
-
-            # Add format parameter to mailto URLs to ensure proper text/html handling
-            # https://github.com/caronc/apprise/issues/633#issuecomment-1191449321
-            # Note: Custom handlers (post://, get://, etc.) don't need this as we handle them
-            # differently by passing an invalid body_format to prevent HTML conversion
-            if not 'format=' in url and url.startswith(('mailto', 'mailtos')):
-                parsed = urlparse(url)
-                prefix = '?' if not parsed.query else '&'
-                # Apprise format is already lowercase from notification_format_align_with_apprise()
-                url = f"{url}{prefix}format={n_format}"
+            (url, n_body, n_title) = apply_service_tweaks(url=url, n_body=n_body, n_title=n_title)
 
             apobj.add(url)
 
             sent_objs.append({'title': n_title,
                               'body': n_body,
-                              'url': url,
-                              'body_format': n_format})
-
-        # Blast off the notifications tht are set in .add()
-        # Check if we have any custom HTTP handlers (post://, get://, etc.)
-        # These handlers created with @notify decorator don't handle format conversion properly
-        # and will strip HTML if we pass a valid format. So we pass an invalid format string
-        # to prevent Apprise from converting HTML->TEXT
-
-        # Create list of custom handler protocols (both http and https versions)
-        custom_handler_protocols = [f"{method}://" for method in SUPPORTED_HTTP_METHODS]
-        custom_handler_protocols += [f"{method}s://" for method in SUPPORTED_HTTP_METHODS]
-
-        has_custom_handler = any(
-            url.startswith(tuple(custom_handler_protocols))
-            for url in n_object['notification_urls']
-        )
-
-        # If we have custom handlers, use invalid format to prevent conversion
-        # Otherwise use the proper format
-        notify_format = 'raw-no-convert' if has_custom_handler else n_format
+                              'url': url})
 
         apobj.notify(
             title=n_title,
             body=n_body,
-            body_format=notify_format,
+            # `body_format` Tell apprise what format the INPUT is in
+            # &format= in URL Tell apprise what format the OUTPUT should be in (it can convert between)
+            body_format=input_format,
             # False is not an option for AppRise, must be type None
             attach=n_object.get('screenshot', None)
         )
 
-
         # Returns empty string if nothing found, multi-line string otherwise
         log_value = logs.getvalue()
 

changedetectionio/tests/smtp/test_notification_smtp.py

@@ -1,7 +1,4 @@
-import json
-import os
 import time
-import re
 from flask import url_for
 from email import message_from_string
 from email.policy import default as email_policy
@@ -10,9 +7,9 @@ from changedetectionio.diff import REMOVED_STYLE, ADDED_STYLE
 from changedetectionio.tests.util import set_original_response, set_modified_response, set_more_modified_response, live_server_setup, \
     wait_for_all_checks, \
     set_longer_modified_response, delete_all_watches
-from changedetectionio.tests.util import extract_UUID_from_client
+
 import logging
-import base64
+
 
 # NOTE - RELIES ON mailserver as hostname running, see github build recipes
 smtp_test_server = 'mailserver'
@@ -124,7 +121,7 @@ def test_check_notification_plaintext_format(client, live_server, measure_memory
         data={"application-notification_urls": notification_url,
               "application-notification_title": "fallback-title " + default_notification_title,
               "application-notification_body": "some text\n" + default_notification_body,
-              "application-notification_format": 'Text',
+              "application-notification_format": 'Plain Text',
               "requests-time_between_check-minutes": 180,
               'application-fetch_backend': "html_requests"},
         follow_redirects=True
@@ -134,18 +131,11 @@ def test_check_notification_plaintext_format(client, live_server, measure_memory
 
     # Add a watch and trigger a HTTP POST
     test_url = url_for('test_endpoint', _external=True)
-    res = client.post(
-        url_for("ui.ui_views.form_quick_watch_add"),
-        data={"url": test_url, "tags": 'nice one'},
-        follow_redirects=True
-    )
-
-    assert b"Watch added" in res.data
-
-    wait_for_all_checks(client)
-    set_longer_modified_response()
+    uuid = client.application.config.get('DATASTORE').add_watch(url=test_url)
+    client.get(url_for("ui.form_watch_checknow"), follow_redirects=True)
     time.sleep(2)
 
+    set_longer_modified_response()
     client.get(url_for("ui.form_watch_checknow"), follow_redirects=True)
     wait_for_all_checks(client)
 
@@ -241,9 +231,76 @@ def test_check_notification_html_color_format(client, live_server, measure_memor
     assert 'some text<br>' in html_content
     delete_all_watches(client)
 
+def test_check_notification_markdown_format(client, live_server, measure_memory_usage):
+    set_original_response()
+
+    notification_url = f'mailto://changedetection@{smtp_test_server}:11025/?to=fff@home.com'
+
+    #####################
+    # Set this up for when we remove the notification from the watch, it should fallback with these details
+    res = client.post(
+        url_for("settings.settings_page"),
+        data={"application-notification_urls": notification_url,
+              "application-notification_title": "fallback-title " + default_notification_title,
+              "application-notification_body": "*header*\n\nsome text\n" + default_notification_body,
+              "application-notification_format": 'Markdown to HTML',
+              "requests-time_between_check-minutes": 180,
+              'application-fetch_backend': "html_requests"},
+        follow_redirects=True
+    )
+
+    assert b"Settings updated." in res.data
+
+    # Add a watch and trigger a HTTP POST
+    test_url = url_for('test_endpoint', _external=True)
+    res = client.post(
+        url_for("ui.ui_views.form_quick_watch_add"),
+        data={"url": test_url, "tags": 'nice one'},
+        follow_redirects=True
+    )
+
+    assert b"Watch added" in res.data
+
+    wait_for_all_checks(client)
+    set_longer_modified_response()
+    time.sleep(2)
+
+    client.get(url_for("ui.form_watch_checknow"), follow_redirects=True)
+    wait_for_all_checks(client)
+
+    time.sleep(3)
+
+    msg_raw = get_last_message_from_smtp_server()
+    assert len(msg_raw) >= 1
+
+    # Parse the email properly using Python's email library
+    msg = message_from_string(msg_raw, policy=email_policy)
+
+    # The email should have two bodies (multipart/alternative with text/plain and text/html)
+    assert msg.is_multipart()
+    assert msg.get_content_type() == 'multipart/alternative'
+
+    # Get the parts
+    parts = list(msg.iter_parts())
+    assert len(parts) == 2
+
+    # First part should be text/plain (the auto-generated plaintext version)
+    text_part = parts[0]
+    assert text_part.get_content_type() == 'text/plain'
+    text_content = text_part.get_content()
+    assert '(added) So let\'s see what happens.\r\n' in text_content  # The plaintext part
+
+
+    # Second part should be text/html and roughly converted from markdown to HTML
+    html_part = parts[1]
+    assert html_part.get_content_type() == 'text/html'
+    html_content = html_part.get_content()
+    assert '<p><em>header</em></p>' in html_content
+    assert '(added) So let\'s see what happens.<br' in html_content
+    delete_all_watches(client)
+
 
 def test_check_notification_email_formats_default_Text_override_HTML(client, live_server, measure_memory_usage):
-    ##  live_server_setup(live_server) # Setup on conftest per function
 
     # HTML problems? see this
     # https://github.com/caronc/apprise/issues/633
@@ -269,7 +326,7 @@ def test_check_notification_email_formats_default_Text_override_HTML(client, liv
         data={"application-notification_urls": notification_url,
               "application-notification_title": "fallback-title " + default_notification_title,
               "application-notification_body": notification_body,
-              "application-notification_format": 'Text',
+              "application-notification_format": 'Plain Text',
               "requests-time_between_check-minutes": 180,
               'application-fetch_backend': "html_requests"},
         follow_redirects=True

changedetectionio/tests/test_filter_exist_changes.py

@@ -86,7 +86,7 @@ def test_filter_doesnt_exist_then_exists_should_get_notification(client, live_se
                                                    "Diff Full: {{diff_full}}\n"
                                                    "Diff as Patch: {{diff_patch}}\n"
                                                    ":-)",
-                              "notification_format": "Text"}
+                              "notification_format": 'Plain Text'}
 
     notification_form_data.update({
         "url": test_url,

changedetectionio/tests/test_filter_failure_notification.py

@@ -63,7 +63,7 @@ def run_filter_test(client, live_server, content_filter, app_notification_format
                                        "Diff Full: {{diff_full}}\n"
                                        "Diff as Patch: {{diff_patch}}\n"
                                        ":-)",
-                  "notification_format": "Text",
+                  "notification_format": 'Plain Text',
                   "fetch_backend": "html_requests",
                   "filter_failure_notification_send": 'y',
                   "time_between_check_use_default": "y",
@@ -177,7 +177,7 @@ def test_check_include_filters_failure_notification(client, live_server, measure
     #   #  live_server_setup(live_server) # Setup on conftest per function
     run_filter_test(client=client, live_server=live_server, content_filter='#nope-doesnt-exist', app_notification_format=valid_notification_formats.get('HTML Color'))
     # Check markup send conversion didnt affect plaintext preference
-    run_filter_test(client=client, live_server=live_server, content_filter='#nope-doesnt-exist', app_notification_format=valid_notification_formats.get('Text'))
+    run_filter_test(client=client, live_server=live_server, content_filter='#nope-doesnt-exist', app_notification_format=valid_notification_formats.get('Plain Text'))
 
 def test_check_xpath_filter_failure_notification(client, live_server, measure_memory_usage):
     #   #  live_server_setup(live_server) # Setup on conftest per function

changedetectionio/tests/test_group.py

@@ -195,7 +195,7 @@ def test_group_tag_notification(client, live_server, measure_memory_usage):
                                                    "Diff as Patch: {{diff_patch}}\n"
                                                    ":-)",
                               "notification_screenshot": True,
-                              "notification_format": "Text",
+                              "notification_format": 'Plain Text',
                               "title": "test-tag"}
 
     res = client.post(

changedetectionio/tests/test_notification.py

@@ -101,7 +101,7 @@ def test_check_notification(client, live_server, measure_memory_usage):
                                                    "Diff as Patch: {{diff_patch}}\n"
                                                    ":-)",
                               "notification_screenshot": True,
-                              "notification_format": "Text"}
+                              "notification_format": 'Plain Text'}
 
     notification_form_data.update({
         "url": test_url,
@@ -267,7 +267,7 @@ def test_notification_validation(client, live_server, measure_memory_usage):
 #        data={"notification_urls": 'json://localhost/foobar',
 #              "notification_title": "",
 #              "notification_body": "",
-#              "notification_format": "Text",
+#              "notification_format": 'Plain Text',
 #              "url": test_url,
 #              "tag": "my tag",
 #              "title": "my title",
@@ -383,7 +383,7 @@ def test_notification_custom_endpoint_and_jinja2(client, live_server, measure_me
         assert 'second: hello world "space"' in notification_headers.lower()
 
 
-    # Should always be automatically detected as JSON content type even when we set it as 'Text' (default)
+    # Should always be automatically detected as JSON content type even when we set it as 'Plain Text' (default)
     assert os.path.isfile("test-datastore/notification-content-type.txt")
     with open("test-datastore/notification-content-type.txt", 'r') as f:
         assert 'application/json' in f.read()

changedetectionio/tests/test_notification_errors.py

@@ -30,7 +30,7 @@ def test_check_notification_error_handling(client, live_server, measure_memory_u
         data={"notification_urls": f"{broken_notification_url}\r\n{working_notification_url}",
               "notification_title": "xxx",
               "notification_body": "xxxxx",
-              "notification_format": "Text",
+              "notification_format": 'Plain Text',
               "url": test_url,
               "tags": "",
               "title": "",