diff --git a/changedetectionio/notification/__init__.py b/changedetectionio/notification/__init__.py
index 8f6bd81c..06ed830a 100644
--- a/changedetectionio/notification/__init__.py
+++ b/changedetectionio/notification/__init__.py
@@ -7,10 +7,10 @@ default_notification_title = 'ChangeDetection.io Notification - {{watch_url}}'
# The values (markdown etc) are from apprise NotifyFormat,
# But to avoid importing the whole heavy module just use the same strings here.
valid_notification_formats = {
- 'Text': 'text',
- 'Markdown': 'markdown',
+ 'Plain Text': 'text',
'HTML': 'html',
'HTML Color': 'htmlcolor',
+ 'Markdown to HTML': 'markdown',
# Used only for editing a watch (not for global)
default_notification_format_for_watch: default_notification_format_for_watch
}
diff --git a/changedetectionio/notification/handler.py b/changedetectionio/notification/handler.py
index d5c6edc3..948e4558 100644
--- a/changedetectionio/notification/handler.py
+++ b/changedetectionio/notification/handler.py
@@ -53,6 +53,7 @@ def notification_format_align_with_apprise(n_format : str):
"""
Correctly align changedetection's formats with apprise's formats
Probably these are the same - but good to be sure.
+ These set the expected OUTPUT format type
:param n_format:
:return:
"""
@@ -71,12 +72,63 @@ def notification_format_align_with_apprise(n_format : str):
return n_format
+
+def apply_service_tweaks(url, n_body, n_title):
+ # Re 323 - Limit discord length to their 2000 char limit total or it wont send.
+ # Because different notifications may require different pre-processing, run each sequentially :(
+ # 2000 bytes minus -
+ # 200 bytes for the overhead of the _entire_ json payload, 200 bytes for {tts, wait, content} etc headers
+ # Length of URL - Incase they specify a longer custom avatar_url
+
+ # So if no avatar_url is specified, add one so it can be correctly calculated into the total payload
+ parsed = urlparse(url)
+ k = '?' if not parsed.query else '&'
+ if not 'avatar_url' in url \
+ and not url.startswith('mail') \
+ and not url.startswith('post') \
+ and not url.startswith('get') \
+ and not url.startswith('delete') \
+ and not url.startswith('put'):
+ url += k + f"avatar_url={APPRISE_AVATAR_URL}"
+
+ if url.startswith('tgram://'):
+ # Telegram only supports a limit subset of HTML, remove the '
' we place in.
+ # re https://github.com/dgtlmoon/changedetection.io/issues/555
+ # @todo re-use an existing library we have already imported to strip all non-allowed tags
+ n_body = n_body.replace('
', '\n')
+ n_body = n_body.replace('', '\n')
+ # real limit is 4096, but minus some for extra metadata
+ payload_max_size = 3600
+ body_limit = max(0, payload_max_size - len(n_title))
+ n_title = n_title[0:payload_max_size]
+ n_body = n_body[0:body_limit]
+
+ elif url.startswith('discord://') or url.startswith('https://discordapp.com/api/webhooks') or url.startswith(
+ 'https://discord.com/api'):
+ # real limit is 2000, but minus some for extra metadata
+ payload_max_size = 1700
+ body_limit = max(0, payload_max_size - len(n_title))
+ n_title = n_title[0:payload_max_size]
+ n_body = n_body[0:body_limit]
+
+ return url, n_body, n_title
+
+
def process_notification(n_object: NotificationContextData, datastore):
from changedetectionio.jinja2_custom import render as jinja_render
from . import default_notification_format_for_watch, default_notification_format, valid_notification_formats
# be sure its registered
from .apprise_plugin.custom_handlers import apprise_http_custom_handler
+ # Create list of custom handler protocols (both http and https versions)
+ custom_handler_protocols = [f"{method}://" for method in SUPPORTED_HTTP_METHODS]
+ custom_handler_protocols += [f"{method}s://" for method in SUPPORTED_HTTP_METHODS]
+
+ has_custom_handler = any(
+ url.startswith(tuple(custom_handler_protocols))
+ for url in n_object['notification_urls']
+ )
+
if not isinstance(n_object, NotificationContextData):
raise TypeError(f"Expected NotificationContextData, got {type(n_object)}")
@@ -87,20 +139,25 @@ def process_notification(n_object: NotificationContextData, datastore):
# Insert variables into the notification content
notification_parameters = create_notification_parameters(n_object, datastore)
- n_format = valid_notification_formats.get(
+ requested_output_format = valid_notification_formats.get(
n_object.get('notification_format', default_notification_format),
valid_notification_formats[default_notification_format],
)
# If we arrived with 'System default' then look it up
- if n_format == default_notification_format_for_watch and datastore.data['settings']['application'].get('notification_format') != default_notification_format_for_watch:
+ if requested_output_format == default_notification_format_for_watch and datastore.data['settings']['application'].get('notification_format') != default_notification_format_for_watch:
# Initially text or whatever
- n_format = datastore.data['settings']['application'].get('notification_format', valid_notification_formats[default_notification_format]).lower()
+ requested_output_format = datastore.data['settings']['application'].get('notification_format', valid_notification_formats[default_notification_format]).lower()
- n_format = notification_format_align_with_apprise(n_format=n_format)
+ requested_output_format = notification_format_align_with_apprise(n_format=requested_output_format)
logger.trace(f"Complete notification body including Jinja and placeholders calculated in {time.time() - now:.2f}s")
+ # If we have custom handlers, use invalid format to prevent conversion
+ # Otherwise use the proper format
+ if has_custom_handler:
+ input_format = 'raw-no-convert'
+
# https://github.com/caronc/apprise/wiki/Development_LogCapture
# Anything higher than or equal to WARNING (which covers things like Connection errors)
# raise it as an exception
@@ -117,6 +174,8 @@ def process_notification(n_object: NotificationContextData, datastore):
with apprise.LogCapture(level=apprise.logging.DEBUG) as logs:
for url in n_object['notification_urls']:
+ parsed_url = urlparse(url)
+ prefix_add_to_url = '?' if not parsed_url.query else '&'
# Get the notification body from datastore
n_body = jinja_render(template_str=n_object.get('notification_body', ''), **notification_parameters)
@@ -124,8 +183,32 @@ def process_notification(n_object: NotificationContextData, datastore):
if n_object.get('markup_text_to_html'):
n_body = markup_text_links_to_html(body=n_body)
- if n_format == NotifyFormat.HTML.value:
+ # This actually means we request "Markdown to HTML"
+ if requested_output_format == NotifyFormat.MARKDOWN.value:
+ output_format = NotifyFormat.HTML.value
+ input_format = NotifyFormat.MARKDOWN.value
+ if not 'format=' in url.lower():
+ url = f"{url}{prefix_add_to_url}format={output_format}"
+
+ # Deviation from apprise.
+ # No conversion, its like they want to send raw HTML but we add linebreaks
+ elif requested_output_format == NotifyFormat.HTML.value:
+ # same in and out means apprise wont try to convert
+ input_format = output_format = NotifyFormat.HTML.value
n_body = n_body.replace("\n", '
')
+ if not 'format=' in url.lower():
+ url = f"{url}{prefix_add_to_url}format={output_format}"
+
+ else:
+ # Nothing to be done, leave it as plaintext
+ # `body_format` Tell apprise what format the INPUT is in
+ # &format= in URL Tell apprise what format the OUTPUT should be in (it can convert between)
+ input_format = output_format = NotifyFormat.TEXT.value
+ if not 'format=' in url.lower():
+ url = f"{url}{prefix_add_to_url}format={output_format}"
+
+ if has_custom_handler:
+ input_format='raw-no-convert'
n_title = jinja_render(template_str=n_object.get('notification_title', ''), **notification_parameters)
@@ -141,88 +224,24 @@ def process_notification(n_object: NotificationContextData, datastore):
logger.info(f">> Process Notification: AppRise notifying {url}")
url = jinja_render(template_str=url, **notification_parameters)
- # Re 323 - Limit discord length to their 2000 char limit total or it wont send.
- # Because different notifications may require different pre-processing, run each sequentially :(
- # 2000 bytes minus -
- # 200 bytes for the overhead of the _entire_ json payload, 200 bytes for {tts, wait, content} etc headers
- # Length of URL - Incase they specify a longer custom avatar_url
-
- # So if no avatar_url is specified, add one so it can be correctly calculated into the total payload
- parsed = urlparse(url)
- k = '?' if not parsed.query else '&'
- if not 'avatar_url' in url \
- and not url.startswith('mail') \
- and not url.startswith('post') \
- and not url.startswith('get') \
- and not url.startswith('delete') \
- and not url.startswith('put'):
- url += k + f"avatar_url={APPRISE_AVATAR_URL}"
-
- if url.startswith('tgram://'):
- # Telegram only supports a limit subset of HTML, remove the '
' we place in.
- # re https://github.com/dgtlmoon/changedetection.io/issues/555
- # @todo re-use an existing library we have already imported to strip all non-allowed tags
- n_body = n_body.replace('
', '\n')
- n_body = n_body.replace('', '\n')
- # real limit is 4096, but minus some for extra metadata
- payload_max_size = 3600
- body_limit = max(0, payload_max_size - len(n_title))
- n_title = n_title[0:payload_max_size]
- n_body = n_body[0:body_limit]
-
- elif url.startswith('discord://') or url.startswith('https://discordapp.com/api/webhooks') or url.startswith(
- 'https://discord.com/api'):
- # real limit is 2000, but minus some for extra metadata
- payload_max_size = 1700
- body_limit = max(0, payload_max_size - len(n_title))
- n_title = n_title[0:payload_max_size]
- n_body = n_body[0:body_limit]
-
- # Add format parameter to mailto URLs to ensure proper text/html handling
- # https://github.com/caronc/apprise/issues/633#issuecomment-1191449321
- # Note: Custom handlers (post://, get://, etc.) don't need this as we handle them
- # differently by passing an invalid body_format to prevent HTML conversion
- if not 'format=' in url and url.startswith(('mailto', 'mailtos')):
- parsed = urlparse(url)
- prefix = '?' if not parsed.query else '&'
- # Apprise format is already lowercase from notification_format_align_with_apprise()
- url = f"{url}{prefix}format={n_format}"
+ (url, n_body, n_title) = apply_service_tweaks(url=url, n_body=n_body, n_title=n_title)
apobj.add(url)
sent_objs.append({'title': n_title,
'body': n_body,
- 'url': url,
- 'body_format': n_format})
-
- # Blast off the notifications tht are set in .add()
- # Check if we have any custom HTTP handlers (post://, get://, etc.)
- # These handlers created with @notify decorator don't handle format conversion properly
- # and will strip HTML if we pass a valid format. So we pass an invalid format string
- # to prevent Apprise from converting HTML->TEXT
-
- # Create list of custom handler protocols (both http and https versions)
- custom_handler_protocols = [f"{method}://" for method in SUPPORTED_HTTP_METHODS]
- custom_handler_protocols += [f"{method}s://" for method in SUPPORTED_HTTP_METHODS]
-
- has_custom_handler = any(
- url.startswith(tuple(custom_handler_protocols))
- for url in n_object['notification_urls']
- )
-
- # If we have custom handlers, use invalid format to prevent conversion
- # Otherwise use the proper format
- notify_format = 'raw-no-convert' if has_custom_handler else n_format
+ 'url': url})
apobj.notify(
title=n_title,
body=n_body,
- body_format=notify_format,
+ # `body_format` Tell apprise what format the INPUT is in
+ # &format= in URL Tell apprise what format the OUTPUT should be in (it can convert between)
+ body_format=input_format,
# False is not an option for AppRise, must be type None
attach=n_object.get('screenshot', None)
)
-
# Returns empty string if nothing found, multi-line string otherwise
log_value = logs.getvalue()
diff --git a/changedetectionio/tests/smtp/test_notification_smtp.py b/changedetectionio/tests/smtp/test_notification_smtp.py
index 6f897e17..9c60433d 100644
--- a/changedetectionio/tests/smtp/test_notification_smtp.py
+++ b/changedetectionio/tests/smtp/test_notification_smtp.py
@@ -1,7 +1,4 @@
-import json
-import os
import time
-import re
from flask import url_for
from email import message_from_string
from email.policy import default as email_policy
@@ -10,9 +7,9 @@ from changedetectionio.diff import REMOVED_STYLE, ADDED_STYLE
from changedetectionio.tests.util import set_original_response, set_modified_response, set_more_modified_response, live_server_setup, \
wait_for_all_checks, \
set_longer_modified_response, delete_all_watches
-from changedetectionio.tests.util import extract_UUID_from_client
+
import logging
-import base64
+
# NOTE - RELIES ON mailserver as hostname running, see github build recipes
smtp_test_server = 'mailserver'
@@ -124,7 +121,7 @@ def test_check_notification_plaintext_format(client, live_server, measure_memory
data={"application-notification_urls": notification_url,
"application-notification_title": "fallback-title " + default_notification_title,
"application-notification_body": "some text\n" + default_notification_body,
- "application-notification_format": 'Text',
+ "application-notification_format": 'Plain Text',
"requests-time_between_check-minutes": 180,
'application-fetch_backend': "html_requests"},
follow_redirects=True
@@ -134,18 +131,11 @@ def test_check_notification_plaintext_format(client, live_server, measure_memory
# Add a watch and trigger a HTTP POST
test_url = url_for('test_endpoint', _external=True)
- res = client.post(
- url_for("ui.ui_views.form_quick_watch_add"),
- data={"url": test_url, "tags": 'nice one'},
- follow_redirects=True
- )
-
- assert b"Watch added" in res.data
-
- wait_for_all_checks(client)
- set_longer_modified_response()
+ uuid = client.application.config.get('DATASTORE').add_watch(url=test_url)
+ client.get(url_for("ui.form_watch_checknow"), follow_redirects=True)
time.sleep(2)
+ set_longer_modified_response()
client.get(url_for("ui.form_watch_checknow"), follow_redirects=True)
wait_for_all_checks(client)
@@ -241,9 +231,76 @@ def test_check_notification_html_color_format(client, live_server, measure_memor
assert 'some text
' in html_content
delete_all_watches(client)
+def test_check_notification_markdown_format(client, live_server, measure_memory_usage):
+ set_original_response()
+
+ notification_url = f'mailto://changedetection@{smtp_test_server}:11025/?to=fff@home.com'
+
+ #####################
+ # Set this up for when we remove the notification from the watch, it should fallback with these details
+ res = client.post(
+ url_for("settings.settings_page"),
+ data={"application-notification_urls": notification_url,
+ "application-notification_title": "fallback-title " + default_notification_title,
+ "application-notification_body": "*header*\n\nsome text\n" + default_notification_body,
+ "application-notification_format": 'Markdown to HTML',
+ "requests-time_between_check-minutes": 180,
+ 'application-fetch_backend': "html_requests"},
+ follow_redirects=True
+ )
+
+ assert b"Settings updated." in res.data
+
+ # Add a watch and trigger a HTTP POST
+ test_url = url_for('test_endpoint', _external=True)
+ res = client.post(
+ url_for("ui.ui_views.form_quick_watch_add"),
+ data={"url": test_url, "tags": 'nice one'},
+ follow_redirects=True
+ )
+
+ assert b"Watch added" in res.data
+
+ wait_for_all_checks(client)
+ set_longer_modified_response()
+ time.sleep(2)
+
+ client.get(url_for("ui.form_watch_checknow"), follow_redirects=True)
+ wait_for_all_checks(client)
+
+ time.sleep(3)
+
+ msg_raw = get_last_message_from_smtp_server()
+ assert len(msg_raw) >= 1
+
+ # Parse the email properly using Python's email library
+ msg = message_from_string(msg_raw, policy=email_policy)
+
+ # The email should have two bodies (multipart/alternative with text/plain and text/html)
+ assert msg.is_multipart()
+ assert msg.get_content_type() == 'multipart/alternative'
+
+ # Get the parts
+ parts = list(msg.iter_parts())
+ assert len(parts) == 2
+
+ # First part should be text/plain (the auto-generated plaintext version)
+ text_part = parts[0]
+ assert text_part.get_content_type() == 'text/plain'
+ text_content = text_part.get_content()
+ assert '(added) So let\'s see what happens.\r\n' in text_content # The plaintext part
+
+
+ # Second part should be text/html and roughly converted from markdown to HTML
+ html_part = parts[1]
+ assert html_part.get_content_type() == 'text/html'
+ html_content = html_part.get_content()
+ assert '
header
' in html_content + assert '(added) So let\'s see what happens.