diff --git a/changedetectionio/blueprint/settings/templates/settings.html b/changedetectionio/blueprint/settings/templates/settings.html index 096d7a3c..a6114830 100644 --- a/changedetectionio/blueprint/settings/templates/settings.html +++ b/changedetectionio/blueprint/settings/templates/settings.html @@ -44,10 +44,6 @@ -
- {{ render_field(form.requests.form.jitter_seconds, class="jitter_seconds") }} - Example - 3 seconds random jitter could trigger up to 3 seconds earlier or up to 3 seconds later -
{{ render_field(form.application.form.filter_failure_notification_threshold_attempts, class="filter_failure_notification_threshold_attempts") }} After this many consecutive times that the CSS/xPath filter is missing, send a notification @@ -134,6 +130,10 @@ Number of concurrent workers to process watches. More workers = faster processing but higher memory usage.
Currently running: {{ worker_info.count }} operational {{ worker_info.type }} workers{% if worker_info.active_workers > 0 %} ({{ worker_info.active_workers }} actively processing){% endif %}.
+
+ {{ render_field(form.requests.form.jitter_seconds, class="jitter_seconds") }} + Example - 3 seconds random jitter could trigger up to 3 seconds earlier or up to 3 seconds later +
{{ render_field(form.requests.form.default_ua) }} diff --git a/changedetectionio/blueprint/ui/notification.py b/changedetectionio/blueprint/ui/notification.py index 2d6bf6bd..9299d60f 100644 --- a/changedetectionio/blueprint/ui/notification.py +++ b/changedetectionio/blueprint/ui/notification.py @@ -79,6 +79,7 @@ def construct_blueprint(datastore: ChangeDetectionStore): if not notification_urls: logger.debug("Test notification - Trying by group/tag in the edit form if available") + # @todo this logic is not clear, omegaconf? # On an edit page, we should also fire off to the tags if they have notifications if request.form.get('tags') and request.form['tags'].strip(): for k in request.form['tags'].split(','): @@ -92,7 +93,7 @@ def construct_blueprint(datastore: ChangeDetectionStore): notification_urls = datastore.data['settings']['application']['notification_urls'] if not notification_urls: - return 'Error: No Notification URLs set/found' + return make_response("Error: No Notification URLs set/found.", 400) for n_url in notification_urls: if len(n_url.strip()): diff --git a/changedetectionio/notification/handler.py b/changedetectionio/notification/handler.py index 54049ff2..7b972943 100644 --- a/changedetectionio/notification/handler.py +++ b/changedetectionio/notification/handler.py @@ -35,7 +35,11 @@ def _populate_notification_tokens(n_object, datastore): # Add text that was triggered if len(dates): - snapshot_contents = str(escape(watch.get_history_snapshot(dates[-1]))) + snapshot_contents = watch.get_history_snapshot(dates[-1]) + + if n_object.get('notification_format').lower().startswith('html'): + snapshot_contents = str(escape(snapshot_contents)) + else: snapshot_contents = "No snapshot/history available, the watch should fetch atleast once." @@ -44,18 +48,15 @@ def _populate_notification_tokens(n_object, datastore): n_object['notification_format'] = datastore.data['settings']['application'].get('notification_format') html_colour_enable = False + line_feed_sep = "\n" + # HTML needs linebreak, but MarkDown and Text can use a linefeed - if n_object.get('notification_format') == 'HTML': - line_feed_sep = "
" - # Snapshot will be plaintext on the disk, convert to some kind of HTML - snapshot_contents = snapshot_contents.replace('\n', line_feed_sep) - elif n_object.get('notification_format') == 'HTML Color': + if n_object.get('notification_format').lower().startswith('html'): line_feed_sep = "
" # Snapshot will be plaintext on the disk, convert to some kind of HTML snapshot_contents = snapshot_contents.replace('\n', line_feed_sep) + if n_object.get('notification_format') == 'HTML Color': html_colour_enable = True - else: - line_feed_sep = "\n" triggered_text = '' if len(trigger_text): @@ -69,8 +70,12 @@ def _populate_notification_tokens(n_object, datastore): current_snapshot = "Example text: example test\nExample text: More than 1 watch change needs to exist to build a nice preview!" if len(dates) > 1: - prev_snapshot = str(escape(watch.get_history_snapshot(dates[-2]))) - current_snapshot = str(escape(watch.get_history_snapshot(dates[-1]))) + prev_snapshot = watch.get_history_snapshot(dates[-2]) + current_snapshot = watch.get_history_snapshot(dates[-1]) + if n_object.get('notification_format').lower().startswith('html'): + prev_snapshot = str(escape(prev_snapshot)) + current_snapshot = str(escape(current_snapshot)) + if watch: v = {'url': watch.get('url'), 'label': watch.label} @@ -182,8 +187,10 @@ def process_notification(n_object, datastore): # Get the notification body from datastore n_body = jinja_render(template_str=n_object.get('notification_body', ''), **notification_parameters) + # hmm unsure about this, but why not if n_object.get('notification_format', '').startswith('HTML'): n_body = n_body.replace("\n", '
') + n_title = jinja_render(template_str=n_object.get('notification_title', ''), **notification_parameters) n_body_from_file_template = scan_notification_file_templates(url=url, @@ -238,7 +245,7 @@ def process_notification(n_object, datastore): # Apprise will default to HTML, so we need to override it # So that whats' generated in n_body is in line with what is going to be sent. # https://github.com/caronc/apprise/issues/633#issuecomment-1191449321 - if not 'format=' in url and (n_format == 'Text' or n_format == 'Markdown'): + if not 'format=' in url and (n_format.lower() == 'text' or n_format.lower() == 'markdown'): prefix = '?' if not '?' in url else '&' # Apprise format is lowercase text https://github.com/caronc/apprise/issues/633 n_format = n_format.lower() diff --git a/changedetectionio/static/js/notifications.js b/changedetectionio/static/js/notifications.js index 9ea8ac0a..61064ccc 100644 --- a/changedetectionio/static/js/notifications.js +++ b/changedetectionio/static/js/notifications.js @@ -50,6 +50,8 @@ $(document).ready(function () { function setPreview(data) { const iframe = document.getElementById("notification-iframe"); const isDark = document.documentElement.getAttribute('data-darkmode') === 'true'; + const isTextFormat = $('select.notification-format').val() === 'Text'; + $('#notification-preview-title-text').text(data['title']); iframe.srcdoc = ` @@ -76,9 +78,15 @@ $(document).ready(function () { color: var(--color-text); padding: 5px; } + body.text-format { + font-family: monospace; + white-space: pre; + overflow-wrap: normal; + overflow-x: auto; + } - ${data['body']} + ${data['body']} `; } diff --git a/changedetectionio/templates/_common_fields.html b/changedetectionio/templates/_common_fields.html index 53b1adcd..bd4c6b47 100644 --- a/changedetectionio/templates/_common_fields.html +++ b/changedetectionio/templates/_common_fields.html @@ -159,11 +159,15 @@ Format for all notifications
- diff --git a/changedetectionio/tests/smtp/test_notification_smtp.py b/changedetectionio/tests/smtp/test_notification_smtp.py index a400901c..d2b076ef 100644 --- a/changedetectionio/tests/smtp/test_notification_smtp.py +++ b/changedetectionio/tests/smtp/test_notification_smtp.py @@ -1,17 +1,17 @@ -import json import os import time -import re from flask import url_for from changedetectionio.tests.util import set_original_response, set_modified_response, set_more_modified_response, live_server_setup, \ wait_for_all_checks, \ set_longer_modified_response -from changedetectionio.tests.util import extract_UUID_from_client import logging -import base64 # NOTE - RELIES ON mailserver as hostname running, see github build recipes -smtp_test_server = 'mailserver' +# Should be hostname (never IP), looks for our test mailserver that repeats the content +# python3 changedetectionio/tests/smtp/smtp-test-server.py & +# mailserver=localhost pytest tests/smtp/test_notification_smtp.py::test_check_notification_email_formats_default_HTML +smtp_test_server = os.getenv('mailserver', 'mailserver') + from changedetectionio.notification import ( default_notification_body, @@ -20,7 +20,35 @@ from changedetectionio.notification import ( valid_notification_formats, ) +from email import policy +from email.parser import BytesParser, Parser +def parse_mime(raw): + """Return (EmailMessage, dict[str, list[str]] bodies by content-type).""" + if isinstance(raw, (bytes, bytearray)): + msg = BytesParser(policy=policy.default).parsebytes(raw) + else: + msg = Parser(policy=policy.default).parsestr(raw) + + parts_by_type = {} + if msg.is_multipart(): + for part in msg.walk(): + if part.get_content_maintype() == "multipart": + continue + ctype = part.get_content_type() # e.g. "text/plain" + text = part.get_content() # decoded str + parts_by_type.setdefault(ctype, []).append(text) + else: + parts_by_type.setdefault(msg.get_content_type(), []).append(msg.get_content()) + + return msg, parts_by_type + +def one_or_join(parts_dict, ctype): + """Join multiple parts of the same type (rare but possible).""" + return "\n".join(parts_dict.get(ctype, [])) + +def norm_newlines(s: str) -> str: + return s.replace("\r\n", "\n").replace("\r", "\n") def get_last_message_from_smtp_server(): import socket @@ -77,37 +105,36 @@ def test_check_notification_email_formats_default_HTML(client, live_server, meas time.sleep(3) - msg = get_last_message_from_smtp_server() - assert len(msg) >= 1 + raw = get_last_message_from_smtp_server() + assert raw # not empty + + msg, bodies = parse_mime(raw) + + plain = norm_newlines(one_or_join(bodies, "text/plain")) + html = norm_newlines(one_or_join(bodies, "text/html")) + + # Now assert against the decoded bodies + assert "(added) So let's see what happens.\n" in plain # plaintext uses a literal apostrophe + assert "(added) So let's see what happens.
" in html # html uses ' and
+ + # You can also check counts, boundaries, etc. + assert html.count("So let's see what happens.") == 3 + assert "modified head title had a change." in plain + assert "modified head title had a change.
" in html + + - # The email should have two bodies, and the text/html part should be
- assert 'Content-Type: text/plain' in msg - assert '(added) So let\'s see what happens.\r\n' in msg # The plaintext part with \r\n - assert 'Content-Type: text/html' in msg - assert '(added) So let\'s see what happens.
' in msg # the html part res = client.get(url_for("ui.form_delete", uuid="all"), follow_redirects=True) assert b'Deleted' in res.data def test_check_notification_email_formats_default_Text_override_HTML(client, live_server, measure_memory_usage): - ## live_server_setup(live_server) # Setup on conftest per function # HTML problems? see this # https://github.com/caronc/apprise/issues/633 - set_original_response() notification_url = f'mailto://changedetection@{smtp_test_server}:11025/?to=fff@home.com' - notification_body = f""" - - - My Webpage - - -

Test

- {default_notification_body} - - -""" + notification_body = f"""{default_notification_body}""" ##################### # Set this up for when we remove the notification from the watch, it should fallback with these details @@ -116,11 +143,12 @@ def test_check_notification_email_formats_default_Text_override_HTML(client, liv data={"application-notification_urls": notification_url, "application-notification_title": "fallback-title " + default_notification_title, "application-notification_body": notification_body, - "application-notification_format": 'Text', + "application-notification_format": 'Text', # handler.py should be sure to add &format=text to override default html from apprise "requests-time_between_check-minutes": 180, 'application-fetch_backend': "html_requests"}, follow_redirects=True ) + assert b"Settings updated." in res.data # Add a watch and trigger a HTTP POST @@ -140,43 +168,56 @@ def test_check_notification_email_formats_default_Text_override_HTML(client, liv wait_for_all_checks(client) time.sleep(3) - msg = get_last_message_from_smtp_server() - assert len(msg) >= 1 - # with open('/tmp/m.txt', 'w') as f: - # f.write(msg) + raw = get_last_message_from_smtp_server() + assert raw - # The email should not have two bodies, should be TEXT only + msg, bodies = parse_mime(raw) - assert 'Content-Type: text/plain' in msg - assert '(added) So let\'s see what happens.\r\n' in msg # The plaintext part with \r\n + plain = norm_newlines(one_or_join(bodies, "text/plain")) + html = norm_newlines(one_or_join(bodies, "text/html")) + assert not html # should be no HTML here + # Expect ONLY text/plain body + assert "text/plain" in bodies + assert "text/html" not in bodies + + # Assert on decoded plaintext (literal apostrophe, not ') + # Should be NO markup when in text mode + assert "(added) So let's see what happens.\n" in plain + + + # ---------- Flip to HTML format, then expect multipart with both ---------- set_original_response() - # Now override as HTML format res = client.post( url_for("ui.ui_edit.edit_page", uuid="first"), - data={ - "url": test_url, - "notification_format": 'HTML', - 'fetch_backend': "html_requests"}, - follow_redirects=True + data={"url": test_url, + "notification_format": "HTML", + "fetch_backend": "html_requests", + "time_between_check_use_default": "y"}, + follow_redirects=True, ) + assert b"Updated watch." in res.data wait_for_all_checks(client) time.sleep(3) - msg = get_last_message_from_smtp_server() - assert len(msg) >= 1 - # The email should have two bodies, and the text/html part should be
- assert 'Content-Type: text/plain' in msg - assert '(removed) So let\'s see what happens.\r\n' in msg # The plaintext part with \n - assert 'Content-Type: text/html' in msg - assert '(removed) So let\'s see what happens.
' in msg # the html part + raw = get_last_message_from_smtp_server() + assert raw - # https://github.com/dgtlmoon/changedetection.io/issues/2103 - assert '

Test

' in msg - assert '<' not in msg - assert 'Content-Type: text/html' in msg + msg, bodies = parse_mime(raw) + plain = norm_newlines(one_or_join(bodies, "text/plain")) + html = norm_newlines(one_or_join(bodies, "text/html")) - res = client.get(url_for("ui.form_delete", uuid="all"), follow_redirects=True) - assert b'Deleted' in res.data + # Expect both text/plain and text/html bodies now + assert "text/plain" in bodies + assert "text/html" in bodies + + # Plaintext reflects the removal line (literal apostrophe) + assert "(removed) So let's see what happens.\n" in plain + assert "(removed) So let's see what happens.
" in html + + # Optional: ensure we got multipart/alternative (typical for dual bodies) + if msg.is_multipart(): + # most senders do "multipart/alternative" for text/plain + text/html + assert msg.get_content_subtype() in ("alternative", "mixed", "related")