HTML improvements

changedetectionio/blueprint/settings/templates/settings.html

@@ -44,10 +44,6 @@
                                 </div>
                         </div>
                     </div>
-                    <div class="pure-control-group">
-                        {{ render_field(form.requests.form.jitter_seconds, class="jitter_seconds") }}
-                        <span class="pure-form-message-inline">Example - 3 seconds random jitter could trigger up to 3 seconds earlier or up to 3 seconds later</span>
-                    </div>
                     <div class="pure-control-group">
                         {{ render_field(form.application.form.filter_failure_notification_threshold_attempts, class="filter_failure_notification_threshold_attempts") }}
                         <span class="pure-form-message-inline">After this many consecutive times that the CSS/xPath filter is missing, send a notification
@@ -134,6 +130,10 @@
                     <span class="pure-form-message-inline">Number of concurrent workers to process watches. More workers = faster processing but higher memory usage.<br>
                     Currently running: <strong>{{ worker_info.count }}</strong> operational {{ worker_info.type }} workers{% if worker_info.active_workers > 0 %} ({{ worker_info.active_workers }} actively processing){% endif %}.</span>
                 </div>
+                <div class="pure-control-group">
+                    {{ render_field(form.requests.form.jitter_seconds, class="jitter_seconds") }}
+                    <span class="pure-form-message-inline">Example - 3 seconds random jitter could trigger up to 3 seconds earlier or up to 3 seconds later</span>
+                </div>
                 <div class="pure-control-group inline-radio">
                     {{ render_field(form.requests.form.default_ua) }}
                     <span class="pure-form-message-inline">

changedetectionio/blueprint/ui/notification.py

@@ -79,6 +79,7 @@ def construct_blueprint(datastore: ChangeDetectionStore):
 
         if not notification_urls:
             logger.debug("Test notification - Trying by group/tag in the edit form if available")
+            # @todo this logic is not clear, omegaconf?
             # On an edit page, we should also fire off to the tags if they have notifications
             if request.form.get('tags') and request.form['tags'].strip():
                 for k in request.form['tags'].split(','):
@@ -92,7 +93,7 @@ def construct_blueprint(datastore: ChangeDetectionStore):
                 notification_urls = datastore.data['settings']['application']['notification_urls']
 
         if not notification_urls:
-            return 'Error: No Notification URLs set/found'
+            return make_response("Error: No Notification URLs set/found.", 400)
 
         for n_url in notification_urls:
             if len(n_url.strip()):

changedetectionio/notification/handler.py

@@ -35,7 +35,11 @@ def _populate_notification_tokens(n_object, datastore):
 
     # Add text that was triggered
     if len(dates):
-        snapshot_contents = str(escape(watch.get_history_snapshot(dates[-1])))
+        snapshot_contents = watch.get_history_snapshot(dates[-1])
+
+        if n_object.get('notification_format').lower().startswith('html'):
+            snapshot_contents = str(escape(snapshot_contents))
+
     else:
         snapshot_contents = "No snapshot/history available, the watch should fetch atleast once."
 
@@ -44,18 +48,15 @@ def _populate_notification_tokens(n_object, datastore):
         n_object['notification_format'] = datastore.data['settings']['application'].get('notification_format')
 
     html_colour_enable = False
+    line_feed_sep = "\n"
+
     # HTML needs linebreak, but MarkDown and Text can use a linefeed
-    if n_object.get('notification_format') == 'HTML':
-        line_feed_sep = "<br>"
-        # Snapshot will be plaintext on the disk, convert to some kind of HTML
-        snapshot_contents = snapshot_contents.replace('\n', line_feed_sep)
-    elif n_object.get('notification_format') == 'HTML Color':
+    if n_object.get('notification_format').lower().startswith('html'):
         line_feed_sep = "<br>"
         # Snapshot will be plaintext on the disk, convert to some kind of HTML
         snapshot_contents = snapshot_contents.replace('\n', line_feed_sep)
+    if n_object.get('notification_format') == 'HTML Color':
         html_colour_enable = True
-    else:
-        line_feed_sep = "\n"
 
     triggered_text = ''
     if len(trigger_text):
@@ -69,8 +70,12 @@ def _populate_notification_tokens(n_object, datastore):
     current_snapshot = "Example text: example test\nExample text: More than 1 watch change needs to exist to build a nice preview!"
 
     if len(dates) > 1:
-        prev_snapshot = str(escape(watch.get_history_snapshot(dates[-2])))
-        current_snapshot = str(escape(watch.get_history_snapshot(dates[-1])))
+        prev_snapshot = watch.get_history_snapshot(dates[-2])
+        current_snapshot = watch.get_history_snapshot(dates[-1])
+        if n_object.get('notification_format').lower().startswith('html'):
+            prev_snapshot = str(escape(prev_snapshot))
+            current_snapshot = str(escape(current_snapshot))
+
 
     if watch:
         v = {'url': watch.get('url'), 'label': watch.label}
@@ -182,8 +187,10 @@ def process_notification(n_object, datastore):
 
             # Get the notification body from datastore
             n_body = jinja_render(template_str=n_object.get('notification_body', ''), **notification_parameters)
+            # hmm unsure about this, but why not
             if n_object.get('notification_format', '').startswith('HTML'):
                 n_body = n_body.replace("\n", '<br>')
+
             n_title = jinja_render(template_str=n_object.get('notification_title', ''), **notification_parameters)
 
             n_body_from_file_template = scan_notification_file_templates(url=url,
@@ -238,7 +245,7 @@ def process_notification(n_object, datastore):
                 # Apprise will default to HTML, so we need to override it
                 # So that whats' generated in n_body is in line with what is going to be sent.
                 # https://github.com/caronc/apprise/issues/633#issuecomment-1191449321
-                if not 'format=' in url and (n_format == 'Text' or n_format == 'Markdown'):
+                if not 'format=' in url and (n_format.lower() == 'text' or n_format.lower() == 'markdown'):
                     prefix = '?' if not '?' in url else '&'
                     # Apprise format is lowercase text https://github.com/caronc/apprise/issues/633
                     n_format = n_format.lower()

changedetectionio/static/js/notifications.js

@@ -50,6 +50,8 @@ $(document).ready(function () {
     function setPreview(data) {
         const iframe = document.getElementById("notification-iframe");
         const isDark = document.documentElement.getAttribute('data-darkmode') === 'true';
+        const isTextFormat = $('select.notification-format').val() === 'Text';
+
         $('#notification-preview-title-text').text(data['title']);
 
         iframe.srcdoc = `
@@ -76,9 +78,15 @@ $(document).ready(function () {
                         color: var(--color-text);
                         padding: 5px;
                     }
+                    body.text-format {
+                        font-family: monospace;
+                        white-space: pre;
+                        overflow-wrap: normal;
+                        overflow-x: auto;
+                    }
             </style>
           </head>
-          <body>${data['body']}</body>
+          <body class="${isTextFormat ? 'text-format' : ''}">${data['body']}</body>
         </html>`;
     }
 

changedetectionio/templates/_common_fields.html

@@ -159,11 +159,15 @@
                                             <span class="pure-form-message-inline">Format for all notifications</span>
                                         </div>
                                     </div>
-                                    <div id="notification-preview" style="display: none;" class="">
+                                    <div id="notification-preview" style="display: none; height:100%; display:flex; flex-direction:column;">
                                         <p><strong>Title: </strong><span id="notification-preview-title-text">Preview loading..</span></p>
-                                        <p><strong>Body: </strong><!-- setPreview in notifications.js sets this safely --->
-                                          <iframe id="notification-iframe" style="width:100%;border-radius: 4px;margin-top:0.5rem; height:95%; border:none;">Preview loading...</iframe>
-                                        </p>
+                                        <div style="flex:1; display:flex; flex-direction:column;">
+                                            <strong>Body: </strong>
+                                            <iframe id="notification-iframe"
+                                                    style="flex:1; height:95%; width:100%; border-radius:4px; margin-top:0.5rem; border:none;">
+                                                Preview loading...
+                                            </iframe>
+                                        </div>
                                     </div>
                                 </div>
                             </div>

changedetectionio/tests/smtp/test_notification_smtp.py

@@ -1,17 +1,17 @@
-import json
 import os
 import time
-import re
 from flask import url_for
 from changedetectionio.tests.util import set_original_response, set_modified_response, set_more_modified_response, live_server_setup, \
     wait_for_all_checks, \
     set_longer_modified_response
-from changedetectionio.tests.util import extract_UUID_from_client
 import logging
-import base64
 
 # NOTE - RELIES ON mailserver as hostname running, see github build recipes
-smtp_test_server = 'mailserver'
+# Should be hostname (never IP), looks for our test mailserver that repeats the content
+# python3 changedetectionio/tests/smtp/smtp-test-server.py &
+# mailserver=localhost pytest tests/smtp/test_notification_smtp.py::test_check_notification_email_formats_default_HTML
+smtp_test_server = os.getenv('mailserver', 'mailserver')
+
 
 from changedetectionio.notification import (
     default_notification_body,
@@ -20,7 +20,35 @@ from changedetectionio.notification import (
     valid_notification_formats,
 )
 
+from email import policy
+from email.parser import BytesParser, Parser
 
+def parse_mime(raw):
+    """Return (EmailMessage, dict[str, list[str]] bodies by content-type)."""
+    if isinstance(raw, (bytes, bytearray)):
+        msg = BytesParser(policy=policy.default).parsebytes(raw)
+    else:
+        msg = Parser(policy=policy.default).parsestr(raw)
+
+    parts_by_type = {}
+    if msg.is_multipart():
+        for part in msg.walk():
+            if part.get_content_maintype() == "multipart":
+                continue
+            ctype = part.get_content_type()           # e.g. "text/plain"
+            text = part.get_content()                 # decoded str
+            parts_by_type.setdefault(ctype, []).append(text)
+    else:
+        parts_by_type.setdefault(msg.get_content_type(), []).append(msg.get_content())
+
+    return msg, parts_by_type
+
+def one_or_join(parts_dict, ctype):
+    """Join multiple parts of the same type (rare but possible)."""
+    return "\n".join(parts_dict.get(ctype, []))
+
+def norm_newlines(s: str) -> str:
+    return s.replace("\r\n", "\n").replace("\r", "\n")
 
 def get_last_message_from_smtp_server():
     import socket
@@ -77,37 +105,36 @@ def test_check_notification_email_formats_default_HTML(client, live_server, meas
 
     time.sleep(3)
 
-    msg = get_last_message_from_smtp_server()
-    assert len(msg) >= 1
+    raw = get_last_message_from_smtp_server()
+    assert raw  # not empty
+
+    msg, bodies = parse_mime(raw)
+
+    plain = norm_newlines(one_or_join(bodies, "text/plain"))
+    html = norm_newlines(one_or_join(bodies, "text/html"))
+
+    # Now assert against the decoded bodies
+    assert "(added) So let's see what happens.\n" in plain  # plaintext uses a literal apostrophe
+    assert "(added) So let&#39;s see what happens.<br>" in html  # html uses &#39; and <br>
+
+    # You can also check counts, boundaries, etc.
+    assert html.count("So let&#39;s see what happens.") == 3
+    assert "modified head title had a change." in plain
+    assert "modified head title had a change.<br>" in html
+
+
 
-    # The email should have two bodies, and the text/html part should be <br>
-    assert 'Content-Type: text/plain' in msg
-    assert '(added) So let\'s see what happens.\r\n' in msg  # The plaintext part with \r\n
-    assert 'Content-Type: text/html' in msg
-    assert '(added) So let\'s see what happens.<br>' in msg  # the html part
     res = client.get(url_for("ui.form_delete", uuid="all"), follow_redirects=True)
     assert b'Deleted' in res.data
 
 
 def test_check_notification_email_formats_default_Text_override_HTML(client, live_server, measure_memory_usage):
-    ##  live_server_setup(live_server) # Setup on conftest per function
 
     # HTML problems? see this
     # https://github.com/caronc/apprise/issues/633
-
     set_original_response()
     notification_url = f'mailto://changedetection@{smtp_test_server}:11025/?to=fff@home.com'
-    notification_body = f"""<!DOCTYPE html>
-<html lang="en">
-<head>
-    <title>My Webpage</title>
-</head>
-<body>
-    <h1>Test</h1>
-    {default_notification_body}
-</body>
-</html>
-"""
+    notification_body = f"""{default_notification_body}"""
 
     #####################
     # Set this up for when we remove the notification from the watch, it should fallback with these details
@@ -116,11 +143,12 @@ def test_check_notification_email_formats_default_Text_override_HTML(client, liv
         data={"application-notification_urls": notification_url,
               "application-notification_title": "fallback-title " + default_notification_title,
               "application-notification_body": notification_body,
-              "application-notification_format": 'Text',
+              "application-notification_format": 'Text', # handler.py should be sure to add &format=text to override default html from apprise
               "requests-time_between_check-minutes": 180,
               'application-fetch_backend': "html_requests"},
         follow_redirects=True
     )
+
     assert b"Settings updated." in res.data
 
     # Add a watch and trigger a HTTP POST
@@ -140,43 +168,56 @@ def test_check_notification_email_formats_default_Text_override_HTML(client, liv
     wait_for_all_checks(client)
 
     time.sleep(3)
-    msg = get_last_message_from_smtp_server()
-    assert len(msg) >= 1
-    #    with open('/tmp/m.txt', 'w') as f:
-    #        f.write(msg)
+    raw = get_last_message_from_smtp_server()
+    assert raw
 
-    # The email should not have two bodies, should be TEXT only
+    msg, bodies = parse_mime(raw)
 
-    assert 'Content-Type: text/plain' in msg
-    assert '(added) So let\'s see what happens.\r\n' in msg  # The plaintext part with \r\n
+    plain = norm_newlines(one_or_join(bodies, "text/plain"))
+    html = norm_newlines(one_or_join(bodies, "text/html"))
+    assert not html # should be no HTML here
 
+    # Expect ONLY text/plain body
+    assert "text/plain" in bodies
+    assert "text/html" not in bodies
+
+    # Assert on decoded plaintext (literal apostrophe, not &#39;)
+    # Should be NO markup when in text mode
+    assert "(added) So let's see what happens.\n" in plain
+
+
+    # ---------- Flip to HTML format, then expect multipart with both ----------
     set_original_response()
-    # Now override as HTML format
     res = client.post(
         url_for("ui.ui_edit.edit_page", uuid="first"),
-        data={
-            "url": test_url,
-            "notification_format": 'HTML',
-            'fetch_backend': "html_requests"},
-        follow_redirects=True
+        data={"url": test_url,
+              "notification_format": "HTML",
+              "fetch_backend": "html_requests",
+              "time_between_check_use_default": "y"},
+        follow_redirects=True,
     )
+
     assert b"Updated watch." in res.data
     wait_for_all_checks(client)
 
     time.sleep(3)
-    msg = get_last_message_from_smtp_server()
-    assert len(msg) >= 1
 
-    # The email should have two bodies, and the text/html part should be <br>
-    assert 'Content-Type: text/plain' in msg
-    assert '(removed) So let\'s see what happens.\r\n' in msg  # The plaintext part with \n
-    assert 'Content-Type: text/html' in msg
-    assert '(removed) So let\'s see what happens.<br>' in msg  # the html part
+    raw = get_last_message_from_smtp_server()
+    assert raw
 
-    # https://github.com/dgtlmoon/changedetection.io/issues/2103
-    assert '<h1>Test</h1>' in msg
-    assert '&lt;' not in msg
-    assert 'Content-Type: text/html' in msg
+    msg, bodies = parse_mime(raw)
+    plain = norm_newlines(one_or_join(bodies, "text/plain"))
+    html = norm_newlines(one_or_join(bodies, "text/html"))
 
-    res = client.get(url_for("ui.form_delete", uuid="all"), follow_redirects=True)
-    assert b'Deleted' in res.data
+    # Expect both text/plain and text/html bodies now
+    assert "text/plain" in bodies
+    assert "text/html" in bodies
+
+    # Plaintext reflects the removal line (literal apostrophe)
+    assert "(removed) So let's see what happens.\n" in plain
+    assert "(removed) So let&#39;s see what happens.<br>" in html
+
+    # Optional: ensure we got multipart/alternative (typical for dual bodies)
+    if msg.is_multipart():
+        # most senders do "multipart/alternative" for text/plain + text/html
+        assert msg.get_content_subtype() in ("alternative", "mixed", "related")