Merge branch 'master' into 3159-test-notification-send

changedetectionio/async_update_worker.py

@@ -91,7 +91,7 @@ async def async_update_worker(worker_id, q, notification_q, app, datastore):
                     try:
                         processor_module = importlib.import_module(processor_module_name)
                     except ModuleNotFoundError as e:
-                        print(f"Processor module '{processor}' not found.")
+                        logger.error(f"Processor module '{processor}' not found.")
                         raise e
 
                     update_handler = processor_module.perform_site_check(datastore=datastore,

changedetectionio/blueprint/settings/templates/settings.html

@@ -5,6 +5,7 @@
 {% from '_common_fields.html' import render_common_settings_form %}
 <script>
     const notification_base_url="{{url_for('ui.ui_notification.ajax_callback_send_notification_test', mode="global-settings")}}";
+    const notification_test_render_preview_url="{{url_for('ui.ui_notification.ajax_callback_test_render_preview', mode="global-settings")}}";
 {% if emailprefix %}
     const email_notification_prefix=JSON.parse('{{emailprefix|tojson}}');
 {% endif %}
@@ -43,10 +44,6 @@
                                 </div>
                         </div>
                     </div>
-                    <div class="pure-control-group">
-                        {{ render_field(form.requests.form.jitter_seconds, class="jitter_seconds") }}
-                        <span class="pure-form-message-inline">Example - 3 seconds random jitter could trigger up to 3 seconds earlier or up to 3 seconds later</span>
-                    </div>
                     <div class="pure-control-group">
                         {{ render_field(form.application.form.filter_failure_notification_threshold_attempts, class="filter_failure_notification_threshold_attempts") }}
                         <span class="pure-form-message-inline">After this many consecutive times that the CSS/xPath filter is missing, send a notification
@@ -133,6 +130,10 @@
                     <span class="pure-form-message-inline">Number of concurrent workers to process watches. More workers = faster processing but higher memory usage.<br>
                     Currently running: <strong>{{ worker_info.count }}</strong> operational {{ worker_info.type }} workers{% if worker_info.active_workers > 0 %} ({{ worker_info.active_workers }} actively processing){% endif %}.</span>
                 </div>
+                <div class="pure-control-group">
+                    {{ render_field(form.requests.form.jitter_seconds, class="jitter_seconds") }}
+                    <span class="pure-form-message-inline">Example - 3 seconds random jitter could trigger up to 3 seconds earlier or up to 3 seconds later</span>
+                </div>
                 <div class="pure-control-group inline-radio">
                     {{ render_field(form.requests.form.default_ua) }}
                     <span class="pure-form-message-inline">
@@ -200,21 +201,12 @@ nav
            </div>
 
             <div class="tab-pane-inner" id="api">
-                <h4>API Access</h4>
-                <p>Drive your changedetection.io via API, More about <a href="https://changedetection.io/docs/api_v1/index.html">API access and examples here</a>.</p>
+                <p>
+                    <strong>Chrome extension and API Access</strong><br>
+                </p>
 
-                <div class="pure-control-group">
-                    {{ render_checkbox_field(form.application.form.api_access_token_enabled) }}
-                    <div class="pure-form-message-inline">Restrict API access limit by using <code>x-api-key</code> header - required for the Chrome Extension to work</div><br>
-                    <div class="pure-form-message-inline"><br>API Key <span id="api-key">{{api_key}}</span>
-                        <span style="display:none;" id="api-key-copy" >copy</span>
-                    </div>
-                </div>
-                <div class="pure-control-group">
-                    <a href="{{url_for('settings.settings_reset_api_key')}}" class="pure-button button-small button-cancel">Regenerate API key</a>
-                </div>
-                <div class="pure-control-group">
-                    <h4>Chrome Extension</h4>
+                <div class="pure-control-group border-fieldset">
+                    <strong>Chrome Extension</strong><br>
                     <p>Easily add any web-page to your changedetection.io installation from within Chrome.</p>
                     <strong>Step 1</strong> Install the extension, <strong>Step 2</strong> Navigate to this page,
                     <strong>Step 3</strong> Open the extension from the toolbar and click "<i>Sync API Access</i>"
@@ -227,6 +219,20 @@ nav
                         </a>
                     </p>
                 </div>
+                <div class="pure-control-group  border-fieldset">
+                    Drive your changedetection.io via API, More about <a href="https://changedetection.io/docs/api_v1/index.html">API access and examples here</a>.<br>
+                    <p>
+                        {{ render_checkbox_field(form.application.form.api_access_token_enabled) }}
+                    </p>
+                    <div class="pure-form-message-inline">Restrict API access limit by using <code>x-api-key</code> header - required for the Chrome Extension to work</div><br>
+                    <div class="pure-form-message-inline"><br>API Key <span id="api-key">{{api_key}}</span>
+                        <span style="display:none;" id="api-key-copy" >copy</span>
+                    </div>
+                    <p>
+                        <a href="{{url_for('settings.settings_reset_api_key')}}" class="pure-button button-small button-cancel">Regenerate API key</a>
+                    </p>
+                </div>
+
             </div>
             <div class="tab-pane-inner" id="timedate">
                 <div class="pure-control-group">

changedetectionio/blueprint/tags/templates/edit-tag.html

@@ -4,6 +4,8 @@
 {% from '_common_fields.html' import render_common_settings_form %}
 <script>
     const notification_base_url="{{url_for('ui.ui_notification.ajax_callback_send_notification_test', mode="group-settings")}}";
+    const notification_test_render_preview_url="{{url_for('ui.ui_notification.ajax_callback_test_render_preview', mode="group-settings", watch_uuid=data.uuid)}}";
+    //alert(notification_test_render_preview_url)
 </script>
 
 <script src="{{url_for('static_content', group='js', filename='tabs.js')}}" defer></script>
@@ -19,6 +21,8 @@
 
 <script src="{{url_for('static_content', group='js', filename='watch-settings.js')}}" defer></script>
 <script src="{{url_for('static_content', group='js', filename='notifications.js')}}" defer></script>
+<script src="{{url_for('static_content', group='js', filename='plugins.js')}}" defer></script>
+
 
 <div class="edit-form monospaced-textarea">
 

changedetectionio/blueprint/ui/__init__.py

@@ -106,14 +106,14 @@ def _handle_operations(op, uuids, datastore, worker_handler, update_q, queuedWat
         for uuid in uuids:
             watch_check_update.send(watch_uuid=uuid)
 
-def construct_blueprint(datastore: ChangeDetectionStore, update_q, worker_handler, queuedWatchMetaData, watch_check_update):
+def construct_blueprint(datastore: ChangeDetectionStore, update_q, worker_handler, queuedWatchMetaData, watch_check_update, notification_q):
     ui_blueprint = Blueprint('ui', __name__, template_folder="templates")
     
     # Register the edit blueprint
     edit_blueprint = construct_edit_blueprint(datastore, update_q, queuedWatchMetaData)
     ui_blueprint.register_blueprint(edit_blueprint)
     
-    # Register the notification blueprint
+    # Register the notification blueprint - mostly used for sending test
     notification_blueprint = construct_notification_blueprint(datastore)
     ui_blueprint.register_blueprint(notification_blueprint)
     

changedetectionio/blueprint/ui/notification.py

@@ -1,50 +1,85 @@
-from flask import Blueprint, request, make_response
+from flask import Blueprint, request, make_response, jsonify
 import random
 from loguru import logger
 
+from changedetectionio.notification.handler import process_notification
 from changedetectionio.store import ChangeDetectionStore
 from changedetectionio.auth_decorator import login_optionally_required
 
 def construct_blueprint(datastore: ChangeDetectionStore):
     notification_blueprint = Blueprint('ui_notification', __name__, template_folder="../ui/templates")
-    
+
+
+    @notification_blueprint.route("/notification/render-preview/<string:watch_uuid>", methods=['POST'])
+    @notification_blueprint.route("/notification/render-preview", methods=['POST'])
+    @notification_blueprint.route("/notification/render-preview/", methods=['POST'])
+    @login_optionally_required
+    def ajax_callback_test_render_preview(watch_uuid=None):
+        return ajax_callback_send_notification_test(watch_uuid=watch_uuid, send_as_null_test=True)
+
     # AJAX endpoint for sending a test
     @notification_blueprint.route("/notification/send-test/<string:watch_uuid>", methods=['POST'])
     @notification_blueprint.route("/notification/send-test", methods=['POST'])
     @notification_blueprint.route("/notification/send-test/", methods=['POST'])
     @login_optionally_required
-    def ajax_callback_send_notification_test(watch_uuid=None):
+    def ajax_callback_send_notification_test(watch_uuid=None, send_as_null_test=False):
 
         # Watch_uuid could be unset in the case it`s used in tag editor, global settings
         import apprise
-        from changedetectionio.notification.handler import process_notification
+        from urllib.parse import urlparse
         from changedetectionio.notification.apprise_plugin.assets import apprise_asset
 
-        from changedetectionio.notification.apprise_plugin.custom_handlers import apprise_http_custom_handler
+        # Necessary so that we import our custom handlers
+        from changedetectionio.notification.apprise_plugin.custom_handlers import apprise_http_custom_handler, apprise_null_custom_handler
 
         apobj = apprise.Apprise(asset=apprise_asset)
+        sent_obj = {}
 
         is_global_settings_form = request.args.get('mode', '') == 'global-settings'
         is_group_settings_form = request.args.get('mode', '') == 'group-settings'
 
         # Use an existing random one on the global/main settings form
-        if not watch_uuid and (is_global_settings_form or is_group_settings_form) \
-                and datastore.data.get('watching'):
-            logger.debug(f"Send test notification - Choosing random Watch {watch_uuid}")
+        if not watch_uuid and is_global_settings_form and datastore.data.get('watching'):
             watch_uuid = random.choice(list(datastore.data['watching'].keys()))
+            logger.debug(f"Send test notification - Chose random watch UUID: {watch_uuid}")
+
+        if is_group_settings_form  and datastore.data.get('watching'):
+            logger.debug(f"Send test notification - Choosing random Watch from group {watch_uuid}")
+            matching_watches = [uuid for uuid, watch in datastore.data['watching'].items() if watch.get('tags') and watch_uuid in watch['tags']]
+            if matching_watches:
+                watch_uuid = random.choice(matching_watches)
+            else:
+                # Just fallback to any
+                watch_uuid = random.choice(list(datastore.data['watching'].keys()))
 
         if not watch_uuid:
             return make_response("Error: You must have atleast one watch configured for 'test notification' to work", 400)
 
         watch = datastore.data['watching'].get(watch_uuid)
 
-        notification_urls = None
+        notification_urls = []
 
-        if request.form.get('notification_urls'):
-            notification_urls = request.form['notification_urls'].strip().splitlines()
+        if send_as_null_test:
+            test_schema = ''
+            try:
+                if request.form.get('notification_urls') and '://' in request.form.get('notification_urls'):
+                    first_test_notification_url = request.form['notification_urls'].strip().splitlines()[0]
+                    test_schema = urlparse(first_test_notification_url).scheme.lower().strip()
+            except Exception as e:
+                logger.error(f"Error trying to get a test schema based on the first notification_url {str(e)}")
+
+            notification_urls = [
+                # Null lets us do the whole chain of the same code without any extra repeated code
+                f'null://null-test-just-to-render-everything-on-the-same-codepath-and-get-preview?test_schema={test_schema}'
+            ]
+
+        else:
+            if request.form.get('notification_urls'):
+                notification_urls += request.form['notification_urls'].strip().splitlines()
 
         if not notification_urls:
             logger.debug("Test notification - Trying by group/tag in the edit form if available")
+            # @todo this logic is not clear, omegaconf?
             # On an edit page, we should also fire off to the tags if they have notifications
             if request.form.get('tags') and request.form['tags'].strip():
                 for k in request.form['tags'].split(','):
@@ -58,23 +93,28 @@ def construct_blueprint(datastore: ChangeDetectionStore):
                 notification_urls = datastore.data['settings']['application']['notification_urls']
 
         if not notification_urls:
-            return 'Error: No Notification URLs set/found'
+            return make_response("Error: No Notification URLs set/found.", 400)
 
         for n_url in notification_urls:
             if len(n_url.strip()):
                 if not apobj.add(n_url):
-                    return f'Error:  {n_url} is not a valid AppRise URL.'
+                    return make_response(f'Error: {n_url} is not a valid AppRise URL.', 400)
 
         try:
             # use the same as when it is triggered, but then override it with the form test values
             n_object = {
                 'watch_url': request.form.get('window_url', "https://changedetection.io"),
-                'notification_urls': notification_urls
+                'notification_urls': notification_urls,
+                'uuid': watch_uuid  # Ensure uuid is present so diff rendering works
             }
 
             # Only use if present, if not set in n_object it should use the default system value
-            if 'notification_format' in request.form and request.form['notification_format'].strip():
-                n_object['notification_format'] = request.form.get('notification_format', '').strip()
+            notif_format = request.form.get('notification_format', '').strip()
+            # Use it if provided and not "System default", otherwise fall back
+            if notif_format and notif_format != 'System default':
+                n_object['notification_format'] = notif_format
+            else:
+                n_object['notification_format'] = datastore.data['settings']['application'].get('notification_format')
 
             if 'notification_title' in request.form and request.form['notification_title'].strip():
                 n_object['notification_title'] = request.form.get('notification_title', '').strip()
@@ -92,7 +132,13 @@ def construct_blueprint(datastore: ChangeDetectionStore):
 
             n_object['as_async'] = False
             n_object.update(watch.extra_notification_token_values())
-            sent_obj = process_notification(n_object, datastore)
+
+            # This uses the same processor that the queue runner uses
+            # @todo - Split the notification URLs so we know which one worked, maybe highlight them in green in the UI
+            result = process_notification(n_object, datastore)
+            if result:
+                sent_obj['result'] = result[0]
+                sent_obj['status'] = 'OK - Sent test notifications'
 
         except Exception as e:
             e_str = str(e)
@@ -100,9 +146,9 @@ def construct_blueprint(datastore: ChangeDetectionStore):
             e_str = e_str.replace(
                 "DEBUG - <class 'apprise.decorators.base.CustomNotifyPlugin.instantiate_plugin.<locals>.CustomNotifyPluginWrapper'>",
                 '')
-
             return make_response(e_str, 400)
 
-        return 'OK - Sent test notifications'
+        # it will be a list of things reached, for this purpose just the first is good so we can see the body that was sent
+        return make_response(sent_obj, 200)
 
     return notification_blueprint

changedetectionio/blueprint/ui/templates/edit.html

@@ -21,6 +21,7 @@
     const email_notification_prefix=JSON.parse('{{ emailprefix|tojson }}');
 {% endif %}
     const notification_base_url="{{url_for('ui.ui_notification.ajax_callback_send_notification_test', watch_uuid=uuid)}}";
+    const notification_test_render_preview_url="{{url_for('ui.ui_notification.ajax_callback_test_render_preview', watch_uuid=uuid)}}";
     const playwright_enabled={% if playwright_enabled %}true{% else %}false{% endif %};
     const recheck_proxy_start_url="{{url_for('check_proxies.start_check', uuid=uuid)}}";
     const proxy_recheck_status_url="{{url_for('check_proxies.get_recheck_status', uuid=uuid)}}";
@@ -356,12 +357,12 @@ Math: {{ 1 + 1 }}") }}
                     </script>
                     <br>
                     {#<div id="text-preview-controls"><span id="text-preview-refresh" class="pure-button button-xsmall">Refresh</span></div>#}
-                    <div class="minitabs-wrapper">
+                    <div class="minitabs-wrapper" id="filter-preview-minitabs">
                       <div class="minitabs-content">
-                          <div id="text-preview-inner" class="monospace-preview">
+                          <div id="text-preview-inner" class="tab-contents-monospace-preview">
                               <p>Loading...</p>
                           </div>
-                          <div id="text-preview-before-inner" style="display: none;" class="monospace-preview">
+                          <div id="text-preview-before-inner" style="display: none;" class="tab-contents-monospace-preview">
                               <p>Loading...</p>
                           </div>
                       </div>

changedetectionio/conditions/pluggy_interface.py

@@ -1,7 +1,7 @@
 import pluggy
 import os
 import importlib
-import sys
+from loguru import logger
 from . import default_plugin
 
 # ✅ Ensure that the namespace in HookspecMarker matches PluginManager
@@ -65,7 +65,7 @@ def load_plugins_from_directory():
                 # Register the plugin with pluggy
                 plugin_manager.register(module, module_name)
             except (ImportError, AttributeError) as e:
-                print(f"Error loading plugin {module_name}: {e}")
+                logger.critical(f"Error loading plugin {module_name}: {e}")
 
 # Load plugins from the plugins directory
 load_plugins_from_directory()

changedetectionio/flask_app.py

@@ -519,7 +519,7 @@ def changedetection_app(config=None, datastore_o=None):
 
     # watchlist UI buttons etc
     import changedetectionio.blueprint.ui as ui
-    app.register_blueprint(ui.construct_blueprint(datastore, update_q, worker_handler, queuedWatchMetaData, watch_check_update))
+    app.register_blueprint(ui.construct_blueprint(datastore, update_q, worker_handler, queuedWatchMetaData, watch_check_update, notification_q))
 
     import changedetectionio.blueprint.watchlist as watchlist
     app.register_blueprint(watchlist.construct_blueprint(datastore=datastore, update_q=update_q, queuedWatchMetaData=queuedWatchMetaData), url_prefix='')

changedetectionio/model/Watch.py

@@ -642,8 +642,10 @@ class model(watch_base):
 
     def extra_notification_token_placeholder_info(self):
         # Used for providing extra tokens
-        # return [('widget', "Get widget amounts")]
-        return []
+        values = []
+        values.append(('watch_html_link', "Link to URL as <a href>"))
+        values.append(('watch_url_raw', "Raw URL/link before any jinja2 macro"))
+        return values
 
 
     def extract_regex_from_all_history(self, regex):

changedetectionio/notification/__init__.py

@@ -2,8 +2,8 @@ from changedetectionio.model import default_notification_format_for_watch
 
 ult_notification_format_for_watch = 'System default'
 default_notification_format = 'HTML Color'
-default_notification_body = '{{watch_url}} had a change.\n---\n{{diff}}\n---\n'
-default_notification_title = 'ChangeDetection.io Notification - {{watch_url}}'
+default_notification_body = '{{watch_title}} had a change.\n---\n{{diff}}\n---\n'
+default_notification_title = 'ChangeDetection.io Notification - {{watch_title}}'
 
 # The values (markdown etc) are from apprise NotifyFormat,
 # But to avoid importing the whole heavy module just use the same strings here.

changedetectionio/notification/apprise_plugin/custom_handlers.py

@@ -19,6 +19,11 @@ def notify_supported_methods(func):
     return func
 
 
+def notify_null_method(func):
+    func = notify(on="null")(func)
+    return func
+
+
 def _get_auth(parsed_url: dict) -> str | tuple[str, str]:
     user: str | None = parsed_url.get("user")
     password: str | None = parsed_url.get("password")
@@ -110,3 +115,21 @@ def apprise_http_custom_handler(
     except Exception as e:
         logger.error(f"Unexpected error occurred while sending custom notification to {url}: {e}")
         return False
+
+
+@notify_null_method
+def apprise_null_custom_handler(
+    body: str,
+    title: str,
+    notify_type: str,
+    meta: dict,
+    *args,
+    **kwargs,
+) -> bool:
+    url: str = meta.get("url")
+    schema: str = meta.get("schema")
+    method: str = re.sub(r"s$", "", schema).upper()
+    logger.info(f"Processed 'null' notification")
+
+    return True
+

changedetectionio/notification/handler.py

@@ -1,31 +1,167 @@
 
+import os
 import time
 import apprise
 from loguru import logger
 from .apprise_plugin.assets import apprise_asset, APPRISE_AVATAR_URL
+from changedetectionio.safe_jinja import render as jinja_render
+from urllib.parse import urlparse
+
+def _populate_notification_tokens(n_object, datastore):
+    """
+    Populate notification tokens (diff, current_snapshot, etc.) if not already present.
+    This ensures both queued notifications and test notifications have the same data.
+    """
+    from changedetectionio import diff
+    from changedetectionio.notification import default_notification_format_for_watch
+    from markupsafe import escape
+
+    watch_uuid = n_object.get('uuid')
+    if not watch_uuid:
+        return
+        
+    watch = datastore.data['watching'].get(watch_uuid)
+    if not watch:
+        return
+
+    dates = []
+    trigger_text = ''
+    watch_html_link = ''
+
+    if watch:
+        watch_history = watch.history
+        dates = list(watch_history.keys())
+        trigger_text = watch.get('trigger_text', [])
+
+    # Add text that was triggered
+    if len(dates):
+        snapshot_contents = watch.get_history_snapshot(dates[-1])
+
+        if n_object.get('notification_format').lower().startswith('html'):
+            snapshot_contents = str(escape(snapshot_contents))
+
+    else:
+        snapshot_contents = "No snapshot/history available, the watch should fetch atleast once."
+
+    # If we ended up here with "System default"
+    if n_object.get('notification_format') == default_notification_format_for_watch:
+        n_object['notification_format'] = datastore.data['settings']['application'].get('notification_format')
+
+    html_colour_enable = False
+    line_feed_sep = "\n"
+
+    # HTML needs linebreak, but MarkDown and Text can use a linefeed
+    if n_object.get('notification_format').lower().startswith('html'):
+        line_feed_sep = "<br>"
+        # Snapshot will be plaintext on the disk, convert to some kind of HTML
+        snapshot_contents = snapshot_contents.replace('\n', line_feed_sep)
+    if n_object.get('notification_format') == 'HTML Color':
+        html_colour_enable = True
+
+    triggered_text = ''
+    if len(trigger_text):
+        from changedetectionio import html_tools
+        triggered_text = html_tools.get_triggered_text(content=snapshot_contents, trigger_text=trigger_text)
+        if triggered_text:
+            triggered_text = line_feed_sep.join(triggered_text)
+
+    # Could be called as a 'test notification' with only 1 snapshot available
+    prev_snapshot = "Example text: example test\nExample text: change detection is cool\nExample text: some more examples\n"
+    current_snapshot = "Example text: example test\nExample text: More than 1 watch change needs to exist to build a nice preview!"
+
+    if len(dates) > 1:
+        prev_snapshot = watch.get_history_snapshot(dates[-2])
+        current_snapshot = watch.get_history_snapshot(dates[-1])
+        if n_object.get('notification_format').lower().startswith('html'):
+            prev_snapshot = str(escape(prev_snapshot))
+            current_snapshot = str(escape(current_snapshot))
+
+
+    if watch:
+        v = {'url': watch.get('url'), 'label': watch.label}
+        watch_html_link = jinja_render(template_str='<a href="{{ label or url | e }}" rel="noopener noreferrer">{{ url | e }}</a>', **v)
+
+
+    n_object.update({
+        'current_snapshot': snapshot_contents,
+        'diff': diff.render_diff(prev_snapshot, current_snapshot, line_feed_sep=line_feed_sep, html_colour=html_colour_enable),
+        'diff_added': diff.render_diff(prev_snapshot, current_snapshot, include_removed=False, line_feed_sep=line_feed_sep, html_colour=html_colour_enable),
+        'diff_full': diff.render_diff(prev_snapshot, current_snapshot, include_equal=True, line_feed_sep=line_feed_sep, html_colour=html_colour_enable),
+        'diff_patch': diff.render_diff(prev_snapshot, current_snapshot, line_feed_sep=line_feed_sep, patch_format=True),
+        'diff_removed': diff.render_diff(prev_snapshot, current_snapshot, include_added=False, line_feed_sep=line_feed_sep, html_colour=html_colour_enable),
+        'screenshot': watch.get_screenshot() if watch and watch.get('notification_screenshot') else None,
+        'triggered_text': triggered_text,
+        'watch_html_link': watch_html_link,
+        'watch_url': watch.link,
+        'watch_url_raw': watch.get('url'),
+    })
+
+    if watch:
+        n_object.update(watch.extra_notification_token_values())
+
+def scan_notification_file_templates(url, datastore, n_body, notification_parameters):
+    import glob
+    from urllib.parse import urlparse, parse_qs
+
+    try:
+        scheme = urlparse(url).scheme.lower().strip()
+
+        # schema could be overriden dynamically
+        if scheme == 'null' and 'test_schema=' in url:
+            scheme = parse_qs(urlparse(url).query).get("test_schema", [None])[0]
+
+        logger.debug(f"Looking for '{scheme}' notification wrapper templates...")
+
+        # Try exact match first, then wildcard matches
+        candidates = [
+            os.path.join(datastore.datastore_path, f"notification-wrapper-{scheme}.html"),
+            *[f for f in glob.glob(os.path.join(datastore.datastore_path, "notification-wrapper-*--.html"))
+              if scheme.startswith(os.path.basename(f).replace("notification-wrapper-", "").replace("--.html", ""))]
+        ]
+        
+        for tpl_name in candidates:
+            if os.path.isfile(tpl_name):
+                template_params = notification_parameters.copy()
+                template_params['notification_body'] = n_body
+                
+                with open(tpl_name, 'r', encoding='utf-8') as f:
+                    logger.info(f"Using HTML notification template wrapper from '{tpl_name}'")
+                    return jinja_render(template_str=f.read(), **template_params)
+                    
+    except Exception as e:
+        logger.warning(f"Failed to load notification template: {e}")
+
+    return None
 
 def process_notification(n_object, datastore):
-    from changedetectionio.safe_jinja import render as jinja_render
     from . import default_notification_format_for_watch, default_notification_format, valid_notification_formats
     # be sure its registered
-    from .apprise_plugin.custom_handlers import apprise_http_custom_handler
+    from .apprise_plugin.custom_handlers import apprise_http_custom_handler, apprise_null_custom_handler
+
+    n_body = ''
+    n_title = ''
 
     now = time.time()
     if n_object.get('notification_timestamp'):
         logger.trace(f"Time since queued {now-n_object['notification_timestamp']:.3f}s")
-    # Insert variables into the notification content
-    notification_parameters = create_notification_parameters(n_object, datastore)
 
     n_format = valid_notification_formats.get(
         n_object.get('notification_format', default_notification_format),
         valid_notification_formats[default_notification_format],
     )
-
     # If we arrived with 'System default' then look it up
     if n_format == default_notification_format_for_watch and datastore.data['settings']['application'].get('notification_format') != default_notification_format_for_watch:
         # Initially text or whatever
         n_format = datastore.data['settings']['application'].get('notification_format', valid_notification_formats[default_notification_format])
 
+    # Ensure diff rendering is done if not already present (for test notifications)
+    if not n_object.get('diff') and n_object.get('uuid'):
+        _populate_notification_tokens(n_object, datastore)
+
+    # Insert variables into the notification content
+    notification_parameters = create_notification_parameters(n_object, datastore)
+
+
     logger.trace(f"Complete notification body including Jinja and placeholders calculated in  {time.time() - now:.2f}s")
 
     # https://github.com/caronc/apprise/wiki/Development_LogCapture
@@ -44,22 +180,27 @@ def process_notification(n_object, datastore):
 
     with apprise.LogCapture(level=apprise.logging.DEBUG) as logs:
         for url in n_object['notification_urls']:
+            # Commented out is OK
+            if url.startswith('#') or not url or not url.strip():
+                logger.trace(f"Skipping notification URL - '{url}'")
+                continue
 
             # Get the notification body from datastore
             n_body = jinja_render(template_str=n_object.get('notification_body', ''), **notification_parameters)
+            # hmm unsure about this, but why not
             if n_object.get('notification_format', '').startswith('HTML'):
                 n_body = n_body.replace("\n", '<br>')
 
             n_title = jinja_render(template_str=n_object.get('notification_title', ''), **notification_parameters)
 
-            url = url.strip()
-            if url.startswith('#'):
-                logger.trace(f"Skipping commented out notification URL - {url}")
-                continue
+            n_body_from_file_template = scan_notification_file_templates(url=url,
+                                                                         datastore=datastore,
+                                                                         n_body=n_body,
+                                                                         notification_parameters=notification_parameters)
+            if n_body_from_file_template:
+                n_body = n_body_from_file_template
+
 
-            if not url:
-                logger.warning(f"Process Notification: skipping empty notification URL.")
-                continue
 
             logger.info(f">> Process Notification: AppRise notifying {url}")
             url = jinja_render(template_str=url, **notification_parameters)
@@ -104,7 +245,7 @@ def process_notification(n_object, datastore):
                 # Apprise will default to HTML, so we need to override it
                 # So that whats' generated in n_body is in line with what is going to be sent.
                 # https://github.com/caronc/apprise/issues/633#issuecomment-1191449321
-                if not 'format=' in url and (n_format == 'Text' or n_format == 'Markdown'):
+                if not 'format=' in url and (n_format.lower() == 'text' or n_format.lower() == 'markdown'):
                     prefix = '?' if not '?' in url else '&'
                     # Apprise format is lowercase text https://github.com/caronc/apprise/issues/633
                     n_format = n_format.lower()
@@ -118,14 +259,15 @@ def process_notification(n_object, datastore):
                               'url': url,
                               'body_format': n_format})
 
-        # Blast off the notifications tht are set in .add()
-        apobj.notify(
-            title=n_title,
-            body=n_body,
-            body_format=n_format,
-            # False is not an option for AppRise, must be type None
-            attach=n_object.get('screenshot', None)
-        )
+        if n_object.get('notification_urls'):
+            # Blast off the notifications tht are set in .add()
+            apobj.notify(
+                title=n_title,
+                body=n_body,
+                body_format=n_format,
+                # False is not an option for AppRise, must be type None
+                attach=n_object.get('screenshot', None)
+            )
 
 
         # Returns empty string if nothing found, multi-line string otherwise

changedetectionio/notification/notification-wrapper-HTML-mail--.html

@@ -0,0 +1,15 @@
+{#  Copy this to your data-store directory if you wish to enable it for HTML style notifications, applies to all as a wrapper :) #}
+<html>
+<body>
+Hello,<br>
+
+<p>A change was detected on your web page watch for <p>{{ watch_html_link }}.</p>
+
+[ view history ] [ pause checks ] [ mute notifications ]
+
+<div>
+    {{  notification_body }}
+</div>
+
+</body>
+</html>

changedetectionio/notification/readme.md

@@ -0,0 +1,17 @@
+## Notification syntax
+
+All notifications use the https://github.com/caronc/apprise syntax, there are some custom ones such as `posts` etc for general web-services usability.
+
+
+## Template file notification wrappers
+
+You can by default wrap all notifications by creating a `notification-wrapper-HTML-schema.html` in your datastore directory.
+
+For example
+
+You can use "`--`" in the filename where the _schema_ is to symbolize a wildcard. For example `notification-wrapper-HTML-mail--.html` would
+apply to `mail://` `mailto://` etc etc
+
+See is `notification-wrapper-HTML-mail--.html` which applies to `mail://`, `mailto://foobar..` etc notifications
+
+

changedetectionio/notification_service.py

@@ -22,70 +22,14 @@ class NotificationService:
     
     def queue_notification_for_watch(self, n_object, watch):
         """
-        Queue a notification for a watch with full diff rendering and template variables
+        Queue a notification for a watch. Diff rendering and template variables will be
+        handled by process_notification() to ensure consistency with test notifications.
         """
-        from changedetectionio import diff
-        from changedetectionio.notification import default_notification_format_for_watch
-
-        dates = []
-        trigger_text = ''
-
         now = time.time()
 
-        if watch:
-            watch_history = watch.history
-            dates = list(watch_history.keys())
-            trigger_text = watch.get('trigger_text', [])
-
-        # Add text that was triggered
-        if len(dates):
-            snapshot_contents = watch.get_history_snapshot(dates[-1])
-        else:
-            snapshot_contents = "No snapshot/history available, the watch should fetch atleast once."
-
-        # If we ended up here with "System default"
-        if n_object.get('notification_format') == default_notification_format_for_watch:
-            n_object['notification_format'] = self.datastore.data['settings']['application'].get('notification_format')
-
-        html_colour_enable = False
-        # HTML needs linebreak, but MarkDown and Text can use a linefeed
-        if n_object.get('notification_format') == 'HTML':
-            line_feed_sep = "<br>"
-            # Snapshot will be plaintext on the disk, convert to some kind of HTML
-            snapshot_contents = snapshot_contents.replace('\n', line_feed_sep)
-        elif n_object.get('notification_format') == 'HTML Color':
-            line_feed_sep = "<br>"
-            # Snapshot will be plaintext on the disk, convert to some kind of HTML
-            snapshot_contents = snapshot_contents.replace('\n', line_feed_sep)
-            html_colour_enable = True
-        else:
-            line_feed_sep = "\n"
-
-        triggered_text = ''
-        if len(trigger_text):
-            from . import html_tools
-            triggered_text = html_tools.get_triggered_text(content=snapshot_contents, trigger_text=trigger_text)
-            if triggered_text:
-                triggered_text = line_feed_sep.join(triggered_text)
-
-        # Could be called as a 'test notification' with only 1 snapshot available
-        prev_snapshot = "Example text: example test\nExample text: change detection is cool\nExample text: some more examples\n"
-        current_snapshot = "Example text: example test\nExample text: change detection is fantastic\nExample text: even more examples\nExample text: a lot more examples"
-
-        if len(dates) > 1:
-            prev_snapshot = watch.get_history_snapshot(dates[-2])
-            current_snapshot = watch.get_history_snapshot(dates[-1])
-
+        # Add basic metadata for the notification
         n_object.update({
-            'current_snapshot': snapshot_contents,
-            'diff': diff.render_diff(prev_snapshot, current_snapshot, line_feed_sep=line_feed_sep, html_colour=html_colour_enable),
-            'diff_added': diff.render_diff(prev_snapshot, current_snapshot, include_removed=False, line_feed_sep=line_feed_sep),
-            'diff_full': diff.render_diff(prev_snapshot, current_snapshot, include_equal=True, line_feed_sep=line_feed_sep, html_colour=html_colour_enable),
-            'diff_patch': diff.render_diff(prev_snapshot, current_snapshot, line_feed_sep=line_feed_sep, patch_format=True),
-            'diff_removed': diff.render_diff(prev_snapshot, current_snapshot, include_added=False, line_feed_sep=line_feed_sep),
             'notification_timestamp': now,
-            'screenshot': watch.get_screenshot() if watch and watch.get('notification_screenshot') else None,
-            'triggered_text': triggered_text,
             'uuid': watch.get('uuid') if watch else None,
             'watch_url': watch.get('url') if watch else None,
         })
@@ -93,7 +37,6 @@ class NotificationService:
         if watch:
             n_object.update(watch.extra_notification_token_values())
 
-        logger.trace(f"Main rendered notification placeholders (diff_added etc) calculated in {time.time()-now:.3f}s")
         logger.debug("Queued notification for sending")
         self.notification_q.put(n_object)
 

changedetectionio/pluggy_interface.py

@@ -1,7 +1,7 @@
 import pluggy
 import os
 import importlib
-import sys
+from loguru import logger
 
 # Global plugin namespace for changedetection.io
 PLUGIN_NAMESPACE = "changedetectionio"
@@ -57,7 +57,7 @@ def load_plugins_from_directories():
                     # Register the plugin with pluggy
                     plugin_manager.register(module, module_name)
                 except (ImportError, AttributeError) as e:
-                    print(f"Error loading plugin {module_name}: {e}")
+                    logger.critical(f"Error loading plugin {module_name}: {e}")
 
 # Load plugins
 load_plugins_from_directories()

changedetectionio/safe_jinja.py

@@ -18,7 +18,7 @@ def render(template_str, **args: t.Any) -> str:
     return output[:JINJA2_MAX_RETURN_PAYLOAD_SIZE]
 
 def render_fully_escaped(content):
-    env = jinja2.sandbox.ImmutableSandboxedEnvironment(autoescape=True)
+    env = jinja2.sandbox.ImmutableSandboxedEnvironment(autoescape=True, extensions=['jinja2_time.TimeExtension'])
     template = env.from_string("{{ some_html|e }}")
     return template.render(some_html=content)
 

changedetectionio/static/js/notifications.js

@@ -1,5 +1,18 @@
 $(document).ready(function () {
 
+    // Could be from 'watch' or system settings or other
+    function getNotificationData() {
+        data = {
+            notification_body: $('textarea.notification-body').val(),
+            notification_format: $('select.notification-format').val(),
+            notification_title: $('input.notification-title').val(),
+            notification_urls: $('textarea.notification-urls').val(),
+            tags: $('#tags').val(),
+            window_url: window.location.href,
+        }
+        return data
+    }
+
     $('#add-email-helper').click(function (e) {
         e.preventDefault();
         email = prompt("Destination email");
@@ -10,17 +23,82 @@ $(document).ready(function () {
         }
     });
 
+    $('#notifications-minitabs').miniTabs({
+        "Customise": "#notification-setup",
+        "Preview": "#notification-preview",
+    });
+
+    $(document).on('click', '[data-target="#notification-preview"]', function (e) {
+        var data = getNotificationData();
+        $('#notification-iframe-html-preview').contents().find('body').html('Loading...');
+        $.ajax({
+            type: "POST",
+            url: notification_test_render_preview_url,
+            data: data,
+            statusCode: {
+                400: function (data) {
+                    $('#notification-test-log').show().toggleClass('error', true);
+                    $("#notification-test-log>span").text(data.responseText);
+                },
+            }
+        }).done(function (data) {
+            $('#notification-test-log').toggleClass('error', false);
+            setPreview(data['result']);
+        })
+
+    });
+
+    function setPreview(data) {
+        const iframe = document.getElementById("notification-iframe-html-preview");
+        const isDark = document.documentElement.getAttribute('data-darkmode') === 'true';
+
+        // this should come back in the data objk
+        const isTextFormat = $('select.notification-format').val() === 'Text';
+
+        $('#notification-preview-title-text').text(data['title']);
+        $('#notification-div-text-preview').text(data['body']);
+        return;
+        iframe.srcdoc = `
+        <html data-darkmode="${isDark}">
+          <head>            
+            <style>
+                :root {
+                  --color-white: #fff;
+                  --color-grey-200: #333;
+                  --color-grey-800: #e0e0e0;
+                  --color-black: #000;
+                  --color-dark-red: #a00;
+                  --color-light-red: #dd0000;
+                  --color-background: var(--color-grey-800);
+                  --color-text: var(--color-grey-200);
+                  }
+                  
+                    html[data-darkmode="true"] {
+                      --color-background: var(--color-grey-200);
+                      --color-text: var(--color-white);
+                    }
+                    body { /* no darkmode */
+                        background-color: var(--color-background);
+                        color: var(--color-text);
+                        padding: 5px;
+                    }
+                    body.text-format {
+                        font-family: monospace;
+                        white-space: pre;
+                        overflow-wrap: normal;
+                        overflow-x: auto;
+                    }
+            </style>
+          </head>
+          <body class="${isTextFormat ? 'text-format' : ''}">${data['body']}</body>
+        </html>`;
+    }
+
+
     $('#send-test-notification').click(function (e) {
         e.preventDefault();
 
-        data = {
-            notification_body: $('#notification_body').val(),
-            notification_format: $('#notification_format').val(),
-            notification_title: $('#notification_title').val(),
-            notification_urls: $('.notification-urls').val(),
-            tags: $('#tags').val(),
-            window_url: window.location.href,
-        }
+        var data = getNotificationData();
 
         $('.notifications-wrapper .spinner').fadeIn();
         $('#notification-test-log').show();
@@ -30,11 +108,14 @@ $(document).ready(function () {
             data: data,
             statusCode: {
                 400: function (data) {
+                    $("#notification-test-log").toggleClass('error', true);
                     $("#notification-test-log>span").text(data.responseText);
                 },
             }
         }).done(function (data) {
-            $("#notification-test-log>span").text(data);
+            $("#notification-test-log").toggleClass('error', false);
+            $("#notification-test-log>span").text(data['status']);
+
         }).fail(function (jqXHR, textStatus, errorThrown) {
             // Handle connection refused or other errors
             if (textStatus === "error" && errorThrown === "") {
@@ -42,11 +123,13 @@ $(document).ready(function () {
                 $("#notification-test-log>span").text("Error: Connection refused or server is unreachable.");
             } else {
                 console.error("Error:", textStatus, errorThrown);
-                $("#notification-test-log>span").text("An error occurred: " + textStatus);
+                $("#notification-test-log>span").text("An error occurred: " + errorThrown);
             }
         }).always(function () {
             $('.notifications-wrapper .spinner').hide();
         })
     });
+
+
 });
 

changedetectionio/static/js/tabs.js

@@ -1,11 +1,11 @@
 // Rewrite this is a plugin.. is all this JS really 'worth it?'
 
 window.addEventListener('hashchange', function () {
-    var tabs = document.getElementsByClassName('active');
-    while (tabs[0]) {
-        tabs[0].classList.remove('active');
+    var tabs = document.querySelectorAll('.tabs .active');
+    tabs.forEach(function (tab) {
+        tab.classList.remove('active');
         document.body.classList.remove('full-width');
-    }
+    });
     set_active_tab();
 }, false);
 

changedetectionio/static/js/watch-settings.js

@@ -74,7 +74,7 @@ $(document).ready(function () {
         $('#filters-and-triggers input')[method]('change', request_textpreview_update.throttle(1000));
         $("#filters-and-triggers-tab")[method]('click', request_textpreview_update.throttle(1000));
     });
-    $('.minitabs-wrapper').miniTabs({
+    $('#filter-preview-minitabs').miniTabs({
         "Content after filters": "#text-preview-inner",
         "Content raw/before filters": "#text-preview-before-inner"
     });

changedetectionio/static/styles/scss/parts/_darkmode.scss

@@ -18,8 +18,15 @@ html[data-darkmode="true"] {
       display: block;
     }
   }
+
+  .minitabs-content {
+    > div {
+       background-color: rgb(249 249 249 / 13%) !important;
+    }
+  }
 }
 
 
 
 
+

changedetectionio/static/styles/scss/parts/_minitabs.scss

@@ -1,6 +1,13 @@
 .minitabs-wrapper {
   width: 100%;
 
+  .tab-contents-monospace-preview {
+    font-family: "Courier New", Courier, monospace; /* Sets the font to a monospace type */
+    font-size: 70%;
+    word-break: break-word;
+    white-space: pre-wrap; /* Preserves whitespace and line breaks like <pre> */
+  }
+
   > div[id] {
     padding: 20px;
     border: 1px solid #ccc;
@@ -10,38 +17,45 @@
   .minitabs-content {
     width: 100%;
     display: flex;
+
     > div {
       flex: 1 1 auto;
       min-width: 0;
-      overflow: scroll;
+      padding: 1rem;
+      border: 1px solid #ddd;
+      background-color: #eee;
     }
   }
 
   .minitabs {
     display: flex;
     border-bottom: 1px solid #ccc;
-  }
 
-  .minitab {
-    flex: 1;
-    text-align: center;
-    padding: 12px 0;
-    text-decoration: none;
-    color: #333;
-    background-color: #f1f1f1;
-    border: 1px solid #ccc;
-    border-bottom: none;
-    cursor: pointer;
-    transition: background-color 0.3s;
-  }
+    .minitab {
+      flex: 1;
+      text-align: center;
+      padding: 12px 0;
+      text-decoration: none;
+      color: #333;
+      background-color: #f1f1f1;
+      border: 1px solid #ccc;
+      border-bottom: none;
+      cursor: pointer;
+      transition: background-color 0.3s;
+      border-top-left-radius: 5px;
+      border-top-right-radius: 5px;
+      opacity: 0.45;
+      &:hover {
+        background-color: #ddd;
+      }
 
-  .minitab:hover {
-    background-color: #ddd;
+      &.active {
+        background-color: #eee;
+        font-weight: bold;
+        opacity: 1.0;
+      }
+    }
   }
-
-  .minitab.active {
-    background-color: #fff;
-    font-weight: bold;
-  }
-
 }
+
+

changedetectionio/static/styles/scss/parts/_notification.scss

@@ -0,0 +1,12 @@
+#notification-preview {
+  resize: both;
+  overflow: hidden;
+}
+
+#notification-iframe-html-preview {
+  width: 100%;
+  height: 100%;
+  border: 0;
+  display: block;
+  overflow: auto;
+}

changedetectionio/static/styles/scss/parts/_preview_text_filter.scss

@@ -1,5 +1,3 @@
-@use "minitabs";
-
 body.preview-text-enabled {
 
   @media (min-width: 800px) {
@@ -31,19 +29,7 @@ body.preview-text-enabled {
   }
 
   #activate-text-preview {
-      background-color: var(--color-grey-500);
-  }
-
-  /* actual preview area */
-  .monospace-preview {
-    background: var(--color-background-input);
-    border: 1px solid var(--color-grey-600);
-    padding: 1rem;
-    color: var(--color-text-input);
-    font-family: "Courier New", Courier, monospace; /* Sets the font to a monospace type */
-    font-size: 70%;
-    word-break: break-word;
-    white-space: pre-wrap; /* Preserves whitespace and line breaks like <pre> */
+    background-color: var(--color-grey-500);
   }
 }
 
@@ -53,3 +39,11 @@ body.preview-text-enabled {
   z-index: 3;
   box-shadow: 1px 1px 4px var(--color-shadow-jump);
 }
+
+#filter-preview-minitabs {
+  .minitabs-content {
+    > div {
+      overflow: scroll;
+    }
+  }
+}

changedetectionio/static/styles/scss/styles.scss

@@ -20,6 +20,8 @@
 @use "parts/lister_extra";
 @use "parts/socket";
 @use "parts/visualselector";
+@use "parts/_minitabs";
+@use "parts/_notification";
 @use "parts/widgets";
 
 body {
@@ -335,6 +337,11 @@ a.pure-button-selected {
     overflow-wrap: break-word;
     max-width: 100%;
     box-sizing: border-box;
+    &.error {
+      > span {
+        color: var(--color-error) !important;
+      }
+    }
   }
 }
 
@@ -344,11 +351,6 @@ label {
  }  
 }
 
-#notification-customisation {
-  border: 1px solid var(--color-border-notification);
-  padding: 0.5rem;
-  border-radius: 5px;
-}
 
 #notification-error-log {
   border: 1px solid var(--color-border-notification);

changedetectionio/templates/_common_fields.html

@@ -24,121 +24,153 @@
                               </ul>
                             </div>
                             <div class="notifications-wrapper">
-                              <a id="send-test-notification" class="pure-button button-secondary button-xsmall" >Send test notification</a> <div class="spinner"  style="display: none;"></div>
+                              <a id="send-test-notification" class="pure-button button-secondary" >Send test notification</a> <div class="spinner"  style="display: none;"></div>
                             {% if emailprefix %}
-                              <a id="add-email-helper" class="pure-button button-secondary button-xsmall" >Add email <img style="height: 1em; display: inline-block" src="{{url_for('static_content', group='images', filename='email.svg')}}" alt="Add an email address"> </a>
+                              <a id="add-email-helper" class="pure-button button-secondary" >Add email <img style="height: 1em; display: inline-block" src="{{url_for('static_content', group='images', filename='email.svg')}}" alt="Add an email address"> </a>
                             {% endif %}
-                              <a href="{{url_for('settings.notification_logs')}}" class="pure-button button-secondary button-xsmall" >Notification debug logs</a>
+                              <a href="{{url_for('settings.notification_logs')}}" class="pure-button button-secondary " >Notification debug logs</a>
                               <br>
                                 <div id="notification-test-log" style="display: none;"><span class="pure-form-message-inline">Processing..</span></div>
                             </div>
                         </div>
-                        <div id="notification-customisation" class="pure-control-group">
-                            <div class="pure-control-group">
-                                {{ render_field(form.notification_title, class="m-d notification-title", placeholder=settings_application['notification_title']) }}
-                                <span class="pure-form-message-inline">Title for all notifications</span>
-                            </div>
-                            <div class="pure-control-group">
-                                {{ render_field(form.notification_body , rows=5, class="notification-body", placeholder=settings_application['notification_body']) }}
-                                <span class="pure-form-message-inline">Body for all notifications &dash; You can use <a target="newwindow" href="https://jinja.palletsprojects.com/en/3.0.x/templates/">Jinja2</a> templating in the notification title, body and URL, and tokens from below.
-                                </span>
+                        <div class="pure-control-group">
+                        <p>Customise the contents of the notification using the form below, this is not necessary but you can create quite interesting integrations :-)</p>
+                            <div class="minitabs-wrapper" id="notifications-minitabs">
+                                <div class="minitabs-content">
+                                    <div id="notification-setup">
+                                        <div class="pure-control-group">
+                                            {{ render_field(form.notification_title, class="m-d notification-title", placeholder=settings_application['notification_title']) }}
+                                        </div>
+                                        <div class="pure-control-group">
+                                            {{ render_field(form.notification_body , rows=5, class="notification-body", placeholder=settings_application['notification_body']) }}
+                                            <span class="pure-form-message-inline">Body for the notification &dash; You can use <a
+                                                    target="newwindow"
+                                                    href="https://jinja.palletsprojects.com/en/3.0.x/templates/">Jinja2</a> templating in the notification title, body and URL, and tokens from below.
+                                            </span>
 
-                            </div>
-                            <div class="pure-controls">
-                                <div data-target="#notification-tokens-info" class="toggle-show pure-button button-tag button-xsmall">Show token/placeholders</div>
-                            </div>
-                            <div class="pure-controls" style="display: none;" id="notification-tokens-info">
-                                <table class="pure-table" id="token-table">
-                                    <thead>
-                                    <tr>
-                                        <th>Token</th>
-                                        <th>Description</th>
-                                    </tr>
-                                    </thead>
-                                    <tbody>
-                                    <tr>
-                                        <td><code>{{ '{{base_url}}' }}</code></td>
-                                        <td>The URL of the changedetection.io instance you are running.</td>
-                                    </tr>
-                                    <tr>
-                                        <td><code>{{ '{{watch_url}}' }}</code></td>
-                                        <td>The URL being watched.</td>
-                                    </tr>
-                                    <tr>
-                                        <td><code>{{ '{{watch_uuid}}' }}</code></td>
-                                        <td>The UUID of the watch.</td>
-                                    </tr>
-                                    <tr>
-                                        <td><code>{{ '{{watch_title}}' }}</code></td>
-                                        <td>The page title of the watch, uses &lt;title&gt; if not set, falls back to URL</td>
-                                    </tr>
-                                    <tr>
-                                        <td><code>{{ '{{watch_tag}}' }}</code></td>
-                                        <td>The watch group / tag</td>
-                                    </tr>
-                                    <tr>
-                                        <td><code>{{ '{{preview_url}}' }}</code></td>
-                                        <td>The URL of the preview page generated by changedetection.io.</td>
-                                    </tr>
-                                    <tr>
-                                        <td><code>{{ '{{diff_url}}' }}</code></td>
-                                        <td>The URL of the diff output for the watch.</td>
-                                    </tr>
-									<tr>
-                                        <td><code>{{ '{{diff}}' }}</code></td>
-                                        <td>The diff output - only changes, additions, and removals</td>
-                                    </tr>
-									<tr>
-                                        <td><code>{{ '{{diff_added}}' }}</code></td>
-                                        <td>The diff output - only changes and additions</td>
-                                    </tr>
-									<tr>
-                                        <td><code>{{ '{{diff_removed}}' }}</code></td>
-                                        <td>The diff output - only changes and removals</td>
-                                    </tr>
-                                    <tr>
-                                        <td><code>{{ '{{diff_full}}' }}</code></td>
-                                        <td>The diff output - full difference output</td>
-                                    </tr>
-                                    <tr>
-                                        <td><code>{{ '{{diff_patch}}' }}</code></td>
-                                        <td>The diff output - patch in unified format</td>
-                                    </tr>
-                                    <tr>
-                                        <td><code>{{ '{{current_snapshot}}' }}</code></td>
-                                        <td>The current snapshot text contents value, useful when combined with JSON or CSS filters
-                                        </td>
-                                    </tr>
-                                    <tr>
-                                        <td><code>{{ '{{triggered_text}}' }}</code></td>
-                                        <td>Text that tripped the trigger from filters</td>
-
-                                        {% if extra_notification_token_placeholder_info %}
-                                            {% for token in extra_notification_token_placeholder_info %}
+                                        </div>
+                                        <div class="pure-controls">
+                                            <div data-target="#notification-tokens-info"
+                                                 class="toggle-show pure-button button-tag button-xsmall">Show
+                                                token/placeholders
+                                            </div>
+                                        </div>
+                                        <div class="pure-controls" style="display: none;" id="notification-tokens-info">
+                                            <table class="pure-table" id="token-table">
+                                                <thead>
                                                 <tr>
-                                                    <td><code>{{ '{{' }}{{ token[0] }}{{ '}}' }}</code></td>
-                                                    <td>{{ token[1] }}</td>
+                                                    <th>Token</th>
+                                                    <th>Description</th>
                                                 </tr>
-                                            {% endfor %}
-                                        {% endif %}
-                                    </tbody>
-                                </table>
-                                <div class="pure-form-message-inline">
-                                    <p>
-									Warning: Contents of <code>{{ '{{diff}}' }}</code>, <code>{{ '{{diff_removed}}' }}</code>, and <code>{{ '{{diff_added}}' }}</code> depend on how the difference algorithm perceives the change. <br>
-                                    For example, an addition or removal could be perceived as a change in some cases. <a target="newwindow" href="https://github.com/dgtlmoon/changedetection.io/wiki/Using-the-%7B%7Bdiff%7D%7D,-%7B%7Bdiff_added%7D%7D,-and-%7B%7Bdiff_removed%7D%7D-notification-tokens">More Here</a> <br>
-                                    </p>
-                                    <p>
-                                        For JSON payloads, use <strong>|tojson</strong> without quotes for automatic escaping, for example - <code>{ "name": {{ '{{ watch_title|tojson }}' }} }</code>
-                                    </p>
-                                    <p>
-                                        URL encoding, use <strong>|urlencode</strong>, for example - <code>gets://hook-website.com/test.php?title={{ '{{ watch_title|urlencode }}' }}</code>
-                                    </p>
+                                                </thead>
+                                                <tbody>
+                                                <tr>
+                                                    <td><code>{{ '{{base_url}}' }}</code></td>
+                                                    <td>The URL of the changedetection.io instance you are running.</td>
+                                                </tr>
+                                                <tr>
+                                                    <td><code>{{ '{{watch_url}}' }}</code></td>
+                                                    <td>The URL being watched.</td>
+                                                </tr>
+                                                <tr>
+                                                    <td><code>{{ '{{watch_uuid}}' }}</code></td>
+                                                    <td>The UUID of the watch.</td>
+                                                </tr>
+                                                <tr>
+                                                    <td><code>{{ '{{watch_title}}' }}</code></td>
+                                                    <td>The page title of the watch, uses &lt;title&gt; if not set, falls back to URL</td>
+                                                </tr>
+                                                <tr>
+                                                    <td><code>{{ '{{watch_tag}}' }}</code></td>
+                                                    <td>The watch group / tag</td>
+                                                </tr>
+                                                <tr>
+                                                    <td><code>{{ '{{preview_url}}' }}</code></td>
+                                                    <td>The URL of the preview page generated by changedetection.io.
+                                                    </td>
+                                                </tr>
+                                                <tr>
+                                                    <td><code>{{ '{{diff_url}}' }}</code></td>
+                                                    <td>The URL of the diff output for the watch.</td>
+                                                </tr>
+                                                <tr>
+                                                    <td><code>{{ '{{diff}}' }}</code></td>
+                                                    <td>The diff output - only changes, additions, and removals</td>
+                                                </tr>
+                                                <tr>
+                                                    <td><code>{{ '{{diff_added}}' }}</code></td>
+                                                    <td>The diff output - only changes and additions</td>
+                                                </tr>
+                                                <tr>
+                                                    <td><code>{{ '{{diff_removed}}' }}</code></td>
+                                                    <td>The diff output - only changes and removals</td>
+                                                </tr>
+                                                <tr>
+                                                    <td><code>{{ '{{diff_full}}' }}</code></td>
+                                                    <td>The diff output - full difference output</td>
+                                                </tr>
+                                                <tr>
+                                                    <td><code>{{ '{{diff_patch}}' }}</code></td>
+                                                    <td>The diff output - patch in unified format</td>
+                                                </tr>
+                                                <tr>
+                                                    <td><code>{{ '{{current_snapshot}}' }}</code></td>
+                                                    <td>The current snapshot text contents value, useful when combined
+                                                        with JSON or CSS filters
+                                                    </td>
+                                                </tr>
+                                                <tr>
+                                                    <td><code>{{ '{{triggered_text}}' }}</code></td>
+                                                    <td>Text that tripped the trigger from filters</td>
+                                                </tr>
+                                                    {% if extra_notification_token_placeholder_info %}
+                                                        {% for token in extra_notification_token_placeholder_info %}
+                                                            <tr>
+                                                                <td><code>{{ '{{' }}{{ token[0] }}{{ '}}' }}</code></td>
+                                                                <td>{{ token[1] }}</td>
+                                                            </tr>
+                                                        {% endfor %}
+                                                    {% endif %}
+                                                </tbody>
+                                            </table>
+                                            <div class="pure-form-message-inline">
+                                                <p>
+                                                    Warning: Contents of <code>{{ '{{diff}}' }}</code>,
+                                                    <code>{{ '{{diff_removed}}' }}</code>, and
+                                                    <code>{{ '{{diff_added}}' }}</code> depend on how the difference
+                                                    algorithm perceives the change. <br>
+                                                    For example, an addition or removal could be perceived as a change
+                                                    in some cases. <a target="newwindow"
+                                                                      href="https://github.com/dgtlmoon/changedetection.io/wiki/Using-the-%7B%7Bdiff%7D%7D,-%7B%7Bdiff_added%7D%7D,-and-%7B%7Bdiff_removed%7D%7D-notification-tokens">More
+                                                    Here</a> <br>
+                                                </p>
+                                                <p>
+                                                    For JSON payloads, use <strong>|tojson</strong> without quotes for
+                                                    automatic escaping, for example - <code>{
+                                                    "name": {{ '{{ watch_title|tojson }}' }} }</code>
+                                                </p>
+                                                <p>
+                                                    URL encoding, use <strong>|urlencode</strong>, for example - <code>gets://hook-website.com/test.php?title={{ '{{ watch_title|urlencode }}' }}</code>
+                                                </p>
+                                            </div>
+                                        </div>
+                                        <div class="pure-control-group">
+                                            {{ render_field(form.notification_format , class="notification-format") }}
+                                            <span class="pure-form-message-inline">Format for all notifications</span>
+                                        </div>
+                                    </div>
+                                    <div id="notification-preview" style="display: none; height:100%; display:flex; flex-direction:column;">
+                                        <p><strong>Title: </strong><span id="notification-preview-title-text">Preview loading..</span></p>
+                                        <div style="flex:1; display:flex; flex-direction:column;">
+                                            <strong>Body: </strong>
+                                            <div id="notification-div-text-preview" style="flex:1; height:95%; width:100%; border-radius:4px; margin-top:0.5rem; border:none;"></div>
+                                            <iframe id="notification-iframe-html-preview"
+                                                    style="flex:1; height:95%; width:100%; border-radius:4px; margin-top:0.5rem; border:none;">
+                                                Preview loading...
+                                            </iframe>
+                                        </div>
+                                    </div>
                                 </div>
                             </div>
-                            <div class="pure-control-group">
-                                {{ render_field(form.notification_format , class="notification-format") }}
-                                <span class="pure-form-message-inline">Format for all notifications</span>
-                            </div>
                         </div>
-{% endmacro %}
+{% endmacro %}

changedetectionio/templates/_helpers.html

@@ -27,8 +27,15 @@
 {% endmacro %}
 
 {% macro render_checkbox_field(field) %}
-  <div class="checkbox {% if field.errors %} error {% endif %}">
+  <div class="checkbox {% if field.errors or field.top_errors %} error {% endif %}">
   {{ field(**kwargs)|safe }} {{ field.label }}
+  {% if field.top_errors %}
+    <ul class="errors top-errors">
+    {% for error in field.top_errors %}
+      <li>{{ error }}</li>
+    {% endfor %}
+    </ul>
+  {% endif %}
   {% if field.errors %}
     <ul class=errors>
     {% for error in field.errors %}
@@ -43,9 +50,16 @@
   {% if BooleanField %}
     {% set _ = field.__setattr__('boolean_mode', true) %}
   {% endif %}
-  <div class="ternary-field {% if field.errors %} error {% endif %}">
+  <div class="ternary-field {% if field.errors or field.top_errors %} error {% endif %}">
     <div class="ternary-field-label">{{ field.label }}</div>
     <div class="ternary-field-widget">{{ field(**kwargs)|safe }}</div>
+    {% if field.top_errors %}
+      <ul class="errors top-errors">
+      {% for error in field.top_errors %}
+        <li>{{ error }}</li>
+      {% endfor %}
+      </ul>
+    {% endif %}
     {% if field.errors %}
       <ul class=errors>
       {% for error in field.errors %}
@@ -58,8 +72,15 @@
 
 
 {% macro render_simple_field(field) %}
-  <span class="label {% if field.errors %}error{% endif %}">{{ field.label }}</span>
-  <span {% if field.errors %} class="error" {% endif %}>{{ field(**kwargs)|safe }}
+  <span class="label {% if field.errors or field.top_errors %}error{% endif %}">{{ field.label }}</span>
+  <span {% if field.errors or field.top_errors %} class="error" {% endif %}>{{ field(**kwargs)|safe }}
+  {% if field.top_errors %}
+    <ul class="errors top-errors">
+    {% for error in field.top_errors %}
+      <li>{{ error }}</li>
+    {% endfor %}
+    </ul>
+  {% endif %}
   {% if field.errors %}
     <ul class=errors>
     {% for error in field.errors %}
@@ -74,8 +95,15 @@
 {% macro render_nolabel_field(field) %}
     <span>
     {{ field(**kwargs)|safe }}
-        {% if field.errors %}
+        {% if field.top_errors or field.errors %}
             <span class="error">
+      {% if field.top_errors %}
+          <ul class="errors top-errors">
+        {% for error in field.top_errors %}
+            <li>{{ error }}</li>
+        {% endfor %}
+        </ul>
+      {% endif %}
       {% if field.errors %}
           <ul class=errors>
         {% for error in field.errors %}

changedetectionio/tests/smtp/test_notification_smtp.py

@@ -1,17 +1,17 @@
-import json
 import os
 import time
-import re
 from flask import url_for
 from changedetectionio.tests.util import set_original_response, set_modified_response, set_more_modified_response, live_server_setup, \
     wait_for_all_checks, \
     set_longer_modified_response
-from changedetectionio.tests.util import extract_UUID_from_client
 import logging
-import base64
 
 # NOTE - RELIES ON mailserver as hostname running, see github build recipes
-smtp_test_server = 'mailserver'
+# Should be hostname (never IP), looks for our test mailserver that repeats the content
+# python3 changedetectionio/tests/smtp/smtp-test-server.py &
+# mailserver=localhost pytest tests/smtp/test_notification_smtp.py::test_check_notification_email_formats_default_HTML
+smtp_test_server = os.getenv('mailserver', 'mailserver')
+
 
 from changedetectionio.notification import (
     default_notification_body,
@@ -20,7 +20,35 @@ from changedetectionio.notification import (
     valid_notification_formats,
 )
 
+from email import policy
+from email.parser import BytesParser, Parser
 
+def parse_mime(raw):
+    """Return (EmailMessage, dict[str, list[str]] bodies by content-type)."""
+    if isinstance(raw, (bytes, bytearray)):
+        msg = BytesParser(policy=policy.default).parsebytes(raw)
+    else:
+        msg = Parser(policy=policy.default).parsestr(raw)
+
+    parts_by_type = {}
+    if msg.is_multipart():
+        for part in msg.walk():
+            if part.get_content_maintype() == "multipart":
+                continue
+            ctype = part.get_content_type()           # e.g. "text/plain"
+            text = part.get_content()                 # decoded str
+            parts_by_type.setdefault(ctype, []).append(text)
+    else:
+        parts_by_type.setdefault(msg.get_content_type(), []).append(msg.get_content())
+
+    return msg, parts_by_type
+
+def one_or_join(parts_dict, ctype):
+    """Join multiple parts of the same type (rare but possible)."""
+    return "\n".join(parts_dict.get(ctype, []))
+
+def norm_newlines(s: str) -> str:
+    return s.replace("\r\n", "\n").replace("\r", "\n")
 
 def get_last_message_from_smtp_server():
     import socket
@@ -77,37 +105,36 @@ def test_check_notification_email_formats_default_HTML(client, live_server, meas
 
     time.sleep(3)
 
-    msg = get_last_message_from_smtp_server()
-    assert len(msg) >= 1
+    raw = get_last_message_from_smtp_server()
+    assert raw  # not empty
+
+    msg, bodies = parse_mime(raw)
+
+    plain = norm_newlines(one_or_join(bodies, "text/plain"))
+    html = norm_newlines(one_or_join(bodies, "text/html"))
+
+    # Now assert against the decoded bodies
+    assert "(added) So let's see what happens.\n" in plain  # plaintext uses a literal apostrophe
+    assert "(added) So let&#39;s see what happens.<br>" in html  # html uses &#39; and <br>
+
+    # You can also check counts, boundaries, etc.
+    assert html.count("So let&#39;s see what happens.") == 3
+    assert "modified head title had a change." in plain
+    assert "modified head title had a change.<br>" in html
+
+
 
-    # The email should have two bodies, and the text/html part should be <br>
-    assert 'Content-Type: text/plain' in msg
-    assert '(added) So let\'s see what happens.\r\n' in msg  # The plaintext part with \r\n
-    assert 'Content-Type: text/html' in msg
-    assert '(added) So let\'s see what happens.<br>' in msg  # the html part
     res = client.get(url_for("ui.form_delete", uuid="all"), follow_redirects=True)
     assert b'Deleted' in res.data
 
 
 def test_check_notification_email_formats_default_Text_override_HTML(client, live_server, measure_memory_usage):
-    ##  live_server_setup(live_server) # Setup on conftest per function
 
     # HTML problems? see this
     # https://github.com/caronc/apprise/issues/633
-
     set_original_response()
     notification_url = f'mailto://changedetection@{smtp_test_server}:11025/?to=fff@home.com'
-    notification_body = f"""<!DOCTYPE html>
-<html lang="en">
-<head>
-    <title>My Webpage</title>
-</head>
-<body>
-    <h1>Test</h1>
-    {default_notification_body}
-</body>
-</html>
-"""
+    notification_body = f"""{default_notification_body}"""
 
     #####################
     # Set this up for when we remove the notification from the watch, it should fallback with these details
@@ -116,11 +143,12 @@ def test_check_notification_email_formats_default_Text_override_HTML(client, liv
         data={"application-notification_urls": notification_url,
               "application-notification_title": "fallback-title " + default_notification_title,
               "application-notification_body": notification_body,
-              "application-notification_format": 'Text',
+              "application-notification_format": 'Text', # handler.py should be sure to add &format=text to override default html from apprise
               "requests-time_between_check-minutes": 180,
               'application-fetch_backend': "html_requests"},
         follow_redirects=True
     )
+
     assert b"Settings updated." in res.data
 
     # Add a watch and trigger a HTTP POST
@@ -140,18 +168,26 @@ def test_check_notification_email_formats_default_Text_override_HTML(client, liv
     wait_for_all_checks(client)
 
     time.sleep(3)
-    msg = get_last_message_from_smtp_server()
-    assert len(msg) >= 1
-    #    with open('/tmp/m.txt', 'w') as f:
-    #        f.write(msg)
+    raw = get_last_message_from_smtp_server()
+    assert raw
 
-    # The email should not have two bodies, should be TEXT only
+    msg, bodies = parse_mime(raw)
 
-    assert 'Content-Type: text/plain' in msg
-    assert '(added) So let\'s see what happens.\r\n' in msg  # The plaintext part with \r\n
+    plain = norm_newlines(one_or_join(bodies, "text/plain"))
+    html = norm_newlines(one_or_join(bodies, "text/html"))
+    assert not html # should be no HTML here
 
+    # Expect ONLY text/plain body
+    assert "text/plain" in bodies
+    assert "text/html" not in bodies
+
+    # Assert on decoded plaintext (literal apostrophe, not &#39;)
+    # Should be NO markup when in text mode
+    assert "(added) So let's see what happens.\n" in plain
+
+
+    # ---------- Flip to HTML format, then expect multipart with both ----------
     set_original_response()
-    # Now override as HTML format
     res = client.post(
         url_for("ui.ui_edit.edit_page", uuid="first"),
         data={
@@ -161,23 +197,28 @@ def test_check_notification_email_formats_default_Text_override_HTML(client, liv
             "time_between_check_use_default": "y"},
         follow_redirects=True
     )
+
     assert b"Updated watch." in res.data
     wait_for_all_checks(client)
 
     time.sleep(3)
-    msg = get_last_message_from_smtp_server()
-    assert len(msg) >= 1
 
-    # The email should have two bodies, and the text/html part should be <br>
-    assert 'Content-Type: text/plain' in msg
-    assert '(removed) So let\'s see what happens.\r\n' in msg  # The plaintext part with \n
-    assert 'Content-Type: text/html' in msg
-    assert '(removed) So let\'s see what happens.<br>' in msg  # the html part
+    raw = get_last_message_from_smtp_server()
+    assert raw
 
-    # https://github.com/dgtlmoon/changedetection.io/issues/2103
-    assert '<h1>Test</h1>' in msg
-    assert '&lt;' not in msg
-    assert 'Content-Type: text/html' in msg
+    msg, bodies = parse_mime(raw)
+    plain = norm_newlines(one_or_join(bodies, "text/plain"))
+    html = norm_newlines(one_or_join(bodies, "text/html"))
 
-    res = client.get(url_for("ui.form_delete", uuid="all"), follow_redirects=True)
-    assert b'Deleted' in res.data
+    # Expect both text/plain and text/html bodies now
+    assert "text/plain" in bodies
+    assert "text/html" in bodies
+
+    # Plaintext reflects the removal line (literal apostrophe)
+    assert "(removed) So let's see what happens.\n" in plain
+    assert "(removed) So let&#39;s see what happens.<br>" in html
+
+    # Optional: ensure we got multipart/alternative (typical for dual bodies)
+    if msg.is_multipart():
+        # most senders do "multipart/alternative" for text/plain + text/html
+        assert msg.get_content_subtype() in ("alternative", "mixed", "related")

changedetectionio/tests/test_jinja2.py

@@ -3,6 +3,7 @@
 import time
 from flask import url_for
 from .util import live_server_setup, wait_for_all_checks
+from ..notification import default_notification_title, default_notification_body, default_notification_format
 
 
 # def test_setup(client, live_server, measure_memory_usage):
@@ -10,7 +11,6 @@ from .util import live_server_setup, wait_for_all_checks
 
 # If there was only a change in the whitespacing, then we shouldnt have a change detected
 def test_jinja2_in_url_query(client, live_server, measure_memory_usage):
-    
 
     # Add our URL to the import page
     test_url = url_for('test_return_query', _external=True)
@@ -56,3 +56,20 @@ def test_jinja2_security_url_query(client, live_server, measure_memory_usage):
     assert b'is invalid and cannot be used' in res.data
     # Some of the spewed output from the subclasses
     assert b'dict_values' not in res.data
+
+
+def test_jinja2_notification(client, live_server, measure_memory_usage):
+
+    res = client.post(
+        url_for("settings.settings_page"),
+        data={"application-notification_urls": "posts://127.0.0.1",
+              "application-notification_title": "on the {% now  'America/New_York', '%Y-%m-%d' %}",
+              "application-notification_body": "on the {% now  'America/New_York', '%Y-%m-%d' %}",
+              "application-notification_format": default_notification_format,
+              "requests-time_between_check-minutes": 180,
+              'application-fetch_backend': "html_requests"},
+        follow_redirects=True
+    )
+
+    assert b"Settings updated." in res.data
+    assert b"Settings updated." in res.data

changedetectionio/tests/test_notification.py

@@ -291,6 +291,20 @@ def test_notification_custom_endpoint_and_jinja2(client, live_server, measure_me
     # test_endpoint - that sends the contents of a file
     # test_notification_endpoint - that takes a POST and writes it to file (test-datastore/notification.txt)
 
+    # Drop in a custom wrapping template
+    with open("test-datastore/notification-wrapper.html", "w" ) as f:
+        f.write("""<html>
+        <body id="notification-wrapper">
+        
+        A change was detected at {{watch_html_link}}
+                        template_params = notification_parameters.copy()
+                template_params['notification_body'] = n_body
+                template_params['notification_url_current'] = url
+                n_body = jinja_render(template_str=notification_template, **template_params)
+        </body>
+        """)
+
+
     # CUSTOM JSON BODY CHECK for POST://
     set_original_response()
     # https://github.com/caronc/apprise/wiki/Notify_Custom_JSON#header-manipulation