Merge branch 'master' into 3159-test-notification-send

.github/dependabot.yml

@@ -4,11 +4,11 @@ updates:
     directory: /
     schedule:
       interval: "weekly"
-    "caronc/apprise":
-      versioning-strategy: "increase"
-      schedule:
-        interval: "daily"
     groups:
       all:
         patterns:
         - "*"
+  - package-ecosystem: pip
+    directory: /
+    schedule:
+      interval: "weekly"

.github/workflows/containers.yml

@@ -95,7 +95,7 @@ jobs:
           push: true
           tags: |
             ${{ secrets.DOCKER_HUB_USERNAME }}/changedetection.io:dev,ghcr.io/${{ github.repository }}:dev
-          platforms: linux/amd64,linux/arm64,linux/arm/v7,linux/arm/v8,linux/arm64/v8
+          platforms: linux/amd64,linux/arm64,linux/arm/v7,linux/arm/v8
           cache-from: type=gha
           cache-to: type=gha,mode=max
 
@@ -133,7 +133,7 @@ jobs:
           file: ./Dockerfile
           push: true
           tags: ${{ steps.meta.outputs.tags }}
-          platforms: linux/amd64,linux/arm64,linux/arm/v7,linux/arm/v8,linux/arm64/v8
+          platforms: linux/amd64,linux/arm64,linux/arm/v7,linux/arm/v8
           cache-from: type=gha
           cache-to: type=gha,mode=max
 # Looks like this was disabled

.github/workflows/test-container-build.yml

@@ -38,8 +38,6 @@ jobs:
             dockerfile: ./Dockerfile
           - platform: linux/arm/v8
             dockerfile: ./Dockerfile
-          - platform: linux/arm64/v8
-            dockerfile: ./Dockerfile
           # Alpine Dockerfile platforms (musl via alpine check)
           - platform: linux/amd64
             dockerfile: ./.github/test/Dockerfile-alpine
@@ -76,5 +74,5 @@ jobs:
             file: ${{ matrix.dockerfile }}
             platforms: ${{ matrix.platform }}
             cache-from: type=gha
-            cache-to: type=gha,mode=max
+            cache-to: type=gha,mode=min
 

.github/workflows/test-stack-reusable-workflow.yml

@@ -71,7 +71,6 @@ jobs:
           docker run test-changedetectionio  bash -c 'python3 -m unittest changedetectionio.tests.unit.test_watch_model'
           docker run test-changedetectionio  bash -c 'python3 -m unittest changedetectionio.tests.unit.test_jinja2_security'
           docker run test-changedetectionio  bash -c 'python3 -m unittest changedetectionio.tests.unit.test_semver'
-          docker run test-changedetectionio  bash -c 'python3 -m unittest changedetectionio.tests.unit.test_browser_notifications'
 
       - name: Test built container with Pytest (generally as requests/plaintext fetching)
         run: |

changedetectionio/__init__.py

@@ -2,7 +2,7 @@
 
 # Read more https://github.com/dgtlmoon/changedetection.io/wiki
 
-__version__ = '0.50.14'
+__version__ = '0.50.17'
 
 from changedetectionio.strtobool import strtobool
 from json.decoder import JSONDecodeError

changedetectionio/async_update_worker.py

@@ -91,7 +91,7 @@ async def async_update_worker(worker_id, q, notification_q, app, datastore):
                     try:
                         processor_module = importlib.import_module(processor_module_name)
                     except ModuleNotFoundError as e:
-                        print(f"Processor module '{processor}' not found.")
+                        logger.error(f"Processor module '{processor}' not found.")
                         raise e
 
                     update_handler = processor_module.perform_site_check(datastore=datastore,

changedetectionio/blueprint/browser_notifications/__init__.py

@@ -1 +0,0 @@
-# Browser notifications blueprint

changedetectionio/blueprint/browser_notifications/browser_notifications.py

@@ -1,76 +0,0 @@
-from flask import Blueprint, jsonify, request
-from loguru import logger
-
-
-def construct_blueprint(datastore):
-    browser_notifications_blueprint = Blueprint('browser_notifications', __name__)
-
-    @browser_notifications_blueprint.route("/test", methods=['POST'])
-    def test_browser_notification():
-        """Send a test browser notification using the apprise handler"""
-        try:
-            from changedetectionio.notification.apprise_plugin.custom_handlers import apprise_browser_notification_handler
-            
-            # Check if there are any subscriptions
-            browser_subscriptions = datastore.data.get('settings', {}).get('application', {}).get('browser_subscriptions', [])
-            if not browser_subscriptions:
-                return jsonify({'success': False, 'message': 'No browser subscriptions found'}), 404
-            
-            # Get notification data from request or use defaults
-            data = request.get_json() or {}
-            title = data.get('title', 'Test Notification')
-            body = data.get('body', 'This is a test notification from changedetection.io')
-            
-            # Use the apprise handler directly
-            success = apprise_browser_notification_handler(
-                body=body,
-                title=title,
-                notify_type='info',
-                meta={'url': 'browser://test'}
-            )
-            
-            if success:
-                subscription_count = len(browser_subscriptions)
-                return jsonify({
-                    'success': True,
-                    'message': f'Test notification sent successfully to {subscription_count} subscriber(s)'
-                })
-            else:
-                return jsonify({'success': False, 'message': 'Failed to send test notification'}), 500
-                
-        except ImportError:
-            logger.error("Browser notification handler not available")
-            return jsonify({'success': False, 'message': 'Browser notification handler not available'}), 500
-        except Exception as e:
-            logger.error(f"Failed to send test browser notification: {e}")
-            return jsonify({'success': False, 'message': f'Error: {str(e)}'}), 500
-
-    @browser_notifications_blueprint.route("/clear", methods=['POST'])
-    def clear_all_browser_notifications():
-        """Clear all browser notification subscriptions from the datastore"""
-        try:
-            # Get current subscription count
-            browser_subscriptions = datastore.data.get('settings', {}).get('application', {}).get('browser_subscriptions', [])
-            subscription_count = len(browser_subscriptions)
-            
-            # Clear all subscriptions
-            if 'settings' not in datastore.data:
-                datastore.data['settings'] = {}
-            if 'application' not in datastore.data['settings']:
-                datastore.data['settings']['application'] = {}
-                
-            datastore.data['settings']['application']['browser_subscriptions'] = []
-            datastore.needs_write = True
-            
-            logger.info(f"Cleared {subscription_count} browser notification subscriptions")
-            
-            return jsonify({
-                'success': True, 
-                'message': f'Cleared {subscription_count} browser notification subscription(s)'
-            })
-            
-        except Exception as e:
-            logger.error(f"Failed to clear all browser notifications: {e}")
-            return jsonify({'success': False, 'message': f'Clear all failed: {str(e)}'}), 500
-
-    return browser_notifications_blueprint

changedetectionio/blueprint/settings/templates/settings.html

@@ -5,6 +5,7 @@
 {% from '_common_fields.html' import render_common_settings_form %}
 <script>
     const notification_base_url="{{url_for('ui.ui_notification.ajax_callback_send_notification_test', mode="global-settings")}}";
+    const notification_test_render_preview_url="{{url_for('ui.ui_notification.ajax_callback_test_render_preview', mode="global-settings")}}";
 {% if emailprefix %}
     const email_notification_prefix=JSON.parse('{{emailprefix|tojson}}');
 {% endif %}
@@ -43,10 +44,6 @@
                                 </div>
                         </div>
                     </div>
-                    <div class="pure-control-group">
-                        {{ render_field(form.requests.form.jitter_seconds, class="jitter_seconds") }}
-                        <span class="pure-form-message-inline">Example - 3 seconds random jitter could trigger up to 3 seconds earlier or up to 3 seconds later</span>
-                    </div>
                     <div class="pure-control-group">
                         {{ render_field(form.application.form.filter_failure_notification_threshold_attempts, class="filter_failure_notification_threshold_attempts") }}
                         <span class="pure-form-message-inline">After this many consecutive times that the CSS/xPath filter is missing, send a notification
@@ -133,6 +130,10 @@
                     <span class="pure-form-message-inline">Number of concurrent workers to process watches. More workers = faster processing but higher memory usage.<br>
                     Currently running: <strong>{{ worker_info.count }}</strong> operational {{ worker_info.type }} workers{% if worker_info.active_workers > 0 %} ({{ worker_info.active_workers }} actively processing){% endif %}.</span>
                 </div>
+                <div class="pure-control-group">
+                    {{ render_field(form.requests.form.jitter_seconds, class="jitter_seconds") }}
+                    <span class="pure-form-message-inline">Example - 3 seconds random jitter could trigger up to 3 seconds earlier or up to 3 seconds later</span>
+                </div>
                 <div class="pure-control-group inline-radio">
                     {{ render_field(form.requests.form.default_ua) }}
                     <span class="pure-form-message-inline">
@@ -191,24 +192,21 @@ nav
                         </ul>
                      </span>
                     </fieldset>
+                    <fieldset class="pure-group">
+                        {{ render_checkbox_field(form.application.form.strip_ignored_lines) }}
+                        <span class="pure-form-message-inline">Remove any text that appears in the "Ignore text" from the output (otherwise its just ignored for change-detection)<br>
+                        <i>Note:</i> Changing this will change the status of your existing watches, possibly trigger alerts etc.
+                        </span>
+                    </fieldset>
            </div>
 
             <div class="tab-pane-inner" id="api">
-                <h4>API Access</h4>
-                <p>Drive your changedetection.io via API, More about <a href="https://changedetection.io/docs/api_v1/index.html">API access and examples here</a>.</p>
+                <p>
+                    <strong>Chrome extension and API Access</strong><br>
+                </p>
 
-                <div class="pure-control-group">
-                    {{ render_checkbox_field(form.application.form.api_access_token_enabled) }}
-                    <div class="pure-form-message-inline">Restrict API access limit by using <code>x-api-key</code> header - required for the Chrome Extension to work</div><br>
-                    <div class="pure-form-message-inline"><br>API Key <span id="api-key">{{api_key}}</span>
-                        <span style="display:none;" id="api-key-copy" >copy</span>
-                    </div>
-                </div>
-                <div class="pure-control-group">
-                    <a href="{{url_for('settings.settings_reset_api_key')}}" class="pure-button button-small button-cancel">Regenerate API key</a>
-                </div>
-                <div class="pure-control-group">
-                    <h4>Chrome Extension</h4>
+                <div class="pure-control-group border-fieldset">
+                    <strong>Chrome Extension</strong><br>
                     <p>Easily add any web-page to your changedetection.io installation from within Chrome.</p>
                     <strong>Step 1</strong> Install the extension, <strong>Step 2</strong> Navigate to this page,
                     <strong>Step 3</strong> Open the extension from the toolbar and click "<i>Sync API Access</i>"
@@ -221,6 +219,20 @@ nav
                         </a>
                     </p>
                 </div>
+                <div class="pure-control-group  border-fieldset">
+                    Drive your changedetection.io via API, More about <a href="https://changedetection.io/docs/api_v1/index.html">API access and examples here</a>.<br>
+                    <p>
+                        {{ render_checkbox_field(form.application.form.api_access_token_enabled) }}
+                    </p>
+                    <div class="pure-form-message-inline">Restrict API access limit by using <code>x-api-key</code> header - required for the Chrome Extension to work</div><br>
+                    <div class="pure-form-message-inline"><br>API Key <span id="api-key">{{api_key}}</span>
+                        <span style="display:none;" id="api-key-copy" >copy</span>
+                    </div>
+                    <p>
+                        <a href="{{url_for('settings.settings_reset_api_key')}}" class="pure-button button-small button-cancel">Regenerate API key</a>
+                    </p>
+                </div>
+
             </div>
             <div class="tab-pane-inner" id="timedate">
                 <div class="pure-control-group">

changedetectionio/blueprint/tags/templates/edit-tag.html

@@ -4,6 +4,8 @@
 {% from '_common_fields.html' import render_common_settings_form %}
 <script>
     const notification_base_url="{{url_for('ui.ui_notification.ajax_callback_send_notification_test', mode="group-settings")}}";
+    const notification_test_render_preview_url="{{url_for('ui.ui_notification.ajax_callback_test_render_preview', mode="group-settings", watch_uuid=data.uuid)}}";
+    //alert(notification_test_render_preview_url)
 </script>
 
 <script src="{{url_for('static_content', group='js', filename='tabs.js')}}" defer></script>
@@ -19,6 +21,8 @@
 
 <script src="{{url_for('static_content', group='js', filename='watch-settings.js')}}" defer></script>
 <script src="{{url_for('static_content', group='js', filename='notifications.js')}}" defer></script>
+<script src="{{url_for('static_content', group='js', filename='plugins.js')}}" defer></script>
+
 
 <div class="edit-form monospaced-textarea">
 

changedetectionio/blueprint/ui/__init__.py

@@ -106,14 +106,14 @@ def _handle_operations(op, uuids, datastore, worker_handler, update_q, queuedWat
         for uuid in uuids:
             watch_check_update.send(watch_uuid=uuid)
 
-def construct_blueprint(datastore: ChangeDetectionStore, update_q, worker_handler, queuedWatchMetaData, watch_check_update):
+def construct_blueprint(datastore: ChangeDetectionStore, update_q, worker_handler, queuedWatchMetaData, watch_check_update, notification_q):
     ui_blueprint = Blueprint('ui', __name__, template_folder="templates")
     
     # Register the edit blueprint
     edit_blueprint = construct_edit_blueprint(datastore, update_q, queuedWatchMetaData)
     ui_blueprint.register_blueprint(edit_blueprint)
     
-    # Register the notification blueprint
+    # Register the notification blueprint - mostly used for sending test
     notification_blueprint = construct_notification_blueprint(datastore)
     ui_blueprint.register_blueprint(notification_blueprint)
     

changedetectionio/blueprint/ui/notification.py

@@ -1,50 +1,85 @@
-from flask import Blueprint, request, make_response
+from flask import Blueprint, request, make_response, jsonify
 import random
 from loguru import logger
 
+from changedetectionio.notification.handler import process_notification
 from changedetectionio.store import ChangeDetectionStore
 from changedetectionio.auth_decorator import login_optionally_required
 
 def construct_blueprint(datastore: ChangeDetectionStore):
     notification_blueprint = Blueprint('ui_notification', __name__, template_folder="../ui/templates")
-    
+
+
+    @notification_blueprint.route("/notification/render-preview/<string:watch_uuid>", methods=['POST'])
+    @notification_blueprint.route("/notification/render-preview", methods=['POST'])
+    @notification_blueprint.route("/notification/render-preview/", methods=['POST'])
+    @login_optionally_required
+    def ajax_callback_test_render_preview(watch_uuid=None):
+        return ajax_callback_send_notification_test(watch_uuid=watch_uuid, send_as_null_test=True)
+
     # AJAX endpoint for sending a test
     @notification_blueprint.route("/notification/send-test/<string:watch_uuid>", methods=['POST'])
     @notification_blueprint.route("/notification/send-test", methods=['POST'])
     @notification_blueprint.route("/notification/send-test/", methods=['POST'])
     @login_optionally_required
-    def ajax_callback_send_notification_test(watch_uuid=None):
+    def ajax_callback_send_notification_test(watch_uuid=None, send_as_null_test=False):
 
         # Watch_uuid could be unset in the case it`s used in tag editor, global settings
         import apprise
-        from changedetectionio.notification.handler import process_notification
+        from urllib.parse import urlparse
         from changedetectionio.notification.apprise_plugin.assets import apprise_asset
 
-        from changedetectionio.notification.apprise_plugin.custom_handlers import apprise_http_custom_handler
+        # Necessary so that we import our custom handlers
+        from changedetectionio.notification.apprise_plugin.custom_handlers import apprise_http_custom_handler, apprise_null_custom_handler
 
         apobj = apprise.Apprise(asset=apprise_asset)
+        sent_obj = {}
 
         is_global_settings_form = request.args.get('mode', '') == 'global-settings'
         is_group_settings_form = request.args.get('mode', '') == 'group-settings'
 
         # Use an existing random one on the global/main settings form
-        if not watch_uuid and (is_global_settings_form or is_group_settings_form) \
-                and datastore.data.get('watching'):
-            logger.debug(f"Send test notification - Choosing random Watch {watch_uuid}")
+        if not watch_uuid and is_global_settings_form and datastore.data.get('watching'):
             watch_uuid = random.choice(list(datastore.data['watching'].keys()))
+            logger.debug(f"Send test notification - Chose random watch UUID: {watch_uuid}")
+
+        if is_group_settings_form  and datastore.data.get('watching'):
+            logger.debug(f"Send test notification - Choosing random Watch from group {watch_uuid}")
+            matching_watches = [uuid for uuid, watch in datastore.data['watching'].items() if watch.get('tags') and watch_uuid in watch['tags']]
+            if matching_watches:
+                watch_uuid = random.choice(matching_watches)
+            else:
+                # Just fallback to any
+                watch_uuid = random.choice(list(datastore.data['watching'].keys()))
 
         if not watch_uuid:
             return make_response("Error: You must have atleast one watch configured for 'test notification' to work", 400)
 
         watch = datastore.data['watching'].get(watch_uuid)
 
-        notification_urls = None
+        notification_urls = []
 
-        if request.form.get('notification_urls'):
-            notification_urls = request.form['notification_urls'].strip().splitlines()
+        if send_as_null_test:
+            test_schema = ''
+            try:
+                if request.form.get('notification_urls') and '://' in request.form.get('notification_urls'):
+                    first_test_notification_url = request.form['notification_urls'].strip().splitlines()[0]
+                    test_schema = urlparse(first_test_notification_url).scheme.lower().strip()
+            except Exception as e:
+                logger.error(f"Error trying to get a test schema based on the first notification_url {str(e)}")
+
+            notification_urls = [
+                # Null lets us do the whole chain of the same code without any extra repeated code
+                f'null://null-test-just-to-render-everything-on-the-same-codepath-and-get-preview?test_schema={test_schema}'
+            ]
+
+        else:
+            if request.form.get('notification_urls'):
+                notification_urls += request.form['notification_urls'].strip().splitlines()
 
         if not notification_urls:
             logger.debug("Test notification - Trying by group/tag in the edit form if available")
+            # @todo this logic is not clear, omegaconf?
             # On an edit page, we should also fire off to the tags if they have notifications
             if request.form.get('tags') and request.form['tags'].strip():
                 for k in request.form['tags'].split(','):
@@ -58,23 +93,28 @@ def construct_blueprint(datastore: ChangeDetectionStore):
                 notification_urls = datastore.data['settings']['application']['notification_urls']
 
         if not notification_urls:
-            return 'Error: No Notification URLs set/found'
+            return make_response("Error: No Notification URLs set/found.", 400)
 
         for n_url in notification_urls:
             if len(n_url.strip()):
                 if not apobj.add(n_url):
-                    return f'Error:  {n_url} is not a valid AppRise URL.'
+                    return make_response(f'Error: {n_url} is not a valid AppRise URL.', 400)
 
         try:
             # use the same as when it is triggered, but then override it with the form test values
             n_object = {
                 'watch_url': request.form.get('window_url', "https://changedetection.io"),
-                'notification_urls': notification_urls
+                'notification_urls': notification_urls,
+                'uuid': watch_uuid  # Ensure uuid is present so diff rendering works
             }
 
             # Only use if present, if not set in n_object it should use the default system value
-            if 'notification_format' in request.form and request.form['notification_format'].strip():
-                n_object['notification_format'] = request.form.get('notification_format', '').strip()
+            notif_format = request.form.get('notification_format', '').strip()
+            # Use it if provided and not "System default", otherwise fall back
+            if notif_format and notif_format != 'System default':
+                n_object['notification_format'] = notif_format
+            else:
+                n_object['notification_format'] = datastore.data['settings']['application'].get('notification_format')
 
             if 'notification_title' in request.form and request.form['notification_title'].strip():
                 n_object['notification_title'] = request.form.get('notification_title', '').strip()
@@ -92,7 +132,13 @@ def construct_blueprint(datastore: ChangeDetectionStore):
 
             n_object['as_async'] = False
             n_object.update(watch.extra_notification_token_values())
-            sent_obj = process_notification(n_object, datastore)
+
+            # This uses the same processor that the queue runner uses
+            # @todo - Split the notification URLs so we know which one worked, maybe highlight them in green in the UI
+            result = process_notification(n_object, datastore)
+            if result:
+                sent_obj['result'] = result[0]
+                sent_obj['status'] = 'OK - Sent test notifications'
 
         except Exception as e:
             e_str = str(e)
@@ -100,9 +146,9 @@ def construct_blueprint(datastore: ChangeDetectionStore):
             e_str = e_str.replace(
                 "DEBUG - <class 'apprise.decorators.base.CustomNotifyPlugin.instantiate_plugin.<locals>.CustomNotifyPluginWrapper'>",
                 '')
-
             return make_response(e_str, 400)
 
-        return 'OK - Sent test notifications'
+        # it will be a list of things reached, for this purpose just the first is good so we can see the body that was sent
+        return make_response(sent_obj, 200)
 
     return notification_blueprint

changedetectionio/blueprint/ui/templates/edit.html

@@ -21,6 +21,7 @@
     const email_notification_prefix=JSON.parse('{{ emailprefix|tojson }}');
 {% endif %}
     const notification_base_url="{{url_for('ui.ui_notification.ajax_callback_send_notification_test', watch_uuid=uuid)}}";
+    const notification_test_render_preview_url="{{url_for('ui.ui_notification.ajax_callback_test_render_preview', watch_uuid=uuid)}}";
     const playwright_enabled={% if playwright_enabled %}true{% else %}false{% endif %};
     const recheck_proxy_start_url="{{url_for('check_proxies.start_check', uuid=uuid)}}";
     const proxy_recheck_status_url="{{url_for('check_proxies.get_recheck_status', uuid=uuid)}}";
@@ -356,12 +357,12 @@ Math: {{ 1 + 1 }}") }}
                     </script>
                     <br>
                     {#<div id="text-preview-controls"><span id="text-preview-refresh" class="pure-button button-xsmall">Refresh</span></div>#}
-                    <div class="minitabs-wrapper">
+                    <div class="minitabs-wrapper" id="filter-preview-minitabs">
                       <div class="minitabs-content">
-                          <div id="text-preview-inner" class="monospace-preview">
+                          <div id="text-preview-inner" class="tab-contents-monospace-preview">
                               <p>Loading...</p>
                           </div>
-                          <div id="text-preview-before-inner" style="display: none;" class="monospace-preview">
+                          <div id="text-preview-before-inner" style="display: none;" class="tab-contents-monospace-preview">
                               <p>Loading...</p>
                           </div>
                       </div>

changedetectionio/blueprint/watchlist/__init__.py

@@ -87,7 +87,6 @@ def construct_blueprint(datastore: ChangeDetectionStore, update_q, queuedWatchMe
             form=form,
             guid=datastore.data['app_guid'],
             has_proxies=datastore.proxy_list,
-            has_unviewed=datastore.has_unviewed,
             hosted_sticky=os.getenv("SALTED_PASS", False) == False,
             now_time_server=round(time.time()),
             pagination=pagination,
@@ -97,6 +96,7 @@ def construct_blueprint(datastore: ChangeDetectionStore, update_q, queuedWatchMe
             sort_order=request.args.get('order') if request.args.get('order') else request.cookies.get('order'),
             system_default_fetcher=datastore.data['settings']['application'].get('fetch_backend'),
             tags=sorted_tags,
+            unread_changes_count=datastore.unread_changes_count,
             watches=sorted_watches
         )
 

changedetectionio/blueprint/watchlist/templates/watch-overview.html

@@ -82,8 +82,11 @@ document.addEventListener('DOMContentLoaded', function() {
         {%- set cols_required = cols_required + 1 -%}
     {%- endif -%}
     {%- set ui_settings = datastore.data['settings']['application']['ui'] -%}
-
-    <div id="watch-table-wrapper">
+    {%- set wrapper_classes = [
+        'has-unread-changes' if unread_changes_count else '',
+        'has-error' if errored_count else '',
+    ] -%}
+    <div id="watch-table-wrapper" class="{{ wrapper_classes | reject('equalto', '') | join(' ') }}">
         {%- set table_classes = [
             'favicon-enabled' if 'favicons_enabled' not in ui_settings or ui_settings['favicons_enabled'] else 'favicon-not-enabled',
         ] -%}
@@ -241,10 +244,10 @@ document.addEventListener('DOMContentLoaded', function() {
             </tbody>
         </table>
         <ul id="post-list-buttons">
-            <li id="post-list-with-errors" class="{%- if errored_count -%}has-error{%- endif -%}" style="display: none;" >
+            <li id="post-list-with-errors" style="display: none;" >
                 <a href="{{url_for('watchlist.index', with_errors=1, tag=request.args.get('tag')) }}" class="pure-button button-tag button-error">With errors ({{ errored_count }})</a>
             </li>
-            <li id="post-list-mark-views" class="{%- if has_unviewed -%}has-unviewed{%- endif -%}" style="display: none;" >
+            <li id="post-list-mark-views" style="display: none;" >
                 <a href="{{url_for('ui.mark_all_viewed',with_errors=request.args.get('with_errors',0)) }}" class="pure-button button-tag " id="mark-all-viewed">Mark all viewed</a>
             </li>
         {%-  if active_tag_uuid -%}
@@ -252,8 +255,8 @@ document.addEventListener('DOMContentLoaded', function() {
                 <a href="{{url_for('ui.mark_all_viewed', tag=active_tag_uuid) }}" class="pure-button button-tag " id="mark-all-viewed">Mark all viewed in '{{active_tag.title}}'</a>
             </li>
         {%-  endif -%}
-            <li id="post-list-unread" class="{%- if has_unviewed -%}has-unviewed{%- endif -%}" style="display: none;" >
-                <a href="{{url_for('watchlist.index', unread=1, tag=request.args.get('tag')) }}" class="pure-button button-tag">Unread</a>
+            <li id="post-list-unread" style="display: none;" >
+                <a href="{{url_for('watchlist.index', unread=1, tag=request.args.get('tag')) }}" class="pure-button button-tag">Unread (<span id="unread-tab-counter">{{ unread_changes_count }}</span>)</a>
             </li>
             <li>
                <a href="{{ url_for('ui.form_watch_checknow', tag=active_tag_uuid, with_errors=request.args.get('with_errors',0)) }}" class="pure-button button-tag" id="recheck-all">Recheck

changedetectionio/conditions/pluggy_interface.py

@@ -1,7 +1,7 @@
 import pluggy
 import os
 import importlib
-import sys
+from loguru import logger
 from . import default_plugin
 
 # ✅ Ensure that the namespace in HookspecMarker matches PluginManager
@@ -65,7 +65,7 @@ def load_plugins_from_directory():
                 # Register the plugin with pluggy
                 plugin_manager.register(module, module_name)
             except (ImportError, AttributeError) as e:
-                print(f"Error loading plugin {module_name}: {e}")
+                logger.critical(f"Error loading plugin {module_name}: {e}")
 
 # Load plugins from the plugins directory
 load_plugins_from_directory()

changedetectionio/flask_app.py

@@ -39,11 +39,6 @@ from loguru import logger
 from changedetectionio import __version__
 from changedetectionio import queuedWatchMetaData
 from changedetectionio.api import Watch, WatchHistory, WatchSingleHistory, CreateWatch, Import, SystemInfo, Tag, Tags, Notifications, WatchFavicon
-from changedetectionio.notification.BrowserNotifications import (
-    BrowserNotificationsVapidPublicKey,
-    BrowserNotificationsSubscribe, 
-    BrowserNotificationsUnsubscribe
-)
 from changedetectionio.api.Search import Search
 from .time_handler import is_within_schedule
 
@@ -99,7 +94,6 @@ except locale.Error:
     logger.warning(f"Unable to set locale {default_locale}, locale is not installed maybe?")
 
 watch_api = Api(app, decorators=[csrf.exempt])
-browser_notification_api = Api(app, decorators=[csrf.exempt])
 
 def init_app_secret(datastore_path):
     secret = ""
@@ -342,11 +336,6 @@ def changedetection_app(config=None, datastore_o=None):
 
     watch_api.add_resource(Notifications, '/api/v1/notifications',
                            resource_class_kwargs={'datastore': datastore})
-    
-    # Browser notification endpoints
-    browser_notification_api.add_resource(BrowserNotificationsVapidPublicKey, '/browser-notifications-api/vapid-public-key')
-    browser_notification_api.add_resource(BrowserNotificationsSubscribe, '/browser-notifications-api/subscribe')
-    browser_notification_api.add_resource(BrowserNotificationsUnsubscribe, '/browser-notifications-api/unsubscribe')
 
     @login_manager.user_loader
     def user_loader(email):
@@ -500,29 +489,10 @@ def changedetection_app(config=None, datastore_o=None):
         except FileNotFoundError:
             abort(404)
 
-    @app.route("/service-worker.js", methods=['GET'])
-    def service_worker():
-        from flask import make_response
-        try:
-            # Serve from the changedetectionio/static/js directory
-            static_js_path = os.path.join(os.path.dirname(__file__), 'static', 'js')
-            response = make_response(send_from_directory(static_js_path, "service-worker.js"))
-            response.headers['Content-Type'] = 'application/javascript'
-            response.headers['Service-Worker-Allowed'] = '/'
-            response.headers['Cache-Control'] = 'no-cache, no-store, must-revalidate'
-            response.headers['Pragma'] = 'no-cache'
-            response.headers['Expires'] = '0'
-            return response
-        except FileNotFoundError:
-            abort(404)
-
 
     import changedetectionio.blueprint.browser_steps as browser_steps
     app.register_blueprint(browser_steps.construct_blueprint(datastore), url_prefix='/browser-steps')
 
-    import changedetectionio.blueprint.browser_notifications.browser_notifications as browser_notifications
-    app.register_blueprint(browser_notifications.construct_blueprint(datastore), url_prefix='/browser-notifications')
-
     from changedetectionio.blueprint.imports import construct_blueprint as construct_import_blueprint
     app.register_blueprint(construct_import_blueprint(datastore, update_q, queuedWatchMetaData), url_prefix='/imports')
 
@@ -549,7 +519,7 @@ def changedetection_app(config=None, datastore_o=None):
 
     # watchlist UI buttons etc
     import changedetectionio.blueprint.ui as ui
-    app.register_blueprint(ui.construct_blueprint(datastore, update_q, worker_handler, queuedWatchMetaData, watch_check_update))
+    app.register_blueprint(ui.construct_blueprint(datastore, update_q, worker_handler, queuedWatchMetaData, watch_check_update, notification_q))
 
     import changedetectionio.blueprint.watchlist as watchlist
     app.register_blueprint(watchlist.construct_blueprint(datastore=datastore, update_q=update_q, queuedWatchMetaData=queuedWatchMetaData), url_prefix='')

changedetectionio/forms.py

@@ -707,7 +707,6 @@ class commonSettingsForm(Form):
     processor = RadioField( label=u"Processor - What do you want to achieve?", choices=processors.available_processors(), default="text_json_diff")
     timezone = StringField("Timezone for watch schedule", render_kw={"list": "timezones"}, validators=[validateTimeZoneName()])
     webdriver_delay = IntegerField('Wait seconds before extracting text', validators=[validators.Optional(), validators.NumberRange(min=1, message="Should contain one or more seconds")])
-    
 
 
 class importForm(Form):
@@ -760,6 +759,7 @@ class processor_text_json_diff_form(commonSettingsForm):
     check_unique_lines = BooleanField('Only trigger when unique lines appear in all history', default=False)
     remove_duplicate_lines = BooleanField('Remove duplicate lines of text', default=False)
     sort_text_alphabetically =  BooleanField('Sort text alphabetically', default=False)
+    strip_ignored_lines = TernaryNoneBooleanField('Strip ignored lines', default=None)
     trim_text_whitespace = BooleanField('Trim whitespace before and after text', default=False)
 
     filter_text_added = BooleanField('Added lines', default=True)
@@ -937,6 +937,7 @@ class globalSettingsApplicationForm(commonSettingsForm):
     removepassword_button = SubmitField('Remove password', render_kw={"class": "pure-button pure-button-primary"})
     render_anchor_tag_content = BooleanField('Render anchor tag content', default=False)
     shared_diff_access = BooleanField('Allow anonymous access to watch history page when password is enabled', default=False, validators=[validators.Optional()])
+    strip_ignored_lines = BooleanField('Strip ignored lines')
     rss_hide_muted_watches = BooleanField('Hide muted watches from RSS feed', default=True,
                                       validators=[validators.Optional()])
     filter_failure_notification_threshold_attempts = IntegerField('Number of times the filter can be missing before sending a notification',

changedetectionio/model/App.py

@@ -57,6 +57,7 @@ class model(dict):
                     'rss_hide_muted_watches': True,
                     'schema_version' : 0,
                     'shared_diff_access': False,
+                    'strip_ignored_lines': False,
                     'tags': {}, #@todo use Tag.model initialisers
                     'timezone': None, # Default IANA timezone name
                     'webdriver_delay': None , # Extra delay in seconds before extracting text
@@ -66,11 +67,6 @@ class model(dict):
                         'socket_io_enabled': True,
                         'favicons_enabled': True
                     },
-                    'vapid': {
-                        'private_key': None,
-                        'public_key': None,
-                        'contact_email': None
-                    },
                 }
             }
         }

changedetectionio/model/Watch.py

@@ -642,8 +642,10 @@ class model(watch_base):
 
     def extra_notification_token_placeholder_info(self):
         # Used for providing extra tokens
-        # return [('widget', "Get widget amounts")]
-        return []
+        values = []
+        values.append(('watch_html_link', "Link to URL as <a href>"))
+        values.append(('watch_url_raw', "Raw URL/link before any jinja2 macro"))
+        return values
 
 
     def extract_regex_from_all_history(self, regex):

changedetectionio/model/__init__.py

@@ -58,6 +58,7 @@ class watch_base(dict):
             'proxy': None,  # Preferred proxy connection
             'remote_server_reply': None,  # From 'server' reply header
             'sort_text_alphabetically': False,
+            'strip_ignored_lines': None,
             'subtractive_selectors': [],
             'tag': '',  # Old system of text name for a tag, to be removed
             'tags': [],  # list of UUIDs to App.Tags

changedetectionio/notification/BrowserNotifications.py

@@ -1,217 +0,0 @@
-import json
-from flask import request, current_app
-from flask_restful import Resource, marshal_with, fields
-from loguru import logger
-
-
-browser_notifications_fields = {
-    'success': fields.Boolean,
-    'message': fields.String,
-}
-
-vapid_public_key_fields = {
-    'publicKey': fields.String,
-}
-
-test_notification_fields = {
-    'success': fields.Boolean,
-    'message': fields.String,
-    'sent_count': fields.Integer,
-}
-
-
-class BrowserNotificationsVapidPublicKey(Resource):
-    """Get VAPID public key for browser push notifications"""
-    
-    @marshal_with(vapid_public_key_fields)
-    def get(self):
-        try:
-            from changedetectionio.notification.apprise_plugin.browser_notification_helpers import (
-                get_vapid_config_from_datastore, convert_pem_public_key_for_browser
-            )
-            
-            datastore = current_app.config.get('DATASTORE')
-            if not datastore:
-                return {'publicKey': None}, 500
-                
-            private_key, public_key_pem, contact_email = get_vapid_config_from_datastore(datastore)
-            
-            if not public_key_pem:
-                return {'publicKey': None}, 404
-            
-            # Convert PEM format to URL-safe base64 format for browser
-            public_key_b64 = convert_pem_public_key_for_browser(public_key_pem)
-            
-            if public_key_b64:
-                return {'publicKey': public_key_b64}
-            else:
-                return {'publicKey': None}, 500
-                
-        except Exception as e:
-            logger.error(f"Failed to get VAPID public key: {e}")
-            return {'publicKey': None}, 500
-
-
-class BrowserNotificationsSubscribe(Resource):
-    """Subscribe to browser notifications"""
-    
-    @marshal_with(browser_notifications_fields)
-    def post(self):
-        try:
-            data = request.get_json()
-            if not data:
-                return {'success': False, 'message': 'No data provided'}, 400
-                
-            subscription = data.get('subscription')
-            
-            if not subscription:
-                return {'success': False, 'message': 'Subscription is required'}, 400
-                
-            # Validate subscription format
-            required_fields = ['endpoint', 'keys']
-            for field in required_fields:
-                if field not in subscription:
-                    return {'success': False, 'message': f'Missing subscription field: {field}'}, 400
-                    
-            if 'p256dh' not in subscription['keys'] or 'auth' not in subscription['keys']:
-                return {'success': False, 'message': 'Missing subscription keys'}, 400
-                
-            # Get datastore
-            datastore = current_app.config.get('DATASTORE')
-            if not datastore:
-                return {'success': False, 'message': 'Datastore not available'}, 500
-                
-            # Initialize browser_subscriptions if it doesn't exist
-            if 'browser_subscriptions' not in datastore.data['settings']['application']:
-                datastore.data['settings']['application']['browser_subscriptions'] = []
-                
-            # Check if subscription already exists
-            existing_subscriptions = datastore.data['settings']['application']['browser_subscriptions']
-            for existing_sub in existing_subscriptions:
-                if existing_sub.get('endpoint') == subscription.get('endpoint'):
-                    return {'success': True, 'message': 'Already subscribed to browser notifications'}
-                    
-            # Add new subscription
-            datastore.data['settings']['application']['browser_subscriptions'].append(subscription)
-            datastore.needs_write = True
-            
-            logger.info(f"New browser notification subscription: {subscription.get('endpoint')}")
-            
-            return {'success': True, 'message': 'Successfully subscribed to browser notifications'}
-            
-        except Exception as e:
-            logger.error(f"Failed to subscribe to browser notifications: {e}")
-            return {'success': False, 'message': f'Subscription failed: {str(e)}'}, 500
-
-
-class BrowserNotificationsUnsubscribe(Resource):
-    """Unsubscribe from browser notifications"""
-    
-    @marshal_with(browser_notifications_fields)
-    def post(self):
-        try:
-            data = request.get_json()
-            if not data:
-                return {'success': False, 'message': 'No data provided'}, 400
-                
-            subscription = data.get('subscription')
-            
-            if not subscription or not subscription.get('endpoint'):
-                return {'success': False, 'message': 'Valid subscription is required'}, 400
-                
-            # Get datastore
-            datastore = current_app.config.get('DATASTORE')
-            if not datastore:
-                return {'success': False, 'message': 'Datastore not available'}, 500
-                
-            # Check if subscriptions exist
-            browser_subscriptions = datastore.data.get('settings', {}).get('application', {}).get('browser_subscriptions', [])
-            if not browser_subscriptions:
-                return {'success': True, 'message': 'No subscriptions found'}
-                
-            # Remove subscription with matching endpoint
-            endpoint = subscription.get('endpoint')
-            original_count = len(browser_subscriptions)
-            
-            datastore.data['settings']['application']['browser_subscriptions'] = [
-                sub for sub in browser_subscriptions 
-                if sub.get('endpoint') != endpoint
-            ]
-            
-            removed_count = original_count - len(datastore.data['settings']['application']['browser_subscriptions'])
-            
-            if removed_count > 0:
-                datastore.needs_write = True
-                logger.info(f"Removed {removed_count} browser notification subscription(s)")
-                return {'success': True, 'message': 'Successfully unsubscribed from browser notifications'}
-            else:
-                return {'success': True, 'message': 'No matching subscription found'}
-                
-        except Exception as e:
-            logger.error(f"Failed to unsubscribe from browser notifications: {e}")
-            return {'success': False, 'message': f'Unsubscribe failed: {str(e)}'}, 500
-
-
-
-class BrowserNotificationsTest(Resource):
-    """Send a test browser notification"""
-    
-    @marshal_with(test_notification_fields)
-    def post(self):
-        try:
-            data = request.get_json()
-            if not data:
-                return {'success': False, 'message': 'No data provided', 'sent_count': 0}, 400
-                
-            title = data.get('title', 'Test Notification')
-            body = data.get('body', 'This is a test notification from changedetection.io')
-            
-            # Get datastore to check if subscriptions exist
-            datastore = current_app.config.get('DATASTORE')
-            if not datastore:
-                return {'success': False, 'message': 'Datastore not available', 'sent_count': 0}, 500
-                
-            # Check if there are subscriptions before attempting to send
-            browser_subscriptions = datastore.data.get('settings', {}).get('application', {}).get('browser_subscriptions', [])
-            if not browser_subscriptions:
-                return {'success': False, 'message': 'No subscriptions found', 'sent_count': 0}, 404
-            
-            # Use the apprise handler directly
-            try:
-                from changedetectionio.notification.apprise_plugin.custom_handlers import apprise_browser_notification_handler
-                
-                # Call the apprise handler with test data
-                success = apprise_browser_notification_handler(
-                    body=body,
-                    title=title,
-                    notify_type='info',
-                    meta={'url': 'browser://test'}
-                )
-                
-                # Count how many subscriptions we have after sending (some may have been removed if invalid)
-                final_subscriptions = datastore.data.get('settings', {}).get('application', {}).get('browser_subscriptions', [])
-                sent_count = len(browser_subscriptions)  # Original count
-                
-                if success:
-                    return {
-                        'success': True,
-                        'message': f'Test notification sent successfully to {sent_count} subscriber(s)',
-                        'sent_count': sent_count
-                    }
-                else:
-                    return {
-                        'success': False,
-                        'message': 'Failed to send test notification',
-                        'sent_count': 0
-                    }, 500
-                    
-            except ImportError:
-                return {'success': False, 'message': 'Browser notification handler not available', 'sent_count': 0}, 500
-                
-        except Exception as e:
-            logger.error(f"Failed to send test browser notification: {e}")
-            return {'success': False, 'message': f'Test failed: {str(e)}', 'sent_count': 0}, 500
-
-
-
-

changedetectionio/notification/__init__.py

@@ -2,8 +2,8 @@ from changedetectionio.model import default_notification_format_for_watch
 
 ult_notification_format_for_watch = 'System default'
 default_notification_format = 'HTML Color'
-default_notification_body = '{{watch_url}} had a change.\n---\n{{diff}}\n---\n'
-default_notification_title = 'ChangeDetection.io Notification - {{watch_url}}'
+default_notification_body = '{{watch_title}} had a change.\n---\n{{diff}}\n---\n'
+default_notification_title = 'ChangeDetection.io Notification - {{watch_title}}'
 
 # The values (markdown etc) are from apprise NotifyFormat,
 # But to avoid importing the whole heavy module just use the same strings here.

changedetectionio/notification/apprise_plugin/browser_notification_helpers.py

@@ -1,273 +0,0 @@
-"""
-Browser notification helpers for Web Push API
-Shared utility functions for VAPID key handling and notification sending
-"""
-
-import json
-import re
-import time
-from loguru import logger
-
-
-def convert_pem_private_key_for_pywebpush(private_key):
-    """
-    Convert PEM private key to the format that pywebpush expects
-    
-    Args:
-        private_key: PEM private key string or already converted key
-        
-    Returns:
-        Vapid instance for pywebpush (avoids PEM parsing compatibility issues)
-    """
-    try:
-        from py_vapid import Vapid
-        import tempfile
-        import os
-        
-        # If we get a string, assume it's PEM and create a Vapid instance from it
-        if isinstance(private_key, str) and private_key.startswith('-----BEGIN'):
-            # Write PEM to temporary file and load with Vapid.from_file
-            with tempfile.NamedTemporaryFile(mode='w', suffix='.pem', delete=False) as tmp_file:
-                tmp_file.write(private_key)
-                tmp_file.flush()
-                temp_path = tmp_file.name
-                
-            try:
-                # Load using Vapid.from_file - this is more compatible with pywebpush
-                vapid_instance = Vapid.from_file(temp_path)
-                os.unlink(temp_path)  # Clean up
-                logger.debug("Successfully created Vapid instance from PEM")
-                return vapid_instance
-            except Exception as e:
-                os.unlink(temp_path)  # Clean up even on error
-                logger.error(f"Failed to create Vapid instance from PEM: {e}")
-                # Fall back to returning the original PEM string
-                return private_key
-        else:
-            # Return as-is if not a PEM string  
-            return private_key
-            
-    except Exception as e:
-        logger.error(f"Failed to convert private key: {e}")
-        return private_key
-
-
-def convert_pem_public_key_for_browser(public_key_pem):
-    """
-    Convert PEM public key to URL-safe base64 format for browser applicationServerKey
-    
-    Args:
-        public_key_pem: PEM public key string
-        
-    Returns:
-        URL-safe base64 encoded public key without padding
-    """
-    try:
-        from cryptography.hazmat.primitives import serialization
-        import base64
-        
-        # Parse PEM directly using cryptography library
-        pem_bytes = public_key_pem.encode() if isinstance(public_key_pem, str) else public_key_pem
-        
-        # Load the public key from PEM
-        public_key_crypto = serialization.load_pem_public_key(pem_bytes)
-        
-        # Get the raw public key bytes in uncompressed format (what browsers expect)
-        public_key_raw = public_key_crypto.public_bytes(
-            encoding=serialization.Encoding.X962,
-            format=serialization.PublicFormat.UncompressedPoint
-        )
-        
-        # Convert to URL-safe base64 (remove padding)
-        public_key_b64 = base64.urlsafe_b64encode(public_key_raw).decode('ascii').rstrip('=')
-        
-        return public_key_b64
-        
-    except Exception as e:
-        logger.error(f"Failed to convert public key format: {e}")
-        return None
-
-
-def send_push_notifications(subscriptions, notification_payload, private_key, contact_email, datastore):
-    """
-    Send push notifications to a list of subscriptions
-    
-    Args:
-        subscriptions: List of push subscriptions
-        notification_payload: Dict with notification data (title, body, etc.)
-        private_key: VAPID private key (will be converted if needed)
-        contact_email: Contact email for VAPID claims
-        datastore: Datastore object for updating subscriptions
-        
-    Returns:
-        Tuple of (success_count, total_count)
-    """
-    try:
-        from pywebpush import webpush, WebPushException
-    except ImportError:
-        logger.error("pywebpush not available - cannot send browser notifications")
-        return 0, len(subscriptions)
-    
-    # Convert private key to format pywebpush expects
-    private_key_for_push = convert_pem_private_key_for_pywebpush(private_key)
-    
-    success_count = 0
-    total_count = len(subscriptions)
-    
-    # Send to all subscriptions
-    for subscription in subscriptions[:]:  # Copy list to avoid modification issues
-        try:
-            webpush(
-                subscription_info=subscription,
-                data=json.dumps(notification_payload),
-                vapid_private_key=private_key_for_push,
-                vapid_claims={
-                    "sub": f"mailto:{contact_email}",
-                    "aud": f"https://{subscription['endpoint'].split('/')[2]}"
-                }
-            )
-            success_count += 1
-            
-        except WebPushException as e:
-            logger.warning(f"Failed to send browser notification to subscription: {e}")
-            # Remove invalid subscriptions (410 = Gone, 404 = Not Found)
-            if e.response and e.response.status_code in [404, 410]:
-                logger.info("Removing invalid browser notification subscription")
-                try:
-                    subscriptions.remove(subscription)
-                    datastore.needs_write = True
-                except ValueError:
-                    pass  # Already removed
-                    
-        except Exception as e:
-            logger.error(f"Unexpected error sending browser notification: {e}")
-    
-    return success_count, total_count
-
-
-def create_notification_payload(title, body, icon_path=None):
-    """
-    Create a standard notification payload
-    
-    Args:
-        title: Notification title
-        body: Notification body
-        icon_path: Optional icon path (defaults to favicon)
-        
-    Returns:
-        Dict with notification payload
-    """
-    return {
-        'title': title,
-        'body': body,
-        'icon': icon_path or '/static/favicons/favicon-32x32.png',
-        'badge': '/static/favicons/favicon-32x32.png',
-        'timestamp': int(time.time() * 1000),
-    }
-
-
-def get_vapid_config_from_datastore(datastore):
-    """
-    Get VAPID configuration from datastore with proper error handling
-    
-    Args:
-        datastore: Datastore object
-        
-    Returns:
-        Tuple of (private_key, public_key, contact_email) or (None, None, None) if error
-    """
-    try:
-        if not datastore:
-            return None, None, None
-            
-        vapid_config = datastore.data.get('settings', {}).get('application', {}).get('vapid', {})
-        private_key = vapid_config.get('private_key')
-        public_key = vapid_config.get('public_key')
-        contact_email = vapid_config.get('contact_email', 'citizen@example.com')
-        
-        return private_key, public_key, contact_email
-        
-    except Exception as e:
-        logger.error(f"Failed to get VAPID config from datastore: {e}")
-        return None, None, None
-
-
-
-def get_browser_subscriptions(datastore):
-    """
-    Get browser subscriptions from datastore
-    
-    Args:
-        datastore: Datastore object
-        
-    Returns:
-        List of subscriptions
-    """
-    try:
-        if not datastore:
-            return []
-            
-        return datastore.data.get('settings', {}).get('application', {}).get('browser_subscriptions', [])
-        
-    except Exception as e:
-        logger.error(f"Failed to get browser subscriptions: {e}")
-        return []
-
-
-def save_browser_subscriptions(datastore, subscriptions):
-    """
-    Save browser subscriptions to datastore
-    
-    Args:
-        datastore: Datastore object
-        subscriptions: List of subscriptions to save
-    """
-    try:
-        if not datastore:
-            return
-            
-        # Ensure the settings structure exists
-        if 'settings' not in datastore.data:
-            datastore.data['settings'] = {}
-        if 'application' not in datastore.data['settings']:
-            datastore.data['settings']['application'] = {}
-            
-        datastore.data['settings']['application']['browser_subscriptions'] = subscriptions
-        datastore.needs_write = True
-        
-    except Exception as e:
-        logger.error(f"Failed to save browser subscriptions: {e}")
-
-
-
-
-def create_error_response(message, sent_count=0, status_code=500):
-    """
-    Create standardized error response for API endpoints
-    
-    Args:
-        message: Error message
-        sent_count: Number of notifications sent (for test endpoints)
-        status_code: HTTP status code
-        
-    Returns:
-        Tuple of (response_dict, status_code)
-    """
-    return {'success': False, 'message': message, 'sent_count': sent_count}, status_code
-
-
-def create_success_response(message, sent_count=None):
-    """
-    Create standardized success response for API endpoints
-    
-    Args:
-        message: Success message
-        sent_count: Number of notifications sent (optional)
-        
-    Returns:
-        Response dict
-    """
-    response = {'success': True, 'message': message}
-    if sent_count is not None:
-        response['sent_count'] = sent_count
-    return response

changedetectionio/notification/apprise_plugin/custom_handlers.py

@@ -1,6 +1,5 @@
 import json
 import re
-import time
 from urllib.parse import unquote_plus
 
 import requests
@@ -20,6 +19,11 @@ def notify_supported_methods(func):
     return func
 
 
+def notify_null_method(func):
+    func = notify(on="null")(func)
+    return func
+
+
 def _get_auth(parsed_url: dict) -> str | tuple[str, str]:
     user: str | None = parsed_url.get("user")
     password: str | None = parsed_url.get("password")
@@ -113,8 +117,8 @@ def apprise_http_custom_handler(
         return False
 
 
-@notify(on="browser")
-def apprise_browser_notification_handler(
+@notify_null_method
+def apprise_null_custom_handler(
     body: str,
     title: str,
     notify_type: str,
@@ -122,69 +126,10 @@ def apprise_browser_notification_handler(
     *args,
     **kwargs,
 ) -> bool:
-    """
-    Browser push notification handler for browser:// URLs
-    Ignores anything after browser:// and uses single default channel
-    """
-    try:
-        from pywebpush import webpush, WebPushException
-        from flask import current_app
-        
-        # Get VAPID keys from app settings
-        try:
-            datastore = current_app.config.get('DATASTORE')
-            if not datastore:
-                logger.error("No datastore available for browser notifications")
-                return False
-                
-            vapid_config = datastore.data.get('settings', {}).get('application', {}).get('vapid', {})
-            private_key = vapid_config.get('private_key')
-            public_key = vapid_config.get('public_key')
-            contact_email = vapid_config.get('contact_email', 'admin@changedetection.io')
-            
-            if not private_key or not public_key:
-                logger.error("VAPID keys not configured for browser notifications")
-                return False
-                
-        except Exception as e:
-            logger.error(f"Failed to get VAPID configuration: {e}")
-            return False
-        
-        # Get subscriptions from datastore
-        browser_subscriptions = datastore.data.get('settings', {}).get('application', {}).get('browser_subscriptions', [])
-        
-        if not browser_subscriptions:
-            logger.info("No browser subscriptions found")
-            return True  # Not an error - just no subscribers
-            
-        # Import helper functions
-        try:
-            from .browser_notification_helpers import create_notification_payload, send_push_notifications
-        except ImportError:
-            logger.error("Browser notification helpers not available")
-            return False
-        
-        # Prepare notification payload
-        notification_payload = create_notification_payload(title, body)
-        
-        # Send notifications using shared helper
-        success_count, total_count = send_push_notifications(
-            subscriptions=browser_subscriptions,
-            notification_payload=notification_payload,
-            private_key=private_key,
-            contact_email=contact_email,
-            datastore=datastore
-        )
-                
-        # Update datastore with cleaned subscriptions
-        datastore.data['settings']['application']['browser_subscriptions'] = browser_subscriptions
-        
-        logger.info(f"Sent browser notifications: {success_count}/{total_count} successful")
-        return success_count > 0
-        
-    except ImportError:
-        logger.error("pywebpush not available - cannot send browser notifications")
-        return False
-    except Exception as e:
-        logger.error(f"Unexpected error in browser notification handler: {e}")
-        return False
+    url: str = meta.get("url")
+    schema: str = meta.get("schema")
+    method: str = re.sub(r"s$", "", schema).upper()
+    logger.info(f"Processed 'null' notification")
+
+    return True
+

changedetectionio/notification/handler.py

@@ -1,31 +1,167 @@
 
+import os
 import time
 import apprise
 from loguru import logger
 from .apprise_plugin.assets import apprise_asset, APPRISE_AVATAR_URL
+from changedetectionio.safe_jinja import render as jinja_render
+from urllib.parse import urlparse
+
+def _populate_notification_tokens(n_object, datastore):
+    """
+    Populate notification tokens (diff, current_snapshot, etc.) if not already present.
+    This ensures both queued notifications and test notifications have the same data.
+    """
+    from changedetectionio import diff
+    from changedetectionio.notification import default_notification_format_for_watch
+    from markupsafe import escape
+
+    watch_uuid = n_object.get('uuid')
+    if not watch_uuid:
+        return
+        
+    watch = datastore.data['watching'].get(watch_uuid)
+    if not watch:
+        return
+
+    dates = []
+    trigger_text = ''
+    watch_html_link = ''
+
+    if watch:
+        watch_history = watch.history
+        dates = list(watch_history.keys())
+        trigger_text = watch.get('trigger_text', [])
+
+    # Add text that was triggered
+    if len(dates):
+        snapshot_contents = watch.get_history_snapshot(dates[-1])
+
+        if n_object.get('notification_format').lower().startswith('html'):
+            snapshot_contents = str(escape(snapshot_contents))
+
+    else:
+        snapshot_contents = "No snapshot/history available, the watch should fetch atleast once."
+
+    # If we ended up here with "System default"
+    if n_object.get('notification_format') == default_notification_format_for_watch:
+        n_object['notification_format'] = datastore.data['settings']['application'].get('notification_format')
+
+    html_colour_enable = False
+    line_feed_sep = "\n"
+
+    # HTML needs linebreak, but MarkDown and Text can use a linefeed
+    if n_object.get('notification_format').lower().startswith('html'):
+        line_feed_sep = "<br>"
+        # Snapshot will be plaintext on the disk, convert to some kind of HTML
+        snapshot_contents = snapshot_contents.replace('\n', line_feed_sep)
+    if n_object.get('notification_format') == 'HTML Color':
+        html_colour_enable = True
+
+    triggered_text = ''
+    if len(trigger_text):
+        from changedetectionio import html_tools
+        triggered_text = html_tools.get_triggered_text(content=snapshot_contents, trigger_text=trigger_text)
+        if triggered_text:
+            triggered_text = line_feed_sep.join(triggered_text)
+
+    # Could be called as a 'test notification' with only 1 snapshot available
+    prev_snapshot = "Example text: example test\nExample text: change detection is cool\nExample text: some more examples\n"
+    current_snapshot = "Example text: example test\nExample text: More than 1 watch change needs to exist to build a nice preview!"
+
+    if len(dates) > 1:
+        prev_snapshot = watch.get_history_snapshot(dates[-2])
+        current_snapshot = watch.get_history_snapshot(dates[-1])
+        if n_object.get('notification_format').lower().startswith('html'):
+            prev_snapshot = str(escape(prev_snapshot))
+            current_snapshot = str(escape(current_snapshot))
+
+
+    if watch:
+        v = {'url': watch.get('url'), 'label': watch.label}
+        watch_html_link = jinja_render(template_str='<a href="{{ label or url | e }}" rel="noopener noreferrer">{{ url | e }}</a>', **v)
+
+
+    n_object.update({
+        'current_snapshot': snapshot_contents,
+        'diff': diff.render_diff(prev_snapshot, current_snapshot, line_feed_sep=line_feed_sep, html_colour=html_colour_enable),
+        'diff_added': diff.render_diff(prev_snapshot, current_snapshot, include_removed=False, line_feed_sep=line_feed_sep, html_colour=html_colour_enable),
+        'diff_full': diff.render_diff(prev_snapshot, current_snapshot, include_equal=True, line_feed_sep=line_feed_sep, html_colour=html_colour_enable),
+        'diff_patch': diff.render_diff(prev_snapshot, current_snapshot, line_feed_sep=line_feed_sep, patch_format=True),
+        'diff_removed': diff.render_diff(prev_snapshot, current_snapshot, include_added=False, line_feed_sep=line_feed_sep, html_colour=html_colour_enable),
+        'screenshot': watch.get_screenshot() if watch and watch.get('notification_screenshot') else None,
+        'triggered_text': triggered_text,
+        'watch_html_link': watch_html_link,
+        'watch_url': watch.link,
+        'watch_url_raw': watch.get('url'),
+    })
+
+    if watch:
+        n_object.update(watch.extra_notification_token_values())
+
+def scan_notification_file_templates(url, datastore, n_body, notification_parameters):
+    import glob
+    from urllib.parse import urlparse, parse_qs
+
+    try:
+        scheme = urlparse(url).scheme.lower().strip()
+
+        # schema could be overriden dynamically
+        if scheme == 'null' and 'test_schema=' in url:
+            scheme = parse_qs(urlparse(url).query).get("test_schema", [None])[0]
+
+        logger.debug(f"Looking for '{scheme}' notification wrapper templates...")
+
+        # Try exact match first, then wildcard matches
+        candidates = [
+            os.path.join(datastore.datastore_path, f"notification-wrapper-{scheme}.html"),
+            *[f for f in glob.glob(os.path.join(datastore.datastore_path, "notification-wrapper-*--.html"))
+              if scheme.startswith(os.path.basename(f).replace("notification-wrapper-", "").replace("--.html", ""))]
+        ]
+        
+        for tpl_name in candidates:
+            if os.path.isfile(tpl_name):
+                template_params = notification_parameters.copy()
+                template_params['notification_body'] = n_body
+                
+                with open(tpl_name, 'r', encoding='utf-8') as f:
+                    logger.info(f"Using HTML notification template wrapper from '{tpl_name}'")
+                    return jinja_render(template_str=f.read(), **template_params)
+                    
+    except Exception as e:
+        logger.warning(f"Failed to load notification template: {e}")
+
+    return None
 
 def process_notification(n_object, datastore):
-    from changedetectionio.safe_jinja import render as jinja_render
     from . import default_notification_format_for_watch, default_notification_format, valid_notification_formats
     # be sure its registered
-    from .apprise_plugin.custom_handlers import apprise_http_custom_handler, apprise_browser_notification_handler
+    from .apprise_plugin.custom_handlers import apprise_http_custom_handler, apprise_null_custom_handler
+
+    n_body = ''
+    n_title = ''
 
     now = time.time()
     if n_object.get('notification_timestamp'):
         logger.trace(f"Time since queued {now-n_object['notification_timestamp']:.3f}s")
-    # Insert variables into the notification content
-    notification_parameters = create_notification_parameters(n_object, datastore)
 
     n_format = valid_notification_formats.get(
         n_object.get('notification_format', default_notification_format),
         valid_notification_formats[default_notification_format],
     )
-
     # If we arrived with 'System default' then look it up
     if n_format == default_notification_format_for_watch and datastore.data['settings']['application'].get('notification_format') != default_notification_format_for_watch:
         # Initially text or whatever
         n_format = datastore.data['settings']['application'].get('notification_format', valid_notification_formats[default_notification_format])
 
+    # Ensure diff rendering is done if not already present (for test notifications)
+    if not n_object.get('diff') and n_object.get('uuid'):
+        _populate_notification_tokens(n_object, datastore)
+
+    # Insert variables into the notification content
+    notification_parameters = create_notification_parameters(n_object, datastore)
+
+
     logger.trace(f"Complete notification body including Jinja and placeholders calculated in  {time.time() - now:.2f}s")
 
     # https://github.com/caronc/apprise/wiki/Development_LogCapture
@@ -44,22 +180,27 @@ def process_notification(n_object, datastore):
 
     with apprise.LogCapture(level=apprise.logging.DEBUG) as logs:
         for url in n_object['notification_urls']:
+            # Commented out is OK
+            if url.startswith('#') or not url or not url.strip():
+                logger.trace(f"Skipping notification URL - '{url}'")
+                continue
 
             # Get the notification body from datastore
             n_body = jinja_render(template_str=n_object.get('notification_body', ''), **notification_parameters)
+            # hmm unsure about this, but why not
             if n_object.get('notification_format', '').startswith('HTML'):
                 n_body = n_body.replace("\n", '<br>')
 
             n_title = jinja_render(template_str=n_object.get('notification_title', ''), **notification_parameters)
 
-            url = url.strip()
-            if url.startswith('#'):
-                logger.trace(f"Skipping commented out notification URL - {url}")
-                continue
+            n_body_from_file_template = scan_notification_file_templates(url=url,
+                                                                         datastore=datastore,
+                                                                         n_body=n_body,
+                                                                         notification_parameters=notification_parameters)
+            if n_body_from_file_template:
+                n_body = n_body_from_file_template
+
 
-            if not url:
-                logger.warning(f"Process Notification: skipping empty notification URL.")
-                continue
 
             logger.info(f">> Process Notification: AppRise notifying {url}")
             url = jinja_render(template_str=url, **notification_parameters)
@@ -104,7 +245,7 @@ def process_notification(n_object, datastore):
                 # Apprise will default to HTML, so we need to override it
                 # So that whats' generated in n_body is in line with what is going to be sent.
                 # https://github.com/caronc/apprise/issues/633#issuecomment-1191449321
-                if not 'format=' in url and (n_format == 'Text' or n_format == 'Markdown'):
+                if not 'format=' in url and (n_format.lower() == 'text' or n_format.lower() == 'markdown'):
                     prefix = '?' if not '?' in url else '&'
                     # Apprise format is lowercase text https://github.com/caronc/apprise/issues/633
                     n_format = n_format.lower()
@@ -118,14 +259,15 @@ def process_notification(n_object, datastore):
                               'url': url,
                               'body_format': n_format})
 
-        # Blast off the notifications tht are set in .add()
-        apobj.notify(
-            title=n_title,
-            body=n_body,
-            body_format=n_format,
-            # False is not an option for AppRise, must be type None
-            attach=n_object.get('screenshot', None)
-        )
+        if n_object.get('notification_urls'):
+            # Blast off the notifications tht are set in .add()
+            apobj.notify(
+                title=n_title,
+                body=n_body,
+                body_format=n_format,
+                # False is not an option for AppRise, must be type None
+                attach=n_object.get('screenshot', None)
+            )
 
 
         # Returns empty string if nothing found, multi-line string otherwise

changedetectionio/notification/notification-wrapper-HTML-mail--.html

@@ -0,0 +1,15 @@
+{#  Copy this to your data-store directory if you wish to enable it for HTML style notifications, applies to all as a wrapper :) #}
+<html>
+<body>
+Hello,<br>
+
+<p>A change was detected on your web page watch for <p>{{ watch_html_link }}.</p>
+
+[ view history ] [ pause checks ] [ mute notifications ]
+
+<div>
+    {{  notification_body }}
+</div>
+
+</body>
+</html>

changedetectionio/notification/readme.md

@@ -0,0 +1,17 @@
+## Notification syntax
+
+All notifications use the https://github.com/caronc/apprise syntax, there are some custom ones such as `posts` etc for general web-services usability.
+
+
+## Template file notification wrappers
+
+You can by default wrap all notifications by creating a `notification-wrapper-HTML-schema.html` in your datastore directory.
+
+For example
+
+You can use "`--`" in the filename where the _schema_ is to symbolize a wildcard. For example `notification-wrapper-HTML-mail--.html` would
+apply to `mail://` `mailto://` etc etc
+
+See is `notification-wrapper-HTML-mail--.html` which applies to `mail://`, `mailto://foobar..` etc notifications
+
+

changedetectionio/notification_service.py

@@ -22,70 +22,14 @@ class NotificationService:
     
     def queue_notification_for_watch(self, n_object, watch):
         """
-        Queue a notification for a watch with full diff rendering and template variables
+        Queue a notification for a watch. Diff rendering and template variables will be
+        handled by process_notification() to ensure consistency with test notifications.
         """
-        from changedetectionio import diff
-        from changedetectionio.notification import default_notification_format_for_watch
-
-        dates = []
-        trigger_text = ''
-
         now = time.time()
 
-        if watch:
-            watch_history = watch.history
-            dates = list(watch_history.keys())
-            trigger_text = watch.get('trigger_text', [])
-
-        # Add text that was triggered
-        if len(dates):
-            snapshot_contents = watch.get_history_snapshot(dates[-1])
-        else:
-            snapshot_contents = "No snapshot/history available, the watch should fetch atleast once."
-
-        # If we ended up here with "System default"
-        if n_object.get('notification_format') == default_notification_format_for_watch:
-            n_object['notification_format'] = self.datastore.data['settings']['application'].get('notification_format')
-
-        html_colour_enable = False
-        # HTML needs linebreak, but MarkDown and Text can use a linefeed
-        if n_object.get('notification_format') == 'HTML':
-            line_feed_sep = "<br>"
-            # Snapshot will be plaintext on the disk, convert to some kind of HTML
-            snapshot_contents = snapshot_contents.replace('\n', line_feed_sep)
-        elif n_object.get('notification_format') == 'HTML Color':
-            line_feed_sep = "<br>"
-            # Snapshot will be plaintext on the disk, convert to some kind of HTML
-            snapshot_contents = snapshot_contents.replace('\n', line_feed_sep)
-            html_colour_enable = True
-        else:
-            line_feed_sep = "\n"
-
-        triggered_text = ''
-        if len(trigger_text):
-            from . import html_tools
-            triggered_text = html_tools.get_triggered_text(content=snapshot_contents, trigger_text=trigger_text)
-            if triggered_text:
-                triggered_text = line_feed_sep.join(triggered_text)
-
-        # Could be called as a 'test notification' with only 1 snapshot available
-        prev_snapshot = "Example text: example test\nExample text: change detection is cool\nExample text: some more examples\n"
-        current_snapshot = "Example text: example test\nExample text: change detection is fantastic\nExample text: even more examples\nExample text: a lot more examples"
-
-        if len(dates) > 1:
-            prev_snapshot = watch.get_history_snapshot(dates[-2])
-            current_snapshot = watch.get_history_snapshot(dates[-1])
-
+        # Add basic metadata for the notification
         n_object.update({
-            'current_snapshot': snapshot_contents,
-            'diff': diff.render_diff(prev_snapshot, current_snapshot, line_feed_sep=line_feed_sep, html_colour=html_colour_enable),
-            'diff_added': diff.render_diff(prev_snapshot, current_snapshot, include_removed=False, line_feed_sep=line_feed_sep),
-            'diff_full': diff.render_diff(prev_snapshot, current_snapshot, include_equal=True, line_feed_sep=line_feed_sep, html_colour=html_colour_enable),
-            'diff_patch': diff.render_diff(prev_snapshot, current_snapshot, line_feed_sep=line_feed_sep, patch_format=True),
-            'diff_removed': diff.render_diff(prev_snapshot, current_snapshot, include_added=False, line_feed_sep=line_feed_sep),
             'notification_timestamp': now,
-            'screenshot': watch.get_screenshot() if watch and watch.get('notification_screenshot') else None,
-            'triggered_text': triggered_text,
             'uuid': watch.get('uuid') if watch else None,
             'watch_url': watch.get('url') if watch else None,
         })
@@ -93,7 +37,6 @@ class NotificationService:
         if watch:
             n_object.update(watch.extra_notification_token_values())
 
-        logger.trace(f"Main rendered notification placeholders (diff_added etc) calculated in {time.time()-now:.3f}s")
         logger.debug("Queued notification for sending")
         self.notification_q.put(n_object)
 

changedetectionio/pluggy_interface.py

@@ -1,7 +1,7 @@
 import pluggy
 import os
 import importlib
-import sys
+from loguru import logger
 
 # Global plugin namespace for changedetection.io
 PLUGIN_NAMESPACE = "changedetectionio"
@@ -57,7 +57,7 @@ def load_plugins_from_directories():
                     # Register the plugin with pluggy
                     plugin_manager.register(module, module_name)
                 except (ImportError, AttributeError) as e:
-                    print(f"Error loading plugin {module_name}: {e}")
+                    logger.critical(f"Error loading plugin {module_name}: {e}")
 
 # Load plugins
 load_plugins_from_directories()

changedetectionio/processors/magic.py

@@ -0,0 +1,138 @@
+"""
+Content Type Detection and Stream Classification
+
+This module provides intelligent content-type detection for changedetection.io.
+It addresses the common problem where HTTP Content-Type headers are missing, incorrect,
+or too generic, which would otherwise cause the wrong processor to be used.
+
+The guess_stream_type class combines:
+1. HTTP Content-Type headers (when available and reliable)
+2. Python-magic library for MIME detection (analyzing actual file content)
+3. Content-based pattern matching for text formats (HTML tags, XML declarations, etc.)
+
+This multi-layered approach ensures accurate detection of RSS feeds, JSON, HTML, PDF,
+plain text, CSV, YAML, and XML formats - even when servers provide misleading headers.
+
+Used by: processors/text_json_diff/processor.py and other content processors
+"""
+
+# When to apply the 'cdata to real HTML' hack
+RSS_XML_CONTENT_TYPES = [
+    "application/rss+xml",
+    "application/rdf+xml",
+    "text/xml",
+    "application/xml",
+    "application/atom+xml",
+    "text/rss+xml",  # rare, non-standard
+    "application/x-rss+xml",  # legacy (older feed software)
+    "application/x-atom+xml",  # legacy (older Atom)
+]
+
+# JSON Content-types
+JSON_CONTENT_TYPES = [
+    "application/activity+json",
+    "application/feed+json",
+    "application/json",
+    "application/ld+json",
+    "application/vnd.api+json",
+]
+
+# CSV Content-types
+CSV_CONTENT_TYPES = [
+    "text/csv",
+    "application/csv",
+]
+
+# Generic XML Content-types (non-RSS/Atom)
+XML_CONTENT_TYPES = [
+    "text/xml",
+    "application/xml",
+]
+
+# YAML Content-types
+YAML_CONTENT_TYPES = [
+    "text/yaml",
+    "text/x-yaml",
+    "application/yaml",
+    "application/x-yaml",
+]
+
+HTML_PATTERNS = ['<!doctype html', '<html', '<head', '<body', '<script', '<iframe', '<div']
+
+import re
+import magic
+from loguru import logger
+
+
+class guess_stream_type():
+    is_pdf = False
+    is_json = False
+    is_html = False
+    is_plaintext = False
+    is_rss = False
+    is_csv = False
+    is_xml = False  # Generic XML, not RSS/Atom
+    is_yaml = False
+
+    def __init__(self, http_content_header, content):
+
+        magic_content_header = http_content_header
+        test_content = content[:200].lower().strip()
+
+        # Remove whitespace between < and tag name for robust detection (handles '< html', '<\nhtml', etc.)
+        test_content_normalized = re.sub(r'<\s+', '<', test_content)
+
+        # Magic will sometimes call text/plain as text/html!
+        magic_result = None
+        try:
+            mime = magic.from_buffer(content[:200], mime=True) # Send the original content
+            logger.debug(f"Guessing mime type, original content_type '{http_content_header}', mime type detected '{mime}'")
+            if mime and "/" in mime:
+                magic_result = mime
+                # Ignore generic/fallback mime types from magic
+                if mime in ['application/octet-stream', 'application/x-empty', 'binary']:
+                    logger.debug(f"Ignoring generic mime type '{mime}' from magic library")
+                # Trust magic for non-text types immediately
+                elif mime not in ['text/html', 'text/plain']:
+                    magic_content_header = mime
+
+        except Exception as e:
+            logger.error(f"Error getting a more precise mime type from 'magic' library ({str(e)}), using content-based detection")
+
+        # Content-based detection (most reliable for text formats)
+        # Check for HTML patterns first - if found, override magic's text/plain
+        has_html_patterns = any(p in test_content_normalized for p in HTML_PATTERNS)
+
+        # Always trust headers first
+        if any(s in http_content_header for s in RSS_XML_CONTENT_TYPES) or any(s in magic_content_header for s in RSS_XML_CONTENT_TYPES):
+            self.is_rss = True
+        elif any(s in http_content_header for s in JSON_CONTENT_TYPES) or any(s in magic_content_header for s in JSON_CONTENT_TYPES):
+            self.is_json = True
+        elif any(s in http_content_header for s in CSV_CONTENT_TYPES) or any(s in magic_content_header for s in CSV_CONTENT_TYPES):
+            self.is_csv = True
+        elif any(s in http_content_header for s in XML_CONTENT_TYPES) or any(s in magic_content_header for s in XML_CONTENT_TYPES):
+            # Only mark as generic XML if not already detected as RSS
+            if not self.is_rss:
+                self.is_xml = True
+        elif any(s in http_content_header for s in YAML_CONTENT_TYPES) or any(s in magic_content_header for s in YAML_CONTENT_TYPES):
+            self.is_yaml = True
+        elif 'pdf' in magic_content_header:
+            self.is_pdf = True
+###
+        elif has_html_patterns or http_content_header == 'text/html':
+            self.is_html = True
+        # If magic says text/plain and we found no HTML patterns, trust it
+        elif magic_result == 'text/plain':
+            self.is_plaintext = True
+            logger.debug(f"Trusting magic's text/plain result (no HTML patterns detected)")
+        elif '<rss' in test_content_normalized or '<feed' in test_content_normalized:
+            self.is_rss = True
+        elif test_content_normalized.startswith('<?xml'):
+            # Generic XML that's not RSS/Atom (RSS/Atom checked above)
+            self.is_xml = True
+        elif '%pdf-1' in test_content:
+            self.is_pdf = True
+        # Only trust magic for 'text' if no other patterns matched
+        elif 'text' in magic_content_header:
+            self.is_plaintext = True
+

changedetectionio/processors/text_json_diff/processor.py

@@ -13,6 +13,8 @@ from changedetectionio import html_tools, content_fetchers
 from changedetectionio.blueprint.price_data_follower import PRICE_DATA_TRACK_ACCEPT, PRICE_DATA_TRACK_REJECT
 from loguru import logger
 
+from changedetectionio.processors.magic import guess_stream_type
+
 urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)
 
 name = 'Webpage Text/HTML, JSON and PDF changes'
@@ -20,6 +22,9 @@ description = 'Detects all text changes where possible'
 
 json_filter_prefixes = ['json:', 'jq:', 'jqraw:']
 
+# Assume it's this type if the server says nothing on content-type
+DEFAULT_WHEN_NO_CONTENT_TYPE_HEADER = 'text/html'
+
 class FilterNotFoundInResponse(ValueError):
     def __init__(self, msg, screenshot=None, xpath_data=None):
         self.screenshot = screenshot
@@ -45,6 +50,9 @@ class perform_site_check(difference_detection_processor):
         if not watch:
             raise Exception("Watch no longer exists.")
 
+        ctype_header = self.fetcher.get_all_headers().get('content-type', DEFAULT_WHEN_NO_CONTENT_TYPE_HEADER).lower()
+        stream_content_type = guess_stream_type(http_content_header=ctype_header, content=self.fetcher.content)
+
         # Unset any existing notification error
         update_obj = {'last_notification_error': False, 'last_error': False}
 
@@ -54,7 +62,7 @@ class perform_site_check(difference_detection_processor):
         self.xpath_data = self.fetcher.xpath_data
 
         # Track the content type
-        update_obj['content_type'] = self.fetcher.get_all_headers().get('content-type', '').lower()
+        update_obj['content_type'] = ctype_header
 
         # Watches added automatically in the queue manager will skip if its the same checksum as the previous run
         # Saves a lot of CPU
@@ -69,24 +77,12 @@ class perform_site_check(difference_detection_processor):
         # https://stackoverflow.com/questions/41817578/basic-method-chaining ?
         # return content().textfilter().jsonextract().checksumcompare() ?
 
-        is_json = 'application/json' in self.fetcher.get_all_headers().get('content-type', '').lower()
-        is_html = not is_json
-        is_rss = False
 
-        ctype_header = self.fetcher.get_all_headers().get('content-type', '').lower()
         # Go into RSS preprocess for converting CDATA/comment to usable text
-        if any(substring in ctype_header for substring in ['application/xml', 'application/rss', 'text/xml']):
-            if '<rss' in self.fetcher.content[:100].lower():
-                self.fetcher.content = cdata_in_document_to_text(html_content=self.fetcher.content)
-                is_rss = True
+        if stream_content_type.is_rss:
+            self.fetcher.content = cdata_in_document_to_text(html_content=self.fetcher.content)
 
-        # source: support, basically treat it as plaintext
-        if watch.is_source_type_url:
-            is_html = False
-            is_json = False
-
-        inline_pdf = self.fetcher.get_all_headers().get('content-disposition', '') and '%PDF-1' in self.fetcher.content[:10]
-        if watch.is_pdf or 'application/pdf' in self.fetcher.get_all_headers().get('content-type', '').lower() or inline_pdf:
+        if watch.is_pdf or stream_content_type.is_pdf:
             from shutil import which
             tool = os.getenv("PDF_TO_HTML_TOOL", "pdftohtml")
             if not which(tool):
@@ -130,11 +126,12 @@ class perform_site_check(difference_detection_processor):
         has_filter_rule = len(include_filters_rule) and len(include_filters_rule[0].strip())
         has_subtractive_selectors = len(subtractive_selectors) and len(subtractive_selectors[0].strip())
 
-        if is_json and not has_filter_rule:
-            include_filters_rule.append("json:$")
-            has_filter_rule = True
+        if stream_content_type.is_json:
+            if not has_filter_rule:
+                # Force a reformat
+                include_filters_rule.append("json:$")
+                has_filter_rule = True
 
-        if is_json:
             # Sort the JSON so we dont get false alerts when the content is just re-ordered
             try:
                 self.fetcher.content = json.dumps(json.loads(self.fetcher.content), sort_keys=True)
@@ -142,23 +139,28 @@ class perform_site_check(difference_detection_processor):
                 # Might have just been a snippet, or otherwise bad JSON, continue
                 pass
 
-        if has_filter_rule:
-            for filter in include_filters_rule:
-                if any(prefix in filter for prefix in json_filter_prefixes):
-                    stripped_text_from_html += html_tools.extract_json_as_string(content=self.fetcher.content, json_filter=filter)
-                    is_html = False
+            if has_filter_rule:
+                for filter in include_filters_rule:
+                    if any(prefix in filter for prefix in json_filter_prefixes):
+                        stripped_text_from_html += html_tools.extract_json_as_string(content=self.fetcher.content, json_filter=filter)
+                        if stripped_text_from_html:
+                            stream_content_type.is_json = True
+                            stream_content_type.is_html = False
 
-        if is_html or watch.is_source_type_url:
+        # We have 'watch.is_source_type_url' because we should be able to use selectors on the raw HTML but return just that selected HTML
+        if stream_content_type.is_html or watch.is_source_type_url or stream_content_type.is_plaintext or stream_content_type.is_rss or stream_content_type.is_xml or stream_content_type.is_pdf:
 
             # CSS Filter, extract the HTML that matches and feed that into the existing inscriptis::get_text
             self.fetcher.content = html_tools.workarounds_for_obfuscations(self.fetcher.content)
             html_content = self.fetcher.content
 
-            # If not JSON,  and if it's not text/plain..
-            if 'text/plain' in self.fetcher.get_all_headers().get('content-type', '').lower():
+            # Some kind of "text" but definitely not RSS looking
+            if stream_content_type.is_plaintext:
                 # Don't run get_text or xpath/css filters on plaintext
+                # We are not HTML, we are not any kind of RSS, doesnt even look like HTML
                 stripped_text_from_html = html_content
             else:
+                # If not JSON, and if it's not text/plain..
                 # Does it have some ld+json price data? used for easier monitoring
                 update_obj['has_ldjson_price_data'] = html_tools.has_ldjson_product_info(self.fetcher.content)
 
@@ -172,13 +174,13 @@ class perform_site_check(difference_detection_processor):
                             html_content += html_tools.xpath_filter(xpath_filter=filter_rule.replace('xpath:', ''),
                                                                     html_content=self.fetcher.content,
                                                                     append_pretty_line_formatting=not watch.is_source_type_url,
-                                                                    is_rss=is_rss)
+                                                                    is_rss=stream_content_type.is_rss)
 
                         elif filter_rule.startswith('xpath1:'):
                             html_content += html_tools.xpath1_filter(xpath_filter=filter_rule.replace('xpath1:', ''),
                                                                      html_content=self.fetcher.content,
                                                                      append_pretty_line_formatting=not watch.is_source_type_url,
-                                                                     is_rss=is_rss)
+                                                                     is_rss=stream_content_type.is_rss)
                         else:
                             html_content += html_tools.include_filters(include_filters=filter_rule,
                                                                        html_content=self.fetcher.content,
@@ -197,7 +199,7 @@ class perform_site_check(difference_detection_processor):
                     do_anchor = self.datastore.data["settings"]["application"].get("render_anchor_tag_content", False)
                     stripped_text_from_html = html_tools.html_to_text(html_content=html_content,
                                                                       render_anchor_tag_content=do_anchor,
-                                                                      is_rss=is_rss)  # 1874 activate the <title workaround hack
+                                                                      is_rss=stream_content_type.is_rss)  # 1874 activate the <title workaround hack
 
         if watch.get('trim_text_whitespace'):
             stripped_text_from_html = '\n'.join(line.strip() for line in stripped_text_from_html.replace("\n\n", "\n").splitlines())
@@ -236,7 +238,7 @@ class perform_site_check(difference_detection_processor):
 
         # Treat pages with no renderable text content as a change? No by default
         empty_pages_are_a_change = self.datastore.data['settings']['application'].get('empty_pages_are_a_change', False)
-        if not is_json and not empty_pages_are_a_change and len(stripped_text_from_html.strip()) == 0:
+        if not stream_content_type.is_json and not empty_pages_are_a_change and len(stripped_text_from_html.strip()) == 0:
             raise content_fetchers.exceptions.ReplyWithContentButNoText(url=url,
                                                             status_code=self.fetcher.get_last_status_code(),
                                                             screenshot=self.fetcher.screenshot,
@@ -301,6 +303,11 @@ class perform_site_check(difference_detection_processor):
         text_for_checksuming = stripped_text_from_html
         if text_to_ignore:
             text_for_checksuming = html_tools.strip_ignore_text(stripped_text_from_html, text_to_ignore)
+            # Some people prefer to also completely remove it
+            strip_ignored_lines = watch.get('strip_ignored_lines') if watch.get('strip_ignored_lines') is not None else self.datastore.data['settings']['application'].get('strip_ignored_lines')
+            if strip_ignored_lines:
+                # @todo add test in the 'preview' mode, check the widget works? compare to datastruct
+                stripped_text_from_html = text_for_checksuming
 
         # Re #133 - if we should strip whitespaces from triggering the change detected comparison
         if text_for_checksuming and self.datastore.data['settings']['application'].get('ignore_whitespace', False):

changedetectionio/realtime/socket_server.py

@@ -243,14 +243,15 @@ def handle_watch_update(socketio, **kwargs):
 
         general_stats = {
             'count_errors': errored_count,
-            'has_unviewed': datastore.has_unviewed
+            'unread_changes_count': datastore.unread_changes_count
         }
 
         # Debug what's being emitted
         # logger.debug(f"Emitting 'watch_update' event for {watch.get('uuid')}, data: {watch_data}")
 
         # Emit to all clients (no 'broadcast' parameter needed - it's the default behavior)
-        socketio.emit("watch_update", {'watch': watch_data, 'general_stats': general_stats})
+        socketio.emit("watch_update", {'watch': watch_data})
+        socketio.emit("general_stats_update", general_stats)
 
         # Log after successful emit - use watch_data['uuid'] to avoid variable shadowing issues
         logger.trace(f"Socket.IO: Emitted update for watch {watch_data['uuid']}, Checking now: {watch_data['checking_now']}")

changedetectionio/run_socks_proxy_tests.sh

@@ -9,7 +9,7 @@ set -x
 # SOCKS5 related - start simple Socks5 proxy server
 # SOCKSTEST=xyz should show in the logs of this service to confirm it fetched
 docker run --network changedet-network -d --hostname socks5proxy --rm  --name socks5proxy -p 1080:1080 -e PROXY_USER=proxy_user123 -e PROXY_PASSWORD=proxy_pass123 serjs/go-socks5-proxy
-docker run --network changedet-network -d --hostname socks5proxy-noauth --rm  -p 1081:1080 --name socks5proxy-noauth  serjs/go-socks5-proxy
+docker run --network changedet-network -d --hostname socks5proxy-noauth --rm -p 1081:1080 --name socks5proxy-noauth -e REQUIRE_AUTH=false serjs/go-socks5-proxy
 
 echo "---------------------------------- SOCKS5 -------------------"
 # SOCKS5 related - test from proxies.json

changedetectionio/safe_jinja.py

@@ -18,7 +18,7 @@ def render(template_str, **args: t.Any) -> str:
     return output[:JINJA2_MAX_RETURN_PAYLOAD_SIZE]
 
 def render_fully_escaped(content):
-    env = jinja2.sandbox.ImmutableSandboxedEnvironment(autoescape=True)
+    env = jinja2.sandbox.ImmutableSandboxedEnvironment(autoescape=True, extensions=['jinja2_time.TimeExtension'])
     template = env.from_string("{{ some_html|e }}")
     return template.render(some_html=content)
 

changedetectionio/static/favicons/site.webmanifest

@@ -1,6 +1,6 @@
 {
-    "name": "changedetection.io",
-    "short_name": "changedetection",
+    "name": "",
+    "short_name": "",
     "icons": [
         {
             "src": "android-chrome-192x192.png",
@@ -15,8 +15,5 @@
     ],
     "theme_color": "#ffffff",
     "background_color": "#ffffff",
-    "display": "standalone",
-    "start_url": "/",
-    "scope": "/",
-    "gcm_sender_id": "103953800507"
+    "display": "standalone"
 }

changedetectionio/static/js/browser-notifications.js

@@ -1,450 +0,0 @@
-/**
- * changedetection.io Browser Push Notifications
- * Handles service worker registration, push subscription management, and notification permissions
- */
-
-class BrowserNotifications {
-    constructor() {
-        this.serviceWorkerRegistration = null;
-        this.vapidPublicKey = null;
-        this.isSubscribed = false;
-        this.init();
-    }
-
-    async init() {
-        if (!this.isSupported()) {
-            console.warn('Push notifications are not supported in this browser');
-            return;
-        }
-
-        try {
-            // Get VAPID public key from server
-            await this.fetchVapidPublicKey();
-            
-            // Register service worker
-            await this.registerServiceWorker();
-            
-            // Check existing subscription state
-            await this.checkExistingSubscription();
-            
-            // Initialize UI elements
-            this.initializeUI();
-            
-            // Set up notification URL monitoring
-            this.setupNotificationUrlMonitoring();
-            
-        } catch (error) {
-            console.error('Failed to initialize browser notifications:', error);
-        }
-    }
-
-    isSupported() {
-        return 'serviceWorker' in navigator && 
-               'PushManager' in window && 
-               'Notification' in window;
-    }
-
-    async fetchVapidPublicKey() {
-        try {
-            const response = await fetch('/browser-notifications-api/vapid-public-key');
-            if (!response.ok) {
-                throw new Error(`HTTP ${response.status}: ${response.statusText}`);
-            }
-            const data = await response.json();
-            this.vapidPublicKey = data.publicKey;
-        } catch (error) {
-            console.error('Failed to fetch VAPID public key:', error);
-            throw error;
-        }
-    }
-
-    async registerServiceWorker() {
-        try {
-            this.serviceWorkerRegistration = await navigator.serviceWorker.register('/service-worker.js', {
-                scope: '/'
-            });
-
-            console.log('Service Worker registered successfully');
-
-            // Wait for service worker to be ready
-            await navigator.serviceWorker.ready;
-
-        } catch (error) {
-            console.error('Service Worker registration failed:', error);
-            throw error;
-        }
-    }
-
-    initializeUI() {
-        // Bind event handlers to existing elements in the template
-        this.bindEventHandlers();
-        
-        // Update UI based on current permission state
-        this.updatePermissionStatus();
-    }
-
-    bindEventHandlers() {
-        const enableBtn = document.querySelector('#enable-notifications-btn');
-        const testBtn = document.querySelector('#test-notification-btn');
-
-        if (enableBtn) {
-            enableBtn.addEventListener('click', () => this.requestNotificationPermission());
-        }
-
-        if (testBtn) {
-            testBtn.addEventListener('click', () => this.sendTestNotification());
-        }
-    }
-
-    setupNotificationUrlMonitoring() {
-        // Monitor the notification URLs textarea for browser:// URLs
-        const notificationUrlsField = document.querySelector('textarea[name*="notification_urls"]');
-        if (notificationUrlsField) {
-            const checkForBrowserUrls = async () => {
-                const urls = notificationUrlsField.value || '';
-                const hasBrowserUrls = /browser:\/\//.test(urls);
-                
-                // If browser URLs are detected and we're not subscribed, auto-subscribe
-                if (hasBrowserUrls && !this.isSubscribed && Notification.permission === 'default') {
-                    const shouldSubscribe = confirm('Browser notifications detected! Would you like to enable browser notifications now?');
-                    if (shouldSubscribe) {
-                        await this.requestNotificationPermission();
-                    }
-                } else if (hasBrowserUrls && !this.isSubscribed && Notification.permission === 'granted') {
-                    // Permission already granted but not subscribed - auto-subscribe silently
-                    console.log('Auto-subscribing to browser notifications...');
-                    await this.subscribe();
-                }
-            };
-            
-            // Check immediately
-            checkForBrowserUrls();
-            
-            // Check on input changes
-            notificationUrlsField.addEventListener('input', checkForBrowserUrls);
-        }
-    }
-
-    async updatePermissionStatus() {
-        const statusElement = document.querySelector('#permission-status');
-        const enableBtn = document.querySelector('#enable-notifications-btn');
-        const testBtn = document.querySelector('#test-notification-btn');
-
-        if (!statusElement) return;
-
-        const permission = Notification.permission;
-        statusElement.textContent = permission;
-        statusElement.className = `permission-${permission}`;
-
-        // Show/hide controls based on permission
-        if (permission === 'default') {
-            if (enableBtn) enableBtn.style.display = 'inline-block';
-            if (testBtn) testBtn.style.display = 'none';
-        } else if (permission === 'granted') {
-            if (enableBtn) enableBtn.style.display = 'none';
-            if (testBtn) testBtn.style.display = 'inline-block';
-        } else { // denied
-            if (enableBtn) enableBtn.style.display = 'none';
-            if (testBtn) testBtn.style.display = 'none';
-        }
-    }
-
-    async requestNotificationPermission() {
-        try {
-            const permission = await Notification.requestPermission();
-            this.updatePermissionStatus();
-            
-            if (permission === 'granted') {
-                console.log('Notification permission granted');
-                // Automatically subscribe to browser notifications
-                this.subscribe();
-            } else {
-                console.log('Notification permission denied');
-            }
-        } catch (error) {
-            console.error('Error requesting notification permission:', error);
-        }
-    }
-
-    async subscribe() {
-        if (Notification.permission !== 'granted') {
-            alert('Please enable notifications first');
-            return;
-        }
-
-        if (this.isSubscribed) {
-            console.log('Already subscribed to browser notifications');
-            return;
-        }
-
-        try {
-            // First, try to clear any existing subscription with different keys
-            await this.clearExistingSubscription();
-
-            // Create push subscription
-            const subscription = await this.serviceWorkerRegistration.pushManager.subscribe({
-                userVisibleOnly: true,
-                applicationServerKey: this.urlBase64ToUint8Array(this.vapidPublicKey)
-            });
-
-            // Send subscription to server
-            const response = await fetch('/browser-notifications-api/subscribe', {
-                method: 'POST',
-                headers: {
-                    'Content-Type': 'application/json',
-                    'X-CSRFToken': document.querySelector('input[name=csrf_token]')?.value
-                },
-                body: JSON.stringify({
-                    subscription: subscription.toJSON()
-                })
-            });
-
-            if (!response.ok) {
-                throw new Error(`HTTP ${response.status}: ${response.statusText}`);
-            }
-
-            // Store subscription status
-            this.isSubscribed = true;
-            
-            console.log('Successfully subscribed to browser notifications');
-
-        } catch (error) {
-            console.error('Failed to subscribe to browser notifications:', error);
-            
-            // Show user-friendly error message
-            if (error.message.includes('different applicationServerKey')) {
-                this.showSubscriptionConflictDialog(error);
-            } else {
-                alert(`Failed to subscribe: ${error.message}`);
-            }
-        }
-    }
-
-    async unsubscribe() {
-        try {
-            if (!this.isSubscribed) return;
-
-            // Get current subscription
-            const subscription = await this.serviceWorkerRegistration.pushManager.getSubscription();
-            if (!subscription) {
-                this.isSubscribed = false;
-                return;
-            }
-
-            // Unsubscribe from server
-            const response = await fetch('/browser-notifications-api/unsubscribe', {
-                method: 'POST',
-                headers: {
-                    'Content-Type': 'application/json',
-                    'X-CSRFToken': document.querySelector('input[name=csrf_token]')?.value
-                },
-                body: JSON.stringify({
-                    subscription: subscription.toJSON()
-                })
-            });
-
-            if (!response.ok) {
-                console.warn(`Server unsubscribe failed: ${response.status}`);
-            }
-
-            // Unsubscribe locally
-            await subscription.unsubscribe();
-
-            // Update status
-            this.isSubscribed = false;
-            
-            console.log('Unsubscribed from browser notifications');
-
-        } catch (error) {
-            console.error('Failed to unsubscribe from browser notifications:', error);
-        }
-    }
-
-    async sendTestNotification() {
-        try {
-            // First, check if we're subscribed
-            if (!this.isSubscribed) {
-                const shouldSubscribe = confirm('You need to subscribe to browser notifications first. Subscribe now?');
-                if (shouldSubscribe) {
-                    await this.subscribe();
-                    // Give a moment for subscription to complete
-                    await new Promise(resolve => setTimeout(resolve, 1000));
-                } else {
-                    return;
-                }
-            }
-
-            const response = await fetch('/browser-notifications/test', {
-                method: 'POST',
-                headers: {
-                    'Content-Type': 'application/json',
-                    'X-CSRFToken': document.querySelector('input[name=csrf_token]')?.value
-                }
-            });
-
-            if (!response.ok) {
-                if (response.status === 404) {
-                    // No subscriptions found on server - try subscribing
-                    alert('No browser subscriptions found. Subscribing now...');
-                    await this.subscribe();
-                    return;
-                }
-                throw new Error(`HTTP ${response.status}: ${response.statusText}`);
-            }
-
-            const result = await response.json();
-            alert(result.message);
-            console.log('Test notification result:', result);
-        } catch (error) {
-            console.error('Failed to send test notification:', error);
-            alert(`Failed to send test notification: ${error.message}`);
-        }
-    }
-
-
-
-
-    urlBase64ToUint8Array(base64String) {
-        const padding = '='.repeat((4 - base64String.length % 4) % 4);
-        const base64 = (base64String + padding)
-            .replace(/-/g, '+')
-            .replace(/_/g, '/');
-
-        const rawData = window.atob(base64);
-        const outputArray = new Uint8Array(rawData.length);
-
-        for (let i = 0; i < rawData.length; ++i) {
-            outputArray[i] = rawData.charCodeAt(i);
-        }
-        return outputArray;
-    }
-
-    async checkExistingSubscription() {
-        /**
-         * Check if we already have a valid browser subscription
-         * Updates this.isSubscribed based on actual browser state
-         */
-        try {
-            if (!this.serviceWorkerRegistration) {
-                this.isSubscribed = false;
-                return;
-            }
-            
-            const existingSubscription = await this.serviceWorkerRegistration.pushManager.getSubscription();
-            
-            if (existingSubscription) {
-                // We have a subscription - verify it's still valid and matches our VAPID key
-                const subscriptionJson = existingSubscription.toJSON();
-                
-                // Check if the endpoint is still active (basic validation)
-                if (subscriptionJson.endpoint && subscriptionJson.keys) {
-                    console.log('Found existing valid subscription');
-                    this.isSubscribed = true;
-                } else {
-                    console.log('Found invalid subscription, clearing...');
-                    await existingSubscription.unsubscribe();
-                    this.isSubscribed = false;
-                }
-            } else {
-                console.log('No existing subscription found');
-                this.isSubscribed = false;
-            }
-        } catch (error) {
-            console.warn('Failed to check existing subscription:', error);
-            this.isSubscribed = false;
-        }
-    }
-
-    async clearExistingSubscription() {
-        /**
-         * Clear any existing push subscription that might conflict with our VAPID keys
-         */
-        try {
-            const existingSubscription = await this.serviceWorkerRegistration.pushManager.getSubscription();
-            
-            if (existingSubscription) {
-                console.log('Found existing subscription, unsubscribing...');
-                await existingSubscription.unsubscribe();
-                console.log('Successfully cleared existing subscription');
-            }
-        } catch (error) {
-            console.warn('Failed to clear existing subscription:', error);
-            // Don't throw - this is just cleanup
-        }
-    }
-
-    showSubscriptionConflictDialog(error) {
-        /**
-         * Show user-friendly dialog for subscription conflicts
-         */
-        const message = `Browser notifications are already set up for a different changedetection.io instance or with different settings.
-
-To fix this:
-1. Clear your existing subscription 
-2. Try subscribing again
-
-Would you like to automatically clear the old subscription and retry?`;
-
-        if (confirm(message)) {
-            this.clearExistingSubscription().then(() => {
-                // Retry subscription after clearing
-                setTimeout(() => {
-                    this.subscribe();
-                }, 500);
-            });
-        } else {
-            alert('To use browser notifications, please manually clear your browser notifications for this site in browser settings, then try again.');
-        }
-    }
-
-    async clearAllNotifications() {
-        /**
-         * Clear all browser notification subscriptions (admin function)
-         */
-        try {
-            // Call the server to clear ALL subscriptions from datastore
-            const response = await fetch('/browser-notifications/clear', {
-                method: 'POST',
-                headers: {
-                    'Content-Type': 'application/json',
-                    'X-CSRFToken': document.querySelector('input[name=csrf_token]')?.value
-                }
-            });
-            
-            if (response.ok) {
-                const result = await response.json();
-                console.log('Server response:', result.message);
-                
-                // Also clear the current browser's subscription if it exists
-                const existingSubscription = await this.serviceWorkerRegistration.pushManager.getSubscription();
-                if (existingSubscription) {
-                    await existingSubscription.unsubscribe();
-                    console.log('Cleared current browser subscription');
-                }
-                
-                // Update status
-                this.isSubscribed = false;
-                
-                alert(result.message + '. All browser notifications have been cleared.');
-            } else {
-                const error = await response.json();
-                console.error('Server clear failed:', error.message);
-                alert('Failed to clear server subscriptions: ' + error.message);
-            }
-            
-        } catch (error) {
-            console.error('Failed to clear all notifications:', error);
-            alert('Failed to clear notifications: ' + error.message);
-        }
-    }
-
-}
-
-// Initialize when DOM is ready
-if (document.readyState === 'loading') {
-    document.addEventListener('DOMContentLoaded', () => {
-        window.browserNotifications = new BrowserNotifications();
-    });
-} else {
-    window.browserNotifications = new BrowserNotifications();
-}

changedetectionio/static/js/notifications.js

@@ -1,5 +1,18 @@
 $(document).ready(function () {
 
+    // Could be from 'watch' or system settings or other
+    function getNotificationData() {
+        data = {
+            notification_body: $('textarea.notification-body').val(),
+            notification_format: $('select.notification-format').val(),
+            notification_title: $('input.notification-title').val(),
+            notification_urls: $('textarea.notification-urls').val(),
+            tags: $('#tags').val(),
+            window_url: window.location.href,
+        }
+        return data
+    }
+
     $('#add-email-helper').click(function (e) {
         e.preventDefault();
         email = prompt("Destination email");
@@ -10,17 +23,82 @@ $(document).ready(function () {
         }
     });
 
+    $('#notifications-minitabs').miniTabs({
+        "Customise": "#notification-setup",
+        "Preview": "#notification-preview",
+    });
+
+    $(document).on('click', '[data-target="#notification-preview"]', function (e) {
+        var data = getNotificationData();
+        $('#notification-iframe-html-preview').contents().find('body').html('Loading...');
+        $.ajax({
+            type: "POST",
+            url: notification_test_render_preview_url,
+            data: data,
+            statusCode: {
+                400: function (data) {
+                    $('#notification-test-log').show().toggleClass('error', true);
+                    $("#notification-test-log>span").text(data.responseText);
+                },
+            }
+        }).done(function (data) {
+            $('#notification-test-log').toggleClass('error', false);
+            setPreview(data['result']);
+        })
+
+    });
+
+    function setPreview(data) {
+        const iframe = document.getElementById("notification-iframe-html-preview");
+        const isDark = document.documentElement.getAttribute('data-darkmode') === 'true';
+
+        // this should come back in the data objk
+        const isTextFormat = $('select.notification-format').val() === 'Text';
+
+        $('#notification-preview-title-text').text(data['title']);
+        $('#notification-div-text-preview').text(data['body']);
+        return;
+        iframe.srcdoc = `
+        <html data-darkmode="${isDark}">
+          <head>            
+            <style>
+                :root {
+                  --color-white: #fff;
+                  --color-grey-200: #333;
+                  --color-grey-800: #e0e0e0;
+                  --color-black: #000;
+                  --color-dark-red: #a00;
+                  --color-light-red: #dd0000;
+                  --color-background: var(--color-grey-800);
+                  --color-text: var(--color-grey-200);
+                  }
+                  
+                    html[data-darkmode="true"] {
+                      --color-background: var(--color-grey-200);
+                      --color-text: var(--color-white);
+                    }
+                    body { /* no darkmode */
+                        background-color: var(--color-background);
+                        color: var(--color-text);
+                        padding: 5px;
+                    }
+                    body.text-format {
+                        font-family: monospace;
+                        white-space: pre;
+                        overflow-wrap: normal;
+                        overflow-x: auto;
+                    }
+            </style>
+          </head>
+          <body class="${isTextFormat ? 'text-format' : ''}">${data['body']}</body>
+        </html>`;
+    }
+
+
     $('#send-test-notification').click(function (e) {
         e.preventDefault();
 
-        data = {
-            notification_body: $('#notification_body').val(),
-            notification_format: $('#notification_format').val(),
-            notification_title: $('#notification_title').val(),
-            notification_urls: $('.notification-urls').val(),
-            tags: $('#tags').val(),
-            window_url: window.location.href,
-        }
+        var data = getNotificationData();
 
         $('.notifications-wrapper .spinner').fadeIn();
         $('#notification-test-log').show();
@@ -30,11 +108,14 @@ $(document).ready(function () {
             data: data,
             statusCode: {
                 400: function (data) {
+                    $("#notification-test-log").toggleClass('error', true);
                     $("#notification-test-log>span").text(data.responseText);
                 },
             }
         }).done(function (data) {
-            $("#notification-test-log>span").text(data);
+            $("#notification-test-log").toggleClass('error', false);
+            $("#notification-test-log>span").text(data['status']);
+
         }).fail(function (jqXHR, textStatus, errorThrown) {
             // Handle connection refused or other errors
             if (textStatus === "error" && errorThrown === "") {
@@ -42,11 +123,13 @@ $(document).ready(function () {
                 $("#notification-test-log>span").text("Error: Connection refused or server is unreachable.");
             } else {
                 console.error("Error:", textStatus, errorThrown);
-                $("#notification-test-log>span").text("An error occurred: " + textStatus);
+                $("#notification-test-log>span").text("An error occurred: " + errorThrown);
             }
         }).always(function () {
             $('.notifications-wrapper .spinner').hide();
         })
     });
+
+
 });
 

changedetectionio/static/js/realtime.js

@@ -117,15 +117,16 @@ $(document).ready(function () {
                 }
             })
 
+            socket.on('general_stats_update', function (general_stats) {
+                // Tabs at bottom of list
+                $('#watch-table-wrapper').toggleClass("has-unread-changes", general_stats.unread_changes_count !==0)
+                $('#watch-table-wrapper').toggleClass("has-error", general_stats.count_errors !== 0)
+                $('#post-list-with-errors a').text(`With errors (${ new Intl.NumberFormat(navigator.language).format(general_stats.count_errors) })`);
+                $('#unread-tab-counter').text(new Intl.NumberFormat(navigator.language).format(general_stats.unread_changes_count));
+            });
+
             socket.on('watch_update', function (data) {
                 const watch = data.watch;
-                const general_stats = data.general_stats;
-
-                // Log the entire watch object for debugging
-                console.log('!!! WATCH UPDATE EVENT RECEIVED !!!');
-                console.log(`${watch.event_timestamp} - Watch update ${watch.uuid} - Checking now - ${watch.checking_now} - UUID in URL ${window.location.href.includes(watch.uuid)}`);
-                console.log('Watch data:', watch);
-                console.log('General stats:', general_stats);
 
                 // Updating watch table rows
                 const $watchRow = $('tr[data-watch-uuid="' + watch.uuid + '"]');
@@ -150,13 +151,6 @@ $(document).ready(function () {
 
                     console.log('Updated UI for watch:', watch.uuid);
                 }
-
-                // Tabs at bottom of list
-                $('#post-list-mark-views').toggleClass("has-unviewed", general_stats.has_unviewed);
-                $('#post-list-unread').toggleClass("has-unviewed", general_stats.has_unviewed);
-                $('#post-list-with-errors').toggleClass("has-error", general_stats.count_errors !== 0)
-                $('#post-list-with-errors a').text(`With errors (${ general_stats.count_errors })`);
-
                 $('body').toggleClass('checking-now', watch.checking_now && window.location.href.includes(watch.uuid));
             });
 

changedetectionio/static/js/service-worker.js

@@ -1,95 +0,0 @@
-// changedetection.io Service Worker for Browser Push Notifications
-
-self.addEventListener('install', function(event) {
-    console.log('Service Worker installing');
-    self.skipWaiting();
-});
-
-self.addEventListener('activate', function(event) {
-    console.log('Service Worker activating');
-    event.waitUntil(self.clients.claim());
-});
-
-self.addEventListener('push', function(event) {
-    console.log('Push message received', event);
-    
-    let notificationData = {
-        title: 'changedetection.io',
-        body: 'A watched page has changed',
-        icon: '/static/favicons/favicon-32x32.png',
-        badge: '/static/favicons/favicon-32x32.png',
-        tag: 'changedetection-notification',
-        requireInteraction: false,
-        timestamp: Date.now()
-    };
-    
-    // Parse push data if available
-    if (event.data) {
-        try {
-            const pushData = event.data.json();
-            notificationData = {
-                ...notificationData,
-                ...pushData
-            };
-        } catch (e) {
-            console.warn('Failed to parse push data:', e);
-            notificationData.body = event.data.text() || notificationData.body;
-        }
-    }
-    
-    const promiseChain = self.registration.showNotification(
-        notificationData.title,
-        {
-            body: notificationData.body,
-            icon: notificationData.icon,
-            badge: notificationData.badge,
-            tag: notificationData.tag,
-            requireInteraction: notificationData.requireInteraction,
-            timestamp: notificationData.timestamp,
-            data: {
-                url: notificationData.url || '/',
-                timestamp: notificationData.timestamp
-            }
-        }
-    );
-    
-    event.waitUntil(promiseChain);
-});
-
-self.addEventListener('notificationclick', function(event) {
-    console.log('Notification clicked', event);
-    
-    event.notification.close();
-    
-    const targetUrl = event.notification.data?.url || '/';
-    
-    event.waitUntil(
-        clients.matchAll().then(function(clientList) {
-            // Check if there's already a window/tab open with our app
-            for (let i = 0; i < clientList.length; i++) {
-                const client = clientList[i];
-                if (client.url.includes(self.location.origin) && 'focus' in client) {
-                    client.navigate(targetUrl);
-                    return client.focus();
-                }
-            }
-            // If no existing window, open a new one
-            if (clients.openWindow) {
-                return clients.openWindow(targetUrl);
-            }
-        })
-    );
-});
-
-self.addEventListener('notificationclose', function(event) {
-    console.log('Notification closed', event);
-});
-
-// Handle messages from the main thread
-self.addEventListener('message', function(event) {
-    console.log('Service Worker received message:', event.data);
-    
-    if (event.data && event.data.type === 'SKIP_WAITING') {
-        self.skipWaiting();
-    }
-});

changedetectionio/static/js/tabs.js

@@ -1,11 +1,11 @@
 // Rewrite this is a plugin.. is all this JS really 'worth it?'
 
 window.addEventListener('hashchange', function () {
-    var tabs = document.getElementsByClassName('active');
-    while (tabs[0]) {
-        tabs[0].classList.remove('active');
+    var tabs = document.querySelectorAll('.tabs .active');
+    tabs.forEach(function (tab) {
+        tab.classList.remove('active');
         document.body.classList.remove('full-width');
-    }
+    });
     set_active_tab();
 }, false);
 

changedetectionio/static/js/watch-settings.js

@@ -74,7 +74,7 @@ $(document).ready(function () {
         $('#filters-and-triggers input')[method]('change', request_textpreview_update.throttle(1000));
         $("#filters-and-triggers-tab")[method]('click', request_textpreview_update.throttle(1000));
     });
-    $('.minitabs-wrapper').miniTabs({
+    $('#filter-preview-minitabs').miniTabs({
         "Content after filters": "#text-preview-inner",
         "Content raw/before filters": "#text-preview-before-inner"
     });

changedetectionio/static/styles/scss/parts/_darkmode.scss

@@ -18,8 +18,15 @@ html[data-darkmode="true"] {
       display: block;
     }
   }
+
+  .minitabs-content {
+    > div {
+       background-color: rgb(249 249 249 / 13%) !important;
+    }
+  }
 }
 
 
 
 
+

changedetectionio/static/styles/scss/parts/_minitabs.scss

@@ -1,6 +1,13 @@
 .minitabs-wrapper {
   width: 100%;
 
+  .tab-contents-monospace-preview {
+    font-family: "Courier New", Courier, monospace; /* Sets the font to a monospace type */
+    font-size: 70%;
+    word-break: break-word;
+    white-space: pre-wrap; /* Preserves whitespace and line breaks like <pre> */
+  }
+
   > div[id] {
     padding: 20px;
     border: 1px solid #ccc;
@@ -10,38 +17,45 @@
   .minitabs-content {
     width: 100%;
     display: flex;
+
     > div {
       flex: 1 1 auto;
       min-width: 0;
-      overflow: scroll;
+      padding: 1rem;
+      border: 1px solid #ddd;
+      background-color: #eee;
     }
   }
 
   .minitabs {
     display: flex;
     border-bottom: 1px solid #ccc;
-  }
 
-  .minitab {
-    flex: 1;
-    text-align: center;
-    padding: 12px 0;
-    text-decoration: none;
-    color: #333;
-    background-color: #f1f1f1;
-    border: 1px solid #ccc;
-    border-bottom: none;
-    cursor: pointer;
-    transition: background-color 0.3s;
-  }
+    .minitab {
+      flex: 1;
+      text-align: center;
+      padding: 12px 0;
+      text-decoration: none;
+      color: #333;
+      background-color: #f1f1f1;
+      border: 1px solid #ccc;
+      border-bottom: none;
+      cursor: pointer;
+      transition: background-color 0.3s;
+      border-top-left-radius: 5px;
+      border-top-right-radius: 5px;
+      opacity: 0.45;
+      &:hover {
+        background-color: #ddd;
+      }
 
-  .minitab:hover {
-    background-color: #ddd;
+      &.active {
+        background-color: #eee;
+        font-weight: bold;
+        opacity: 1.0;
+      }
+    }
   }
-
-  .minitab.active {
-    background-color: #fff;
-    font-weight: bold;
-  }
-
 }
+
+

changedetectionio/static/styles/scss/parts/_notification.scss

@@ -0,0 +1,12 @@
+#notification-preview {
+  resize: both;
+  overflow: hidden;
+}
+
+#notification-iframe-html-preview {
+  width: 100%;
+  height: 100%;
+  border: 0;
+  display: block;
+  overflow: auto;
+}

changedetectionio/static/styles/scss/parts/_preview_text_filter.scss

@@ -1,5 +1,3 @@
-@use "minitabs";
-
 body.preview-text-enabled {
 
   @media (min-width: 800px) {
@@ -31,19 +29,7 @@ body.preview-text-enabled {
   }
 
   #activate-text-preview {
-      background-color: var(--color-grey-500);
-  }
-
-  /* actual preview area */
-  .monospace-preview {
-    background: var(--color-background-input);
-    border: 1px solid var(--color-grey-600);
-    padding: 1rem;
-    color: var(--color-text-input);
-    font-family: "Courier New", Courier, monospace; /* Sets the font to a monospace type */
-    font-size: 70%;
-    word-break: break-word;
-    white-space: pre-wrap; /* Preserves whitespace and line breaks like <pre> */
+    background-color: var(--color-grey-500);
   }
 }
 
@@ -53,3 +39,11 @@ body.preview-text-enabled {
   z-index: 3;
   box-shadow: 1px 1px 4px var(--color-shadow-jump);
 }
+
+#filter-preview-minitabs {
+  .minitabs-content {
+    > div {
+      overflow: scroll;
+    }
+  }
+}

changedetectionio/static/styles/scss/parts/_socket.scss

@@ -17,18 +17,6 @@ body.checking-now {
   position: fixed;
 }
 
-#post-list-buttons {
-  #post-list-with-errors.has-error {
-    display: inline-block !important;
-  }
-  #post-list-mark-views.has-unviewed {
-    display: inline-block !important;
-  }
-  #post-list-unread.has-unviewed {
-    display: inline-block !important;
-  }
-}
-
 
 
 

changedetectionio/static/styles/scss/parts/_watch_table.scss

@@ -127,5 +127,44 @@
       display: inline-block !important;
     }
   }
+
+
 }
 
+#watch-table-wrapper {
+  /* general styling */
+  #post-list-buttons {
+    text-align: right;
+    padding: 0px;
+    margin: 0px;
+
+    li {
+      display: inline-block;
+    }
+
+    a {
+      border-top-left-radius: initial;
+      border-top-right-radius: initial;
+      border-bottom-left-radius: 5px;
+      border-bottom-right-radius: 5px;
+    }
+  }
+
+  /* post list dynamically on/off stuff */
+
+  &.has-error {
+    #post-list-buttons {
+      #post-list-with-errors {
+        display: inline-block !important;
+      }
+    }
+  }
+
+  &.has-unread-changes {
+    #post-list-buttons {
+      #post-list-unread, #post-list-mark-views, #post-list-unread {
+        display: inline-block !important;
+      }
+    }
+  }
+}

changedetectionio/static/styles/scss/parts/_widgets.scss

@@ -34,7 +34,6 @@
       transition: all 0.2s ease;
       cursor: pointer;
       display: block;
-      min-width: 60px;
       text-align: center;
     }
 

changedetectionio/static/styles/scss/styles.scss

@@ -20,6 +20,8 @@
 @use "parts/lister_extra";
 @use "parts/socket";
 @use "parts/visualselector";
+@use "parts/_minitabs";
+@use "parts/_notification";
 @use "parts/widgets";
 
 body {
@@ -203,24 +205,6 @@ code {
 }
 
 
-#post-list-buttons {
-  text-align: right;
-  padding: 0px;
-  margin: 0px;
-
-  li {
-    display: inline-block;
-  }
-
-  a {
-    border-top-left-radius: initial;
-    border-top-right-radius: initial;
-    border-bottom-left-radius: 5px;
-    border-bottom-right-radius: 5px;
-  }
-}
-
-
 body:after {
   content: "";
   background: linear-gradient(130deg, var(--color-background-gradient-first), var(--color-background-gradient-second) 41.07%, var(--color-background-gradient-third) 84.05%);
@@ -353,6 +337,11 @@ a.pure-button-selected {
     overflow-wrap: break-word;
     max-width: 100%;
     box-sizing: border-box;
+    &.error {
+      > span {
+        color: var(--color-error) !important;
+      }
+    }
   }
 }
 
@@ -362,11 +351,6 @@ label {
  }  
 }
 
-#notification-customisation {
-  border: 1px solid var(--color-border-notification);
-  padding: 0.5rem;
-  border-radius: 5px;
-}
 
 #notification-error-log {
   border: 1px solid var(--color-border-notification);

changedetectionio/store.py

@@ -140,28 +140,6 @@ class ChangeDetectionStore:
             secret = secrets.token_hex(16)
             self.__data['settings']['application']['api_access_token'] = secret
 
-        # Generate VAPID keys for browser push notifications
-        if not self.__data['settings']['application']['vapid'].get('private_key'):
-            try:
-                from py_vapid import Vapid
-                vapid = Vapid()
-                vapid.generate_keys()
-                # Convert bytes to strings for JSON serialization
-                private_pem = vapid.private_pem()
-                public_pem = vapid.public_pem()
-                
-                self.__data['settings']['application']['vapid']['private_key'] = private_pem.decode() if isinstance(private_pem, bytes) else private_pem
-                self.__data['settings']['application']['vapid']['public_key'] = public_pem.decode() if isinstance(public_pem, bytes) else public_pem
-                
-                # Set default contact email if not present
-                if not self.__data['settings']['application']['vapid'].get('contact_email'):
-                    self.__data['settings']['application']['vapid']['contact_email'] = 'citizen@example.com'
-                logger.info("Generated new VAPID keys for browser push notifications")
-            except ImportError:
-                logger.warning("py_vapid not available - browser notifications will not work")
-            except Exception as e:
-                logger.warning(f"Failed to generate VAPID keys: {e}")
-
         self.needs_write = True
 
         # Finally start the thread that will manage periodic data saves to JSON
@@ -224,14 +202,13 @@ class ChangeDetectionStore:
         return seconds
 
     @property
-    def has_unviewed(self):
-        if not self.__data.get('watching'):
-            return None
-
+    def unread_changes_count(self):
+        unread_changes_count = 0
         for uuid, watch in self.__data['watching'].items():
             if watch.history_n >= 2 and watch.viewed == False:
-                return True
-        return False
+                unread_changes_count += 1
+
+        return unread_changes_count
 
     @property
     def data(self):

changedetectionio/templates/_common_fields.html

@@ -24,149 +24,153 @@
                               </ul>
                             </div>
                             <div class="notifications-wrapper">
-                              <a id="send-test-notification" class="pure-button button-secondary button-xsmall" >Send test notification</a> <div class="spinner"  style="display: none;"></div>
+                              <a id="send-test-notification" class="pure-button button-secondary" >Send test notification</a> <div class="spinner"  style="display: none;"></div>
                             {% if emailprefix %}
-                              <a id="add-email-helper" class="pure-button button-secondary button-xsmall" >Add email <img style="height: 1em; display: inline-block" src="{{url_for('static_content', group='images', filename='email.svg')}}" alt="Add an email address"> </a>
+                              <a id="add-email-helper" class="pure-button button-secondary" >Add email <img style="height: 1em; display: inline-block" src="{{url_for('static_content', group='images', filename='email.svg')}}" alt="Add an email address"> </a>
                             {% endif %}
-                              <a href="{{url_for('settings.notification_logs')}}" class="pure-button button-secondary button-xsmall" >Notification debug logs</a>
+                              <a href="{{url_for('settings.notification_logs')}}" class="pure-button button-secondary " >Notification debug logs</a>
                               <br>
                                 <div id="notification-test-log" style="display: none;"><span class="pure-form-message-inline">Processing..</span></div>
                             </div>
                         </div>
-                        
-                        <!-- Browser Notifications -->
-                        <div id="browser-notification-section">
-                            <div class="pure-control-group">
-                                <label>Browser Notifications</label>
-                                <div class="pure-form-message-inline">
-                                    <p><strong>Browser push notifications!</strong> Use <code>browser://</code> URLs in your notification settings to receive real-time push notifications even when this tab is closed.</p>
-                                    <p><small><strong>Troubleshooting:</strong> If you get "different applicationServerKey" errors, click "Clear All Notifications" below and try again. This happens when switching between different changedetection.io instances.</small></p>
-                                    <div id="browser-notification-controls" style="margin-top: 1em;">
-                                        <div id="notification-permission-status">
-                                            <p>Browser notifications: <span id="permission-status">checking...</span></p>
+                        <div class="pure-control-group">
+                        <p>Customise the contents of the notification using the form below, this is not necessary but you can create quite interesting integrations :-)</p>
+                            <div class="minitabs-wrapper" id="notifications-minitabs">
+                                <div class="minitabs-content">
+                                    <div id="notification-setup">
+                                        <div class="pure-control-group">
+                                            {{ render_field(form.notification_title, class="m-d notification-title", placeholder=settings_application['notification_title']) }}
                                         </div>
-                                        <div id="browser-notification-actions">
-                                            <button type="button" id="enable-notifications-btn" class="pure-button button-secondary button-xsmall" style="display: none;">
-                                                Enable Browser Notifications
-                                            </button>
-                                            <button type="button" id="test-notification-btn" class="pure-button button-secondary button-xsmall" style="display: none;">
-                                                Send browser test notification
-                                            </button>
-                                            <button type="button" id="clear-notifications-btn" class="pure-button button-secondary button-xsmall" onclick="window.browserNotifications?.clearAllNotifications()" style="margin-left: 0.5em;">
-                                                Clear All Notifications
-                                            </button>
+                                        <div class="pure-control-group">
+                                            {{ render_field(form.notification_body , rows=5, class="notification-body", placeholder=settings_application['notification_body']) }}
+                                            <span class="pure-form-message-inline">Body for the notification &dash; You can use <a
+                                                    target="newwindow"
+                                                    href="https://jinja.palletsprojects.com/en/3.0.x/templates/">Jinja2</a> templating in the notification title, body and URL, and tokens from below.
+                                            </span>
+
+                                        </div>
+                                        <div class="pure-controls">
+                                            <div data-target="#notification-tokens-info"
+                                                 class="toggle-show pure-button button-tag button-xsmall">Show
+                                                token/placeholders
+                                            </div>
+                                        </div>
+                                        <div class="pure-controls" style="display: none;" id="notification-tokens-info">
+                                            <table class="pure-table" id="token-table">
+                                                <thead>
+                                                <tr>
+                                                    <th>Token</th>
+                                                    <th>Description</th>
+                                                </tr>
+                                                </thead>
+                                                <tbody>
+                                                <tr>
+                                                    <td><code>{{ '{{base_url}}' }}</code></td>
+                                                    <td>The URL of the changedetection.io instance you are running.</td>
+                                                </tr>
+                                                <tr>
+                                                    <td><code>{{ '{{watch_url}}' }}</code></td>
+                                                    <td>The URL being watched.</td>
+                                                </tr>
+                                                <tr>
+                                                    <td><code>{{ '{{watch_uuid}}' }}</code></td>
+                                                    <td>The UUID of the watch.</td>
+                                                </tr>
+                                                <tr>
+                                                    <td><code>{{ '{{watch_title}}' }}</code></td>
+                                                    <td>The page title of the watch, uses &lt;title&gt; if not set, falls back to URL</td>
+                                                </tr>
+                                                <tr>
+                                                    <td><code>{{ '{{watch_tag}}' }}</code></td>
+                                                    <td>The watch group / tag</td>
+                                                </tr>
+                                                <tr>
+                                                    <td><code>{{ '{{preview_url}}' }}</code></td>
+                                                    <td>The URL of the preview page generated by changedetection.io.
+                                                    </td>
+                                                </tr>
+                                                <tr>
+                                                    <td><code>{{ '{{diff_url}}' }}</code></td>
+                                                    <td>The URL of the diff output for the watch.</td>
+                                                </tr>
+                                                <tr>
+                                                    <td><code>{{ '{{diff}}' }}</code></td>
+                                                    <td>The diff output - only changes, additions, and removals</td>
+                                                </tr>
+                                                <tr>
+                                                    <td><code>{{ '{{diff_added}}' }}</code></td>
+                                                    <td>The diff output - only changes and additions</td>
+                                                </tr>
+                                                <tr>
+                                                    <td><code>{{ '{{diff_removed}}' }}</code></td>
+                                                    <td>The diff output - only changes and removals</td>
+                                                </tr>
+                                                <tr>
+                                                    <td><code>{{ '{{diff_full}}' }}</code></td>
+                                                    <td>The diff output - full difference output</td>
+                                                </tr>
+                                                <tr>
+                                                    <td><code>{{ '{{diff_patch}}' }}</code></td>
+                                                    <td>The diff output - patch in unified format</td>
+                                                </tr>
+                                                <tr>
+                                                    <td><code>{{ '{{current_snapshot}}' }}</code></td>
+                                                    <td>The current snapshot text contents value, useful when combined
+                                                        with JSON or CSS filters
+                                                    </td>
+                                                </tr>
+                                                <tr>
+                                                    <td><code>{{ '{{triggered_text}}' }}</code></td>
+                                                    <td>Text that tripped the trigger from filters</td>
+                                                </tr>
+                                                    {% if extra_notification_token_placeholder_info %}
+                                                        {% for token in extra_notification_token_placeholder_info %}
+                                                            <tr>
+                                                                <td><code>{{ '{{' }}{{ token[0] }}{{ '}}' }}</code></td>
+                                                                <td>{{ token[1] }}</td>
+                                                            </tr>
+                                                        {% endfor %}
+                                                    {% endif %}
+                                                </tbody>
+                                            </table>
+                                            <div class="pure-form-message-inline">
+                                                <p>
+                                                    Warning: Contents of <code>{{ '{{diff}}' }}</code>,
+                                                    <code>{{ '{{diff_removed}}' }}</code>, and
+                                                    <code>{{ '{{diff_added}}' }}</code> depend on how the difference
+                                                    algorithm perceives the change. <br>
+                                                    For example, an addition or removal could be perceived as a change
+                                                    in some cases. <a target="newwindow"
+                                                                      href="https://github.com/dgtlmoon/changedetection.io/wiki/Using-the-%7B%7Bdiff%7D%7D,-%7B%7Bdiff_added%7D%7D,-and-%7B%7Bdiff_removed%7D%7D-notification-tokens">More
+                                                    Here</a> <br>
+                                                </p>
+                                                <p>
+                                                    For JSON payloads, use <strong>|tojson</strong> without quotes for
+                                                    automatic escaping, for example - <code>{
+                                                    "name": {{ '{{ watch_title|tojson }}' }} }</code>
+                                                </p>
+                                                <p>
+                                                    URL encoding, use <strong>|urlencode</strong>, for example - <code>gets://hook-website.com/test.php?title={{ '{{ watch_title|urlencode }}' }}</code>
+                                                </p>
+                                            </div>
+                                        </div>
+                                        <div class="pure-control-group">
+                                            {{ render_field(form.notification_format , class="notification-format") }}
+                                            <span class="pure-form-message-inline">Format for all notifications</span>
+                                        </div>
+                                    </div>
+                                    <div id="notification-preview" style="display: none; height:100%; display:flex; flex-direction:column;">
+                                        <p><strong>Title: </strong><span id="notification-preview-title-text">Preview loading..</span></p>
+                                        <div style="flex:1; display:flex; flex-direction:column;">
+                                            <strong>Body: </strong>
+                                            <div id="notification-div-text-preview" style="flex:1; height:95%; width:100%; border-radius:4px; margin-top:0.5rem; border:none;"></div>
+                                            <iframe id="notification-iframe-html-preview"
+                                                    style="flex:1; height:95%; width:100%; border-radius:4px; margin-top:0.5rem; border:none;">
+                                                Preview loading...
+                                            </iframe>
                                         </div>
                                     </div>
                                 </div>
                             </div>
                         </div>
-                        
-                        <div id="notification-customisation" class="pure-control-group">
-                            <div class="pure-control-group">
-                                {{ render_field(form.notification_title, class="m-d notification-title", placeholder=settings_application['notification_title']) }}
-                                <span class="pure-form-message-inline">Title for all notifications</span>
-                            </div>
-                            <div class="pure-control-group">
-                                {{ render_field(form.notification_body , rows=5, class="notification-body", placeholder=settings_application['notification_body']) }}
-                                <span class="pure-form-message-inline">Body for all notifications &dash; You can use <a target="newwindow" href="https://jinja.palletsprojects.com/en/3.0.x/templates/">Jinja2</a> templating in the notification title, body and URL, and tokens from below.
-                                </span>
-
-                            </div>
-                            <div class="pure-controls">
-                                <div data-target="#notification-tokens-info" class="toggle-show pure-button button-tag button-xsmall">Show token/placeholders</div>
-                            </div>
-                            <div class="pure-controls" style="display: none;" id="notification-tokens-info">
-                                <table class="pure-table" id="token-table">
-                                    <thead>
-                                    <tr>
-                                        <th>Token</th>
-                                        <th>Description</th>
-                                    </tr>
-                                    </thead>
-                                    <tbody>
-                                    <tr>
-                                        <td><code>{{ '{{base_url}}' }}</code></td>
-                                        <td>The URL of the changedetection.io instance you are running.</td>
-                                    </tr>
-                                    <tr>
-                                        <td><code>{{ '{{watch_url}}' }}</code></td>
-                                        <td>The URL being watched.</td>
-                                    </tr>
-                                    <tr>
-                                        <td><code>{{ '{{watch_uuid}}' }}</code></td>
-                                        <td>The UUID of the watch.</td>
-                                    </tr>
-                                    <tr>
-                                        <td><code>{{ '{{watch_title}}' }}</code></td>
-                                        <td>The page title of the watch, uses &lt;title&gt; if not set, falls back to URL</td>
-                                    </tr>
-                                    <tr>
-                                        <td><code>{{ '{{watch_tag}}' }}</code></td>
-                                        <td>The watch group / tag</td>
-                                    </tr>
-                                    <tr>
-                                        <td><code>{{ '{{preview_url}}' }}</code></td>
-                                        <td>The URL of the preview page generated by changedetection.io.</td>
-                                    </tr>
-                                    <tr>
-                                        <td><code>{{ '{{diff_url}}' }}</code></td>
-                                        <td>The URL of the diff output for the watch.</td>
-                                    </tr>
-									<tr>
-                                        <td><code>{{ '{{diff}}' }}</code></td>
-                                        <td>The diff output - only changes, additions, and removals</td>
-                                    </tr>
-									<tr>
-                                        <td><code>{{ '{{diff_added}}' }}</code></td>
-                                        <td>The diff output - only changes and additions</td>
-                                    </tr>
-									<tr>
-                                        <td><code>{{ '{{diff_removed}}' }}</code></td>
-                                        <td>The diff output - only changes and removals</td>
-                                    </tr>
-                                    <tr>
-                                        <td><code>{{ '{{diff_full}}' }}</code></td>
-                                        <td>The diff output - full difference output</td>
-                                    </tr>
-                                    <tr>
-                                        <td><code>{{ '{{diff_patch}}' }}</code></td>
-                                        <td>The diff output - patch in unified format</td>
-                                    </tr>
-                                    <tr>
-                                        <td><code>{{ '{{current_snapshot}}' }}</code></td>
-                                        <td>The current snapshot text contents value, useful when combined with JSON or CSS filters
-                                        </td>
-                                    </tr>
-                                    <tr>
-                                        <td><code>{{ '{{triggered_text}}' }}</code></td>
-                                        <td>Text that tripped the trigger from filters</td>
-
-                                        {% if extra_notification_token_placeholder_info %}
-                                            {% for token in extra_notification_token_placeholder_info %}
-                                                <tr>
-                                                    <td><code>{{ '{{' }}{{ token[0] }}{{ '}}' }}</code></td>
-                                                    <td>{{ token[1] }}</td>
-                                                </tr>
-                                            {% endfor %}
-                                        {% endif %}
-                                    </tbody>
-                                </table>
-                                <div class="pure-form-message-inline">
-                                    <p>
-									Warning: Contents of <code>{{ '{{diff}}' }}</code>, <code>{{ '{{diff_removed}}' }}</code>, and <code>{{ '{{diff_added}}' }}</code> depend on how the difference algorithm perceives the change. <br>
-                                    For example, an addition or removal could be perceived as a change in some cases. <a target="newwindow" href="https://github.com/dgtlmoon/changedetection.io/wiki/Using-the-%7B%7Bdiff%7D%7D,-%7B%7Bdiff_added%7D%7D,-and-%7B%7Bdiff_removed%7D%7D-notification-tokens">More Here</a> <br>
-                                    </p>
-                                    <p>
-                                        For JSON payloads, use <strong>|tojson</strong> without quotes for automatic escaping, for example - <code>{ "name": {{ '{{ watch_title|tojson }}' }} }</code>
-                                    </p>
-                                    <p>
-                                        URL encoding, use <strong>|urlencode</strong>, for example - <code>gets://hook-website.com/test.php?title={{ '{{ watch_title|urlencode }}' }}</code>
-                                    </p>
-                                </div>
-                            </div>
-                            <div class="pure-control-group">
-                                {{ render_field(form.notification_format , class="notification-format") }}
-                                <span class="pure-form-message-inline">Format for all notifications</span>
-                            </div>
-                        </div>
-{% endmacro %}
+{% endmacro %}

changedetectionio/templates/_helpers.html

@@ -27,8 +27,15 @@
 {% endmacro %}
 
 {% macro render_checkbox_field(field) %}
-  <div class="checkbox {% if field.errors %} error {% endif %}">
+  <div class="checkbox {% if field.errors or field.top_errors %} error {% endif %}">
   {{ field(**kwargs)|safe }} {{ field.label }}
+  {% if field.top_errors %}
+    <ul class="errors top-errors">
+    {% for error in field.top_errors %}
+      <li>{{ error }}</li>
+    {% endfor %}
+    </ul>
+  {% endif %}
   {% if field.errors %}
     <ul class=errors>
     {% for error in field.errors %}
@@ -43,9 +50,16 @@
   {% if BooleanField %}
     {% set _ = field.__setattr__('boolean_mode', true) %}
   {% endif %}
-  <div class="ternary-field {% if field.errors %} error {% endif %}">
+  <div class="ternary-field {% if field.errors or field.top_errors %} error {% endif %}">
     <div class="ternary-field-label">{{ field.label }}</div>
     <div class="ternary-field-widget">{{ field(**kwargs)|safe }}</div>
+    {% if field.top_errors %}
+      <ul class="errors top-errors">
+      {% for error in field.top_errors %}
+        <li>{{ error }}</li>
+      {% endfor %}
+      </ul>
+    {% endif %}
     {% if field.errors %}
       <ul class=errors>
       {% for error in field.errors %}
@@ -58,8 +72,15 @@
 
 
 {% macro render_simple_field(field) %}
-  <span class="label {% if field.errors %}error{% endif %}">{{ field.label }}</span>
-  <span {% if field.errors %} class="error" {% endif %}>{{ field(**kwargs)|safe }}
+  <span class="label {% if field.errors or field.top_errors %}error{% endif %}">{{ field.label }}</span>
+  <span {% if field.errors or field.top_errors %} class="error" {% endif %}>{{ field(**kwargs)|safe }}
+  {% if field.top_errors %}
+    <ul class="errors top-errors">
+    {% for error in field.top_errors %}
+      <li>{{ error }}</li>
+    {% endfor %}
+    </ul>
+  {% endif %}
   {% if field.errors %}
     <ul class=errors>
     {% for error in field.errors %}
@@ -74,8 +95,15 @@
 {% macro render_nolabel_field(field) %}
     <span>
     {{ field(**kwargs)|safe }}
-        {% if field.errors %}
+        {% if field.top_errors or field.errors %}
             <span class="error">
+      {% if field.top_errors %}
+          <ul class="errors top-errors">
+        {% for error in field.top_errors %}
+            <li>{{ error }}</li>
+        {% endfor %}
+        </ul>
+      {% endif %}
       {% if field.errors %}
           <ul class=errors>
         {% for error in field.errors %}

changedetectionio/templates/base.html

@@ -35,7 +35,6 @@
     <script src="{{url_for('static_content', group='js', filename='jquery-3.6.0.min.js')}}"></script>
     <script src="{{url_for('static_content', group='js', filename='csrf.js')}}" defer></script>
     <script src="{{url_for('static_content', group='js', filename='feather-icons.min.js')}}" defer></script>
-    <script src="{{url_for('static_content', group='js', filename='browser-notifications.js')}}" defer></script>
     {% if socket_io_enabled %}
     <script src="{{url_for('static_content', group='js', filename='socket.io.min.js')}}"></script>
     <script src="{{url_for('static_content', group='js', filename='realtime.js')}}" defer></script>

changedetectionio/templates/edit/text-options.html

@@ -26,7 +26,10 @@
                             <li>Changing this will affect the comparison checksum which may trigger an alert</li>
                         </ul>
                 </span>
-
+                <br><br>
+                    <div class="pure-control-group">
+                      {{ render_ternary_field(form.strip_ignored_lines) }}
+                    </div>
                 </fieldset>
 
                 <fieldset>

changedetectionio/tests/smtp/test_notification_smtp.py

@@ -1,17 +1,17 @@
-import json
 import os
 import time
-import re
 from flask import url_for
 from changedetectionio.tests.util import set_original_response, set_modified_response, set_more_modified_response, live_server_setup, \
     wait_for_all_checks, \
     set_longer_modified_response
-from changedetectionio.tests.util import extract_UUID_from_client
 import logging
-import base64
 
 # NOTE - RELIES ON mailserver as hostname running, see github build recipes
-smtp_test_server = 'mailserver'
+# Should be hostname (never IP), looks for our test mailserver that repeats the content
+# python3 changedetectionio/tests/smtp/smtp-test-server.py &
+# mailserver=localhost pytest tests/smtp/test_notification_smtp.py::test_check_notification_email_formats_default_HTML
+smtp_test_server = os.getenv('mailserver', 'mailserver')
+
 
 from changedetectionio.notification import (
     default_notification_body,
@@ -20,7 +20,35 @@ from changedetectionio.notification import (
     valid_notification_formats,
 )
 
+from email import policy
+from email.parser import BytesParser, Parser
 
+def parse_mime(raw):
+    """Return (EmailMessage, dict[str, list[str]] bodies by content-type)."""
+    if isinstance(raw, (bytes, bytearray)):
+        msg = BytesParser(policy=policy.default).parsebytes(raw)
+    else:
+        msg = Parser(policy=policy.default).parsestr(raw)
+
+    parts_by_type = {}
+    if msg.is_multipart():
+        for part in msg.walk():
+            if part.get_content_maintype() == "multipart":
+                continue
+            ctype = part.get_content_type()           # e.g. "text/plain"
+            text = part.get_content()                 # decoded str
+            parts_by_type.setdefault(ctype, []).append(text)
+    else:
+        parts_by_type.setdefault(msg.get_content_type(), []).append(msg.get_content())
+
+    return msg, parts_by_type
+
+def one_or_join(parts_dict, ctype):
+    """Join multiple parts of the same type (rare but possible)."""
+    return "\n".join(parts_dict.get(ctype, []))
+
+def norm_newlines(s: str) -> str:
+    return s.replace("\r\n", "\n").replace("\r", "\n")
 
 def get_last_message_from_smtp_server():
     import socket
@@ -77,37 +105,36 @@ def test_check_notification_email_formats_default_HTML(client, live_server, meas
 
     time.sleep(3)
 
-    msg = get_last_message_from_smtp_server()
-    assert len(msg) >= 1
+    raw = get_last_message_from_smtp_server()
+    assert raw  # not empty
+
+    msg, bodies = parse_mime(raw)
+
+    plain = norm_newlines(one_or_join(bodies, "text/plain"))
+    html = norm_newlines(one_or_join(bodies, "text/html"))
+
+    # Now assert against the decoded bodies
+    assert "(added) So let's see what happens.\n" in plain  # plaintext uses a literal apostrophe
+    assert "(added) So let&#39;s see what happens.<br>" in html  # html uses &#39; and <br>
+
+    # You can also check counts, boundaries, etc.
+    assert html.count("So let&#39;s see what happens.") == 3
+    assert "modified head title had a change." in plain
+    assert "modified head title had a change.<br>" in html
+
+
 
-    # The email should have two bodies, and the text/html part should be <br>
-    assert 'Content-Type: text/plain' in msg
-    assert '(added) So let\'s see what happens.\r\n' in msg  # The plaintext part with \r\n
-    assert 'Content-Type: text/html' in msg
-    assert '(added) So let\'s see what happens.<br>' in msg  # the html part
     res = client.get(url_for("ui.form_delete", uuid="all"), follow_redirects=True)
     assert b'Deleted' in res.data
 
 
 def test_check_notification_email_formats_default_Text_override_HTML(client, live_server, measure_memory_usage):
-    ##  live_server_setup(live_server) # Setup on conftest per function
 
     # HTML problems? see this
     # https://github.com/caronc/apprise/issues/633
-
     set_original_response()
     notification_url = f'mailto://changedetection@{smtp_test_server}:11025/?to=fff@home.com'
-    notification_body = f"""<!DOCTYPE html>
-<html lang="en">
-<head>
-    <title>My Webpage</title>
-</head>
-<body>
-    <h1>Test</h1>
-    {default_notification_body}
-</body>
-</html>
-"""
+    notification_body = f"""{default_notification_body}"""
 
     #####################
     # Set this up for when we remove the notification from the watch, it should fallback with these details
@@ -116,11 +143,12 @@ def test_check_notification_email_formats_default_Text_override_HTML(client, liv
         data={"application-notification_urls": notification_url,
               "application-notification_title": "fallback-title " + default_notification_title,
               "application-notification_body": notification_body,
-              "application-notification_format": 'Text',
+              "application-notification_format": 'Text', # handler.py should be sure to add &format=text to override default html from apprise
               "requests-time_between_check-minutes": 180,
               'application-fetch_backend': "html_requests"},
         follow_redirects=True
     )
+
     assert b"Settings updated." in res.data
 
     # Add a watch and trigger a HTTP POST
@@ -140,18 +168,26 @@ def test_check_notification_email_formats_default_Text_override_HTML(client, liv
     wait_for_all_checks(client)
 
     time.sleep(3)
-    msg = get_last_message_from_smtp_server()
-    assert len(msg) >= 1
-    #    with open('/tmp/m.txt', 'w') as f:
-    #        f.write(msg)
+    raw = get_last_message_from_smtp_server()
+    assert raw
 
-    # The email should not have two bodies, should be TEXT only
+    msg, bodies = parse_mime(raw)
 
-    assert 'Content-Type: text/plain' in msg
-    assert '(added) So let\'s see what happens.\r\n' in msg  # The plaintext part with \r\n
+    plain = norm_newlines(one_or_join(bodies, "text/plain"))
+    html = norm_newlines(one_or_join(bodies, "text/html"))
+    assert not html # should be no HTML here
 
+    # Expect ONLY text/plain body
+    assert "text/plain" in bodies
+    assert "text/html" not in bodies
+
+    # Assert on decoded plaintext (literal apostrophe, not &#39;)
+    # Should be NO markup when in text mode
+    assert "(added) So let's see what happens.\n" in plain
+
+
+    # ---------- Flip to HTML format, then expect multipart with both ----------
     set_original_response()
-    # Now override as HTML format
     res = client.post(
         url_for("ui.ui_edit.edit_page", uuid="first"),
         data={
@@ -161,23 +197,28 @@ def test_check_notification_email_formats_default_Text_override_HTML(client, liv
             "time_between_check_use_default": "y"},
         follow_redirects=True
     )
+
     assert b"Updated watch." in res.data
     wait_for_all_checks(client)
 
     time.sleep(3)
-    msg = get_last_message_from_smtp_server()
-    assert len(msg) >= 1
 
-    # The email should have two bodies, and the text/html part should be <br>
-    assert 'Content-Type: text/plain' in msg
-    assert '(removed) So let\'s see what happens.\r\n' in msg  # The plaintext part with \n
-    assert 'Content-Type: text/html' in msg
-    assert '(removed) So let\'s see what happens.<br>' in msg  # the html part
+    raw = get_last_message_from_smtp_server()
+    assert raw
 
-    # https://github.com/dgtlmoon/changedetection.io/issues/2103
-    assert '<h1>Test</h1>' in msg
-    assert '&lt;' not in msg
-    assert 'Content-Type: text/html' in msg
+    msg, bodies = parse_mime(raw)
+    plain = norm_newlines(one_or_join(bodies, "text/plain"))
+    html = norm_newlines(one_or_join(bodies, "text/html"))
 
-    res = client.get(url_for("ui.form_delete", uuid="all"), follow_redirects=True)
-    assert b'Deleted' in res.data
+    # Expect both text/plain and text/html bodies now
+    assert "text/plain" in bodies
+    assert "text/html" in bodies
+
+    # Plaintext reflects the removal line (literal apostrophe)
+    assert "(removed) So let's see what happens.\n" in plain
+    assert "(removed) So let&#39;s see what happens.<br>" in html
+
+    # Optional: ensure we got multipart/alternative (typical for dual bodies)
+    if msg.is_multipart():
+        # most senders do "multipart/alternative" for text/plain + text/html
+        assert msg.get_content_subtype() in ("alternative", "mixed", "related")

changedetectionio/tests/test_add_replace_remove_filter.py

@@ -75,7 +75,7 @@ def test_check_removed_line_contains_trigger(client, live_server, measure_memory
     wait_for_all_checks(client)
     time.sleep(0.5)
     res = client.get(url_for("watchlist.index"))
-    assert b'unviewed' not in res.data
+    assert b'has-unread-changes' not in res.data
 
     # The trigger line is REMOVED,  this should trigger
     set_original(excluding='The golden line')
@@ -84,7 +84,7 @@ def test_check_removed_line_contains_trigger(client, live_server, measure_memory
     client.get(url_for("ui.form_watch_checknow"), follow_redirects=True)
     wait_for_all_checks(client)
     res = client.get(url_for("watchlist.index"))
-    assert b'unviewed' in res.data
+    assert b'has-unread-changes' in res.data
 
     time.sleep(1)
 
@@ -98,14 +98,14 @@ def test_check_removed_line_contains_trigger(client, live_server, measure_memory
     wait_for_all_checks(client)
     time.sleep(1)
     res = client.get(url_for("watchlist.index"))
-    assert b'unviewed' not in res.data
+    assert b'has-unread-changes' not in res.data
 
     # Remove it again, and we should get a trigger
     set_original(excluding='The golden line')
     client.get(url_for("ui.form_watch_checknow"), follow_redirects=True)
     wait_for_all_checks(client)
     res = client.get(url_for("watchlist.index"))
-    assert b'unviewed' in res.data
+    assert b'has-unread-changes' in res.data
 
     res = client.get(url_for("ui.form_delete", uuid="all"), follow_redirects=True)
     assert b'Deleted' in res.data
@@ -169,7 +169,7 @@ def test_check_add_line_contains_trigger(client, live_server, measure_memory_usa
 
     wait_for_all_checks(client)
     res = client.get(url_for("watchlist.index"))
-    assert b'unviewed' not in res.data
+    assert b'has-unread-changes' not in res.data
 
     # The trigger line is ADDED,  this should trigger
     set_original(add_line='<p>Oh yes please</p>')
@@ -177,7 +177,7 @@ def test_check_add_line_contains_trigger(client, live_server, measure_memory_usa
     wait_for_all_checks(client)
     res = client.get(url_for("watchlist.index"))
 
-    assert b'unviewed' in res.data
+    assert b'has-unread-changes' in res.data
 
     # Takes a moment for apprise to fire
     wait_for_notification_endpoint_output()

changedetectionio/tests/test_backend.py

@@ -38,9 +38,9 @@ def test_check_basic_change_detection_functionality(client, live_server, measure
         # Give the thread time to pick it up
         wait_for_all_checks(client)
 
-        # It should report nothing found (no new 'unviewed' class)
+        # It should report nothing found (no new 'has-unread-changes' class)
         res = client.get(url_for("watchlist.index"))
-        assert b'unviewed' not in res.data
+        assert b'has-unread-changes' not in res.data
         assert b'test-endpoint' in res.data
 
         # Default no password set, this stuff should be always available.
@@ -74,9 +74,9 @@ def test_check_basic_change_detection_functionality(client, live_server, measure
     res = client.get(url_for("ui.ui_edit.watch_get_latest_html", uuid=uuid))
     assert b'which has this one new line' in res.data
 
-    # Now something should be ready, indicated by having a 'unviewed' class
+    # Now something should be ready, indicated by having a 'has-unread-changes' class
     res = client.get(url_for("watchlist.index"))
-    assert b'unviewed' in res.data
+    assert b'has-unread-changes' in res.data
 
     # #75, and it should be in the RSS feed
     rss_token = extract_rss_token_from_UI(client)
@@ -90,7 +90,7 @@ def test_check_basic_change_detection_functionality(client, live_server, measure
 
     assert expected_url.encode('utf-8') in res.data
 #
-    # Following the 'diff' link, it should no longer display as 'unviewed' even after we recheck it a few times
+    # Following the 'diff' link, it should no longer display as 'has-unread-changes' even after we recheck it a few times
     res = client.get(url_for("ui.ui_views.diff_history_page", uuid=uuid))
     assert b'selected=""' in res.data, "Confirm diff history page loaded"
 
@@ -111,12 +111,12 @@ def test_check_basic_change_detection_functionality(client, live_server, measure
         # Give the thread time to pick it up
         wait_for_all_checks(client)
 
-        # It should report nothing found (no new 'unviewed' class)
+        # It should report nothing found (no new 'has-unread-changes' class)
         res = client.get(url_for("watchlist.index"))
 
 
-        assert b'unviewed' not in res.data
-        assert b'class="has-unviewed' not in res.data
+        assert b'has-unread-changes' not in res.data
+        assert b'class="has-unread-changes' not in res.data
         assert b'head title' in res.data  # Should be ON by default
         assert b'test-endpoint' in res.data
 
@@ -140,8 +140,8 @@ def test_check_basic_change_detection_functionality(client, live_server, measure
     wait_for_all_checks(client)
 
     res = client.get(url_for("watchlist.index"))
-    assert b'unviewed' in res.data
-    assert b'class="has-unviewed' in res.data
+    assert b'has-unread-changes' in res.data
+    assert b'class="has-unread-changes' in res.data
     assert b'head title' not in res.data  # should now be off
 
 
@@ -151,8 +151,8 @@ def test_check_basic_change_detection_functionality(client, live_server, measure
     # hit the mark all viewed link
     res = client.get(url_for("ui.mark_all_viewed"), follow_redirects=True)
 
-    assert b'class="has-unviewed' not in res.data
-    assert b'unviewed' not in res.data
+    assert b'class="has-unread-changes' not in res.data
+    assert b'has-unread-changes' not in res.data
 
     # #2458 "clear history" should make the Watch object update its status correctly when the first snapshot lands again
     client.get(url_for("ui.clear_watch_history", uuid=uuid))
@@ -165,3 +165,133 @@ def test_check_basic_change_detection_functionality(client, live_server, measure
     # Cleanup everything
     res = client.get(url_for("ui.form_delete", uuid="all"), follow_redirects=True)
     assert b'Deleted' in res.data
+
+def test_non_text_mime_or_downloads(client, live_server, measure_memory_usage):
+    """
+
+    https://github.com/dgtlmoon/changedetection.io/issues/3434
+    I noticed that a watched website can be monitored fine as long as the server sends content-type: text/plain; charset=utf-8,
+    but once the server sends content-type: application/octet-stream (which is usually done to force the browser to show the Download dialog),
+    changedetection somehow ignores all line breaks and treats the document file as if everything is on one line.
+
+    WHAT THIS DOES - makes the system rely on 'magic' to determine what is it
+
+    :param client:
+    :param live_server:
+    :param measure_memory_usage:
+    :return:
+    """
+    with open("test-datastore/endpoint-content.txt", "w") as f:
+        f.write("""some random text that should be split by line
+and not parsed with html_to_text
+this way we know that it correctly parsed as plain text
+\r\n
+ok\r\n
+got it\r\n
+""")
+
+    test_url = url_for('test_endpoint', content_type="application/octet-stream", _external=True)
+
+    # Add our URL to the import page
+    res = client.post(
+        url_for("imports.import_page"),
+        data={"urls": test_url},
+        follow_redirects=True
+    )
+
+    assert b"1 Imported" in res.data
+
+    wait_for_all_checks(client)
+
+    ### check the front end
+    res = client.get(
+        url_for("ui.ui_views.preview_page", uuid="first"),
+        follow_redirects=True
+    )
+    assert b"some random text that should be split by line\n" in res.data
+    ####
+
+    # Check the snapshot by API that it has linefeeds too
+    watch_uuid = next(iter(live_server.app.config['DATASTORE'].data['watching']))
+    api_key = live_server.app.config['DATASTORE'].data['settings']['application'].get('api_access_token')
+    res = client.get(
+        url_for("watchhistory", uuid=watch_uuid),
+        headers={'x-api-key': api_key},
+    )
+
+    # Fetch a snapshot by timestamp, check the right one was found
+    res = client.get(
+        url_for("watchsinglehistory", uuid=watch_uuid, timestamp=list(res.json.keys())[-1]),
+        headers={'x-api-key': api_key},
+    )
+    assert b"some random text that should be split by line\n" in res.data
+
+
+    res = client.get(url_for("ui.form_delete", uuid="all"), follow_redirects=True)
+
+
+def test_standard_text_plain(client, live_server, measure_memory_usage):
+    """
+
+    https://github.com/dgtlmoon/changedetection.io/issues/3434
+    I noticed that a watched website can be monitored fine as long as the server sends content-type: text/plain; charset=utf-8,
+    but once the server sends content-type: application/octet-stream (which is usually done to force the browser to show the Download dialog),
+    changedetection somehow ignores all line breaks and treats the document file as if everything is on one line.
+
+    The real bug here can be that it will try to process plain-text as HTML, losing <etc>
+
+    :param client:
+    :param live_server:
+    :param measure_memory_usage:
+    :return:
+    """
+    with open("test-datastore/endpoint-content.txt", "w") as f:
+        f.write("""some random text that should be split by line
+and not parsed with html_to_text
+<title>Even this title should stay because we are just plain text</title>
+this way we know that it correctly parsed as plain text
+\r\n
+ok\r\n
+got it\r\n
+""")
+
+    test_url = url_for('test_endpoint', content_type="text/plain", _external=True)
+
+    # Add our URL to the import page
+    res = client.post(
+        url_for("imports.import_page"),
+        data={"urls": test_url},
+        follow_redirects=True
+    )
+
+    assert b"1 Imported" in res.data
+
+    wait_for_all_checks(client)
+
+    ### check the front end
+    res = client.get(
+        url_for("ui.ui_views.preview_page", uuid="first"),
+        follow_redirects=True
+    )
+
+    assert b"some random text that should be split by line\n" in res.data
+    ####
+
+    # Check the snapshot by API that it has linefeeds too
+    watch_uuid = next(iter(live_server.app.config['DATASTORE'].data['watching']))
+    api_key = live_server.app.config['DATASTORE'].data['settings']['application'].get('api_access_token')
+    res = client.get(
+        url_for("watchhistory", uuid=watch_uuid),
+        headers={'x-api-key': api_key},
+    )
+
+    # Fetch a snapshot by timestamp, check the right one was found
+    res = client.get(
+        url_for("watchsinglehistory", uuid=watch_uuid, timestamp=list(res.json.keys())[-1]),
+        headers={'x-api-key': api_key},
+    )
+    assert b"some random text that should be split by line\n" in res.data
+    assert b"<title>Even this title should stay because we are just plain text</title>" in res.data
+
+    res = client.get(url_for("ui.form_delete", uuid="all"), follow_redirects=True)
+

changedetectionio/tests/test_basic_socketio.py

@@ -58,6 +58,7 @@ def run_socketio_watch_update_test(client, live_server, password_mode=""):
 
     has_watch_update = False
     has_unviewed_update = False
+    got_general_stats_update = False
 
     for i in range(10):
         # Get received events
@@ -65,15 +66,11 @@ def run_socketio_watch_update_test(client, live_server, password_mode=""):
 
         if received:
             logger.info(f"Received {len(received)} events after {i+1} seconds")
-
-            # Check for watch_update events with unviewed=True
             for event in received:
                 if event['name'] == 'watch_update':
                     has_watch_update = True
-                    if event['args'][0]['watch'].get('unviewed', False):
-                        has_unviewed_update = True
-                        logger.info("Found unviewed update event!")
-                        break
+                if event['name'] == 'general_stats_update':
+                    got_general_stats_update = True
 
         if has_unviewed_update:
             break
@@ -92,7 +89,7 @@ def run_socketio_watch_update_test(client, live_server, password_mode=""):
     assert has_watch_update, "No watch_update events received"
 
     # Verify we received an unviewed event
-    assert has_unviewed_update, "No watch_update event with unviewed=True received"
+    assert got_general_stats_update, "Got general stats update event"
 
     # Alternatively, check directly if the watch in the datastore is marked as unviewed
     from changedetectionio.flask_app import app

changedetectionio/tests/test_block_while_text_present.py

@@ -107,9 +107,9 @@ def test_check_block_changedetection_text_NOT_present(client, live_server, measu
     # Give the thread time to pick it up
     wait_for_all_checks(client)
 
-    # It should report nothing found (no new 'unviewed' class)
+    # It should report nothing found (no new 'has-unread-changes' class)
     res = client.get(url_for("watchlist.index"))
-    assert b'unviewed' not in res.data
+    assert b'has-unread-changes' not in res.data
     assert b'/test-endpoint' in res.data
 
     # The page changed, BUT the text is still there, just the rest of it changes, we should not see a change
@@ -120,9 +120,9 @@ def test_check_block_changedetection_text_NOT_present(client, live_server, measu
     # Give the thread time to pick it up
     wait_for_all_checks(client)
 
-    # It should report nothing found (no new 'unviewed' class)
+    # It should report nothing found (no new 'has-unread-changes' class)
     res = client.get(url_for("watchlist.index"))
-    assert b'unviewed' not in res.data
+    assert b'has-unread-changes' not in res.data
     assert b'/test-endpoint' in res.data
 
     # 2548
@@ -131,7 +131,7 @@ def test_check_block_changedetection_text_NOT_present(client, live_server, measu
     client.get(url_for("ui.form_watch_checknow"), follow_redirects=True)
     wait_for_all_checks(client)
     res = client.get(url_for("watchlist.index"))
-    assert b'unviewed' not in res.data
+    assert b'has-unread-changes' not in res.data
 
 
     # Now we set a change where the text is gone AND its different content, it should now trigger
@@ -139,7 +139,7 @@ def test_check_block_changedetection_text_NOT_present(client, live_server, measu
     client.get(url_for("ui.form_watch_checknow"), follow_redirects=True)
     wait_for_all_checks(client)
     res = client.get(url_for("watchlist.index"))
-    assert b'unviewed' in res.data
+    assert b'has-unread-changes' in res.data
 
 
 

changedetectionio/tests/test_conditions.py

@@ -125,7 +125,7 @@ def test_conditions_with_text_and_number(client, live_server):
     time.sleep(2)
     # 75 is > 20 and < 100 and contains "5"
     res = client.get(url_for("watchlist.index"))
-    assert b'unviewed' in res.data
+    assert b'has-unread-changes' in res.data
 
 
     # Case 2: Change with one condition violated
@@ -141,7 +141,7 @@ def test_conditions_with_text_and_number(client, live_server):
 
     # Should NOT be marked as having changes since not all conditions are met
     res = client.get(url_for("watchlist.index"))
-    assert b'unviewed' not in res.data
+    assert b'has-unread-changes' not in res.data
 
     res = client.get(url_for("ui.form_delete", uuid="all"), follow_redirects=True)
     assert b'Deleted' in res.data
@@ -299,7 +299,7 @@ def test_lev_conditions_plugin(client, live_server, measure_memory_usage):
 
     wait_for_all_checks(client)
     res = client.get(url_for("watchlist.index"))
-    assert b'unviewed' not in res.data
+    assert b'has-unread-changes' not in res.data
 
     # Check the content saved initially, even tho a condition was set - this is the first snapshot so shouldnt be affected by conditions
     res = client.get(
@@ -326,7 +326,7 @@ def test_lev_conditions_plugin(client, live_server, measure_memory_usage):
     wait_for_all_checks(client)
 
     res = client.get(url_for("watchlist.index"))
-    assert b'unviewed' not in res.data #because this will be like 0.90 not 0.8 threshold
+    assert b'has-unread-changes' not in res.data #because this will be like 0.90 not 0.8 threshold
 
     ############### Now change it a MORE THAN 50%
     test_return_data = """<html>
@@ -345,7 +345,7 @@ def test_lev_conditions_plugin(client, live_server, measure_memory_usage):
     assert b'Queued 1 watch for rechecking.' in res.data
     wait_for_all_checks(client)
     res = client.get(url_for("watchlist.index"))
-    assert b'unviewed' in res.data
+    assert b'has-unread-changes' in res.data
     # cleanup for the next
     client.get(
         url_for("ui.form_delete", uuid="all"),

changedetectionio/tests/test_css_selector.py

@@ -116,10 +116,10 @@ def test_check_markup_include_filters_restriction(client, live_server, measure_m
     # Give the thread time to pick it up
     time.sleep(sleep_time_for_fetch_thread)
 
-    # It should have 'unviewed' still
+    # It should have 'has-unread-changes' still
     # Because it should be looking at only that 'sametext' id
     res = client.get(url_for("watchlist.index"))
-    assert b'unviewed' in res.data
+    assert b'has-unread-changes' in res.data
 
 
 # Tests the whole stack works with the CSS Filter

changedetectionio/tests/test_element_removal.py

@@ -190,7 +190,7 @@ def test_element_removal_full(client, live_server, measure_memory_usage):
 
     wait_for_all_checks(client)
 
-    # so that we set the state to 'unviewed' after all the edits
+    # so that we set the state to 'has-unread-changes' after all the edits
     client.get(url_for("ui.ui_views.diff_history_page", uuid="first"))
 
     #  Make a change to header/footer/nav

changedetectionio/tests/test_errorhandling.py

@@ -31,7 +31,7 @@ def _runner_test_http_errors(client, live_server, http_code, expected_text):
 
     res = client.get(url_for("watchlist.index"))
     # no change
-    assert b'unviewed' not in res.data
+    assert b'has-unread-changes' not in res.data
     assert bytes(expected_text.encode('utf-8')) in res.data
 
 

changedetectionio/tests/test_extract_regex.py

@@ -174,10 +174,10 @@ def test_check_filter_and_regex_extract(client, live_server, measure_memory_usag
     # Give the thread time to pick it up
     wait_for_all_checks(client)
 
-    # It should have 'unviewed' still
+    # It should have 'has-unread-changes' still
     # Because it should be looking at only that 'sametext' id
     res = client.get(url_for("watchlist.index"))
-    assert b'unviewed' in res.data
+    assert b'has-unread-changes' in res.data
 
     # Check HTML conversion detected and workd
     res = client.get(

changedetectionio/tests/test_group.py

@@ -264,8 +264,6 @@ def test_limit_tag_ui(client, live_server, measure_memory_usage):
     client.get(url_for('ui.mark_all_viewed', tag=tag_uuid), follow_redirects=True)
     wait_for_all_checks(client)
 
-    with open('/tmp/fuck.html', 'wb') as f:
-        f.write(res.data)
     # Should be only 1 unviewed
     res = client.get(url_for("watchlist.index"))
     assert res.data.count(b' unviewed ') == 1

changedetectionio/tests/test_history_consistency.py

@@ -3,9 +3,8 @@
 import time
 import os
 import json
-import logging
 from flask import url_for
-from .util import live_server_setup, wait_for_all_checks
+from .util import wait_for_all_checks
 from urllib.parse import urlparse, parse_qs
 
 def test_consistent_history(client, live_server, measure_memory_usage):

changedetectionio/tests/test_ignore.py

@@ -58,3 +58,39 @@ def test_ignore(client, live_server, measure_memory_usage):
     # Should be in base.html
     assert b'csrftoken' in res.data
 
+
+def test_strip_ignore_lines(client, live_server, measure_memory_usage):
+   #  live_server_setup(live_server) # Setup on conftest per function
+    set_original_ignore_response()
+
+
+    # Goto the settings page, add our ignore text
+    res = client.post(
+        url_for("settings.settings_page"),
+        data={
+            "requests-time_between_check-minutes": 180,
+            "application-ignore_whitespace": "y",
+            "application-strip_ignored_lines": "y",
+            "application-global_ignore_text": "Which is across multiple",
+            'application-fetch_backend': "html_requests"
+        },
+        follow_redirects=True
+    )
+    assert b"Settings updated." in res.data
+
+    test_url = url_for('test_endpoint', _external=True)
+    res = client.post(
+        url_for("imports.import_page"),
+        data={"urls": test_url},
+        follow_redirects=True
+    )
+    assert b"1 Imported" in res.data
+
+    # Give the thread time to pick it up
+    wait_for_all_checks(client)
+    uuid = next(iter(live_server.app.config['DATASTORE'].data['watching']))
+
+    # It should not be in the preview anymore
+    res = client.get(url_for("ui.ui_views.preview_page", uuid=uuid))
+    assert b'<div class="ignored">' not in res.data
+    assert b'Which is across multiple' not in res.data

changedetectionio/tests/test_ignore_text.py

@@ -128,9 +128,9 @@ def test_check_ignore_text_functionality(client, live_server, measure_memory_usa
     # Give the thread time to pick it up
     wait_for_all_checks(client)
 
-    # It should report nothing found (no new 'unviewed' class)
+    # It should report nothing found (no new 'has-unread-changes' class)
     res = client.get(url_for("watchlist.index"))
-    assert b'unviewed' not in res.data
+    assert b'has-unread-changes' not in res.data
     assert b'/test-endpoint' in res.data
 
     #  Make a change
@@ -141,9 +141,9 @@ def test_check_ignore_text_functionality(client, live_server, measure_memory_usa
     # Give the thread time to pick it up
     wait_for_all_checks(client)
 
-    # It should report nothing found (no new 'unviewed' class)
+    # It should report nothing found (no new 'has-unread-changes' class)
     res = client.get(url_for("watchlist.index"))
-    assert b'unviewed' not in res.data
+    assert b'has-unread-changes' not in res.data
     assert b'/test-endpoint' in res.data
 
 
@@ -154,7 +154,7 @@ def test_check_ignore_text_functionality(client, live_server, measure_memory_usa
     wait_for_all_checks(client)
 
     res = client.get(url_for("watchlist.index"))
-    assert b'unviewed' in res.data
+    assert b'has-unread-changes' in res.data
 
     res = client.get(url_for("ui.ui_views.preview_page", uuid="first"))
 
@@ -222,9 +222,9 @@ def _run_test_global_ignore(client, as_source=False, extra_ignore=""):
     # Trigger a check
     client.get(url_for("ui.form_watch_checknow"), follow_redirects=True)
     wait_for_all_checks(client)
-    # It should report nothing found (no new 'unviewed' class), adding random ignore text should not cause a change
+    # It should report nothing found (no new 'has-unread-changes' class), adding random ignore text should not cause a change
     res = client.get(url_for("watchlist.index"))
-    assert b'unviewed' not in res.data
+    assert b'has-unread-changes' not in res.data
     assert b'/test-endpoint' in res.data
 #####
 
@@ -238,10 +238,10 @@ def _run_test_global_ignore(client, as_source=False, extra_ignore=""):
     # Give the thread time to pick it up
     wait_for_all_checks(client)
 
-    # It should report nothing found (no new 'unviewed' class)
+    # It should report nothing found (no new 'has-unread-changes' class)
     res = client.get(url_for("watchlist.index"))
 
-    assert b'unviewed' not in res.data
+    assert b'has-unread-changes' not in res.data
     assert b'/test-endpoint' in res.data
 
     # Just to be sure.. set a regular modified change that will trigger it
@@ -249,7 +249,7 @@ def _run_test_global_ignore(client, as_source=False, extra_ignore=""):
     client.get(url_for("ui.form_watch_checknow"), follow_redirects=True)
     wait_for_all_checks(client)
     res = client.get(url_for("watchlist.index"))
-    assert b'unviewed' in res.data
+    assert b'has-unread-changes' in res.data
 
     res = client.get(url_for("ui.form_delete", uuid="all"), follow_redirects=True)
     assert b'Deleted' in res.data

changedetectionio/tests/test_ignorehyperlinks.py

@@ -111,7 +111,7 @@ def test_render_anchor_tag_content_true(client, live_server, measure_memory_usag
     assert '(/modified_link)' in res.data.decode()
 
     # since the link has changed, and we chose to render anchor tag content,
-    # we should detect a change (new 'unviewed' class)
+    # we should detect a change (new 'has-unread-changes' class)
     res = client.get(url_for("watchlist.index"))
     assert b"unviewed" in res.data
     assert b"/test-endpoint" in res.data

changedetectionio/tests/test_ignorestatuscode.py

@@ -77,9 +77,9 @@ def test_normal_page_check_works_with_ignore_status_code(client, live_server, me
     # Give the thread time to pick it up
     wait_for_all_checks(client)
 
-    # It should report nothing found (no new 'unviewed' class)
+    # It should report nothing found (no new 'has-unread-changes' class)
     res = client.get(url_for("watchlist.index"))
-    assert b'unviewed' in res.data
+    assert b'has-unread-changes' in res.data
     assert b'/test-endpoint' in res.data
 
 
@@ -124,8 +124,8 @@ def test_403_page_check_works_with_ignore_status_code(client, live_server, measu
     # Give the thread time to pick it up
     wait_for_all_checks(client)
 
-    # It should have 'unviewed' still
+    # It should have 'has-unread-changes' still
     # Because it should be looking at only that 'sametext' id
     res = client.get(url_for("watchlist.index"))
-    assert b'unviewed' in res.data
+    assert b'has-unread-changes' in res.data
 

changedetectionio/tests/test_ignorewhitespace.py

@@ -89,7 +89,7 @@ def test_check_ignore_whitespace(client, live_server, measure_memory_usage):
     # Give the thread time to pick it up
     time.sleep(sleep_time_for_fetch_thread)
 
-    # It should report nothing found (no new 'unviewed' class)
+    # It should report nothing found (no new 'has-unread-changes' class)
     res = client.get(url_for("watchlist.index"))
-    assert b'unviewed' not in res.data
+    assert b'has-unread-changes' not in res.data
     assert b'/test-endpoint' in res.data

changedetectionio/tests/test_jinja2.py

@@ -3,6 +3,7 @@
 import time
 from flask import url_for
 from .util import live_server_setup, wait_for_all_checks
+from ..notification import default_notification_title, default_notification_body, default_notification_format
 
 
 # def test_setup(client, live_server, measure_memory_usage):
@@ -10,7 +11,6 @@ from .util import live_server_setup, wait_for_all_checks
 
 # If there was only a change in the whitespacing, then we shouldnt have a change detected
 def test_jinja2_in_url_query(client, live_server, measure_memory_usage):
-    
 
     # Add our URL to the import page
     test_url = url_for('test_return_query', _external=True)
@@ -26,7 +26,7 @@ def test_jinja2_in_url_query(client, live_server, measure_memory_usage):
     assert b"Watch added" in res.data
     wait_for_all_checks(client)
 
-    # It should report nothing found (no new 'unviewed' class)
+    # It should report nothing found (no new 'has-unread-changes' class)
     res = client.get(
         url_for("ui.ui_views.preview_page", uuid="first"),
         follow_redirects=True
@@ -51,8 +51,25 @@ def test_jinja2_security_url_query(client, live_server, measure_memory_usage):
     assert b"Watch added" in res.data
     wait_for_all_checks(client)
 
-    # It should report nothing found (no new 'unviewed' class)
+    # It should report nothing found (no new 'has-unread-changes' class)
     res = client.get(url_for("watchlist.index"))
     assert b'is invalid and cannot be used' in res.data
     # Some of the spewed output from the subclasses
     assert b'dict_values' not in res.data
+
+
+def test_jinja2_notification(client, live_server, measure_memory_usage):
+
+    res = client.post(
+        url_for("settings.settings_page"),
+        data={"application-notification_urls": "posts://127.0.0.1",
+              "application-notification_title": "on the {% now  'America/New_York', '%Y-%m-%d' %}",
+              "application-notification_body": "on the {% now  'America/New_York', '%Y-%m-%d' %}",
+              "application-notification_format": default_notification_format,
+              "requests-time_between_check-minutes": 180,
+              'application-fetch_backend': "html_requests"},
+        follow_redirects=True
+    )
+
+    assert b"Settings updated." in res.data
+    assert b"Settings updated." in res.data

changedetectionio/tests/test_jsonpath_jq_selector.py

@@ -280,9 +280,9 @@ def check_json_filter(json_filter, client, live_server):
     # Give the thread time to pick it up
     wait_for_all_checks(client)
 
-    # It should have 'unviewed' still
+    # It should have 'has-unread-changes' still
     res = client.get(url_for("watchlist.index"))
-    assert b'unviewed' in res.data
+    assert b'has-unread-changes' in res.data
 
     # Should not see this, because its not in the JSONPath we entered
     res = client.get(url_for("ui.ui_views.diff_history_page", uuid="first"))
@@ -418,14 +418,14 @@ def check_json_ext_filter(json_filter, client, live_server):
     # Give the thread time to pick it up
     wait_for_all_checks(client)
 
-    # It should have 'unviewed'
+    # It should have 'has-unread-changes'
     res = client.get(url_for("watchlist.index"))
-    assert b'unviewed' in res.data
+    assert b'has-unread-changes' in res.data
 
     res = client.get(url_for("ui.ui_views.preview_page", uuid="first"))
 
     # We should never see 'ForSale' because we are selecting on 'Sold' in the rule,
-    # But we should know it triggered ('unviewed' assert above)
+    # But we should know it triggered ('has-unread-changes' assert above)
     assert b'ForSale' not in res.data
     assert b'Sold' in res.data
 
@@ -465,7 +465,7 @@ def test_ignore_json_order(client, live_server, measure_memory_usage):
     wait_for_all_checks(client)
 
     res = client.get(url_for("watchlist.index"))
-    assert b'unviewed' not in res.data
+    assert b'has-unread-changes' not in res.data
 
     # Just to be sure it still works
     with open("test-datastore/endpoint-content.txt", "w") as f:
@@ -476,7 +476,7 @@ def test_ignore_json_order(client, live_server, measure_memory_usage):
     wait_for_all_checks(client)
 
     res = client.get(url_for("watchlist.index"))
-    assert b'unviewed' in res.data
+    assert b'has-unread-changes' in res.data
 
     res = client.get(url_for("ui.form_delete", uuid="all"), follow_redirects=True)
     assert b'Deleted' in res.data

changedetectionio/tests/test_nonrenderable_pages.py

@@ -40,9 +40,9 @@ def test_check_basic_change_detection_functionality(client, live_server, measure
 
     wait_for_all_checks(client)
 
-    # It should report nothing found (no new 'unviewed' class)
+    # It should report nothing found (no new 'has-unread-changes' class)
     res = client.get(url_for("watchlist.index"))
-    assert b'unviewed' not in res.data
+    assert b'has-unread-changes' not in res.data
 
 
     #####################
@@ -62,9 +62,9 @@ def test_check_basic_change_detection_functionality(client, live_server, measure
     # Give the thread time to pick it up
     wait_for_all_checks(client)
 
-    # It should report nothing found (no new 'unviewed' class)
+    # It should report nothing found (no new 'has-unread-changes' class)
     res = client.get(url_for("watchlist.index"))
-    assert b'unviewed' not in res.data
+    assert b'has-unread-changes' not in res.data
 
     uuid = next(iter(live_server.app.config['DATASTORE'].data['watching']))
     watch = live_server.app.config['DATASTORE'].data['watching'][uuid]
@@ -92,9 +92,9 @@ def test_check_basic_change_detection_functionality(client, live_server, measure
     # Give the thread time to pick it up
     wait_for_all_checks(client)
 
-    # It should report nothing found (no new 'unviewed' class)
+    # It should report nothing found (no new 'has-unread-changes' class)
     res = client.get(url_for("watchlist.index"))
-    assert b'unviewed' in res.data
+    assert b'has-unread-changes' in res.data
     client.get(url_for("ui.mark_all_viewed"), follow_redirects=True)
     time.sleep(0.2)
 
@@ -108,7 +108,7 @@ def test_check_basic_change_detection_functionality(client, live_server, measure
 
     wait_for_all_checks(client)
     res = client.get(url_for("watchlist.index"))
-    assert b'unviewed' in res.data # A change should have registered because empty_pages_are_a_change is ON
+    assert b'has-unread-changes' in res.data # A change should have registered because empty_pages_are_a_change is ON
     assert b'fetch-error' not in res.data
 
     #

changedetectionio/tests/test_notification.py

@@ -291,6 +291,20 @@ def test_notification_custom_endpoint_and_jinja2(client, live_server, measure_me
     # test_endpoint - that sends the contents of a file
     # test_notification_endpoint - that takes a POST and writes it to file (test-datastore/notification.txt)
 
+    # Drop in a custom wrapping template
+    with open("test-datastore/notification-wrapper.html", "w" ) as f:
+        f.write("""<html>
+        <body id="notification-wrapper">
+        
+        A change was detected at {{watch_html_link}}
+                        template_params = notification_parameters.copy()
+                template_params['notification_body'] = n_body
+                template_params['notification_url_current'] = url
+                n_body = jinja_render(template_str=notification_template, **template_params)
+        </body>
+        """)
+
+
     # CUSTOM JSON BODY CHECK for POST://
     set_original_response()
     # https://github.com/caronc/apprise/wiki/Notify_Custom_JSON#header-manipulation

changedetectionio/tests/test_pdf.py

@@ -49,9 +49,9 @@ def test_fetch_pdf(client, live_server, measure_memory_usage):
 
     wait_for_all_checks(client)
 
-    # Now something should be ready, indicated by having a 'unviewed' class
+    # Now something should be ready, indicated by having a 'has-unread-changes' class
     res = client.get(url_for("watchlist.index"))
-    assert b'unviewed' in res.data
+    assert b'has-unread-changes' in res.data
 
     # The original checksum should be not be here anymore (cdio adds it to the bottom of the text)
 

changedetectionio/tests/test_preview_endpoints.py

@@ -47,9 +47,9 @@ def test_fetch_pdf(client, live_server, measure_memory_usage):
 
     wait_for_all_checks(client)
 
-    # Now something should be ready, indicated by having a 'unviewed' class
+    # Now something should be ready, indicated by having a 'has-unread-changes' class
     res = client.get(url_for("watchlist.index"))
-    assert b'unviewed' in res.data
+    assert b'has-unread-changes' in res.data
 
     # The original checksum should be not be here anymore (cdio adds it to the bottom of the text)
 

changedetectionio/tests/test_restock_itemprop.py

@@ -112,7 +112,7 @@ def test_itemprop_price_change(client, live_server):
     wait_for_all_checks(client)
     res = client.get(url_for("watchlist.index"))
     assert b'180.45' in res.data
-    assert b'unviewed' in res.data
+    assert b'has-unread-changes' in res.data
     client.get(url_for("ui.mark_all_viewed"), follow_redirects=True)
     time.sleep(0.2)
 
@@ -129,7 +129,7 @@ def test_itemprop_price_change(client, live_server):
     wait_for_all_checks(client)
     res = client.get(url_for("watchlist.index"))
     assert b'120.45' in res.data
-    assert b'unviewed' not in res.data
+    assert b'has-unread-changes' not in res.data
 
 
     res = client.get(url_for("ui.form_delete", uuid="all"), follow_redirects=True)
@@ -178,7 +178,7 @@ def _run_test_minmax_limit(client, extra_watch_edit_form):
     assert b'more than one price detected' not in res.data
     # BUT the new price should show, even tho its within limits
     assert b'1,000.45' or b'1000.45' in res.data #depending on locale
-    assert b'unviewed' not in res.data
+    assert b'has-unread-changes' not in res.data
 
     # price changed to something LESS than min (900), SHOULD be a change
     set_original_response(props_markup=instock_props[0], price='890.45')
@@ -188,7 +188,7 @@ def _run_test_minmax_limit(client, extra_watch_edit_form):
     wait_for_all_checks(client)
     res = client.get(url_for("watchlist.index"))
     assert b'890.45' in res.data
-    assert b'unviewed' in res.data
+    assert b'has-unread-changes' in res.data
 
     client.get(url_for("ui.mark_all_viewed"))
 
@@ -200,7 +200,7 @@ def _run_test_minmax_limit(client, extra_watch_edit_form):
     wait_for_all_checks(client)
     res = client.get(url_for("watchlist.index"))
     assert b'820.45' in res.data
-    assert b'unviewed' in res.data
+    assert b'has-unread-changes' in res.data
     client.get(url_for("ui.mark_all_viewed"))
 
     # price changed to something MORE than max (1100.10), SHOULD be a change
@@ -210,7 +210,7 @@ def _run_test_minmax_limit(client, extra_watch_edit_form):
     res = client.get(url_for("watchlist.index"))
     # Depending on the LOCALE it may be either of these (generally for US/default/etc)
     assert b'1,890.45' in res.data or b'1890.45' in res.data
-    assert b'unviewed' in res.data
+    assert b'has-unread-changes' in res.data
 
     res = client.get(url_for("ui.form_delete", uuid="all"), follow_redirects=True)
     assert b'Deleted' in res.data
@@ -294,7 +294,7 @@ def test_itemprop_percent_threshold(client, live_server):
     wait_for_all_checks(client)
     res = client.get(url_for("watchlist.index"))
     assert b'960.45' in res.data
-    assert b'unviewed' not in res.data
+    assert b'has-unread-changes' not in res.data
 
     # Bigger INCREASE change than the threshold should trigger
     set_original_response(props_markup=instock_props[0], price='1960.45')
@@ -302,7 +302,7 @@ def test_itemprop_percent_threshold(client, live_server):
     wait_for_all_checks(client)
     res = client.get(url_for("watchlist.index"))
     assert b'1,960.45' or b'1960.45' in res.data #depending on locale
-    assert b'unviewed' in res.data
+    assert b'has-unread-changes' in res.data
 
 
     # Small decrease should NOT trigger
@@ -312,7 +312,7 @@ def test_itemprop_percent_threshold(client, live_server):
     wait_for_all_checks(client)
     res = client.get(url_for("watchlist.index"))
     assert b'1,950.45' or b'1950.45' in res.data #depending on locale
-    assert b'unviewed' not in res.data
+    assert b'has-unread-changes' not in res.data
 
 
 

changedetectionio/tests/test_source.py

@@ -43,9 +43,9 @@ def test_check_basic_change_detection_functionality_source(client, live_server,
 
     wait_for_all_checks(client)
 
-    # Now something should be ready, indicated by having a 'unviewed' class
+    # Now something should be ready, indicated by having a 'has-unread-changes' class
     res = client.get(url_for("watchlist.index"))
-    assert b'unviewed' in res.data
+    assert b'has-unread-changes' in res.data
 
     res = client.get(
         url_for("ui.ui_views.diff_history_page", uuid="first"),

changedetectionio/tests/test_trigger.py

@@ -96,7 +96,7 @@ def test_trigger_functionality(client, live_server, measure_memory_usage):
 
 
     
-    # so that we set the state to 'unviewed' after all the edits
+    # so that we set the state to 'has-unread-changes' after all the edits
     client.get(url_for("ui.ui_views.diff_history_page", uuid="first"))
 
     # Trigger a check
@@ -104,9 +104,9 @@ def test_trigger_functionality(client, live_server, measure_memory_usage):
 
     wait_for_all_checks(client)
 
-    # It should report nothing found (no new 'unviewed' class)
+    # It should report nothing found (no new 'has-unread-changes' class)
     res = client.get(url_for("watchlist.index"))
-    assert b'unviewed' not in res.data
+    assert b'has-unread-changes' not in res.data
     assert b'/test-endpoint' in res.data
 
     #  Make a change
@@ -116,9 +116,9 @@ def test_trigger_functionality(client, live_server, measure_memory_usage):
     client.get(url_for("ui.form_watch_checknow"), follow_redirects=True)
     wait_for_all_checks(client)
 
-    # It should report nothing found (no new 'unviewed' class)
+    # It should report nothing found (no new 'has-unread-changes' class)
     res = client.get(url_for("watchlist.index"))
-    assert b'unviewed' not in res.data
+    assert b'has-unread-changes' not in res.data
 
     # Now set the content which contains the trigger text
     set_modified_with_trigger_text_response()
@@ -126,7 +126,7 @@ def test_trigger_functionality(client, live_server, measure_memory_usage):
     client.get(url_for("ui.form_watch_checknow"), follow_redirects=True)
     wait_for_all_checks(client)
     res = client.get(url_for("watchlist.index"))
-    assert b'unviewed' in res.data
+    assert b'has-unread-changes' in res.data
     
     # https://github.com/dgtlmoon/changedetection.io/issues/616
     # Apparently the actual snapshot that contains the trigger never shows

changedetectionio/tests/test_trigger_regex.py

@@ -42,7 +42,7 @@ def test_trigger_regex_functionality(client, live_server, measure_memory_usage):
 
     # It should report nothing found (just a new one shouldnt have anything)
     res = client.get(url_for("watchlist.index"))
-    assert b'unviewed' not in res.data
+    assert b'has-unread-changes' not in res.data
 
     ### test regex
     res = client.post(
@@ -54,7 +54,7 @@ def test_trigger_regex_functionality(client, live_server, measure_memory_usage):
         follow_redirects=True
     )
     wait_for_all_checks(client)
-    # so that we set the state to 'unviewed' after all the edits
+    # so that we set the state to 'has-unread-changes' after all the edits
     client.get(url_for("ui.ui_views.diff_history_page", uuid="first"))
 
     with open("test-datastore/endpoint-content.txt", "w") as f:
@@ -65,7 +65,7 @@ def test_trigger_regex_functionality(client, live_server, measure_memory_usage):
 
     # It should report nothing found (nothing should match the regex)
     res = client.get(url_for("watchlist.index"))
-    assert b'unviewed' not in res.data
+    assert b'has-unread-changes' not in res.data
 
     with open("test-datastore/endpoint-content.txt", "w") as f:
         f.write("regex test123<br>\nsomething 123")
@@ -73,7 +73,7 @@ def test_trigger_regex_functionality(client, live_server, measure_memory_usage):
     client.get(url_for("ui.form_watch_checknow"), follow_redirects=True)
     wait_for_all_checks(client)
     res = client.get(url_for("watchlist.index"))
-    assert b'unviewed' in res.data
+    assert b'has-unread-changes' in res.data
 
     # Cleanup everything
     res = client.get(url_for("ui.form_delete", uuid="all"), follow_redirects=True)

changedetectionio/tests/test_trigger_regex_with_filter.py

@@ -69,7 +69,7 @@ def test_trigger_regex_functionality_with_filter(client, live_server, measure_me
 
     # It should report nothing found (nothing should match the regex and filter)
     res = client.get(url_for("watchlist.index"))
-    assert b'unviewed' not in res.data
+    assert b'has-unread-changes' not in res.data
 
     # now this should trigger something
     with open("test-datastore/endpoint-content.txt", "w") as f:
@@ -78,7 +78,7 @@ def test_trigger_regex_functionality_with_filter(client, live_server, measure_me
     client.get(url_for("ui.form_watch_checknow"), follow_redirects=True)
     time.sleep(sleep_time_for_fetch_thread)
     res = client.get(url_for("watchlist.index"))
-    assert b'unviewed' in res.data
+    assert b'has-unread-changes' in res.data
 
 # Cleanup everything
     res = client.get(url_for("ui.form_delete", uuid="all"), follow_redirects=True)

changedetectionio/tests/test_ui.py

@@ -248,3 +248,44 @@ def test_page_title_listing_behaviour(client, live_server):
     res = client.get(url_for("watchlist.index"))
     assert b"head titlecustom html" in res.data
 
+
+def test_ui_viewed_unread_flag(client, live_server):
+
+    import time
+
+    set_original_response(extra_title="custom html")
+
+    # Add our URL to the import page
+    res = client.post(
+        url_for("imports.import_page"),
+        data={"urls": url_for('test_endpoint', _external=True)+"\r\n"+url_for('test_endpoint', _external=True)},
+        follow_redirects=True
+    )
+
+    assert b"2 Imported" in res.data
+    wait_for_all_checks(client)
+
+    set_modified_response()
+    res = client.get(url_for("ui.form_watch_checknow"), follow_redirects=True)
+    assert b'Queued 2 watches for rechecking.' in res.data
+    wait_for_all_checks(client)
+    res = client.get(url_for("watchlist.index"))
+    assert b'<span id="unread-tab-counter">2</span>' in res.data
+    assert res.data.count(b'data-watch-uuid') == 2
+
+    # one should now be viewed, but two in total still
+    client.get(url_for("ui.ui_views.diff_history_page", uuid="first"))
+    res = client.get(url_for("watchlist.index"))
+    assert b'<span id="unread-tab-counter">1</span>' in res.data
+    assert res.data.count(b'data-watch-uuid') == 2
+
+    # check ?unread=1 works
+    res = client.get(url_for("watchlist.index")+"?unread=1")
+    assert res.data.count(b'data-watch-uuid') == 1
+    assert b'<span id="unread-tab-counter">1</span>' in res.data
+
+    # Mark all viewed test again
+    client.get(url_for("ui.mark_all_viewed"), follow_redirects=True)
+    time.sleep(0.2)
+    res = client.get(url_for("watchlist.index"))
+    assert b'<span id="unread-tab-counter">0</span>' in res.data

changedetectionio/tests/test_unique_lines.py

@@ -97,7 +97,7 @@ def test_unique_lines_functionality(client, live_server, measure_memory_usage):
         follow_redirects=True
     )
     assert b"Updated watch." in res.data
-    assert b'unviewed' not in res.data
+    assert b'has-unread-changes' not in res.data
 
     #  Make a change
     set_modified_swapped_lines()
@@ -108,16 +108,16 @@ def test_unique_lines_functionality(client, live_server, measure_memory_usage):
     # Give the thread time to pick it up
     wait_for_all_checks(client)
 
-    # It should report nothing found (no new 'unviewed' class)
+    # It should report nothing found (no new 'has-unread-changes' class)
     res = client.get(url_for("watchlist.index"))
-    assert b'unviewed' not in res.data
+    assert b'has-unread-changes' not in res.data
 
     # Now set the content which contains the new text and re-ordered existing text
     set_modified_with_trigger_text_response()
     client.get(url_for("ui.form_watch_checknow"), follow_redirects=True)
     wait_for_all_checks(client)
     res = client.get(url_for("watchlist.index"))
-    assert b'unviewed' in res.data
+    assert b'has-unread-changes' in res.data
     res = client.get(url_for("ui.form_delete", uuid="all"), follow_redirects=True)
     assert b'Deleted' in res.data
 
@@ -157,7 +157,7 @@ def test_sort_lines_functionality(client, live_server, measure_memory_usage):
 
     res = client.get(url_for("watchlist.index"))
     # Should be a change registered
-    assert b'unviewed' in res.data
+    assert b'has-unread-changes' in res.data
 
     res = client.get(
         url_for("ui.ui_views.preview_page", uuid="first"),

changedetectionio/tests/test_xpath_selector.py

@@ -1,12 +1,42 @@
 # -*- coding: utf-8 -*-
 
-import time
+
 from flask import url_for
-from .util import live_server_setup, wait_for_all_checks
-
-from ..html_tools import *
+from .util import  wait_for_all_checks
+from ..processors.magic import RSS_XML_CONTENT_TYPES
 
 
+def set_rss_atom_feed_response(header=''):
+    test_return_data = f"""{header}<!-- Generated on Wed, 08 Oct 2025 08:42:33 -0700, really really honestly  -->
+<rss xmlns:atom="http://www.w3.org/2005/Atom" version="2.0">
+<channel>
+    <atom:link href="https://store.waterpowered.com/news/collection//" rel="self" type="application/rss+xml"/>
+    <title>RSS Feed</title>
+    <link>
+        <![CDATA[ https://store.waterpowered.com/news/collection// ]]>
+    </link>
+    <description>
+        <![CDATA[ Events and Announcements for ]]>
+    </description>
+    <language>en-us</language>
+    <generator>water News RSS</generator>
+    <item>
+        <title> 🍁 Lets go discount</title>
+        <description><p class="bb_paragraph">ok heres the description</p></description>
+        <link>
+        <![CDATA[ https://store.waterpowered.com/news/app/1643320/view/511845698831908921 ]]>
+        </link>
+        <pubDate>Wed, 08 Oct 2025 15:28:55 +0000</pubDate>
+        <guid isPermaLink="true">https://store.waterpowered.com/news/app/1643320/view/511845698831908921</guid>
+        <enclosure url="https://clan.fastly.waterstatic.com/images/40721482/42822e5f00b2becf520ace9500981bb56f3a89f2.jpg" length="0" type="image/jpeg"/>
+    </item>
+</channel>
+</rss>"""
+
+    with open("test-datastore/endpoint-content.txt", "w") as f:
+        f.write(test_return_data)
+
+    return None
 
 
 
@@ -208,7 +238,7 @@ def test_check_markup_xpath_filter_restriction(client, live_server, measure_memo
     wait_for_all_checks(client)
 
     res = client.get(url_for("watchlist.index"))
-    assert b'unviewed' not in res.data
+    assert b'has-unread-changes' not in res.data
     res = client.get(url_for("ui.form_delete", uuid="all"), follow_redirects=True)
     assert b'Deleted' in res.data
 
@@ -575,3 +605,47 @@ def test_xpath_20_function_string_join_matches(client, live_server, measure_memo
 
     client.get(url_for("ui.form_delete", uuid="all"), follow_redirects=True)
 
+
+def _subtest_xpath_rss(client, content_type='text/html'):
+
+    # Add our URL to the import page
+    test_url = url_for('test_endpoint', content_type=content_type, _external=True)
+    res = client.post(
+        url_for("ui.ui_views.form_quick_watch_add"),
+        data={"url": test_url, "tags": '', 'edit_and_watch_submit_button': 'Edit > Watch'},
+        follow_redirects=True
+    )
+
+    assert b"Watch added in Paused state, saving will unpause" in res.data
+
+    res = client.post(
+        url_for("ui.ui_edit.edit_page", uuid="first", unpause_on_save=1),
+        data={
+            "url": test_url,
+            "include_filters": "xpath://item",
+            "tags": '',
+            "fetch_backend": "html_requests",
+            "time_between_check_use_default": "y",
+        },
+        follow_redirects=True
+    )
+
+    assert b"unpaused" in res.data
+    wait_for_all_checks(client)
+
+    res = client.get(
+        url_for("ui.ui_views.preview_page", uuid="first"),
+        follow_redirects=True
+    )
+
+    assert b"Lets go discount" in res.data, f"When testing for Lets go discount called with content type '{content_type}'"
+    assert b"Events and Announcements" not in res.data, f"When testing for Lets go discount called with content type '{content_type}'" # It should not be here because thats not our selector target
+
+    client.get(url_for("ui.form_delete", uuid="all"), follow_redirects=True)
+
+# Be sure all-in-the-wild types of RSS feeds work with xpath
+def test_rss_xpath(client, live_server):
+    for feed_header in ['', '<?xml version="1.0" encoding="utf-8"?>']:
+        set_rss_atom_feed_response(header=feed_header)
+        for content_type in RSS_XML_CONTENT_TYPES:
+            _subtest_xpath_rss(client, content_type=content_type)

changedetectionio/tests/unit/test_browser_notifications.py

@@ -1,436 +0,0 @@
-"""
-Tests for browser notification functionality
-Tests VAPID key handling, subscription management, and notification sending
-"""
-
-import json
-import sys
-import tempfile
-import os
-import unittest
-from unittest.mock import patch, Mock, MagicMock
-from py_vapid import Vapid
-
-from changedetectionio.notification.apprise_plugin.browser_notification_helpers import (
-    convert_pem_private_key_for_pywebpush,
-    convert_pem_public_key_for_browser,
-    send_push_notifications,
-    create_notification_payload,
-    get_vapid_config_from_datastore,
-    get_browser_subscriptions,
-    save_browser_subscriptions
-)
-
-
-class TestVAPIDKeyHandling(unittest.TestCase):
-    """Test VAPID key generation, conversion, and validation"""
-    
-    def test_create_notification_payload(self):
-        """Test notification payload creation"""
-        payload = create_notification_payload("Test Title", "Test Body", "/test-icon.png")
-        
-        self.assertEqual(payload['title'], "Test Title")
-        self.assertEqual(payload['body'], "Test Body")
-        self.assertEqual(payload['icon'], "/test-icon.png")
-        self.assertEqual(payload['badge'], "/static/favicons/favicon-32x32.png")
-        self.assertIn('timestamp', payload)
-        self.assertIsInstance(payload['timestamp'], int)
-    
-    def test_create_notification_payload_defaults(self):
-        """Test notification payload with default values"""
-        payload = create_notification_payload("Title", "Body")
-        
-        self.assertEqual(payload['icon'], "/static/favicons/favicon-32x32.png")
-        self.assertEqual(payload['badge'], "/static/favicons/favicon-32x32.png")
-    
-    def test_convert_pem_private_key_for_pywebpush_with_valid_pem(self):
-        """Test conversion of valid PEM private key to Vapid instance"""
-        # Generate a real VAPID key
-        vapid = Vapid()
-        vapid.generate_keys()
-        private_pem = vapid.private_pem().decode()
-        
-        # Convert using our function
-        converted_key = convert_pem_private_key_for_pywebpush(private_pem)
-        
-        # Should return a Vapid instance
-        self.assertIsInstance(converted_key, Vapid)
-    
-    def test_convert_pem_private_key_invalid_input(self):
-        """Test conversion with invalid input returns original"""
-        invalid_key = "not-a-pem-key"
-        result = convert_pem_private_key_for_pywebpush(invalid_key)
-        self.assertEqual(result, invalid_key)
-        
-        none_key = None
-        result = convert_pem_private_key_for_pywebpush(none_key)
-        self.assertEqual(result, none_key)
-    
-    def test_convert_pem_public_key_for_browser(self):
-        """Test conversion of PEM public key to browser format"""
-        # Generate a real VAPID key pair
-        vapid = Vapid()
-        vapid.generate_keys()
-        public_pem = vapid.public_pem().decode()
-        
-        # Convert to browser format
-        browser_key = convert_pem_public_key_for_browser(public_pem)
-        
-        # Should return URL-safe base64 string
-        self.assertIsInstance(browser_key, str)
-        self.assertGreater(len(browser_key), 0)
-        # Should not contain padding
-        self.assertFalse(browser_key.endswith('='))
-    
-    def test_convert_pem_public_key_invalid(self):
-        """Test public key conversion with invalid input"""
-        result = convert_pem_public_key_for_browser("invalid-pem")
-        self.assertIsNone(result)
-
-
-class TestDatastoreIntegration(unittest.TestCase):
-    """Test datastore operations for VAPID and subscriptions"""
-    
-    def test_get_vapid_config_from_datastore(self):
-        """Test retrieving VAPID config from datastore"""
-        mock_datastore = Mock()
-        mock_datastore.data = {
-            'settings': {
-                'application': {
-                    'vapid': {
-                        'private_key': 'test-private-key',
-                        'public_key': 'test-public-key',
-                        'contact_email': 'test@example.com'
-                    }
-                }
-            }
-        }
-        
-        private_key, public_key, contact_email = get_vapid_config_from_datastore(mock_datastore)
-        
-        self.assertEqual(private_key, 'test-private-key')
-        self.assertEqual(public_key, 'test-public-key')
-        self.assertEqual(contact_email, 'test@example.com')
-    
-    def test_get_vapid_config_missing_email(self):
-        """Test VAPID config with missing contact email uses default"""
-        mock_datastore = Mock()
-        mock_datastore.data = {
-            'settings': {
-                'application': {
-                    'vapid': {
-                        'private_key': 'test-private-key',
-                        'public_key': 'test-public-key'
-                    }
-                }
-            }
-        }
-        
-        private_key, public_key, contact_email = get_vapid_config_from_datastore(mock_datastore)
-        
-        self.assertEqual(contact_email, 'citizen@example.com')
-    
-    def test_get_vapid_config_empty_datastore(self):
-        """Test VAPID config with empty datastore returns None values"""
-        mock_datastore = Mock()
-        mock_datastore.data = {}
-        
-        private_key, public_key, contact_email = get_vapid_config_from_datastore(mock_datastore)
-        
-        self.assertIsNone(private_key)
-        self.assertIsNone(public_key)
-        self.assertEqual(contact_email, 'citizen@example.com')
-    
-    def test_get_browser_subscriptions(self):
-        """Test retrieving browser subscriptions from datastore"""
-        mock_datastore = Mock()
-        test_subscriptions = [
-            {
-                'endpoint': 'https://fcm.googleapis.com/fcm/send/test1',
-                'keys': {'p256dh': 'key1', 'auth': 'auth1'}
-            },
-            {
-                'endpoint': 'https://fcm.googleapis.com/fcm/send/test2', 
-                'keys': {'p256dh': 'key2', 'auth': 'auth2'}
-            }
-        ]
-        mock_datastore.data = {
-            'settings': {
-                'application': {
-                    'browser_subscriptions': test_subscriptions
-                }
-            }
-        }
-        
-        subscriptions = get_browser_subscriptions(mock_datastore)
-        
-        self.assertEqual(len(subscriptions), 2)
-        self.assertEqual(subscriptions, test_subscriptions)
-    
-    def test_get_browser_subscriptions_empty(self):
-        """Test getting subscriptions from empty datastore returns empty list"""
-        mock_datastore = Mock()
-        mock_datastore.data = {}
-        
-        subscriptions = get_browser_subscriptions(mock_datastore)
-        
-        self.assertEqual(subscriptions, [])
-    
-    def test_save_browser_subscriptions(self):
-        """Test saving browser subscriptions to datastore"""
-        mock_datastore = Mock()
-        mock_datastore.data = {'settings': {'application': {}}}
-        
-        test_subscriptions = [
-            {'endpoint': 'test1', 'keys': {'p256dh': 'key1', 'auth': 'auth1'}}
-        ]
-        
-        save_browser_subscriptions(mock_datastore, test_subscriptions)
-        
-        self.assertEqual(mock_datastore.data['settings']['application']['browser_subscriptions'], test_subscriptions)
-        self.assertTrue(mock_datastore.needs_write)
-
-
-class TestNotificationSending(unittest.TestCase):
-    """Test notification sending with mocked pywebpush"""
-    
-    @patch('pywebpush.webpush')
-    def test_send_push_notifications_success(self, mock_webpush):
-        """Test successful notification sending"""
-        mock_webpush.return_value = True
-        
-        mock_datastore = Mock()
-        mock_datastore.needs_write = False
-        
-        subscriptions = [
-            {
-                'endpoint': 'https://fcm.googleapis.com/fcm/send/test1',
-                'keys': {'p256dh': 'key1', 'auth': 'auth1'}
-            }
-        ]
-        
-        # Generate a real VAPID key for testing
-        vapid = Vapid()
-        vapid.generate_keys()
-        private_key = vapid.private_pem().decode()
-        
-        notification_payload = {
-            'title': 'Test Title',
-            'body': 'Test Body'
-        }
-        
-        success_count, total_count = send_push_notifications(
-            subscriptions=subscriptions,
-            notification_payload=notification_payload,
-            private_key=private_key,
-            contact_email='test@example.com',
-            datastore=mock_datastore
-        )
-        
-        self.assertEqual(success_count, 1)
-        self.assertEqual(total_count, 1)
-        self.assertTrue(mock_webpush.called)
-        
-        # Verify webpush was called with correct parameters
-        call_args = mock_webpush.call_args
-        self.assertEqual(call_args[1]['subscription_info'], subscriptions[0])
-        self.assertEqual(json.loads(call_args[1]['data']), notification_payload)
-        self.assertIn('vapid_private_key', call_args[1])
-        self.assertEqual(call_args[1]['vapid_claims']['sub'], 'mailto:test@example.com')
-    
-    @patch('pywebpush.webpush')
-    def test_send_push_notifications_webpush_exception(self, mock_webpush):
-        """Test handling of WebPushException with invalid subscription removal"""
-        from pywebpush import WebPushException
-        
-        # Mock a 410 response (subscription gone)
-        mock_response = Mock()
-        mock_response.status_code = 410
-        
-        mock_webpush.side_effect = WebPushException("Subscription expired", response=mock_response)
-        
-        mock_datastore = Mock()
-        mock_datastore.needs_write = False
-        
-        subscriptions = [
-            {
-                'endpoint': 'https://fcm.googleapis.com/fcm/send/test1',
-                'keys': {'p256dh': 'key1', 'auth': 'auth1'}
-            }
-        ]
-        
-        vapid = Vapid()
-        vapid.generate_keys()
-        private_key = vapid.private_pem().decode()
-        
-        success_count, total_count = send_push_notifications(
-            subscriptions=subscriptions,
-            notification_payload={'title': 'Test', 'body': 'Test'},
-            private_key=private_key,
-            contact_email='test@example.com',
-            datastore=mock_datastore
-        )
-        
-        self.assertEqual(success_count, 0)
-        self.assertEqual(total_count, 1)
-        self.assertTrue(mock_datastore.needs_write)  # Should mark for subscription cleanup
-    
-    def test_send_push_notifications_no_pywebpush(self):
-        """Test graceful handling when pywebpush is not available"""
-        with patch.dict('sys.modules', {'pywebpush': None}):
-            subscriptions = [{'endpoint': 'test', 'keys': {}}]
-            
-            success_count, total_count = send_push_notifications(
-                subscriptions=subscriptions,
-                notification_payload={'title': 'Test', 'body': 'Test'},
-                private_key='test-key',
-                contact_email='test@example.com',
-                datastore=Mock()
-            )
-            
-            self.assertEqual(success_count, 0)
-            self.assertEqual(total_count, 1)
-
-
-class TestBrowserIntegration(unittest.TestCase):
-    """Test browser integration aspects (file existence)"""
-    
-    def test_javascript_browser_notifications_class_exists(self):
-        """Test that browser notifications JavaScript file exists and has expected structure"""
-        js_file = "/var/www/changedetection.io/changedetectionio/static/js/browser-notifications.js"
-        
-        self.assertTrue(os.path.exists(js_file))
-        
-        with open(js_file, 'r') as f:
-            content = f.read()
-            
-        # Check for key class and methods
-        self.assertIn('class BrowserNotifications', content)
-        self.assertIn('async init()', content)
-        self.assertIn('async subscribe()', content)
-        self.assertIn('async sendTestNotification()', content)
-        self.assertIn('setupNotificationUrlMonitoring()', content)
-    
-    def test_service_worker_exists(self):
-        """Test that service worker file exists"""
-        sw_file = "/var/www/changedetection.io/changedetectionio/static/js/service-worker.js"
-        
-        self.assertTrue(os.path.exists(sw_file))
-        
-        with open(sw_file, 'r') as f:
-            content = f.read()
-            
-        # Check for key service worker functionality
-        self.assertIn('push', content)
-        self.assertIn('notificationclick', content)
-
-
-class TestAPIEndpoints(unittest.TestCase):
-    """Test browser notification API endpoints"""
-    
-    def test_browser_notifications_module_exists(self):
-        """Test that BrowserNotifications API module exists"""
-        api_file = "/var/www/changedetection.io/changedetectionio/notification/BrowserNotifications.py"
-        
-        self.assertTrue(os.path.exists(api_file))
-        
-        with open(api_file, 'r') as f:
-            content = f.read()
-            
-        # Check for key API classes
-        self.assertIn('BrowserNotificationsVapidPublicKey', content)
-        self.assertIn('BrowserNotificationsSubscribe', content)  
-        self.assertIn('BrowserNotificationsUnsubscribe', content)
-    
-    def test_vapid_public_key_conversion(self):
-        """Test VAPID public key conversion for browser use"""
-        # Generate a real key pair
-        vapid = Vapid()
-        vapid.generate_keys()
-        public_pem = vapid.public_pem().decode()
-        
-        # Convert to browser format
-        browser_key = convert_pem_public_key_for_browser(public_pem)
-        
-        # Verify it's a valid URL-safe base64 string
-        self.assertIsInstance(browser_key, str)
-        self.assertGreater(len(browser_key), 80)  # P-256 uncompressed point should be ~88 chars
-        
-        # Should not have padding
-        self.assertFalse(browser_key.endswith('='))
-        
-        # Should only contain URL-safe base64 characters
-        import re
-        self.assertRegex(browser_key, r'^[A-Za-z0-9_-]+$')
-
-
-class TestIntegrationFlow(unittest.TestCase):
-    """Test complete integration flow"""
-    
-    @patch('pywebpush.webpush')
-    def test_complete_notification_flow(self, mock_webpush):
-        """Test complete flow from subscription to notification"""
-        mock_webpush.return_value = True
-        
-        # Create mock datastore with VAPID keys
-        mock_datastore = Mock()
-        vapid = Vapid()
-        vapid.generate_keys()
-        
-        mock_datastore.data = {
-            'settings': {
-                'application': {
-                    'vapid': {
-                        'private_key': vapid.private_pem().decode(),
-                        'public_key': vapid.public_pem().decode(),
-                        'contact_email': 'test@example.com'
-                    },
-                    'browser_subscriptions': [
-                        {
-                            'endpoint': 'https://fcm.googleapis.com/fcm/send/test123',
-                            'keys': {
-                                'p256dh': 'test-p256dh-key',
-                                'auth': 'test-auth-key'
-                            }
-                        }
-                    ]
-                }
-            }
-        }
-        mock_datastore.needs_write = False
-        
-        # Get configuration
-        private_key, public_key, contact_email = get_vapid_config_from_datastore(mock_datastore)
-        subscriptions = get_browser_subscriptions(mock_datastore)
-        
-        # Create notification
-        payload = create_notification_payload("Test Title", "Test Message")
-        
-        # Send notification
-        success_count, total_count = send_push_notifications(
-            subscriptions=subscriptions,
-            notification_payload=payload,
-            private_key=private_key,
-            contact_email=contact_email,
-            datastore=mock_datastore
-        )
-        
-        # Verify success
-        self.assertEqual(success_count, 1)
-        self.assertEqual(total_count, 1)
-        self.assertTrue(mock_webpush.called)
-        
-        # Verify webpush call parameters
-        call_args = mock_webpush.call_args
-        self.assertIn('subscription_info', call_args[1])
-        self.assertIn('vapid_private_key', call_args[1])
-        self.assertIn('vapid_claims', call_args[1])
-        
-        # Verify vapid_claims format
-        vapid_claims = call_args[1]['vapid_claims']
-        self.assertEqual(vapid_claims['sub'], 'mailto:test@example.com')
-        self.assertEqual(vapid_claims['aud'], 'https://fcm.googleapis.com')
-
-
-if __name__ == '__main__':
-    unittest.main()

requirements.txt

@@ -39,7 +39,7 @@ jsonpath-ng~=1.5.3
 # jq not available on Windows so must be installed manually
 
 # Notification library
-apprise==1.9.3
+apprise==1.9.5
 
 # - Needed for apprise/spush, and maybe others? hopefully doesnt trigger a rust compile.
 # - Requires extra wheel for rPi, adds build time for arm/v8 which is not in piwheels
@@ -51,8 +51,8 @@ cryptography==44.0.1
 # use any version other than 2.0.x due to https://github.com/eclipse/paho.mqtt.python/issues/814
 paho-mqtt!=2.0.*
 
-# Used for CSS filtering
-beautifulsoup4>=4.0.0
+# Used for CSS filtering, JSON extraction from HTML
+beautifulsoup4>=4.0.0,<=4.13.5
 
 # XPath filtering, lxml is required by bs4 anyway, but put it here to be safe.
 # #2328 - 5.2.0 and 5.2.1 had extra CPU flag CFLAGS set which was not compatible on older hardware
@@ -135,13 +135,10 @@ tzdata
 pluggy ~= 1.5
 
 # Needed for testing, cross-platform for process and system monitoring
-psutil==7.0.0
+psutil==7.1.0
 
 ruff >= 0.11.2
 pre_commit >= 4.2.0
 
 # For events between checking and socketio updates
 blinker
-
-# For Web Push notifications (browser notifications)
-pywebpush