Re #2945 - Handle ByteOrderMark in JSON strings

.github/workflows/containers.yml

@@ -103,6 +103,19 @@ jobs:
 #          provenance: false
 
       # A new tagged release is required, which builds :tag and :latest
+      - name: Docker meta :tag
+        if: github.event_name == 'release' && startsWith(github.event.release.tag_name, '0.')
+        uses: docker/metadata-action@v5
+        id: meta
+        with:
+            images: |
+                ${{ secrets.DOCKER_HUB_USERNAME }}/changedetection.io
+                ghcr.io/dgtlmoon/changedetection.io
+            tags: |
+                type=semver,pattern={{version}}
+                type=semver,pattern={{major}}.{{minor}}
+                type=semver,pattern={{major}}
+
       - name: Build and push :tag
         id: docker_build_tag_release
         if: github.event_name == 'release' && startsWith(github.event.release.tag_name, '0.')
@@ -111,11 +124,7 @@ jobs:
           context: ./
           file: ./Dockerfile
           push: true
-          tags: |
-            ${{ secrets.DOCKER_HUB_USERNAME }}/changedetection.io:${{ github.event.release.tag_name }}
-            ghcr.io/dgtlmoon/changedetection.io:${{ github.event.release.tag_name }}
-            ${{ secrets.DOCKER_HUB_USERNAME }}/changedetection.io:latest
-            ghcr.io/dgtlmoon/changedetection.io:latest
+          tags: ${{ steps.meta.outputs.tags }}
           platforms: linux/amd64,linux/arm64,linux/arm/v7,linux/arm/v8,linux/arm64/v8
           cache-from: type=gha
           cache-to: type=gha,mode=max

.github/workflows/test-stack-reusable-workflow.yml

@@ -107,22 +107,22 @@ jobs:
         if: ${{ inputs.skip-pypuppeteer == false }}
         run: |
           # Playwright via Sockpuppetbrowser fetch 
-          docker run --rm -e "FLASK_SERVER_NAME=cdio" -e "FAST_PUPPETEER_CHROME_FETCHER=True" -e "PLAYWRIGHT_DRIVER_URL=ws://sockpuppetbrowser:3000" --network changedet-network --hostname=cdio test-changedetectionio  bash -c 'cd changedetectionio;pytest --live-server-host=0.0.0.0 --live-server-port=5004 --live-server-wait=20 tests/fetchers/test_content.py'
-          docker run --rm -e "FLASK_SERVER_NAME=cdio" -e "FAST_PUPPETEER_CHROME_FETCHER=True" -e "PLAYWRIGHT_DRIVER_URL=ws://sockpuppetbrowser:3000" --network changedet-network --hostname=cdio test-changedetectionio  bash -c 'cd changedetectionio;pytest --live-server-host=0.0.0.0 --live-server-port=5004 --live-server-wait=20 tests/test_errorhandling.py'
-          docker run --rm -e "FLASK_SERVER_NAME=cdio" -e "FAST_PUPPETEER_CHROME_FETCHER=True" -e "PLAYWRIGHT_DRIVER_URL=ws://sockpuppetbrowser:3000" --network changedet-network --hostname=cdio test-changedetectionio  bash -c 'cd changedetectionio;pytest --live-server-host=0.0.0.0 --live-server-port=5004 --live-server-wait=20 tests/visualselector/test_fetch_data.py'
-          docker run --rm -e "FLASK_SERVER_NAME=cdio" -e "FAST_PUPPETEER_CHROME_FETCHER=True" -e "PLAYWRIGHT_DRIVER_URL=ws://sockpuppetbrowser:3000" --network changedet-network --hostname=cdio test-changedetectionio  bash -c 'cd changedetectionio;pytest --live-server-host=0.0.0.0 --live-server-port=5004 --live-server-wait=20 tests/fetchers/test_custom_js_before_content.py'
+          docker run --rm -e "FLASK_SERVER_NAME=cdio" -e "FAST_PUPPETEER_CHROME_FETCHER=True" -e "PLAYWRIGHT_DRIVER_URL=ws://sockpuppetbrowser:3000" --network changedet-network --hostname=cdio test-changedetectionio  bash -c 'cd changedetectionio;pytest --live-server-host=0.0.0.0 --live-server-port=5004 tests/fetchers/test_content.py'
+          docker run --rm -e "FLASK_SERVER_NAME=cdio" -e "FAST_PUPPETEER_CHROME_FETCHER=True" -e "PLAYWRIGHT_DRIVER_URL=ws://sockpuppetbrowser:3000" --network changedet-network --hostname=cdio test-changedetectionio  bash -c 'cd changedetectionio;pytest --live-server-host=0.0.0.0 --live-server-port=5004 tests/test_errorhandling.py'
+          docker run --rm -e "FLASK_SERVER_NAME=cdio" -e "FAST_PUPPETEER_CHROME_FETCHER=True" -e "PLAYWRIGHT_DRIVER_URL=ws://sockpuppetbrowser:3000" --network changedet-network --hostname=cdio test-changedetectionio  bash -c 'cd changedetectionio;pytest --live-server-host=0.0.0.0 --live-server-port=5004 tests/visualselector/test_fetch_data.py'
+          docker run --rm -e "FLASK_SERVER_NAME=cdio" -e "FAST_PUPPETEER_CHROME_FETCHER=True" -e "PLAYWRIGHT_DRIVER_URL=ws://sockpuppetbrowser:3000" --network changedet-network --hostname=cdio test-changedetectionio  bash -c 'cd changedetectionio;pytest --live-server-host=0.0.0.0 --live-server-port=5004 tests/fetchers/test_custom_js_before_content.py'
 
       - name: Pyppeteer and SocketPuppetBrowser - Headers and requests checks
         if: ${{ inputs.skip-pypuppeteer == false }}
         run: |
           # Settings headers playwright tests - Call back in from Sockpuppetbrowser, check headers
-          docker run --name "changedet" --hostname changedet --rm  -e "FAST_PUPPETEER_CHROME_FETCHER=True" -e "FLASK_SERVER_NAME=changedet" -e "PLAYWRIGHT_DRIVER_URL=ws://sockpuppetbrowser:3000?dumpio=true" --network changedet-network test-changedetectionio  bash -c 'cd changedetectionio; pytest --live-server-host=0.0.0.0  --live-server-port=5004  --live-server-wait=20  tests/test_request.py'
+          docker run --name "changedet" --hostname changedet --rm  -e "FAST_PUPPETEER_CHROME_FETCHER=True" -e "FLASK_SERVER_NAME=changedet" -e "PLAYWRIGHT_DRIVER_URL=ws://sockpuppetbrowser:3000?dumpio=true" --network changedet-network test-changedetectionio  bash -c 'cd changedetectionio; pytest --live-server-host=0.0.0.0  --live-server-port=5004 tests/test_request.py'
 
       - name: Pyppeteer and SocketPuppetBrowser - Restock detection
         if: ${{ inputs.skip-pypuppeteer == false }}
         run: |                            
           # restock detection via playwright - added name=changedet here so that playwright and sockpuppetbrowser can connect to it
-          docker run --rm --name "changedet" -e "FLASK_SERVER_NAME=changedet"  -e "FAST_PUPPETEER_CHROME_FETCHER=True"  -e "PLAYWRIGHT_DRIVER_URL=ws://sockpuppetbrowser:3000" --network changedet-network test-changedetectionio  bash -c 'cd changedetectionio;pytest --live-server-port=5004 --live-server-host=0.0.0.0  --live-server-wait=20  tests/restock/test_restock.py'
+          docker run --rm --name "changedet" -e "FLASK_SERVER_NAME=changedet"  -e "FAST_PUPPETEER_CHROME_FETCHER=True"  -e "PLAYWRIGHT_DRIVER_URL=ws://sockpuppetbrowser:3000" --network changedet-network test-changedetectionio  bash -c 'cd changedetectionio;pytest --live-server-port=5004 --live-server-host=0.0.0.0 tests/restock/test_restock.py'
 
 # SELENIUM
       - name: Specific tests in built container for Selenium
@@ -132,7 +132,7 @@ jobs:
 
       - name: Specific tests in built container for headers and requests checks with Selenium
         run: |
-          docker run --name "changedet" --hostname changedet --rm -e "FLASK_SERVER_NAME=changedet" -e "WEBDRIVER_URL=http://selenium:4444/wd/hub" --network changedet-network test-changedetectionio  bash -c 'cd changedetectionio; pytest --live-server-host=0.0.0.0  --live-server-port=5004   --live-server-wait=20  tests/test_request.py'
+          docker run --name "changedet" --hostname changedet --rm -e "FLASK_SERVER_NAME=changedet" -e "WEBDRIVER_URL=http://selenium:4444/wd/hub" --network changedet-network test-changedetectionio  bash -c 'cd changedetectionio; pytest --live-server-host=0.0.0.0  --live-server-port=5004 tests/test_request.py'
 
 # OTHER STUFF
       - name: Test SMTP notification mime types

changedetectionio/diff.py

@@ -43,19 +43,19 @@ def customSequenceMatcher(
             yield before[alo:ahi]
         elif include_removed and tag == 'delete':
             if html_colour:
-                yield [f'<span class="cdio" style="{REMOVED_STYLE}">{line}</span>' for line in same_slicer(before, alo, ahi)]
+                yield [f'<span style="{REMOVED_STYLE}">{line}</span>' for line in same_slicer(before, alo, ahi)]
             else:
                 yield [f"(removed) {line}" for line in same_slicer(before, alo, ahi)] if include_change_type_prefix else same_slicer(before, alo, ahi)
         elif include_replaced and tag == 'replace':
             if html_colour:
-                yield [f'<span class="cdio" style="{REMOVED_STYLE}">{line}</span>' for line in same_slicer(before, alo, ahi)] + \
-                      [f'<span class="cdio" style="{ADDED_STYLE}">{line}</span>' for line in same_slicer(after, blo, bhi)]
+                yield [f'<span style="{REMOVED_STYLE}">{line}</span>' for line in same_slicer(before, alo, ahi)] + \
+                      [f'<span style="{ADDED_STYLE}">{line}</span>' for line in same_slicer(after, blo, bhi)]
             else:
                 yield [f"(changed) {line}" for line in same_slicer(before, alo, ahi)] + \
                       [f"(into) {line}" for line in same_slicer(after, blo, bhi)] if include_change_type_prefix else same_slicer(before, alo, ahi) + same_slicer(after, blo, bhi)
         elif include_added and tag == 'insert':
             if html_colour:
-                yield [f'<span class="cdio" style="{ADDED_STYLE}">{line}</span>' for line in same_slicer(after, blo, bhi)]
+                yield [f'<span style="{ADDED_STYLE}">{line}</span>' for line in same_slicer(after, blo, bhi)]
             else:
                 yield [f"(added) {line}" for line in same_slicer(after, blo, bhi)] if include_change_type_prefix else same_slicer(after, blo, bhi)
 

changedetectionio/flask_app.py

@@ -12,7 +12,6 @@ import threading
 import time
 import timeago
 
-from .html_tools import escape_mixed_content
 from .processors import find_processors, get_parent_module, get_custom_watch_obj_for_processor
 from .safe_jinja import render as jinja_render
 from changedetectionio.strtobool import strtobool
@@ -540,9 +539,6 @@ def changedetection_app(config=None, datastore_o=None):
         import apprise
         import random
         from .apprise_asset import asset
-        from .notification import default_notification_format
-        from .update_worker import build_notification_object_for_watch
-
         apobj = apprise.Apprise(asset=asset)
 
         # so that the custom endpoints are registered
@@ -599,8 +595,6 @@ def changedetection_app(config=None, datastore_o=None):
             # Only use if present, if not set in n_object it should use the default system value
             if 'notification_format' in request.form and request.form['notification_format'].strip():
                 n_object['notification_format'] = request.form.get('notification_format', '').strip()
-            else:
-                n_object['notification_format'] = default_notification_format
 
             if 'notification_title' in request.form and request.form['notification_title'].strip():
                 n_object['notification_title'] = request.form.get('notification_title', '').strip()
@@ -616,14 +610,9 @@ def changedetection_app(config=None, datastore_o=None):
             else:
                 n_object['notification_body'] = "Test body"
 
-            n_object = build_notification_object_for_watch(watch, n_object, datastore.data['settings']['application'].get('notification_body'))
-
-            if n_object['notification_format'].startswith('HTML'):
-                n_object['notification_body'] = escape_mixed_content(n_object['notification_body'])
-
-            from .notification import process_notification
             n_object['as_async'] = False
-            # Now we send the notification_body after everything is compiled
+            n_object.update(watch.extra_notification_token_values())
+            from .notification import process_notification
             sent_obj = process_notification(n_object, datastore)
 
         except Exception as e:

changedetectionio/html_tools.py

@@ -1,5 +1,6 @@
-from typing import List
+from loguru import logger
 from lxml import etree
+from typing import List
 import json
 import re
 
@@ -298,8 +299,10 @@ def extract_json_as_string(content, json_filter, ensure_is_ldjson_info_type=None
 # https://github.com/dgtlmoon/changedetection.io/pull/2041#issuecomment-1848397161w
     # Try to parse/filter out the JSON, if we get some parser error, then maybe it's embedded within HTML tags
     try:
-        stripped_text_from_html = _parse_json(json.loads(content), json_filter)
-    except json.JSONDecodeError:
+        # .lstrip("\ufeff") strings ByteOrderMark from UTF8 and still lets the UTF work
+        stripped_text_from_html = _parse_json(json.loads(content.lstrip("\ufeff") ), json_filter)
+    except json.JSONDecodeError as e:
+        logger.warning(str(e))
 
         # Foreach <script json></script> blob.. just return the first that matches json_filter
         # As a last resort, try to parse the whole <body>
@@ -500,40 +503,3 @@ def get_triggered_text(content, trigger_text):
         i += 1
 
     return triggered_text
-
-
-from bs4 import BeautifulSoup
-import html
-
-
-def escape_mixed_content(document):
-    import uuid
-    # Parse the document as HTML
-
-    # Generate a single random hash for placeholders
-    random_hash = f"__PLACEHOLDER_{uuid.uuid4().hex}__"
-    placeholder_map = []
-
-    # <br> to something else so we can preserve them
-    random_hash_br = f"__BR_{uuid.uuid4().hex}__"
-    document = document.replace('<br>', random_hash_br)
-
-    soup = BeautifulSoup(document, 'html.parser')
-
-    # Find all <span class="cdio"> and <br>/<br/>
-    for tag in soup.find_all("span", class_="cdio"):
-        placeholder_map.append(str(tag))  # Save the tag as a string
-        tag.replace_with(random_hash)  # Replace tag with the placeholder
-
-
-
-    # Escape the entire document
-    escaped_html = html.escape(str(soup))
-
-    # Restore all occurrences of placeholders with the original tags
-    for original_tag in placeholder_map:
-        escaped_html = escaped_html.replace(random_hash, original_tag, 1)  # Replace one occurrence at a time
-
-    escaped_html = escaped_html.replace( random_hash_br, "<br>")
-    return escaped_html
-

changedetectionio/notification.py

@@ -4,7 +4,6 @@ from apprise import NotifyFormat
 import apprise
 from loguru import logger
 
-from changedetectionio.html_tools import escape_mixed_content
 
 valid_tokens = {
     'base_url': '',
@@ -86,8 +85,6 @@ def process_notification(n_object, datastore):
                 n_body = n_body.replace("\n", '<br>')
 
             n_title = jinja_render(template_str=n_object.get('notification_title', ''), **notification_parameters)
-            if n_object['notification_format'].startswith('HTML'):
-                n_body = escape_mixed_content(n_body)
 
             url = url.strip()
             if url.startswith('#'):
@@ -164,6 +161,7 @@ def process_notification(n_object, datastore):
             attach=n_object.get('screenshot', None)
         )
 
+
         # Returns empty string if nothing found, multi-line string otherwise
         log_value = logs.getvalue()
 

changedetectionio/tests/test_jsonpath_jq_selector.py

@@ -514,3 +514,15 @@ def test_check_jq_ext_filter(client, live_server, measure_memory_usage):
 def test_check_jqraw_ext_filter(client, live_server, measure_memory_usage):
     if jq_support:
         check_json_ext_filter('jq:.[] | select(.status | contains("Sold"))', client, live_server)
+
+def test_jsonpath_BOM_utf8(client, live_server, measure_memory_usage):
+    from .. import html_tools
+
+    # JSON string with BOM and correct double-quoted keys
+    json_str = '\ufeff{"name": "José", "emoji": "😊", "language": "中文", "greeting": "Привет"}'
+
+    # See that we can find the second <script> one, which is not broken, and matches our filter
+    text = html_tools.extract_json_as_string(json_str, "json:$.name")
+    assert text == '"José"'
+
+    

changedetectionio/tests/test_notification.py

@@ -454,18 +454,8 @@ def test_global_send_test_notification(client, live_server, measure_memory_usage
     assert b"Error: You must have atleast one watch configured for 'test notification' to work" in res.data
 
 
-    client.get(
-        url_for("form_delete", uuid="all"),
-        follow_redirects=True
-    )
-
 def _test_color_notifications(client, notification_body_token):
 
-    client.get(
-        url_for("form_delete", uuid="all"),
-        follow_redirects=True
-    )
-
     from changedetectionio.diff import ADDED_STYLE, REMOVED_STYLE
 
     set_original_response()
@@ -504,9 +494,9 @@ def _test_color_notifications(client, notification_body_token):
     wait_for_all_checks(client)
 
     set_modified_response()
-    
-    res = client.get(url_for("form_watch_checknow"), follow_redirects=True)
 
+
+    res = client.get(url_for("form_watch_checknow"), follow_redirects=True)
     assert b'1 watches queued for rechecking.' in res.data
 
     wait_for_all_checks(client)
@@ -514,8 +504,7 @@ def _test_color_notifications(client, notification_body_token):
 
     with open("test-datastore/notification.txt", 'r') as f:
         x = f.read()
-        assert f'<span class="cdio" style="{REMOVED_STYLE}">Which is across multiple lines' in x
-        assert f'<br>' in x
+        assert f'<span style="{REMOVED_STYLE}">Which is across multiple lines' in x
 
 
     client.get(

changedetectionio/update_worker.py

@@ -16,77 +16,6 @@ import time
 
 from loguru import logger
 
-def build_notification_object_for_watch(watch, n_object, default_app_settings_notification_format):
-    from changedetectionio import diff
-    from changedetectionio.notification import default_notification_format_for_watch
-
-    dates = []
-    trigger_text = ''
-
-    if watch:
-        watch_history = watch.history
-        dates = list(watch_history.keys())
-        trigger_text = watch.get('trigger_text', [])
-
-    # Add text that was triggered
-    if len(dates):
-        snapshot_contents = watch.get_history_snapshot(dates[-1])
-    else:
-        snapshot_contents = "No snapshot/history available, the watch should fetch atleast once."
-
-    # If we ended up here with "System default"
-    if n_object.get('notification_format') == default_notification_format_for_watch:
-        n_object['notification_format'] = default_app_settings_notification_format
-
-    html_colour_enable = False
-    # HTML needs linebreak, but MarkDown and Text can use a linefeed
-    if n_object.get('notification_format') == 'HTML':
-        line_feed_sep = "<br>"
-        # Snapshot will be plaintext on the disk, convert to some kind of HTML
-        snapshot_contents = snapshot_contents.replace('\n', line_feed_sep)
-    elif n_object.get('notification_format') == 'HTML Color':
-        line_feed_sep = "<br>"
-        # Snapshot will be plaintext on the disk, convert to some kind of HTML
-        snapshot_contents = snapshot_contents.replace('\n', line_feed_sep)
-        html_colour_enable = True
-    else:
-        line_feed_sep = "\n"
-
-    triggered_text = ''
-    if len(trigger_text):
-        from . import html_tools
-        triggered_text = html_tools.get_triggered_text(content=snapshot_contents, trigger_text=trigger_text)
-        if triggered_text:
-            triggered_text = line_feed_sep.join(triggered_text)
-
-    # Could be called as a 'test notification' with only 1 snapshot available
-    prev_snapshot = "Example text: example test\nExample text: change detection is cool\nExample text: some more examples\n"
-    current_snapshot = "Example text: example test\nExample text: change detection is fantastic\nExample text: even more examples\nExample text: a lot more examples"
-
-    if len(dates) > 1:
-        prev_snapshot = watch.get_history_snapshot(dates[-2])
-        current_snapshot = watch.get_history_snapshot(dates[-1])
-
-    n_object.update({
-        'current_snapshot': snapshot_contents,
-        'diff': diff.render_diff(prev_snapshot, current_snapshot, line_feed_sep=line_feed_sep, html_colour=html_colour_enable),
-        'diff_added': diff.render_diff(prev_snapshot, current_snapshot, include_removed=False, line_feed_sep=line_feed_sep),
-        'diff_full': diff.render_diff(prev_snapshot, current_snapshot, include_equal=True, line_feed_sep=line_feed_sep,
-                                      html_colour=html_colour_enable),
-        'diff_patch': diff.render_diff(prev_snapshot, current_snapshot, line_feed_sep=line_feed_sep, patch_format=True),
-        'diff_removed': diff.render_diff(prev_snapshot, current_snapshot, include_added=False, line_feed_sep=line_feed_sep),
-        'notification_timestamp': time.time(),
-        'screenshot': watch.get_screenshot() if watch and watch.get('notification_screenshot') else None,
-        'triggered_text': triggered_text,
-        'uuid': watch.get('uuid') if watch else None,
-        'watch_url': watch.get('url') if watch else None,
-    })
-
-    if watch:
-        n_object.update(watch.extra_notification_token_values())
-
-    return n_object
-
 class update_worker(threading.Thread):
     current_uuid = None
 
@@ -98,8 +27,75 @@ class update_worker(threading.Thread):
         super().__init__(*args, **kwargs)
 
     def queue_notification_for_watch(self, notification_q, n_object, watch):
+        from changedetectionio import diff
+        from changedetectionio.notification import default_notification_format_for_watch
+
+        dates = []
+        trigger_text = ''
+
         now = time.time()
-        n_object = build_notification_object_for_watch(watch, n_object, self.datastore.data['settings']['application'].get('notification_format'))
+
+        if watch:
+            watch_history = watch.history
+            dates = list(watch_history.keys())
+            trigger_text = watch.get('trigger_text', [])
+
+        # Add text that was triggered
+        if len(dates):
+            snapshot_contents = watch.get_history_snapshot(dates[-1])
+        else:
+            snapshot_contents = "No snapshot/history available, the watch should fetch atleast once."
+
+        # If we ended up here with "System default"
+        if n_object.get('notification_format') == default_notification_format_for_watch:
+            n_object['notification_format'] = self.datastore.data['settings']['application'].get('notification_format')
+
+        html_colour_enable = False
+        # HTML needs linebreak, but MarkDown and Text can use a linefeed
+        if n_object.get('notification_format') == 'HTML':
+            line_feed_sep = "<br>"
+            # Snapshot will be plaintext on the disk, convert to some kind of HTML
+            snapshot_contents = snapshot_contents.replace('\n', line_feed_sep)
+        elif n_object.get('notification_format') == 'HTML Color':
+            line_feed_sep = "<br>"
+            # Snapshot will be plaintext on the disk, convert to some kind of HTML
+            snapshot_contents = snapshot_contents.replace('\n', line_feed_sep)
+            html_colour_enable = True
+        else:
+            line_feed_sep = "\n"
+
+        triggered_text = ''
+        if len(trigger_text):
+            from . import html_tools
+            triggered_text = html_tools.get_triggered_text(content=snapshot_contents, trigger_text=trigger_text)
+            if triggered_text:
+                triggered_text = line_feed_sep.join(triggered_text)
+
+        # Could be called as a 'test notification' with only 1 snapshot available
+        prev_snapshot = "Example text: example test\nExample text: change detection is cool\nExample text: some more examples\n"
+        current_snapshot = "Example text: example test\nExample text: change detection is fantastic\nExample text: even more examples\nExample text: a lot more examples"
+
+        if len(dates) > 1:
+            prev_snapshot = watch.get_history_snapshot(dates[-2])
+            current_snapshot = watch.get_history_snapshot(dates[-1])
+
+        n_object.update({
+            'current_snapshot': snapshot_contents,
+            'diff': diff.render_diff(prev_snapshot, current_snapshot, line_feed_sep=line_feed_sep, html_colour=html_colour_enable),
+            'diff_added': diff.render_diff(prev_snapshot, current_snapshot, include_removed=False, line_feed_sep=line_feed_sep),
+            'diff_full': diff.render_diff(prev_snapshot, current_snapshot, include_equal=True, line_feed_sep=line_feed_sep, html_colour=html_colour_enable),
+            'diff_patch': diff.render_diff(prev_snapshot, current_snapshot, line_feed_sep=line_feed_sep, patch_format=True),
+            'diff_removed': diff.render_diff(prev_snapshot, current_snapshot, include_added=False, line_feed_sep=line_feed_sep),
+            'notification_timestamp': now,
+            'screenshot': watch.get_screenshot() if watch and watch.get('notification_screenshot') else None,
+            'triggered_text': triggered_text,
+            'uuid': watch.get('uuid') if watch else None,
+            'watch_url': watch.get('url') if watch else None,
+        })
+
+        if watch:
+            n_object.update(watch.extra_notification_token_values())
+
         logger.trace(f"Main rendered notification placeholders (diff_added etc) calculated in {time.time()-now:.3f}s")
         logger.debug("Queued notification for sending")
         notification_q.put(n_object)