Fix unset default var

.github/workflows/containers.yml

@@ -103,19 +103,6 @@ jobs:
 #          provenance: false
 
       # A new tagged release is required, which builds :tag and :latest
-      - name: Docker meta :tag
-        if: github.event_name == 'release' && startsWith(github.event.release.tag_name, '0.')
-        uses: docker/metadata-action@v5
-        id: meta
-        with:
-            images: |
-                ${{ secrets.DOCKER_HUB_USERNAME }}/changedetection.io
-                ghcr.io/dgtlmoon/changedetection.io
-            tags: |
-                type=semver,pattern={{version}}
-                type=semver,pattern={{major}}.{{minor}}
-                type=semver,pattern={{major}}
-
       - name: Build and push :tag
         id: docker_build_tag_release
         if: github.event_name == 'release' && startsWith(github.event.release.tag_name, '0.')
@@ -124,7 +111,11 @@ jobs:
           context: ./
           file: ./Dockerfile
           push: true
-          tags: ${{ steps.meta.outputs.tags }}
+          tags: |
+            ${{ secrets.DOCKER_HUB_USERNAME }}/changedetection.io:${{ github.event.release.tag_name }}
+            ghcr.io/dgtlmoon/changedetection.io:${{ github.event.release.tag_name }}
+            ${{ secrets.DOCKER_HUB_USERNAME }}/changedetection.io:latest
+            ghcr.io/dgtlmoon/changedetection.io:latest
           platforms: linux/amd64,linux/arm64,linux/arm/v7,linux/arm/v8,linux/arm64/v8
           cache-from: type=gha
           cache-to: type=gha,mode=max

.github/workflows/test-stack-reusable-workflow.yml

@@ -107,22 +107,22 @@ jobs:
         if: ${{ inputs.skip-pypuppeteer == false }}
         run: |
           # Playwright via Sockpuppetbrowser fetch 
-          docker run --rm -e "FLASK_SERVER_NAME=cdio" -e "FAST_PUPPETEER_CHROME_FETCHER=True" -e "PLAYWRIGHT_DRIVER_URL=ws://sockpuppetbrowser:3000" --network changedet-network --hostname=cdio test-changedetectionio  bash -c 'cd changedetectionio;pytest --live-server-host=0.0.0.0 --live-server-port=5004 tests/fetchers/test_content.py'
-          docker run --rm -e "FLASK_SERVER_NAME=cdio" -e "FAST_PUPPETEER_CHROME_FETCHER=True" -e "PLAYWRIGHT_DRIVER_URL=ws://sockpuppetbrowser:3000" --network changedet-network --hostname=cdio test-changedetectionio  bash -c 'cd changedetectionio;pytest --live-server-host=0.0.0.0 --live-server-port=5004 tests/test_errorhandling.py'
-          docker run --rm -e "FLASK_SERVER_NAME=cdio" -e "FAST_PUPPETEER_CHROME_FETCHER=True" -e "PLAYWRIGHT_DRIVER_URL=ws://sockpuppetbrowser:3000" --network changedet-network --hostname=cdio test-changedetectionio  bash -c 'cd changedetectionio;pytest --live-server-host=0.0.0.0 --live-server-port=5004 tests/visualselector/test_fetch_data.py'
-          docker run --rm -e "FLASK_SERVER_NAME=cdio" -e "FAST_PUPPETEER_CHROME_FETCHER=True" -e "PLAYWRIGHT_DRIVER_URL=ws://sockpuppetbrowser:3000" --network changedet-network --hostname=cdio test-changedetectionio  bash -c 'cd changedetectionio;pytest --live-server-host=0.0.0.0 --live-server-port=5004 tests/fetchers/test_custom_js_before_content.py'
+          docker run --rm -e "FLASK_SERVER_NAME=cdio" -e "FAST_PUPPETEER_CHROME_FETCHER=True" -e "PLAYWRIGHT_DRIVER_URL=ws://sockpuppetbrowser:3000" --network changedet-network --hostname=cdio test-changedetectionio  bash -c 'cd changedetectionio;pytest --live-server-host=0.0.0.0 --live-server-port=5004 --live-server-wait=20 tests/fetchers/test_content.py'
+          docker run --rm -e "FLASK_SERVER_NAME=cdio" -e "FAST_PUPPETEER_CHROME_FETCHER=True" -e "PLAYWRIGHT_DRIVER_URL=ws://sockpuppetbrowser:3000" --network changedet-network --hostname=cdio test-changedetectionio  bash -c 'cd changedetectionio;pytest --live-server-host=0.0.0.0 --live-server-port=5004 --live-server-wait=20 tests/test_errorhandling.py'
+          docker run --rm -e "FLASK_SERVER_NAME=cdio" -e "FAST_PUPPETEER_CHROME_FETCHER=True" -e "PLAYWRIGHT_DRIVER_URL=ws://sockpuppetbrowser:3000" --network changedet-network --hostname=cdio test-changedetectionio  bash -c 'cd changedetectionio;pytest --live-server-host=0.0.0.0 --live-server-port=5004 --live-server-wait=20 tests/visualselector/test_fetch_data.py'
+          docker run --rm -e "FLASK_SERVER_NAME=cdio" -e "FAST_PUPPETEER_CHROME_FETCHER=True" -e "PLAYWRIGHT_DRIVER_URL=ws://sockpuppetbrowser:3000" --network changedet-network --hostname=cdio test-changedetectionio  bash -c 'cd changedetectionio;pytest --live-server-host=0.0.0.0 --live-server-port=5004 --live-server-wait=20 tests/fetchers/test_custom_js_before_content.py'
 
       - name: Pyppeteer and SocketPuppetBrowser - Headers and requests checks
         if: ${{ inputs.skip-pypuppeteer == false }}
         run: |
           # Settings headers playwright tests - Call back in from Sockpuppetbrowser, check headers
-          docker run --name "changedet" --hostname changedet --rm  -e "FAST_PUPPETEER_CHROME_FETCHER=True" -e "FLASK_SERVER_NAME=changedet" -e "PLAYWRIGHT_DRIVER_URL=ws://sockpuppetbrowser:3000?dumpio=true" --network changedet-network test-changedetectionio  bash -c 'cd changedetectionio; pytest --live-server-host=0.0.0.0  --live-server-port=5004 tests/test_request.py'
+          docker run --name "changedet" --hostname changedet --rm  -e "FAST_PUPPETEER_CHROME_FETCHER=True" -e "FLASK_SERVER_NAME=changedet" -e "PLAYWRIGHT_DRIVER_URL=ws://sockpuppetbrowser:3000?dumpio=true" --network changedet-network test-changedetectionio  bash -c 'cd changedetectionio; pytest --live-server-host=0.0.0.0  --live-server-port=5004  --live-server-wait=20  tests/test_request.py'
 
       - name: Pyppeteer and SocketPuppetBrowser - Restock detection
         if: ${{ inputs.skip-pypuppeteer == false }}
         run: |                            
           # restock detection via playwright - added name=changedet here so that playwright and sockpuppetbrowser can connect to it
-          docker run --rm --name "changedet" -e "FLASK_SERVER_NAME=changedet"  -e "FAST_PUPPETEER_CHROME_FETCHER=True"  -e "PLAYWRIGHT_DRIVER_URL=ws://sockpuppetbrowser:3000" --network changedet-network test-changedetectionio  bash -c 'cd changedetectionio;pytest --live-server-port=5004 --live-server-host=0.0.0.0 tests/restock/test_restock.py'
+          docker run --rm --name "changedet" -e "FLASK_SERVER_NAME=changedet"  -e "FAST_PUPPETEER_CHROME_FETCHER=True"  -e "PLAYWRIGHT_DRIVER_URL=ws://sockpuppetbrowser:3000" --network changedet-network test-changedetectionio  bash -c 'cd changedetectionio;pytest --live-server-port=5004 --live-server-host=0.0.0.0  --live-server-wait=20  tests/restock/test_restock.py'
 
 # SELENIUM
       - name: Specific tests in built container for Selenium
@@ -132,7 +132,7 @@ jobs:
 
       - name: Specific tests in built container for headers and requests checks with Selenium
         run: |
-          docker run --name "changedet" --hostname changedet --rm -e "FLASK_SERVER_NAME=changedet" -e "WEBDRIVER_URL=http://selenium:4444/wd/hub" --network changedet-network test-changedetectionio  bash -c 'cd changedetectionio; pytest --live-server-host=0.0.0.0  --live-server-port=5004 tests/test_request.py'
+          docker run --name "changedet" --hostname changedet --rm -e "FLASK_SERVER_NAME=changedet" -e "WEBDRIVER_URL=http://selenium:4444/wd/hub" --network changedet-network test-changedetectionio  bash -c 'cd changedetectionio; pytest --live-server-host=0.0.0.0  --live-server-port=5004   --live-server-wait=20  tests/test_request.py'
 
 # OTHER STUFF
       - name: Test SMTP notification mime types

changedetectionio/__init__.py

@@ -2,7 +2,7 @@
 
 # Read more https://github.com/dgtlmoon/changedetection.io/wiki
 
-__version__ = '0.49.1'
+__version__ = '0.49.0'
 
 from changedetectionio.strtobool import strtobool
 from json.decoder import JSONDecodeError

changedetectionio/conditions/__init__.py

@@ -1,113 +0,0 @@
-from json_logic import jsonLogic
-from json_logic.builtins import BUILTINS
-from .pluggy_interface import plugin_manager  # Import the pluggy plugin manager
-from . import default_plugin
-
-import re
-
-# List of all supported JSON Logic operators
-operator_choices = [
-    (">", "Greater Than"),
-    ("<", "Less Than"),
-    (">=", "Greater Than or Equal To"),
-    ("<=", "Less Than or Equal To"),
-    ("==", "Equals"),
-    ("!=", "Not Equals"),
-    ("in", "Contains"),
-    ("!in", "Does Not Contain"),
-    ("contains_regex", "Text Matches Regex"),
-    ("!contains_regex", "Text Does NOT Match Regex"),
-    ("changed > minutes", "Changed more than X minutes ago"),
-#    ("watch_uuid_unviewed_change", "Watch UUID had an unviewed change"),  #('if'? )
-#    ("watch_uuid_not_unviewed_change", "Watch UUID NOT had an unviewed change") #('if'? )
-#    ("watch_uuid_changed", "Watch UUID had unviewed change"),
-#    ("watch_uuid_not_changed", "Watch UUID did NOT have unviewed change"),
-#    ("!!", "Is Truthy"),
-#    ("!", "Is Falsy"),
-#    ("and", "All Conditions Must Be True"),
-#    ("or", "At Least One Condition Must Be True"),
-#    ("max", "Maximum of Values"),
-#    ("min", "Minimum of Values"),
-#    ("+", "Addition"),
-#    ("-", "Subtraction"),
-#    ("*", "Multiplication"),
-#    ("/", "Division"),
-#    ("%", "Modulo"),
-#    ("log", "Logarithm"),
-#    ("if", "Conditional If-Else")
-]
-
-# Fields available in the rules
-field_choices = [
-    ("extracted_number", "Extracted Number"),
-    ("page_filtered_text", "Page text After Filters"),
-    ("page_title", "Page Title"), # actual page title <title>
-    ("watch_uuid", "Watch UUID"),
-    #("watch_history_length", "History Length"),  # Would never equate
-    ("watch_history", "All Watch Text History"),
-    ("watch_check_count", "Watch Check Count"),
-    ("watch_uuid", "Other Watch by UUID"), # (Maybe this is 'if' ??)
-    #("requests_get", "Web GET requests (https://..)")
-]
-
-
-# ✅ Custom function for case-insensitive regex matching
-def contains_regex(_, text, pattern):
-    """Returns True if `text` contains `pattern` (case-insensitive regex match)."""
-    return bool(re.search(pattern, text, re.IGNORECASE))
-
-# ✅ Custom function for NOT matching case-insensitive regex
-def not_contains_regex(_, text, pattern):
-    """Returns True if `text` does NOT contain `pattern` (case-insensitive regex match)."""
-    return not bool(re.search(pattern, text, re.IGNORECASE))
-
-
-# ✅ Custom function to check if "watch_uuid" has changed
-def watch_uuid_changed(_, previous_uuid, current_uuid):
-    """Returns True if the watch UUID has changed."""
-    return previous_uuid != current_uuid
-
-# ✅ Custom function to check if "watch_uuid" has NOT changed
-def watch_uuid_not_changed(_, previous_uuid, current_uuid):
-    """Returns True if the watch UUID has NOT changed."""
-    return previous_uuid == current_uuid
-
-# Define the extended operations dictionary
-CUSTOM_OPERATIONS = {
-    **BUILTINS,  # Include all standard operators
-    "watch_uuid_changed": watch_uuid_changed,
-    "watch_uuid_not_changed": watch_uuid_not_changed,
-    "contains_regex": contains_regex,
-    "!contains_regex": not_contains_regex
-}
-
-# ✅ Load plugins dynamically
-for plugin in plugin_manager.get_plugins():
-    new_ops = plugin.register_operators()
-    if isinstance(new_ops, dict):
-        CUSTOM_OPERATIONS.update(new_ops)
-
-    new_operator_choices = plugin.register_operator_choices()
-    if isinstance(new_operator_choices, list):
-        operator_choices.extend(new_operator_choices)
-
-    new_field_choices = plugin.register_field_choices()
-    if isinstance(new_field_choices, list):
-        field_choices.extend(new_field_choices)
-
-def run(ruleset, data):
-    """
-    Execute a JSON Logic rule against given data.
-
-    :param ruleset: JSON Logic rule dictionary.
-    :param data: Dictionary containing the facts.
-    :return: Boolean result of rule evaluation.
-    """
-
-
-    try:
-        return jsonLogic(ruleset, data, CUSTOM_OPERATIONS)
-    except Exception as e:
-        # raise some custom nice handler
-        print(f"❌ Error evaluating JSON Logic: {e}")
-        return False

changedetectionio/conditions/default_plugin.py

@@ -1,30 +0,0 @@
-import pluggy
-
-hookimpl = pluggy.HookimplMarker("conditions")
-
-@hookimpl
-def register_operators():
-    def starts_with(_, text, prefix):
-        return text.lower().startswith(prefix.lower())
-
-    def ends_with(_, text, suffix):
-        return text.lower().endswith(suffix.lower())
-
-    return {
-        "starts_with": starts_with,
-        "ends_with": ends_with
-    }
-
-@hookimpl
-def register_operator_choices():
-    return [
-        ("starts_with", "Text Starts With"),
-        ("ends_with", "Text Ends With"),
-    ]
-
-@hookimpl
-def register_field_choices():
-    return [
-        ("meta_description", "Meta Description"),
-        ("meta_keywords", "Meta Keywords"),
-    ]

changedetectionio/conditions/pluggy_interface.py

@@ -1,32 +0,0 @@
-import pluggy
-
-# Define `pluggy` hookspecs (Specifications for Plugins)
-hookspec = pluggy.HookspecMarker("conditions")
-hookimpl = pluggy.HookimplMarker("conditions")
-
-
-class ConditionsSpec:
-    """Hook specifications for extending JSON Logic conditions."""
-
-    @hookspec
-    def register_operators():
-        """Return a dictionary of new JSON Logic operators."""
-        pass
-
-    @hookspec
-    def register_operator_choices():
-        """Return a list of new operator choices."""
-        pass
-
-    @hookspec
-    def register_field_choices():
-        """Return a list of new field choices."""
-        pass
-
-
-# ✅ Set up `pluggy` Plugin Manager
-plugin_manager = pluggy.PluginManager("conditions")
-plugin_manager.add_hookspecs(ConditionsSpec)
-
-# Discover installed plugins
-plugin_manager.load_setuptools_entrypoints("conditions")

changedetectionio/content_fetchers/res/stock-not-in-stock.js

@@ -52,7 +52,6 @@ function isItemInStock() {
         'niet leverbaar',
         'niet op voorraad',
         'no disponible',
-        'non disponibile',
         'no longer in stock',
         'no tickets available',
         'not available',

changedetectionio/diff.py

@@ -43,19 +43,19 @@ def customSequenceMatcher(
             yield before[alo:ahi]
         elif include_removed and tag == 'delete':
             if html_colour:
-                yield [f'<span style="{REMOVED_STYLE}">{line}</span>' for line in same_slicer(before, alo, ahi)]
+                yield [f'<span class="cdio" style="{REMOVED_STYLE}">{line}</span>' for line in same_slicer(before, alo, ahi)]
             else:
                 yield [f"(removed) {line}" for line in same_slicer(before, alo, ahi)] if include_change_type_prefix else same_slicer(before, alo, ahi)
         elif include_replaced and tag == 'replace':
             if html_colour:
-                yield [f'<span style="{REMOVED_STYLE}">{line}</span>' for line in same_slicer(before, alo, ahi)] + \
-                      [f'<span style="{ADDED_STYLE}">{line}</span>' for line in same_slicer(after, blo, bhi)]
+                yield [f'<span class="cdio" style="{REMOVED_STYLE}">{line}</span>' for line in same_slicer(before, alo, ahi)] + \
+                      [f'<span class="cdio" style="{ADDED_STYLE}">{line}</span>' for line in same_slicer(after, blo, bhi)]
             else:
                 yield [f"(changed) {line}" for line in same_slicer(before, alo, ahi)] + \
                       [f"(into) {line}" for line in same_slicer(after, blo, bhi)] if include_change_type_prefix else same_slicer(before, alo, ahi) + same_slicer(after, blo, bhi)
         elif include_added and tag == 'insert':
             if html_colour:
-                yield [f'<span style="{ADDED_STYLE}">{line}</span>' for line in same_slicer(after, blo, bhi)]
+                yield [f'<span class="cdio" style="{ADDED_STYLE}">{line}</span>' for line in same_slicer(after, blo, bhi)]
             else:
                 yield [f"(added) {line}" for line in same_slicer(after, blo, bhi)] if include_change_type_prefix else same_slicer(after, blo, bhi)
 

changedetectionio/flask_app.py

@@ -12,6 +12,7 @@ import threading
 import time
 import timeago
 
+from .html_tools import escape_mixed_content
 from .processors import find_processors, get_parent_module, get_custom_watch_obj_for_processor
 from .safe_jinja import render as jinja_render
 from changedetectionio.strtobool import strtobool
@@ -539,6 +540,9 @@ def changedetection_app(config=None, datastore_o=None):
         import apprise
         import random
         from .apprise_asset import asset
+        from .notification import default_notification_format
+        from .update_worker import build_notification_object_for_watch
+
         apobj = apprise.Apprise(asset=asset)
 
         # so that the custom endpoints are registered
@@ -595,6 +599,8 @@ def changedetection_app(config=None, datastore_o=None):
             # Only use if present, if not set in n_object it should use the default system value
             if 'notification_format' in request.form and request.form['notification_format'].strip():
                 n_object['notification_format'] = request.form.get('notification_format', '').strip()
+            else:
+                n_object['notification_format'] = default_notification_format
 
             if 'notification_title' in request.form and request.form['notification_title'].strip():
                 n_object['notification_title'] = request.form.get('notification_title', '').strip()
@@ -610,9 +616,14 @@ def changedetection_app(config=None, datastore_o=None):
             else:
                 n_object['notification_body'] = "Test body"
 
-            n_object['as_async'] = False
-            n_object.update(watch.extra_notification_token_values())
+            n_object = build_notification_object_for_watch(watch, n_object, datastore.data['settings']['application'].get('notification_body'))
+
+            if n_object['notification_format'].startswith('HTML'):
+                n_object['notification_body'] = escape_mixed_content(n_object['notification_body'])
+
             from .notification import process_notification
+            n_object['as_async'] = False
+            # Now we send the notification_body after everything is compiled
             sent_obj = process_notification(n_object, datastore)
 
         except Exception as e:
@@ -758,25 +769,6 @@ def changedetection_app(config=None, datastore_o=None):
             for p in datastore.proxy_list:
                 form.proxy.choices.append(tuple((p, datastore.proxy_list[p]['label'])))
 
-        # Example JSON Rule
-        DEFAULT_RULE = {
-            "and": [
-                {">": [{"var": "extracted_number"}, 5000]},
-                {"<": [{"var": "extracted_number"}, 80000]},
-                {"in": ["rock", {"var": "page_text"}]}
-            ]
-        }
-        form.conditions.pop_entry()  # Remove the default empty row
-        for condition in DEFAULT_RULE["and"]:
-            operator, values = list(condition.items())[0]
-            field = values[0]["var"] if isinstance(values[0], dict) else values[1]["var"]
-            value = values[1] if isinstance(values[1], (str, int)) else values[0]
-
-            form.conditions.append_entry({
-                "operator": operator,
-                "field": field,
-                "value": value
-            })
 
         if request.method == 'POST' and form.validate():
 
@@ -812,14 +804,6 @@ def changedetection_app(config=None, datastore_o=None):
                 extra_update_obj['filter_text_replaced'] = True
                 extra_update_obj['filter_text_removed'] = True
 
-            # Convert form input into JSON Logic format
-            extra_update_obj["conditions"] = {
-                "and": [
-                    {c.operator.data: [{"var": c.field.data}, c.value.data]}
-                    for c in getattr(form, "conditions", []) or []
-                ]
-            } if getattr(form, "conditions", None) else {}
-
             # Because wtforms doesn't support accessing other data in process_ , but we convert the CSV list of tags back to a list of UUIDs
             tag_uuids = []
             if form.data.get('tags'):

changedetectionio/forms.py

@@ -4,7 +4,6 @@ from loguru import logger
 from wtforms.widgets.core import TimeInput
 
 from changedetectionio.strtobool import strtobool
-from flask_wtf import FlaskForm
 
 from wtforms import (
     BooleanField,
@@ -307,6 +306,7 @@ class ValidateAppRiseServers(object):
         import apprise
         apobj = apprise.Apprise()
         # so that the custom endpoints are registered
+        from changedetectionio.apprise_plugin import apprise_custom_api_call_wrapper
         for server_url in field.data:
             url = server_url.strip()
             if url.startswith("#"):
@@ -509,23 +509,6 @@ class quickWatchForm(Form):
     edit_and_watch_submit_button = SubmitField('Edit > Watch', render_kw={"class": "pure-button pure-button-primary"})
 
 
-
-# Condition Rule Form (for each rule row)
-class ConditionForm(FlaskForm):
-    from changedetectionio.conditions import operator_choices, field_choices
-
-    operator = SelectField(
-        "Operator",
-        choices=operator_choices,
-        validators=[validators.Optional()]
-    )
-    field = SelectField(
-        "Field",
-        choices=field_choices,
-        validators=[validators.Optional()]
-    )
-    value = StringField("Value", validators=[validators.Optional()])
-
 # Common to a single watch and the global settings
 class commonSettingsForm(Form):
     from . import processors
@@ -613,9 +596,6 @@ class processor_text_json_diff_form(commonSettingsForm):
     notification_muted = BooleanField('Notifications Muted / Off', default=False)
     notification_screenshot = BooleanField('Attach screenshot to notification (where possible)', default=False)
 
-    conditions = FieldList(FormField(ConditionForm), min_entries=1)  # Add rule logic here
-
-
     def extra_tab_content(self):
         return None
 

changedetectionio/html_tools.py

@@ -1,6 +1,5 @@
-from loguru import logger
-from lxml import etree
 from typing import List
+from lxml import etree
 import json
 import re
 
@@ -299,10 +298,8 @@ def extract_json_as_string(content, json_filter, ensure_is_ldjson_info_type=None
 # https://github.com/dgtlmoon/changedetection.io/pull/2041#issuecomment-1848397161w
     # Try to parse/filter out the JSON, if we get some parser error, then maybe it's embedded within HTML tags
     try:
-        # .lstrip("\ufeff") strings ByteOrderMark from UTF8 and still lets the UTF work
-        stripped_text_from_html = _parse_json(json.loads(content.lstrip("\ufeff") ), json_filter)
-    except json.JSONDecodeError as e:
-        logger.warning(str(e))
+        stripped_text_from_html = _parse_json(json.loads(content), json_filter)
+    except json.JSONDecodeError:
 
         # Foreach <script json></script> blob.. just return the first that matches json_filter
         # As a last resort, try to parse the whole <body>
@@ -503,3 +500,40 @@ def get_triggered_text(content, trigger_text):
         i += 1
 
     return triggered_text
+
+
+from bs4 import BeautifulSoup
+import html
+
+
+def escape_mixed_content(document):
+    import uuid
+    # Parse the document as HTML
+
+    # Generate a single random hash for placeholders
+    random_hash = f"__PLACEHOLDER_{uuid.uuid4().hex}__"
+    placeholder_map = []
+
+    # <br> to something else so we can preserve them
+    random_hash_br = f"__BR_{uuid.uuid4().hex}__"
+    document = document.replace('<br>', random_hash_br)
+
+    soup = BeautifulSoup(document, 'html.parser')
+
+    # Find all <span class="cdio"> and <br>/<br/>
+    for tag in soup.find_all("span", class_="cdio"):
+        placeholder_map.append(str(tag))  # Save the tag as a string
+        tag.replace_with(random_hash)  # Replace tag with the placeholder
+
+
+
+    # Escape the entire document
+    escaped_html = html.escape(str(soup))
+
+    # Restore all occurrences of placeholders with the original tags
+    for original_tag in placeholder_map:
+        escaped_html = escaped_html.replace(random_hash, original_tag, 1)  # Replace one occurrence at a time
+
+    escaped_html = escaped_html.replace( random_hash_br, "<br>")
+    return escaped_html
+

changedetectionio/notification.py

@@ -4,6 +4,7 @@ from apprise import NotifyFormat
 import apprise
 from loguru import logger
 
+from changedetectionio.html_tools import escape_mixed_content
 
 valid_tokens = {
     'base_url': '',
@@ -85,6 +86,8 @@ def process_notification(n_object, datastore):
                 n_body = n_body.replace("\n", '<br>')
 
             n_title = jinja_render(template_str=n_object.get('notification_title', ''), **notification_parameters)
+            if n_object['notification_format'].startswith('HTML'):
+                n_body = escape_mixed_content(n_body)
 
             url = url.strip()
             if url.startswith('#'):
@@ -161,7 +164,6 @@ def process_notification(n_object, datastore):
             attach=n_object.get('screenshot', None)
         )
 
-
         # Returns empty string if nothing found, multi-line string otherwise
         log_value = logs.getvalue()
 

changedetectionio/templates/edit.html

@@ -6,39 +6,6 @@
 <script src="{{url_for('static_content', group='js', filename='vis.js')}}" defer></script>
 <script src="{{url_for('static_content', group='js', filename='global-settings.js')}}" defer></script>
 <script src="{{url_for('static_content', group='js', filename='scheduler.js')}}" defer></script>
-
-<script>
-    function addRuleRow() {
-        let rulesContainer = document.getElementById("rules-container");
-        let lastRule = document.querySelector(".rule-row:last-child");
-        let newRule = lastRule.cloneNode(true);
-
-        // Get the new unique index for the added row
-        let ruleCount = document.querySelectorAll(".rule-row").length;
-
-        // Update all IDs, names, and labels to have the correct index
-        newRule.querySelectorAll("select, input, label").forEach(element => {
-            if (element.id) {
-                element.id = element.id.replace(/\d+/, ruleCount);
-            }
-            if (element.name) {
-                element.name = element.name.replace(/\d+/, ruleCount);
-            }
-            if (element.hasAttribute("for")) {
-                element.setAttribute("for", element.getAttribute("for").replace(/\d+/, ruleCount));
-            }
-            if (element.tagName === "INPUT") {
-                element.value = ""; // Clear input field value
-            }
-        });
-
-        // Append the new rule to the grid container
-        rulesContainer.appendChild(newRule);
-    }
-</script>
-
-
-
 <script>
     const browser_steps_available_screenshots=JSON.parse('{{ watch.get_browsersteps_available_screenshots|tojson }}');
     const browser_steps_config=JSON.parse('{{ browser_steps_config|tojson }}');
@@ -78,12 +45,12 @@
             {% if extra_tab_content %}
             <li class="tab"><a href="#extras_tab">{{ extra_tab_content }}</a></li>
             {% endif %}
+            {% if playwright_enabled %}
             <li class="tab"><a id="browsersteps-tab" href="#browser-steps">Browser Steps</a></li>
-        <!-- should goto extra forms? -->
+            {% endif %}
             {% if watch['processor'] == 'text_json_diff' %}
             <li class="tab"><a id="visualselector-tab" href="#visualselector">Visual Filter Selector</a></li>
             <li class="tab" id="filters-and-triggers-tab"><a href="#filters-and-triggers">Filters &amp; Triggers</a></li>
-            <li class="tab" id="conditions-tab"><a href="#conditions">Conditions</a></li>
             {% endif %}
             <li class="tab"><a href="#notifications">Notifications</a></li>
             <li class="tab"><a href="#stats">Stats</a></li>
@@ -232,15 +199,8 @@ Math: {{ 1 + 1 }}") }}
                     </div>
             </fieldset>
             </div>
-
-            <div class="tab-pane-inner" id="browser-steps">
-                            <span class="pure-form-message-inline">
-                                <p>Sorry, this functionality only works with Playwright/Chrome enabled watches.</p>
-                                <p>Enable the Playwright Chrome fetcher, or alternatively try our <a href="https://lemonade.changedetection.io/start">very affordable subscription based service</a>.</p>
-                                <p>This is because Selenium/WebDriver can not extract full page screenshots reliably.</p>
-                            </span>
-
             {% if playwright_enabled %}
+            <div class="tab-pane-inner" id="browser-steps">
                 <img class="beta-logo" src="{{url_for('static_content', group='images', filename='beta-logo.png')}}" alt="New beta functionality">
                 <fieldset>
                     <div class="pure-control-group">
@@ -280,9 +240,8 @@ Math: {{ 1 + 1 }}") }}
                         </div>
                     </div>
                 </fieldset>
-                {% endif %}
             </div>
-
+            {% endif %}
 
             <div class="tab-pane-inner" id="notifications">
                 <fieldset>
@@ -312,36 +271,6 @@ Math: {{ 1 + 1 }}") }}
             </div>
 
             {% if watch['processor'] == 'text_json_diff' %}
-
-            <div class="tab-pane-inner" id="conditions">
-                <!-- Grid Header -->
-                <table>
-                    <thead>
-                    <td>In Value</td>
-                    <td>Operator</td>
-                    <td>Value</td>
-                    <td></td>
-
-                    </thead>
-                    <tbody>
-                    <!-- Rule Rows (Dynamic Content) -->
-
-                    {% for rule in form.conditions %}
-                        <tr>
-                            <td>{{ rule.field() }}</td>
-
-                            <td>{{ rule.operator() }}</td>
-                            <td>{{ rule.value() }}</td>
-
-                            <td>
-                                <button type="button" onclick="addRuleRow()">AND +</button>
-                            </td>
-                        </tr>
-                    {% endfor %}
-                    </tbody>
-                </table>
-
-            </div>
             <div class="tab-pane-inner" id="filters-and-triggers">
                 <span id="activate-text-preview" class="pure-button pure-button-primary button-xsmall">Activate preview</span>
               <div>

changedetectionio/tests/test_jsonpath_jq_selector.py

@@ -514,15 +514,3 @@ def test_check_jq_ext_filter(client, live_server, measure_memory_usage):
 def test_check_jqraw_ext_filter(client, live_server, measure_memory_usage):
     if jq_support:
         check_json_ext_filter('jq:.[] | select(.status | contains("Sold"))', client, live_server)
-
-def test_jsonpath_BOM_utf8(client, live_server, measure_memory_usage):
-    from .. import html_tools
-
-    # JSON string with BOM and correct double-quoted keys
-    json_str = '\ufeff{"name": "José", "emoji": "😊", "language": "中文", "greeting": "Привет"}'
-
-    # See that we can find the second <script> one, which is not broken, and matches our filter
-    text = html_tools.extract_json_as_string(json_str, "json:$.name")
-    assert text == '"José"'
-
-    

changedetectionio/tests/test_notification.py

@@ -454,8 +454,18 @@ def test_global_send_test_notification(client, live_server, measure_memory_usage
     assert b"Error: You must have atleast one watch configured for 'test notification' to work" in res.data
 
 
+    client.get(
+        url_for("form_delete", uuid="all"),
+        follow_redirects=True
+    )
+
 def _test_color_notifications(client, notification_body_token):
 
+    client.get(
+        url_for("form_delete", uuid="all"),
+        follow_redirects=True
+    )
+
     from changedetectionio.diff import ADDED_STYLE, REMOVED_STYLE
 
     set_original_response()
@@ -494,9 +504,9 @@ def _test_color_notifications(client, notification_body_token):
     wait_for_all_checks(client)
 
     set_modified_response()
-
-
+    
     res = client.get(url_for("form_watch_checknow"), follow_redirects=True)
+
     assert b'1 watches queued for rechecking.' in res.data
 
     wait_for_all_checks(client)
@@ -504,7 +514,8 @@ def _test_color_notifications(client, notification_body_token):
 
     with open("test-datastore/notification.txt", 'r') as f:
         x = f.read()
-        assert f'<span style="{REMOVED_STYLE}">Which is across multiple lines' in x
+        assert f'<span class="cdio" style="{REMOVED_STYLE}">Which is across multiple lines' in x
+        assert f'<br>' in x
 
 
     client.get(

changedetectionio/update_worker.py

@@ -16,6 +16,77 @@ import time
 
 from loguru import logger
 
+def build_notification_object_for_watch(watch, n_object, default_app_settings_notification_format):
+    from changedetectionio import diff
+    from changedetectionio.notification import default_notification_format_for_watch
+
+    dates = []
+    trigger_text = ''
+
+    if watch:
+        watch_history = watch.history
+        dates = list(watch_history.keys())
+        trigger_text = watch.get('trigger_text', [])
+
+    # Add text that was triggered
+    if len(dates):
+        snapshot_contents = watch.get_history_snapshot(dates[-1])
+    else:
+        snapshot_contents = "No snapshot/history available, the watch should fetch atleast once."
+
+    # If we ended up here with "System default"
+    if n_object.get('notification_format') == default_notification_format_for_watch:
+        n_object['notification_format'] = default_app_settings_notification_format
+
+    html_colour_enable = False
+    # HTML needs linebreak, but MarkDown and Text can use a linefeed
+    if n_object.get('notification_format') == 'HTML':
+        line_feed_sep = "<br>"
+        # Snapshot will be plaintext on the disk, convert to some kind of HTML
+        snapshot_contents = snapshot_contents.replace('\n', line_feed_sep)
+    elif n_object.get('notification_format') == 'HTML Color':
+        line_feed_sep = "<br>"
+        # Snapshot will be plaintext on the disk, convert to some kind of HTML
+        snapshot_contents = snapshot_contents.replace('\n', line_feed_sep)
+        html_colour_enable = True
+    else:
+        line_feed_sep = "\n"
+
+    triggered_text = ''
+    if len(trigger_text):
+        from . import html_tools
+        triggered_text = html_tools.get_triggered_text(content=snapshot_contents, trigger_text=trigger_text)
+        if triggered_text:
+            triggered_text = line_feed_sep.join(triggered_text)
+
+    # Could be called as a 'test notification' with only 1 snapshot available
+    prev_snapshot = "Example text: example test\nExample text: change detection is cool\nExample text: some more examples\n"
+    current_snapshot = "Example text: example test\nExample text: change detection is fantastic\nExample text: even more examples\nExample text: a lot more examples"
+
+    if len(dates) > 1:
+        prev_snapshot = watch.get_history_snapshot(dates[-2])
+        current_snapshot = watch.get_history_snapshot(dates[-1])
+
+    n_object.update({
+        'current_snapshot': snapshot_contents,
+        'diff': diff.render_diff(prev_snapshot, current_snapshot, line_feed_sep=line_feed_sep, html_colour=html_colour_enable),
+        'diff_added': diff.render_diff(prev_snapshot, current_snapshot, include_removed=False, line_feed_sep=line_feed_sep),
+        'diff_full': diff.render_diff(prev_snapshot, current_snapshot, include_equal=True, line_feed_sep=line_feed_sep,
+                                      html_colour=html_colour_enable),
+        'diff_patch': diff.render_diff(prev_snapshot, current_snapshot, line_feed_sep=line_feed_sep, patch_format=True),
+        'diff_removed': diff.render_diff(prev_snapshot, current_snapshot, include_added=False, line_feed_sep=line_feed_sep),
+        'notification_timestamp': time.time(),
+        'screenshot': watch.get_screenshot() if watch and watch.get('notification_screenshot') else None,
+        'triggered_text': triggered_text,
+        'uuid': watch.get('uuid') if watch else None,
+        'watch_url': watch.get('url') if watch else None,
+    })
+
+    if watch:
+        n_object.update(watch.extra_notification_token_values())
+
+    return n_object
+
 class update_worker(threading.Thread):
     current_uuid = None
 
@@ -27,75 +98,8 @@ class update_worker(threading.Thread):
         super().__init__(*args, **kwargs)
 
     def queue_notification_for_watch(self, notification_q, n_object, watch):
-        from changedetectionio import diff
-        from changedetectionio.notification import default_notification_format_for_watch
-
-        dates = []
-        trigger_text = ''
-
         now = time.time()
-
-        if watch:
-            watch_history = watch.history
-            dates = list(watch_history.keys())
-            trigger_text = watch.get('trigger_text', [])
-
-        # Add text that was triggered
-        if len(dates):
-            snapshot_contents = watch.get_history_snapshot(dates[-1])
-        else:
-            snapshot_contents = "No snapshot/history available, the watch should fetch atleast once."
-
-        # If we ended up here with "System default"
-        if n_object.get('notification_format') == default_notification_format_for_watch:
-            n_object['notification_format'] = self.datastore.data['settings']['application'].get('notification_format')
-
-        html_colour_enable = False
-        # HTML needs linebreak, but MarkDown and Text can use a linefeed
-        if n_object.get('notification_format') == 'HTML':
-            line_feed_sep = "<br>"
-            # Snapshot will be plaintext on the disk, convert to some kind of HTML
-            snapshot_contents = snapshot_contents.replace('\n', line_feed_sep)
-        elif n_object.get('notification_format') == 'HTML Color':
-            line_feed_sep = "<br>"
-            # Snapshot will be plaintext on the disk, convert to some kind of HTML
-            snapshot_contents = snapshot_contents.replace('\n', line_feed_sep)
-            html_colour_enable = True
-        else:
-            line_feed_sep = "\n"
-
-        triggered_text = ''
-        if len(trigger_text):
-            from . import html_tools
-            triggered_text = html_tools.get_triggered_text(content=snapshot_contents, trigger_text=trigger_text)
-            if triggered_text:
-                triggered_text = line_feed_sep.join(triggered_text)
-
-        # Could be called as a 'test notification' with only 1 snapshot available
-        prev_snapshot = "Example text: example test\nExample text: change detection is cool\nExample text: some more examples\n"
-        current_snapshot = "Example text: example test\nExample text: change detection is fantastic\nExample text: even more examples\nExample text: a lot more examples"
-
-        if len(dates) > 1:
-            prev_snapshot = watch.get_history_snapshot(dates[-2])
-            current_snapshot = watch.get_history_snapshot(dates[-1])
-
-        n_object.update({
-            'current_snapshot': snapshot_contents,
-            'diff': diff.render_diff(prev_snapshot, current_snapshot, line_feed_sep=line_feed_sep, html_colour=html_colour_enable),
-            'diff_added': diff.render_diff(prev_snapshot, current_snapshot, include_removed=False, line_feed_sep=line_feed_sep),
-            'diff_full': diff.render_diff(prev_snapshot, current_snapshot, include_equal=True, line_feed_sep=line_feed_sep, html_colour=html_colour_enable),
-            'diff_patch': diff.render_diff(prev_snapshot, current_snapshot, line_feed_sep=line_feed_sep, patch_format=True),
-            'diff_removed': diff.render_diff(prev_snapshot, current_snapshot, include_added=False, line_feed_sep=line_feed_sep),
-            'notification_timestamp': now,
-            'screenshot': watch.get_screenshot() if watch and watch.get('notification_screenshot') else None,
-            'triggered_text': triggered_text,
-            'uuid': watch.get('uuid') if watch else None,
-            'watch_url': watch.get('url') if watch else None,
-        })
-
-        if watch:
-            n_object.update(watch.extra_notification_token_values())
-
+        n_object = build_notification_object_for_watch(watch, n_object, self.datastore.data['settings']['application'].get('notification_format'))
         logger.trace(f"Main rendered notification placeholders (diff_added etc) calculated in {time.time()-now:.3f}s")
         logger.debug("Queued notification for sending")
         notification_q.put(n_object)

requirements.txt

@@ -98,14 +98,5 @@ greenlet >= 3.0.3
 # Pinned or it causes problems with flask_expects_json which seems unmaintained
 referencing==0.35.1
 
-panzi-json-logic
-
 # Scheduler - Windows seemed to miss a lot of default timezone info (even "UTC" !)
 tzdata
-
-#typing_extensions ==4.8.0
-
-pluggy ~= 1.5
-
-
-