woops

UI - "Browser Steps" tab should be always available with helpful info
UI - Fix mute/unmute alt/title label alt/title text in watch overview (#2951 )
2026-07-02 13:31:18 +00:00 · 2025-02-09 21:12:46 +01:00 · 2025-02-09 21:02:46 +01:00 · 2025-02-08 18:01:26 +01:00 · 2025-02-08 10:14:19 +01:00 · 2025-02-08 00:23:20 +01:00
51 changed files with 566 additions and 401 deletions
@@ -2,32 +2,33 @@
 # Test that we can still build on Alpine (musl modified libc https://musl.libc.org/)
 # Some packages wont install via pypi because they dont have a wheel available under this architecture.

-FROM ghcr.io/linuxserver/baseimage-alpine:3.18
+FROM ghcr.io/linuxserver/baseimage-alpine:3.21
 ENV PYTHONUNBUFFERED=1

 COPY requirements.txt /requirements.txt

 RUN \
-  apk add --update --no-cache --virtual=build-dependencies \
+ apk add --update --no-cache --virtual=build-dependencies \
+    build-base \
    cargo \
-    g++ \
-    gcc \
+    git \
    jpeg-dev \
    libc-dev \
    libffi-dev \
-    libjpeg \
    libxslt-dev \
-    make \
    openssl-dev \
-    py3-wheel \
    python3-dev \
+    zip \
    zlib-dev && \
  apk add --update --no-cache \
+    libjpeg \
    libxslt \
-    python3 \
-    py3-pip && \
+    nodejs \
+    poppler-utils \
+    python3 && \
  echo "**** pip3 install test of changedetection.io ****" && \
-  pip3 install -U pip wheel setuptools && \
-  pip3 install -U --no-cache-dir --find-links https://wheel-index.linuxserver.io/alpine-3.18/ -r /requirements.txt && \
+  python3 -m venv /lsiopy  && \
+  pip install -U pip wheel setuptools && \
+  pip install -U --no-cache-dir --find-links https://wheel-index.linuxserver.io/alpine-3.21/ -r /requirements.txt && \
  apk del --purge \
    build-dependencies
@@ -103,6 +103,19 @@ jobs:
 #          provenance: false

      # A new tagged release is required, which builds :tag and :latest
+      - name: Docker meta :tag
+        if: github.event_name == 'release' && startsWith(github.event.release.tag_name, '0.')
+        uses: docker/metadata-action@v5
+        id: meta
+        with:
+            images: |
+                ${{ secrets.DOCKER_HUB_USERNAME }}/changedetection.io
+                ghcr.io/dgtlmoon/changedetection.io
+            tags: |
+                type=semver,pattern={{version}}
+                type=semver,pattern={{major}}.{{minor}}
+                type=semver,pattern={{major}}
+
      - name: Build and push :tag
        id: docker_build_tag_release
        if: github.event_name == 'release' && startsWith(github.event.release.tag_name, '0.')
@@ -111,11 +124,7 @@ jobs:
          context: ./
          file: ./Dockerfile
          push: true
-          tags: |
-            ${{ secrets.DOCKER_HUB_USERNAME }}/changedetection.io:${{ github.event.release.tag_name }}
-            ghcr.io/dgtlmoon/changedetection.io:${{ github.event.release.tag_name }}
-            ${{ secrets.DOCKER_HUB_USERNAME }}/changedetection.io:latest
-            ghcr.io/dgtlmoon/changedetection.io:latest
+          tags: ${{ steps.meta.outputs.tags }}
          platforms: linux/amd64,linux/arm64,linux/arm/v7,linux/arm/v8,linux/arm64/v8
          cache-from: type=gha
          cache-to: type=gha,mode=max
@@ -45,7 +45,6 @@ jobs:
    - name: Test that the basic pip built package runs without error
      run: |
        set -ex
-        sudo pip3 install --upgrade pip 
        pip3 install dist/changedetection.io*.whl
        changedetection.io -d /tmp -p 10000 &
        sleep 3
@@ -64,14 +64,16 @@ jobs:
          echo "Running processes in docker..."
          docker ps

-      - name: Test built container with Pytest (generally as requests/plaintext fetching)
+      - name: Run Unit Tests
        run: |
          # Unit tests
-          echo "run test with unittest"
          docker run test-changedetectionio  bash -c 'python3 -m unittest changedetectionio.tests.unit.test_notification_diff'
          docker run test-changedetectionio  bash -c 'python3 -m unittest changedetectionio.tests.unit.test_watch_model'
          docker run test-changedetectionio  bash -c 'python3 -m unittest changedetectionio.tests.unit.test_jinja2_security'
-          
+          docker run test-changedetectionio  bash -c 'python3 -m unittest changedetectionio.tests.unit.test_semver'
+
+      - name: Test built container with Pytest (generally as requests/plaintext fetching)
+        run: |
          # All tests
          echo "run test with pytest"
          # The default pytest logger_level is TRACE
@@ -120,7 +120,7 @@ Easily add the current web page to your changedetection.io tool, simply install

 [<img src="./docs/chrome-extension-screenshot.png" style="max-width:80%;" alt="Chrome Extension to easily add the current web-page to detect a change."  title="Chrome Extension to easily add the current web-page to detect a change."  />](https://chromewebstore.google.com/detail/changedetectionio-website/kefcfmgmlhmankjmnbijimhofdjekbop)

-[Goto the Chrome Webstore to download the extension.](https://chromewebstore.google.com/detail/changedetectionio-website/kefcfmgmlhmankjmnbijimhofdjekbop)
+[Goto the Chrome Webstore to download the extension.](https://chromewebstore.google.com/detail/changedetectionio-website/kefcfmgmlhmankjmnbijimhofdjekbop) ( Or check out the [GitHub repo](https://github.com/dgtlmoon/changedetection.io-browser-extension) ) 

 ## Installation

@@ -2,7 +2,7 @@

 # Read more https://github.com/dgtlmoon/changedetection.io/wiki

-__version__ = '0.48.05'
+__version__ = '0.49.1'

 from changedetectionio.strtobool import strtobool
 from json.decoder import JSONDecodeError
@@ -24,6 +24,9 @@ from loguru import logger
 app = None
 datastore = None

+def get_version():
+    return __version__
+
 # Parent wrapper or OS sends us a SIGTERM/SIGINT, do everything required for a clean shutdown
 def sigshutdown_handler(_signo, _stack_frame):
    global app
@@ -12,10 +12,10 @@ import copy
 # See docs/README.md for rebuilding the docs/apidoc information

 from . import api_schema
-from ..model import WatchBase
+from ..model import watch_base

 # Build a JSON Schema atleast partially based on our Watch model
-watch_base_config = WatchBase()
+watch_base_config = watch_base()
 schema = api_schema.build_watch_json_schema(watch_base_config)

 schema_create_watch = copy.deepcopy(schema)
@@ -52,8 +52,8 @@ class Watch(Resource):
        @apiSuccess (200) {String} OK When paused/muted/recheck operation OR full JSON object of the watch
        @apiSuccess (200) {JSON} WatchJSON JSON Full JSON object of the watch
        """
-        watch = self.datastore.data['watching'].get(uuid)
-
+        from copy import deepcopy
+        watch = deepcopy(self.datastore.data['watching'].get(uuid))
        if not watch:
            abort(404, message='No watch exists with the UUID of {}'.format(uuid))

@@ -75,11 +75,11 @@ class Watch(Resource):

        # Return without history, get that via another API call
        # Properties are not returned as a JSON, so add the required props manually
-        result = watch.as_dict()
-        result['history_n'] = watch.history_n
-        result['last_changed'] = watch.last_changed
-        result['viewed'] = watch.viewed
-        return result
+        watch['history_n'] = watch.history_n
+        # attr .last_changed will check for the last written text snapshot on change
+        watch['last_changed'] = watch.last_changed
+        watch['viewed'] = watch.viewed
+        return watch

    @auth.check_token
    def delete(self, uuid):
@@ -1,3 +1,4 @@
+from changedetectionio import apprise_plugin
 import apprise

 # Create our AppriseAsset and populate it with some of our new values:
@@ -1,6 +1,8 @@
 # include the decorator
 from apprise.decorators import notify
 from loguru import logger
+from requests.structures import CaseInsensitiveDict
+

@notify(on="delete")
@notify(on="deletes")
@@ -13,70 +15,84 @@ from loguru import logger
 def apprise_custom_api_call_wrapper(body, title, notify_type, *args, **kwargs):
    import requests
    import json
+    import re
+
    from urllib.parse import unquote_plus
-    from apprise.utils import parse_url as apprise_parse_url
-    from apprise import URLBase
+    from apprise.utils.parse import parse_url as apprise_parse_url

    url = kwargs['meta'].get('url')
+    schema = kwargs['meta'].get('schema').lower().strip()

-    if url.startswith('post'):
-        r = requests.post
-    elif url.startswith('get'):
-        r = requests.get
-    elif url.startswith('put'):
-        r = requests.put
-    elif url.startswith('delete'):
-        r = requests.delete
+    # Choose POST, GET etc from requests
+    method =  re.sub(rf's$', '', schema)
+    requests_method = getattr(requests, method)

-    url = url.replace('post://', 'http://')
-    url = url.replace('posts://', 'https://')
-    url = url.replace('put://', 'http://')
-    url = url.replace('puts://', 'https://')
-    url = url.replace('get://', 'http://')
-    url = url.replace('gets://', 'https://')
-    url = url.replace('put://', 'http://')
-    url = url.replace('puts://', 'https://')
-    url = url.replace('delete://', 'http://')
-    url = url.replace('deletes://', 'https://')
-
-    headers = {}
-    params = {}
+    params = CaseInsensitiveDict({}) # Added to requests
    auth = None
+    has_error = False

    # Convert /foobar?+some-header=hello to proper header dictionary
    results = apprise_parse_url(url)
-    if results:
-        # Add our headers that the user can potentially over-ride if they wish
-        # to to our returned result set and tidy entries by unquoting them
-        headers = {unquote_plus(x): unquote_plus(y)
-                   for x, y in results['qsd+'].items()}

-        # https://github.com/caronc/apprise/wiki/Notify_Custom_JSON#get-parameter-manipulation
-        # In Apprise, it relies on prefixing each request arg with "-", because it uses say &method=update as a flag for apprise
-        # but here we are making straight requests, so we need todo convert this against apprise's logic
-        for k, v in results['qsd'].items():
-            if not k.strip('+-') in results['qsd+'].keys():
-                params[unquote_plus(k)] = unquote_plus(v)
+    # Add our headers that the user can potentially over-ride if they wish
+    # to to our returned result set and tidy entries by unquoting them
+    headers = CaseInsensitiveDict({unquote_plus(x): unquote_plus(y)
+               for x, y in results['qsd+'].items()})

-        # Determine Authentication
-        auth = ''
-        if results.get('user') and results.get('password'):
-            auth = (unquote_plus(results.get('user')), unquote_plus(results.get('user')))
-        elif results.get('user'):
-            auth = (unquote_plus(results.get('user')))
+    # https://github.com/caronc/apprise/wiki/Notify_Custom_JSON#get-parameter-manipulation
+    # In Apprise, it relies on prefixing each request arg with "-", because it uses say &method=update as a flag for apprise
+    # but here we are making straight requests, so we need todo convert this against apprise's logic
+    for k, v in results['qsd'].items():
+        if not k.strip('+-') in results['qsd+'].keys():
+            params[unquote_plus(k)] = unquote_plus(v)

-    # Try to auto-guess if it's JSON
-    h = 'application/json; charset=utf-8'
+    # Determine Authentication
+    auth = ''
+    if results.get('user') and results.get('password'):
+        auth = (unquote_plus(results.get('user')), unquote_plus(results.get('user')))
+    elif results.get('user'):
+        auth = (unquote_plus(results.get('user')))
+
+    # If it smells like it could be JSON and no content-type was already set, offer a default content type.
+    if body and '{' in body[:100] and not headers.get('Content-Type'):
+        json_header = 'application/json; charset=utf-8'
+        try:
+            # Try if it's JSON
+            json.loads(body)
+            headers['Content-Type'] = json_header
+        except ValueError as e:
+            logger.warning(f"Could not automatically add '{json_header}' header to the notification because the document failed to parse as JSON: {e}")
+            pass
+
+    # POSTS -> HTTPS etc
+    if schema.lower().endswith('s'):
+        url = re.sub(rf'^{schema}', 'https', results.get('url'))
+    else:
+        url = re.sub(rf'^{schema}', 'http', results.get('url'))
+
+    status_str = ''
    try:
-        json.loads(body)
-        headers['Content-Type'] = h
-    except ValueError as e:
-        logger.warning(f"Could not automatically add '{h}' header to the {kwargs['meta'].get('schema')}:// notification because the document failed to parse as JSON: {e}")
-        pass
+        r = requests_method(url,
+          auth=auth,
+          data=body.encode('utf-8') if type(body) is str else body,
+          headers=headers,
+          params=params
+        )

-    r(results.get('url'),
-      auth=auth,
-      data=body.encode('utf-8') if type(body) is str else body,
-      headers=headers,
-      params=params
-      )
+        if not (200 <= r.status_code < 300):
+            status_str = f"Error sending '{method.upper()}' request to {url} - Status: {r.status_code}: '{r.reason}'"
+            logger.error(status_str)
+            has_error = True
+        else:
+            logger.info(f"Sent '{method.upper()}' request to {url}")
+            has_error = False
+
+    except requests.RequestException as e:
+        status_str = f"Error sending '{method.upper()}' request to {url} - {str(e)}"
+        logger.error(status_str)
+        has_error = True
+
+    if has_error:
+        raise TypeError(status_str)
+
+    return True
@@ -41,7 +41,6 @@ def construct_blueprint(datastore: ChangeDetectionStore):
        now = time.time()
        try:
            processor_module = importlib.import_module("changedetectionio.processors.text_json_diff.processor")
-            # @todo can now just pass the watch here?
            update_handler = processor_module.perform_site_check(datastore=datastore,
                                                                 watch_uuid=uuid
                                                                 )
@@ -52,6 +52,7 @@ function isItemInStock() {
        'niet leverbaar',
        'niet op voorraad',
        'no disponible',
+        'non disponibile',
        'no longer in stock',
        'no tickets available',
        'not available',
@@ -43,7 +43,6 @@ from loguru import logger
 from changedetectionio import html_tools, __version__
 from changedetectionio import queuedWatchMetaData
 from changedetectionio.api import api_v1
-from .store import CustomEncoder
 from .time_handler import is_within_schedule

 datastore = None
@@ -599,17 +598,31 @@ def changedetection_app(config=None, datastore_o=None):

            if 'notification_title' in request.form and request.form['notification_title'].strip():
                n_object['notification_title'] = request.form.get('notification_title', '').strip()
+            elif datastore.data['settings']['application'].get('notification_title'):
+                n_object['notification_title'] = datastore.data['settings']['application'].get('notification_title')
+            else:
+                n_object['notification_title'] = "Test title"

            if 'notification_body' in request.form and request.form['notification_body'].strip():
                n_object['notification_body'] = request.form.get('notification_body', '').strip()
+            elif datastore.data['settings']['application'].get('notification_body'):
+                n_object['notification_body'] = datastore.data['settings']['application'].get('notification_body')
+            else:
+                n_object['notification_body'] = "Test body"

+            n_object['as_async'] = False
            n_object.update(watch.extra_notification_token_values())
+            from .notification import process_notification
+            sent_obj = process_notification(n_object, datastore)

-            from . import update_worker
-            new_worker = update_worker.update_worker(update_q, notification_q, app, datastore)
-            new_worker.queue_notification_for_watch(notification_q=notification_q, n_object=n_object, watch=watch)
        except Exception as e:
-            return make_response(f"Error: str(e)", 400)
+            e_str = str(e)
+            # Remove this text which is not important and floods the container
+            e_str = e_str.replace(
+                "DEBUG - <class 'apprise.decorators.base.CustomNotifyPlugin.instantiate_plugin.<locals>.CustomNotifyPluginWrapper'>",
+                '')
+
+            return make_response(e_str, 400)

        return 'OK - Sent test notifications'

@@ -801,7 +814,7 @@ def changedetection_app(config=None, datastore_o=None):

            # Recast it if need be to right data Watch handler
            watch_class = get_custom_watch_obj_for_processor(form.data.get('processor'))
-            datastore.data['watching'][uuid] = watch_class(__datastore=datastore_o, default=datastore.data['watching'][uuid])
+            datastore.data['watching'][uuid] = watch_class(datastore_path=datastore_o.datastore_path, default=datastore.data['watching'][uuid])
            flash("Updated watch - unpaused!" if request.args.get('unpause_on_save') else "Updated watch.")

            # Re #286 - We wait for syncing new data to disk in another thread every 60 seconds
@@ -1614,7 +1627,7 @@ def changedetection_app(config=None, datastore_o=None):
        watch['ignore_text'] += datastore.data['settings']['application']['global_ignore_text']
        watch['subtractive_selectors'] += datastore.data['settings']['application']['global_subtractive_selectors']

-        watch_json = json.dumps(watch, cls=CustomEncoder)
+        watch_json = json.dumps(watch)

        try:
            r = requests.request(method="POST",
@@ -1,5 +1,6 @@
-from typing import List
+from loguru import logger
 from lxml import etree
+from typing import List
 import json
 import re

@@ -298,8 +299,10 @@ def extract_json_as_string(content, json_filter, ensure_is_ldjson_info_type=None
 # https://github.com/dgtlmoon/changedetection.io/pull/2041#issuecomment-1848397161w
    # Try to parse/filter out the JSON, if we get some parser error, then maybe it's embedded within HTML tags
    try:
-        stripped_text_from_html = _parse_json(json.loads(content), json_filter)
-    except json.JSONDecodeError:
+        # .lstrip("\ufeff") strings ByteOrderMark from UTF8 and still lets the UTF work
+        stripped_text_from_html = _parse_json(json.loads(content.lstrip("\ufeff") ), json_filter)
+    except json.JSONDecodeError as e:
+        logger.warning(str(e))

        # Foreach <script json></script> blob.. just return the first that matches json_filter
        # As a last resort, try to parse the whole <body>
@@ -10,7 +10,7 @@ _FILTER_FAILURE_THRESHOLD_ATTEMPTS_DEFAULT = 6
 DEFAULT_SETTINGS_HEADERS_USERAGENT='Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/87.0.4280.66 Safari/537.36'

 class model(dict):
-    __base_config = {
+    base_config = {
            'note': "Hello! If you change this file manually, please be sure to restart your changedetection.io instance!",
            'watching': {},
            'settings': {
@@ -60,7 +60,7 @@ class model(dict):

    def __init__(self, *arg, **kw):
        super(model, self).__init__(*arg, **kw)
-        self.update(self.__base_config)
+        self.update(self.base_config)


 def parse_headers_from_text_file(filepath):
@@ -69,7 +69,7 @@ def parse_headers_from_text_file(filepath):
        for l in f.readlines():
            l = l.strip()
            if not l.startswith('#') and ':' in l:
-                (k, v) = l.split(':')
+                (k, v) = l.split(':', 1)  # Split only on the first colon
                headers[k.strip()] = v.strip()

    return headers
@@ -1,8 +1,8 @@

-from changedetectionio.model import WatchBase
+from changedetectionio.model import watch_base


-class model(WatchBase):
+class model(watch_base):

    def __init__(self, *arg, **kw):
        super(model, self).__init__(*arg, **kw)
@@ -1,17 +1,17 @@
 from changedetectionio.strtobool import strtobool
 from changedetectionio.safe_jinja import render as jinja_render
+from . import watch_base
 import os
 import re
 from pathlib import Path
 from loguru import logger

-from . import WatchBase
 from ..html_tools import TRANSLATE_WHITESPACE_TABLE

 # Allowable protocols, protects against javascript: etc
 # file:// is further checked by ALLOW_FILE_URI
 SAFE_PROTOCOL_REGEX='^(http|https|ftp|file):'
-WATCH_DB_JSON_FILENAME = 'watch.json'
+
 minimum_seconds_recheck_time = int(os.getenv('MINIMUM_SECONDS_RECHECK_TIME', 3))
 mtable = {'seconds': 1, 'minutes': 60, 'hours': 3600, 'days': 86400, 'weeks': 86400 * 7}

@@ -32,20 +32,15 @@ def is_safe_url(test_url):
    return True


-class model(WatchBase):
-    __datastore = None
-    __datastore_checksum = None
-
-    __history_n = 0
+class model(watch_base):
    __newest_history_key = None
+    __history_n = 0
    jitter_seconds = 0
-    
+
    def __init__(self, *arg, **kw):
-        if not kw.get('__datastore'):
-            logger.critical('No __datastore reference was set!')
-
-        self.__datastore = kw.get('__datastore')
-
+        self.__datastore_path = kw.get('datastore_path')
+        if kw.get('datastore_path'):
+            del kw['datastore_path']
        super(model, self).__init__(*arg, **kw)
        if kw.get('default'):
            self.update(kw['default'])
@@ -184,7 +179,7 @@ class model(WatchBase):
        tmp_history = {}

        # In the case we are only using the watch for processing without history
-        if not self.__datastore or not self.watch_data_dir:
+        if not self.watch_data_dir:
            return []

        # Read the history file as a dict
@@ -424,7 +419,7 @@ class model(WatchBase):
    @property
    def watch_data_dir(self):
        # The base dir of the watch data
-        return os.path.join(self.__datastore.datastore_path, self['uuid']) if self.__datastore.datastore_path else None
+        return os.path.join(self.__datastore_path, self['uuid']) if self.__datastore_path else None

    def get_error_text(self):
        """Return the text saved from a previous request that resulted in a non-200 error"""
@@ -529,22 +524,6 @@ class model(WatchBase):
        # None is set
        return False

-    def save_data(self):
-        import json
-        # @todo dict change?
-        # Save it to a temp file first so that if the disk is full or other error it wont corrupt (hopefully).
-
-        dest = os.path.join(self.watch_data_dir, WATCH_DB_JSON_FILENAME)
-        logger.debug(f"Saving watch {dest}")
-        try:
-            with open(dest + '.tmp', 'w') as json_file:
-                json.dump(self.as_dict(), json_file, indent=2)
-            os.replace(dest + '.tmp', dest)
-
-        except Exception as e:
-            logger.critical(f"Exception saving watch JSON {dest} - {e}")
-
-
    def save_error_text(self, contents):
        self.ensure_data_dir_exists()
        target_path = os.path.join(self.watch_data_dir, "last-error.txt")
@@ -1,14 +1,13 @@
 import os
 import uuid
-from collections.abc import MutableMapping

 from changedetectionio import strtobool
 from changedetectionio.notification import default_notification_format_for_watch

-class WatchBase(MutableMapping):
-    __data_checksum = None
-    def __init__(self, *args, **kwargs):
-        self.__internal_dict = {
+class watch_base(dict):
+
+    def __init__(self, *arg, **kw):
+        self.update({
            # Custom notification content
            # Re #110, so then if this is set to None, we know to use the default value instead
            # Requires setting to None on submit if it's the same as the default
@@ -128,37 +127,9 @@ class WatchBase(MutableMapping):
            'uuid': str(uuid.uuid4()),
            'webdriver_delay': None,
            'webdriver_js_execute_code': None,  # Run before change-detection
-        }
+        })

-        # Update with any provided arguments
-        self.update(*args, **kwargs)
+        super(watch_base, self).__init__(*arg, **kw)

        if self.get('default'):
-            del self['default']
-
-
-    # Implement abstract methods required by MutableMapping
-    def __getitem__(self, key):
-        return self.__internal_dict[key]
-
-    def __setitem__(self, key, value):
-        if key == '__datastore':
-            self.__datastore = value
-        else:
-            self.__internal_dict[key] = value
-
-    def __delitem__(self, key):
-        del self.__internal_dict[key]
-
-    def __iter__(self):
-        return iter(self.__internal_dict)
-
-    def __len__(self):
-        return len(self.__internal_dict)
-
-    # Optional: Implement additional methods for convenience
-    def __repr__(self):
-        return f"{self.__class__.__name__}({self.__internal_dict})"
-
-    def as_dict(self):
-        return self.__internal_dict
+            del self['default']
@@ -67,6 +67,10 @@ def process_notification(n_object, datastore):

    sent_objs = []
    from .apprise_asset import asset
+
+    if 'as_async' in n_object:
+        asset.async_mode = n_object.get('as_async')
+
    apobj = apprise.Apprise(debug=True, asset=asset)

    if not n_object.get('notification_urls'):
@@ -157,8 +161,6 @@ def process_notification(n_object, datastore):
            attach=n_object.get('screenshot', None)
        )

-        # Give apprise time to register an error
-        time.sleep(3)

        # Returns empty string if nothing found, multi-line string otherwise
        log_value = logs.getvalue()
@@ -1,42 +1,52 @@
-$(document).ready(function() {
+$(document).ready(function () {

-  $('#add-email-helper').click(function (e) {
-    e.preventDefault();
-    email = prompt("Destination email");
-    if(email) {
-      var n = $(".notification-urls");
-      var p=email_notification_prefix;
-      $(n).val( $.trim( $(n).val() )+"\n"+email_notification_prefix+email );
-    }
-  });
-
-  $('#send-test-notification').click(function (e) {
-    e.preventDefault();
-
-    data = {
-      notification_body: $('#notification_body').val(),
-      notification_format: $('#notification_format').val(),
-      notification_title: $('#notification_title').val(),
-      notification_urls: $('.notification-urls').val(),
-      tags: $('#tags').val(),
-      window_url: window.location.href,
-    }
-
-
-    $.ajax({
-      type: "POST",
-      url: notification_base_url,
-      data : data,
-        statusCode: {
-        400: function(data) {
-          // More than likely the CSRF token was lost when the server restarted
-          alert(data.responseText);
+    $('#add-email-helper').click(function (e) {
+        e.preventDefault();
+        email = prompt("Destination email");
+        if (email) {
+            var n = $(".notification-urls");
+            var p = email_notification_prefix;
+            $(n).val($.trim($(n).val()) + "\n" + email_notification_prefix + email);
        }
-      }
-    }).done(function(data){
-      console.log(data);
-      alert(data);
-    })
-  });
+    });
+
+    $('#send-test-notification').click(function (e) {
+        e.preventDefault();
+
+        data = {
+            notification_body: $('#notification_body').val(),
+            notification_format: $('#notification_format').val(),
+            notification_title: $('#notification_title').val(),
+            notification_urls: $('.notification-urls').val(),
+            tags: $('#tags').val(),
+            window_url: window.location.href,
+        }
+
+        $('.notifications-wrapper .spinner').fadeIn();
+        $('#notification-test-log').show();
+        $.ajax({
+            type: "POST",
+            url: notification_base_url,
+            data: data,
+            statusCode: {
+                400: function (data) {
+                    $("#notification-test-log>span").text(data.responseText);
+                },
+            }
+        }).done(function (data) {
+            $("#notification-test-log>span").text(data);
+        }).fail(function (jqXHR, textStatus, errorThrown) {
+            // Handle connection refused or other errors
+            if (textStatus === "error" && errorThrown === "") {
+                console.error("Connection refused or server unreachable");
+                $("#notification-test-log>span").text("Error: Connection refused or server is unreachable.");
+            } else {
+                console.error("Error:", textStatus, errorThrown);
+                $("#notification-test-log>span").text("An error occurred: " + textStatus);
+            }
+        }).always(function () {
+            $('.notifications-wrapper .spinner').hide();
+        })
+    });
 });

@@ -380,7 +380,15 @@ a.pure-button-selected {
 }

 .notifications-wrapper {
-  padding: 0.5rem 0 1rem 0;
+  padding-top: 0.5rem;
+  #notification-test-log {
+    padding-top: 1rem;
+    white-space: pre-wrap;
+    word-break: break-word;
+    overflow-wrap: break-word;
+    max-width: 100%;
+    box-sizing: border-box;
+  }
 }

 label {
@@ -780,7 +780,14 @@ a.pure-button-selected {
  cursor: pointer; }

 .notifications-wrapper {
-  padding: 0.5rem 0 1rem 0; }
+  padding-top: 0.5rem; }
+  .notifications-wrapper #notification-test-log {
+    padding-top: 1rem;
+    white-space: pre-wrap;
+    word-break: break-word;
+    overflow-wrap: break-word;
+    max-width: 100%;
+    box-sizing: border-box; }

 label:hover {
  cursor: pointer; }
@@ -1,5 +1,3 @@
-import glob
-
 from changedetectionio.strtobool import strtobool

 from flask import (
@@ -7,8 +5,8 @@ from flask import (
 )

 from .html_tools import TRANSLATE_WHITESPACE_TABLE
-from .model import App, Watch, WatchBase
-from copy import deepcopy
+from . model import App, Watch
+from copy import deepcopy, copy
 from os import path, unlink
 from threading import Lock
 import json
@@ -20,7 +18,6 @@ import time
 import uuid as uuid_builder
 from loguru import logger

-from .model.Watch import WATCH_DB_JSON_FILENAME
 from .processors import get_custom_watch_obj_for_processor
 from .processors.restock_diff import Restock

@@ -29,13 +26,6 @@ BASE_URL_NOT_SET_TEXT = '("Base URL" not set - see settings - notifications)'

 dictfilt = lambda x, y: dict([ (i,x[i]) for i in x if i in set(y) ])

-class CustomEncoder(json.JSONEncoder):
-    def default(self, obj):
-        if obj and isinstance(obj, WatchBase):
-            return obj.as_dict()
-        # Add more custom type handlers here
-        return super().default(obj)
-
 # Is there an existing library to ensure some data store (JSON etc) is in sync with CRUD methods?
 # Open a github issue if you know something :)
 # https://stackoverflow.com/questions/6190468/how-to-trigger-function-on-value-change
@@ -52,14 +42,16 @@ class ChangeDetectionStore:
    def __init__(self, datastore_path="/datastore", include_default_watches=True, version_tag="0.0.0"):
        # Should only be active for docker
        # logging.basicConfig(filename='/dev/stdout', level=logging.INFO)
-        from os.path import join
        self.__data = App.model()
        self.datastore_path = datastore_path
-        self.json_store_path = join(self.datastore_path, 'url-watches.json')
+        self.json_store_path = "{}/url-watches.json".format(self.datastore_path)
        logger.info(f"Datastore path is '{self.json_store_path}'")
        self.needs_write = False
        self.start_time = time.time()
        self.stop_thread = False
+        # Base definition for all watchers
+        # deepcopy part of #569 - not sure why its needed exactly
+        self.generic_definition = deepcopy(Watch.model(datastore_path = datastore_path, default={}))

        if path.isfile('changedetectionio/source.txt'):
            with open('changedetectionio/source.txt') as f:
@@ -73,6 +65,10 @@ class ChangeDetectionStore:
                from_disk = json.load(json_file)

                # @todo isnt there a way todo this dict.update recursively?
+                # Problem here is if the one on the disk is missing a sub-struct, it wont be present anymore.
+                if 'watching' in from_disk:
+                    self.__data['watching'].update(from_disk['watching'])
+
                if 'app_guid' in from_disk:
                    self.__data['app_guid'] = from_disk['app_guid']

@@ -86,7 +82,10 @@ class ChangeDetectionStore:
                    if 'application' in from_disk['settings']:
                        self.__data['settings']['application'].update(from_disk['settings']['application'])

-                self.scan_and_load_watches()
+                # Convert each existing watch back to the Watch.model object
+                for uuid, watch in self.__data['watching'].items():
+                    self.__data['watching'][uuid] = self.rehydrate_entity(uuid, watch)
+                    logger.info(f"Watching: {uuid} {watch['url']}")

                # And for Tags also, should be Restock type because it has extra settings
                for uuid, tag in self.__data['settings']['application']['tags'].items():
@@ -159,29 +158,9 @@ class ChangeDetectionStore:
        if entity.get('uuid') != 'text_json_diff':
            logger.trace(f"Loading Watch object '{watch_class.__module__}.{watch_class.__name__}' for UUID {uuid}")

-        entity = watch_class(__datastore=self, default=entity)
+        entity = watch_class(datastore_path=self.datastore_path, default=entity)
        return entity

-    def scan_and_load_watches(self):
-
-        # Use glob to find all occurrences of 'watch.json' in subdirectories
-        # @todo move to some other function so we can trigger a rescan in a thread
-        for file_path in glob.glob(f"{self.datastore_path}/*/{WATCH_DB_JSON_FILENAME}", recursive=True):
-            try:
-                with open(file_path, 'r') as json_file:
-                    data = json.load(json_file)
-                    # So that we can always move it to another UUID by renaming the dir
-                    directory_path = os.path.dirname(file_path)
-                    uuid = os.path.basename(directory_path)
-                    if data.get('uuid'):
-                        del data['uuid']
-                    self.__data['watching'][uuid] = self.rehydrate_entity(uuid, data)
-
-            except json.JSONDecodeError as e:
-                logger.error(f"Error decoding JSON in file {file_path}: {e}")
-            except Exception as e:
-                logger.critical(f"Exception decoding JSON in file {file_path}: {e}")
-
    def set_last_viewed(self, uuid, timestamp):
        logger.debug(f"Setting watch UUID: {uuid} last viewed to {int(timestamp)}")
        self.data['watching'][uuid].update({'last_viewed': int(timestamp)})
@@ -198,15 +177,13 @@ class ChangeDetectionStore:
            return

        with self.lock:
-            # deepcopy part of #569 - not sure why its needed exactly
-#            self.generic_definition = deepcopy(Watch.model(default={}))

-#            # In python 3.9 we have the |= dict operator, but that still will lose data on nested structures...
-#            for dict_key, d in self.generic_definition.items():
-#                if isinstance(d, dict):
-#                    if update_obj is not None and dict_key in update_obj:
-#                        self.__data['watching'][uuid][dict_key].update(update_obj[dict_key])
-#                        del (update_obj[dict_key])
+            # In python 3.9 we have the |= dict operator, but that still will lose data on nested structures...
+            for dict_key, d in self.generic_definition.items():
+                if isinstance(d, dict):
+                    if update_obj is not None and dict_key in update_obj:
+                        self.__data['watching'][uuid][dict_key].update(update_obj[dict_key])
+                        del (update_obj[dict_key])

            self.__data['watching'][uuid].update(update_obj)
        self.needs_write = True
@@ -369,7 +346,7 @@ class ChangeDetectionStore:

        # If the processor also has its own Watch implementation
        watch_class = get_custom_watch_obj_for_processor(apply_extras.get('processor'))
-        new_watch = watch_class(__datastore=self, url=url)
+        new_watch = watch_class(datastore_path=self.datastore_path, url=url)

        new_uuid = new_watch.get('uuid')

@@ -406,8 +383,7 @@ class ChangeDetectionStore:
    def sync_to_json(self):
        logger.info("Saving JSON..")
        try:
-            data = {key: deepcopy(value) for key, value in self.__data.items() if key != 'watching'}
-
+            data = deepcopy(self.__data)
        except RuntimeError as e:
            # Try again in 15 seconds
            time.sleep(15)
@@ -421,15 +397,11 @@ class ChangeDetectionStore:
                # This is a fairly basic strategy to deal with the case that the file is corrupted,
                # system was out of memory, out of RAM etc
                with open(self.json_store_path+".tmp", 'w') as json_file:
-                    json.dump(data, json_file, indent=2, cls=CustomEncoder)
+                    json.dump(data, json_file, indent=4)
                os.replace(self.json_store_path+".tmp", self.json_store_path)
            except Exception as e:
                logger.error(f"Error writing JSON!! (Main JSON file save was skipped) : {str(e)}")

-            # Write each watch to the disk (data in their own subdir) if it changed
-            for watch_uuid, watch in self.__data['watching'].items():
-                watch.save_data()
-
            self.needs_write = False
            self.needs_write_urgent = False

@@ -952,25 +924,3 @@ class ChangeDetectionStore:
                        f_d.write(zlib.compress(f_j.read()))
                        os.unlink(json_path)

-    # Move each 'watching' from a big JSON file to their own datafile in their data subdirectory
-    def update_20(self):
-        with open(self.json_store_path) as json_file:
-            data = json.load(json_file)
-            if data.get('watching'):
-                for uuid, watch in data['watching'].items():
-                    watch_data_dir = os.path.join(self.datastore_path, uuid)
-                    dest = os.path.join(watch_data_dir, WATCH_DB_JSON_FILENAME)
-
-                    try:
-                        if not os.path.isdir(watch_data_dir):
-                            logger.debug(f"> Creating data dir {watch_data_dir}")
-                            os.mkdir(watch_data_dir)
-                        with open(dest + '.tmp', 'w') as json_file:
-                            json.dump(watch, json_file, indent=2)
-                        os.replace(dest + '.tmp', dest)
-                        logger.info(f"Saved watch to {dest}")
-                    except Exception as e:
-                        logger.critical(f"Exception saving watch JSON {dest} - {e}")
-
-            self.data['watching'] = {}
-            self.scan_and_load_watches()
@@ -24,11 +24,13 @@
                              </ul>
                            </div>
                            <div class="notifications-wrapper">
-                              <a id="send-test-notification" class="pure-button button-secondary button-xsmall" >Send test notification</a>
+                              <a id="send-test-notification" class="pure-button button-secondary button-xsmall" >Send test notification</a> <div class="spinner"  style="display: none;"></div>
                            {% if emailprefix %}
                              <a id="add-email-helper" class="pure-button button-secondary button-xsmall" >Add email <img style="height: 1em; display: inline-block" src="{{url_for('static_content', group='images', filename='email.svg')}}" alt="Add an email address"> </a>
                            {% endif %}
                              <a href="{{url_for('notification_logs')}}" class="pure-button button-secondary button-xsmall" >Notification debug logs</a>
+                              <br>
+                                <div id="notification-test-log" style="display: none;"><span class="pure-form-message-inline">Processing..</span></div>
                            </div>
                        </div>
                        <div id="notification-customisation" class="pure-control-group">
@@ -45,9 +45,8 @@
            {% if extra_tab_content %}
            <li class="tab"><a href="#extras_tab">{{ extra_tab_content }}</a></li>
            {% endif %}
-            {% if playwright_enabled %}
            <li class="tab"><a id="browsersteps-tab" href="#browser-steps">Browser Steps</a></li>
-            {% endif %}
+        <!-- should goto extra forms? -->
            {% if watch['processor'] == 'text_json_diff' %}
            <li class="tab"><a id="visualselector-tab" href="#visualselector">Visual Filter Selector</a></li>
            <li class="tab" id="filters-and-triggers-tab"><a href="#filters-and-triggers">Filters &amp; Triggers</a></li>
@@ -199,8 +198,9 @@ Math: {{ 1 + 1 }}") }}
                    </div>
            </fieldset>
            </div>
-            {% if playwright_enabled %}
+
            <div class="tab-pane-inner" id="browser-steps">
+            {% if playwright_enabled %}
                <img class="beta-logo" src="{{url_for('static_content', group='images', filename='beta-logo.png')}}" alt="New beta functionality">
                <fieldset>
                    <div class="pure-control-group">
@@ -240,8 +240,16 @@ Math: {{ 1 + 1 }}") }}
                        </div>
                    </div>
                </fieldset>
+                {% else %}
+                    <span class="pure-form-message-inline">
+                        <p>Sorry, this functionality only works with Playwright/Chrome enabled watches.</p>
+                        <p>Enable the Playwright Chrome fetcher, or alternatively try our <a href="https://lemonade.changedetection.io/start">very affordable subscription based service</a>.</p>
+                        <p>This is because Selenium/WebDriver can not extract full page screenshots reliably.</p>
+                        <p>You may need to <a href="https://github.com/dgtlmoon/changedetection.io/blob/09ebc6ec6338545bdd694dc6eee57f2e9d2b8075/docker-compose.yml#L31">Enable playwright environment variable</a> and uncomment the <strong>sockpuppetbrowser</strong> from the docker-compose.yml file.</p>
+                    </span>
+                {% endif %}
            </div>
-            {% endif %}
+

            <div class="tab-pane-inner" id="notifications">
                <fieldset>
@@ -493,6 +501,7 @@ keyword") }}
                                <p>Sorry, this functionality only works with Playwright/Chrome enabled watches.</p>
                                <p>Enable the Playwright Chrome fetcher, or alternatively try our <a href="https://lemonade.changedetection.io/start">very affordable subscription based service</a>.</p>
                                <p>This is because Selenium/WebDriver can not extract full page screenshots reliably.</p>
+                                <p>You may need to <a href="https://github.com/dgtlmoon/changedetection.io/blob/09ebc6ec6338545bdd694dc6eee57f2e9d2b8075/docker-compose.yml#L31">Enable playwright environment variable</a> and uncomment the <strong>sockpuppetbrowser</strong> from the docker-compose.yml file.</p>
                            </span>
                        {% endif %}
                    </div>
@@ -108,7 +108,8 @@
                    {% else %}
                    <a class="state-on" href="{{url_for('index', op='pause', uuid=watch.uuid, tag=active_tag_uuid)}}"><img src="{{url_for('static_content', group='images', filename='play.svg')}}" alt="UnPause checks" title="UnPause checks" class="icon icon-unpause" ></a>
                    {% endif %}
-                    <a class="link-mute state-{{'on' if watch.notification_muted else 'off'}}" href="{{url_for('index', op='mute', uuid=watch.uuid, tag=active_tag_uuid)}}"><img src="{{url_for('static_content', group='images', filename='bell-off.svg')}}" alt="Mute notifications" title="Mute notifications" class="icon icon-mute" ></a>
+                    {% set mute_label = 'UnMute notification' if watch.notification_muted else 'Mute notification' %}
+                    <a class="link-mute state-{{'on' if watch.notification_muted else 'off'}}" href="{{url_for('index', op='mute', uuid=watch.uuid, tag=active_tag_uuid)}}"><img src="{{url_for('static_content', group='images', filename='bell-off.svg')}}" alt="{{ mute_label }}" title="{{ mute_label }}" class="icon icon-mute" ></a>
                </td>
                <td class="title-col inline">{{watch.title if watch.title is not none and watch.title|length > 0 else watch.url}}
                    <a class="external" target="_blank" rel="noopener" href="{{ watch.link.replace('source:','') }}"></a>
@@ -34,7 +34,7 @@ def test_execute_custom_js(client, live_server, measure_memory_usage):
    assert b"unpaused" in res.data
    wait_for_all_checks(client)

-    uuid = extract_UUID_from_client(client)
+    uuid = next(iter(live_server.app.config['DATASTORE'].data['watching']))
    assert live_server.app.config['DATASTORE'].data['watching'][uuid].history_n >= 1, "Watch history had atleast 1 (everything fetched OK)"

    assert b"This text should be removed" not in res.data
@@ -48,7 +48,7 @@ def test_noproxy_option(client, live_server, measure_memory_usage):
        follow_redirects=True
    )
    assert b"Watch added in Paused state, saving will unpause" in res.data
-    uuid = extract_UUID_from_client(client)
+    uuid = next(iter(live_server.app.config['DATASTORE'].data['watching']))
    res = client.get(
        url_for("edit_page", uuid=uuid, unpause_on_save=1))
    assert b'No proxy' in res.data
@@ -81,7 +81,7 @@ def test_socks5(client, live_server, measure_memory_usage):
    assert "Awesome, you made it".encode('utf-8') in res.data

    # PROXY CHECKER WIDGET CHECK - this needs more checking
-    uuid = extract_UUID_from_client(client)
+    uuid = next(iter(live_server.app.config['DATASTORE'].data['watching']))

    res = client.get(
        url_for("check_proxies.start_check", uuid=uuid),
@@ -1,9 +1,9 @@
 #!/usr/bin/env python3
+
 import os.path
-import time
+
 from flask import url_for
 from .util import live_server_setup, wait_for_all_checks, wait_for_notification_endpoint_output
-from changedetectionio import html_tools


 def set_original(excluding=None, add_line=None):
@@ -44,7 +44,6 @@ def set_modified_response():

    return None

-
 def is_valid_uuid(val):
    try:
        uuid.UUID(str(val))
@@ -56,8 +55,9 @@ def is_valid_uuid(val):
 def test_setup(client, live_server, measure_memory_usage):
    live_server_setup(live_server)

+
 def test_api_simple(client, live_server, measure_memory_usage):
-    #live_server_setup(live_server)
+#    live_server_setup(live_server)

    api_key = extract_api_key_from_UI(client)

@@ -129,6 +129,9 @@ def test_api_simple(client, live_server, measure_memory_usage):
    assert after_recheck_info['last_checked'] != before_recheck_info['last_checked']
    assert after_recheck_info['last_changed'] != 0

+    # #2877 When run in a slow fetcher like playwright etc
+    assert after_recheck_info['last_changed'] ==  after_recheck_info['last_checked']
+
    # Check history index list
    res = client.get(
        url_for("watchhistory", uuid=watch_uuid),
@@ -99,7 +99,7 @@ def test_check_ldjson_price_autodetect(client, live_server, measure_memory_usage
    assert b'ldjson-price-track-offer' in res.data

    # Accept it
-    uuid = extract_UUID_from_client(client)
+    uuid = next(iter(live_server.app.config['DATASTORE'].data['watching']))
    #time.sleep(1)
    client.get(url_for('price_data_follower.accept', uuid=uuid, follow_redirects=True))
    client.get(url_for("form_watch_checknow"), follow_redirects=True)
@@ -2,7 +2,6 @@

 import time
 from flask import url_for
-from urllib.request import urlopen
 from .util import set_original_response, set_modified_response, live_server_setup, wait_for_all_checks, extract_rss_token_from_UI, \
    extract_UUID_from_client

@@ -69,7 +68,7 @@ def test_check_basic_change_detection_functionality(client, live_server, measure

    wait_for_all_checks(client)

-    uuid = extract_UUID_from_client(client)
+    uuid = next(iter(live_server.app.config['DATASTORE'].data['watching']))

    # Check the 'get latest snapshot works'
    res = client.get(url_for("watch_get_latest_html", uuid=uuid))
@@ -40,7 +40,7 @@ def test_check_encoding_detection(client, live_server, measure_memory_usage):


    # Content type recording worked
-    uuid = extract_UUID_from_client(client)
+    uuid = next(iter(live_server.app.config['DATASTORE'].data['watching']))
    assert live_server.app.config['DATASTORE'].data['watching'][uuid]['content-type'] == "text/html"

    res = client.get(
@@ -51,7 +51,7 @@ def run_filter_test(client, live_server, content_filter):
    assert b"1 Imported" in res.data
    wait_for_all_checks(client)

-    uuid = extract_UUID_from_client(client)
+    uuid = next(iter(live_server.app.config['DATASTORE'].data['watching']))

    assert live_server.app.config['DATASTORE'].data['watching'][uuid]['consecutive_filter_failures'] == 0, "No filter = No filter failure"

@@ -288,7 +288,7 @@ def test_clone_tag_on_import(client, live_server, measure_memory_usage):
    assert b'test-tag' in res.data
    assert b'another-tag' in res.data

-    watch_uuid = extract_UUID_from_client(client)
+    watch_uuid = next(iter(live_server.app.config['DATASTORE'].data['watching']))
    res = client.get(url_for("form_clone", uuid=watch_uuid), follow_redirects=True)

    assert b'Cloned' in res.data
@@ -315,7 +315,7 @@ def test_clone_tag_on_quickwatchform_add(client, live_server, measure_memory_usa
    assert b'test-tag' in res.data
    assert b'another-tag' in res.data

-    watch_uuid = extract_UUID_from_client(client)
+    watch_uuid = next(iter(live_server.app.config['DATASTORE'].data['watching']))
    res = client.get(url_for("form_clone", uuid=watch_uuid), follow_redirects=True)

    assert b'Cloned' in res.data
@@ -8,19 +8,11 @@ from flask import url_for
 from .util import live_server_setup, wait_for_all_checks
 from urllib.parse import urlparse, parse_qs

-from ..model.Watch import WATCH_DB_JSON_FILENAME
-
-
 def test_consistent_history(client, live_server, measure_memory_usage):
    live_server_setup(live_server)

-    import glob
    r = range(1, 30)

-    # incase some exist from a previous test
-    for f in glob.glob(f"{live_server.app.config['DATASTORE'].datastore_path}/*/{WATCH_DB_JSON_FILENAME}", recursive=True):
-        os.unlink(f)
-
    for one in r:
        test_url = url_for('test_endpoint', content_type="text/html", content=str(one), _external=True)
        res = client.post(
@@ -52,17 +44,11 @@ def test_consistent_history(client, live_server, measure_memory_usage):
    with open(json_db_file, 'r') as f:
        json_obj = json.load(f)

-
-    found_db_jsons = glob.glob(f"{live_server.app.config['DATASTORE'].datastore_path}/*/{WATCH_DB_JSON_FILENAME}", recursive=True)
    # assert the right amount of watches was found in the JSON
-    assert len(found_db_jsons) == len(r), "Correct number of watches was found in the JSON"
+    assert len(json_obj['watching']) == len(r), "Correct number of watches was found in the JSON"

    # each one should have a history.txt containing just one line
-    for json_db_file in found_db_jsons:
-
-        directory_path = os.path.dirname(json_db_file)
-        w = os.path.basename(directory_path)
-
+    for w in json_obj['watching'].keys():
        history_txt_index_file = os.path.join(live_server.app.config['DATASTORE'].datastore_path, w, 'history.txt')
        assert os.path.isfile(history_txt_index_file), f"History.txt should exist where I expect it at {history_txt_index_file}"

@@ -72,21 +58,22 @@ def test_consistent_history(client, live_server, measure_memory_usage):
            assert len(tmp_history) == 1, "History.txt should contain 1 line"

        # Should be two files,. the history.txt , and the snapshot.txt
-        files_in_watch_dir = os.listdir(os.path.join(live_server.app.config['DATASTORE'].datastore_path,w))
+        files_in_watch_dir = os.listdir(os.path.join(live_server.app.config['DATASTORE'].datastore_path,
+                                                     w))
        # Find the snapshot one
-#        for fname in files_in_watch_dir:
-#            if fname != 'history.txt' and 'html' not in fname and fname != WATCH_DB_JSON_FILENAME:
-#                # contents should match what we requested as content returned from the test url
-#                with open(os.path.join(live_server.app.config['DATASTORE'].datastore_path, w, fname), 'r') as snapshot_f:
-#                    contents = snapshot_f.read()
-#                    watch_url = json_obj['watching'][w]['url']
-#                    u = urlparse(watch_url)
-#                    q = parse_qs(u[4])
-#                    assert q['content'][0] == contents.strip(), f"Snapshot file {fname} should contain {q['content'][0]}"
+        for fname in files_in_watch_dir:
+            if fname != 'history.txt' and 'html' not in fname:
+                # contents should match what we requested as content returned from the test url
+                with open(os.path.join(live_server.app.config['DATASTORE'].datastore_path, w, fname), 'r') as snapshot_f:
+                    contents = snapshot_f.read()
+                    watch_url = json_obj['watching'][w]['url']
+                    u = urlparse(watch_url)
+                    q = parse_qs(u[4])
+                    assert q['content'][0] == contents.strip(), f"Snapshot file {fname} should contain {q['content'][0]}"



-        assert len(files_in_watch_dir) == 4, "Should be just four files in the dir, html.br snapshot, history.txt, watch.json and the extracted text snapshot"
+        assert len(files_in_watch_dir) == 3, "Should be just three files in the dir, html.br snapshot, history.txt and the extracted text snapshot"


    json_db_file = os.path.join(live_server.app.config['DATASTORE'].datastore_path, 'url-watches.json')
@@ -36,7 +36,7 @@ def test_ignore(client, live_server, measure_memory_usage):

    # Give the thread time to pick it up
    wait_for_all_checks(client)
-    uuid = extract_UUID_from_client(client)
+    uuid = next(iter(live_server.app.config['DATASTORE'].data['watching']))
    # use the highlighter endpoint
    res = client.post(
        url_for("highlight_submit_ignore_url", uuid=uuid),
@@ -514,3 +514,15 @@ def test_check_jq_ext_filter(client, live_server, measure_memory_usage):
 def test_check_jqraw_ext_filter(client, live_server, measure_memory_usage):
    if jq_support:
        check_json_ext_filter('jq:.[] | select(.status | contains("Sold"))', client, live_server)
+
+def test_jsonpath_BOM_utf8(client, live_server, measure_memory_usage):
+    from .. import html_tools
+
+    # JSON string with BOM and correct double-quoted keys
+    json_str = '\ufeff{"name": "José", "emoji": "😊", "language": "中文", "greeting": "Привет"}'
+
+    # See that we can find the second <script> one, which is not broken, and matches our filter
+    text = html_tools.extract_json_as_string(json_str, "json:$.name")
+    assert text == '"José"'
+
+    
@@ -29,7 +29,7 @@ def test_content_filter_live_preview(client, live_server, measure_memory_usage):
        data={"url": test_url, "tags": ''},
        follow_redirects=True
    )
-    uuid = extract_UUID_from_client(client)
+    uuid = next(iter(live_server.app.config['DATASTORE'].data['watching']))
    res = client.post(
        url_for("edit_page", uuid=uuid),
        data={
@@ -48,7 +48,7 @@ def test_check_basic_change_detection_functionality(client, live_server, measure
    #####################
    client.post(
        url_for("settings_page"),
-        data={"application-empty_pages_are_a_change": "",
+        data={"application-empty_pages_are_a_change": "", # default, OFF, they are NOT a change
              "requests-time_between_check-minutes": 180,
              'application-fetch_backend': "html_requests"},
        follow_redirects=True
@@ -66,6 +66,14 @@ def test_check_basic_change_detection_functionality(client, live_server, measure
    res = client.get(url_for("index"))
    assert b'unviewed' not in res.data

+    uuid = next(iter(live_server.app.config['DATASTORE'].data['watching']))
+    watch = live_server.app.config['DATASTORE'].data['watching'][uuid]
+
+    assert watch.last_changed == 0
+    assert watch['last_checked'] != 0
+
+
+

    # ok now do the opposite

@@ -92,6 +100,10 @@ def test_check_basic_change_detection_functionality(client, live_server, measure
    # A totally zero byte (#2528) response should also not trigger an error
    set_zero_byte_response()
    client.get(url_for("form_watch_checknow"), follow_redirects=True)
+
+    # 2877
+    assert watch.last_changed == watch['last_checked']
+
    wait_for_all_checks(client)
    res = client.get(url_for("index"))
    assert b'unviewed' in res.data # A change should have registered because empty_pages_are_a_change is ON
@@ -6,7 +6,7 @@ from flask import url_for
 from loguru import logger

 from .util import set_original_response, set_modified_response, set_more_modified_response, live_server_setup, wait_for_all_checks, \
-    set_longer_modified_response
+    set_longer_modified_response, get_index
 from . util import  extract_UUID_from_client
 import logging
 import base64
@@ -29,7 +29,7 @@ def test_check_notification(client, live_server, measure_memory_usage):

    # Re 360 - new install should have defaults set
    res = client.get(url_for("settings_page"))
-    notification_url = url_for('test_notification_endpoint', _external=True).replace('http', 'json')
+    notification_url = url_for('test_notification_endpoint', _external=True).replace('http', 'json')+"?status_code=204"

    assert default_notification_body.encode() in res.data
    assert default_notification_title.encode() in res.data
@@ -76,7 +76,7 @@ def test_check_notification(client, live_server, measure_memory_usage):
    testimage_png = 'iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mNkYAAAAAYAAjCB0C8AAAAASUVORK5CYII='


-    uuid = extract_UUID_from_client(client)
+    uuid = next(iter(live_server.app.config['DATASTORE'].data['watching']))
    datastore = 'test-datastore'
    with open(os.path.join(datastore, str(uuid), 'last-screenshot.png'), 'wb') as f:
        f.write(base64.b64decode(testimage_png))
@@ -135,7 +135,14 @@ def test_check_notification(client, live_server, measure_memory_usage):

    # Trigger a check
    client.get(url_for("form_watch_checknow"), follow_redirects=True)
+    wait_for_all_checks(client)
    time.sleep(3)
+
+    # Check no errors were recorded
+    res = client.get(url_for("index"))
+    assert b'notification-error' not in res.data
+
+
    # Verify what was sent as a notification, this file should exist
    with open("test-datastore/notification.txt", "r") as f:
        notification_submission = f.read()
@@ -284,7 +291,7 @@ def test_notification_custom_endpoint_and_jinja2(client, live_server, measure_me
    # CUSTOM JSON BODY CHECK for POST://
    set_original_response()
    # https://github.com/caronc/apprise/wiki/Notify_Custom_JSON#header-manipulation
-    test_notification_url = url_for('test_notification_endpoint', _external=True).replace('http://', 'post://')+"?xxx={{ watch_url }}&+custom-header=123&+second=hello+world%20%22space%22"
+    test_notification_url = url_for('test_notification_endpoint', _external=True).replace('http://', 'post://')+"?status_code=204&xxx={{ watch_url }}&+custom-header=123&+second=hello+world%20%22space%22"

    res = client.post(
        url_for("settings_page"),
@@ -319,6 +326,11 @@ def test_notification_custom_endpoint_and_jinja2(client, live_server, measure_me

    time.sleep(2) # plus extra delay for notifications to fire

+
+    # Check no errors were recorded, because we asked for 204 which is slightly uncommon but is still OK
+    res = get_index(client)
+    assert b'notification-error' not in res.data
+
    with open("test-datastore/notification.txt", 'r') as f:
        x = f.read()
        j = json.loads(x)
@@ -360,7 +372,10 @@ def test_global_send_test_notification(client, live_server, measure_memory_usage
    #live_server_setup(live_server)
    set_original_response()
    if os.path.isfile("test-datastore/notification.txt"):
-        os.unlink("test-datastore/notification.txt")
+        os.unlink("test-datastore/notification.txt") \
+
+    # 1995 UTF-8 content should be encoded
+    test_body = 'change detection is cool 网站监测 内容更新了'

    # otherwise other settings would have already existed from previous tests in this file
    res = client.post(
@@ -368,8 +383,7 @@ def test_global_send_test_notification(client, live_server, measure_memory_usage
        data={
            "application-fetch_backend": "html_requests",
            "application-minutes_between_check": 180,
-            #1995 UTF-8 content should be encoded
-            "application-notification_body": 'change detection is cool 网站监测 内容更新了',
+            "application-notification_body": test_body,
            "application-notification_format": default_notification_format,
            "application-notification_urls": "",
            "application-notification_title": "New ChangeDetection.io Notification - {{ watch_url }}",
@@ -399,12 +413,10 @@ def test_global_send_test_notification(client, live_server, measure_memory_usage
    assert res.status_code != 400
    assert res.status_code != 500

-    # Give apprise time to fire
-    time.sleep(4)

    with open("test-datastore/notification.txt", 'r') as f:
        x = f.read()
-        assert 'change detection is cool 网站监测 内容更新了' in x
+        assert test_body in x

    os.unlink("test-datastore/notification.txt")

@@ -2,7 +2,7 @@ import json
 import os
 import time
 from flask import url_for
-from .util import live_server_setup, wait_for_all_checks, extract_UUID_from_client
+from . util import set_original_response, set_modified_response, live_server_setup, wait_for_all_checks, extract_UUID_from_client

 def test_setup(live_server):
    live_server_setup(live_server)
@@ -74,8 +74,8 @@ def test_headers_in_request(client, live_server, measure_memory_usage):
    # Re #137 -  It should have only one set of headers entered
    watches_with_headers = 0
    for k, watch in client.application.config.get('DATASTORE').data.get('watching').items():
-        if (len(watch['headers'])):
-            watches_with_headers += 1
+            if (len(watch['headers'])):
+                watches_with_headers += 1
    assert watches_with_headers == 1

    # 'server' http header was automatically recorded
@@ -156,10 +156,11 @@ def test_body_in_request(client, live_server, measure_memory_usage):
    assert b"1 Imported" in res.data

    watches_with_body = 0
-
-    for k, watch in client.application.config.get('DATASTORE').data.get('watching').items():
-        if watch['body'] == body_value:
-            watches_with_body += 1
+    with open('test-datastore/url-watches.json') as f:
+        app_struct = json.load(f)
+        for uuid in app_struct['watching']:
+            if app_struct['watching'][uuid]['body']==body_value:
+                watches_with_body += 1

    # Should be only one with body set
    assert watches_with_body==1
@@ -243,9 +244,11 @@ def test_method_in_request(client, live_server, measure_memory_usage):
    wait_for_all_checks(client)

    watches_with_method = 0
-    for k, watch in client.application.config.get('DATASTORE').data.get('watching').items():
-        if watch['method'] == 'PATCH':
-            watches_with_method += 1
+    with open('test-datastore/url-watches.json') as f:
+        app_struct = json.load(f)
+        for uuid in app_struct['watching']:
+            if app_struct['watching'][uuid]['method'] == 'PATCH':
+                watches_with_method += 1

    # Should be only one with method set to PATCH
    assert watches_with_method == 1
@@ -370,13 +373,14 @@ def test_headers_textfile_in_request(client, live_server, measure_memory_usage):
    wait_for_all_checks(client)

    with open('test-datastore/headers-testtag.txt', 'w') as f:
-        f.write("tag-header: test")
+        f.write("tag-header: test\r\nurl-header: http://example.com")

    with open('test-datastore/headers.txt', 'w') as f:
-        f.write("global-header: nice\r\nnext-global-header: nice")
+        f.write("global-header: nice\r\nnext-global-header: nice\r\nurl-header-global: http://example.com/global")

-    with open('test-datastore/' + extract_UUID_from_client(client) + '/headers.txt', 'w') as f:
-        f.write("watch-header: nice")
+    uuid = next(iter(live_server.app.config['DATASTORE'].data['watching']))
+    with open(f'test-datastore/{uuid}/headers.txt', 'w') as f:
+        f.write("watch-header: nice\r\nurl-header-watch: http://example.com/watch")

    wait_for_all_checks(client)
    client.get(url_for("form_watch_checknow"), follow_redirects=True)
@@ -407,6 +411,9 @@ def test_headers_textfile_in_request(client, live_server, measure_memory_usage):
    assert b"Xxx:ooo" in res.data
    assert b"Watch-Header:nice" in res.data
    assert b"Tag-Header:test" in res.data
+    assert b"Url-Header:http://example.com" in res.data
+    assert b"Url-Header-Global:http://example.com/global" in res.data
+    assert b"Url-Header-Watch:http://example.com/watch" in res.data

    # Check the custom UA from system settings page made it through
    if os.getenv('PLAYWRIGHT_DRIVER_URL'):
@@ -189,6 +189,17 @@ def _run_test_minmax_limit(client, extra_watch_edit_form):

    client.get(url_for("mark_all_viewed"))

+
+    # 2715 - Price detection (once it crosses the "lower" threshold) again with a lower price - should trigger again!
+    set_original_response(props_markup=instock_props[0], price='820.45')
+    res = client.get(url_for("form_watch_checknow"), follow_redirects=True)
+    assert b'1 watches queued for rechecking.' in res.data
+    wait_for_all_checks(client)
+    res = client.get(url_for("index"))
+    assert b'820.45' in res.data
+    assert b'unviewed' in res.data
+    client.get(url_for("mark_all_viewed"))
+
    # price changed to something MORE than max (1100.10), SHOULD be a change
    set_original_response(props_markup=instock_props[0], price='1890.45')
    client.get(url_for("form_watch_checknow"), follow_redirects=True)
@@ -203,7 +214,7 @@ def _run_test_minmax_limit(client, extra_watch_edit_form):


 def test_restock_itemprop_minmax(client, live_server):
-#    live_server_setup(live_server)
+    #live_server_setup(live_server)
    extras = {
        "restock_settings-follow_price_changes": "y",
        "restock_settings-price_change_min": 900.0,
@@ -369,7 +380,7 @@ def test_change_with_notification_values(client, live_server):

    ## Now test the "SEND TEST NOTIFICATION" is working
    os.unlink("test-datastore/notification.txt")
-    uuid = extract_UUID_from_client(client)
+    uuid = next(iter(live_server.app.config['DATASTORE'].data['watching']))
    res = client.post(url_for("ajax_callback_send_notification_test", watch_uuid=uuid), data={}, follow_redirects=True)
    time.sleep(5)
    assert os.path.isfile("test-datastore/notification.txt"), "Notification received"
@@ -132,7 +132,7 @@ def test_rss_xpath_filtering(client, live_server, measure_memory_usage):
    )
    assert b"Watch added in Paused state, saving will unpause" in res.data

-    uuid = extract_UUID_from_client(client)
+    uuid = next(iter(live_server.app.config['DATASTORE'].data['watching']))
    res = client.post(
        url_for("edit_page", uuid=uuid, unpause_on_save=1),
        data={
@@ -39,7 +39,7 @@ def test_check_basic_scheduler_functionality(client, live_server, measure_memory

    assert b"1 Imported" in res.data
    wait_for_all_checks(client)
-    uuid = extract_UUID_from_client(client)
+    uuid = next(iter(live_server.app.config['DATASTORE'].data['watching']))

    # Setup all the days of the weeks using XXX as the placeholder for monday/tuesday/etc

@@ -104,7 +104,7 @@ def test_check_basic_global_scheduler_functionality(client, live_server, measure

    assert b"1 Imported" in res.data
    wait_for_all_checks(client)
-    uuid = extract_UUID_from_client(client)
+    uuid = next(iter(live_server.app.config['DATASTORE'].data['watching']))

    # Setup all the days of the weeks using XXX as the placeholder for monday/tuesday/etc

@@ -0,0 +1,64 @@
+#!/usr/bin/env python3
+
+# run from dir above changedetectionio/ dir
+# python3 -m unittest changedetectionio.tests.unit.test_semver
+
+import re
+import unittest
+
+
+# The SEMVER regex
+SEMVER_REGEX = r"^(?P<major>0|[1-9]\d*)\.(?P<minor>0|[1-9]\d*)\.(?P<patch>0|[1-9]\d*)(?:-(?P<prerelease>(?:0|[1-9]\d*|\d*[a-zA-Z-][0-9a-zA-Z-]*)(?:\.(?:0|[1-9]\d*|\d*[a-zA-Z-][0-9a-zA-Z-]*))*))?(?:\+(?P<buildmetadata>[0-9a-zA-Z-]+(?:\.[0-9a-zA-Z-]+)*))?$"
+
+# Compile the regex
+semver_pattern = re.compile(SEMVER_REGEX)
+
+class TestSemver(unittest.TestCase):
+    def test_valid_versions(self):
+        """Test valid semantic version strings"""
+        valid_versions = [
+            "1.0.0",
+            "0.1.0",
+            "0.0.1",
+            "1.0.0-alpha",
+            "1.0.0-alpha.1",
+            "1.0.0-0.3.7",
+            "1.0.0-x.7.z.92",
+            "1.0.0-alpha+001",
+            "1.0.0+20130313144700",
+            "1.0.0-beta+exp.sha.5114f85"
+        ]
+        for version in valid_versions:
+            with self.subTest(version=version):
+                self.assertIsNotNone(semver_pattern.match(version), f"Version {version} should be valid")
+
+    def test_invalid_versions(self):
+        """Test invalid semantic version strings"""
+        invalid_versions = [
+            "0.48.06",
+            "1.0",
+            "1.0.0-",
+# Seems to pass the semver.org regex?
+#            "1.0.0-alpha-",
+            "1.0.0+",
+            "1.0.0-alpha+",
+            "1.0.0-",
+            "01.0.0",
+            "1.01.0",
+            "1.0.01",
+            ".1.0.0",
+            "1..0.0"
+        ]
+        for version in invalid_versions:
+            with self.subTest(version=version):
+                res = semver_pattern.match(version)
+                self.assertIsNone(res, f"Version '{version}' should be invalid")
+
+    def test_our_version(self):
+        from changedetectionio import get_version
+        our_version = get_version()
+        self.assertIsNotNone(semver_pattern.match(our_version), f"Our version '{our_version}' should be a valid SEMVER string")
+
+
+if __name__ == '__main__':
+    unittest.main()
@@ -6,7 +6,6 @@
 import unittest
 import os

-from changedetectionio import store
 from changedetectionio.model import Watch

 # mostly
@@ -14,8 +13,7 @@ class TestDiffBuilder(unittest.TestCase):

    def test_watch_get_suggested_from_diff_timestamp(self):
        import uuid as uuid_builder
-        datastore = store.ChangeDetectionStore(datastore_path='/tmp')
-        watch = Watch.model(__datastore=datastore, default={})
+        watch = Watch.model(datastore_path='/tmp', default={})
        watch.ensure_data_dir_exists()


@@ -51,7 +49,7 @@ class TestDiffBuilder(unittest.TestCase):
        assert p == "109", "Correct when its the same time"

        # new empty one
-        watch = Watch.model(__datastore=datastore, default={})
+        watch = Watch.model(datastore_path='/tmp', default={})
        p = watch.get_from_version_based_on_last_viewed
        assert p == None, "None when no history available"

@@ -63,6 +61,5 @@ class TestDiffBuilder(unittest.TestCase):
        p = watch.get_from_version_based_on_last_viewed
        assert p == "100", "Correct with only one history snapshot"

-        datastore.stop_thread = True
 if __name__ == '__main__':
    unittest.main()
@@ -76,11 +76,18 @@ def set_more_modified_response():
    return None


+def set_empty_text_response():
+    test_return_data = """<html><body></body></html>"""
+
+    with open("test-datastore/endpoint-content.txt", "w") as f:
+        f.write(test_return_data)
+
+    return None
+
 def wait_for_notification_endpoint_output():
    '''Apprise can take a few seconds to fire'''
    #@todo - could check the apprise object directly instead of looking for this file
    from os.path import isfile
-
    for i in range(1, 20):
        time.sleep(1)
        if isfile("test-datastore/notification.txt"):
@@ -216,9 +223,10 @@ def live_server_setup(live_server):
    def test_method():
        return request.method

-    # Where we POST to as a notification
-    @live_server.app.route('/test_notification_endpoint', methods=['POST', 'GET'])
+    # Where we POST to as a notification, also use a space here to test URL escaping is OK across all tests that use this. ( #2868 )
+    @live_server.app.route('/test_notification endpoint', methods=['POST', 'GET'])
    def test_notification_endpoint():
+
        with open("test-datastore/notification.txt", "wb") as f:
            # Debug method, dump all POST to file also, used to prove #65
            data = request.stream.read()
@@ -236,8 +244,11 @@ def live_server_setup(live_server):
                f.write(request.content_type)

        print("\n>> Test notification endpoint was hit.\n", data)
-        return "Text was set"

+        content = "Text was set"
+        status_code = request.args.get('status_code',200)
+        resp = make_response(content, status_code)
+        return resp

    # Just return the verb in the request
    @live_server.app.route('/test-basicauth', methods=['GET'])
@@ -274,15 +285,43 @@ def live_server_setup(live_server):
            <p id="remove">This text should be removed</p>
              <form onsubmit="event.preventDefault();">
            <!-- obfuscated text so that we dont accidentally get a false positive due to conversion of the source :) --->
-                <button name="test-button" onclick="getElementById('remove').remove();getElementById('some-content').innerHTML = atob('SSBzbWVsbCBKYXZhU2NyaXB0IGJlY2F1c2UgdGhlIGJ1dHRvbiB3YXMgcHJlc3NlZCE=')">Click here</button>
-                <div id=some-content></div>
+                <button name="test-button" onclick="
+                getElementById('remove').remove();
+                getElementById('some-content').innerHTML = atob('SSBzbWVsbCBKYXZhU2NyaXB0IGJlY2F1c2UgdGhlIGJ1dHRvbiB3YXMgcHJlc3NlZCE=');
+                getElementById('reflect-text').innerHTML = getElementById('test-input-text').value;
+                ">Click here</button>
+                
+                <div id="some-content"></div>
+                
                <pre>
                {header_text.lower()}
                </pre>
-              </body>
+                
+                <br>
+                <!-- used for testing that the jinja2 compiled here --->
+                <input type="text" value="" id="test-input-text" /><br>
+                <div id="reflect-text">Waiting to reflect text from #test-input-text here</div>
+              </form>
+                
+           </body>
         </html>""", 200)
        resp.headers['Content-Type'] = 'text/html'
        return resp

    live_server.start()

+def get_index(client):
+    import inspect
+    # Get the caller's frame (parent function)
+    frame = inspect.currentframe()
+    caller_frame = frame.f_back  # Go back to the caller's frame
+    caller_name = caller_frame.f_code.co_name
+    caller_line = caller_frame.f_lineno
+
+    print(f"Called by: {caller_name}, Line: {caller_line}")
+
+    res = client.get(url_for("index"))
+    with open(f"test-datastore/index-{caller_name}-{caller_line}.html", 'wb') as f:
+        f.write(res.data)
+
+    return res
@@ -2,14 +2,16 @@

 import os
 from flask import url_for
-from ..util import live_server_setup, wait_for_all_checks, extract_UUID_from_client
+from ..util import live_server_setup, wait_for_all_checks, get_index

-def test_setup(client, live_server, measure_memory_usage):
+def test_setup(client, live_server):
    live_server_setup(live_server)


 # Add a site in paused mode, add an invalid filter, we should still have visual selector data ready
 def test_visual_selector_content_ready(client, live_server, measure_memory_usage):
+    live_server.stop()
+    live_server.start()

    import os
    import json
@@ -27,7 +29,7 @@ def test_visual_selector_content_ready(client, live_server, measure_memory_usage
        follow_redirects=True
    )
    assert b"Watch added in Paused state, saving will unpause" in res.data
-    uuid = extract_UUID_from_client(client)
+    uuid = next(iter(live_server.app.config['DATASTORE'].data['watching']))
    res = client.post(
        url_for("edit_page", uuid=uuid, unpause_on_save=1),
        data={
@@ -87,7 +89,9 @@ def test_visual_selector_content_ready(client, live_server, measure_memory_usage

 def test_basic_browserstep(client, live_server, measure_memory_usage):

-    #live_server_setup(live_server)
+    live_server.stop()
+    live_server.start()
+
    assert os.getenv('PLAYWRIGHT_DRIVER_URL'), "Needs PLAYWRIGHT_DRIVER_URL set for this test"

    test_url = url_for('test_interactive_html_endpoint', _external=True)
@@ -108,9 +112,13 @@ def test_basic_browserstep(client, live_server, measure_memory_usage):
            "url": test_url,
            "tags": "",
            'fetch_backend': "html_webdriver",
-            'browser_steps-0-operation': 'Click element',
-            'browser_steps-0-selector': 'button[name=test-button]',
-            'browser_steps-0-optional_value': '',
+            'browser_steps-0-operation': 'Enter text in field',
+            'browser_steps-0-selector': '#test-input-text',
+            # Should get set to the actual text (jinja2 rendered)
+            'browser_steps-0-optional_value': "Hello-Jinja2-{% now  'Europe/Berlin', '%Y-%m-%d' %}",
+            'browser_steps-1-operation': 'Click element',
+            'browser_steps-1-selector': 'button[name=test-button]',
+            'browser_steps-1-optional_value': '',
            # For now, cookies doesnt work in headers because it must be a full cookiejar object
            'headers': "testheader: yes\buser-agent: MyCustomAgent",
        },
@@ -119,7 +127,7 @@ def test_basic_browserstep(client, live_server, measure_memory_usage):
    assert b"unpaused" in res.data
    wait_for_all_checks(client)

-    uuid = extract_UUID_from_client(client)
+    uuid = next(iter(live_server.app.config['DATASTORE'].data['watching']))
    assert live_server.app.config['DATASTORE'].data['watching'][uuid].history_n >= 1, "Watch history had atleast 1 (everything fetched OK)"

    assert b"This text should be removed" not in res.data
@@ -132,13 +140,32 @@ def test_basic_browserstep(client, live_server, measure_memory_usage):
    assert b"This text should be removed" not in res.data
    assert b"I smell JavaScript because the button was pressed" in res.data

+    assert b'Hello-Jinja2-20' in res.data
+
    assert b"testheader: yes" in res.data
    assert b"user-agent: mycustomagent" in res.data
+    live_server.stop()
+
+def test_non_200_errors_report_browsersteps(client, live_server):
+
+    live_server.stop()
+    live_server.start()

    four_o_four_url =  url_for('test_endpoint', status_code=404, _external=True)
    four_o_four_url = four_o_four_url.replace('localhost.localdomain', 'cdio')
    four_o_four_url = four_o_four_url.replace('localhost', 'cdio')

+    res = client.post(
+        url_for("form_quick_watch_add"),
+        data={"url": four_o_four_url, "tags": '', 'edit_and_watch_submit_button': 'Edit > Watch'},
+        follow_redirects=True
+    )
+
+    assert b"Watch added in Paused state, saving will unpause" in res.data
+    assert os.getenv('PLAYWRIGHT_DRIVER_URL'), "Needs PLAYWRIGHT_DRIVER_URL set for this test"
+
+    uuid = next(iter(live_server.app.config['DATASTORE'].data['watching']))
+
    # now test for 404 errors
    res = client.post(
        url_for("edit_page", uuid=uuid, unpause_on_save=1),
@@ -153,12 +180,14 @@ def test_basic_browserstep(client, live_server, measure_memory_usage):
        follow_redirects=True
    )
    assert b"unpaused" in res.data
+
    wait_for_all_checks(client)

-    res = client.get(url_for("index"))
+    res = get_index(client)
+
    assert b'Error - 404' in res.data

    client.get(
        url_for("form_delete", uuid="all"),
        follow_redirects=True
-    )
+    )
@@ -243,7 +243,6 @@ class update_worker(threading.Thread):
                os.unlink(full_path)

    def run(self):
-        now = time.time()
        
        while not self.app.config.exit.is_set():
            update_handler = None
@@ -254,6 +253,7 @@ class update_worker(threading.Thread):
                pass

            else:
+                fetch_start_time = time.time()
                uuid = queued_item_data.item.get('uuid')
                self.current_uuid = uuid
                if uuid in list(self.datastore.data['watching'].keys()) and self.datastore.data['watching'][uuid].get('url'):
@@ -268,7 +268,6 @@ class update_worker(threading.Thread):
                    watch = self.datastore.data['watching'].get(uuid)

                    logger.info(f"Processing watch UUID {uuid} Priority {queued_item_data.priority} URL {watch['url']}")
-                    now = time.time()

                    try:
                        # Processor is what we are using for detecting the "Change"
@@ -282,13 +281,16 @@ class update_worker(threading.Thread):
                            print(f"Processor module '{processor}' not found.")
                            raise e

-                        # Can pass just the watch here?
                        update_handler = processor_module.perform_site_check(datastore=self.datastore,
                                                                             watch_uuid=uuid
                                                                             )

                        update_handler.call_browser()

+                        # In reality, the actual time of when the change was detected could be a few seconds after this
+                        # For example it should include when the page stopped rendering if using a playwright/chrome type fetch
+                        fetch_start_time = time.time()
+
                        changed_detected, update_obj, contents = update_handler.run_changedetection(watch=watch)

                        # Re #342
@@ -513,7 +515,7 @@ class update_worker(threading.Thread):

                    if not self.datastore.data['watching'].get(uuid):
                        continue
-                    #
+
                    # Different exceptions mean that we may or may not want to bump the snapshot, trigger notifications etc
                    if process_changedetection_results:

@@ -526,8 +528,6 @@ class update_worker(threading.Thread):
                                except Exception as e:
                                    logger.warning(f"UUID: {uuid} Extract <title> as watch title was enabled, but couldn't find a <title>.")

-                        # Now update after running everything
-                        timestamp = round(time.time())
                        try:
                            self.datastore.update_watch(uuid=uuid, update_obj=update_obj)

@@ -543,24 +543,28 @@ class update_worker(threading.Thread):

                                # Small hack so that we sleep just enough to allow 1 second  between history snapshots
                                # this is because history.txt indexes/keys snapshots by epoch seconds and we dont want dupe keys
-
-                                if watch.newest_history_key and int(timestamp) == int(watch.newest_history_key):
+                                # @also - the keys are one per second at the most (for now)
+                                if watch.newest_history_key and int(fetch_start_time) == int(watch.newest_history_key):
                                    logger.warning(
-                                        f"Timestamp {timestamp} already exists, waiting 1 seconds so we have a unique key in history.txt")
-                                    timestamp = str(int(timestamp) + 1)
+                                        f"Timestamp {fetch_start_time} already exists, waiting 1 seconds so we have a unique key in history.txt")
+                                    fetch_start_time += 1
                                    time.sleep(1)

                                watch.save_history_text(contents=contents,
-                                                        timestamp=timestamp,
+                                                        timestamp=int(fetch_start_time),
                                                        snapshot_id=update_obj.get('previous_md5', 'none'))

-                                if update_handler.fetcher.content:
-                                    watch.save_last_fetched_html(contents=update_handler.fetcher.content, timestamp=timestamp)
+
+                                empty_pages_are_a_change = self.datastore.data['settings']['application'].get('empty_pages_are_a_change', False)
+                                if update_handler.fetcher.content or (not update_handler.fetcher.content and empty_pages_are_a_change):
+                                    # attribute .last_changed is then based on this data
+                                    watch.save_last_fetched_html(contents=update_handler.fetcher.content, timestamp=int(fetch_start_time))

                                # Notifications should only trigger on the second time (first time, we gather the initial snapshot)
                                if watch.history_n >= 2:
                                    logger.info(f"Change detected in UUID {uuid} - {watch['url']}")
                                    if not watch.get('notification_muted'):
+                                        # @todo only run this if notifications exist
                                        self.send_content_changed_notification(watch_uuid=uuid)

                        except Exception as e:
@@ -582,15 +586,15 @@ class update_worker(threading.Thread):
                    except Exception as e:
                        pass

-                    self.datastore.update_watch(uuid=uuid, update_obj={'fetch_time': round(time.time() - now, 3),
-                                                                       'last_checked': round(time.time()),
+                    self.datastore.update_watch(uuid=uuid, update_obj={'fetch_time': round(time.time() - fetch_start_time, 3),
+                                                                       'last_checked': int(fetch_start_time),
                                                                       'check_count': count
                                                                       })


                self.current_uuid = None  # Done
                self.q.task_done()
-                logger.debug(f"Watch {uuid} done in {time.time()-now:.2f}s")
+                logger.debug(f"Watch {uuid} done in {time.time()-fetch_start_time:.2f}s")

                # Give the CPU time to interrupt
                time.sleep(0.1)
@@ -35,7 +35,7 @@ dnspython==2.6.1 # related to eventlet fixes
 # jq not available on Windows so must be installed manually

 # Notification library
-apprise==1.9.0
+apprise==1.9.2

 # apprise mqtt https://github.com/dgtlmoon/changedetection.io/issues/315
 # use any version other than 2.0.x due to https://github.com/eclipse/paho.mqtt.python/issues/814
@@ -95,5 +95,8 @@ babel
 # Needed for > 3.10, https://github.com/microsoft/playwright-python/issues/2096
 greenlet >= 3.0.3

+# Pinned or it causes problems with flask_expects_json which seems unmaintained
+referencing==0.35.1
+
 # Scheduler - Windows seemed to miss a lot of default timezone info (even "UTC" !)
 tzdata
Author	SHA1	Message	Date
dgtlmoon	2483e26593	woops	2025-02-09 21:12:46 +01:00
dgtlmoon	c4001ec3ee	UI - "Browser Steps" tab should be always available with helpful info	2025-02-09 21:02:46 +01:00
dgtlmoon	09ebc6ec63	UI - Fix mute/unmute alt/title label alt/title text in watch overview (#2951 ) Build and push containers / metadata (push) Waiting to run Details Build and push containers / build-push-containers (push) Waiting to run Details Publish Python 🐍distribution 📦 to PyPI and TestPyPI / Build distribution 📦 (push) Waiting to run Details Publish Python 🐍distribution 📦 to PyPI and TestPyPI / Test the built 📦 package works basically. (push) Blocked by required conditions Details Publish Python 🐍distribution 📦 to PyPI and TestPyPI / Publish Python 🐍 distribution 📦 to PyPI (push) Blocked by required conditions Details ChangeDetection.io App Test / lint-code (push) Waiting to run Details ChangeDetection.io App Test / test-application-3-10 (push) Blocked by required conditions Details ChangeDetection.io App Test / test-application-3-11 (push) Blocked by required conditions Details ChangeDetection.io App Test / test-application-3-12 (push) Blocked by required conditions Details ChangeDetection.io App Test / test-application-3-13 (push) Blocked by required conditions Details	2025-02-08 18:01:26 +01:00
dgtlmoon	6b1065502e	0.49.1 Build and push containers / metadata (push) Waiting to run Details Build and push containers / build-push-containers (push) Waiting to run Details Publish Python 🐍distribution 📦 to PyPI and TestPyPI / Build distribution 📦 (push) Waiting to run Details Publish Python 🐍distribution 📦 to PyPI and TestPyPI / Test the built 📦 package works basically. (push) Blocked by required conditions Details Publish Python 🐍distribution 📦 to PyPI and TestPyPI / Publish Python 🐍 distribution 📦 to PyPI (push) Blocked by required conditions Details ChangeDetection.io App Test / lint-code (push) Waiting to run Details ChangeDetection.io App Test / test-application-3-10 (push) Blocked by required conditions Details ChangeDetection.io App Test / test-application-3-11 (push) Blocked by required conditions Details ChangeDetection.io App Test / test-application-3-12 (push) Blocked by required conditions Details ChangeDetection.io App Test / test-application-3-13 (push) Blocked by required conditions Details	2025-02-08 10:14:19 +01:00
vin86	d4c470984a	Update stock-not-in-stock.js - Italian (#2948 ) Build and push containers / metadata (push) Waiting to run Details Build and push containers / build-push-containers (push) Waiting to run Details Publish Python 🐍distribution 📦 to PyPI and TestPyPI / Build distribution 📦 (push) Waiting to run Details Publish Python 🐍distribution 📦 to PyPI and TestPyPI / Test the built 📦 package works basically. (push) Blocked by required conditions Details Publish Python 🐍distribution 📦 to PyPI and TestPyPI / Publish Python 🐍 distribution 📦 to PyPI (push) Blocked by required conditions Details ChangeDetection.io App Test / lint-code (push) Waiting to run Details ChangeDetection.io App Test / test-application-3-10 (push) Blocked by required conditions Details ChangeDetection.io App Test / test-application-3-11 (push) Blocked by required conditions Details ChangeDetection.io App Test / test-application-3-12 (push) Blocked by required conditions Details ChangeDetection.io App Test / test-application-3-13 (push) Blocked by required conditions Details	2025-02-08 00:23:20 +01:00
dgtlmoon	55da48f719	Re #2945 - Handle/Strip UTF-8 ByteOrderMark in JSON strings correctly (fixes `"Exception: No parsable JSON found in this document"` error) (#2947 ) Build and push containers / metadata (push) Waiting to run Details Build and push containers / build-push-containers (push) Waiting to run Details Publish Python 🐍distribution 📦 to PyPI and TestPyPI / Build distribution 📦 (push) Waiting to run Details Publish Python 🐍distribution 📦 to PyPI and TestPyPI / Test the built 📦 package works basically. (push) Blocked by required conditions Details Publish Python 🐍distribution 📦 to PyPI and TestPyPI / Publish Python 🐍 distribution 📦 to PyPI (push) Blocked by required conditions Details ChangeDetection.io App Test / lint-code (push) Waiting to run Details ChangeDetection.io App Test / test-application-3-10 (push) Blocked by required conditions Details ChangeDetection.io App Test / test-application-3-11 (push) Blocked by required conditions Details ChangeDetection.io App Test / test-application-3-12 (push) Blocked by required conditions Details ChangeDetection.io App Test / test-application-3-13 (push) Blocked by required conditions Details	2025-02-07 22:19:23 +01:00
RoboMagus	dbd4adf23a	Add major and minor tags for Docker release workflow (#2938 ) Build and push containers / metadata (push) Has been cancelled Details Build and push containers / build-push-containers (push) Has been cancelled Details Publish Python 🐍distribution 📦 to PyPI and TestPyPI / Build distribution 📦 (push) Has been cancelled Details ChangeDetection.io Container Build Test / test-container-build (push) Has been cancelled Details ChangeDetection.io App Test / lint-code (push) Has been cancelled Details Publish Python 🐍distribution 📦 to PyPI and TestPyPI / Test the built 📦 package works basically. (push) Has been cancelled Details Publish Python 🐍distribution 📦 to PyPI and TestPyPI / Publish Python 🐍 distribution 📦 to PyPI (push) Has been cancelled Details ChangeDetection.io App Test / test-application-3-10 (push) Has been cancelled Details ChangeDetection.io App Test / test-application-3-11 (push) Has been cancelled Details ChangeDetection.io App Test / test-application-3-12 (push) Has been cancelled Details ChangeDetection.io App Test / test-application-3-13 (push) Has been cancelled Details CodeQL / Analyze (javascript) (push) Has been cancelled Details CodeQL / Analyze (python) (push) Has been cancelled Details	2025-02-01 10:52:04 +01:00
dgtlmoon	b1e700b3ff	Adding jinja2/browsersteps test (#2915 ) Build and push containers / metadata (push) Has been cancelled Details Build and push containers / build-push-containers (push) Has been cancelled Details Publish Python 🐍distribution 📦 to PyPI and TestPyPI / Build distribution 📦 (push) Has been cancelled Details ChangeDetection.io App Test / lint-code (push) Has been cancelled Details Publish Python 🐍distribution 📦 to PyPI and TestPyPI / Test the built 📦 package works basically. (push) Has been cancelled Details Publish Python 🐍distribution 📦 to PyPI and TestPyPI / Publish Python 🐍 distribution 📦 to PyPI (push) Has been cancelled Details ChangeDetection.io App Test / test-application-3-10 (push) Has been cancelled Details ChangeDetection.io App Test / test-application-3-11 (push) Has been cancelled Details ChangeDetection.io App Test / test-application-3-12 (push) Has been cancelled Details ChangeDetection.io App Test / test-application-3-13 (push) Has been cancelled Details CodeQL / Analyze (javascript) (push) Has been cancelled Details CodeQL / Analyze (python) (push) Has been cancelled Details	2025-01-28 18:14:49 +01:00
Iftekhar Alam Fuad	1c61b5a623	Header handling - Fix header parsing to split on the first colon only (headers where the value contained :// type may have been broken) (#2929 ) Build and push containers / metadata (push) Has been cancelled Details Build and push containers / build-push-containers (push) Has been cancelled Details Publish Python 🐍distribution 📦 to PyPI and TestPyPI / Build distribution 📦 (push) Has been cancelled Details ChangeDetection.io App Test / lint-code (push) Has been cancelled Details Publish Python 🐍distribution 📦 to PyPI and TestPyPI / Test the built 📦 package works basically. (push) Has been cancelled Details Publish Python 🐍distribution 📦 to PyPI and TestPyPI / Publish Python 🐍 distribution 📦 to PyPI (push) Has been cancelled Details ChangeDetection.io App Test / test-application-3-10 (push) Has been cancelled Details ChangeDetection.io App Test / test-application-3-11 (push) Has been cancelled Details ChangeDetection.io App Test / test-application-3-12 (push) Has been cancelled Details ChangeDetection.io App Test / test-application-3-13 (push) Has been cancelled Details	2025-01-26 00:08:09 +01:00
dgtlmoon	e799a1cdcb	0.49.00 Build and push containers / metadata (push) Has been cancelled Details Build and push containers / build-push-containers (push) Has been cancelled Details Publish Python 🐍distribution 📦 to PyPI and TestPyPI / Build distribution 📦 (push) Has been cancelled Details ChangeDetection.io Container Build Test / test-container-build (push) Has been cancelled Details ChangeDetection.io App Test / lint-code (push) Has been cancelled Details Publish Python 🐍distribution 📦 to PyPI and TestPyPI / Test the built 📦 package works basically. (push) Has been cancelled Details Publish Python 🐍distribution 📦 to PyPI and TestPyPI / Publish Python 🐍 distribution 📦 to PyPI (push) Has been cancelled Details ChangeDetection.io App Test / test-application-3-10 (push) Has been cancelled Details ChangeDetection.io App Test / test-application-3-11 (push) Has been cancelled Details ChangeDetection.io App Test / test-application-3-12 (push) Has been cancelled Details ChangeDetection.io App Test / test-application-3-13 (push) Has been cancelled Details CodeQL / Analyze (javascript) (push) Has been cancelled Details CodeQL / Analyze (python) (push) Has been cancelled Details	2025-01-21 13:40:01 +01:00
dgtlmoon	938065db6f	Update README.md Build and push containers / metadata (push) Waiting to run Details Build and push containers / build-push-containers (push) Waiting to run Details Publish Python 🐍distribution 📦 to PyPI and TestPyPI / Build distribution 📦 (push) Waiting to run Details Publish Python 🐍distribution 📦 to PyPI and TestPyPI / Test the built 📦 package works basically. (push) Blocked by required conditions Details Publish Python 🐍distribution 📦 to PyPI and TestPyPI / Publish Python 🐍 distribution 📦 to PyPI (push) Blocked by required conditions Details ChangeDetection.io App Test / lint-code (push) Waiting to run Details ChangeDetection.io App Test / test-application-3-10 (push) Blocked by required conditions Details ChangeDetection.io App Test / test-application-3-11 (push) Blocked by required conditions Details ChangeDetection.io App Test / test-application-3-12 (push) Blocked by required conditions Details ChangeDetection.io App Test / test-application-3-13 (push) Blocked by required conditions Details	2025-01-20 16:10:54 +01:00
dgtlmoon	4f2d38ff49	Build/Libraries - Pin `referencing` library which breaks due to out-dated flask_expects_json, remove pip upgrade in test(#2912 ) Build and push containers / metadata (push) Has been cancelled Details Build and push containers / build-push-containers (push) Has been cancelled Details Publish Python 🐍distribution 📦 to PyPI and TestPyPI / Build distribution 📦 (push) Has been cancelled Details ChangeDetection.io Container Build Test / test-container-build (push) Has been cancelled Details ChangeDetection.io App Test / lint-code (push) Has been cancelled Details Publish Python 🐍distribution 📦 to PyPI and TestPyPI / Test the built 📦 package works basically. (push) Has been cancelled Details Publish Python 🐍distribution 📦 to PyPI and TestPyPI / Publish Python 🐍 distribution 📦 to PyPI (push) Has been cancelled Details ChangeDetection.io App Test / test-application-3-10 (push) Has been cancelled Details ChangeDetection.io App Test / test-application-3-11 (push) Has been cancelled Details ChangeDetection.io App Test / test-application-3-12 (push) Has been cancelled Details ChangeDetection.io App Test / test-application-3-13 (push) Has been cancelled Details	2025-01-18 23:20:58 +01:00
dgtlmoon	8960f401b7	Notifications - Custom POST:// GET:// etc endpoints - returning 204 and other 20x responses are OK (don't show an error was detected)(#2897 ) Build and push containers / metadata (push) Has been cancelled Details Build and push containers / build-push-containers (push) Has been cancelled Details Publish Python 🐍distribution 📦 to PyPI and TestPyPI / Build distribution 📦 (push) Has been cancelled Details ChangeDetection.io App Test / lint-code (push) Has been cancelled Details Publish Python 🐍distribution 📦 to PyPI and TestPyPI / Test the built 📦 package works basically. (push) Has been cancelled Details Publish Python 🐍distribution 📦 to PyPI and TestPyPI / Publish Python 🐍 distribution 📦 to PyPI (push) Has been cancelled Details ChangeDetection.io App Test / test-application-3-10 (push) Has been cancelled Details ChangeDetection.io App Test / test-application-3-11 (push) Has been cancelled Details ChangeDetection.io App Test / test-application-3-12 (push) Has been cancelled Details ChangeDetection.io App Test / test-application-3-13 (push) Has been cancelled Details CodeQL / Analyze (javascript) (push) Has been cancelled Details CodeQL / Analyze (python) (push) Has been cancelled Details	2025-01-13 13:13:18 +01:00
dgtlmoon	1c1f1c6f6b	0.48.06	2025-01-09 23:02:29 +01:00
dgtlmoon	a2a98811a5	Restock - Add test for new lower/higher price notification Re #2715 (#2892 )	2025-01-09 22:59:55 +01:00
dgtlmoon	5a0ef8fc01	Update integration test for "linuxserver" test build (#2891 )	2025-01-09 21:36:39 +01:00
dgtlmoon	d90de0851d	Notifications - Update Apprise to 1.9.2 - Fixes custom posts:// gets:// etc URL's being double-encoded, fixes chantify:// notifications (#2868 ) (#2875 ) (#2870 )	2025-01-09 21:16:32 +01:00
dgtlmoon	360b4f0d8b	Custom posts:// get:// notifications etc - Be sure our custom extensions are imported (#2890 )	2025-01-09 21:10:09 +01:00
dgtlmoon	6fc04d7f1c	"Send test notification" button - Easier to understand test send results, Improved error handling, code refactor (#2888 )	2025-01-08 14:35:41 +01:00
dgtlmoon	66fb05527b	Improve `last_checked` vs `last_changed` time information precision (#2883 )	2025-01-06 20:38:50 +01:00
William Brawner	202e47d728	Update Apprise to 1.9.1 (#2876 )	2025-01-02 20:06:25 +01:00