Migrate watches from main DB file to a subfile

.github/test/Dockerfile-alpine

@@ -2,33 +2,32 @@
 # Test that we can still build on Alpine (musl modified libc https://musl.libc.org/)
 # Some packages wont install via pypi because they dont have a wheel available under this architecture.
 
-FROM ghcr.io/linuxserver/baseimage-alpine:3.21
+FROM ghcr.io/linuxserver/baseimage-alpine:3.18
 ENV PYTHONUNBUFFERED=1
 
 COPY requirements.txt /requirements.txt
 
 RUN \
- apk add --update --no-cache --virtual=build-dependencies \
-    build-base \
+  apk add --update --no-cache --virtual=build-dependencies \
     cargo \
-    git \
+    g++ \
+    gcc \
     jpeg-dev \
     libc-dev \
     libffi-dev \
+    libjpeg \
     libxslt-dev \
+    make \
     openssl-dev \
+    py3-wheel \
     python3-dev \
-    zip \
     zlib-dev && \
   apk add --update --no-cache \
-    libjpeg \
     libxslt \
-    nodejs \
-    poppler-utils \
-    python3 && \
+    python3 \
+    py3-pip && \
   echo "**** pip3 install test of changedetection.io ****" && \
-  python3 -m venv /lsiopy  && \
-  pip install -U pip wheel setuptools && \
-  pip install -U --no-cache-dir --find-links https://wheel-index.linuxserver.io/alpine-3.21/ -r /requirements.txt && \
+  pip3 install -U pip wheel setuptools && \
+  pip3 install -U --no-cache-dir --find-links https://wheel-index.linuxserver.io/alpine-3.18/ -r /requirements.txt && \
   apk del --purge \
     build-dependencies

.github/workflows/containers.yml

@@ -103,19 +103,6 @@ jobs:
 #          provenance: false
 
       # A new tagged release is required, which builds :tag and :latest
-      - name: Docker meta :tag
-        if: github.event_name == 'release' && startsWith(github.event.release.tag_name, '0.')
-        uses: docker/metadata-action@v5
-        id: meta
-        with:
-            images: |
-                ${{ secrets.DOCKER_HUB_USERNAME }}/changedetection.io
-                ghcr.io/dgtlmoon/changedetection.io
-            tags: |
-                type=semver,pattern={{version}}
-                type=semver,pattern={{major}}.{{minor}}
-                type=semver,pattern={{major}}
-
       - name: Build and push :tag
         id: docker_build_tag_release
         if: github.event_name == 'release' && startsWith(github.event.release.tag_name, '0.')
@@ -124,7 +111,11 @@ jobs:
           context: ./
           file: ./Dockerfile
           push: true
-          tags: ${{ steps.meta.outputs.tags }}
+          tags: |
+            ${{ secrets.DOCKER_HUB_USERNAME }}/changedetection.io:${{ github.event.release.tag_name }}
+            ghcr.io/dgtlmoon/changedetection.io:${{ github.event.release.tag_name }}
+            ${{ secrets.DOCKER_HUB_USERNAME }}/changedetection.io:latest
+            ghcr.io/dgtlmoon/changedetection.io:latest
           platforms: linux/amd64,linux/arm64,linux/arm/v7,linux/arm/v8,linux/arm64/v8
           cache-from: type=gha
           cache-to: type=gha,mode=max

.github/workflows/pypi-release.yml

@@ -45,6 +45,7 @@ jobs:
     - name: Test that the basic pip built package runs without error
       run: |
         set -ex
+        sudo pip3 install --upgrade pip 
         pip3 install dist/changedetection.io*.whl
         changedetection.io -d /tmp -p 10000 &
         sleep 3

.github/workflows/test-stack-reusable-workflow.yml

@@ -64,16 +64,14 @@ jobs:
           echo "Running processes in docker..."
           docker ps
 
-      - name: Run Unit Tests
+      - name: Test built container with Pytest (generally as requests/plaintext fetching)
         run: |
           # Unit tests
+          echo "run test with unittest"
           docker run test-changedetectionio  bash -c 'python3 -m unittest changedetectionio.tests.unit.test_notification_diff'
           docker run test-changedetectionio  bash -c 'python3 -m unittest changedetectionio.tests.unit.test_watch_model'
           docker run test-changedetectionio  bash -c 'python3 -m unittest changedetectionio.tests.unit.test_jinja2_security'
-          docker run test-changedetectionio  bash -c 'python3 -m unittest changedetectionio.tests.unit.test_semver'
-
-      - name: Test built container with Pytest (generally as requests/plaintext fetching)
-        run: |
+          
           # All tests
           echo "run test with pytest"
           # The default pytest logger_level is TRACE

README.md

@@ -120,7 +120,7 @@ Easily add the current web page to your changedetection.io tool, simply install
 
 [<img src="./docs/chrome-extension-screenshot.png" style="max-width:80%;" alt="Chrome Extension to easily add the current web-page to detect a change."  title="Chrome Extension to easily add the current web-page to detect a change."  />](https://chromewebstore.google.com/detail/changedetectionio-website/kefcfmgmlhmankjmnbijimhofdjekbop)
 
-[Goto the Chrome Webstore to download the extension.](https://chromewebstore.google.com/detail/changedetectionio-website/kefcfmgmlhmankjmnbijimhofdjekbop) ( Or check out the [GitHub repo](https://github.com/dgtlmoon/changedetection.io-browser-extension) ) 
+[Goto the Chrome Webstore to download the extension.](https://chromewebstore.google.com/detail/changedetectionio-website/kefcfmgmlhmankjmnbijimhofdjekbop)
 
 ## Installation
 

changedetectionio/__init__.py

@@ -2,7 +2,7 @@
 
 # Read more https://github.com/dgtlmoon/changedetection.io/wiki
 
-__version__ = '0.49.1'
+__version__ = '0.48.05'
 
 from changedetectionio.strtobool import strtobool
 from json.decoder import JSONDecodeError
@@ -24,9 +24,6 @@ from loguru import logger
 app = None
 datastore = None
 
-def get_version():
-    return __version__
-
 # Parent wrapper or OS sends us a SIGTERM/SIGINT, do everything required for a clean shutdown
 def sigshutdown_handler(_signo, _stack_frame):
     global app

changedetectionio/api/api_v1.py

@@ -12,10 +12,10 @@ import copy
 # See docs/README.md for rebuilding the docs/apidoc information
 
 from . import api_schema
-from ..model import watch_base
+from ..model import WatchBase
 
 # Build a JSON Schema atleast partially based on our Watch model
-watch_base_config = watch_base()
+watch_base_config = WatchBase()
 schema = api_schema.build_watch_json_schema(watch_base_config)
 
 schema_create_watch = copy.deepcopy(schema)
@@ -52,8 +52,8 @@ class Watch(Resource):
         @apiSuccess (200) {String} OK When paused/muted/recheck operation OR full JSON object of the watch
         @apiSuccess (200) {JSON} WatchJSON JSON Full JSON object of the watch
         """
-        from copy import deepcopy
-        watch = deepcopy(self.datastore.data['watching'].get(uuid))
+        watch = self.datastore.data['watching'].get(uuid)
+
         if not watch:
             abort(404, message='No watch exists with the UUID of {}'.format(uuid))
 
@@ -75,11 +75,11 @@ class Watch(Resource):
 
         # Return without history, get that via another API call
         # Properties are not returned as a JSON, so add the required props manually
-        watch['history_n'] = watch.history_n
-        # attr .last_changed will check for the last written text snapshot on change
-        watch['last_changed'] = watch.last_changed
-        watch['viewed'] = watch.viewed
-        return watch
+        result = watch.as_dict()
+        result['history_n'] = watch.history_n
+        result['last_changed'] = watch.last_changed
+        result['viewed'] = watch.viewed
+        return result
 
     @auth.check_token
     def delete(self, uuid):

changedetectionio/apprise_asset.py

@@ -1,4 +1,3 @@
-from changedetectionio import apprise_plugin
 import apprise
 
 # Create our AppriseAsset and populate it with some of our new values:

changedetectionio/apprise_plugin/__init__.py

@@ -1,8 +1,6 @@
 # include the decorator
 from apprise.decorators import notify
 from loguru import logger
-from requests.structures import CaseInsensitiveDict
-
 
 @notify(on="delete")
 @notify(on="deletes")
@@ -15,84 +13,70 @@ from requests.structures import CaseInsensitiveDict
 def apprise_custom_api_call_wrapper(body, title, notify_type, *args, **kwargs):
     import requests
     import json
-    import re
-
     from urllib.parse import unquote_plus
-    from apprise.utils.parse import parse_url as apprise_parse_url
+    from apprise.utils import parse_url as apprise_parse_url
+    from apprise import URLBase
 
     url = kwargs['meta'].get('url')
-    schema = kwargs['meta'].get('schema').lower().strip()
 
-    # Choose POST, GET etc from requests
-    method =  re.sub(rf's$', '', schema)
-    requests_method = getattr(requests, method)
+    if url.startswith('post'):
+        r = requests.post
+    elif url.startswith('get'):
+        r = requests.get
+    elif url.startswith('put'):
+        r = requests.put
+    elif url.startswith('delete'):
+        r = requests.delete
 
-    params = CaseInsensitiveDict({}) # Added to requests
+    url = url.replace('post://', 'http://')
+    url = url.replace('posts://', 'https://')
+    url = url.replace('put://', 'http://')
+    url = url.replace('puts://', 'https://')
+    url = url.replace('get://', 'http://')
+    url = url.replace('gets://', 'https://')
+    url = url.replace('put://', 'http://')
+    url = url.replace('puts://', 'https://')
+    url = url.replace('delete://', 'http://')
+    url = url.replace('deletes://', 'https://')
+
+    headers = {}
+    params = {}
     auth = None
-    has_error = False
 
     # Convert /foobar?+some-header=hello to proper header dictionary
     results = apprise_parse_url(url)
+    if results:
+        # Add our headers that the user can potentially over-ride if they wish
+        # to to our returned result set and tidy entries by unquoting them
+        headers = {unquote_plus(x): unquote_plus(y)
+                   for x, y in results['qsd+'].items()}
 
-    # Add our headers that the user can potentially over-ride if they wish
-    # to to our returned result set and tidy entries by unquoting them
-    headers = CaseInsensitiveDict({unquote_plus(x): unquote_plus(y)
-               for x, y in results['qsd+'].items()})
+        # https://github.com/caronc/apprise/wiki/Notify_Custom_JSON#get-parameter-manipulation
+        # In Apprise, it relies on prefixing each request arg with "-", because it uses say &method=update as a flag for apprise
+        # but here we are making straight requests, so we need todo convert this against apprise's logic
+        for k, v in results['qsd'].items():
+            if not k.strip('+-') in results['qsd+'].keys():
+                params[unquote_plus(k)] = unquote_plus(v)
 
-    # https://github.com/caronc/apprise/wiki/Notify_Custom_JSON#get-parameter-manipulation
-    # In Apprise, it relies on prefixing each request arg with "-", because it uses say &method=update as a flag for apprise
-    # but here we are making straight requests, so we need todo convert this against apprise's logic
-    for k, v in results['qsd'].items():
-        if not k.strip('+-') in results['qsd+'].keys():
-            params[unquote_plus(k)] = unquote_plus(v)
+        # Determine Authentication
+        auth = ''
+        if results.get('user') and results.get('password'):
+            auth = (unquote_plus(results.get('user')), unquote_plus(results.get('user')))
+        elif results.get('user'):
+            auth = (unquote_plus(results.get('user')))
 
-    # Determine Authentication
-    auth = ''
-    if results.get('user') and results.get('password'):
-        auth = (unquote_plus(results.get('user')), unquote_plus(results.get('user')))
-    elif results.get('user'):
-        auth = (unquote_plus(results.get('user')))
-
-    # If it smells like it could be JSON and no content-type was already set, offer a default content type.
-    if body and '{' in body[:100] and not headers.get('Content-Type'):
-        json_header = 'application/json; charset=utf-8'
-        try:
-            # Try if it's JSON
-            json.loads(body)
-            headers['Content-Type'] = json_header
-        except ValueError as e:
-            logger.warning(f"Could not automatically add '{json_header}' header to the notification because the document failed to parse as JSON: {e}")
-            pass
-
-    # POSTS -> HTTPS etc
-    if schema.lower().endswith('s'):
-        url = re.sub(rf'^{schema}', 'https', results.get('url'))
-    else:
-        url = re.sub(rf'^{schema}', 'http', results.get('url'))
-
-    status_str = ''
+    # Try to auto-guess if it's JSON
+    h = 'application/json; charset=utf-8'
     try:
-        r = requests_method(url,
-          auth=auth,
-          data=body.encode('utf-8') if type(body) is str else body,
-          headers=headers,
-          params=params
-        )
+        json.loads(body)
+        headers['Content-Type'] = h
+    except ValueError as e:
+        logger.warning(f"Could not automatically add '{h}' header to the {kwargs['meta'].get('schema')}:// notification because the document failed to parse as JSON: {e}")
+        pass
 
-        if not (200 <= r.status_code < 300):
-            status_str = f"Error sending '{method.upper()}' request to {url} - Status: {r.status_code}: '{r.reason}'"
-            logger.error(status_str)
-            has_error = True
-        else:
-            logger.info(f"Sent '{method.upper()}' request to {url}")
-            has_error = False
-
-    except requests.RequestException as e:
-        status_str = f"Error sending '{method.upper()}' request to {url} - {str(e)}"
-        logger.error(status_str)
-        has_error = True
-
-    if has_error:
-        raise TypeError(status_str)
-
-    return True
+    r(results.get('url'),
+      auth=auth,
+      data=body.encode('utf-8') if type(body) is str else body,
+      headers=headers,
+      params=params
+      )

changedetectionio/blueprint/check_proxies/__init__.py

@@ -41,6 +41,7 @@ def construct_blueprint(datastore: ChangeDetectionStore):
         now = time.time()
         try:
             processor_module = importlib.import_module("changedetectionio.processors.text_json_diff.processor")
+            # @todo can now just pass the watch here?
             update_handler = processor_module.perform_site_check(datastore=datastore,
                                                                  watch_uuid=uuid
                                                                  )

changedetectionio/conditions/__init__.py

@@ -1,113 +0,0 @@
-from json_logic import jsonLogic
-from json_logic.builtins import BUILTINS
-from .pluggy_interface import plugin_manager  # Import the pluggy plugin manager
-from . import default_plugin
-
-import re
-
-# List of all supported JSON Logic operators
-operator_choices = [
-    (">", "Greater Than"),
-    ("<", "Less Than"),
-    (">=", "Greater Than or Equal To"),
-    ("<=", "Less Than or Equal To"),
-    ("==", "Equals"),
-    ("!=", "Not Equals"),
-    ("in", "Contains"),
-    ("!in", "Does Not Contain"),
-    ("contains_regex", "Text Matches Regex"),
-    ("!contains_regex", "Text Does NOT Match Regex"),
-    ("changed > minutes", "Changed more than X minutes ago"),
-#    ("watch_uuid_unviewed_change", "Watch UUID had an unviewed change"),  #('if'? )
-#    ("watch_uuid_not_unviewed_change", "Watch UUID NOT had an unviewed change") #('if'? )
-#    ("watch_uuid_changed", "Watch UUID had unviewed change"),
-#    ("watch_uuid_not_changed", "Watch UUID did NOT have unviewed change"),
-#    ("!!", "Is Truthy"),
-#    ("!", "Is Falsy"),
-#    ("and", "All Conditions Must Be True"),
-#    ("or", "At Least One Condition Must Be True"),
-#    ("max", "Maximum of Values"),
-#    ("min", "Minimum of Values"),
-#    ("+", "Addition"),
-#    ("-", "Subtraction"),
-#    ("*", "Multiplication"),
-#    ("/", "Division"),
-#    ("%", "Modulo"),
-#    ("log", "Logarithm"),
-#    ("if", "Conditional If-Else")
-]
-
-# Fields available in the rules
-field_choices = [
-    ("extracted_number", "Extracted Number"),
-    ("page_filtered_text", "Page text After Filters"),
-    ("page_title", "Page Title"), # actual page title <title>
-    ("watch_uuid", "Watch UUID"),
-    #("watch_history_length", "History Length"),  # Would never equate
-    ("watch_history", "All Watch Text History"),
-    ("watch_check_count", "Watch Check Count"),
-    ("watch_uuid", "Other Watch by UUID"), # (Maybe this is 'if' ??)
-    #("requests_get", "Web GET requests (https://..)")
-]
-
-
-# ✅ Custom function for case-insensitive regex matching
-def contains_regex(_, text, pattern):
-    """Returns True if `text` contains `pattern` (case-insensitive regex match)."""
-    return bool(re.search(pattern, text, re.IGNORECASE))
-
-# ✅ Custom function for NOT matching case-insensitive regex
-def not_contains_regex(_, text, pattern):
-    """Returns True if `text` does NOT contain `pattern` (case-insensitive regex match)."""
-    return not bool(re.search(pattern, text, re.IGNORECASE))
-
-
-# ✅ Custom function to check if "watch_uuid" has changed
-def watch_uuid_changed(_, previous_uuid, current_uuid):
-    """Returns True if the watch UUID has changed."""
-    return previous_uuid != current_uuid
-
-# ✅ Custom function to check if "watch_uuid" has NOT changed
-def watch_uuid_not_changed(_, previous_uuid, current_uuid):
-    """Returns True if the watch UUID has NOT changed."""
-    return previous_uuid == current_uuid
-
-# Define the extended operations dictionary
-CUSTOM_OPERATIONS = {
-    **BUILTINS,  # Include all standard operators
-    "watch_uuid_changed": watch_uuid_changed,
-    "watch_uuid_not_changed": watch_uuid_not_changed,
-    "contains_regex": contains_regex,
-    "!contains_regex": not_contains_regex
-}
-
-# ✅ Load plugins dynamically
-for plugin in plugin_manager.get_plugins():
-    new_ops = plugin.register_operators()
-    if isinstance(new_ops, dict):
-        CUSTOM_OPERATIONS.update(new_ops)
-
-    new_operator_choices = plugin.register_operator_choices()
-    if isinstance(new_operator_choices, list):
-        operator_choices.extend(new_operator_choices)
-
-    new_field_choices = plugin.register_field_choices()
-    if isinstance(new_field_choices, list):
-        field_choices.extend(new_field_choices)
-
-def run(ruleset, data):
-    """
-    Execute a JSON Logic rule against given data.
-
-    :param ruleset: JSON Logic rule dictionary.
-    :param data: Dictionary containing the facts.
-    :return: Boolean result of rule evaluation.
-    """
-
-
-    try:
-        return jsonLogic(ruleset, data, CUSTOM_OPERATIONS)
-    except Exception as e:
-        # raise some custom nice handler
-        print(f"❌ Error evaluating JSON Logic: {e}")
-        return False

changedetectionio/conditions/default_plugin.py

@@ -1,30 +0,0 @@
-import pluggy
-
-hookimpl = pluggy.HookimplMarker("conditions")
-
-@hookimpl
-def register_operators():
-    def starts_with(_, text, prefix):
-        return text.lower().startswith(prefix.lower())
-
-    def ends_with(_, text, suffix):
-        return text.lower().endswith(suffix.lower())
-
-    return {
-        "starts_with": starts_with,
-        "ends_with": ends_with
-    }
-
-@hookimpl
-def register_operator_choices():
-    return [
-        ("starts_with", "Text Starts With"),
-        ("ends_with", "Text Ends With"),
-    ]
-
-@hookimpl
-def register_field_choices():
-    return [
-        ("meta_description", "Meta Description"),
-        ("meta_keywords", "Meta Keywords"),
-    ]

changedetectionio/conditions/pluggy_interface.py

@@ -1,32 +0,0 @@
-import pluggy
-
-# Define `pluggy` hookspecs (Specifications for Plugins)
-hookspec = pluggy.HookspecMarker("conditions")
-hookimpl = pluggy.HookimplMarker("conditions")
-
-
-class ConditionsSpec:
-    """Hook specifications for extending JSON Logic conditions."""
-
-    @hookspec
-    def register_operators():
-        """Return a dictionary of new JSON Logic operators."""
-        pass
-
-    @hookspec
-    def register_operator_choices():
-        """Return a list of new operator choices."""
-        pass
-
-    @hookspec
-    def register_field_choices():
-        """Return a list of new field choices."""
-        pass
-
-
-# ✅ Set up `pluggy` Plugin Manager
-plugin_manager = pluggy.PluginManager("conditions")
-plugin_manager.add_hookspecs(ConditionsSpec)
-
-# Discover installed plugins
-plugin_manager.load_setuptools_entrypoints("conditions")

changedetectionio/content_fetchers/res/stock-not-in-stock.js

@@ -52,7 +52,6 @@ function isItemInStock() {
         'niet leverbaar',
         'niet op voorraad',
         'no disponible',
-        'non disponibile',
         'no longer in stock',
         'no tickets available',
         'not available',

changedetectionio/flask_app.py

@@ -43,6 +43,7 @@ from loguru import logger
 from changedetectionio import html_tools, __version__
 from changedetectionio import queuedWatchMetaData
 from changedetectionio.api import api_v1
+from .store import CustomEncoder
 from .time_handler import is_within_schedule
 
 datastore = None
@@ -598,31 +599,17 @@ def changedetection_app(config=None, datastore_o=None):
 
             if 'notification_title' in request.form and request.form['notification_title'].strip():
                 n_object['notification_title'] = request.form.get('notification_title', '').strip()
-            elif datastore.data['settings']['application'].get('notification_title'):
-                n_object['notification_title'] = datastore.data['settings']['application'].get('notification_title')
-            else:
-                n_object['notification_title'] = "Test title"
 
             if 'notification_body' in request.form and request.form['notification_body'].strip():
                 n_object['notification_body'] = request.form.get('notification_body', '').strip()
-            elif datastore.data['settings']['application'].get('notification_body'):
-                n_object['notification_body'] = datastore.data['settings']['application'].get('notification_body')
-            else:
-                n_object['notification_body'] = "Test body"
 
-            n_object['as_async'] = False
             n_object.update(watch.extra_notification_token_values())
-            from .notification import process_notification
-            sent_obj = process_notification(n_object, datastore)
 
+            from . import update_worker
+            new_worker = update_worker.update_worker(update_q, notification_q, app, datastore)
+            new_worker.queue_notification_for_watch(notification_q=notification_q, n_object=n_object, watch=watch)
         except Exception as e:
-            e_str = str(e)
-            # Remove this text which is not important and floods the container
-            e_str = e_str.replace(
-                "DEBUG - <class 'apprise.decorators.base.CustomNotifyPlugin.instantiate_plugin.<locals>.CustomNotifyPluginWrapper'>",
-                '')
-
-            return make_response(e_str, 400)
+            return make_response(f"Error: str(e)", 400)
 
         return 'OK - Sent test notifications'
 
@@ -758,25 +745,6 @@ def changedetection_app(config=None, datastore_o=None):
             for p in datastore.proxy_list:
                 form.proxy.choices.append(tuple((p, datastore.proxy_list[p]['label'])))
 
-        # Example JSON Rule
-        DEFAULT_RULE = {
-            "and": [
-                {">": [{"var": "extracted_number"}, 5000]},
-                {"<": [{"var": "extracted_number"}, 80000]},
-                {"in": ["rock", {"var": "page_text"}]}
-            ]
-        }
-        form.conditions.pop_entry()  # Remove the default empty row
-        for condition in DEFAULT_RULE["and"]:
-            operator, values = list(condition.items())[0]
-            field = values[0]["var"] if isinstance(values[0], dict) else values[1]["var"]
-            value = values[1] if isinstance(values[1], (str, int)) else values[0]
-
-            form.conditions.append_entry({
-                "operator": operator,
-                "field": field,
-                "value": value
-            })
 
         if request.method == 'POST' and form.validate():
 
@@ -812,14 +780,6 @@ def changedetection_app(config=None, datastore_o=None):
                 extra_update_obj['filter_text_replaced'] = True
                 extra_update_obj['filter_text_removed'] = True
 
-            # Convert form input into JSON Logic format
-            extra_update_obj["conditions"] = {
-                "and": [
-                    {c.operator.data: [{"var": c.field.data}, c.value.data]}
-                    for c in getattr(form, "conditions", []) or []
-                ]
-            } if getattr(form, "conditions", None) else {}
-
             # Because wtforms doesn't support accessing other data in process_ , but we convert the CSV list of tags back to a list of UUIDs
             tag_uuids = []
             if form.data.get('tags'):
@@ -841,7 +801,7 @@ def changedetection_app(config=None, datastore_o=None):
 
             # Recast it if need be to right data Watch handler
             watch_class = get_custom_watch_obj_for_processor(form.data.get('processor'))
-            datastore.data['watching'][uuid] = watch_class(datastore_path=datastore_o.datastore_path, default=datastore.data['watching'][uuid])
+            datastore.data['watching'][uuid] = watch_class(__datastore=datastore_o, default=datastore.data['watching'][uuid])
             flash("Updated watch - unpaused!" if request.args.get('unpause_on_save') else "Updated watch.")
 
             # Re #286 - We wait for syncing new data to disk in another thread every 60 seconds
@@ -1654,7 +1614,7 @@ def changedetection_app(config=None, datastore_o=None):
         watch['ignore_text'] += datastore.data['settings']['application']['global_ignore_text']
         watch['subtractive_selectors'] += datastore.data['settings']['application']['global_subtractive_selectors']
 
-        watch_json = json.dumps(watch)
+        watch_json = json.dumps(watch, cls=CustomEncoder)
 
         try:
             r = requests.request(method="POST",

changedetectionio/forms.py

@@ -4,7 +4,6 @@ from loguru import logger
 from wtforms.widgets.core import TimeInput
 
 from changedetectionio.strtobool import strtobool
-from flask_wtf import FlaskForm
 
 from wtforms import (
     BooleanField,
@@ -307,6 +306,7 @@ class ValidateAppRiseServers(object):
         import apprise
         apobj = apprise.Apprise()
         # so that the custom endpoints are registered
+        from changedetectionio.apprise_plugin import apprise_custom_api_call_wrapper
         for server_url in field.data:
             url = server_url.strip()
             if url.startswith("#"):
@@ -509,23 +509,6 @@ class quickWatchForm(Form):
     edit_and_watch_submit_button = SubmitField('Edit > Watch', render_kw={"class": "pure-button pure-button-primary"})
 
 
-
-# Condition Rule Form (for each rule row)
-class ConditionForm(FlaskForm):
-    from changedetectionio.conditions import operator_choices, field_choices
-
-    operator = SelectField(
-        "Operator",
-        choices=operator_choices,
-        validators=[validators.Optional()]
-    )
-    field = SelectField(
-        "Field",
-        choices=field_choices,
-        validators=[validators.Optional()]
-    )
-    value = StringField("Value", validators=[validators.Optional()])
-
 # Common to a single watch and the global settings
 class commonSettingsForm(Form):
     from . import processors
@@ -613,9 +596,6 @@ class processor_text_json_diff_form(commonSettingsForm):
     notification_muted = BooleanField('Notifications Muted / Off', default=False)
     notification_screenshot = BooleanField('Attach screenshot to notification (where possible)', default=False)
 
-    conditions = FieldList(FormField(ConditionForm), min_entries=1)  # Add rule logic here
-
-
     def extra_tab_content(self):
         return None
 

changedetectionio/html_tools.py

@@ -1,6 +1,5 @@
-from loguru import logger
-from lxml import etree
 from typing import List
+from lxml import etree
 import json
 import re
 
@@ -299,10 +298,8 @@ def extract_json_as_string(content, json_filter, ensure_is_ldjson_info_type=None
 # https://github.com/dgtlmoon/changedetection.io/pull/2041#issuecomment-1848397161w
     # Try to parse/filter out the JSON, if we get some parser error, then maybe it's embedded within HTML tags
     try:
-        # .lstrip("\ufeff") strings ByteOrderMark from UTF8 and still lets the UTF work
-        stripped_text_from_html = _parse_json(json.loads(content.lstrip("\ufeff") ), json_filter)
-    except json.JSONDecodeError as e:
-        logger.warning(str(e))
+        stripped_text_from_html = _parse_json(json.loads(content), json_filter)
+    except json.JSONDecodeError:
 
         # Foreach <script json></script> blob.. just return the first that matches json_filter
         # As a last resort, try to parse the whole <body>

changedetectionio/model/App.py

@@ -10,7 +10,7 @@ _FILTER_FAILURE_THRESHOLD_ATTEMPTS_DEFAULT = 6
 DEFAULT_SETTINGS_HEADERS_USERAGENT='Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/87.0.4280.66 Safari/537.36'
 
 class model(dict):
-    base_config = {
+    __base_config = {
             'note': "Hello! If you change this file manually, please be sure to restart your changedetection.io instance!",
             'watching': {},
             'settings': {
@@ -60,7 +60,7 @@ class model(dict):
 
     def __init__(self, *arg, **kw):
         super(model, self).__init__(*arg, **kw)
-        self.update(self.base_config)
+        self.update(self.__base_config)
 
 
 def parse_headers_from_text_file(filepath):
@@ -69,7 +69,7 @@ def parse_headers_from_text_file(filepath):
         for l in f.readlines():
             l = l.strip()
             if not l.startswith('#') and ':' in l:
-                (k, v) = l.split(':', 1)  # Split only on the first colon
+                (k, v) = l.split(':')
                 headers[k.strip()] = v.strip()
 
     return headers

changedetectionio/model/Tag.py

@@ -1,8 +1,8 @@
 
-from changedetectionio.model import watch_base
+from changedetectionio.model import WatchBase
 
 
-class model(watch_base):
+class model(WatchBase):
 
     def __init__(self, *arg, **kw):
         super(model, self).__init__(*arg, **kw)

changedetectionio/model/Watch.py

@@ -1,17 +1,17 @@
 from changedetectionio.strtobool import strtobool
 from changedetectionio.safe_jinja import render as jinja_render
-from . import watch_base
 import os
 import re
 from pathlib import Path
 from loguru import logger
 
+from . import WatchBase
 from ..html_tools import TRANSLATE_WHITESPACE_TABLE
 
 # Allowable protocols, protects against javascript: etc
 # file:// is further checked by ALLOW_FILE_URI
 SAFE_PROTOCOL_REGEX='^(http|https|ftp|file):'
-
+WATCH_DB_JSON_FILENAME = 'watch.json'
 minimum_seconds_recheck_time = int(os.getenv('MINIMUM_SECONDS_RECHECK_TIME', 3))
 mtable = {'seconds': 1, 'minutes': 60, 'hours': 3600, 'days': 86400, 'weeks': 86400 * 7}
 
@@ -32,15 +32,20 @@ def is_safe_url(test_url):
     return True
 
 
-class model(watch_base):
-    __newest_history_key = None
-    __history_n = 0
-    jitter_seconds = 0
+class model(WatchBase):
+    __datastore = None
+    __datastore_checksum = None
 
+    __history_n = 0
+    __newest_history_key = None
+    jitter_seconds = 0
+    
     def __init__(self, *arg, **kw):
-        self.__datastore_path = kw.get('datastore_path')
-        if kw.get('datastore_path'):
-            del kw['datastore_path']
+        if not kw.get('__datastore'):
+            logger.critical('No __datastore reference was set!')
+
+        self.__datastore = kw.get('__datastore')
+
         super(model, self).__init__(*arg, **kw)
         if kw.get('default'):
             self.update(kw['default'])
@@ -179,7 +184,7 @@ class model(watch_base):
         tmp_history = {}
 
         # In the case we are only using the watch for processing without history
-        if not self.watch_data_dir:
+        if not self.__datastore or not self.watch_data_dir:
             return []
 
         # Read the history file as a dict
@@ -419,7 +424,7 @@ class model(watch_base):
     @property
     def watch_data_dir(self):
         # The base dir of the watch data
-        return os.path.join(self.__datastore_path, self['uuid']) if self.__datastore_path else None
+        return os.path.join(self.__datastore.datastore_path, self['uuid']) if self.__datastore.datastore_path else None
 
     def get_error_text(self):
         """Return the text saved from a previous request that resulted in a non-200 error"""
@@ -524,6 +529,22 @@ class model(watch_base):
         # None is set
         return False
 
+    def save_data(self):
+        import json
+        # @todo dict change?
+        # Save it to a temp file first so that if the disk is full or other error it wont corrupt (hopefully).
+
+        dest = os.path.join(self.watch_data_dir, WATCH_DB_JSON_FILENAME)
+        logger.debug(f"Saving watch {dest}")
+        try:
+            with open(dest + '.tmp', 'w') as json_file:
+                json.dump(self.as_dict(), json_file, indent=2)
+            os.replace(dest + '.tmp', dest)
+
+        except Exception as e:
+            logger.critical(f"Exception saving watch JSON {dest} - {e}")
+
+
     def save_error_text(self, contents):
         self.ensure_data_dir_exists()
         target_path = os.path.join(self.watch_data_dir, "last-error.txt")

changedetectionio/model/__init__.py

@@ -1,13 +1,14 @@
 import os
 import uuid
+from collections.abc import MutableMapping
 
 from changedetectionio import strtobool
 from changedetectionio.notification import default_notification_format_for_watch
 
-class watch_base(dict):
-
-    def __init__(self, *arg, **kw):
-        self.update({
+class WatchBase(MutableMapping):
+    __data_checksum = None
+    def __init__(self, *args, **kwargs):
+        self.__internal_dict = {
             # Custom notification content
             # Re #110, so then if this is set to None, we know to use the default value instead
             # Requires setting to None on submit if it's the same as the default
@@ -127,9 +128,37 @@ class watch_base(dict):
             'uuid': str(uuid.uuid4()),
             'webdriver_delay': None,
             'webdriver_js_execute_code': None,  # Run before change-detection
-        })
+        }
 
-        super(watch_base, self).__init__(*arg, **kw)
+        # Update with any provided arguments
+        self.update(*args, **kwargs)
 
         if self.get('default'):
-            del self['default']
+            del self['default']
+
+
+    # Implement abstract methods required by MutableMapping
+    def __getitem__(self, key):
+        return self.__internal_dict[key]
+
+    def __setitem__(self, key, value):
+        if key == '__datastore':
+            self.__datastore = value
+        else:
+            self.__internal_dict[key] = value
+
+    def __delitem__(self, key):
+        del self.__internal_dict[key]
+
+    def __iter__(self):
+        return iter(self.__internal_dict)
+
+    def __len__(self):
+        return len(self.__internal_dict)
+
+    # Optional: Implement additional methods for convenience
+    def __repr__(self):
+        return f"{self.__class__.__name__}({self.__internal_dict})"
+
+    def as_dict(self):
+        return self.__internal_dict

changedetectionio/notification.py

@@ -67,10 +67,6 @@ def process_notification(n_object, datastore):
 
     sent_objs = []
     from .apprise_asset import asset
-
-    if 'as_async' in n_object:
-        asset.async_mode = n_object.get('as_async')
-
     apobj = apprise.Apprise(debug=True, asset=asset)
 
     if not n_object.get('notification_urls'):
@@ -161,6 +157,8 @@ def process_notification(n_object, datastore):
             attach=n_object.get('screenshot', None)
         )
 
+        # Give apprise time to register an error
+        time.sleep(3)
 
         # Returns empty string if nothing found, multi-line string otherwise
         log_value = logs.getvalue()

changedetectionio/static/js/notifications.js

@@ -1,52 +1,42 @@
-$(document).ready(function () {
+$(document).ready(function() {
 
-    $('#add-email-helper').click(function (e) {
-        e.preventDefault();
-        email = prompt("Destination email");
-        if (email) {
-            var n = $(".notification-urls");
-            var p = email_notification_prefix;
-            $(n).val($.trim($(n).val()) + "\n" + email_notification_prefix + email);
+  $('#add-email-helper').click(function (e) {
+    e.preventDefault();
+    email = prompt("Destination email");
+    if(email) {
+      var n = $(".notification-urls");
+      var p=email_notification_prefix;
+      $(n).val( $.trim( $(n).val() )+"\n"+email_notification_prefix+email );
+    }
+  });
+
+  $('#send-test-notification').click(function (e) {
+    e.preventDefault();
+
+    data = {
+      notification_body: $('#notification_body').val(),
+      notification_format: $('#notification_format').val(),
+      notification_title: $('#notification_title').val(),
+      notification_urls: $('.notification-urls').val(),
+      tags: $('#tags').val(),
+      window_url: window.location.href,
+    }
+
+
+    $.ajax({
+      type: "POST",
+      url: notification_base_url,
+      data : data,
+        statusCode: {
+        400: function(data) {
+          // More than likely the CSRF token was lost when the server restarted
+          alert(data.responseText);
         }
-    });
-
-    $('#send-test-notification').click(function (e) {
-        e.preventDefault();
-
-        data = {
-            notification_body: $('#notification_body').val(),
-            notification_format: $('#notification_format').val(),
-            notification_title: $('#notification_title').val(),
-            notification_urls: $('.notification-urls').val(),
-            tags: $('#tags').val(),
-            window_url: window.location.href,
-        }
-
-        $('.notifications-wrapper .spinner').fadeIn();
-        $('#notification-test-log').show();
-        $.ajax({
-            type: "POST",
-            url: notification_base_url,
-            data: data,
-            statusCode: {
-                400: function (data) {
-                    $("#notification-test-log>span").text(data.responseText);
-                },
-            }
-        }).done(function (data) {
-            $("#notification-test-log>span").text(data);
-        }).fail(function (jqXHR, textStatus, errorThrown) {
-            // Handle connection refused or other errors
-            if (textStatus === "error" && errorThrown === "") {
-                console.error("Connection refused or server unreachable");
-                $("#notification-test-log>span").text("Error: Connection refused or server is unreachable.");
-            } else {
-                console.error("Error:", textStatus, errorThrown);
-                $("#notification-test-log>span").text("An error occurred: " + textStatus);
-            }
-        }).always(function () {
-            $('.notifications-wrapper .spinner').hide();
-        })
-    });
+      }
+    }).done(function(data){
+      console.log(data);
+      alert(data);
+    })
+  });
 });
 

changedetectionio/static/styles/scss/styles.scss

@@ -380,15 +380,7 @@ a.pure-button-selected {
 }
 
 .notifications-wrapper {
-  padding-top: 0.5rem;
-  #notification-test-log {
-    padding-top: 1rem;
-    white-space: pre-wrap;
-    word-break: break-word;
-    overflow-wrap: break-word;
-    max-width: 100%;
-    box-sizing: border-box;
-  }
+  padding: 0.5rem 0 1rem 0;
 }
 
 label {

changedetectionio/static/styles/styles.css

@@ -780,14 +780,7 @@ a.pure-button-selected {
   cursor: pointer; }
 
 .notifications-wrapper {
-  padding-top: 0.5rem; }
-  .notifications-wrapper #notification-test-log {
-    padding-top: 1rem;
-    white-space: pre-wrap;
-    word-break: break-word;
-    overflow-wrap: break-word;
-    max-width: 100%;
-    box-sizing: border-box; }
+  padding: 0.5rem 0 1rem 0; }
 
 label:hover {
   cursor: pointer; }

changedetectionio/store.py

@@ -1,3 +1,5 @@
+import glob
+
 from changedetectionio.strtobool import strtobool
 
 from flask import (
@@ -5,8 +7,8 @@ from flask import (
 )
 
 from .html_tools import TRANSLATE_WHITESPACE_TABLE
-from . model import App, Watch
-from copy import deepcopy, copy
+from .model import App, Watch, WatchBase
+from copy import deepcopy
 from os import path, unlink
 from threading import Lock
 import json
@@ -18,6 +20,7 @@ import time
 import uuid as uuid_builder
 from loguru import logger
 
+from .model.Watch import WATCH_DB_JSON_FILENAME
 from .processors import get_custom_watch_obj_for_processor
 from .processors.restock_diff import Restock
 
@@ -26,6 +29,13 @@ BASE_URL_NOT_SET_TEXT = '("Base URL" not set - see settings - notifications)'
 
 dictfilt = lambda x, y: dict([ (i,x[i]) for i in x if i in set(y) ])
 
+class CustomEncoder(json.JSONEncoder):
+    def default(self, obj):
+        if obj and isinstance(obj, WatchBase):
+            return obj.as_dict()
+        # Add more custom type handlers here
+        return super().default(obj)
+
 # Is there an existing library to ensure some data store (JSON etc) is in sync with CRUD methods?
 # Open a github issue if you know something :)
 # https://stackoverflow.com/questions/6190468/how-to-trigger-function-on-value-change
@@ -42,16 +52,14 @@ class ChangeDetectionStore:
     def __init__(self, datastore_path="/datastore", include_default_watches=True, version_tag="0.0.0"):
         # Should only be active for docker
         # logging.basicConfig(filename='/dev/stdout', level=logging.INFO)
+        from os.path import join
         self.__data = App.model()
         self.datastore_path = datastore_path
-        self.json_store_path = "{}/url-watches.json".format(self.datastore_path)
+        self.json_store_path = join(self.datastore_path, 'url-watches.json')
         logger.info(f"Datastore path is '{self.json_store_path}'")
         self.needs_write = False
         self.start_time = time.time()
         self.stop_thread = False
-        # Base definition for all watchers
-        # deepcopy part of #569 - not sure why its needed exactly
-        self.generic_definition = deepcopy(Watch.model(datastore_path = datastore_path, default={}))
 
         if path.isfile('changedetectionio/source.txt'):
             with open('changedetectionio/source.txt') as f:
@@ -65,10 +73,6 @@ class ChangeDetectionStore:
                 from_disk = json.load(json_file)
 
                 # @todo isnt there a way todo this dict.update recursively?
-                # Problem here is if the one on the disk is missing a sub-struct, it wont be present anymore.
-                if 'watching' in from_disk:
-                    self.__data['watching'].update(from_disk['watching'])
-
                 if 'app_guid' in from_disk:
                     self.__data['app_guid'] = from_disk['app_guid']
 
@@ -82,10 +86,7 @@ class ChangeDetectionStore:
                     if 'application' in from_disk['settings']:
                         self.__data['settings']['application'].update(from_disk['settings']['application'])
 
-                # Convert each existing watch back to the Watch.model object
-                for uuid, watch in self.__data['watching'].items():
-                    self.__data['watching'][uuid] = self.rehydrate_entity(uuid, watch)
-                    logger.info(f"Watching: {uuid} {watch['url']}")
+                self.scan_and_load_watches()
 
                 # And for Tags also, should be Restock type because it has extra settings
                 for uuid, tag in self.__data['settings']['application']['tags'].items():
@@ -158,9 +159,29 @@ class ChangeDetectionStore:
         if entity.get('uuid') != 'text_json_diff':
             logger.trace(f"Loading Watch object '{watch_class.__module__}.{watch_class.__name__}' for UUID {uuid}")
 
-        entity = watch_class(datastore_path=self.datastore_path, default=entity)
+        entity = watch_class(__datastore=self, default=entity)
         return entity
 
+    def scan_and_load_watches(self):
+
+        # Use glob to find all occurrences of 'watch.json' in subdirectories
+        # @todo move to some other function so we can trigger a rescan in a thread
+        for file_path in glob.glob(f"{self.datastore_path}/*/{WATCH_DB_JSON_FILENAME}", recursive=True):
+            try:
+                with open(file_path, 'r') as json_file:
+                    data = json.load(json_file)
+                    # So that we can always move it to another UUID by renaming the dir
+                    directory_path = os.path.dirname(file_path)
+                    uuid = os.path.basename(directory_path)
+                    if data.get('uuid'):
+                        del data['uuid']
+                    self.__data['watching'][uuid] = self.rehydrate_entity(uuid, data)
+
+            except json.JSONDecodeError as e:
+                logger.error(f"Error decoding JSON in file {file_path}: {e}")
+            except Exception as e:
+                logger.critical(f"Exception decoding JSON in file {file_path}: {e}")
+
     def set_last_viewed(self, uuid, timestamp):
         logger.debug(f"Setting watch UUID: {uuid} last viewed to {int(timestamp)}")
         self.data['watching'][uuid].update({'last_viewed': int(timestamp)})
@@ -177,13 +198,15 @@ class ChangeDetectionStore:
             return
 
         with self.lock:
+            # deepcopy part of #569 - not sure why its needed exactly
+#            self.generic_definition = deepcopy(Watch.model(default={}))
 
-            # In python 3.9 we have the |= dict operator, but that still will lose data on nested structures...
-            for dict_key, d in self.generic_definition.items():
-                if isinstance(d, dict):
-                    if update_obj is not None and dict_key in update_obj:
-                        self.__data['watching'][uuid][dict_key].update(update_obj[dict_key])
-                        del (update_obj[dict_key])
+#            # In python 3.9 we have the |= dict operator, but that still will lose data on nested structures...
+#            for dict_key, d in self.generic_definition.items():
+#                if isinstance(d, dict):
+#                    if update_obj is not None and dict_key in update_obj:
+#                        self.__data['watching'][uuid][dict_key].update(update_obj[dict_key])
+#                        del (update_obj[dict_key])
 
             self.__data['watching'][uuid].update(update_obj)
         self.needs_write = True
@@ -346,7 +369,7 @@ class ChangeDetectionStore:
 
         # If the processor also has its own Watch implementation
         watch_class = get_custom_watch_obj_for_processor(apply_extras.get('processor'))
-        new_watch = watch_class(datastore_path=self.datastore_path, url=url)
+        new_watch = watch_class(__datastore=self, url=url)
 
         new_uuid = new_watch.get('uuid')
 
@@ -383,7 +406,8 @@ class ChangeDetectionStore:
     def sync_to_json(self):
         logger.info("Saving JSON..")
         try:
-            data = deepcopy(self.__data)
+            data = {key: deepcopy(value) for key, value in self.__data.items() if key != 'watching'}
+
         except RuntimeError as e:
             # Try again in 15 seconds
             time.sleep(15)
@@ -397,11 +421,15 @@ class ChangeDetectionStore:
                 # This is a fairly basic strategy to deal with the case that the file is corrupted,
                 # system was out of memory, out of RAM etc
                 with open(self.json_store_path+".tmp", 'w') as json_file:
-                    json.dump(data, json_file, indent=4)
+                    json.dump(data, json_file, indent=2, cls=CustomEncoder)
                 os.replace(self.json_store_path+".tmp", self.json_store_path)
             except Exception as e:
                 logger.error(f"Error writing JSON!! (Main JSON file save was skipped) : {str(e)}")
 
+            # Write each watch to the disk (data in their own subdir) if it changed
+            for watch_uuid, watch in self.__data['watching'].items():
+                watch.save_data()
+
             self.needs_write = False
             self.needs_write_urgent = False
 
@@ -924,3 +952,25 @@ class ChangeDetectionStore:
                         f_d.write(zlib.compress(f_j.read()))
                         os.unlink(json_path)
 
+    # Move each 'watching' from a big JSON file to their own datafile in their data subdirectory
+    def update_20(self):
+        with open(self.json_store_path) as json_file:
+            data = json.load(json_file)
+            if data.get('watching'):
+                for uuid, watch in data['watching'].items():
+                    watch_data_dir = os.path.join(self.datastore_path, uuid)
+                    dest = os.path.join(watch_data_dir, WATCH_DB_JSON_FILENAME)
+
+                    try:
+                        if not os.path.isdir(watch_data_dir):
+                            logger.debug(f"> Creating data dir {watch_data_dir}")
+                            os.mkdir(watch_data_dir)
+                        with open(dest + '.tmp', 'w') as json_file:
+                            json.dump(watch, json_file, indent=2)
+                        os.replace(dest + '.tmp', dest)
+                        logger.info(f"Saved watch to {dest}")
+                    except Exception as e:
+                        logger.critical(f"Exception saving watch JSON {dest} - {e}")
+
+            self.data['watching'] = {}
+            self.scan_and_load_watches()

changedetectionio/templates/_common_fields.html

@@ -24,13 +24,11 @@
                               </ul>
                             </div>
                             <div class="notifications-wrapper">
-                              <a id="send-test-notification" class="pure-button button-secondary button-xsmall" >Send test notification</a> <div class="spinner"  style="display: none;"></div>
+                              <a id="send-test-notification" class="pure-button button-secondary button-xsmall" >Send test notification</a>
                             {% if emailprefix %}
                               <a id="add-email-helper" class="pure-button button-secondary button-xsmall" >Add email <img style="height: 1em; display: inline-block" src="{{url_for('static_content', group='images', filename='email.svg')}}" alt="Add an email address"> </a>
                             {% endif %}
                               <a href="{{url_for('notification_logs')}}" class="pure-button button-secondary button-xsmall" >Notification debug logs</a>
-                              <br>
-                                <div id="notification-test-log" style="display: none;"><span class="pure-form-message-inline">Processing..</span></div>
                             </div>
                         </div>
                         <div id="notification-customisation" class="pure-control-group">

changedetectionio/templates/edit.html

@@ -6,39 +6,6 @@
 <script src="{{url_for('static_content', group='js', filename='vis.js')}}" defer></script>
 <script src="{{url_for('static_content', group='js', filename='global-settings.js')}}" defer></script>
 <script src="{{url_for('static_content', group='js', filename='scheduler.js')}}" defer></script>
-
-<script>
-    function addRuleRow() {
-        let rulesContainer = document.getElementById("rules-container");
-        let lastRule = document.querySelector(".rule-row:last-child");
-        let newRule = lastRule.cloneNode(true);
-
-        // Get the new unique index for the added row
-        let ruleCount = document.querySelectorAll(".rule-row").length;
-
-        // Update all IDs, names, and labels to have the correct index
-        newRule.querySelectorAll("select, input, label").forEach(element => {
-            if (element.id) {
-                element.id = element.id.replace(/\d+/, ruleCount);
-            }
-            if (element.name) {
-                element.name = element.name.replace(/\d+/, ruleCount);
-            }
-            if (element.hasAttribute("for")) {
-                element.setAttribute("for", element.getAttribute("for").replace(/\d+/, ruleCount));
-            }
-            if (element.tagName === "INPUT") {
-                element.value = ""; // Clear input field value
-            }
-        });
-
-        // Append the new rule to the grid container
-        rulesContainer.appendChild(newRule);
-    }
-</script>
-
-
-
 <script>
     const browser_steps_available_screenshots=JSON.parse('{{ watch.get_browsersteps_available_screenshots|tojson }}');
     const browser_steps_config=JSON.parse('{{ browser_steps_config|tojson }}');
@@ -78,12 +45,12 @@
             {% if extra_tab_content %}
             <li class="tab"><a href="#extras_tab">{{ extra_tab_content }}</a></li>
             {% endif %}
+            {% if playwright_enabled %}
             <li class="tab"><a id="browsersteps-tab" href="#browser-steps">Browser Steps</a></li>
-        <!-- should goto extra forms? -->
+            {% endif %}
             {% if watch['processor'] == 'text_json_diff' %}
             <li class="tab"><a id="visualselector-tab" href="#visualselector">Visual Filter Selector</a></li>
             <li class="tab" id="filters-and-triggers-tab"><a href="#filters-and-triggers">Filters &amp; Triggers</a></li>
-            <li class="tab" id="conditions-tab"><a href="#conditions">Conditions</a></li>
             {% endif %}
             <li class="tab"><a href="#notifications">Notifications</a></li>
             <li class="tab"><a href="#stats">Stats</a></li>
@@ -232,15 +199,8 @@ Math: {{ 1 + 1 }}") }}
                     </div>
             </fieldset>
             </div>
-
-            <div class="tab-pane-inner" id="browser-steps">
-                            <span class="pure-form-message-inline">
-                                <p>Sorry, this functionality only works with Playwright/Chrome enabled watches.</p>
-                                <p>Enable the Playwright Chrome fetcher, or alternatively try our <a href="https://lemonade.changedetection.io/start">very affordable subscription based service</a>.</p>
-                                <p>This is because Selenium/WebDriver can not extract full page screenshots reliably.</p>
-                            </span>
-
             {% if playwright_enabled %}
+            <div class="tab-pane-inner" id="browser-steps">
                 <img class="beta-logo" src="{{url_for('static_content', group='images', filename='beta-logo.png')}}" alt="New beta functionality">
                 <fieldset>
                     <div class="pure-control-group">
@@ -280,9 +240,8 @@ Math: {{ 1 + 1 }}") }}
                         </div>
                     </div>
                 </fieldset>
-                {% endif %}
             </div>
-
+            {% endif %}
 
             <div class="tab-pane-inner" id="notifications">
                 <fieldset>
@@ -312,36 +271,6 @@ Math: {{ 1 + 1 }}") }}
             </div>
 
             {% if watch['processor'] == 'text_json_diff' %}
-
-            <div class="tab-pane-inner" id="conditions">
-                <!-- Grid Header -->
-                <table>
-                    <thead>
-                    <td>In Value</td>
-                    <td>Operator</td>
-                    <td>Value</td>
-                    <td></td>
-
-                    </thead>
-                    <tbody>
-                    <!-- Rule Rows (Dynamic Content) -->
-
-                    {% for rule in form.conditions %}
-                        <tr>
-                            <td>{{ rule.field() }}</td>
-
-                            <td>{{ rule.operator() }}</td>
-                            <td>{{ rule.value() }}</td>
-
-                            <td>
-                                <button type="button" onclick="addRuleRow()">AND +</button>
-                            </td>
-                        </tr>
-                    {% endfor %}
-                    </tbody>
-                </table>
-
-            </div>
             <div class="tab-pane-inner" id="filters-and-triggers">
                 <span id="activate-text-preview" class="pure-button pure-button-primary button-xsmall">Activate preview</span>
               <div>

changedetectionio/tests/fetchers/test_custom_js_before_content.py

@@ -34,7 +34,7 @@ def test_execute_custom_js(client, live_server, measure_memory_usage):
     assert b"unpaused" in res.data
     wait_for_all_checks(client)
 
-    uuid = next(iter(live_server.app.config['DATASTORE'].data['watching']))
+    uuid = extract_UUID_from_client(client)
     assert live_server.app.config['DATASTORE'].data['watching'][uuid].history_n >= 1, "Watch history had atleast 1 (everything fetched OK)"
 
     assert b"This text should be removed" not in res.data

changedetectionio/tests/proxy_list/test_noproxy.py

@@ -48,7 +48,7 @@ def test_noproxy_option(client, live_server, measure_memory_usage):
         follow_redirects=True
     )
     assert b"Watch added in Paused state, saving will unpause" in res.data
-    uuid = next(iter(live_server.app.config['DATASTORE'].data['watching']))
+    uuid = extract_UUID_from_client(client)
     res = client.get(
         url_for("edit_page", uuid=uuid, unpause_on_save=1))
     assert b'No proxy' in res.data

changedetectionio/tests/proxy_socks5/test_socks5_proxy.py

@@ -81,7 +81,7 @@ def test_socks5(client, live_server, measure_memory_usage):
     assert "Awesome, you made it".encode('utf-8') in res.data
 
     # PROXY CHECKER WIDGET CHECK - this needs more checking
-    uuid = next(iter(live_server.app.config['DATASTORE'].data['watching']))
+    uuid = extract_UUID_from_client(client)
 
     res = client.get(
         url_for("check_proxies.start_check", uuid=uuid),

changedetectionio/tests/test_add_replace_remove_filter.py

@@ -1,9 +1,9 @@
 #!/usr/bin/env python3
-
 import os.path
-
+import time
 from flask import url_for
 from .util import live_server_setup, wait_for_all_checks, wait_for_notification_endpoint_output
+from changedetectionio import html_tools
 
 
 def set_original(excluding=None, add_line=None):

changedetectionio/tests/test_api.py

@@ -44,6 +44,7 @@ def set_modified_response():
 
     return None
 
+
 def is_valid_uuid(val):
     try:
         uuid.UUID(str(val))
@@ -55,9 +56,8 @@ def is_valid_uuid(val):
 def test_setup(client, live_server, measure_memory_usage):
     live_server_setup(live_server)
 
-
 def test_api_simple(client, live_server, measure_memory_usage):
-#    live_server_setup(live_server)
+    #live_server_setup(live_server)
 
     api_key = extract_api_key_from_UI(client)
 
@@ -129,9 +129,6 @@ def test_api_simple(client, live_server, measure_memory_usage):
     assert after_recheck_info['last_checked'] != before_recheck_info['last_checked']
     assert after_recheck_info['last_changed'] != 0
 
-    # #2877 When run in a slow fetcher like playwright etc
-    assert after_recheck_info['last_changed'] ==  after_recheck_info['last_checked']
-
     # Check history index list
     res = client.get(
         url_for("watchhistory", uuid=watch_uuid),

changedetectionio/tests/test_automatic_follow_ldjson_price.py

@@ -99,7 +99,7 @@ def test_check_ldjson_price_autodetect(client, live_server, measure_memory_usage
     assert b'ldjson-price-track-offer' in res.data
 
     # Accept it
-    uuid = next(iter(live_server.app.config['DATASTORE'].data['watching']))
+    uuid = extract_UUID_from_client(client)
     #time.sleep(1)
     client.get(url_for('price_data_follower.accept', uuid=uuid, follow_redirects=True))
     client.get(url_for("form_watch_checknow"), follow_redirects=True)

changedetectionio/tests/test_backend.py

@@ -2,6 +2,7 @@
 
 import time
 from flask import url_for
+from urllib.request import urlopen
 from .util import set_original_response, set_modified_response, live_server_setup, wait_for_all_checks, extract_rss_token_from_UI, \
     extract_UUID_from_client
 
@@ -68,7 +69,7 @@ def test_check_basic_change_detection_functionality(client, live_server, measure
 
     wait_for_all_checks(client)
 
-    uuid = next(iter(live_server.app.config['DATASTORE'].data['watching']))
+    uuid = extract_UUID_from_client(client)
 
     # Check the 'get latest snapshot works'
     res = client.get(url_for("watch_get_latest_html", uuid=uuid))

changedetectionio/tests/test_encoding.py

@@ -40,7 +40,7 @@ def test_check_encoding_detection(client, live_server, measure_memory_usage):
 
 
     # Content type recording worked
-    uuid = next(iter(live_server.app.config['DATASTORE'].data['watching']))
+    uuid = extract_UUID_from_client(client)
     assert live_server.app.config['DATASTORE'].data['watching'][uuid]['content-type'] == "text/html"
 
     res = client.get(

changedetectionio/tests/test_filter_failure_notification.py

@@ -51,7 +51,7 @@ def run_filter_test(client, live_server, content_filter):
     assert b"1 Imported" in res.data
     wait_for_all_checks(client)
 
-    uuid = next(iter(live_server.app.config['DATASTORE'].data['watching']))
+    uuid = extract_UUID_from_client(client)
 
     assert live_server.app.config['DATASTORE'].data['watching'][uuid]['consecutive_filter_failures'] == 0, "No filter = No filter failure"
 

changedetectionio/tests/test_group.py

@@ -288,7 +288,7 @@ def test_clone_tag_on_import(client, live_server, measure_memory_usage):
     assert b'test-tag' in res.data
     assert b'another-tag' in res.data
 
-    watch_uuid = next(iter(live_server.app.config['DATASTORE'].data['watching']))
+    watch_uuid = extract_UUID_from_client(client)
     res = client.get(url_for("form_clone", uuid=watch_uuid), follow_redirects=True)
 
     assert b'Cloned' in res.data
@@ -315,7 +315,7 @@ def test_clone_tag_on_quickwatchform_add(client, live_server, measure_memory_usa
     assert b'test-tag' in res.data
     assert b'another-tag' in res.data
 
-    watch_uuid = next(iter(live_server.app.config['DATASTORE'].data['watching']))
+    watch_uuid = extract_UUID_from_client(client)
     res = client.get(url_for("form_clone", uuid=watch_uuid), follow_redirects=True)
 
     assert b'Cloned' in res.data

changedetectionio/tests/test_history_consistency.py

@@ -8,11 +8,19 @@ from flask import url_for
 from .util import live_server_setup, wait_for_all_checks
 from urllib.parse import urlparse, parse_qs
 
+from ..model.Watch import WATCH_DB_JSON_FILENAME
+
+
 def test_consistent_history(client, live_server, measure_memory_usage):
     live_server_setup(live_server)
 
+    import glob
     r = range(1, 30)
 
+    # incase some exist from a previous test
+    for f in glob.glob(f"{live_server.app.config['DATASTORE'].datastore_path}/*/{WATCH_DB_JSON_FILENAME}", recursive=True):
+        os.unlink(f)
+
     for one in r:
         test_url = url_for('test_endpoint', content_type="text/html", content=str(one), _external=True)
         res = client.post(
@@ -44,11 +52,17 @@ def test_consistent_history(client, live_server, measure_memory_usage):
     with open(json_db_file, 'r') as f:
         json_obj = json.load(f)
 
+
+    found_db_jsons = glob.glob(f"{live_server.app.config['DATASTORE'].datastore_path}/*/{WATCH_DB_JSON_FILENAME}", recursive=True)
     # assert the right amount of watches was found in the JSON
-    assert len(json_obj['watching']) == len(r), "Correct number of watches was found in the JSON"
+    assert len(found_db_jsons) == len(r), "Correct number of watches was found in the JSON"
 
     # each one should have a history.txt containing just one line
-    for w in json_obj['watching'].keys():
+    for json_db_file in found_db_jsons:
+
+        directory_path = os.path.dirname(json_db_file)
+        w = os.path.basename(directory_path)
+
         history_txt_index_file = os.path.join(live_server.app.config['DATASTORE'].datastore_path, w, 'history.txt')
         assert os.path.isfile(history_txt_index_file), f"History.txt should exist where I expect it at {history_txt_index_file}"
 
@@ -58,22 +72,21 @@ def test_consistent_history(client, live_server, measure_memory_usage):
             assert len(tmp_history) == 1, "History.txt should contain 1 line"
 
         # Should be two files,. the history.txt , and the snapshot.txt
-        files_in_watch_dir = os.listdir(os.path.join(live_server.app.config['DATASTORE'].datastore_path,
-                                                     w))
+        files_in_watch_dir = os.listdir(os.path.join(live_server.app.config['DATASTORE'].datastore_path,w))
         # Find the snapshot one
-        for fname in files_in_watch_dir:
-            if fname != 'history.txt' and 'html' not in fname:
-                # contents should match what we requested as content returned from the test url
-                with open(os.path.join(live_server.app.config['DATASTORE'].datastore_path, w, fname), 'r') as snapshot_f:
-                    contents = snapshot_f.read()
-                    watch_url = json_obj['watching'][w]['url']
-                    u = urlparse(watch_url)
-                    q = parse_qs(u[4])
-                    assert q['content'][0] == contents.strip(), f"Snapshot file {fname} should contain {q['content'][0]}"
+#        for fname in files_in_watch_dir:
+#            if fname != 'history.txt' and 'html' not in fname and fname != WATCH_DB_JSON_FILENAME:
+#                # contents should match what we requested as content returned from the test url
+#                with open(os.path.join(live_server.app.config['DATASTORE'].datastore_path, w, fname), 'r') as snapshot_f:
+#                    contents = snapshot_f.read()
+#                    watch_url = json_obj['watching'][w]['url']
+#                    u = urlparse(watch_url)
+#                    q = parse_qs(u[4])
+#                    assert q['content'][0] == contents.strip(), f"Snapshot file {fname} should contain {q['content'][0]}"
 
 
 
-        assert len(files_in_watch_dir) == 3, "Should be just three files in the dir, html.br snapshot, history.txt and the extracted text snapshot"
+        assert len(files_in_watch_dir) == 4, "Should be just four files in the dir, html.br snapshot, history.txt, watch.json and the extracted text snapshot"
 
 
     json_db_file = os.path.join(live_server.app.config['DATASTORE'].datastore_path, 'url-watches.json')

changedetectionio/tests/test_ignore.py

@@ -36,7 +36,7 @@ def test_ignore(client, live_server, measure_memory_usage):
 
     # Give the thread time to pick it up
     wait_for_all_checks(client)
-    uuid = next(iter(live_server.app.config['DATASTORE'].data['watching']))
+    uuid = extract_UUID_from_client(client)
     # use the highlighter endpoint
     res = client.post(
         url_for("highlight_submit_ignore_url", uuid=uuid),

changedetectionio/tests/test_jsonpath_jq_selector.py

@@ -514,15 +514,3 @@ def test_check_jq_ext_filter(client, live_server, measure_memory_usage):
 def test_check_jqraw_ext_filter(client, live_server, measure_memory_usage):
     if jq_support:
         check_json_ext_filter('jq:.[] | select(.status | contains("Sold"))', client, live_server)
-
-def test_jsonpath_BOM_utf8(client, live_server, measure_memory_usage):
-    from .. import html_tools
-
-    # JSON string with BOM and correct double-quoted keys
-    json_str = '\ufeff{"name": "José", "emoji": "😊", "language": "中文", "greeting": "Привет"}'
-
-    # See that we can find the second <script> one, which is not broken, and matches our filter
-    text = html_tools.extract_json_as_string(json_str, "json:$.name")
-    assert text == '"José"'
-
-    

changedetectionio/tests/test_live_preview.py

@@ -29,7 +29,7 @@ def test_content_filter_live_preview(client, live_server, measure_memory_usage):
         data={"url": test_url, "tags": ''},
         follow_redirects=True
     )
-    uuid = next(iter(live_server.app.config['DATASTORE'].data['watching']))
+    uuid = extract_UUID_from_client(client)
     res = client.post(
         url_for("edit_page", uuid=uuid),
         data={

changedetectionio/tests/test_nonrenderable_pages.py

@@ -48,7 +48,7 @@ def test_check_basic_change_detection_functionality(client, live_server, measure
     #####################
     client.post(
         url_for("settings_page"),
-        data={"application-empty_pages_are_a_change": "", # default, OFF, they are NOT a change
+        data={"application-empty_pages_are_a_change": "",
               "requests-time_between_check-minutes": 180,
               'application-fetch_backend': "html_requests"},
         follow_redirects=True
@@ -66,14 +66,6 @@ def test_check_basic_change_detection_functionality(client, live_server, measure
     res = client.get(url_for("index"))
     assert b'unviewed' not in res.data
 
-    uuid = next(iter(live_server.app.config['DATASTORE'].data['watching']))
-    watch = live_server.app.config['DATASTORE'].data['watching'][uuid]
-
-    assert watch.last_changed == 0
-    assert watch['last_checked'] != 0
-
-
-
 
     # ok now do the opposite
 
@@ -100,10 +92,6 @@ def test_check_basic_change_detection_functionality(client, live_server, measure
     # A totally zero byte (#2528) response should also not trigger an error
     set_zero_byte_response()
     client.get(url_for("form_watch_checknow"), follow_redirects=True)
-
-    # 2877
-    assert watch.last_changed == watch['last_checked']
-
     wait_for_all_checks(client)
     res = client.get(url_for("index"))
     assert b'unviewed' in res.data # A change should have registered because empty_pages_are_a_change is ON

changedetectionio/tests/test_notification.py

@@ -6,7 +6,7 @@ from flask import url_for
 from loguru import logger
 
 from .util import set_original_response, set_modified_response, set_more_modified_response, live_server_setup, wait_for_all_checks, \
-    set_longer_modified_response, get_index
+    set_longer_modified_response
 from . util import  extract_UUID_from_client
 import logging
 import base64
@@ -29,7 +29,7 @@ def test_check_notification(client, live_server, measure_memory_usage):
 
     # Re 360 - new install should have defaults set
     res = client.get(url_for("settings_page"))
-    notification_url = url_for('test_notification_endpoint', _external=True).replace('http', 'json')+"?status_code=204"
+    notification_url = url_for('test_notification_endpoint', _external=True).replace('http', 'json')
 
     assert default_notification_body.encode() in res.data
     assert default_notification_title.encode() in res.data
@@ -76,7 +76,7 @@ def test_check_notification(client, live_server, measure_memory_usage):
     testimage_png = 'iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mNkYAAAAAYAAjCB0C8AAAAASUVORK5CYII='
 
 
-    uuid = next(iter(live_server.app.config['DATASTORE'].data['watching']))
+    uuid = extract_UUID_from_client(client)
     datastore = 'test-datastore'
     with open(os.path.join(datastore, str(uuid), 'last-screenshot.png'), 'wb') as f:
         f.write(base64.b64decode(testimage_png))
@@ -135,14 +135,7 @@ def test_check_notification(client, live_server, measure_memory_usage):
 
     # Trigger a check
     client.get(url_for("form_watch_checknow"), follow_redirects=True)
-    wait_for_all_checks(client)
     time.sleep(3)
-
-    # Check no errors were recorded
-    res = client.get(url_for("index"))
-    assert b'notification-error' not in res.data
-
-
     # Verify what was sent as a notification, this file should exist
     with open("test-datastore/notification.txt", "r") as f:
         notification_submission = f.read()
@@ -291,7 +284,7 @@ def test_notification_custom_endpoint_and_jinja2(client, live_server, measure_me
     # CUSTOM JSON BODY CHECK for POST://
     set_original_response()
     # https://github.com/caronc/apprise/wiki/Notify_Custom_JSON#header-manipulation
-    test_notification_url = url_for('test_notification_endpoint', _external=True).replace('http://', 'post://')+"?status_code=204&xxx={{ watch_url }}&+custom-header=123&+second=hello+world%20%22space%22"
+    test_notification_url = url_for('test_notification_endpoint', _external=True).replace('http://', 'post://')+"?xxx={{ watch_url }}&+custom-header=123&+second=hello+world%20%22space%22"
 
     res = client.post(
         url_for("settings_page"),
@@ -326,11 +319,6 @@ def test_notification_custom_endpoint_and_jinja2(client, live_server, measure_me
 
     time.sleep(2) # plus extra delay for notifications to fire
 
-
-    # Check no errors were recorded, because we asked for 204 which is slightly uncommon but is still OK
-    res = get_index(client)
-    assert b'notification-error' not in res.data
-
     with open("test-datastore/notification.txt", 'r') as f:
         x = f.read()
         j = json.loads(x)
@@ -372,10 +360,7 @@ def test_global_send_test_notification(client, live_server, measure_memory_usage
     #live_server_setup(live_server)
     set_original_response()
     if os.path.isfile("test-datastore/notification.txt"):
-        os.unlink("test-datastore/notification.txt") \
-
-    # 1995 UTF-8 content should be encoded
-    test_body = 'change detection is cool 网站监测 内容更新了'
+        os.unlink("test-datastore/notification.txt")
 
     # otherwise other settings would have already existed from previous tests in this file
     res = client.post(
@@ -383,7 +368,8 @@ def test_global_send_test_notification(client, live_server, measure_memory_usage
         data={
             "application-fetch_backend": "html_requests",
             "application-minutes_between_check": 180,
-            "application-notification_body": test_body,
+            #1995 UTF-8 content should be encoded
+            "application-notification_body": 'change detection is cool 网站监测 内容更新了',
             "application-notification_format": default_notification_format,
             "application-notification_urls": "",
             "application-notification_title": "New ChangeDetection.io Notification - {{ watch_url }}",
@@ -413,10 +399,12 @@ def test_global_send_test_notification(client, live_server, measure_memory_usage
     assert res.status_code != 400
     assert res.status_code != 500
 
+    # Give apprise time to fire
+    time.sleep(4)
 
     with open("test-datastore/notification.txt", 'r') as f:
         x = f.read()
-        assert test_body in x
+        assert 'change detection is cool 网站监测 内容更新了' in x
 
     os.unlink("test-datastore/notification.txt")
 

changedetectionio/tests/test_request.py

@@ -2,7 +2,7 @@ import json
 import os
 import time
 from flask import url_for
-from . util import set_original_response, set_modified_response, live_server_setup, wait_for_all_checks, extract_UUID_from_client
+from .util import live_server_setup, wait_for_all_checks, extract_UUID_from_client
 
 def test_setup(live_server):
     live_server_setup(live_server)
@@ -74,8 +74,8 @@ def test_headers_in_request(client, live_server, measure_memory_usage):
     # Re #137 -  It should have only one set of headers entered
     watches_with_headers = 0
     for k, watch in client.application.config.get('DATASTORE').data.get('watching').items():
-            if (len(watch['headers'])):
-                watches_with_headers += 1
+        if (len(watch['headers'])):
+            watches_with_headers += 1
     assert watches_with_headers == 1
 
     # 'server' http header was automatically recorded
@@ -156,11 +156,10 @@ def test_body_in_request(client, live_server, measure_memory_usage):
     assert b"1 Imported" in res.data
 
     watches_with_body = 0
-    with open('test-datastore/url-watches.json') as f:
-        app_struct = json.load(f)
-        for uuid in app_struct['watching']:
-            if app_struct['watching'][uuid]['body']==body_value:
-                watches_with_body += 1
+
+    for k, watch in client.application.config.get('DATASTORE').data.get('watching').items():
+        if watch['body'] == body_value:
+            watches_with_body += 1
 
     # Should be only one with body set
     assert watches_with_body==1
@@ -244,11 +243,9 @@ def test_method_in_request(client, live_server, measure_memory_usage):
     wait_for_all_checks(client)
 
     watches_with_method = 0
-    with open('test-datastore/url-watches.json') as f:
-        app_struct = json.load(f)
-        for uuid in app_struct['watching']:
-            if app_struct['watching'][uuid]['method'] == 'PATCH':
-                watches_with_method += 1
+    for k, watch in client.application.config.get('DATASTORE').data.get('watching').items():
+        if watch['method'] == 'PATCH':
+            watches_with_method += 1
 
     # Should be only one with method set to PATCH
     assert watches_with_method == 1
@@ -373,14 +370,13 @@ def test_headers_textfile_in_request(client, live_server, measure_memory_usage):
     wait_for_all_checks(client)
 
     with open('test-datastore/headers-testtag.txt', 'w') as f:
-        f.write("tag-header: test\r\nurl-header: http://example.com")
+        f.write("tag-header: test")
 
     with open('test-datastore/headers.txt', 'w') as f:
-        f.write("global-header: nice\r\nnext-global-header: nice\r\nurl-header-global: http://example.com/global")
+        f.write("global-header: nice\r\nnext-global-header: nice")
 
-    uuid = next(iter(live_server.app.config['DATASTORE'].data['watching']))
-    with open(f'test-datastore/{uuid}/headers.txt', 'w') as f:
-        f.write("watch-header: nice\r\nurl-header-watch: http://example.com/watch")
+    with open('test-datastore/' + extract_UUID_from_client(client) + '/headers.txt', 'w') as f:
+        f.write("watch-header: nice")
 
     wait_for_all_checks(client)
     client.get(url_for("form_watch_checknow"), follow_redirects=True)
@@ -411,9 +407,6 @@ def test_headers_textfile_in_request(client, live_server, measure_memory_usage):
     assert b"Xxx:ooo" in res.data
     assert b"Watch-Header:nice" in res.data
     assert b"Tag-Header:test" in res.data
-    assert b"Url-Header:http://example.com" in res.data
-    assert b"Url-Header-Global:http://example.com/global" in res.data
-    assert b"Url-Header-Watch:http://example.com/watch" in res.data
 
     # Check the custom UA from system settings page made it through
     if os.getenv('PLAYWRIGHT_DRIVER_URL'):

changedetectionio/tests/test_restock_itemprop.py

@@ -189,17 +189,6 @@ def _run_test_minmax_limit(client, extra_watch_edit_form):
 
     client.get(url_for("mark_all_viewed"))
 
-
-    # 2715 - Price detection (once it crosses the "lower" threshold) again with a lower price - should trigger again!
-    set_original_response(props_markup=instock_props[0], price='820.45')
-    res = client.get(url_for("form_watch_checknow"), follow_redirects=True)
-    assert b'1 watches queued for rechecking.' in res.data
-    wait_for_all_checks(client)
-    res = client.get(url_for("index"))
-    assert b'820.45' in res.data
-    assert b'unviewed' in res.data
-    client.get(url_for("mark_all_viewed"))
-
     # price changed to something MORE than max (1100.10), SHOULD be a change
     set_original_response(props_markup=instock_props[0], price='1890.45')
     client.get(url_for("form_watch_checknow"), follow_redirects=True)
@@ -214,7 +203,7 @@ def _run_test_minmax_limit(client, extra_watch_edit_form):
 
 
 def test_restock_itemprop_minmax(client, live_server):
-    #live_server_setup(live_server)
+#    live_server_setup(live_server)
     extras = {
         "restock_settings-follow_price_changes": "y",
         "restock_settings-price_change_min": 900.0,
@@ -380,7 +369,7 @@ def test_change_with_notification_values(client, live_server):
 
     ## Now test the "SEND TEST NOTIFICATION" is working
     os.unlink("test-datastore/notification.txt")
-    uuid = next(iter(live_server.app.config['DATASTORE'].data['watching']))
+    uuid = extract_UUID_from_client(client)
     res = client.post(url_for("ajax_callback_send_notification_test", watch_uuid=uuid), data={}, follow_redirects=True)
     time.sleep(5)
     assert os.path.isfile("test-datastore/notification.txt"), "Notification received"

changedetectionio/tests/test_rss.py

@@ -132,7 +132,7 @@ def test_rss_xpath_filtering(client, live_server, measure_memory_usage):
     )
     assert b"Watch added in Paused state, saving will unpause" in res.data
 
-    uuid = next(iter(live_server.app.config['DATASTORE'].data['watching']))
+    uuid = extract_UUID_from_client(client)
     res = client.post(
         url_for("edit_page", uuid=uuid, unpause_on_save=1),
         data={

changedetectionio/tests/test_scheduler.py

@@ -39,7 +39,7 @@ def test_check_basic_scheduler_functionality(client, live_server, measure_memory
 
     assert b"1 Imported" in res.data
     wait_for_all_checks(client)
-    uuid = next(iter(live_server.app.config['DATASTORE'].data['watching']))
+    uuid = extract_UUID_from_client(client)
 
     # Setup all the days of the weeks using XXX as the placeholder for monday/tuesday/etc
 
@@ -104,7 +104,7 @@ def test_check_basic_global_scheduler_functionality(client, live_server, measure
 
     assert b"1 Imported" in res.data
     wait_for_all_checks(client)
-    uuid = next(iter(live_server.app.config['DATASTORE'].data['watching']))
+    uuid = extract_UUID_from_client(client)
 
     # Setup all the days of the weeks using XXX as the placeholder for monday/tuesday/etc
 

changedetectionio/tests/unit/test_semver.py

@@ -1,64 +0,0 @@
-#!/usr/bin/env python3
-
-# run from dir above changedetectionio/ dir
-# python3 -m unittest changedetectionio.tests.unit.test_semver
-
-import re
-import unittest
-
-
-# The SEMVER regex
-SEMVER_REGEX = r"^(?P<major>0|[1-9]\d*)\.(?P<minor>0|[1-9]\d*)\.(?P<patch>0|[1-9]\d*)(?:-(?P<prerelease>(?:0|[1-9]\d*|\d*[a-zA-Z-][0-9a-zA-Z-]*)(?:\.(?:0|[1-9]\d*|\d*[a-zA-Z-][0-9a-zA-Z-]*))*))?(?:\+(?P<buildmetadata>[0-9a-zA-Z-]+(?:\.[0-9a-zA-Z-]+)*))?$"
-
-# Compile the regex
-semver_pattern = re.compile(SEMVER_REGEX)
-
-class TestSemver(unittest.TestCase):
-    def test_valid_versions(self):
-        """Test valid semantic version strings"""
-        valid_versions = [
-            "1.0.0",
-            "0.1.0",
-            "0.0.1",
-            "1.0.0-alpha",
-            "1.0.0-alpha.1",
-            "1.0.0-0.3.7",
-            "1.0.0-x.7.z.92",
-            "1.0.0-alpha+001",
-            "1.0.0+20130313144700",
-            "1.0.0-beta+exp.sha.5114f85"
-        ]
-        for version in valid_versions:
-            with self.subTest(version=version):
-                self.assertIsNotNone(semver_pattern.match(version), f"Version {version} should be valid")
-
-    def test_invalid_versions(self):
-        """Test invalid semantic version strings"""
-        invalid_versions = [
-            "0.48.06",
-            "1.0",
-            "1.0.0-",
-# Seems to pass the semver.org regex?
-#            "1.0.0-alpha-",
-            "1.0.0+",
-            "1.0.0-alpha+",
-            "1.0.0-",
-            "01.0.0",
-            "1.01.0",
-            "1.0.01",
-            ".1.0.0",
-            "1..0.0"
-        ]
-        for version in invalid_versions:
-            with self.subTest(version=version):
-                res = semver_pattern.match(version)
-                self.assertIsNone(res, f"Version '{version}' should be invalid")
-
-    def test_our_version(self):
-        from changedetectionio import get_version
-        our_version = get_version()
-        self.assertIsNotNone(semver_pattern.match(our_version), f"Our version '{our_version}' should be a valid SEMVER string")
-
-
-if __name__ == '__main__':
-    unittest.main()

changedetectionio/tests/unit/test_watch_model.py

@@ -6,6 +6,7 @@
 import unittest
 import os
 
+from changedetectionio import store
 from changedetectionio.model import Watch
 
 # mostly
@@ -13,7 +14,8 @@ class TestDiffBuilder(unittest.TestCase):
 
     def test_watch_get_suggested_from_diff_timestamp(self):
         import uuid as uuid_builder
-        watch = Watch.model(datastore_path='/tmp', default={})
+        datastore = store.ChangeDetectionStore(datastore_path='/tmp')
+        watch = Watch.model(__datastore=datastore, default={})
         watch.ensure_data_dir_exists()
 
 
@@ -49,7 +51,7 @@ class TestDiffBuilder(unittest.TestCase):
         assert p == "109", "Correct when its the same time"
 
         # new empty one
-        watch = Watch.model(datastore_path='/tmp', default={})
+        watch = Watch.model(__datastore=datastore, default={})
         p = watch.get_from_version_based_on_last_viewed
         assert p == None, "None when no history available"
 
@@ -61,5 +63,6 @@ class TestDiffBuilder(unittest.TestCase):
         p = watch.get_from_version_based_on_last_viewed
         assert p == "100", "Correct with only one history snapshot"
 
+        datastore.stop_thread = True
 if __name__ == '__main__':
     unittest.main()

changedetectionio/tests/util.py

@@ -76,18 +76,11 @@ def set_more_modified_response():
     return None
 
 
-def set_empty_text_response():
-    test_return_data = """<html><body></body></html>"""
-
-    with open("test-datastore/endpoint-content.txt", "w") as f:
-        f.write(test_return_data)
-
-    return None
-
 def wait_for_notification_endpoint_output():
     '''Apprise can take a few seconds to fire'''
     #@todo - could check the apprise object directly instead of looking for this file
     from os.path import isfile
+
     for i in range(1, 20):
         time.sleep(1)
         if isfile("test-datastore/notification.txt"):
@@ -223,10 +216,9 @@ def live_server_setup(live_server):
     def test_method():
         return request.method
 
-    # Where we POST to as a notification, also use a space here to test URL escaping is OK across all tests that use this. ( #2868 )
-    @live_server.app.route('/test_notification endpoint', methods=['POST', 'GET'])
+    # Where we POST to as a notification
+    @live_server.app.route('/test_notification_endpoint', methods=['POST', 'GET'])
     def test_notification_endpoint():
-
         with open("test-datastore/notification.txt", "wb") as f:
             # Debug method, dump all POST to file also, used to prove #65
             data = request.stream.read()
@@ -244,11 +236,8 @@ def live_server_setup(live_server):
                 f.write(request.content_type)
 
         print("\n>> Test notification endpoint was hit.\n", data)
+        return "Text was set"
 
-        content = "Text was set"
-        status_code = request.args.get('status_code',200)
-        resp = make_response(content, status_code)
-        return resp
 
     # Just return the verb in the request
     @live_server.app.route('/test-basicauth', methods=['GET'])
@@ -285,43 +274,15 @@ def live_server_setup(live_server):
             <p id="remove">This text should be removed</p>
               <form onsubmit="event.preventDefault();">
             <!-- obfuscated text so that we dont accidentally get a false positive due to conversion of the source :) --->
-                <button name="test-button" onclick="
-                getElementById('remove').remove();
-                getElementById('some-content').innerHTML = atob('SSBzbWVsbCBKYXZhU2NyaXB0IGJlY2F1c2UgdGhlIGJ1dHRvbiB3YXMgcHJlc3NlZCE=');
-                getElementById('reflect-text').innerHTML = getElementById('test-input-text').value;
-                ">Click here</button>
-                
-                <div id="some-content"></div>
-                
+                <button name="test-button" onclick="getElementById('remove').remove();getElementById('some-content').innerHTML = atob('SSBzbWVsbCBKYXZhU2NyaXB0IGJlY2F1c2UgdGhlIGJ1dHRvbiB3YXMgcHJlc3NlZCE=')">Click here</button>
+                <div id=some-content></div>
                 <pre>
                 {header_text.lower()}
                 </pre>
-                
-                <br>
-                <!-- used for testing that the jinja2 compiled here --->
-                <input type="text" value="" id="test-input-text" /><br>
-                <div id="reflect-text">Waiting to reflect text from #test-input-text here</div>
-              </form>
-                
-           </body>
+              </body>
          </html>""", 200)
         resp.headers['Content-Type'] = 'text/html'
         return resp
 
     live_server.start()
 
-def get_index(client):
-    import inspect
-    # Get the caller's frame (parent function)
-    frame = inspect.currentframe()
-    caller_frame = frame.f_back  # Go back to the caller's frame
-    caller_name = caller_frame.f_code.co_name
-    caller_line = caller_frame.f_lineno
-
-    print(f"Called by: {caller_name}, Line: {caller_line}")
-
-    res = client.get(url_for("index"))
-    with open(f"test-datastore/index-{caller_name}-{caller_line}.html", 'wb') as f:
-        f.write(res.data)
-
-    return res

changedetectionio/tests/visualselector/test_fetch_data.py

@@ -2,16 +2,14 @@
 
 import os
 from flask import url_for
-from ..util import live_server_setup, wait_for_all_checks, get_index
+from ..util import live_server_setup, wait_for_all_checks, extract_UUID_from_client
 
-def test_setup(client, live_server):
+def test_setup(client, live_server, measure_memory_usage):
     live_server_setup(live_server)
 
 
 # Add a site in paused mode, add an invalid filter, we should still have visual selector data ready
 def test_visual_selector_content_ready(client, live_server, measure_memory_usage):
-    live_server.stop()
-    live_server.start()
 
     import os
     import json
@@ -29,7 +27,7 @@ def test_visual_selector_content_ready(client, live_server, measure_memory_usage
         follow_redirects=True
     )
     assert b"Watch added in Paused state, saving will unpause" in res.data
-    uuid = next(iter(live_server.app.config['DATASTORE'].data['watching']))
+    uuid = extract_UUID_from_client(client)
     res = client.post(
         url_for("edit_page", uuid=uuid, unpause_on_save=1),
         data={
@@ -89,9 +87,7 @@ def test_visual_selector_content_ready(client, live_server, measure_memory_usage
 
 def test_basic_browserstep(client, live_server, measure_memory_usage):
 
-    live_server.stop()
-    live_server.start()
-
+    #live_server_setup(live_server)
     assert os.getenv('PLAYWRIGHT_DRIVER_URL'), "Needs PLAYWRIGHT_DRIVER_URL set for this test"
 
     test_url = url_for('test_interactive_html_endpoint', _external=True)
@@ -112,13 +108,9 @@ def test_basic_browserstep(client, live_server, measure_memory_usage):
             "url": test_url,
             "tags": "",
             'fetch_backend': "html_webdriver",
-            'browser_steps-0-operation': 'Enter text in field',
-            'browser_steps-0-selector': '#test-input-text',
-            # Should get set to the actual text (jinja2 rendered)
-            'browser_steps-0-optional_value': "Hello-Jinja2-{% now  'Europe/Berlin', '%Y-%m-%d' %}",
-            'browser_steps-1-operation': 'Click element',
-            'browser_steps-1-selector': 'button[name=test-button]',
-            'browser_steps-1-optional_value': '',
+            'browser_steps-0-operation': 'Click element',
+            'browser_steps-0-selector': 'button[name=test-button]',
+            'browser_steps-0-optional_value': '',
             # For now, cookies doesnt work in headers because it must be a full cookiejar object
             'headers': "testheader: yes\buser-agent: MyCustomAgent",
         },
@@ -127,7 +119,7 @@ def test_basic_browserstep(client, live_server, measure_memory_usage):
     assert b"unpaused" in res.data
     wait_for_all_checks(client)
 
-    uuid = next(iter(live_server.app.config['DATASTORE'].data['watching']))
+    uuid = extract_UUID_from_client(client)
     assert live_server.app.config['DATASTORE'].data['watching'][uuid].history_n >= 1, "Watch history had atleast 1 (everything fetched OK)"
 
     assert b"This text should be removed" not in res.data
@@ -140,32 +132,13 @@ def test_basic_browserstep(client, live_server, measure_memory_usage):
     assert b"This text should be removed" not in res.data
     assert b"I smell JavaScript because the button was pressed" in res.data
 
-    assert b'Hello-Jinja2-20' in res.data
-
     assert b"testheader: yes" in res.data
     assert b"user-agent: mycustomagent" in res.data
-    live_server.stop()
-
-def test_non_200_errors_report_browsersteps(client, live_server):
-
-    live_server.stop()
-    live_server.start()
 
     four_o_four_url =  url_for('test_endpoint', status_code=404, _external=True)
     four_o_four_url = four_o_four_url.replace('localhost.localdomain', 'cdio')
     four_o_four_url = four_o_four_url.replace('localhost', 'cdio')
 
-    res = client.post(
-        url_for("form_quick_watch_add"),
-        data={"url": four_o_four_url, "tags": '', 'edit_and_watch_submit_button': 'Edit > Watch'},
-        follow_redirects=True
-    )
-
-    assert b"Watch added in Paused state, saving will unpause" in res.data
-    assert os.getenv('PLAYWRIGHT_DRIVER_URL'), "Needs PLAYWRIGHT_DRIVER_URL set for this test"
-
-    uuid = next(iter(live_server.app.config['DATASTORE'].data['watching']))
-
     # now test for 404 errors
     res = client.post(
         url_for("edit_page", uuid=uuid, unpause_on_save=1),
@@ -180,14 +153,12 @@ def test_non_200_errors_report_browsersteps(client, live_server):
         follow_redirects=True
     )
     assert b"unpaused" in res.data
-
     wait_for_all_checks(client)
 
-    res = get_index(client)
-
+    res = client.get(url_for("index"))
     assert b'Error - 404' in res.data
 
     client.get(
         url_for("form_delete", uuid="all"),
         follow_redirects=True
-    )
+    )

changedetectionio/update_worker.py

@@ -243,6 +243,7 @@ class update_worker(threading.Thread):
                 os.unlink(full_path)
 
     def run(self):
+        now = time.time()
         
         while not self.app.config.exit.is_set():
             update_handler = None
@@ -253,7 +254,6 @@ class update_worker(threading.Thread):
                 pass
 
             else:
-                fetch_start_time = time.time()
                 uuid = queued_item_data.item.get('uuid')
                 self.current_uuid = uuid
                 if uuid in list(self.datastore.data['watching'].keys()) and self.datastore.data['watching'][uuid].get('url'):
@@ -268,6 +268,7 @@ class update_worker(threading.Thread):
                     watch = self.datastore.data['watching'].get(uuid)
 
                     logger.info(f"Processing watch UUID {uuid} Priority {queued_item_data.priority} URL {watch['url']}")
+                    now = time.time()
 
                     try:
                         # Processor is what we are using for detecting the "Change"
@@ -281,16 +282,13 @@ class update_worker(threading.Thread):
                             print(f"Processor module '{processor}' not found.")
                             raise e
 
+                        # Can pass just the watch here?
                         update_handler = processor_module.perform_site_check(datastore=self.datastore,
                                                                              watch_uuid=uuid
                                                                              )
 
                         update_handler.call_browser()
 
-                        # In reality, the actual time of when the change was detected could be a few seconds after this
-                        # For example it should include when the page stopped rendering if using a playwright/chrome type fetch
-                        fetch_start_time = time.time()
-
                         changed_detected, update_obj, contents = update_handler.run_changedetection(watch=watch)
 
                         # Re #342
@@ -515,7 +513,7 @@ class update_worker(threading.Thread):
 
                     if not self.datastore.data['watching'].get(uuid):
                         continue
-
+                    #
                     # Different exceptions mean that we may or may not want to bump the snapshot, trigger notifications etc
                     if process_changedetection_results:
 
@@ -528,6 +526,8 @@ class update_worker(threading.Thread):
                                 except Exception as e:
                                     logger.warning(f"UUID: {uuid} Extract <title> as watch title was enabled, but couldn't find a <title>.")
 
+                        # Now update after running everything
+                        timestamp = round(time.time())
                         try:
                             self.datastore.update_watch(uuid=uuid, update_obj=update_obj)
 
@@ -543,28 +543,24 @@ class update_worker(threading.Thread):
 
                                 # Small hack so that we sleep just enough to allow 1 second  between history snapshots
                                 # this is because history.txt indexes/keys snapshots by epoch seconds and we dont want dupe keys
-                                # @also - the keys are one per second at the most (for now)
-                                if watch.newest_history_key and int(fetch_start_time) == int(watch.newest_history_key):
+
+                                if watch.newest_history_key and int(timestamp) == int(watch.newest_history_key):
                                     logger.warning(
-                                        f"Timestamp {fetch_start_time} already exists, waiting 1 seconds so we have a unique key in history.txt")
-                                    fetch_start_time += 1
+                                        f"Timestamp {timestamp} already exists, waiting 1 seconds so we have a unique key in history.txt")
+                                    timestamp = str(int(timestamp) + 1)
                                     time.sleep(1)
 
                                 watch.save_history_text(contents=contents,
-                                                        timestamp=int(fetch_start_time),
+                                                        timestamp=timestamp,
                                                         snapshot_id=update_obj.get('previous_md5', 'none'))
 
-
-                                empty_pages_are_a_change = self.datastore.data['settings']['application'].get('empty_pages_are_a_change', False)
-                                if update_handler.fetcher.content or (not update_handler.fetcher.content and empty_pages_are_a_change):
-                                    # attribute .last_changed is then based on this data
-                                    watch.save_last_fetched_html(contents=update_handler.fetcher.content, timestamp=int(fetch_start_time))
+                                if update_handler.fetcher.content:
+                                    watch.save_last_fetched_html(contents=update_handler.fetcher.content, timestamp=timestamp)
 
                                 # Notifications should only trigger on the second time (first time, we gather the initial snapshot)
                                 if watch.history_n >= 2:
                                     logger.info(f"Change detected in UUID {uuid} - {watch['url']}")
                                     if not watch.get('notification_muted'):
-                                        # @todo only run this if notifications exist
                                         self.send_content_changed_notification(watch_uuid=uuid)
 
                         except Exception as e:
@@ -586,15 +582,15 @@ class update_worker(threading.Thread):
                     except Exception as e:
                         pass
 
-                    self.datastore.update_watch(uuid=uuid, update_obj={'fetch_time': round(time.time() - fetch_start_time, 3),
-                                                                       'last_checked': int(fetch_start_time),
+                    self.datastore.update_watch(uuid=uuid, update_obj={'fetch_time': round(time.time() - now, 3),
+                                                                       'last_checked': round(time.time()),
                                                                        'check_count': count
                                                                        })
 
 
                 self.current_uuid = None  # Done
                 self.q.task_done()
-                logger.debug(f"Watch {uuid} done in {time.time()-fetch_start_time:.2f}s")
+                logger.debug(f"Watch {uuid} done in {time.time()-now:.2f}s")
 
                 # Give the CPU time to interrupt
                 time.sleep(0.1)

requirements.txt

@@ -35,7 +35,7 @@ dnspython==2.6.1 # related to eventlet fixes
 # jq not available on Windows so must be installed manually
 
 # Notification library
-apprise==1.9.2
+apprise==1.9.0
 
 # apprise mqtt https://github.com/dgtlmoon/changedetection.io/issues/315
 # use any version other than 2.0.x due to https://github.com/eclipse/paho.mqtt.python/issues/814
@@ -95,17 +95,5 @@ babel
 # Needed for > 3.10, https://github.com/microsoft/playwright-python/issues/2096
 greenlet >= 3.0.3
 
-# Pinned or it causes problems with flask_expects_json which seems unmaintained
-referencing==0.35.1
-
-panzi-json-logic
-
 # Scheduler - Windows seemed to miss a lot of default timezone info (even "UTC" !)
 tzdata
-
-#typing_extensions ==4.8.0
-
-pluggy ~= 1.5
-
-
-