Merge branch 'master' into conditions-then-plugins

.dockerignore

@@ -1,18 +1,31 @@
-.git
-.github
-changedetectionio/processors/__pycache__
-changedetectionio/api/__pycache__
-changedetectionio/model/__pycache__
-changedetectionio/blueprint/price_data_follower/__pycache__
-changedetectionio/blueprint/tags/__pycache__
-changedetectionio/blueprint/__pycache__
-changedetectionio/blueprint/browser_steps/__pycache__
-changedetectionio/fetchers/__pycache__
-changedetectionio/tests/visualselector/__pycache__
-changedetectionio/tests/restock/__pycache__
-changedetectionio/tests/__pycache__
-changedetectionio/tests/fetchers/__pycache__
-changedetectionio/tests/unit/__pycache__
-changedetectionio/tests/proxy_list/__pycache__
-changedetectionio/__pycache__
+# Git
+.git/
+.gitignore
 
+# GitHub
+.github/
+
+# Byte-compiled / optimized / DLL files
+**/__pycache__
+**/*.py[cod]
+
+# Caches
+.mypy_cache/
+.pytest_cache/
+.ruff_cache/
+
+# Distribution / packaging
+build/
+dist/
+*.egg-info*
+
+# Virtual environment
+.env
+.venv/
+venv/
+
+# IntelliJ IDEA
+.idea/
+
+# Visual Studio
+.vscode/

.github/test/Dockerfile-alpine

@@ -2,32 +2,33 @@
 # Test that we can still build on Alpine (musl modified libc https://musl.libc.org/)
 # Some packages wont install via pypi because they dont have a wheel available under this architecture.
 
-FROM ghcr.io/linuxserver/baseimage-alpine:3.18
+FROM ghcr.io/linuxserver/baseimage-alpine:3.21
 ENV PYTHONUNBUFFERED=1
 
 COPY requirements.txt /requirements.txt
 
 RUN \
-  apk add --update --no-cache --virtual=build-dependencies \
+ apk add --update --no-cache --virtual=build-dependencies \
+    build-base \
     cargo \
-    g++ \
-    gcc \
+    git \
     jpeg-dev \
     libc-dev \
     libffi-dev \
-    libjpeg \
     libxslt-dev \
-    make \
     openssl-dev \
-    py3-wheel \
     python3-dev \
+    zip \
     zlib-dev && \
   apk add --update --no-cache \
+    libjpeg \
     libxslt \
-    python3 \
-    py3-pip && \
+    nodejs \
+    poppler-utils \
+    python3 && \
   echo "**** pip3 install test of changedetection.io ****" && \
-  pip3 install -U pip wheel setuptools && \
-  pip3 install -U --no-cache-dir --find-links https://wheel-index.linuxserver.io/alpine-3.18/ -r /requirements.txt && \
+  python3 -m venv /lsiopy  && \
+  pip install -U pip wheel setuptools && \
+  pip install -U --no-cache-dir --find-links https://wheel-index.linuxserver.io/alpine-3.21/ -r /requirements.txt && \
   apk del --purge \
     build-dependencies

.github/workflows/containers.yml

@@ -103,6 +103,19 @@ jobs:
 #          provenance: false
 
       # A new tagged release is required, which builds :tag and :latest
+      - name: Docker meta :tag
+        if: github.event_name == 'release' && startsWith(github.event.release.tag_name, '0.')
+        uses: docker/metadata-action@v5
+        id: meta
+        with:
+            images: |
+                ${{ secrets.DOCKER_HUB_USERNAME }}/changedetection.io
+                ghcr.io/dgtlmoon/changedetection.io
+            tags: |
+                type=semver,pattern={{version}}
+                type=semver,pattern={{major}}.{{minor}}
+                type=semver,pattern={{major}}
+
       - name: Build and push :tag
         id: docker_build_tag_release
         if: github.event_name == 'release' && startsWith(github.event.release.tag_name, '0.')
@@ -111,11 +124,7 @@ jobs:
           context: ./
           file: ./Dockerfile
           push: true
-          tags: |
-            ${{ secrets.DOCKER_HUB_USERNAME }}/changedetection.io:${{ github.event.release.tag_name }}
-            ghcr.io/dgtlmoon/changedetection.io:${{ github.event.release.tag_name }}
-            ${{ secrets.DOCKER_HUB_USERNAME }}/changedetection.io:latest
-            ghcr.io/dgtlmoon/changedetection.io:latest
+          tags: ${{ steps.meta.outputs.tags }}
           platforms: linux/amd64,linux/arm64,linux/arm/v7,linux/arm/v8,linux/arm64/v8
           cache-from: type=gha
           cache-to: type=gha,mode=max

.github/workflows/pypi-release.yml

@@ -45,9 +45,12 @@ jobs:
     - name: Test that the basic pip built package runs without error
       run: |
         set -ex
-        sudo pip3 install --upgrade pip 
-        pip3 install dist/changedetection.io*.whl
+        ls -alR 
+        
+        # Find and install the first .whl file
+        find dist -type f -name "*.whl" -exec pip3 install {} \; -quit
         changedetection.io -d /tmp -p 10000 &
+        
         sleep 3
         curl --retry-connrefused --retry 6 http://127.0.0.1:10000/static/styles/pure-min.css >/dev/null
         curl --retry-connrefused --retry 6 http://127.0.0.1:10000/ >/dev/null

.github/workflows/test-only.yml

@@ -37,3 +37,10 @@ jobs:
       python-version: '3.12'
       skip-pypuppeteer: true
 
+  test-application-3-13:
+    needs: lint-code
+    uses: ./.github/workflows/test-stack-reusable-workflow.yml
+    with:
+      python-version: '3.13'
+      skip-pypuppeteer: true
+      

.github/workflows/test-stack-reusable-workflow.yml

@@ -64,14 +64,16 @@ jobs:
           echo "Running processes in docker..."
           docker ps
 
-      - name: Test built container with Pytest (generally as requests/plaintext fetching)
+      - name: Run Unit Tests
         run: |
           # Unit tests
-          echo "run test with unittest"
           docker run test-changedetectionio  bash -c 'python3 -m unittest changedetectionio.tests.unit.test_notification_diff'
           docker run test-changedetectionio  bash -c 'python3 -m unittest changedetectionio.tests.unit.test_watch_model'
           docker run test-changedetectionio  bash -c 'python3 -m unittest changedetectionio.tests.unit.test_jinja2_security'
-          
+          docker run test-changedetectionio  bash -c 'python3 -m unittest changedetectionio.tests.unit.test_semver'
+
+      - name: Test built container with Pytest (generally as requests/plaintext fetching)
+        run: |
           # All tests
           echo "run test with pytest"
           # The default pytest logger_level is TRACE

.gitignore

@@ -1,15 +1,29 @@
-__pycache__
-.idea
-*.pyc
-datastore/url-watches.json
-datastore/*
-__pycache__
-.pytest_cache
-build
-dist
-venv
-test-datastore/*
-test-datastore
-test-memory.log
+# Byte-compiled / optimized / DLL files
+**/__pycache__
+**/*.py[cod]
+
+# Caches
+.mypy_cache/
+.pytest_cache/
+.ruff_cache/
+
+# Distribution / packaging
+build/
+dist/
 *.egg-info*
+
+# Virtual environment
+.env
+.venv/
+venv/
+
+# IDEs
+.idea
 .vscode/settings.json
+
+# Datastore files
+datastore/
+test-datastore/
+
+# Memory consumption log
+test-memory.log

COMMERCIAL_LICENCE.md

@@ -4,7 +4,7 @@ In any commercial activity involving 'Hosting' (as defined herein), whether in p
 
 # Commercial License Agreement
 
-This Commercial License Agreement ("Agreement") is entered into by and between Mr Morresi (the original creator of this software) here-in ("Licensor") and (your company or personal name) _____________ ("Licensee"). This Agreement sets forth the terms and conditions under which Licensor provides its software ("Software") and services to Licensee for the purpose of reselling the software either in part or full, as part of any commercial activity where the activity involves a third party.
+This Commercial License Agreement ("Agreement") is entered into by and between Web Technologies s.r.o. here-in ("Licensor") and (your company or personal name) _____________ ("Licensee"). This Agreement sets forth the terms and conditions under which Licensor provides its software ("Software") and services to Licensee for the purpose of reselling the software either in part or full, as part of any commercial activity where the activity involves a third party.
 
 ### Definition of Hosting
 

Dockerfile

@@ -32,7 +32,7 @@ RUN pip install --extra-index-url https://www.piwheels.org/simple  --target=/dep
 # Playwright is an alternative to Selenium
 # Excluded this package from requirements.txt to prevent arm/v6 and arm/v7 builds from failing
 # https://github.com/dgtlmoon/changedetection.io/pull/1067 also musl/alpine (not supported)
-RUN pip install --target=/dependencies playwright~=1.41.2 \
+RUN pip install --target=/dependencies playwright~=1.48.0 \
     || echo "WARN: Failed to install Playwright. The application can still run, but the Playwright option will be disabled."
 
 # Final image stage

MANIFEST.in

@@ -2,6 +2,7 @@ recursive-include changedetectionio/api *
 recursive-include changedetectionio/apprise_plugin *
 recursive-include changedetectionio/blueprint *
 recursive-include changedetectionio/content_fetchers *
+recursive-include changedetectionio/conditions *
 recursive-include changedetectionio/model *
 recursive-include changedetectionio/processors *
 recursive-include changedetectionio/static *

README.md

@@ -105,13 +105,22 @@ We [recommend and use Bright Data](https://brightdata.grsm.io/n0r16zf7eivq) glob
 
 Please :star: star :star: this project and help it grow! https://github.com/dgtlmoon/changedetection.io/
 
+### Schedule web page watches in any timezone, limit by day of week and time.
+
+Easily set a re-check schedule, for example you could limit the web page change detection to only operate during business hours.
+Or perhaps based on a foreign timezone (for example, you want to check for the latest news-headlines in a foreign country at 0900 AM),
+
+<img src="./docs/scheduler.png" style="max-width:80%;" alt="How to monitor web page changes according to a schedule"  title="How to monitor web page changes according to a schedule"  />
+
+Includes quick short-cut buttons to setup a schedule for **business hours only**, or **weekends**.
+
 ### We have a Chrome extension!
 
 Easily add the current web page to your changedetection.io tool, simply install the extension and click "Sync" to connect it to your existing changedetection.io install.
 
 [<img src="./docs/chrome-extension-screenshot.png" style="max-width:80%;" alt="Chrome Extension to easily add the current web-page to detect a change."  title="Chrome Extension to easily add the current web-page to detect a change."  />](https://chromewebstore.google.com/detail/changedetectionio-website/kefcfmgmlhmankjmnbijimhofdjekbop)
 
-[Goto the Chrome Webstore to download the extension.](https://chromewebstore.google.com/detail/changedetectionio-website/kefcfmgmlhmankjmnbijimhofdjekbop)
+[Goto the Chrome Webstore to download the extension.](https://chromewebstore.google.com/detail/changedetectionio-website/kefcfmgmlhmankjmnbijimhofdjekbop) ( Or check out the [GitHub repo](https://github.com/dgtlmoon/changedetection.io-browser-extension) ) 
 
 ## Installation
 

changedetectionio/__init__.py

@@ -2,7 +2,7 @@
 
 # Read more https://github.com/dgtlmoon/changedetection.io/wiki
 
-__version__ = '0.47.06'
+__version__ = '0.49.4'
 
 from changedetectionio.strtobool import strtobool
 from json.decoder import JSONDecodeError
@@ -24,6 +24,9 @@ from loguru import logger
 app = None
 datastore = None
 
+def get_version():
+    return __version__
+
 # Parent wrapper or OS sends us a SIGTERM/SIGINT, do everything required for a clean shutdown
 def sigshutdown_handler(_signo, _stack_frame):
     global app
@@ -160,11 +163,10 @@ def main():
                     )
 
     # Monitored websites will not receive a Referer header when a user clicks on an outgoing link.
-    # @Note: Incompatible with password login (and maybe other features) for now, submit a PR!
     @app.after_request
     def hide_referrer(response):
         if strtobool(os.getenv("HIDE_REFERER", 'false')):
-            response.headers["Referrer-Policy"] = "no-referrer"
+            response.headers["Referrer-Policy"] = "same-origin"
 
         return response
 

changedetectionio/api/api_schema.py

@@ -112,6 +112,35 @@ def build_watch_json_schema(d):
 
     schema['properties']['time_between_check'] = build_time_between_check_json_schema()
 
+    schema['properties']['browser_steps'] = {
+        "anyOf": [
+            {
+                "type": "array",
+                "items": {
+                    "type": "object",
+                    "properties": {
+                        "operation": {
+                            "type": ["string", "null"],
+                            "maxLength": 5000  # Allows null and any string up to 5000 chars (including "")
+                        },
+                        "selector": {
+                            "type": ["string", "null"],
+                            "maxLength": 5000
+                        },
+                        "optional_value": {
+                            "type": ["string", "null"],
+                            "maxLength": 5000
+                        }
+                    },
+                    "required": ["operation", "selector", "optional_value"],
+                    "additionalProperties": False  # No extra keys allowed
+                }
+            },
+            {"type": "null"},  # Allows null for `browser_steps`
+            {"type": "array", "maxItems": 0}  # Allows empty array []
+        ]
+    }
+
     # headers ?
     return schema
 

changedetectionio/api/api_v1.py

@@ -76,6 +76,7 @@ class Watch(Resource):
         # Return without history, get that via another API call
         # Properties are not returned as a JSON, so add the required props manually
         watch['history_n'] = watch.history_n
+        # attr .last_changed will check for the last written text snapshot on change
         watch['last_changed'] = watch.last_changed
         watch['viewed'] = watch.viewed
         return watch

changedetectionio/apprise_asset.py

@@ -1,3 +1,4 @@
+from changedetectionio import apprise_plugin
 import apprise
 
 # Create our AppriseAsset and populate it with some of our new values:

changedetectionio/apprise_plugin/__init__.py

@@ -1,6 +1,8 @@
 # include the decorator
 from apprise.decorators import notify
 from loguru import logger
+from requests.structures import CaseInsensitiveDict
+
 
 @notify(on="delete")
 @notify(on="deletes")
@@ -13,70 +15,84 @@ from loguru import logger
 def apprise_custom_api_call_wrapper(body, title, notify_type, *args, **kwargs):
     import requests
     import json
+    import re
+
     from urllib.parse import unquote_plus
-    from apprise.utils import parse_url as apprise_parse_url
-    from apprise import URLBase
+    from apprise.utils.parse import parse_url as apprise_parse_url
 
     url = kwargs['meta'].get('url')
+    schema = kwargs['meta'].get('schema').lower().strip()
 
-    if url.startswith('post'):
-        r = requests.post
-    elif url.startswith('get'):
-        r = requests.get
-    elif url.startswith('put'):
-        r = requests.put
-    elif url.startswith('delete'):
-        r = requests.delete
+    # Choose POST, GET etc from requests
+    method =  re.sub(rf's$', '', schema)
+    requests_method = getattr(requests, method)
 
-    url = url.replace('post://', 'http://')
-    url = url.replace('posts://', 'https://')
-    url = url.replace('put://', 'http://')
-    url = url.replace('puts://', 'https://')
-    url = url.replace('get://', 'http://')
-    url = url.replace('gets://', 'https://')
-    url = url.replace('put://', 'http://')
-    url = url.replace('puts://', 'https://')
-    url = url.replace('delete://', 'http://')
-    url = url.replace('deletes://', 'https://')
-
-    headers = {}
-    params = {}
+    params = CaseInsensitiveDict({}) # Added to requests
     auth = None
+    has_error = False
 
     # Convert /foobar?+some-header=hello to proper header dictionary
     results = apprise_parse_url(url)
-    if results:
-        # Add our headers that the user can potentially over-ride if they wish
-        # to to our returned result set and tidy entries by unquoting them
-        headers = {unquote_plus(x): unquote_plus(y)
-                   for x, y in results['qsd+'].items()}
 
-        # https://github.com/caronc/apprise/wiki/Notify_Custom_JSON#get-parameter-manipulation
-        # In Apprise, it relies on prefixing each request arg with "-", because it uses say &method=update as a flag for apprise
-        # but here we are making straight requests, so we need todo convert this against apprise's logic
-        for k, v in results['qsd'].items():
-            if not k.strip('+-') in results['qsd+'].keys():
-                params[unquote_plus(k)] = unquote_plus(v)
+    # Add our headers that the user can potentially over-ride if they wish
+    # to to our returned result set and tidy entries by unquoting them
+    headers = CaseInsensitiveDict({unquote_plus(x): unquote_plus(y)
+               for x, y in results['qsd+'].items()})
 
-        # Determine Authentication
-        auth = ''
-        if results.get('user') and results.get('password'):
-            auth = (unquote_plus(results.get('user')), unquote_plus(results.get('user')))
-        elif results.get('user'):
-            auth = (unquote_plus(results.get('user')))
+    # https://github.com/caronc/apprise/wiki/Notify_Custom_JSON#get-parameter-manipulation
+    # In Apprise, it relies on prefixing each request arg with "-", because it uses say &method=update as a flag for apprise
+    # but here we are making straight requests, so we need todo convert this against apprise's logic
+    for k, v in results['qsd'].items():
+        if not k.strip('+-') in results['qsd+'].keys():
+            params[unquote_plus(k)] = unquote_plus(v)
 
-    # Try to auto-guess if it's JSON
-    h = 'application/json; charset=utf-8'
+    # Determine Authentication
+    auth = ''
+    if results.get('user') and results.get('password'):
+        auth = (unquote_plus(results.get('user')), unquote_plus(results.get('user')))
+    elif results.get('user'):
+        auth = (unquote_plus(results.get('user')))
+
+    # If it smells like it could be JSON and no content-type was already set, offer a default content type.
+    if body and '{' in body[:100] and not headers.get('Content-Type'):
+        json_header = 'application/json; charset=utf-8'
+        try:
+            # Try if it's JSON
+            json.loads(body)
+            headers['Content-Type'] = json_header
+        except ValueError as e:
+            logger.warning(f"Could not automatically add '{json_header}' header to the notification because the document failed to parse as JSON: {e}")
+            pass
+
+    # POSTS -> HTTPS etc
+    if schema.lower().endswith('s'):
+        url = re.sub(rf'^{schema}', 'https', results.get('url'))
+    else:
+        url = re.sub(rf'^{schema}', 'http', results.get('url'))
+
+    status_str = ''
     try:
-        json.loads(body)
-        headers['Content-Type'] = h
-    except ValueError as e:
-        logger.warning(f"Could not automatically add '{h}' header to the {kwargs['meta'].get('schema')}:// notification because the document failed to parse as JSON: {e}")
-        pass
+        r = requests_method(url,
+          auth=auth,
+          data=body.encode('utf-8') if type(body) is str else body,
+          headers=headers,
+          params=params
+        )
 
-    r(results.get('url'),
-      auth=auth,
-      data=body.encode('utf-8') if type(body) is str else body,
-      headers=headers,
-      params=params
-      )
+        if not (200 <= r.status_code < 300):
+            status_str = f"Error sending '{method.upper()}' request to {url} - Status: {r.status_code}: '{r.reason}'"
+            logger.error(status_str)
+            has_error = True
+        else:
+            logger.info(f"Sent '{method.upper()}' request to {url}")
+            has_error = False
+
+    except requests.RequestException as e:
+        status_str = f"Error sending '{method.upper()}' request to {url} - {str(e)}"
+        logger.error(status_str)
+        has_error = True
+
+    if has_error:
+        raise TypeError(status_str)
+
+    return True

changedetectionio/blueprint/browser_steps/__init__.py

@@ -22,7 +22,10 @@ from loguru import logger
 
 browsersteps_sessions = {}
 io_interface_context = None
-
+import json
+import base64
+import hashlib
+from flask import Response
 
 def construct_blueprint(datastore: ChangeDetectionStore):
     browser_steps_blueprint = Blueprint('browser_steps', __name__, template_folder="templates")
@@ -85,7 +88,7 @@ def construct_blueprint(datastore: ChangeDetectionStore):
         browsersteps_start_session['browserstepper'] = browser_steps.browsersteps_live_ui(
             playwright_browser=browsersteps_start_session['browser'],
             proxy=proxy,
-            start_url=datastore.data['watching'][watch_uuid].get('url'),
+            start_url=datastore.data['watching'][watch_uuid].link,
             headers=datastore.data['watching'][watch_uuid].get('headers')
         )
 
@@ -160,14 +163,13 @@ def construct_blueprint(datastore: ChangeDetectionStore):
         if not browsersteps_sessions.get(browsersteps_session_id):
             return make_response('No session exists under that ID', 500)
 
-
+        is_last_step = False
         # Actions - step/apply/etc, do the thing and return state
         if request.method == 'POST':
             # @todo - should always be an existing session
             step_operation = request.form.get('operation')
             step_selector = request.form.get('selector')
             step_optional_value = request.form.get('optional_value')
-            step_n = int(request.form.get('step_n'))
             is_last_step = strtobool(request.form.get('is_last_step'))
 
             # @todo try.. accept.. nice errors not popups..
@@ -182,16 +184,6 @@ def construct_blueprint(datastore: ChangeDetectionStore):
                 # Try to find something of value to give back to the user
                 return make_response(str(e).splitlines()[0], 401)
 
-            # Get visual selector ready/update its data (also use the current filter info from the page?)
-            # When the last 'apply' button was pressed
-            # @todo this adds overhead because the xpath selection is happening twice
-            u = browsersteps_sessions[browsersteps_session_id]['browserstepper'].page.url
-            if is_last_step and u:
-                (screenshot, xpath_data) = browsersteps_sessions[browsersteps_session_id]['browserstepper'].request_visualselector_data()
-                watch = datastore.data['watching'].get(uuid)
-                if watch:
-                    watch.save_screenshot(screenshot=screenshot)
-                    watch.save_xpath_data(data=xpath_data)
 
 #        if not this_session.page:
 #            cleanup_playwright_session()
@@ -199,31 +191,35 @@ def construct_blueprint(datastore: ChangeDetectionStore):
 
         # Screenshots and other info only needed on requesting a step (POST)
         try:
-            state = browsersteps_sessions[browsersteps_session_id]['browserstepper'].get_current_state()
+            (screenshot, xpath_data) = browsersteps_sessions[browsersteps_session_id]['browserstepper'].get_current_state()
+            if is_last_step:
+                watch = datastore.data['watching'].get(uuid)
+                u = browsersteps_sessions[browsersteps_session_id]['browserstepper'].page.url
+                if watch and u:
+                    watch.save_screenshot(screenshot=screenshot)
+                    watch.save_xpath_data(data=xpath_data)
+
         except playwright._impl._api_types.Error as e:
             return make_response("Browser session ran out of time :( Please reload this page."+str(e), 401)
+        except Exception as e:
+            return make_response("Error fetching screenshot and element data - " + str(e), 401)
 
-        # Use send_file() which is way faster than read/write loop on bytes
-        import json
-        from tempfile import mkstemp
-        from flask import send_file
-        tmp_fd, tmp_file = mkstemp(text=True, suffix=".json", prefix="changedetectionio-")
+        # SEND THIS BACK TO THE BROWSER
 
-        output = json.dumps({'screenshot': "data:image/jpeg;base64,{}".format(
-            base64.b64encode(state[0]).decode('ascii')),
-            'xpath_data': state[1],
-            'session_age_start': browsersteps_sessions[browsersteps_session_id]['browserstepper'].age_start,
-            'browser_time_remaining': round(remaining)
-        })
+        output = {
+            "screenshot": f"data:image/jpeg;base64,{base64.b64encode(screenshot).decode('ascii')}",
+            "xpath_data": xpath_data,
+            "session_age_start": browsersteps_sessions[browsersteps_session_id]['browserstepper'].age_start,
+            "browser_time_remaining": round(remaining)
+        }
+        json_data = json.dumps(output)
 
-        with os.fdopen(tmp_fd, 'w') as f:
-            f.write(output)
+        # Generate an ETag (hash of the response body)
+        etag_hash = hashlib.md5(json_data.encode('utf-8')).hexdigest()
 
-        response = make_response(send_file(path_or_file=tmp_file,
-                                           mimetype='application/json; charset=UTF-8',
-                                           etag=True))
-        # No longer needed
-        os.unlink(tmp_file)
+        # Create the response with ETag
+        response = Response(json_data, mimetype="application/json; charset=UTF-8")
+        response.set_etag(etag_hash)
 
         return response
 

changedetectionio/blueprint/browser_steps/browser_steps.py

@@ -1,14 +1,15 @@
-#!/usr/bin/env python3
-
 import os
 import time
 import re
 from random import randint
 from loguru import logger
 
+from changedetectionio.content_fetchers.helpers import capture_stitched_together_full_page, SCREENSHOT_SIZE_STITCH_THRESHOLD
 from changedetectionio.content_fetchers.base import manage_user_agent
 from changedetectionio.safe_jinja import render as jinja_render
 
+
+
 # Two flags, tell the JS which of the "Selector" or "Value" field should be enabled in the front end
 # 0- off, 1- on
 browser_step_ui_config = {'Choose one': '0 0',
@@ -31,6 +32,7 @@ browser_step_ui_config = {'Choose one': '0 0',
 #                          'Extract text and use as filter': '1 0',
                           'Goto site': '0 0',
                           'Goto URL': '0 1',
+                          'Make all child elements visible': '1 0',
                           'Press Enter': '0 0',
                           'Select by label': '1 1',
                           'Scroll down': '0 0',
@@ -38,6 +40,7 @@ browser_step_ui_config = {'Choose one': '0 0',
                           'Wait for seconds': '0 1',
                           'Wait for text': '0 1',
                           'Wait for text in element': '1 1',
+                          'Remove elements': '1 0',
                           #                          'Press Page Down': '0 0',
                           #                          'Press Page Up': '0 0',
                           # weird bug, come back to it later
@@ -52,6 +55,8 @@ class steppable_browser_interface():
     page = None
     start_url = None
 
+    action_timeout = 10 * 1000
+
     def __init__(self, start_url):
         self.start_url = start_url
 
@@ -102,7 +107,7 @@ class steppable_browser_interface():
             return
         elem = self.page.get_by_text(value)
         if elem.count():
-            elem.first.click(delay=randint(200, 500), timeout=3000)
+            elem.first.click(delay=randint(200, 500), timeout=self.action_timeout)
 
     def action_click_element_containing_text_if_exists(self, selector=None, value=''):
         logger.debug("Clicking element containing text if exists")
@@ -111,7 +116,7 @@ class steppable_browser_interface():
         elem = self.page.get_by_text(value)
         logger.debug(f"Clicking element containing text - {elem.count()} elements found")
         if elem.count():
-            elem.first.click(delay=randint(200, 500), timeout=3000)
+            elem.first.click(delay=randint(200, 500), timeout=self.action_timeout)
         else:
             return
 
@@ -119,7 +124,7 @@ class steppable_browser_interface():
         if not len(selector.strip()):
             return
 
-        self.page.fill(selector, value, timeout=10 * 1000)
+        self.page.fill(selector, value, timeout=self.action_timeout)
 
     def action_execute_js(self, selector, value):
         response = self.page.evaluate(value)
@@ -130,7 +135,7 @@ class steppable_browser_interface():
         if not len(selector.strip()):
             return
 
-        self.page.click(selector=selector, timeout=30 * 1000, delay=randint(200, 500))
+        self.page.click(selector=selector, timeout=self.action_timeout + 20 * 1000, delay=randint(200, 500))
 
     def action_click_element_if_exists(self, selector, value):
         import playwright._impl._errors as _api_types
@@ -138,7 +143,7 @@ class steppable_browser_interface():
         if not len(selector.strip()):
             return
         try:
-            self.page.click(selector, timeout=10 * 1000, delay=randint(200, 500))
+            self.page.click(selector, timeout=self.action_timeout, delay=randint(200, 500))
         except _api_types.TimeoutError as e:
             return
         except _api_types.Error as e:
@@ -185,11 +190,29 @@ class steppable_browser_interface():
         self.page.keyboard.press("PageDown", delay=randint(200, 500))
 
     def action_check_checkbox(self, selector, value):
-        self.page.locator(selector).check(timeout=1000)
+        self.page.locator(selector).check(timeout=self.action_timeout)
 
     def action_uncheck_checkbox(self, selector, value):
-        self.page.locator(selector, timeout=1000).uncheck(timeout=1000)
+        self.page.locator(selector).uncheck(timeout=self.action_timeout)
 
+    def action_remove_elements(self, selector, value):
+        """Removes all elements matching the given selector from the DOM."""
+        self.page.locator(selector).evaluate_all("els => els.forEach(el => el.remove())")
+
+    def action_make_all_child_elements_visible(self, selector, value):
+        """Recursively makes all child elements inside the given selector fully visible."""
+        self.page.locator(selector).locator("*").evaluate_all("""
+            els => els.forEach(el => {
+                el.style.display = 'block';   // Forces it to be displayed
+                el.style.visibility = 'visible';   // Ensures it's not hidden
+                el.style.opacity = '1';   // Fully opaque
+                el.style.position = 'relative';   // Avoids 'absolute' hiding
+                el.style.height = 'auto';   // Expands collapsed elements
+                el.style.width = 'auto';   // Ensures full visibility
+                el.removeAttribute('hidden');   // Removes hidden attribute
+                el.classList.remove('hidden', 'd-none');  // Removes common CSS hidden classes
+            })
+        """)
 
 # Responsible for maintaining a live 'context' with the chrome CDP
 # @todo - how long do contexts live for anyway?
@@ -257,6 +280,7 @@ class browsersteps_live_ui(steppable_browser_interface):
         logger.debug(f"Time to browser setup {time.time()-now:.2f}s")
         self.page.wait_for_timeout(1 * 1000)
 
+
     def mark_as_closed(self):
         logger.debug("Page closed, cleaning up..")
 
@@ -274,39 +298,30 @@ class browsersteps_live_ui(steppable_browser_interface):
         now = time.time()
         self.page.wait_for_timeout(1 * 1000)
 
-        # The actual screenshot
-        screenshot = self.page.screenshot(type='jpeg', full_page=True, quality=40)
 
+        full_height = self.page.evaluate("document.documentElement.scrollHeight")
+
+        if full_height >= SCREENSHOT_SIZE_STITCH_THRESHOLD:
+            logger.warning(f"Page full Height: {full_height}px longer than {SCREENSHOT_SIZE_STITCH_THRESHOLD}px, using 'stitched screenshot method'.")
+            screenshot = capture_stitched_together_full_page(self.page)
+        else:
+            screenshot = self.page.screenshot(type='jpeg', full_page=True, quality=40)
+
+        logger.debug(f"Time to get screenshot from browser {time.time() - now:.2f}s")
+
+        now = time.time()
         self.page.evaluate("var include_filters=''")
         # Go find the interactive elements
         # @todo in the future, something smarter that can scan for elements with .click/focus etc event handlers?
         elements = 'a,button,input,select,textarea,i,th,td,p,li,h1,h2,h3,h4,div,span'
         xpath_element_js = xpath_element_js.replace('%ELEMENTS%', elements)
+
         xpath_data = self.page.evaluate("async () => {" + xpath_element_js + "}")
         # So the JS will find the smallest one first
         xpath_data['size_pos'] = sorted(xpath_data['size_pos'], key=lambda k: k['width'] * k['height'], reverse=True)
-        logger.debug(f"Time to complete get_current_state of browser {time.time()-now:.2f}s")
-        # except
+        logger.debug(f"Time to scrape xpath element data in browser {time.time()-now:.2f}s")
+
         # playwright._impl._api_types.Error: Browser closed.
         # @todo show some countdown timer?
         return (screenshot, xpath_data)
 
-    def request_visualselector_data(self):
-        """
-        Does the same that the playwright operation in content_fetcher does
-        This is used to just bump the VisualSelector data so it' ready to go if they click on the tab
-        @todo refactor and remove duplicate code, add include_filters
-        :param xpath_data:
-        :param screenshot:
-        :param current_include_filters:
-        :return:
-        """
-        import importlib.resources
-        self.page.evaluate("var include_filters=''")
-        xpath_element_js = importlib.resources.files("changedetectionio.content_fetchers.res").joinpath('xpath_element_scraper.js').read_text()
-        from changedetectionio.content_fetchers import visualselector_xpath_selectors
-        xpath_element_js = xpath_element_js.replace('%ELEMENTS%', visualselector_xpath_selectors)
-        xpath_data = self.page.evaluate("async () => {" + xpath_element_js + "}")
-        screenshot = self.page.screenshot(type='jpeg', full_page=True, quality=int(os.getenv("SCREENSHOT_QUALITY", 72)))
-
-        return (screenshot, xpath_data)

changedetectionio/blueprint/tags/__init__.py

@@ -13,6 +13,7 @@ def construct_blueprint(datastore: ChangeDetectionStore):
     def tags_overview_page():
         from .form import SingleTag
         add_form = SingleTag(request.form)
+
         sorted_tags = sorted(datastore.data['settings']['application'].get('tags').items(), key=lambda x: x[1]['title'])
 
         from collections import Counter
@@ -104,9 +105,11 @@ def construct_blueprint(datastore: ChangeDetectionStore):
 
         default = datastore.data['settings']['application']['tags'].get(uuid)
 
-        form = group_restock_settings_form(formdata=request.form if request.method == 'POST' else None,
+        form = group_restock_settings_form(
+                                       formdata=request.form if request.method == 'POST' else None,
                                        data=default,
-                                       extra_notification_tokens=datastore.get_unique_notification_tokens_available()
+                                       extra_notification_tokens=datastore.get_unique_notification_tokens_available(),
+                                       default_system_settings = datastore.data['settings'],
                                        )
 
         template_args = {

changedetectionio/conditions/__init__.py

@@ -0,0 +1,135 @@
+from flask import Blueprint
+
+from json_logic.builtins import BUILTINS
+
+from .exceptions import EmptyConditionRuleRowNotUsable
+from .pluggy_interface import plugin_manager  # Import the pluggy plugin manager
+from . import default_plugin
+
+# List of all supported JSON Logic operators
+operator_choices = [
+    (None, "Choose one"),
+    (">", "Greater Than"),
+    ("<", "Less Than"),
+    (">=", "Greater Than or Equal To"),
+    ("<=", "Less Than or Equal To"),
+    ("==", "Equals"),
+    ("!=", "Not Equals"),
+    ("in", "Contains"),
+    ("!in", "Does Not Contain"),
+]
+
+# Fields available in the rules
+field_choices = [
+    (None, "Choose one"),
+]
+
+# The data we will feed the JSON Rules to see if it passes the test/conditions or not
+EXECUTE_DATA = {}
+
+
+# Define the extended operations dictionary
+CUSTOM_OPERATIONS = {
+    **BUILTINS,  # Include all standard operators
+}
+
+def filter_complete_rules(ruleset):
+    rules = [
+        rule for rule in ruleset
+        if all(value not in ("", False, "None", None) for value in [rule["operator"], rule["field"], rule["value"]])
+    ]
+    return rules
+
+def convert_to_jsonlogic(logic_operator: str, rule_dict: list):
+    """
+    Convert a structured rule dict into a JSON Logic rule.
+
+    :param rule_dict: Dictionary containing conditions.
+    :return: JSON Logic rule as a dictionary.
+    """
+
+
+    json_logic_conditions = []
+
+    for condition in rule_dict:
+        operator = condition["operator"]
+        field = condition["field"]
+        value = condition["value"]
+
+        if not operator or operator == 'None' or not value or not field:
+            raise EmptyConditionRuleRowNotUsable()
+
+        # Convert value to int/float if possible
+        try:
+            if isinstance(value, str) and "." in value and str != "None":
+                value = float(value)
+            else:
+                value = int(value)
+        except (ValueError, TypeError):
+            pass  # Keep as a string if conversion fails
+
+        # Handle different JSON Logic operators properly
+        if operator == "in":
+            json_logic_conditions.append({"in": [value, {"var": field}]})  # value first
+        elif operator in ("!", "!!", "-"):
+            json_logic_conditions.append({operator: [{"var": field}]})  # Unary operators
+        elif operator in ("min", "max", "cat"):
+            json_logic_conditions.append({operator: value})  # Multi-argument operators
+        else:
+            json_logic_conditions.append({operator: [{"var": field}, value]})  # Standard binary operators
+
+    return {logic_operator: json_logic_conditions} if len(json_logic_conditions) > 1 else json_logic_conditions[0]
+
+
+def execute_ruleset_against_all_plugins(current_watch_uuid: str, application_datastruct, ephemeral_data={} ):
+    """
+    Build our data and options by calling our plugins then pass it to jsonlogic and see if the conditions pass
+
+    :param ruleset: JSON Logic rule dictionary.
+    :param extracted_data: Dictionary containing the facts.   <-- maybe the app struct+uuid
+    :return: Dictionary of plugin results.
+    """
+    from json_logic import jsonLogic
+
+    EXECUTE_DATA = {}
+    result = True
+    
+    ruleset_settings = application_datastruct['watching'].get(current_watch_uuid)
+
+    if ruleset_settings.get("conditions"):
+        logic_operator = "and" if ruleset_settings.get("conditions_match_logic", "ALL") == "ALL" else "or"
+        complete_rules = filter_complete_rules(ruleset_settings['conditions'])
+        if complete_rules:
+            # Give all plugins a chance to update the data dict again (that we will test the conditions against)
+            for plugin in plugin_manager.get_plugins():
+                new_execute_data = plugin.add_data(current_watch_uuid=current_watch_uuid,
+                                                   application_datastruct=application_datastruct,
+                                                   ephemeral_data=ephemeral_data)
+
+                if new_execute_data and isinstance(new_execute_data, dict):
+                    EXECUTE_DATA.update(new_execute_data)
+
+            # Create the ruleset
+            ruleset = convert_to_jsonlogic(logic_operator=logic_operator, rule_dict=complete_rules)
+            
+            # Pass the custom operations dictionary to jsonLogic
+            if not jsonLogic(logic=ruleset, data=EXECUTE_DATA, operations=CUSTOM_OPERATIONS):
+                result = False
+
+    return result
+
+
+# Load plugins dynamically
+for plugin in plugin_manager.get_plugins():
+    new_ops = plugin.register_operators()
+    if isinstance(new_ops, dict):
+        CUSTOM_OPERATIONS.update(new_ops)
+
+    new_operator_choices = plugin.register_operator_choices()
+    if isinstance(new_operator_choices, list):
+        operator_choices.extend(new_operator_choices)
+
+    new_field_choices = plugin.register_field_choices()
+    if isinstance(new_field_choices, list):
+        field_choices.extend(new_field_choices)
+

changedetectionio/conditions/blueprint.py

@@ -0,0 +1,78 @@
+# Flask Blueprint Definition
+import json
+
+from flask import Blueprint
+
+from changedetectionio.conditions import execute_ruleset_against_all_plugins
+
+
+def construct_blueprint(datastore):
+    from changedetectionio.flask_app import login_optionally_required
+
+    conditions_blueprint = Blueprint('conditions', __name__, template_folder="templates")
+
+    @conditions_blueprint.route("/<string:watch_uuid>/verify-condition-single-rule", methods=['POST'])
+    @login_optionally_required
+    def verify_condition_single_rule(watch_uuid):
+        """Verify a single condition rule against the current snapshot"""
+        from changedetectionio.processors.text_json_diff import prepare_filter_prevew
+        from flask import request, jsonify
+        from copy import deepcopy
+
+        ephemeral_data = {}
+
+        # Get the watch data
+        watch = datastore.data['watching'].get(watch_uuid)
+        if not watch:
+            return jsonify({'status': 'error', 'message': 'Watch not found'}), 404
+
+        # First use prepare_filter_prevew to process the form data
+        # This will return text_after_filter which is after all current form settings are applied
+        # Create ephemeral data with the text from the current snapshot
+
+        try:
+            # Call prepare_filter_prevew to get a processed version of the content with current form settings
+            # We'll ignore the returned response and just use the datastore which is modified by the function
+
+            # this should apply all filters etc so then we can run the CONDITIONS against the final output text
+            result = prepare_filter_prevew(datastore=datastore,
+                                           form_data=request.form,
+                                           watch_uuid=watch_uuid)
+
+            ephemeral_data['text'] = result.get('after_filter', '')
+            # Create a temporary watch data structure with this single rule
+            tmp_watch_data = deepcopy(datastore.data['watching'].get(watch_uuid))
+
+            # Override the conditions in the temporary watch
+            rule_json = request.args.get("rule")
+            rule = json.loads(rule_json) if rule_json else None
+            tmp_watch_data['conditions'] = [rule]
+            tmp_watch_data['conditions_match_logic'] = "ALL"  # Single rule, so use ALL
+
+            # Create a temporary application data structure for the rule check
+            temp_app_data = {
+                'watching': {
+                    watch_uuid: tmp_watch_data
+                }
+            }
+
+            # Execute the rule against the current snapshot with form data
+            result = execute_ruleset_against_all_plugins(
+                current_watch_uuid=watch_uuid,
+                application_datastruct=temp_app_data,
+                ephemeral_data=ephemeral_data
+            )
+
+            return jsonify({
+                'status': 'success',
+                'result': result,
+                'message': 'Condition passes' if result else 'Condition does not pass'
+            })
+
+        except Exception as e:
+            return jsonify({
+                'status': 'error',
+                'message': f'Error verifying condition: {str(e)}'
+            }), 500
+
+    return conditions_blueprint

changedetectionio/conditions/default_plugin.py

@@ -0,0 +1,78 @@
+import re
+
+import pluggy
+from price_parser import Price
+from loguru import logger
+
+hookimpl = pluggy.HookimplMarker("changedetectionio_conditions")
+
+
+@hookimpl
+def register_operators():
+    def starts_with(_, text, prefix):
+        return text.lower().strip().startswith(str(prefix).strip().lower())
+
+    def ends_with(_, text, suffix):
+        return text.lower().strip().endswith(str(suffix).strip().lower())
+
+    def length_min(_, text, strlen):
+        return len(text) >= int(strlen)
+
+    def length_max(_, text, strlen):
+        return len(text) <= int(strlen)
+
+    # ✅ Custom function for case-insensitive regex matching
+    def contains_regex(_, text, pattern):
+        """Returns True if `text` contains `pattern` (case-insensitive regex match)."""
+        return bool(re.search(pattern, str(text), re.IGNORECASE))
+
+    # ✅ Custom function for NOT matching case-insensitive regex
+    def not_contains_regex(_, text, pattern):
+        """Returns True if `text` does NOT contain `pattern` (case-insensitive regex match)."""
+        return not bool(re.search(pattern, str(text), re.IGNORECASE))
+
+    return {
+        "!contains_regex": not_contains_regex,
+        "contains_regex": contains_regex,
+        "ends_with": ends_with,
+        "length_max": length_max,
+        "length_min": length_min,
+        "starts_with": starts_with,
+    }
+
+@hookimpl
+def register_operator_choices():
+    return [
+        ("starts_with", "Text Starts With"),
+        ("ends_with", "Text Ends With"),
+        ("length_min", "Length minimum"),
+        ("length_max", "Length maximum"),
+        ("contains_regex", "Text Matches Regex"),
+        ("!contains_regex", "Text Does NOT Match Regex"),
+    ]
+
+@hookimpl
+def register_field_choices():
+    return [
+        ("extracted_number", "Extracted number after 'Filters & Triggers'"),
+#        ("meta_description", "Meta Description"),
+#        ("meta_keywords", "Meta Keywords"),
+        ("page_filtered_text", "Page text after 'Filters & Triggers'"),
+        #("page_title", "Page <title>"), # actual page title <title>
+    ]
+
+@hookimpl
+def add_data(current_watch_uuid, application_datastruct, ephemeral_data):
+
+    res = {}
+    if 'text' in ephemeral_data:
+        res['page_filtered_text'] = ephemeral_data['text']
+
+        # Better to not wrap this in try/except so that the UI can see any errors
+        price = Price.fromstring(ephemeral_data.get('text'))
+        if price and price.amount != None:
+            # This is slightly misleading, it's extracting a PRICE not a Number..
+            res['extracted_number'] = float(price.amount)
+            logger.debug(f"Extracted number result: '{price}' - returning float({res['extracted_number']})")
+
+    return res

changedetectionio/conditions/exceptions.py

@@ -0,0 +1,6 @@
+class EmptyConditionRuleRowNotUsable(Exception):
+    def __init__(self):
+        super().__init__("One of the 'conditions' rulesets is incomplete, cannot run.")
+
+    def __str__(self):
+        return self.args[0]

changedetectionio/conditions/form.py

@@ -0,0 +1,44 @@
+# Condition Rule Form (for each rule row)
+from wtforms import Form, SelectField, StringField, validators
+from wtforms import validators
+
+class ConditionFormRow(Form):
+
+    # ✅ Ensure Plugins Are Loaded BEFORE Importing Choices
+    from changedetectionio.conditions import plugin_manager
+    from changedetectionio.conditions import operator_choices, field_choices
+    field = SelectField(
+        "Field",
+        choices=field_choices,
+        validators=[validators.Optional()]
+    )
+
+    operator = SelectField(
+        "Operator",
+        choices=operator_choices,
+        validators=[validators.Optional()]
+    )
+
+    value = StringField("Value", validators=[validators.Optional()])
+
+    def validate(self, extra_validators=None):
+        # First, run the default validators
+        if not super().validate(extra_validators):
+            return False
+
+        # Custom validation logic
+        # If any of the operator/field/value is set, then they must be all set
+        if any(value not in ("", False, "None", None) for value in [self.operator.data, self.field.data, self.value.data]):
+            if not self.operator.data or self.operator.data == 'None':
+                self.operator.errors.append("Operator is required.")
+                return False
+
+            if not self.field.data or self.field.data == 'None':
+                self.field.errors.append("Field is required.")
+                return False
+
+            if not self.value.data:
+                self.value.errors.append("Value is required.")
+                return False
+
+        return True  # Only return True if all conditions pass

changedetectionio/conditions/pluggy_interface.py

@@ -0,0 +1,44 @@
+import pluggy
+from . import default_plugin  # Import the default plugin
+
+# ✅ Ensure that the namespace in HookspecMarker matches PluginManager
+PLUGIN_NAMESPACE = "changedetectionio_conditions"
+
+hookspec = pluggy.HookspecMarker(PLUGIN_NAMESPACE)
+hookimpl = pluggy.HookimplMarker(PLUGIN_NAMESPACE)
+
+
+class ConditionsSpec:
+    """Hook specifications for extending JSON Logic conditions."""
+
+    @hookspec
+    def register_operators():
+        """Return a dictionary of new JSON Logic operators."""
+        pass
+
+    @hookspec
+    def register_operator_choices():
+        """Return a list of new operator choices."""
+        pass
+
+    @hookspec
+    def register_field_choices():
+        """Return a list of new field choices."""
+        pass
+
+    @hookspec
+    def add_data(current_watch_uuid, application_datastruct, ephemeral_data):
+        """Add to the datadict"""
+        pass
+
+# ✅ Set up Pluggy Plugin Manager
+plugin_manager = pluggy.PluginManager(PLUGIN_NAMESPACE)
+
+# ✅ Register hookspecs (Ensures they are detected)
+plugin_manager.add_hookspecs(ConditionsSpec)
+
+# ✅ Register built-in plugins manually
+plugin_manager.register(default_plugin, "default_plugin")
+
+# ✅ Discover installed plugins from external packages (if any)
+plugin_manager.load_setuptools_entrypoints(PLUGIN_NAMESPACE)

changedetectionio/content_fetchers/helpers.py

@@ -0,0 +1,104 @@
+
+# Pages with a vertical height longer than this will use the 'stitch together' method.
+
+# - Many GPUs have a max texture size of 16384x16384px (or lower on older devices).
+# - If a page is taller than ~8000–10000px, it risks exceeding GPU memory limits.
+# - This is especially important on headless Chromium, where Playwright may fail to allocate a massive full-page buffer.
+
+
+# The size at which we will switch to stitching method
+SCREENSHOT_SIZE_STITCH_THRESHOLD=8000
+
+from loguru import logger
+
+def capture_stitched_together_full_page(page):
+    import io
+    import os
+    import time
+    from PIL import Image, ImageDraw, ImageFont
+
+    MAX_TOTAL_HEIGHT = SCREENSHOT_SIZE_STITCH_THRESHOLD*4  # Maximum total height for the final image (When in stitch mode)
+    MAX_CHUNK_HEIGHT = 4000  # Height per screenshot chunk
+    WARNING_TEXT_HEIGHT = 20  # Height of the warning text overlay
+
+    # Save the original viewport size
+    original_viewport = page.viewport_size
+    now = time.time()
+
+    try:
+        viewport = page.viewport_size
+        page_height = page.evaluate("document.documentElement.scrollHeight")
+
+        # Limit the total capture height
+        capture_height = min(page_height, MAX_TOTAL_HEIGHT)
+
+        images = []
+        total_captured_height = 0
+
+        for offset in range(0, capture_height, MAX_CHUNK_HEIGHT):
+            # Ensure we do not exceed the total height limit
+            chunk_height = min(MAX_CHUNK_HEIGHT, MAX_TOTAL_HEIGHT - total_captured_height)
+
+            # Adjust viewport size for this chunk
+            page.set_viewport_size({"width": viewport["width"], "height": chunk_height})
+
+            # Scroll to the correct position
+            page.evaluate(f"window.scrollTo(0, {offset})")
+
+            # Capture screenshot chunk
+            screenshot_bytes = page.screenshot(type='jpeg', quality=int(os.getenv("SCREENSHOT_QUALITY", 30)))
+            images.append(Image.open(io.BytesIO(screenshot_bytes)))
+
+            total_captured_height += chunk_height
+
+            # Stop if we reached the maximum total height
+            if total_captured_height >= MAX_TOTAL_HEIGHT:
+                break
+
+        # Create the final stitched image
+        stitched_image = Image.new('RGB', (viewport["width"], total_captured_height))
+        y_offset = 0
+
+        # Stitch the screenshot chunks together
+        for img in images:
+            stitched_image.paste(img, (0, y_offset))
+            y_offset += img.height
+
+        logger.debug(f"Screenshot stitched together in {time.time()-now:.2f}s")
+
+        # Overlay warning text if the screenshot was trimmed
+        if page_height > MAX_TOTAL_HEIGHT:
+            draw = ImageDraw.Draw(stitched_image)
+            warning_text = f"WARNING: Screenshot was {page_height}px but trimmed to {MAX_TOTAL_HEIGHT}px because it was too long"
+
+            # Load font (default system font if Arial is unavailable)
+            try:
+                font = ImageFont.truetype("arial.ttf", WARNING_TEXT_HEIGHT)  # Arial (Windows/Mac)
+            except IOError:
+                font = ImageFont.load_default()  # Default font if Arial not found
+
+            # Get text bounding box (correct method for newer Pillow versions)
+            text_bbox = draw.textbbox((0, 0), warning_text, font=font)
+            text_width = text_bbox[2] - text_bbox[0]  # Calculate text width
+            text_height = text_bbox[3] - text_bbox[1]  # Calculate text height
+
+            # Define background rectangle (top of the image)
+            draw.rectangle([(0, 0), (viewport["width"], WARNING_TEXT_HEIGHT)], fill="white")
+
+            # Center text horizontally within the warning area
+            text_x = (viewport["width"] - text_width) // 2
+            text_y = (WARNING_TEXT_HEIGHT - text_height) // 2
+
+            # Draw the warning text in red
+            draw.text((text_x, text_y), warning_text, fill="red", font=font)
+
+        # Save or return the final image
+        output = io.BytesIO()
+        stitched_image.save(output, format="JPEG", quality=int(os.getenv("SCREENSHOT_QUALITY", 30)))
+        screenshot = output.getvalue()
+
+    finally:
+        # Restore the original viewport size
+        page.set_viewport_size(original_viewport)
+
+    return screenshot

changedetectionio/content_fetchers/playwright.py

@@ -4,6 +4,7 @@ from urllib.parse import urlparse
 
 from loguru import logger
 
+from changedetectionio.content_fetchers.helpers import capture_stitched_together_full_page, SCREENSHOT_SIZE_STITCH_THRESHOLD
 from changedetectionio.content_fetchers.base import Fetcher, manage_user_agent
 from changedetectionio.content_fetchers.exceptions import PageUnloadable, Non200ErrorCodeReceived, EmptyReply, ScreenshotUnavailable
 
@@ -89,6 +90,7 @@ class fetcher(Fetcher):
         from playwright.sync_api import sync_playwright
         import playwright._impl._errors
         from changedetectionio.content_fetchers import visualselector_xpath_selectors
+        import time
         self.delete_browser_steps_screenshots()
         response = None
 
@@ -179,6 +181,7 @@ class fetcher(Fetcher):
 
             self.page.wait_for_timeout(extra_wait * 1000)
 
+            now = time.time()
             # So we can find an element on the page where its selector was entered manually (maybe not xPath etc)
             if current_include_filters is not None:
                 self.page.evaluate("var include_filters={}".format(json.dumps(current_include_filters)))
@@ -190,6 +193,8 @@ class fetcher(Fetcher):
             self.instock_data = self.page.evaluate("async () => {" + self.instock_data_js + "}")
 
             self.content = self.page.content()
+            logger.debug(f"Time to scrape xpath element data in browser {time.time() - now:.2f}s")
+
             # Bug 3 in Playwright screenshot handling
             # Some bug where it gives the wrong screenshot size, but making a request with the clip set first seems to solve it
             # JPEG is better here because the screenshots can be very very large
@@ -199,10 +204,15 @@ class fetcher(Fetcher):
             # acceptable screenshot quality here
             try:
                 # The actual screenshot - this always base64 and needs decoding! horrible! huge CPU usage
-                self.screenshot = self.page.screenshot(type='jpeg',
-                                                       full_page=True,
-                                                       quality=int(os.getenv("SCREENSHOT_QUALITY", 72)),
-                                                       )
+                full_height = self.page.evaluate("document.documentElement.scrollHeight")
+
+                if full_height >= SCREENSHOT_SIZE_STITCH_THRESHOLD:
+                    logger.warning(
+                        f"Page full Height: {full_height}px longer than {SCREENSHOT_SIZE_STITCH_THRESHOLD}px, using 'stitched screenshot method'.")
+                    self.screenshot = capture_stitched_together_full_page(self.page)
+                else:
+                    self.screenshot = self.page.screenshot(type='jpeg', full_page=True, quality=int(os.getenv("SCREENSHOT_QUALITY", 30)))
+
             except Exception as e:
                 # It's likely the screenshot was too long/big and something crashed
                 raise ScreenshotUnavailable(url=url, status_code=self.status_code)

changedetectionio/content_fetchers/res/stock-not-in-stock.js

@@ -29,8 +29,11 @@ function isItemInStock() {
         'currently unavailable',
         'dieser artikel ist bald wieder verfügbar',
         'dostępne wkrótce',
+        'en rupture',
         'en rupture de stock',
+        'épuisé',
         'esgotado',
+        'indisponible',
         'indisponível',
         'isn\'t in stock right now',
         'isnt in stock right now',
@@ -39,6 +42,7 @@ function isItemInStock() {
         'let me know when it\'s available',
         'mail me when available',
         'message if back in stock',
+        'mevcut değil',
         'nachricht bei',
         'nicht auf lager',
         'nicht lagernd',
@@ -50,7 +54,9 @@ function isItemInStock() {
         'niet beschikbaar',
         'niet leverbaar',
         'niet op voorraad',
-        'no disponible temporalmente',
+        'no disponible',
+        'non disponibile',
+        'non disponible',
         'no longer in stock',
         'no tickets available',
         'not available',
@@ -63,16 +69,20 @@ function isItemInStock() {
         'não estamos a aceitar encomendas',
         'out of stock',
         'out-of-stock',
+        'plus disponible',
         'prodotto esaurito',
         'produkt niedostępny',
+        'rupture',
         'sold out',
         'sold-out',
+        'stokta yok',
         'temporarily out of stock',
         'temporarily unavailable',
         'there were no search results for',
         'this item is currently unavailable',
         'tickets unavailable',
         'tijdelijk uitverkocht',
+        'tükendi',
         'unavailable nearby',
         'unavailable tickets',
         'vergriffen',

changedetectionio/content_fetchers/res/xpath_element_scraper.js

@@ -41,7 +41,7 @@ const findUpTag = (el) => {
 
     //  Strategy 1: If it's an input, with name, and there's only one, prefer that
     if (el.name !== undefined && el.name.length) {
-        var proposed = el.tagName + "[name=" + el.name + "]";
+        var proposed = el.tagName + "[name=\"" + CSS.escape(el.name) + "\"]";
         var proposed_element = window.document.querySelectorAll(proposed);
         if (proposed_element.length) {
             if (proposed_element.length === 1) {
@@ -102,13 +102,15 @@ function collectVisibleElements(parent, visibleElements) {
     const children = parent.children;
     for (let i = 0; i < children.length; i++) {
         const child = children[i];
+        const computedStyle = window.getComputedStyle(child);
+
         if (
             child.nodeType === Node.ELEMENT_NODE &&
-            window.getComputedStyle(child).display !== 'none' &&
-            window.getComputedStyle(child).visibility !== 'hidden' &&
+            computedStyle.display !== 'none' &&
+            computedStyle.visibility !== 'hidden' &&
             child.offsetWidth >= 0 &&
             child.offsetHeight >= 0 &&
-            window.getComputedStyle(child).contentVisibility !== 'hidden'
+            computedStyle.contentVisibility !== 'hidden'
         ) {
             // If the child is an element and is visible, recursively collect visible elements
             collectVisibleElements(child, visibleElements);
@@ -173,6 +175,7 @@ visibleElementsArray.forEach(function (element) {
 
     // Try to identify any possible currency amounts "Sale: 4000" or "Sale now 3000 Kc", can help with the training.
     const hasDigitCurrency = (/\d/.test(text.slice(0, 6)) || /\d/.test(text.slice(-6)) ) &&  /([€£$¥₩₹]|USD|AUD|EUR|Kč|kr|SEK|,–)/.test(text) ;
+    const computedStyle = window.getComputedStyle(element);
 
     size_pos.push({
         xpath: xpath_result,
@@ -184,10 +187,10 @@ visibleElementsArray.forEach(function (element) {
         tagName: (element.tagName) ? element.tagName.toLowerCase() : '',
         // tagtype used by Browser Steps
         tagtype: (element.tagName.toLowerCase() === 'input' && element.type) ? element.type.toLowerCase() : '',
-        isClickable: window.getComputedStyle(element).cursor === "pointer",
+        isClickable: computedStyle.cursor === "pointer",
         // Used by the keras trainer
-        fontSize: window.getComputedStyle(element).getPropertyValue('font-size'),
-        fontWeight: window.getComputedStyle(element).getPropertyValue('font-weight'),
+        fontSize: computedStyle.getPropertyValue('font-size'),
+        fontWeight: computedStyle.getPropertyValue('font-weight'),
         hasDigitCurrency: hasDigitCurrency,
         label: label,
     });

changedetectionio/diff.py

@@ -1,6 +1,9 @@
 import difflib
 from typing import List, Iterator, Union
 
+REMOVED_STYLE = "background-color: #fadad7; color: #b30000;"
+ADDED_STYLE = "background-color: #eaf2c2; color: #406619;"
+
 def same_slicer(lst: List[str], start: int, end: int) -> List[str]:
     """Return a slice of the list, or a single element if start == end."""
     return lst[start:end] if start != end else [lst[start]]
@@ -12,11 +15,12 @@ def customSequenceMatcher(
     include_removed: bool = True,
     include_added: bool = True,
     include_replaced: bool = True,
-    include_change_type_prefix: bool = True
+    include_change_type_prefix: bool = True,
+    html_colour: bool = False
 ) -> Iterator[List[str]]:
     """
     Compare two sequences and yield differences based on specified parameters.
-    
+
     Args:
         before (List[str]): Original sequence
         after (List[str]): Modified sequence
@@ -25,26 +29,35 @@ def customSequenceMatcher(
         include_added (bool): Include added parts
         include_replaced (bool): Include replaced parts
         include_change_type_prefix (bool): Add prefixes to indicate change types
-    
+        html_colour (bool): Use HTML background colors for differences
+
     Yields:
         List[str]: Differences between sequences
     """
     cruncher = difflib.SequenceMatcher(isjunk=lambda x: x in " \t", a=before, b=after)
-    
+
+
+
     for tag, alo, ahi, blo, bhi in cruncher.get_opcodes():
         if include_equal and tag == 'equal':
             yield before[alo:ahi]
         elif include_removed and tag == 'delete':
-            prefix = "(removed) " if include_change_type_prefix else ''
-            yield [f"{prefix}{line}" for line in same_slicer(before, alo, ahi)]
+            if html_colour:
+                yield [f'<span style="{REMOVED_STYLE}">{line}</span>' for line in same_slicer(before, alo, ahi)]
+            else:
+                yield [f"(removed) {line}" for line in same_slicer(before, alo, ahi)] if include_change_type_prefix else same_slicer(before, alo, ahi)
         elif include_replaced and tag == 'replace':
-            prefix_changed = "(changed) " if include_change_type_prefix else ''
-            prefix_into = "(into) " if include_change_type_prefix else ''
-            yield [f"{prefix_changed}{line}" for line in same_slicer(before, alo, ahi)] + \
-                  [f"{prefix_into}{line}" for line in same_slicer(after, blo, bhi)]
+            if html_colour:
+                yield [f'<span style="{REMOVED_STYLE}">{line}</span>' for line in same_slicer(before, alo, ahi)] + \
+                      [f'<span style="{ADDED_STYLE}">{line}</span>' for line in same_slicer(after, blo, bhi)]
+            else:
+                yield [f"(changed) {line}" for line in same_slicer(before, alo, ahi)] + \
+                      [f"(into) {line}" for line in same_slicer(after, blo, bhi)] if include_change_type_prefix else same_slicer(before, alo, ahi) + same_slicer(after, blo, bhi)
         elif include_added and tag == 'insert':
-            prefix = "(added) " if include_change_type_prefix else ''
-            yield [f"{prefix}{line}" for line in same_slicer(after, blo, bhi)]
+            if html_colour:
+                yield [f'<span style="{ADDED_STYLE}">{line}</span>' for line in same_slicer(after, blo, bhi)]
+            else:
+                yield [f"(added) {line}" for line in same_slicer(after, blo, bhi)] if include_change_type_prefix else same_slicer(after, blo, bhi)
 
 def render_diff(
     previous_version_file_contents: str,
@@ -55,11 +68,12 @@ def render_diff(
     include_replaced: bool = True,
     line_feed_sep: str = "\n",
     include_change_type_prefix: bool = True,
-    patch_format: bool = False
+    patch_format: bool = False,
+    html_colour: bool = False
 ) -> str:
     """
     Render the difference between two file contents.
-    
+
     Args:
         previous_version_file_contents (str): Original file contents
         newest_version_file_contents (str): Modified file contents
@@ -70,7 +84,8 @@ def render_diff(
         line_feed_sep (str): Separator for lines in output
         include_change_type_prefix (bool): Add prefixes to indicate change types
         patch_format (bool): Use patch format for output
-    
+        html_colour (bool): Use HTML background colors for differences
+
     Returns:
         str: Rendered difference
     """
@@ -88,10 +103,11 @@ def render_diff(
         include_removed=include_removed,
         include_added=include_added,
         include_replaced=include_replaced,
-        include_change_type_prefix=include_change_type_prefix
+        include_change_type_prefix=include_change_type_prefix,
+        html_colour=html_colour
     )
 
     def flatten(lst: List[Union[str, List[str]]]) -> str:
         return line_feed_sep.join(flatten(x) if isinstance(x, list) else x for x in lst)
 
-    return flatten(rendered_diff)
+    return flatten(rendered_diff)

changedetectionio/flask_app.py

@@ -1,6 +1,7 @@
 #!/usr/bin/env python3
 
 import datetime
+from zoneinfo import ZoneInfo
 
 import flask_login
 import locale
@@ -42,6 +43,7 @@ from loguru import logger
 from changedetectionio import html_tools, __version__
 from changedetectionio import queuedWatchMetaData
 from changedetectionio.api import api_v1
+from .time_handler import is_within_schedule
 
 datastore = None
 
@@ -84,7 +86,7 @@ csrf = CSRFProtect()
 csrf.init_app(app)
 notification_debug_log=[]
 
-# get locale ready
+# Locale for correct presentation of prices etc
 default_locale = locale.getdefaultlocale()
 logger.info(f"System locale default is {default_locale}")
 try:
@@ -596,17 +598,31 @@ def changedetection_app(config=None, datastore_o=None):
 
             if 'notification_title' in request.form and request.form['notification_title'].strip():
                 n_object['notification_title'] = request.form.get('notification_title', '').strip()
+            elif datastore.data['settings']['application'].get('notification_title'):
+                n_object['notification_title'] = datastore.data['settings']['application'].get('notification_title')
+            else:
+                n_object['notification_title'] = "Test title"
 
             if 'notification_body' in request.form and request.form['notification_body'].strip():
                 n_object['notification_body'] = request.form.get('notification_body', '').strip()
+            elif datastore.data['settings']['application'].get('notification_body'):
+                n_object['notification_body'] = datastore.data['settings']['application'].get('notification_body')
+            else:
+                n_object['notification_body'] = "Test body"
 
+            n_object['as_async'] = False
             n_object.update(watch.extra_notification_token_values())
+            from .notification import process_notification
+            sent_obj = process_notification(n_object, datastore)
 
-            from . import update_worker
-            new_worker = update_worker.update_worker(update_q, notification_q, app, datastore)
-            new_worker.queue_notification_for_watch(notification_q=notification_q, n_object=n_object, watch=watch)
         except Exception as e:
-            return make_response(f"Error: str(e)", 400)
+            e_str = str(e)
+            # Remove this text which is not important and floods the container
+            e_str = e_str.replace(
+                "DEBUG - <class 'apprise.decorators.base.CustomNotifyPlugin.instantiate_plugin.<locals>.CustomNotifyPluginWrapper'>",
+                '')
+
+            return make_response(e_str, 400)
 
         return 'OK - Sent test notifications'
 
@@ -696,27 +712,68 @@ def changedetection_app(config=None, datastore_o=None):
         # Does it use some custom form? does one exist?
         processor_name = datastore.data['watching'][uuid].get('processor', '')
         processor_classes = next((tpl for tpl in find_processors() if tpl[1] == processor_name), None)
+        
+        # If it's not found in traditional processors, check if it's a pluggy plugin
         if not processor_classes:
-            flash(f"Cannot load the edit form for processor/plugin '{processor_classes[1]}', plugin missing?", 'error')
-            return redirect(url_for('index'))
-
-        parent_module = get_parent_module(processor_classes[0])
-
-        try:
-            # Get the parent of the "processor.py" go up one, get the form (kinda spaghetti but its reusing existing code)
-            forms_module = importlib.import_module(f"{parent_module.__name__}.forms")
-            # Access the 'processor_settings_form' class from the 'forms' module
-            form_class = getattr(forms_module, 'processor_settings_form')
-        except ModuleNotFoundError as e:
-            # .forms didnt exist
-            form_class = forms.processor_text_json_diff_form
-        except AttributeError as e:
-            # .forms exists but no useful form
-            form_class = forms.processor_text_json_diff_form
+            try:
+                from changedetectionio.processors.processor_registry import get_processor_form, _get_plugin_name_map
+                
+                # Get all available plugins for debugging
+                available_plugins = list(_get_plugin_name_map().keys())
+                logger.debug(f"Available processor plugins: {available_plugins}")
+                
+                # Try to get the processor form
+                plugin_form_class = get_processor_form(processor_name)
+                
+                if plugin_form_class:
+                    # Use default text_json_diff_form as parent module for plugins
+                    from changedetectionio.processors.text_json_diff import processor as text_json_diff_processor
+                    form_class = forms.processor_text_json_diff_form
+                    parent_module = get_parent_module(text_json_diff_processor)
+                    
+                    # Skip the normal form loading code path
+                    use_plugin_form = True
+                    logger.debug(f"Successfully loaded form for plugin '{processor_name}'")
+                else:
+                    # Check if the plugin is registered but doesn't have a form
+                    if processor_name in available_plugins:
+                        logger.error(f"Plugin '{processor_name}' is registered but has no form class")
+                        flash(f"Plugin '{processor_name}' is registered but has no form class", 'error')
+                    else:
+                        logger.error(f"Cannot find plugin '{processor_name}'. Available plugins: {available_plugins}")
+                        flash(f"Cannot load the edit form for processor/plugin '{processor_name}', plugin missing?", 'error')
+                    return redirect(url_for('index'))
+            except ImportError as e:
+                logger.error(f"Import error when loading plugin form: {str(e)}")
+                flash(f"Cannot load the edit form for processor/plugin '{processor_name}', plugin system not available?", 'error')
+                return redirect(url_for('index'))
+            except Exception as e:
+                logger.error(f"Unexpected error loading plugin form: {str(e)}")
+                flash(f"Error loading plugin form: {str(e)}", 'error')
+                return redirect(url_for('index'))
+        else:
+            # Traditional processor - continue with normal flow
+            parent_module = get_parent_module(processor_classes[0])
+            use_plugin_form = False
+        
+        # Only follow this path for traditional processors
+        if not use_plugin_form:
+            try:
+                # Get the parent of the "processor.py" go up one, get the form (kinda spaghetti but its reusing existing code)
+                forms_module = importlib.import_module(f"{parent_module.__name__}.forms")
+                # Access the 'processor_settings_form' class from the 'forms' module
+                form_class = getattr(forms_module, 'processor_settings_form')
+            except ModuleNotFoundError as e:
+                # .forms didnt exist
+                form_class = forms.processor_text_json_diff_form
+            except AttributeError as e:
+                # .forms exists but no useful form
+                form_class = forms.processor_text_json_diff_form
 
         form = form_class(formdata=request.form if request.method == 'POST' else None,
                           data=default,
-                          extra_notification_tokens=default.extra_notification_token_values()
+                          extra_notification_tokens=default.extra_notification_token_values(),
+                          default_system_settings=datastore.data['settings']
                           )
 
         # For the form widget tag UUID back to "string name" for the field
@@ -804,7 +861,33 @@ def changedetection_app(config=None, datastore_o=None):
             # But in the case something is added we should save straight away
             datastore.needs_write_urgent = True
 
-            if not datastore.data['watching'][uuid].get('paused'):
+            # Do not queue on edit if its not within the time range
+
+            # @todo maybe it should never queue anyway on edit...
+            is_in_schedule = True
+            watch = datastore.data['watching'].get(uuid)
+
+            if watch.get('time_between_check_use_default'):
+                time_schedule_limit = datastore.data['settings']['requests'].get('time_schedule_limit', {})
+            else:
+                time_schedule_limit = watch.get('time_schedule_limit')
+
+            tz_name = time_schedule_limit.get('timezone')
+            if not tz_name:
+                tz_name =  datastore.data['settings']['application'].get('timezone', 'UTC')
+
+            if time_schedule_limit and time_schedule_limit.get('enabled'):
+                try:
+                    is_in_schedule = is_within_schedule(time_schedule_limit=time_schedule_limit,
+                                                        default_tz=tz_name
+                                                        )
+                except Exception as e:
+                    logger.error(
+                        f"{uuid} - Recheck scheduler, error handling timezone, check skipped - TZ name '{tz_name}' - {str(e)}")
+                    return False
+
+            #############################
+            if not datastore.data['watching'][uuid].get('paused') and is_in_schedule:
                 # Queue the watch for immediate recheck, with a higher priority
                 update_q.put(queuedWatchMetaData.PrioritizedItem(priority=1, item={'uuid': uuid}))
 
@@ -812,7 +895,7 @@ def changedetection_app(config=None, datastore_o=None):
             if request.args.get("next") and request.args.get("next") == 'diff':
                 return redirect(url_for('diff_history_page', uuid=uuid))
 
-            return redirect(url_for('index'))
+            return redirect(url_for('index', tag=request.args.get("tag",'')))
 
         else:
             if request.method == 'POST' and not form.validate():
@@ -832,30 +915,33 @@ def changedetection_app(config=None, datastore_o=None):
 
             system_uses_webdriver = datastore.data['settings']['application']['fetch_backend'] == 'html_webdriver'
 
-            is_html_webdriver = False
+            watch_uses_webdriver = False
             if (watch.get('fetch_backend') == 'system' and system_uses_webdriver) or watch.get('fetch_backend') == 'html_webdriver' or watch.get('fetch_backend', '').startswith('extra_browser_'):
-                is_html_webdriver = True
+                watch_uses_webdriver = True
+
+            from zoneinfo import available_timezones
 
             # Only works reliably with Playwright
-            visualselector_enabled = os.getenv('PLAYWRIGHT_DRIVER_URL', False) and is_html_webdriver
+
             template_args = {
                 'available_processors': processors.available_processors(),
+                'available_timezones': sorted(available_timezones()),
                 'browser_steps_config': browser_step_ui_config,
                 'emailprefix': os.getenv('NOTIFICATION_MAIL_BUTTON_PREFIX', False),
-                'extra_title': f" - Edit - {watch.label}",
-                'extra_processor_config': form.extra_tab_content(),
                 'extra_notification_token_placeholder_info': datastore.get_unique_notification_token_placeholders_available(),
+                'extra_processor_config': form.extra_tab_content(),
+                'extra_title': f" - Edit - {watch.label}",
                 'form': form,
                 'has_default_notification_urls': True if len(datastore.data['settings']['application']['notification_urls']) else False,
                 'has_extra_headers_file': len(datastore.get_all_headers_in_textfile_for_watch(uuid=uuid)) > 0,
                 'has_special_tag_options': _watch_has_tag_options_set(watch=watch),
-                'is_html_webdriver': is_html_webdriver,
+                'watch_uses_webdriver': watch_uses_webdriver,
                 'jq_support': jq_support,
                 'playwright_enabled': os.getenv('PLAYWRIGHT_DRIVER_URL', False),
                 'settings_application': datastore.data['settings']['application'],
+                'timezone_default_config': datastore.data['settings']['application'].get('timezone'),
                 'using_global_webdriver_wait': not default['webdriver_delay'],
                 'uuid': uuid,
-                'visualselector_enabled': visualselector_enabled,
                 'watch': watch
             }
 
@@ -882,6 +968,8 @@ def changedetection_app(config=None, datastore_o=None):
     @login_optionally_required
     def settings_page():
         from changedetectionio import forms
+        from datetime import datetime
+        from zoneinfo import available_timezones
 
         default = deepcopy(datastore.data['settings'])
         if datastore.proxy_list is not None:
@@ -949,14 +1037,20 @@ def changedetection_app(config=None, datastore_o=None):
             else:
                 flash("An error occurred, please see below.", "error")
 
+        # Convert to ISO 8601 format, all date/time relative events stored as UTC time
+        utc_time = datetime.now(ZoneInfo("UTC")).isoformat()
+
         output = render_template("settings.html",
                                  api_key=datastore.data['settings']['application'].get('api_access_token'),
+                                 available_timezones=sorted(available_timezones()),
                                  emailprefix=os.getenv('NOTIFICATION_MAIL_BUTTON_PREFIX', False),
                                  extra_notification_token_placeholder_info=datastore.get_unique_notification_token_placeholders_available(),
                                  form=form,
                                  hide_remove_pass=os.getenv("SALTED_PASS", False),
                                  min_system_recheck_seconds=int(os.getenv('MINIMUM_SECONDS_RECHECK_TIME', 3)),
-                                 settings_application=datastore.data['settings']['application']
+                                 settings_application=datastore.data['settings']['application'],
+                                 timezone_default_config=datastore.data['settings']['application'].get('timezone'),
+                                 utc_time=utc_time,
                                  )
 
         return output
@@ -1269,12 +1363,23 @@ def changedetection_app(config=None, datastore_o=None):
 
             # These files should be in our subdirectory
             try:
-                # set nocache, set content-type
-                response = make_response(send_from_directory(os.path.join(datastore_o.datastore_path, filename), "elements.json"))
-                response.headers['Content-type'] = 'application/json'
-                response.headers['Cache-Control'] = 'no-cache, no-store, must-revalidate'
-                response.headers['Pragma'] = 'no-cache'
-                response.headers['Expires'] = 0
+                # set nocache, set content-type,
+                # `filename` is actually directory UUID of the watch
+                watch_directory = str(os.path.join(datastore_o.datastore_path, filename))
+                response = None
+                if os.path.isfile(os.path.join(watch_directory, "elements.deflate")):
+                    response = make_response(send_from_directory(watch_directory, "elements.deflate"))
+                    response.headers['Content-Type'] = 'application/json'
+                    response.headers['Content-Encoding'] = 'deflate'
+                else:
+                    logger.error(f'Request elements.deflate at "{watch_directory}" but was notfound.')
+                    abort(404)
+
+                if response:
+                    response.headers['Cache-Control'] = 'no-cache, no-store, must-revalidate'
+                    response.headers['Pragma'] = 'no-cache'
+                    response.headers['Expires'] = "0"
+
                 return response
 
             except FileNotFoundError:
@@ -1317,8 +1422,10 @@ def changedetection_app(config=None, datastore_o=None):
     @login_optionally_required
     def watch_get_preview_rendered(uuid):
         '''For when viewing the "preview" of the rendered text from inside of Edit'''
+        from flask import jsonify
         from .processors.text_json_diff import prepare_filter_prevew
-        return prepare_filter_prevew(watch_uuid=uuid, datastore=datastore)
+        result = prepare_filter_prevew(watch_uuid=uuid, form_data=request.form, datastore=datastore)
+        return jsonify(result)
 
 
     @app.route("/form/add/quickwatch", methods=['POST'])
@@ -1343,13 +1450,13 @@ def changedetection_app(config=None, datastore_o=None):
         if new_uuid:
             if add_paused:
                 flash('Watch added in Paused state, saving will unpause.')
-                return redirect(url_for('edit_page', uuid=new_uuid, unpause_on_save=1))
+                return redirect(url_for('edit_page', uuid=new_uuid, unpause_on_save=1, tag=request.args.get('tag')))
             else:
                 # Straight into the queue.
                 update_q.put(queuedWatchMetaData.PrioritizedItem(priority=1, item={'uuid': new_uuid}))
                 flash("Watch added.")
 
-        return redirect(url_for('index'))
+        return redirect(url_for('index', tag=request.args.get('tag','')))
 
 
 
@@ -1619,13 +1726,15 @@ def changedetection_app(config=None, datastore_o=None):
     import changedetectionio.blueprint.backups as backups
     app.register_blueprint(backups.construct_blueprint(datastore), url_prefix='/backups')
 
+    import changedetectionio.conditions.blueprint as conditions
+    app.register_blueprint(conditions.construct_blueprint(datastore), url_prefix='/conditions')
 
     # @todo handle ctrl break
     ticker_thread = threading.Thread(target=ticker_thread_check_time_launch_checks).start()
     threading.Thread(target=notification_runner).start()
 
     # Check for new release version, but not when running in test/build or pytest
-    if not os.getenv("GITHUB_REF", False) and not config.get('disable_checkver') == True:
+    if not os.getenv("GITHUB_REF", False) and not strtobool(os.getenv('DISABLE_VERSION_CHECK', 'no')):
         threading.Thread(target=check_for_new_version).start()
 
     return app
@@ -1709,7 +1818,6 @@ def notification_runner():
 def ticker_thread_check_time_launch_checks():
     import random
     from changedetectionio import update_worker
-
     proxy_last_called_time = {}
 
     recheck_time_minimum_seconds = int(os.getenv('MINIMUM_SECONDS_RECHECK_TIME', 3))
@@ -1767,6 +1875,28 @@ def ticker_thread_check_time_launch_checks():
             if watch['paused']:
                 continue
 
+            # @todo - Maybe make this a hook?
+            # Time schedule limit - Decide between watch or global settings
+            if watch.get('time_between_check_use_default'):
+                time_schedule_limit = datastore.data['settings']['requests'].get('time_schedule_limit', {})
+                logger.trace(f"{uuid} Time scheduler - Using system/global settings")
+            else:
+                time_schedule_limit = watch.get('time_schedule_limit')
+                logger.trace(f"{uuid} Time scheduler - Using watch settings (not global settings)")
+            tz_name = datastore.data['settings']['application'].get('timezone', 'UTC')
+
+            if time_schedule_limit and time_schedule_limit.get('enabled'):
+                try:
+                    result = is_within_schedule(time_schedule_limit=time_schedule_limit,
+                                                default_tz=tz_name
+                                                )
+                    if not result:
+                        logger.trace(f"{uuid} Time scheduler - not within schedule skipping.")
+                        continue
+                except Exception as e:
+                    logger.error(
+                        f"{uuid} - Recheck scheduler, error handling timezone, check skipped - TZ name '{tz_name}' - {str(e)}")
+                    return False
             # If they supplied an individual entry minutes to threshold.
             threshold = recheck_time_system_seconds if watch.get('time_between_check_use_default') else watch.threshold_seconds()
 

changedetectionio/forms.py

@@ -1,12 +1,15 @@
 import os
 import re
 from loguru import logger
+from wtforms.widgets.core import TimeInput
 
+from changedetectionio.conditions.form import ConditionFormRow
 from changedetectionio.strtobool import strtobool
 
 from wtforms import (
     BooleanField,
     Form,
+    Field,
     IntegerField,
     RadioField,
     SelectField,
@@ -125,6 +128,87 @@ class StringTagUUID(StringField):
 
         return 'error'
 
+class TimeDurationForm(Form):
+    hours = SelectField(choices=[(f"{i}", f"{i}") for i in range(0, 25)], default="24",  validators=[validators.Optional()])
+    minutes = SelectField(choices=[(f"{i}", f"{i}") for i in range(0, 60)], default="00", validators=[validators.Optional()])
+
+class TimeStringField(Field):
+    """
+    A WTForms field for time inputs (HH:MM) that stores the value as a string.
+    """
+    widget = TimeInput()  # Use the built-in time input widget
+
+    def _value(self):
+        """
+        Returns the value for rendering in the form.
+        """
+        return self.data if self.data is not None else ""
+
+    def process_formdata(self, valuelist):
+        """
+        Processes the raw input from the form and stores it as a string.
+        """
+        if valuelist:
+            time_str = valuelist[0]
+            # Simple validation for HH:MM format
+            if not time_str or len(time_str.split(":")) != 2:
+                raise ValidationError("Invalid time format. Use HH:MM.")
+            self.data = time_str
+
+
+class validateTimeZoneName(object):
+    """
+       Flask wtform validators wont work with basic auth
+    """
+
+    def __init__(self, message=None):
+        self.message = message
+
+    def __call__(self, form, field):
+        from zoneinfo import available_timezones
+        python_timezones = available_timezones()
+        if field.data and field.data not in python_timezones:
+            raise ValidationError("Not a valid timezone name")
+
+class ScheduleLimitDaySubForm(Form):
+    enabled = BooleanField("not set", default=True)
+    start_time = TimeStringField("Start At", default="00:00", validators=[validators.Optional()])
+    duration = FormField(TimeDurationForm, label="Run duration")
+
+class ScheduleLimitForm(Form):
+    enabled = BooleanField("Use time scheduler", default=False)
+    # Because the label for=""" doesnt line up/work with the actual checkbox
+    monday = FormField(ScheduleLimitDaySubForm, label="")
+    tuesday = FormField(ScheduleLimitDaySubForm, label="")
+    wednesday = FormField(ScheduleLimitDaySubForm, label="")
+    thursday = FormField(ScheduleLimitDaySubForm, label="")
+    friday = FormField(ScheduleLimitDaySubForm, label="")
+    saturday = FormField(ScheduleLimitDaySubForm, label="")
+    sunday = FormField(ScheduleLimitDaySubForm, label="")
+
+    timezone = StringField("Optional timezone to run in",
+                                  render_kw={"list": "timezones"},
+                                  validators=[validateTimeZoneName()]
+                                  )
+    def __init__(
+        self,
+        formdata=None,
+        obj=None,
+        prefix="",
+        data=None,
+        meta=None,
+        **kwargs,
+    ):
+        super().__init__(formdata, obj, prefix, data, meta, **kwargs)
+        self.monday.form.enabled.label.text="Monday"
+        self.tuesday.form.enabled.label.text = "Tuesday"
+        self.wednesday.form.enabled.label.text = "Wednesday"
+        self.thursday.form.enabled.label.text = "Thursday"
+        self.friday.form.enabled.label.text = "Friday"
+        self.saturday.form.enabled.label.text = "Saturday"
+        self.sunday.form.enabled.label.text = "Sunday"
+
+
 class TimeBetweenCheckForm(Form):
     weeks = IntegerField('Weeks', validators=[validators.Optional(), validators.NumberRange(min=0, message="Should contain zero or more seconds")])
     days = IntegerField('Days', validators=[validators.Optional(), validators.NumberRange(min=0, message="Should contain zero or more seconds")])
@@ -222,11 +306,17 @@ class ValidateAppRiseServers(object):
     def __call__(self, form, field):
         import apprise
         apobj = apprise.Apprise()
+
         # so that the custom endpoints are registered
-        from changedetectionio.apprise_plugin import apprise_custom_api_call_wrapper
+        from .apprise_asset import asset
+
         for server_url in field.data:
-            if not apobj.add(server_url):
-                message = field.gettext('\'%s\' is not a valid AppRise URL.' % (server_url))
+            url = server_url.strip()
+            if url.startswith("#"):
+                continue
+
+            if not apobj.add(url):
+                message = field.gettext('\'%s\' is not a valid AppRise URL.' % (url))
                 raise ValidationError(message)
 
 class ValidateJinja2Template(object):
@@ -279,6 +369,7 @@ class validateURL(object):
         # This should raise a ValidationError() or not
         validate_url(field.data)
 
+
 def validate_url(test_url):
     # If hosts that only contain alphanumerics are allowed ("localhost" for example)
     try:
@@ -421,6 +512,7 @@ class quickWatchForm(Form):
     edit_and_watch_submit_button = SubmitField('Edit > Watch', render_kw={"class": "pure-button pure-button-primary"})
 
 
+
 # Common to a single watch and the global settings
 class commonSettingsForm(Form):
     from . import processors
@@ -438,6 +530,7 @@ class commonSettingsForm(Form):
     notification_title = StringField('Notification Title', default='ChangeDetection.io Notification - {{ watch_url }}', validators=[validators.Optional(), ValidateJinja2Template()])
     notification_urls = StringListField('Notification URL List', validators=[validators.Optional(), ValidateAppRiseServers(), ValidateJinja2Template()])
     processor = RadioField( label=u"Processor - What do you want to achieve?", choices=processors.available_processors(), default="text_json_diff")
+    timezone = StringField("Timezone for watch schedule", render_kw={"list": "timezones"}, validators=[validateTimeZoneName()])
     webdriver_delay = IntegerField('Wait seconds before extracting text', validators=[validators.Optional(), validators.NumberRange(min=1, message="Should contain one or more seconds")])
 
 
@@ -448,7 +541,6 @@ class importForm(Form):
     xlsx_file = FileField('Upload .xlsx file', validators=[FileAllowed(['xlsx'], 'Must be .xlsx file!')])
     file_mapping = SelectField('File mapping', [validators.DataRequired()], choices={('wachete', 'Wachete mapping'), ('custom','Custom mapping')})
 
-
 class SingleBrowserStep(Form):
 
     operation = SelectField('Operation', [validators.Optional()], choices=browser_step_ui_config.keys())
@@ -466,6 +558,9 @@ class processor_text_json_diff_form(commonSettingsForm):
     tags = StringTagUUID('Group tag', [validators.Optional()], default='')
 
     time_between_check = FormField(TimeBetweenCheckForm)
+
+    time_schedule_limit = FormField(ScheduleLimitForm)
+
     time_between_check_use_default = BooleanField('Use global settings for time between check', default=False)
 
     include_filters = StringListField('CSS/JSONPath/JQ/XPath Filters', [ValidateCSSJSONXPATHInput()], default='')
@@ -505,6 +600,10 @@ class processor_text_json_diff_form(commonSettingsForm):
     notification_muted = BooleanField('Notifications Muted / Off', default=False)
     notification_screenshot = BooleanField('Attach screenshot to notification (where possible)', default=False)
 
+    conditions_match_logic = RadioField(u'Match', choices=[('ALL', 'Match all of the following'),('ANY', 'Match any of the following')], default='ALL')
+    conditions = FieldList(FormField(ConditionFormRow), min_entries=1)  # Add rule logic here
+
+
     def extra_tab_content(self):
         return None
 
@@ -567,6 +666,23 @@ class processor_text_json_diff_form(commonSettingsForm):
 
         return result
 
+    def __init__(
+            self,
+            formdata=None,
+            obj=None,
+            prefix="",
+            data=None,
+            meta=None,
+            **kwargs,
+    ):
+        super().__init__(formdata, obj, prefix, data, meta, **kwargs)
+        if kwargs and kwargs.get('default_system_settings'):
+            default_tz = kwargs.get('default_system_settings').get('application', {}).get('timezone')
+            if default_tz:
+                self.time_schedule_limit.form.timezone.render_kw['placeholder'] = default_tz
+
+
+
 class SingleExtraProxy(Form):
 
     # maybe better to set some <script>var..
@@ -587,6 +703,7 @@ class DefaultUAInputForm(Form):
 # datastore.data['settings']['requests']..
 class globalSettingsRequestForm(Form):
     time_between_check = FormField(TimeBetweenCheckForm)
+    time_schedule_limit = FormField(ScheduleLimitForm)
     proxy = RadioField('Proxy')
     jitter_seconds = IntegerField('Random jitter seconds ± check',
                                   render_kw={"style": "width: 5em;"},

changedetectionio/html_tools.py

@@ -1,5 +1,6 @@
-from typing import List
+from loguru import logger
 from lxml import etree
+from typing import List
 import json
 import re
 
@@ -298,8 +299,10 @@ def extract_json_as_string(content, json_filter, ensure_is_ldjson_info_type=None
 # https://github.com/dgtlmoon/changedetection.io/pull/2041#issuecomment-1848397161w
     # Try to parse/filter out the JSON, if we get some parser error, then maybe it's embedded within HTML tags
     try:
-        stripped_text_from_html = _parse_json(json.loads(content), json_filter)
-    except json.JSONDecodeError:
+        # .lstrip("\ufeff") strings ByteOrderMark from UTF8 and still lets the UTF work
+        stripped_text_from_html = _parse_json(json.loads(content.lstrip("\ufeff") ), json_filter)
+    except json.JSONDecodeError as e:
+        logger.warning(str(e))
 
         # Foreach <script json></script> blob.. just return the first that matches json_filter
         # As a last resort, try to parse the whole <body>

changedetectionio/model/App.py

@@ -52,7 +52,8 @@ class model(dict):
                     'schema_version' : 0,
                     'shared_diff_access': False,
                     'webdriver_delay': None , # Extra delay in seconds before extracting text
-                    'tags': {} #@todo use Tag.model initialisers
+                    'tags': {}, #@todo use Tag.model initialisers
+                    'timezone': None, # Default IANA timezone name
                 }
             }
         }
@@ -68,7 +69,7 @@ def parse_headers_from_text_file(filepath):
         for l in f.readlines():
             l = l.strip()
             if not l.startswith('#') and ':' in l:
-                (k, v) = l.split(':')
+                (k, v) = l.split(':', 1)  # Split only on the first colon
                 headers[k.strip()] = v.strip()
 
     return headers

changedetectionio/model/Watch.py

@@ -67,7 +67,6 @@ class model(watch_base):
 
     @property
     def link(self):
-
         url = self.get('url', '')
         if not is_safe_url(url):
             return 'DISABLED'
@@ -93,6 +92,19 @@ class model(watch_base):
         # Also double check it after any Jinja2 formatting just incase
         if not is_safe_url(ready_url):
             return 'DISABLED'
+            
+        # Check if a processor wants to customize the display link
+        processor_name = self.get('processor')
+        if processor_name:
+            try:
+                # Import here to avoid circular imports
+                from changedetectionio.processors.processor_registry import get_display_link
+                custom_link = get_display_link(url=ready_url, processor_name=processor_name)
+                if custom_link:
+                    return custom_link
+            except Exception as e:
+                logger.error(f"Error getting custom display link for processor {processor_name}: {str(e)}")
+                
         return ready_url
 
     def clear_watch(self):
@@ -247,37 +259,32 @@ class model(watch_base):
         bump = self.history
         return self.__newest_history_key
 
-    # Given an arbitrary timestamp, find the closest next key
-    # For example, last_viewed = 1000 so it should return the next 1001 timestamp
-    #
-    # used for the [diff] button so it can preset a smarter from_version
+    # Given an arbitrary timestamp, find the best history key for the [diff] button so it can preset a smarter from_version
     @property
-    def get_next_snapshot_key_to_last_viewed(self):
+    def get_from_version_based_on_last_viewed(self):
 
         """Unfortunately for now timestamp is stored as string key"""
         keys = list(self.history.keys())
         if not keys:
             return None
+        if len(keys) == 1:
+            return keys[0]
 
         last_viewed = int(self.get('last_viewed'))
-        prev_k = keys[0]
         sorted_keys = sorted(keys, key=lambda x: int(x))
         sorted_keys.reverse()
 
-        # When the 'last viewed' timestamp is greater than the newest snapshot, return second last
-        if last_viewed > int(sorted_keys[0]):
+        # When the 'last viewed' timestamp is greater than or equal the newest snapshot, return second newest
+        if last_viewed >= int(sorted_keys[0]):
             return sorted_keys[1]
+        
+        # When the 'last viewed' timestamp is between snapshots, return the older snapshot
+        for newer, older in list(zip(sorted_keys[0:], sorted_keys[1:])):
+            if last_viewed < int(newer) and last_viewed >= int(older):
+                return older
 
-        for k in sorted_keys:
-            if int(k) < last_viewed:
-                if prev_k == sorted_keys[0]:
-                    # Return the second last one so we dont recommend the same version compares itself
-                    return sorted_keys[1]
-
-                return prev_k
-            prev_k = k
-
-        return keys[0]
+        # When the 'last viewed' timestamp is less than the oldest snapshot, return oldest
+        return sorted_keys[-1]
 
     def get_history_snapshot(self, timestamp):
         import brotli
@@ -339,7 +346,6 @@ class model(watch_base):
         # @todo bump static cache of the last timestamp so we dont need to examine the file to set a proper ''viewed'' status
         return snapshot_fname
 
-    @property
     @property
     def has_empty_checktime(self):
         # using all() + dictionary comprehension
@@ -358,7 +364,7 @@ class model(watch_base):
     # Iterate over all history texts and see if something new exists
     # Always applying .strip() to start/end but optionally replace any other whitespace
     def lines_contain_something_unique_compared_to_history(self, lines: list, ignore_whitespace=False):
-        local_lines = []
+        local_lines = set([])
         if lines:
             if ignore_whitespace:
                 if isinstance(lines[0], str): # Can be either str or bytes depending on what was on the disk
@@ -533,21 +539,22 @@ class model(watch_base):
     def save_error_text(self, contents):
         self.ensure_data_dir_exists()
         target_path = os.path.join(self.watch_data_dir, "last-error.txt")
-        with open(target_path, 'w') as f:
+        with open(target_path, 'w', encoding='utf-8') as f:
             f.write(contents)
 
     def save_xpath_data(self, data, as_error=False):
         import json
+        import zlib
 
         if as_error:
-            target_path = os.path.join(self.watch_data_dir, "elements-error.json")
+            target_path = os.path.join(str(self.watch_data_dir), "elements-error.deflate")
         else:
-            target_path = os.path.join(self.watch_data_dir, "elements.json")
+            target_path = os.path.join(str(self.watch_data_dir), "elements.deflate")
 
         self.ensure_data_dir_exists()
 
-        with open(target_path, 'w') as f:
-            f.write(json.dumps(data))
+        with open(target_path, 'wb') as f:
+            f.write(zlib.compress(json.dumps(data).encode()))
             f.close()
 
     # Save as PNG, PNG is larger but better for doing visual diff in the future

changedetectionio/model/__init__.py

@@ -59,6 +59,65 @@ class watch_base(dict):
             'text_should_not_be_present': [],  # Text that should not present
             'time_between_check': {'weeks': None, 'days': None, 'hours': None, 'minutes': None, 'seconds': None},
             'time_between_check_use_default': True,
+            "time_schedule_limit": {
+                "enabled": False,
+                "monday": {
+                    "enabled": True,
+                    "start_time": "00:00",
+                    "duration": {
+                        "hours": "24",
+                        "minutes": "00"
+                    }
+                },
+                "tuesday": {
+                    "enabled": True,
+                    "start_time": "00:00",
+                    "duration": {
+                        "hours": "24",
+                        "minutes": "00"
+                    }
+                },
+                "wednesday": {
+                    "enabled": True,
+                    "start_time": "00:00",
+                    "duration": {
+                        "hours": "24",
+                        "minutes": "00"
+                    }
+                },
+                "thursday": {
+                    "enabled": True,
+                    "start_time": "00:00",
+                    "duration": {
+                        "hours": "24",
+                        "minutes": "00"
+                    }
+                },
+                "friday": {
+                    "enabled": True,
+                    "start_time": "00:00",
+                    "duration": {
+                        "hours": "24",
+                        "minutes": "00"
+                    }
+                },
+                "saturday": {
+                    "enabled": True,
+                    "start_time": "00:00",
+                    "duration": {
+                        "hours": "24",
+                        "minutes": "00"
+                    }
+                },
+                "sunday": {
+                    "enabled": True,
+                    "start_time": "00:00",
+                    "duration": {
+                        "hours": "24",
+                        "minutes": "00"
+                    }
+                },
+            },
             'title': None,
             'track_ldjson_price_data': None,
             'trim_text_whitespace': False,

changedetectionio/notification.py

@@ -23,7 +23,7 @@ valid_tokens = {
 }
 
 default_notification_format_for_watch = 'System default'
-default_notification_format = 'Text'
+default_notification_format = 'HTML Color'
 default_notification_body = '{{watch_url}} had a change.\n---\n{{diff}}\n---\n'
 default_notification_title = 'ChangeDetection.io Notification - {{watch_url}}'
 
@@ -31,6 +31,7 @@ valid_notification_formats = {
     'Text': NotifyFormat.TEXT,
     'Markdown': NotifyFormat.MARKDOWN,
     'HTML': NotifyFormat.HTML,
+    'HTML Color': 'htmlcolor',
     # Used only for editing a watch (not for global)
     default_notification_format_for_watch: default_notification_format_for_watch
 }
@@ -66,6 +67,10 @@ def process_notification(n_object, datastore):
 
     sent_objs = []
     from .apprise_asset import asset
+
+    if 'as_async' in n_object:
+        asset.async_mode = n_object.get('as_async')
+
     apobj = apprise.Apprise(debug=True, asset=asset)
 
     if not n_object.get('notification_urls'):
@@ -76,9 +81,16 @@ def process_notification(n_object, datastore):
 
             # Get the notification body from datastore
             n_body = jinja_render(template_str=n_object.get('notification_body', ''), **notification_parameters)
+            if n_object.get('notification_format', '').startswith('HTML'):
+                n_body = n_body.replace("\n", '<br>')
+
             n_title = jinja_render(template_str=n_object.get('notification_title', ''), **notification_parameters)
 
             url = url.strip()
+            if url.startswith('#'):
+                logger.trace(f"Skipping commented out notification URL - {url}")
+                continue
+
             if not url:
                 logger.warning(f"Process Notification: skipping empty notification URL.")
                 continue
@@ -149,8 +161,6 @@ def process_notification(n_object, datastore):
             attach=n_object.get('screenshot', None)
         )
 
-        # Give apprise time to register an error
-        time.sleep(3)
 
         # Returns empty string if nothing found, multi-line string otherwise
         log_value = logs.getvalue()

changedetectionio/processors/__init__.py

@@ -3,6 +3,7 @@ from changedetectionio.content_fetchers.base import Fetcher
 from changedetectionio.strtobool import strtobool
 from copy import deepcopy
 from loguru import logger
+
 import hashlib
 import importlib
 import inspect
@@ -10,6 +11,10 @@ import os
 import pkgutil
 import re
 
+# Import the plugin manager
+from .pluggy_interface import plugin_manager
+
+
 class difference_detection_processor():
 
     browser_steps = None
@@ -26,15 +31,101 @@ class difference_detection_processor():
         self.watch = deepcopy(self.datastore.data['watching'].get(watch_uuid))
         # Generic fetcher that should be extended (requests, playwright etc)
         self.fetcher = Fetcher()
+        
+    def _get_proxy_for_watch(self, preferred_proxy_id=None):
+        """Get proxy configuration based on watch settings and preferred proxy ID
+        
+        Args:
+            preferred_proxy_id: Optional explicit proxy ID to use
+            
+        Returns:
+            dict: Proxy configuration or None if no proxy should be used
+            str: Proxy URL or None if no proxy should be used
+        """
+        # Default to no proxy config
+        proxy_config = None
+        proxy_url = None
+        
+        # Check if datastore is available and has get_preferred_proxy_for_watch method
+        if hasattr(self, 'datastore') and self.datastore:
+            try:
+                # Get preferred proxy ID if not provided
+                if not preferred_proxy_id and hasattr(self.datastore, 'get_preferred_proxy_for_watch'):
+                    # Get the watch UUID if available
+                    watch_uuid = None
+                    if hasattr(self.watch, 'get'):
+                        watch_uuid = self.watch.get('uuid')
+                    elif hasattr(self.watch, 'uuid'):
+                        watch_uuid = self.watch.uuid
+                    
+                    if watch_uuid:
+                        preferred_proxy_id = self.datastore.get_preferred_proxy_for_watch(uuid=watch_uuid)
+                
+                # Check if we have a proxy list and a valid proxy ID
+                if preferred_proxy_id and hasattr(self.datastore, 'proxy_list') and self.datastore.proxy_list:
+                    proxy_info = self.datastore.proxy_list.get(preferred_proxy_id)
+                    
+                    if proxy_info and 'url' in proxy_info:
+                        proxy_url = proxy_info.get('url')
+                        logger.debug(f"Selected proxy key '{preferred_proxy_id}' as proxy URL '{proxy_url}'")
+                        
+                        # Parse the proxy URL to build a proxy dict for requests
+                        import urllib.parse
+                        parsed_proxy = urllib.parse.urlparse(proxy_url)
+                        proxy_type = parsed_proxy.scheme
+                        
+                        # Extract credentials if present
+                        username = None
+                        password = None
+                        if parsed_proxy.username:
+                            username = parsed_proxy.username
+                            if parsed_proxy.password:
+                                password = parsed_proxy.password
+                        
+                        # Build the proxy URL without credentials for the proxy dict
+                        netloc = parsed_proxy.netloc
+                        if '@' in netloc:
+                            netloc = netloc.split('@')[1]
+                        
+                        proxy_addr = f"{proxy_type}://{netloc}"
+                        
+                        # Create the proxy configuration
+                        proxy_config = {
+                            'http': proxy_addr,
+                            'https': proxy_addr
+                        }
+                        
+                        # Add credentials if present
+                        if username:
+                            proxy_config['username'] = username
+                            if password:
+                                proxy_config['password'] = password
+            except Exception as e:
+                # Log the error but continue without a proxy
+                logger.error(f"Error setting up proxy: {str(e)}")
+                proxy_config = None
+                proxy_url = None
+                
+        return proxy_config, proxy_url
 
     def call_browser(self, preferred_proxy_id=None):
-
+        """Fetch content using the appropriate browser/fetcher
+        
+        This method will:
+        1. Determine the appropriate fetcher to use based on watch settings
+        2. Set up proxy configuration if needed
+        3. Initialize the fetcher with the correct parameters
+        4. Configure any browser steps if needed
+        
+        Args:
+            preferred_proxy_id: Optional explicit proxy ID to use
+        """
         from requests.structures import CaseInsensitiveDict
 
         url = self.watch.link
 
-        # Protect against file://, file:/ access, check the real "link" without any meta "source:" etc prepended.
-        if re.search(r'^file:/', url.strip(), re.IGNORECASE):
+        # Protect against file:, file:/, file:// access, check the real "link" without any meta "source:" etc prepended.
+        if re.search(r'^file:', url.strip(), re.IGNORECASE):
             if not strtobool(os.getenv('ALLOW_FILE_URI', 'false')):
                 raise Exception(
                     "file:// type access is denied for security reasons."
@@ -43,8 +134,8 @@ class difference_detection_processor():
         # Requests, playwright, other browser via wss:// etc, fetch_extra_something
         prefer_fetch_backend = self.watch.get('fetch_backend', 'system')
 
-        # Proxy ID "key"
-        preferred_proxy_id = preferred_proxy_id if preferred_proxy_id else self.datastore.get_preferred_proxy_for_watch(uuid=self.watch.get('uuid'))
+        # Get proxy configuration
+        proxy_config, proxy_url = self._get_proxy_for_watch(preferred_proxy_id)
 
         # Pluggable content self.fetcher
         if not prefer_fetch_backend or prefer_fetch_backend == 'system':
@@ -82,14 +173,10 @@ class difference_detection_processor():
             # What it referenced doesnt exist, Just use a default
             fetcher_obj = getattr(content_fetchers, "html_requests")
 
-        proxy_url = None
-        if preferred_proxy_id:
-            # Custom browser endpoints should NOT have a proxy added
-            if not prefer_fetch_backend.startswith('extra_browser_'):
-                proxy_url = self.datastore.proxy_list.get(preferred_proxy_id).get('url')
-                logger.debug(f"Selected proxy key '{preferred_proxy_id}' as proxy URL '{proxy_url}' for {url}")
-            else:
-                logger.debug(f"Skipping adding proxy data when custom Browser endpoint is specified. ")
+        # Custom browser endpoints should NOT have a proxy added
+        if proxy_url and prefer_fetch_backend.startswith('extra_browser_'):
+            logger.debug(f"Skipping adding proxy data when custom Browser endpoint is specified.")
+            proxy_url = None
 
         # Now call the fetcher (playwright/requests/etc) with arguments that only a fetcher would need.
         # When browser_connection_url is None, it method should default to working out whats the best defaults (os env vars etc)
@@ -185,9 +272,9 @@ def find_sub_packages(package_name):
 
 def find_processors():
     """
-    Find all subclasses of DifferenceDetectionProcessor in the specified package.
+    Find all subclasses of DifferenceDetectionProcessor in the specified package
+    and also include processors from the plugin system.
 
-    :param package_name: The name of the package to scan for processor modules.
     :return: A list of (module, class) tuples.
     """
     package_name = "changedetectionio.processors"  # Name of the current package/module
@@ -195,6 +282,7 @@ def find_processors():
     processors = []
     sub_packages = find_sub_packages(package_name)
 
+    # Find traditional processors
     for sub_package in sub_packages:
         module_name = f"{package_name}.{sub_package}.processor"
         try:
@@ -207,6 +295,15 @@ def find_processors():
         except (ModuleNotFoundError, ImportError) as e:
             logger.warning(f"Failed to import module {module_name}: {e} (find_processors())")
 
+    # Also include processors from the plugin system
+    try:
+        from .processor_registry import get_plugin_processor_modules
+        plugin_modules = get_plugin_processor_modules()
+        if plugin_modules:
+            processors.extend(plugin_modules)
+    except (ImportError, ModuleNotFoundError) as e:
+        logger.warning(f"Failed to import plugin modules: {e} (find_processors())")
+
     return processors
 
 
@@ -223,8 +320,22 @@ def get_parent_module(module):
     return False
 
 
-
 def get_custom_watch_obj_for_processor(processor_name):
+    """
+    Get the custom watch object for a processor
+    :param processor_name: Name of the processor
+    :return: Watch class or None
+    """
+    # First, try to get the watch model from the pluggy system
+    try:
+        from .processor_registry import get_processor_watch_model
+        watch_model = get_processor_watch_model(processor_name)
+        if watch_model:
+            return watch_model
+    except Exception as e:
+        logger.warning(f"Error getting processor watch model from pluggy: {e}")
+
+    # Fall back to the traditional approach
     from changedetectionio.model import Watch
     watch_class = Watch.model
     processor_classes = find_processors()
@@ -241,14 +352,47 @@ def get_custom_watch_obj_for_processor(processor_name):
 def available_processors():
     """
     Get a list of processors by name and description for the UI elements
-    :return: A list :)
+    :return: A list of tuples (processor_name, description)
     """
-
-    processor_classes = find_processors()
-
-    available = []
-    for package, processor_class in processor_classes:
-        available.append((processor_class, package.name))
-
-    return available
-
+    # Get processors from the pluggy system
+    pluggy_processors = []
+    try:
+        from .processor_registry import get_all_processors
+        pluggy_processors = get_all_processors()
+    except Exception as e:
+        logger.error(f"Error getting processors from pluggy: {str(e)}")
+    
+    # Get processors from the traditional file-based system
+    traditional_processors = []
+    try:
+        # Let's not use find_processors() directly since it now also includes pluggy processors
+        package_name = "changedetectionio.processors"
+        sub_packages = find_sub_packages(package_name)
+        
+        for sub_package in sub_packages:
+            module_name = f"{package_name}.{sub_package}.processor"
+            try:
+                module = importlib.import_module(module_name)
+                # Get the name and description from the module if available
+                name = getattr(module, 'name', f"Traditional processor: {sub_package}")
+                description = getattr(module, 'description', sub_package)
+                traditional_processors.append((sub_package, name))
+            except (ModuleNotFoundError, ImportError, AttributeError) as e:
+                logger.warning(f"Failed to import module {module_name}: {e} (available_processors())")
+    except Exception as e:
+        logger.error(f"Error getting traditional processors: {str(e)}")
+    
+    # Combine the lists, ensuring no duplicates
+    # Pluggy processors take precedence
+    all_processors = []
+    
+    # Add all pluggy processors
+    all_processors.extend(pluggy_processors)
+    
+    # Add traditional processors that aren't already registered via pluggy
+    pluggy_processor_names = [name for name, _ in pluggy_processors]
+    for processor_class, name in traditional_processors:
+        if processor_class not in pluggy_processor_names:
+            all_processors.append((processor_class, name))
+    
+    return all_processors

changedetectionio/processors/form.py

@@ -0,0 +1,17 @@
+from wtforms import (
+    BooleanField,
+    validators,
+    RadioField
+)
+from wtforms.fields.choices import SelectField
+from wtforms.fields.form import FormField
+from wtforms.form import Form
+
+class BaseProcessorForm(Form):
+    """Base class for processor forms"""
+    
+    def extra_tab_content(self):
+        return None
+
+    def extra_form_content(self):
+        return None

changedetectionio/processors/forms.py

@@ -0,0 +1,4 @@
+"""
+Forms for processors
+"""
+from changedetectionio.forms import processor_text_json_diff_form

changedetectionio/processors/pluggy_interface.py

@@ -0,0 +1,69 @@
+import pluggy
+
+# Define the plugin namespace for processors
+PLUGIN_NAMESPACE = "changedetectionio_processors"
+
+hookspec = pluggy.HookspecMarker(PLUGIN_NAMESPACE)
+hookimpl = pluggy.HookimplMarker(PLUGIN_NAMESPACE)
+
+
+class ProcessorSpec:
+    """Hook specifications for processor plugins."""
+
+    @hookspec
+    def get_processor_name():
+        """Return the name of the processor."""
+        pass
+
+    @hookspec
+    def get_processor_description():
+        """Return the description of the processor."""
+        pass
+    
+    @hookspec
+    def get_processor_class():
+        """Return the processor class."""
+        pass
+    
+    @hookspec
+    def get_processor_form():
+        """Return the processor form class."""
+        pass
+    
+    @hookspec
+    def get_processor_watch_model():
+        """Return the watch model class for this processor (if any)."""
+        pass
+    
+    @hookspec
+    def get_display_link(url, processor_name):
+        """Return a custom display link for the given processor.
+        
+        Args:
+            url: The original URL from the watch
+            processor_name: The name of the processor
+            
+        Returns:
+            A string with the custom display link or None to use the default
+        """
+        pass
+    
+    @hookspec
+    def perform_site_check(datastore, watch_uuid):
+        """Create and return a processor instance ready to perform site check.
+        
+        Args:
+            datastore: The application datastore
+            watch_uuid: The UUID of the watch to check
+            
+        Returns:
+            A processor instance ready to perform site check
+        """
+        pass
+
+
+# Set up the plugin manager
+plugin_manager = pluggy.PluginManager(PLUGIN_NAMESPACE)
+
+# Register hook specifications
+plugin_manager.add_hookspecs(ProcessorSpec)

changedetectionio/processors/processor_registry.py

@@ -0,0 +1,222 @@
+from loguru import logger
+from changedetectionio.model import Watch
+from .pluggy_interface import plugin_manager
+from typing import Dict, Any, List, Tuple, Optional, TypeVar, Type
+import functools
+
+# Import and register internal plugins
+from . import whois_plugin
+from . import test_plugin
+
+# Register plugins
+plugin_manager.register(whois_plugin)
+plugin_manager.register(test_plugin)
+
+# Load any setuptools entrypoints
+plugin_manager.load_setuptools_entrypoints("changedetectionio_processors")
+
+# Type definitions for better type hinting
+T = TypeVar('T')
+ProcessorClass = TypeVar('ProcessorClass')
+ProcessorForm = TypeVar('ProcessorForm')
+ProcessorWatchModel = TypeVar('ProcessorWatchModel')
+ProcessorInstance = TypeVar('ProcessorInstance')
+
+# Cache for plugin name mapping to improve performance
+_plugin_name_map: Dict[str, Any] = {}
+
+def register_plugin(plugin_module):
+    """Register a processor plugin"""
+    plugin_manager.register(plugin_module)
+    # Clear the plugin name map cache when a new plugin is registered
+    global _plugin_name_map
+    _plugin_name_map = {}
+
+def _get_plugin_name_map() -> Dict[str, Any]:
+    """Get a mapping of processor names to plugins
+    :return: Dictionary mapping processor names to plugins
+    """
+    global _plugin_name_map
+    
+    # Return cached map if available
+    if _plugin_name_map:
+        return _plugin_name_map
+    
+    # Build the map
+    result = {}
+    
+    # Get all plugins from the plugin manager
+    all_plugins = list(plugin_manager.get_plugins())
+    
+    # First register known internal plugins by name for reliability
+    known_plugins = {
+        'whois': whois_plugin,
+        'test': test_plugin
+    }
+    
+    for name, plugin in known_plugins.items():
+        if plugin in all_plugins:
+            result[name] = plugin
+    
+    # Then process remaining plugins through the hook system
+    for plugin in all_plugins:
+        if plugin in known_plugins.values():
+            continue  # Skip plugins we've already registered
+            
+        try:
+            # Get the processor name from this plugin
+            name_results = plugin_manager.hook.get_processor_name(plugin=plugin)
+            
+            if name_results:
+                plugin_name = name_results[0]
+                
+                # Check for name collisions
+                if plugin_name in result:
+                    logger.warning(f"Plugin name collision: '{plugin_name}' is already registered")
+                    continue
+                    
+                result[plugin_name] = plugin
+        except Exception as e:
+            logger.error(f"Error getting processor name from plugin: {str(e)}")
+    
+    # Cache the map
+    _plugin_name_map = result
+    return result
+
+def _get_plugin_by_name(processor_name: str) -> Optional[Any]:
+    """Get a plugin by its processor name
+    :param processor_name: Name of the processor
+    :return: Plugin object or None
+    """
+    return _get_plugin_name_map().get(processor_name)
+
+def _call_hook_for_plugin(plugin: Any, hook_name: str, default_value: T = None, **kwargs) -> Optional[T]:
+    """Call a hook for a specific plugin and handle exceptions
+    :param plugin: The plugin to call the hook for
+    :param hook_name: Name of the hook to call
+    :param default_value: Default value to return if the hook call fails
+    :param kwargs: Additional arguments to pass to the hook
+    :return: Result of the hook call or default value
+    """
+    if not plugin:
+        return default_value
+    
+    try:
+        hook = getattr(plugin_manager.hook, hook_name)
+        results = hook(plugin=plugin, **kwargs)
+        
+        if results:
+            return results[0]
+    except Exception as e:
+        logger.error(f"Error calling {hook_name} for plugin: {str(e)}")
+    
+    return default_value
+
+def get_all_processors() -> List[Tuple[str, str]]:
+    """Get all processors
+    :return: List of tuples (processor_name, processor_description)
+    """
+    processors = []
+    
+    for processor_name, plugin in _get_plugin_name_map().items():
+        description = _call_hook_for_plugin(plugin, 'get_processor_description')
+        if description:
+            processors.append((processor_name, description))
+    
+    return processors
+
+def get_processor_class(processor_name: str) -> Optional[Type[ProcessorClass]]:
+    """Get processor class by name
+    :param processor_name: Name of the processor
+    :return: Processor class or None
+    """
+    plugin = _get_plugin_by_name(processor_name)
+    return _call_hook_for_plugin(plugin, 'get_processor_class')
+
+def get_processor_form(processor_name: str) -> Optional[Type[ProcessorForm]]:
+    """Get processor form by name
+    :param processor_name: Name of the processor
+    :return: Processor form class or None
+    """
+    plugin = _get_plugin_by_name(processor_name)
+    return _call_hook_for_plugin(plugin, 'get_processor_form')
+
+def get_processor_watch_model(processor_name: str) -> Type[ProcessorWatchModel]:
+    """Get processor watch model by name
+    :param processor_name: Name of the processor
+    :return: Watch model class or default Watch model
+    """
+    plugin = _get_plugin_by_name(processor_name)
+    return _call_hook_for_plugin(plugin, 'get_processor_watch_model', default_value=Watch.model)
+
+def get_processor_site_check(processor_name: str, datastore: Any, watch_uuid: str) -> Optional[ProcessorInstance]:
+    """Get a processor instance ready to perform site check
+    :param processor_name: Name of the processor
+    :param datastore: The application datastore
+    :param watch_uuid: The UUID of the watch to check
+    :return: A processor instance ready to perform site check, or None
+    """
+    plugin = _get_plugin_by_name(processor_name)
+    if not plugin:
+        return None
+    
+    # Try to get the perform_site_check implementation
+    try:
+        processor = _call_hook_for_plugin(
+            plugin, 
+            'perform_site_check', 
+            datastore=datastore, 
+            watch_uuid=watch_uuid
+        )
+        if processor:
+            return processor
+        
+        # If no perform_site_check hook implementation, try getting the class and instantiating it
+        processor_class = _call_hook_for_plugin(plugin, 'get_processor_class')
+        if processor_class:
+            return processor_class(datastore=datastore, watch_uuid=watch_uuid)
+    except Exception as e:
+        logger.error(f"Error getting processor site check for {processor_name}: {str(e)}")
+    
+    return None
+
+def get_display_link(url: str, processor_name: str) -> Optional[str]:
+    """Get a custom display link for the given processor
+    :param url: The original URL from the watch
+    :param processor_name: Name of the processor
+    :return: A string with the custom display link or None to use the default
+    """
+    plugin = _get_plugin_by_name(processor_name)
+    return _call_hook_for_plugin(
+        plugin, 
+        'get_display_link', 
+        url=url, 
+        processor_name=processor_name
+    )
+
+def get_plugin_processor_modules() -> List[Tuple[Any, str]]:
+    """Get processor modules for all plugins that can be used with the find_processors function
+    
+    This function adapts pluggy plugins to be compatible with the traditional find_processors system
+    
+    :return: A list of (module, processor_name) tuples
+    """
+    result = []
+    
+    # Import base modules once to avoid repeated imports
+    from changedetectionio.processors.text_json_diff import processor as text_json_diff_processor
+
+    # For each plugin, map to a suitable module for find_processors
+    for processor_name, plugin in _get_plugin_name_map().items():
+        try:
+            processor_class = _call_hook_for_plugin(plugin, 'get_processor_class')
+            
+            if processor_class:
+                # Check if this processor extends the text_json_diff processor
+                base_class_name = str(processor_class.__bases__[0].__name__)
+                if base_class_name == 'perform_site_check' or 'TextJsonDiffProcessor' in base_class_name:
+                    result.append((text_json_diff_processor, processor_name))
+        except Exception as e:
+            logger.error(f"Error mapping processor module for {processor_name}: {str(e)}")
+    
+    return result

changedetectionio/processors/text_json_diff/__init__.py

@@ -28,13 +28,13 @@ def _task(watch, update_handler):
     return text_after_filter
 
 
-def prepare_filter_prevew(datastore, watch_uuid):
+def prepare_filter_prevew(datastore, watch_uuid, form_data):
     '''Used by @app.route("/edit/<string:uuid>/preview-rendered", methods=['POST'])'''
     from changedetectionio import forms, html_tools
     from changedetectionio.model.Watch import model as watch_model
     from concurrent.futures import ProcessPoolExecutor
     from copy import deepcopy
-    from flask import request, jsonify
+    from flask import request
     import brotli
     import importlib
     import os
@@ -50,12 +50,12 @@ def prepare_filter_prevew(datastore, watch_uuid):
 
     if tmp_watch and tmp_watch.history and os.path.isdir(tmp_watch.watch_data_dir):
         # Splice in the temporary stuff from the form
-        form = forms.processor_text_json_diff_form(formdata=request.form if request.method == 'POST' else None,
-                                                   data=request.form
+        form = forms.processor_text_json_diff_form(formdata=form_data if request.method == 'POST' else None,
+                                                   data=form_data
                                                    )
 
         # Only update vars that came in via the AJAX post
-        p = {k: v for k, v in form.data.items() if k in request.form.keys()}
+        p = {k: v for k, v in form.data.items() if k in form_data.keys()}
         tmp_watch.update(p)
         blank_watch_no_filters = watch_model()
         blank_watch_no_filters['url'] = tmp_watch.get('url')
@@ -103,13 +103,12 @@ def prepare_filter_prevew(datastore, watch_uuid):
 
     logger.trace(f"Parsed in {time.time() - now:.3f}s")
 
-    return jsonify(
-        {
+    return ({
             'after_filter': text_after_filter,
             'before_filter': text_before_filter.decode('utf-8') if isinstance(text_before_filter, bytes) else text_before_filter,
             'duration': time.time() - now,
             'trigger_line_numbers': trigger_line_numbers,
             'ignore_line_numbers': ignore_line_numbers,
-        }
-    )
+        })
+
 

changedetectionio/processors/text_json_diff/processor.py

@@ -6,6 +6,7 @@ import os
 import re
 import urllib3
 
+from changedetectionio.conditions import execute_ruleset_against_all_plugins
 from changedetectionio.processors import difference_detection_processor
 from changedetectionio.html_tools import PERL_STYLE_REGEX, cdata_in_document_to_text, TRANSLATE_WHITESPACE_TABLE
 from changedetectionio import html_tools, content_fetchers
@@ -331,6 +332,16 @@ class perform_site_check(difference_detection_processor):
             if result:
                 blocked = True
 
+        # And check if 'conditions' will let this pass through
+        if watch.get('conditions') and watch.get('conditions_match_logic'):
+            if not execute_ruleset_against_all_plugins(current_watch_uuid=watch.get('uuid'),
+                                                application_datastruct=self.datastore.data,
+                                                ephemeral_data={
+                                                    'text': stripped_text_from_html
+                                                }
+                                                ):
+                # Conditions say "Condition not met" so we block it.
+                blocked = True
 
         # Looks like something changed, but did it match all the rules?
         if blocked:

changedetectionio/processors/whois_plugin.py

@@ -0,0 +1,169 @@
+from loguru import logger
+import re
+import urllib.parse
+from .pluggy_interface import hookimpl
+from requests.structures import CaseInsensitiveDict
+from changedetectionio.content_fetchers.base import Fetcher
+
+# Import the text_json_diff processor
+from changedetectionio.processors.text_json_diff.processor import perform_site_check as TextJsonDiffProcessor
+
+# WHOIS Processor implementation that extends TextJsonDiffProcessor
+class WhoisProcessor(TextJsonDiffProcessor):
+    
+    def _extract_domain_from_url(self, url):
+        """Extract domain from URL, removing www. prefix if present"""
+        parsed_url = urllib.parse.urlparse(url)
+        domain = parsed_url.netloc
+        
+        # Remove www. prefix if present
+        domain = re.sub(r'^www\.', '', domain)
+        
+        return domain
+    
+    def call_browser(self, preferred_proxy_id=None):
+        """Override call_browser to perform WHOIS lookup instead of using a browser
+        
+        Note: The python-whois library doesn't directly support proxies. For real proxy support,
+        we would need to implement a custom socket connection that routes through the proxy.
+        This is a TODO for a future enhancement.
+        """
+        # Initialize a basic fetcher - this is used by the parent class
+        self.fetcher = Fetcher()
+        
+        # Extract URL from watch
+        url = self.watch.link
+        
+        # Check for file:// access
+        if re.search(r'^file:', url.strip(), re.IGNORECASE):
+            if not self.datastore.data.get('settings', {}).get('application', {}).get('allow_file_uri', False):
+                raise Exception("file:// type access is denied for security reasons.")
+        
+        # Extract domain from URL
+        domain = self._extract_domain_from_url(url)
+        
+        # Ensure we have a valid domain
+        if not domain:
+            error_msg = f"Could not extract domain from URL: '{url}'"
+            self.fetcher.content = error_msg
+            self.fetcher.status_code = 400
+            logger.error(error_msg)
+            return
+        
+        # Get proxy configuration using the common method from parent class
+        proxy_config, proxy_url = super()._get_proxy_for_watch(preferred_proxy_id)
+        
+        try:
+            # Use python-whois to get domain information
+            import whois
+            
+            # If we have proxy config, use it for the WHOIS lookup
+            # Note: The python-whois library doesn't directly support proxies,
+            # but we can implement proxy support if necessary using custom socket code
+            if proxy_config:
+                # For now, just log that we would use a proxy
+                logger.info(f"Using proxy for WHOIS lookup: {proxy_config}")
+            
+            # Perform the WHOIS lookup
+            whois_info = whois.whois(domain)
+            
+            # Convert whois_info object to text
+            if hasattr(whois_info, 'text'):
+                # Some whois implementations store raw text in .text attribute
+                whois_text = whois_info.text
+            else:
+                # Otherwise, format it nicely as key-value pairs
+                whois_text = f"WHOIS Information for domain: {domain}\n\n"
+                for key, value in whois_info.items():
+                    if value:
+                        whois_text += f"{key}: {value}\n"
+            
+            # Set the content and status for the fetcher
+            self.fetcher.content = whois_text
+            self.fetcher.status_code = 200
+            
+            # Setup headers dictionary for the fetcher
+            self.fetcher.headers = CaseInsensitiveDict({
+                'content-type': 'text/plain',
+                'server': 'whois-processor'
+            })
+            
+            # Add getters for headers
+            self.fetcher.get_all_headers = lambda: self.fetcher.headers
+            self.fetcher.get_last_status_code = lambda: self.fetcher.status_code
+            
+            # Implement necessary methods
+            self.fetcher.quit = lambda: None
+            
+        except Exception as e:
+            error_msg = f"Error fetching WHOIS data for domain {domain}: {str(e)}"
+            self.fetcher.content = error_msg
+            self.fetcher.status_code = 500
+            self.fetcher.headers = CaseInsensitiveDict({
+                'content-type': 'text/plain',
+                'server': 'whois-processor'
+            })
+            self.fetcher.get_all_headers = lambda: self.fetcher.headers
+            self.fetcher.get_last_status_code = lambda: self.fetcher.status_code
+            self.fetcher.quit = lambda: None
+            logger.error(error_msg)
+
+        return
+    
+    def run_changedetection(self, watch):
+        """Use the parent's run_changedetection which will use our overridden call_browser method"""
+        try:
+            # Let the parent class handle everything now that we've overridden call_browser
+            changed_detected, update_obj, filtered_text = super().run_changedetection(watch)
+            return changed_detected, update_obj, filtered_text
+            
+        except Exception as e:
+            error_msg = f"Error in WHOIS processor: {str(e)}"
+            update_obj = {'last_notification_error': False, 'last_error': error_msg}
+            logger.error(error_msg)
+            return False, update_obj, error_msg.encode('utf-8')
+
+    @staticmethod
+    def perform_site_check(datastore, watch_uuid):
+        """Factory method to create a WhoisProcessor instance - for compatibility with legacy code"""
+        processor = WhoisProcessor(datastore=datastore, watch_uuid=watch_uuid)
+        return processor
+
+@hookimpl
+def perform_site_check(datastore, watch_uuid):
+    """Create and return a processor instance ready to perform site check"""
+    return WhoisProcessor(datastore=datastore, watch_uuid=watch_uuid)
+
+@hookimpl(trylast=True)  # Use trylast to ensure this runs last in case of conflicts
+def get_processor_name():
+    """Return the name of this processor"""
+    from loguru import logger
+    logger.debug("whois_plugin.get_processor_name() called")
+    return "whois"
+
+@hookimpl
+def get_processor_description():
+    """Return the description of this processor"""
+    return "WHOIS Domain Information Changes Detector"
+
+@hookimpl
+def get_processor_class():
+    """Return the processor class"""
+    return WhoisProcessor
+
+@hookimpl
+def get_processor_form():
+    """Return the processor form class"""
+    # Import here to avoid circular imports
+    try:
+        from changedetectionio.forms import processor_text_json_diff_form
+        return processor_text_json_diff_form
+    except Exception as e:
+        from loguru import logger
+        logger.error(f"Error importing form for whois plugin: {str(e)}")
+        return None
+
+@hookimpl
+def get_processor_watch_model():
+    """Return the watch model class for this processor"""
+    return None  # Use default watch model

changedetectionio/static/images/copy.svg

@@ -1,7 +1,7 @@
 <?xml version="1.0" encoding="UTF-8" standalone="no"?>
 <svg
    version="1.1"
-   id="Layer_1"
+   id="copy"
    x="0px"
    y="0px"
    viewBox="0 0 115.77 122.88"

changedetectionio/static/images/email.svg

@@ -6,7 +6,7 @@
    height="7.5005589"
    width="11.248507"
    version="1.1"
-   id="Layer_1"
+   id="email"
    viewBox="0 0 7.1975545 4.7993639"
    xml:space="preserve"
    xmlns="http://www.w3.org/2000/svg"

changedetectionio/static/images/schedule.svg

@@ -0,0 +1,225 @@
+<?xml version="1.0" encoding="UTF-8" standalone="no"?>
+<svg
+   version="1.1"
+   id="schedule"
+   x="0px"
+   y="0px"
+   viewBox="0 0 661.20001 665.40002"
+   xml:space="preserve"
+   width="661.20001"
+   height="665.40002"
+   inkscape:version="1.1.2 (0a00cf5339, 2022-02-04)"
+   sodipodi:docname="schedule.svg"
+   xmlns:inkscape="http://www.inkscape.org/namespaces/inkscape"
+   xmlns:sodipodi="http://sodipodi.sourceforge.net/DTD/sodipodi-0.dtd"
+   xmlns="http://www.w3.org/2000/svg"
+   xmlns:svg="http://www.w3.org/2000/svg"><defs
+   id="defs77" /><sodipodi:namedview
+   id="namedview75"
+   pagecolor="#ffffff"
+   bordercolor="#666666"
+   borderopacity="1.0"
+   inkscape:pageshadow="2"
+   inkscape:pageopacity="0.0"
+   inkscape:pagecheckerboard="0"
+   showgrid="false"
+   fit-margin-top="0"
+   fit-margin-left="0"
+   fit-margin-right="0"
+   fit-margin-bottom="0"
+   inkscape:zoom="1.2458671"
+   inkscape:cx="300.59386"
+   inkscape:cy="332.29869"
+   inkscape:window-width="1920"
+   inkscape:window-height="1051"
+   inkscape:window-x="1920"
+   inkscape:window-y="0"
+   inkscape:window-maximized="1"
+   inkscape:current-layer="g72" /> <style
+   type="text/css"
+   id="style2"> .st0{fill:#FFFFFF;} .st1{fill:#C1272D;} .st2{fill:#991D26;} .st3{fill:#CCCCCC;} .st4{fill:#E6E6E6;} .st5{fill:#F7931E;} .st6{fill:#F2F2F2;} .st7{fill:none;stroke:#999999;stroke-width:17.9763;stroke-linecap:round;stroke-miterlimit:10;} .st8{fill:none;stroke:#333333;stroke-width:8.9882;stroke-linecap:round;stroke-miterlimit:10;} .st9{fill:none;stroke:#C1272D;stroke-width:5.9921;stroke-linecap:round;stroke-miterlimit:10;} .st10{fill:#245F7F;} </style> <g
+   id="g72"
+   transform="translate(-149.4,-147.3)"> <path
+   class="st0"
+   d="M 601.2,699.8 H 205 c -30.7,0 -55.6,-24.9 -55.6,-55.6 V 248 c 0,-30.7 24.9,-55.6 55.6,-55.6 h 396.2 c 30.7,0 55.6,24.9 55.6,55.6 v 396.2 c 0,30.7 -24.9,55.6 -55.6,55.6 z"
+   id="path4"
+   style="fill:#dfdfdf;fill-opacity:1" /> <path
+   class="st1"
+   d="M 601.2,192.4 H 205 c -30.7,0 -55.6,24.9 -55.6,55.6 v 88.5 H 656.8 V 248 c 0,-30.7 -24.9,-55.6 -55.6,-55.6 z"
+   id="path6"
+   style="fill:#d62128;fill-opacity:1" /> <circle
+   class="st2"
+   cx="253.3"
+   cy="264.5"
+   r="36.700001"
+   id="circle8" /> <circle
+   class="st2"
+   cx="551.59998"
+   cy="264.5"
+   r="36.700001"
+   id="circle10" /> <path
+   class="st3"
+   d="m 253.3,275.7 v 0 c -11.8,0 -21.3,-9.6 -21.3,-21.3 v -85.8 c 0,-11.8 9.6,-21.3 21.3,-21.3 v 0 c 11.8,0 21.3,9.6 21.3,21.3 v 85.8 c 0,11.8 -9.5,21.3 -21.3,21.3 z"
+   id="path12" /> <path
+   class="st3"
+   d="m 551.6,275.7 v 0 c -11.8,0 -21.3,-9.6 -21.3,-21.3 v -85.8 c 0,-11.8 9.6,-21.3 21.3,-21.3 v 0 c 11.8,0 21.3,9.6 21.3,21.3 v 85.8 c 0.1,11.8 -9.5,21.3 -21.3,21.3 z"
+   id="path14" /> <rect
+   x="215.7"
+   y="370.89999"
+   class="st4"
+   width="75.199997"
+   height="75.199997"
+   id="rect16"
+   style="fill:#ffffff;fill-opacity:1" /> <rect
+   x="313"
+   y="370.89999"
+   class="st4"
+   width="75.199997"
+   height="75.199997"
+   id="rect18"
+   style="fill:#ffffff;fill-opacity:1" /> <rect
+   x="410.20001"
+   y="370.89999"
+   class="st4"
+   width="75.199997"
+   height="75.199997"
+   id="rect20"
+   style="fill:#ffffff;fill-opacity:1" /> <rect
+   x="507.5"
+   y="370.89999"
+   class="st4"
+   width="75.199997"
+   height="75.199997"
+   id="rect22"
+   style="fill:#ffffff;fill-opacity:1" /> <rect
+   x="215.7"
+   y="465"
+   class="st4"
+   width="75.199997"
+   height="75.199997"
+   id="rect24"
+   style="fill:#ffffff;fill-opacity:1" /> <rect
+   x="313"
+   y="465"
+   class="st1"
+   width="75.199997"
+   height="75.199997"
+   id="rect26"
+   style="fill:#27c12b;fill-opacity:1" /> <rect
+   x="410.20001"
+   y="465"
+   class="st4"
+   width="75.199997"
+   height="75.199997"
+   id="rect28"
+   style="fill:#ffffff;fill-opacity:1" /> <rect
+   x="507.5"
+   y="465"
+   class="st4"
+   width="75.199997"
+   height="75.199997"
+   id="rect30" /> <rect
+   x="215.7"
+   y="559.09998"
+   class="st4"
+   width="75.199997"
+   height="75.199997"
+   id="rect32"
+   style="fill:#ffffff;fill-opacity:1" /> <rect
+   x="313"
+   y="559.09998"
+   class="st4"
+   width="75.199997"
+   height="75.199997"
+   id="rect34"
+   style="fill:#ffffff;fill-opacity:1" /> <rect
+   x="410.20001"
+   y="559.09998"
+   class="st4"
+   width="75.199997"
+   height="75.199997"
+   id="rect36"
+   style="fill:#ffffff;fill-opacity:1" /> <rect
+   x="507.5"
+   y="559.09998"
+   class="st4"
+   width="75.199997"
+   height="75.199997"
+   id="rect38" /> <g
+   id="g70"> <circle
+   class="st5"
+   cx="621.90002"
+   cy="624"
+   r="188.7"
+   id="circle40" /> <circle
+   class="st0"
+   cx="621.90002"
+   cy="624"
+   r="148"
+   id="circle42" /> <path
+   class="st6"
+   d="m 486.6,636.8 c 0,-81.7 66.3,-148 148,-148 37.6,0 72,14.1 98.1,37.2 -27.1,-30.6 -66.7,-49.9 -110.8,-49.9 -81.7,0 -148,66.3 -148,148 0,44.1 19.3,83.7 49.9,110.8 -23.1,-26.2 -37.2,-60.5 -37.2,-98.1 z"
+   id="path44" /> <polyline
+   class="st7"
+   points="621.9,530.4 621.9,624 559,624  "
+   id="polyline46" /> <g
+   id="g64"> <line
+   class="st8"
+   x1="621.90002"
+   y1="508.29999"
+   x2="621.90002"
+   y2="497.10001"
+   id="line48" /> <line
+   class="st8"
+   x1="621.90002"
+   y1="756.29999"
+   x2="621.90002"
+   y2="745.09998"
+   id="line50" /> <line
+   class="st8"
+   x1="740.29999"
+   y1="626.70001"
+   x2="751.5"
+   y2="626.70001"
+   id="line52" /> <line
+   class="st8"
+   x1="492.29999"
+   y1="626.70001"
+   x2="503.5"
+   y2="626.70001"
+   id="line54" /> <line
+   class="st8"
+   x1="705.59998"
+   y1="710.40002"
+   x2="713.5"
+   y2="718.29999"
+   id="line56" /> <line
+   class="st8"
+   x1="530.29999"
+   y1="535.09998"
+   x2="538.20001"
+   y2="543"
+   id="line58" /> <line
+   class="st8"
+   x1="538.20001"
+   y1="710.40002"
+   x2="530.29999"
+   y2="718.29999"
+   id="line60" /> <line
+   class="st8"
+   x1="713.5"
+   y1="535.09998"
+   x2="705.59998"
+   y2="543"
+   id="line62" /> </g> <line
+   class="st9"
+   x1="604.40002"
+   y1="606.29999"
+   x2="684.5"
+   y2="687.40002"
+   id="line66" /> <circle
+   class="st10"
+   cx="621.90002"
+   cy="624"
+   r="16.1"
+   id="circle68" /> </g> </g> </svg>

changedetectionio/static/js/browser-steps.js

@@ -221,7 +221,7 @@ $(document).ready(function () {
                     // If you switch to "Click X,y" after an element here is setup, it will give the last co-ords anyway
                     //if (x['isClickable'] || x['tagName'].startsWith('h') || x['tagName'] === 'a' || x['tagName'] === 'button' || x['tagtype'] === 'submit' || x['tagtype'] === 'checkbox' || x['tagtype'] === 'radio' || x['tagtype'] === 'li') {
                         $('select', first_available).val('Click element').change();
-                        $('input[type=text]', first_available).first().val(x['xpath']);
+                        $('input[type=text]', first_available).first().val(x['xpath']).focus();
                         found_something = true;
                     //}
                 }
@@ -305,7 +305,7 @@ $(document).ready(function () {
 
         if ($(this).val() === 'Click X,Y' && last_click_xy['x'] > 0 && $(elem_value).val().length === 0) {
             // @todo handle scale
-            $(elem_value).val(last_click_xy['x'] + ',' + last_click_xy['y']);
+            $(elem_value).val(last_click_xy['x'] + ',' + last_click_xy['y']).focus();
         }
     }).change();
 

changedetectionio/static/js/conditions.js

@@ -0,0 +1,150 @@
+$(document).ready(function () {
+    // Function to set up button event handlers
+    function setupButtonHandlers() {
+        // Unbind existing handlers first to prevent duplicates
+        $(".addRuleRow, .removeRuleRow, .verifyRuleRow").off("click");
+        
+        // Add row button handler
+        $(".addRuleRow").on("click", function(e) {
+            e.preventDefault();
+            
+            let currentRow = $(this).closest("tr");
+            
+            // Clone without events
+            let newRow = currentRow.clone(false);
+            
+            // Reset input values in the cloned row
+            newRow.find("input").val("");
+            newRow.find("select").prop("selectedIndex", 0);
+            
+            // Insert the new row after the current one
+            currentRow.after(newRow);
+            
+            // Reindex all rows
+            reindexRules();
+        });
+        
+        // Remove row button handler
+        $(".removeRuleRow").on("click", function(e) {
+            e.preventDefault();
+            
+            // Only remove if there's more than one row
+            if ($("#rulesTable tbody tr").length > 1) {
+                $(this).closest("tr").remove();
+                reindexRules();
+            }
+        });
+        
+        // Verify rule button handler
+        $(".verifyRuleRow").on("click", function(e) {
+            e.preventDefault();
+            
+            let row = $(this).closest("tr");
+            let field = row.find("select[name$='field']").val();
+            let operator = row.find("select[name$='operator']").val();
+            let value = row.find("input[name$='value']").val();
+            
+            // Validate that all fields are filled
+            if (!field || field === "None" || !operator || operator === "None" || !value) {
+                alert("Please fill in all fields (Field, Operator, and Value) before verifying.");
+                return;
+            }
+
+            
+            // Create a rule object
+            const rule = {
+                field: field,
+                operator: operator,
+                value: value
+            };
+            
+            // Show a spinner or some indication that verification is in progress
+            const $button = $(this);
+            const originalHTML = $button.html();
+            $button.html("⌛").prop("disabled", true);
+            
+            // Collect form data - similar to request_textpreview_update() in watch-settings.js
+            let formData = new FormData();
+            $('#edit-text-filter textarea, #edit-text-filter input').each(function() {
+                const $element = $(this);
+                const name = $element.attr('name');
+                if (name) {
+                    if ($element.is(':checkbox')) {
+                        formData.append(name, $element.is(':checked') ? $element.val() : false);
+                    } else {
+                        formData.append(name, $element.val());
+                    }
+                }
+            });
+            
+            // Also collect select values
+            $('#edit-text-filter select').each(function() {
+                const $element = $(this);
+                const name = $element.attr('name');
+                if (name) {
+                    formData.append(name, $element.val());
+                }
+            });
+
+
+            // Send the request to verify the rule
+            $.ajax({
+                url: verify_condition_rule_url+"?"+ new URLSearchParams({ rule: JSON.stringify(rule) }).toString(),
+                type: "POST",
+                data: formData,
+                processData: false, // Prevent jQuery from converting FormData to a string
+                contentType: false, // Let the browser set the correct content type
+                success: function (response) {
+                    if (response.status === "success") {
+                        if (response.result) {
+                            alert("✅ Condition PASSES verification against current snapshot!");
+                        } else {
+                            alert("❌ Condition FAILS verification against current snapshot.");
+                        }
+                    } else {
+                        alert("Error: " + response.message);
+                    }
+                    $button.html(originalHTML).prop("disabled", false);
+                },
+                error: function (xhr) {
+                    let errorMsg = "Error verifying condition.";
+                    if (xhr.responseJSON && xhr.responseJSON.message) {
+                        errorMsg = xhr.responseJSON.message;
+                    }
+                    alert(errorMsg);
+                    $button.html(originalHTML).prop("disabled", false);
+                }
+            });
+        });
+    }
+
+    // Function to reindex form elements and re-setup event handlers
+    function reindexRules() {
+        // Unbind all button handlers first
+        $(".addRuleRow, .removeRuleRow, .verifyRuleRow").off("click");
+        
+        // Reindex all form elements
+        $("#rulesTable tbody tr").each(function(index) {
+            $(this).find("select, input").each(function() {
+                let oldName = $(this).attr("name");
+                let oldId = $(this).attr("id");
+
+                if (oldName) {
+                    let newName = oldName.replace(/\d+/, index);
+                    $(this).attr("name", newName);
+                }
+
+                if (oldId) {
+                    let newId = oldId.replace(/\d+/, index);
+                    $(this).attr("id", newId);
+                }
+            });
+        });
+        
+        // Reattach event handlers after reindexing
+        setupButtonHandlers();
+    }
+
+    // Initial setup of button handlers
+    setupButtonHandlers();
+});

changedetectionio/static/js/global-settings.js

@@ -24,5 +24,19 @@ $(document).ready(function () {
         $(target).toggle();
     });
 
+    // Time zone config related
+    $(".local-time").each(function (e) {
+        $(this).text(new Date($(this).data("utc")).toLocaleString());
+    })
+
+    const timezoneInput = $('#application-timezone');
+    if(timezoneInput.length) {
+        const timezone = Intl.DateTimeFormat().resolvedOptions().timeZone;
+        if (!timezoneInput.val().trim()) {
+            timezoneInput.val(timezone);
+            timezoneInput.after('<div class="timezone-message">The timezone was set from your browser, <strong>be sure to press save!</strong></div>');
+        }
+    }
+
 });
 

changedetectionio/static/js/notifications.js

@@ -1,42 +1,52 @@
-$(document).ready(function() {
+$(document).ready(function () {
 
-  $('#add-email-helper').click(function (e) {
-    e.preventDefault();
-    email = prompt("Destination email");
-    if(email) {
-      var n = $(".notification-urls");
-      var p=email_notification_prefix;
-      $(n).val( $.trim( $(n).val() )+"\n"+email_notification_prefix+email );
-    }
-  });
-
-  $('#send-test-notification').click(function (e) {
-    e.preventDefault();
-
-    data = {
-      notification_body: $('#notification_body').val(),
-      notification_format: $('#notification_format').val(),
-      notification_title: $('#notification_title').val(),
-      notification_urls: $('.notification-urls').val(),
-      tags: $('#tags').val(),
-      window_url: window.location.href,
-    }
-
-
-    $.ajax({
-      type: "POST",
-      url: notification_base_url,
-      data : data,
-        statusCode: {
-        400: function(data) {
-          // More than likely the CSRF token was lost when the server restarted
-          alert(data.responseText);
+    $('#add-email-helper').click(function (e) {
+        e.preventDefault();
+        email = prompt("Destination email");
+        if (email) {
+            var n = $(".notification-urls");
+            var p = email_notification_prefix;
+            $(n).val($.trim($(n).val()) + "\n" + email_notification_prefix + email);
         }
-      }
-    }).done(function(data){
-      console.log(data);
-      alert(data);
-    })
-  });
+    });
+
+    $('#send-test-notification').click(function (e) {
+        e.preventDefault();
+
+        data = {
+            notification_body: $('#notification_body').val(),
+            notification_format: $('#notification_format').val(),
+            notification_title: $('#notification_title').val(),
+            notification_urls: $('.notification-urls').val(),
+            tags: $('#tags').val(),
+            window_url: window.location.href,
+        }
+
+        $('.notifications-wrapper .spinner').fadeIn();
+        $('#notification-test-log').show();
+        $.ajax({
+            type: "POST",
+            url: notification_base_url,
+            data: data,
+            statusCode: {
+                400: function (data) {
+                    $("#notification-test-log>span").text(data.responseText);
+                },
+            }
+        }).done(function (data) {
+            $("#notification-test-log>span").text(data);
+        }).fail(function (jqXHR, textStatus, errorThrown) {
+            // Handle connection refused or other errors
+            if (textStatus === "error" && errorThrown === "") {
+                console.error("Connection refused or server unreachable");
+                $("#notification-test-log>span").text("Error: Connection refused or server is unreachable.");
+            } else {
+                console.error("Error:", textStatus, errorThrown);
+                $("#notification-test-log>span").text("An error occurred: " + textStatus);
+            }
+        }).always(function () {
+            $('.notifications-wrapper .spinner').hide();
+        })
+    });
 });
 

changedetectionio/static/js/plugins.js

@@ -159,4 +159,38 @@
         // Return the current request in case it's needed
         return requests[namespace];
     };
-})(jQuery);
+})(jQuery);
+
+
+
+function toggleOpacity(checkboxSelector, fieldSelector, inverted) {
+    const checkbox = document.querySelector(checkboxSelector);
+    const fields = document.querySelectorAll(fieldSelector);
+
+    function updateOpacity() {
+        const opacityValue = !checkbox.checked ? (inverted ? 0.6 : 1) : (inverted ? 1 : 0.6);
+        fields.forEach(field => {
+            field.style.opacity = opacityValue;
+        });
+    }
+
+    // Initial setup
+    updateOpacity();
+    checkbox.addEventListener('change', updateOpacity);
+}
+
+function toggleVisibility(checkboxSelector, fieldSelector, inverted) {
+    const checkbox = document.querySelector(checkboxSelector);
+    const fields = document.querySelectorAll(fieldSelector);
+
+    function updateOpacity() {
+        const opacityValue = !checkbox.checked ? (inverted ? 'none' : 'block') : (inverted ? 'block' : 'none');
+        fields.forEach(field => {
+            field.style.display = opacityValue;
+        });
+    }
+
+    // Initial setup
+    updateOpacity();
+    checkbox.addEventListener('change', updateOpacity);
+}

changedetectionio/static/js/scheduler.js

@@ -0,0 +1,109 @@
+function getTimeInTimezone(timezone) {
+    const now = new Date();
+    const options = {
+        timeZone: timezone,
+        weekday: 'long',
+        year: 'numeric',
+        hour12: false,
+        month: '2-digit',
+        day: '2-digit',
+        hour: '2-digit',
+        minute: '2-digit',
+        second: '2-digit',
+    };
+
+    const formatter = new Intl.DateTimeFormat('en-US', options);
+    return formatter.format(now);
+}
+
+$(document).ready(function () {
+
+    let exceedsLimit = false;
+    const warning_text = $("#timespan-warning")
+    const timezone_text_widget = $("input[id*='time_schedule_limit-timezone']")
+
+    toggleVisibility('#time_schedule_limit-enabled, #requests-time_schedule_limit-enabled', '#schedule-day-limits-wrapper', true)
+
+    setInterval(() => {
+        let success = true;
+        try {
+            // Show the current local time according to either placeholder or entered TZ name
+            if (timezone_text_widget.val().length) {
+                $('#local-time-in-tz').text(getTimeInTimezone(timezone_text_widget.val()));
+            } else {
+                // So maybe use what is in the placeholder (which will be the default settings)
+                $('#local-time-in-tz').text(getTimeInTimezone(timezone_text_widget.attr('placeholder')));
+            }
+        } catch (error) {
+            success = false;
+            $('#local-time-in-tz').text("");
+            console.error(timezone_text_widget.val())
+        }
+
+        $(timezone_text_widget).toggleClass('error', !success);
+
+    }, 500);
+
+    $('#schedule-day-limits-wrapper').on('change click blur', 'input, checkbox, select', function() {
+
+        let allOk = true;
+
+        // Controls setting the warning that the time could overlap into the next day
+        $("li.day-schedule").each(function () {
+            const $schedule = $(this);
+            const $checkbox = $schedule.find("input[type='checkbox']");
+
+            if ($checkbox.is(":checked")) {
+                const timeValue = $schedule.find("input[type='time']").val();
+                const durationHours = parseInt($schedule.find("select[name*='-duration-hours']").val(), 10) || 0;
+                const durationMinutes = parseInt($schedule.find("select[name*='-duration-minutes']").val(), 10) || 0;
+
+                if (timeValue) {
+                    const [startHours, startMinutes] = timeValue.split(":").map(Number);
+                    const totalMinutes = (startHours * 60 + startMinutes) + (durationHours * 60 + durationMinutes);
+
+                    exceedsLimit = totalMinutes > 1440
+                    if (exceedsLimit) {
+                        allOk = false
+                    }
+                    // Set the row/day-of-week highlight
+                    $schedule.toggleClass("warning", exceedsLimit);
+                }
+            } else {
+                $schedule.toggleClass("warning", false);
+            }
+        });
+
+        warning_text.toggle(!allOk)
+    });
+
+    $('table[id*="time_schedule_limit-saturday"], table[id*="time_schedule_limit-sunday"]').addClass("weekend-day")
+
+    // Presets [weekend] [business hours] etc
+    $(document).on('click', '[data-template].set-schedule', function () {
+        // Get the value of the 'data-template' attribute
+        switch ($(this).attr('data-template')) {
+            case 'business-hours':
+                $('.day-schedule table:not(.weekend-day) input[type="time"]').val('09:00')
+                $('.day-schedule table:not(.weekend-day) select[id*="-duration-hours"]').val('8');
+                $('.day-schedule table:not(.weekend-day) select[id*="-duration-minutes"]').val('0');
+                $('.day-schedule input[id*="-enabled"]').prop('checked', true);
+                $('.day-schedule .weekend-day input[id*="-enabled"]').prop('checked', false);
+                break;
+            case 'weekend':
+                $('.day-schedule .weekend-day input[type="time"][id$="start-time"]').val('00:00')
+                $('.day-schedule .weekend-day select[id*="-duration-hours"]').val('24');
+                $('.day-schedule .weekend-day select[id*="-duration-minutes"]').val('0');
+                $('.day-schedule input[id*="-enabled"]').prop('checked', false);
+                $('.day-schedule .weekend-day input[id*="-enabled"]').prop('checked', true);
+                break;
+            case 'reset':
+
+                $('.day-schedule input[type="time"]').val('00:00')
+                $('.day-schedule select[id*="-duration-hours"]').val('24');
+                $('.day-schedule select[id*="-duration-minutes"]').val('0');
+                $('.day-schedule input[id*="-enabled"]').prop('checked', true);
+                break;
+        }
+    });
+});

changedetectionio/static/js/tabs.js

@@ -26,7 +26,6 @@ function set_active_tab() {
     if (tab.length) {
         tab[0].parentElement.className = "active";
     }
-
 }
 
 function focus_error_tab() {

changedetectionio/static/js/visual-selector.js

@@ -132,6 +132,7 @@ $(document).ready(() => {
         }).done((data) => {
             $fetchingUpdateNoticeElem.html("Rendering..");
             selectorData = data;
+
             sortScrapedElementsBySize();
             console.log(`Reported browser width from backend: ${data['browser_width']}`);
 

changedetectionio/static/js/watch-settings.js

@@ -1,17 +1,3 @@
-function toggleOpacity(checkboxSelector, fieldSelector, inverted) {
-    const checkbox = document.querySelector(checkboxSelector);
-    const fields = document.querySelectorAll(fieldSelector);
-    function updateOpacity() {
-        const opacityValue = !checkbox.checked ? (inverted ? 0.6 : 1) : (inverted ? 1 : 0.6);
-        fields.forEach(field => {
-            field.style.opacity = opacityValue;
-        });
-    }
-    // Initial setup
-    updateOpacity();
-    checkbox.addEventListener('change', updateOpacity);
-}
-
 
 function request_textpreview_update() {
     if (!$('body').hasClass('preview-text-enabled')) {
@@ -57,7 +43,9 @@ function request_textpreview_update() {
     })
 }
 
+
 $(document).ready(function () {
+
     $('#notification-setting-reset-to-default').click(function (e) {
         $('#notification_title').val('');
         $('#notification_body').val('');
@@ -70,11 +58,12 @@ $(document).ready(function () {
         $('#notification-tokens-info').toggle();
     });
 
-    toggleOpacity('#time_between_check_use_default', '#time_between_check', false);
+    toggleOpacity('#time_between_check_use_default', '#time-check-widget-wrapper, #time-between-check-schedule', false);
+
 
     const vh = Math.max(document.documentElement.clientHeight || 0, window.innerHeight || 0);
-    $("#text-preview-inner").css('max-height', (vh-300)+"px");
-    $("#text-preview-before-inner").css('max-height', (vh-300)+"px");
+    $("#text-preview-inner").css('max-height', (vh - 300) + "px");
+    $("#text-preview-before-inner").css('max-height', (vh - 300) + "px");
 
     $("#activate-text-preview").click(function (e) {
         $('body').toggleClass('preview-text-enabled')

changedetectionio/static/styles/scss/parts/_browser-steps.scss

@@ -40,19 +40,22 @@
   }
 }
 
+@media only screen and (min-width: 760px) {
 
-#browser-steps .flex-wrapper {
-  display: flex;
-  flex-flow: row;
-  height: 70vh;
-  font-size: 80%;
-  #browser-steps-ui {
-    flex-grow: 1;      /* Allow it to grow and fill the available space */
-    flex-shrink: 1;    /* Allow it to shrink if needed */
-    flex-basis: 0;     /* Start with 0 base width so it stretches as much as possible */
-    background-color: #eee;
-    border-radius: 5px;
+  #browser-steps .flex-wrapper {
+    display: flex;
+    flex-flow: row;
+    height: 70vh;
+    font-size: 80%;
 
+    #browser-steps-ui {
+      flex-grow: 1; /* Allow it to grow and fill the available space */
+      flex-shrink: 1; /* Allow it to shrink if needed */
+      flex-basis: 0; /* Start with 0 base width so it stretches as much as possible */
+      background-color: #eee;
+      border-radius: 5px;
+
+    }
   }
 
   #browser-steps-fieldlist {
@@ -63,15 +66,21 @@
     padding-left: 1rem;
     overflow-y: scroll;
   }
+
+  /*  this is duplicate :( */
+  #browsersteps-selector-wrapper {
+    height: 100% !important;
+  }
 }
 
 /*  this is duplicate :( */
 #browsersteps-selector-wrapper {
-  height: 100%;
+
   width: 100%;
   overflow-y: scroll;
   position: relative;
-  //width: 100%;
+  height: 80vh;
+
   > img {
     position: absolute;
     max-width: 100%;
@@ -91,7 +100,6 @@
     left: 50%;
     top: 50%;
     transform: translate(-50%, -50%);
-    margin-left: -40px;
     z-index: 100;
     max-width: 350px;
     text-align: center;

changedetectionio/static/styles/scss/parts/_edit.scss

@@ -0,0 +1,9 @@
+ul#conditions_match_logic {
+    list-style: none;
+  input, label, li {
+    display: inline-block;
+  }
+  li {
+    padding-right: 1em;
+  }
+}

changedetectionio/static/styles/scss/styles.scss

@@ -13,6 +13,7 @@
 @import "parts/_menu";
 @import "parts/_love";
 @import "parts/preview_text_filter";
+@import "parts/_edit";
 
 body {
   color: var(--color-text);
@@ -380,7 +381,15 @@ a.pure-button-selected {
 }
 
 .notifications-wrapper {
-  padding: 0.5rem 0 1rem 0;
+  padding-top: 0.5rem;
+  #notification-test-log {
+    padding-top: 1rem;
+    white-space: pre-wrap;
+    word-break: break-word;
+    overflow-wrap: break-word;
+    max-width: 100%;
+    box-sizing: border-box;
+  }
 }
 
 label {

changedetectionio/static/styles/styles.css

@@ -46,21 +46,22 @@
     #browser_steps li > label {
       display: none; }
 
-#browser-steps .flex-wrapper {
-  display: flex;
-  flex-flow: row;
-  height: 70vh;
-  font-size: 80%; }
-  #browser-steps .flex-wrapper #browser-steps-ui {
-    flex-grow: 1;
-    /* Allow it to grow and fill the available space */
-    flex-shrink: 1;
-    /* Allow it to shrink if needed */
-    flex-basis: 0;
-    /* Start with 0 base width so it stretches as much as possible */
-    background-color: #eee;
-    border-radius: 5px; }
-  #browser-steps .flex-wrapper #browser-steps-fieldlist {
+@media only screen and (min-width: 760px) {
+  #browser-steps .flex-wrapper {
+    display: flex;
+    flex-flow: row;
+    height: 70vh;
+    font-size: 80%; }
+    #browser-steps .flex-wrapper #browser-steps-ui {
+      flex-grow: 1;
+      /* Allow it to grow and fill the available space */
+      flex-shrink: 1;
+      /* Allow it to shrink if needed */
+      flex-basis: 0;
+      /* Start with 0 base width so it stretches as much as possible */
+      background-color: #eee;
+      border-radius: 5px; }
+  #browser-steps-fieldlist {
     flex-grow: 0;
     /* Don't allow it to grow */
     flex-shrink: 0;
@@ -71,13 +72,16 @@
     /* Set a max width to prevent overflow */
     padding-left: 1rem;
     overflow-y: scroll; }
+  /*  this is duplicate :( */
+  #browsersteps-selector-wrapper {
+    height: 100% !important; } }
 
 /*  this is duplicate :( */
 #browsersteps-selector-wrapper {
-  height: 100%;
   width: 100%;
   overflow-y: scroll;
   position: relative;
+  height: 80vh;
   /* nice tall skinny one */ }
   #browsersteps-selector-wrapper > img {
     position: absolute;
@@ -92,7 +96,6 @@
     left: 50%;
     top: 50%;
     transform: translate(-50%, -50%);
-    margin-left: -40px;
     z-index: 100;
     max-width: 350px;
     text-align: center; }
@@ -520,6 +523,13 @@ body.preview-text-enabled {
   z-index: 3;
   box-shadow: 1px 1px 4px var(--color-shadow-jump); }
 
+ul#conditions_match_logic {
+  list-style: none; }
+  ul#conditions_match_logic input, ul#conditions_match_logic label, ul#conditions_match_logic li {
+    display: inline-block; }
+  ul#conditions_match_logic li {
+    padding-right: 1em; }
+
 body {
   color: var(--color-text);
   background: var(--color-background-page);
@@ -780,7 +790,14 @@ a.pure-button-selected {
   cursor: pointer; }
 
 .notifications-wrapper {
-  padding: 0.5rem 0 1rem 0; }
+  padding-top: 0.5rem; }
+  .notifications-wrapper #notification-test-log {
+    padding-top: 1rem;
+    white-space: pre-wrap;
+    word-break: break-word;
+    overflow-wrap: break-word;
+    max-width: 100%;
+    box-sizing: border-box; }
 
 label:hover {
   cursor: pointer; }

changedetectionio/store.py

@@ -374,7 +374,7 @@ class ChangeDetectionStore:
     def visualselector_data_is_ready(self, watch_uuid):
         output_path = "{}/{}".format(self.datastore_path, watch_uuid)
         screenshot_filename = "{}/last-screenshot.png".format(output_path)
-        elements_index_filename = "{}/elements.json".format(output_path)
+        elements_index_filename = "{}/elements.deflate".format(output_path)
         if path.isfile(screenshot_filename) and  path.isfile(elements_index_filename) :
             return True
 
@@ -909,3 +909,18 @@ class ChangeDetectionStore:
             if self.data['watching'][uuid].get('in_stock_only'):
                 del (self.data['watching'][uuid]['in_stock_only'])
 
+    # Compress old elements.json to elements.deflate, saving disk, this compression is pretty fast.
+    def update_19(self):
+        import zlib
+
+        for uuid, watch in self.data['watching'].items():
+            json_path = os.path.join(self.datastore_path, uuid, "elements.json")
+            deflate_path = os.path.join(self.datastore_path, uuid, "elements.deflate")
+
+            if os.path.exists(json_path):
+                with open(json_path, "rb") as f_j:
+                    with open(deflate_path, "wb") as f_d:
+                        logger.debug(f"Compressing {str(json_path)} to {str(deflate_path)}..")
+                        f_d.write(zlib.compress(f_j.read()))
+                        os.unlink(json_path)
+

changedetectionio/templates/_common_fields.html

@@ -12,23 +12,25 @@
                             }}
                             <div class="pure-form-message-inline">
                                 <p>
-                                <strong>Tip:</strong> Use <a target=_new href="https://github.com/caronc/apprise">AppRise Notification URLs</a> for notification to just about any service! <i><a target=_new href="https://github.com/dgtlmoon/changedetection.io/wiki/Notification-configuration-notes">Please read the notification services wiki here for important configuration notes</a></i>.<br>
+                                <strong>Tip:</strong> Use <a target="newwindow" href="https://github.com/caronc/apprise">AppRise Notification URLs</a> for notification to just about any service! <i><a target="newwindow" href="https://github.com/dgtlmoon/changedetection.io/wiki/Notification-configuration-notes">Please read the notification services wiki here for important configuration notes</a></i>.<br>
 </p>
                                 <div data-target="#advanced-help-notifications" class="toggle-show pure-button button-tag button-xsmall">Show advanced help and tips</div>
                                 <ul style="display: none" id="advanced-help-notifications">
-                                <li><code><a target=_new href="https://github.com/caronc/apprise/wiki/Notify_discord">discord://</a></code> (or <code>https://discord.com/api/webhooks...</code>)) only supports a maximum <strong>2,000 characters</strong> of notification text, including the title.</li>
-                                <li><code><a target=_new href="https://github.com/caronc/apprise/wiki/Notify_telegram">tgram://</a></code> bots can't send messages to other bots, so you should specify chat ID of non-bot user.</li>
-                                <li><code><a target=_new href="https://github.com/caronc/apprise/wiki/Notify_telegram">tgram://</a></code> only supports very limited HTML and can fail when extra tags are sent, <a href="https://core.telegram.org/bots/api#html-style">read more here</a> (or use plaintext/markdown format)</li>
+                                <li><code><a target="newwindow" href="https://github.com/caronc/apprise/wiki/Notify_discord">discord://</a></code> (or <code>https://discord.com/api/webhooks...</code>)) only supports a maximum <strong>2,000 characters</strong> of notification text, including the title.</li>
+                                <li><code><a target="newwindow" href="https://github.com/caronc/apprise/wiki/Notify_telegram">tgram://</a></code> bots can't send messages to other bots, so you should specify chat ID of non-bot user.</li>
+                                <li><code><a target="newwindow" href="https://github.com/caronc/apprise/wiki/Notify_telegram">tgram://</a></code> only supports very limited HTML and can fail when extra tags are sent, <a href="https://core.telegram.org/bots/api#html-style">read more here</a> (or use plaintext/markdown format)</li>
                                 <li><code>gets://</code>, <code>posts://</code>, <code>puts://</code>, <code>deletes://</code> for direct API calls (or omit the "<code>s</code>" for non-SSL ie <code>get://</code>) <a href="https://github.com/dgtlmoon/changedetection.io/wiki/Notification-configuration-notes#postposts">more help here</a></li>
                                   <li>Accepts the <code>{{ '{{token}}' }}</code> placeholders listed below</li>
                               </ul>
                             </div>
                             <div class="notifications-wrapper">
-                              <a id="send-test-notification" class="pure-button button-secondary button-xsmall" >Send test notification</a>
+                              <a id="send-test-notification" class="pure-button button-secondary button-xsmall" >Send test notification</a> <div class="spinner"  style="display: none;"></div>
                             {% if emailprefix %}
                               <a id="add-email-helper" class="pure-button button-secondary button-xsmall" >Add email <img style="height: 1em; display: inline-block" src="{{url_for('static_content', group='images', filename='email.svg')}}" alt="Add an email address"> </a>
                             {% endif %}
                               <a href="{{url_for('notification_logs')}}" class="pure-button button-secondary button-xsmall" >Notification debug logs</a>
+                              <br>
+                                <div id="notification-test-log" style="display: none;"><span class="pure-form-message-inline">Processing..</span></div>
                             </div>
                         </div>
                         <div id="notification-customisation" class="pure-control-group">
@@ -38,7 +40,7 @@
                             </div>
                             <div class="pure-control-group">
                                 {{ render_field(form.notification_body , rows=5, class="notification-body", placeholder=settings_application['notification_body']) }}
-                                <span class="pure-form-message-inline">Body for all notifications &dash; You can use <a target="_new" href="https://jinja.palletsprojects.com/en/3.0.x/templates/">Jinja2</a> templating in the notification title, body and URL, and tokens from below.
+                                <span class="pure-form-message-inline">Body for all notifications &dash; You can use <a target="newwindow" href="https://jinja.palletsprojects.com/en/3.0.x/templates/">Jinja2</a> templating in the notification title, body and URL, and tokens from below.
                                 </span>
 
                             </div>
@@ -124,7 +126,7 @@
                                 <div class="pure-form-message-inline">
                                     <p>
 									Warning: Contents of <code>{{ '{{diff}}' }}</code>, <code>{{ '{{diff_removed}}' }}</code>, and <code>{{ '{{diff_added}}' }}</code> depend on how the difference algorithm perceives the change. <br>
-                                    For example, an addition or removal could be perceived as a change in some cases. <a target="_new" href="https://github.com/dgtlmoon/changedetection.io/wiki/Using-the-%7B%7Bdiff%7D%7D,-%7B%7Bdiff_added%7D%7D,-and-%7B%7Bdiff_removed%7D%7D-notification-tokens">More Here</a> <br>
+                                    For example, an addition or removal could be perceived as a change in some cases. <a target="newwindow" href="https://github.com/dgtlmoon/changedetection.io/wiki/Using-the-%7B%7Bdiff%7D%7D,-%7B%7Bdiff_added%7D%7D,-and-%7B%7Bdiff_removed%7D%7D-notification-tokens">More Here</a> <br>
                                     </p>
                                     <p>
                                         For JSON payloads, use <strong>|tojson</strong> without quotes for automatic escaping, for example - <code>{ "name": {{ '{{ watch_title|tojson }}' }} }</code>

changedetectionio/templates/_helpers.html

@@ -59,4 +59,149 @@
 
 {% macro render_button(field) %}
   {{ field(**kwargs)|safe }}
+{% endmacro %}
+
+{% macro render_fieldlist_of_formfields_as_table(fieldlist, table_id="rulesTable") %}
+  <table class="fieldlist_formfields pure-table" id="{{ table_id }}">
+    <thead>
+      <tr>
+        {% for subfield in fieldlist[0] %}
+          <th>{{ subfield.label }}</th>
+        {% endfor %}
+        <th>Actions</th>
+      </tr>
+    </thead>
+    <tbody>
+      {% for form_row in fieldlist %}
+        <tr {% if form_row.errors %} class="error-row" {% endif %}>
+          {% for subfield in form_row %}
+            <td>
+              {{ subfield()|safe }}
+              {% if subfield.errors %}
+                <ul class="errors">
+                  {% for error in subfield.errors %}
+                    <li class="error">{{ error }}</li>
+                  {% endfor %}
+                </ul>
+              {% endif %}
+            </td>
+          {% endfor %}
+          <td>
+            <button type="button" class="addRuleRow">+</button>
+            <button type="button" class="removeRuleRow">-</button>
+            <button type="button" class="verifyRuleRow" title="Verify this rule against current snapshot">✓</button>
+          </td>
+        </tr>
+      {% endfor %}
+    </tbody>
+  </table>
+{% endmacro %}
+
+
+{% macro playwright_warning() %}
+    <p><strong>Error - Playwright support for Chrome based fetching is not enabled.</strong> Alternatively try our <a href="https://changedetection.io">very affordable subscription based service which has all this setup for you</a>.</p>
+    <p>You may need to <a href="https://github.com/dgtlmoon/changedetection.io/blob/09ebc6ec6338545bdd694dc6eee57f2e9d2b8075/docker-compose.yml#L31">Enable playwright environment variable</a> and uncomment the <strong>sockpuppetbrowser</strong> in the <a href="https://github.com/dgtlmoon/changedetection.io/blob/master/docker-compose.yml">docker-compose.yml</a> file.</p>
+    <br>
+    <p>(Also Selenium/WebDriver can not extract full page screenshots reliably so Playwright is recommended here)</p>
+
+{% endmacro %}
+
+{% macro only_webdriver_type_watches_warning() %}
+    <p><strong>Sorry, this functionality only works with Playwright/Chrome enabled watches.<br>You need to <a href="#request">Set the fetch method to Playwright/Chrome mode and resave</a> and have the Playwright connection enabled.</strong></p><br>
+{% endmacro %}
+
+{% macro render_time_schedule_form(form, available_timezones, timezone_default_config) %}
+    <style>
+    .day-schedule *, .day-schedule select {
+        display: inline-block;
+    }
+
+    .day-schedule label[for*="time_schedule_limit-"][for$="-enabled"] {
+        min-width: 6rem;
+        font-weight: bold;
+    }
+    .day-schedule label {
+        font-weight: normal;
+    }
+
+    .day-schedule table label {
+        padding-left: 0.5rem;
+        padding-right: 0.5rem;
+    }
+    #timespan-warning, input[id*='time_schedule_limit-timezone'].error {
+        color: #ff0000;
+    }
+    .day-schedule.warning table {
+        background-color: #ffbbc2;
+    }
+    ul#day-wrapper {
+        list-style: none;
+    }
+    #timezone-info > * {
+        display: inline-block;
+    }
+
+    #scheduler-icon-label {
+        background-position: left center;
+        background-repeat: no-repeat;
+        background-size: contain;
+        display: inline-block;
+        vertical-align: middle;
+        padding-left: 50px;
+        background-image: url({{ url_for('static_content', group='images', filename='schedule.svg') }});
+    }
+    #timespan-warning {
+        display: none;
+    }
+    </style>
+    <br>
+
+    {% if timezone_default_config %}
+    <div>
+        <span id="scheduler-icon-label" style="">
+            {{ render_checkbox_field(form.time_schedule_limit.enabled) }}
+            <div class="pure-form-message-inline">
+                Set a hourly/week day schedule
+            </div>
+        </span>
+
+    </div>
+    <br>
+    <div id="schedule-day-limits-wrapper">
+        <label>Schedule time limits</label><a data-template="business-hours"
+                                              class="set-schedule pure-button button-secondary button-xsmall">Business
+        hours</a>
+        <a data-template="weekend" class="set-schedule pure-button button-secondary button-xsmall">Weekends</a>
+        <a data-template="reset" class="set-schedule pure-button button-xsmall">Reset</a><br>
+        <br>
+
+        <ul id="day-wrapper">
+            {% for day in ['monday', 'tuesday', 'wednesday', 'thursday', 'friday', 'saturday', 'sunday'] %}
+                <li class="day-schedule" id="schedule-{{ day }}">
+                    {{ render_nolabel_field(form.time_schedule_limit[day]) }}
+                </li>
+            {% endfor %}
+            <li id="timespan-warning">Warning, one or more of your 'days' has a duration that would extend into the next day.<br>
+            This could have unintended consequences.</li>
+            <li id="timezone-info">
+                {{ render_field(form.time_schedule_limit.timezone, placeholder=timezone_default_config) }} <span id="local-time-in-tz"></span>
+                <datalist id="timezones" style="display: none;">
+                    {% for timezone in available_timezones %}
+                        <option value="{{ timezone }}">{{ timezone }}</option>
+                    {% endfor %}
+                </datalist>
+            </li>
+        </ul>
+    <br>
+        <span class="pure-form-message-inline">
+         <a href="https://changedetection.io/tutorials">More help and examples about using the scheduler</a>
+        </span>
+    </div>
+    {% else %}
+        <span class="pure-form-message-inline">
+            Want to use a time schedule? <a href="{{url_for('settings_page')}}#timedate">First confirm/save your Time Zone Settings</a>
+        </span>
+        <br>
+    {% endif %}
+
 {% endmacro %}

changedetectionio/templates/base.html

@@ -159,7 +159,7 @@
                     <a id="chrome-extension-link"
                        title="Try our new Chrome Extension!"
                        href="https://chromewebstore.google.com/detail/changedetectionio-website/kefcfmgmlhmankjmnbijimhofdjekbop">
-                        <img src="{{url_for('static_content', group='images', filename='Google-Chrome-icon.png')}}">
+                        <img alt="Chrome store icon" src="{{url_for('static_content', group='images', filename='Google-Chrome-icon.png')}}">
                         Chrome Webstore
                     </a>
                 </p>

changedetectionio/templates/edit.html

@@ -1,10 +1,14 @@
 {% extends 'base.html' %}
 {% block content %}
-{% from '_helpers.html' import render_field, render_checkbox_field, render_button %}
+{% from '_helpers.html' import render_field, render_checkbox_field, render_button, render_time_schedule_form, playwright_warning, only_webdriver_type_watches_warning, render_fieldlist_of_formfields_as_table %}
 {% from '_common_fields.html' import render_common_settings_form %}
 <script src="{{url_for('static_content', group='js', filename='tabs.js')}}" defer></script>
 <script src="{{url_for('static_content', group='js', filename='vis.js')}}" defer></script>
 <script src="{{url_for('static_content', group='js', filename='global-settings.js')}}" defer></script>
+<script src="{{url_for('static_content', group='js', filename='scheduler.js')}}" defer></script>
+<script src="{{url_for('static_content', group='js', filename='conditions.js')}}" defer></script>
+
+
 <script>
     const browser_steps_available_screenshots=JSON.parse('{{ watch.get_browsersteps_available_screenshots|tojson }}');
     const browser_steps_config=JSON.parse('{{ browser_steps_config|tojson }}');
@@ -39,17 +43,17 @@
 
     <div class="tabs collapsable">
         <ul>
-            <li class="tab" id=""><a href="#general">General</a></li>
+            <li class="tab"><a href="#general">General</a></li>
             <li class="tab"><a href="#request">Request</a></li>
             {% if extra_tab_content %}
             <li class="tab"><a href="#extras_tab">{{ extra_tab_content }}</a></li>
             {% endif %}
-            {% if playwright_enabled %}
             <li class="tab"><a id="browsersteps-tab" href="#browser-steps">Browser Steps</a></li>
-            {% endif %}
+        <!-- should goto extra forms? -->
             {% if watch['processor'] == 'text_json_diff' %}
             <li class="tab"><a id="visualselector-tab" href="#visualselector">Visual Filter Selector</a></li>
             <li class="tab" id="filters-and-triggers-tab"><a href="#filters-and-triggers">Filters &amp; Triggers</a></li>
+            <li class="tab" id="conditions-tab"><a href="#conditions">Conditions</a></li>
             {% endif %}
             <li class="tab"><a href="#notifications">Notifications</a></li>
             <li class="tab"><a href="#stats">Stats</a></li>
@@ -58,7 +62,7 @@
 
     <div class="box-wrap inner">
         <form class="pure-form pure-form-stacked"
-              action="{{ url_for('edit_page', uuid=uuid, next = request.args.get('next'), unpause_on_save = request.args.get('unpause_on_save')) }}" method="POST">
+              action="{{ url_for('edit_page', uuid=uuid, next = request.args.get('next'), unpause_on_save = request.args.get('unpause_on_save'), tag = request.args.get('tag')) }}" method="POST">
              <input type="hidden" name="csrf_token" value="{{ csrf_token() }}">
 
             <div class="tab-pane-inner" id="general">
@@ -79,9 +83,24 @@
                         <span class="pure-form-message-inline">Organisational tag/group name used in the main listing page</span>
                     </div>
                     <div class="pure-control-group time-between-check border-fieldset">
-                        {{ render_field(form.time_between_check, class="time-check-widget") }}
+
                         {{ render_checkbox_field(form.time_between_check_use_default, class="use-default-timecheck") }}
-                    </div>
+                        <br>
+                        <div id="time-check-widget-wrapper">
+                            {{ render_field(form.time_between_check, class="time-check-widget") }}
+
+                            <span class="pure-form-message-inline">
+                             The interval/amount of time between each check.
+                            </span>
+                        </div>
+                        <div id="time-between-check-schedule">
+                            <!-- Start Time and End Time -->
+                            <div id="limit-between-time">
+                                {{ render_time_schedule_form(form, available_timezones, timezone_default_config) }}
+                            </div>
+                        </div>
+<br>
+              </div>
                     <div class="pure-control-group">
                         {{ render_checkbox_field(form.extract_title_as_title) }}
                     </div>
@@ -183,8 +202,9 @@ Math: {{ 1 + 1 }}") }}
                     </div>
             </fieldset>
             </div>
-            {% if playwright_enabled %}
+
             <div class="tab-pane-inner" id="browser-steps">
+            {% if playwright_enabled and watch_uses_webdriver %}
                 <img class="beta-logo" src="{{url_for('static_content', group='images', filename='beta-logo.png')}}" alt="New beta functionality">
                 <fieldset>
                     <div class="pure-control-group">
@@ -208,7 +228,7 @@ Math: {{ 1 + 1 }}") }}
                                     <span class="loader" >
                                         <span id="browsersteps-click-start">
                                             <h2 >Click here to Start</h2>
-                                            <svg style="height: 3.5rem;" version="1.1" viewBox="0 0 32 32"  xml:space="preserve" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink"><g id="Layer_1"/><g id="play_x5F_alt"><path d="M16,0C7.164,0,0,7.164,0,16s7.164,16,16,16s16-7.164,16-16S24.836,0,16,0z M10,24V8l16.008,8L10,24z" style="fill: var(--color-grey-400);"/></g></svg><br>
+                                            <svg style="height: 3.5rem;" version="1.1" viewBox="0 0 32 32"  xml:space="preserve" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink"><g id="start"/><g id="play_x5F_alt"><path d="M16,0C7.164,0,0,7.164,0,16s7.164,16,16,16s16-7.164,16-16S24.836,0,16,0z M10,24V8l16.008,8L10,24z" style="fill: var(--color-grey-400);"/></g></svg><br>
                                             Please allow 10-15 seconds for the browser to connect.<br>
                                         </span>
                                         <div class="spinner"  style="display: none;"></div>
@@ -218,21 +238,31 @@ Math: {{ 1 + 1 }}") }}
                                 </div>
                             </div>
                             <div id="browser-steps-fieldlist" >
-                                <span id="browser-seconds-remaining">Loading</span> <span style="font-size: 80%;"> (<a target=_new href="https://github.com/dgtlmoon/changedetection.io/pull/478/files#diff-1a79d924d1840c485238e66772391268a89c95b781d69091384cf1ea1ac146c9R4">?</a>) </span>
+                                <span id="browser-seconds-remaining">Loading</span> <span style="font-size: 80%;"> (<a target="newwindow" href="https://github.com/dgtlmoon/changedetection.io/pull/478/files#diff-1a79d924d1840c485238e66772391268a89c95b781d69091384cf1ea1ac146c9R4">?</a>) </span>
                                 {{ render_field(form.browser_steps) }}
                             </div>
                         </div>
                     </div>
                 </fieldset>
+                {% else %}
+                    <span class="pure-form-message-inline">
+                        {% if not watch_uses_webdriver %}
+                            {{ only_webdriver_type_watches_warning() }}
+                        {% endif %}
+                        {%  if not playwright_enabled %}
+                            {{ playwright_warning() }}
+                        {% endif %}
+                    </span>
+                {% endif %}
             </div>
-            {% endif %}
+
 
             <div class="tab-pane-inner" id="notifications">
                 <fieldset>
                     <div  class="pure-control-group inline-radio">
                       {{ render_checkbox_field(form.notification_muted) }}
                     </div>
-                    {% if is_html_webdriver %}
+                    {% if watch_uses_webdriver %}
                     <div class="pure-control-group inline-radio">
                       {{ render_checkbox_field(form.notification_screenshot) }}
                         <span class="pure-form-message-inline">
@@ -248,13 +278,39 @@ Math: {{ 1 + 1 }}") }}
                         </div>
                         {% endif %}
                         <a href="#notifications" id="notification-setting-reset-to-default" class="pure-button button-xsmall" style="right: 20px; top: 20px; position: absolute; background-color: #5f42dd; border-radius: 4px; font-size: 70%; color: #fff">Use system defaults</a>
-
                         {{ render_common_settings_form(form, emailprefix, settings_application, extra_notification_token_placeholder_info) }}
                     </div>
                 </fieldset>
             </div>
 
             {% if watch['processor'] == 'text_json_diff' %}
+
+            <div class="tab-pane-inner" id="conditions">
+                    <script>
+                        const verify_condition_rule_url="{{url_for('conditions.verify_condition_single_rule', watch_uuid=uuid)}}";
+                    </script>
+                <style>
+                    .verifyRuleRow {
+                        background-color: #4caf50;
+                        color: white;
+                        border: none;
+                        cursor: pointer;
+                        font-weight: bold;
+                    }
+                    .verifyRuleRow:hover {
+                        background-color: #45a049;
+                    }
+                </style>
+                <div class="pure-control-group">
+                    {{ render_field(form.conditions_match_logic) }}
+                    {{ render_fieldlist_of_formfields_as_table(form.conditions) }}
+                    <div class="pure-form-message-inline">
+                        <br>
+                        Use the verify (✓) button to test if a condition passes against the current snapshot.<br><br>
+                        Did you know that <strong>conditions</strong> can be extended with your own custom plugin? tutorials coming soon!<br>
+                    </div>
+                </div>
+            </div>
             <div class="tab-pane-inner" id="filters-and-triggers">
                 <span id="activate-text-preview" class="pure-button pure-button-primary button-xsmall">Activate preview</span>
               <div>
@@ -282,7 +338,7 @@ xpath://body/div/span[contains(@class, 'example-class')]",
                           <span class="pure-form-message-inline"><strong>Note!: //text() function does not work where the &lt;element&gt; contains &lt;![CDATA[]]&gt;</strong></span><br>
                         {% endif %}
                         <span class="pure-form-message-inline">One CSS, xPath, JSON Path/JQ selector per line, <i>any</i> rules that matches will be used.<br>
-<p><div data-target="#advanced-help-selectors" class="toggle-show pure-button button-tag button-xsmall">Show advanced help and tips</div><br></p>
+                        <span data-target="#advanced-help-selectors" class="toggle-show pure-button button-tag button-xsmall">Show advanced help and tips</span><br>
                     <ul id="advanced-help-selectors" style="display: none;">
                         <li>CSS - Limit text to this CSS rule, only text matching this CSS rule is included.</li>
                         <li>JSON - Limit text to this JSON rule, using either <a href="https://pypi.org/project/jsonpath-ng/" target="new">JSONPath</a> or <a href="https://stedolan.github.io/jq/" target="new">jq</a> (if installed).
@@ -455,7 +511,7 @@ keyword") }}
 
                 <fieldset>
                     <div class="pure-control-group">
-                        {% if visualselector_enabled %}
+                        {% if playwright_enabled and watch_uses_webdriver %}
                             <span class="pure-form-message-inline" id="visual-selector-heading">
                                 The Visual Selector tool lets you select the <i>text</i> elements that will be used for the change detection. It automatically fills-in the filters in the "CSS/JSONPath/JQ/XPath Filters" box of the <a href="#filters-and-triggers">Filters & Triggers</a> tab. Use <strong>Shift+Click</strong> to select multiple items.
                             </span>
@@ -473,11 +529,12 @@ keyword") }}
                             </div>
                             <div id="selector-current-xpath" style="overflow-x: hidden"><strong>Currently:</strong>&nbsp;<span class="text">Loading...</span></div>
                         {% else %}
-                            <span class="pure-form-message-inline">
-                                <p>Sorry, this functionality only works with Playwright/Chrome enabled watches.</p>
-                                <p>Enable the Playwright Chrome fetcher, or alternatively try our <a href="https://lemonade.changedetection.io/start">very affordable subscription based service</a>.</p>
-                                <p>This is because Selenium/WebDriver can not extract full page screenshots reliably.</p>
-                            </span>
+                            {% if not watch_uses_webdriver %}
+                                {{ only_webdriver_type_watches_warning() }}
+                            {% endif %}
+                            {% if not playwright_enabled %}
+                                {{ playwright_warning() }}
+                            {% endif %}
                         {% endif %}
                     </div>
                 </fieldset>

changedetectionio/templates/settings.html

@@ -1,7 +1,7 @@
 {% extends 'base.html' %}
 
 {% block content %}
-{% from '_helpers.html' import render_field, render_checkbox_field, render_button %}
+{% from '_helpers.html' import render_field, render_checkbox_field, render_button, render_time_schedule_form %}
 {% from '_common_fields.html' import render_common_settings_form %}
 <script>
     const notification_base_url="{{url_for('ajax_callback_send_notification_test', mode="global-settings")}}";
@@ -10,9 +10,11 @@
 {% endif %}
 </script>
 <script src="{{url_for('static_content', group='js', filename='tabs.js')}}" defer></script>
+<script src="{{url_for('static_content', group='js', filename='plugins.js')}}" defer></script>
 <script src="{{url_for('static_content', group='js', filename='notifications.js')}}" defer></script>
 <script src="{{url_for('static_content', group='js', filename='vis.js')}}" defer></script>
 <script src="{{url_for('static_content', group='js', filename='global-settings.js')}}" defer></script>
+<script src="{{url_for('static_content', group='js', filename='scheduler.js')}}" defer></script>
 <div class="edit-form">
     <div class="tabs collapsable">
         <ul>
@@ -21,6 +23,7 @@
             <li class="tab"><a href="#fetching">Fetching</a></li>
             <li class="tab"><a href="#filters">Global Filters</a></li>
             <li class="tab"><a href="#api">API</a></li>
+            <li class="tab"><a href="#timedate">Time &amp Date</a></li>
             <li class="tab"><a href="#proxies">CAPTCHA &amp; Proxies</a></li>
         </ul>
     </div>
@@ -32,6 +35,12 @@
                     <div class="pure-control-group">
                         {{ render_field(form.requests.form.time_between_check, class="time-check-widget") }}
                         <span class="pure-form-message-inline">Default recheck time for all watches, current system minimum is <i>{{min_system_recheck_seconds}}</i> seconds (<a href="https://github.com/dgtlmoon/changedetection.io/wiki/Misc-system-settings#enviroment-variables">more info</a>).</span>
+                            <div id="time-between-check-schedule">
+                                <!-- Start Time and End Time -->
+                                <div id="limit-between-time">
+                                    {{ render_time_schedule_form(form.requests, available_timezones, timezone_default_config) }}
+                                </div>
+                        </div>
                     </div>
                     <div class="pure-control-group">
                         {{ render_field(form.requests.form.jitter_seconds, class="jitter_seconds") }}
@@ -205,12 +214,29 @@ nav
                         <a id="chrome-extension-link"
                            title="Try our new Chrome Extension!"
                            href="https://chromewebstore.google.com/detail/changedetectionio-website/kefcfmgmlhmankjmnbijimhofdjekbop">
-                            <img src="{{ url_for('static_content', group='images', filename='Google-Chrome-icon.png') }}" alt="Chrome">
+                            <img alt="Chrome store icon" src="{{ url_for('static_content', group='images', filename='Google-Chrome-icon.png') }}" alt="Chrome">
                             Chrome Webstore
                         </a>
                     </p>
                 </div>
             </div>
+            <div class="tab-pane-inner" id="timedate">
+                <div class="pure-control-group">
+                    Ensure the settings below are correct, they are used to manage the time schedule for checking your web page watches.
+                </div>
+                <div class="pure-control-group">
+                    <p><strong>UTC Time &amp Date from Server:</strong> <span id="utc-time" >{{ utc_time }}</span></p>
+                    <p><strong>Local Time &amp Date in Browser:</strong> <span class="local-time" data-utc="{{ utc_time }}"></span></p>
+                    <p>
+                       {{ render_field(form.application.form.timezone) }}
+                        <datalist id="timezones" style="display: none;">
+                            {% for tz_name in available_timezones %}
+                                <option value="{{ tz_name }}">{{ tz_name }}</option>
+                            {% endfor %}
+                        </datalist>
+                    </p>
+                </div>
+            </div>
             <div class="tab-pane-inner" id="proxies">
                 <div id="recommended-proxy">
                     <div>
@@ -254,9 +280,7 @@ nav
                         
                     </div>
                 </div>
-                <p>
-                    Your proxy provider may need to whitelist our IP of <code>204.15.192.195</code>
-                </p>
+
                <p><strong>Tip</strong>: "Residential" and "Mobile" proxy type can be more successfull than "Data Center" for blocked websites.
 
                 <div class="pure-control-group" id="extra-proxies-setting">

changedetectionio/templates/svgs/search-icon.svg

@@ -1 +1 @@
-<svg version="1.1" id="Layer_1" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" viewBox="0 0 122.879 119.799" enable-background="new 0 0 122.879 119.799" xml:space="preserve"><g><path d="M49.988,0h0.016v0.007C63.803,0.011,76.298,5.608,85.34,14.652c9.027,9.031,14.619,21.515,14.628,35.303h0.007v0.033v0.04 h-0.007c-0.005,5.557-0.917,10.905-2.594,15.892c-0.281,0.837-0.575,1.641-0.877,2.409v0.007c-1.446,3.66-3.315,7.12-5.547,10.307 l29.082,26.139l0.018,0.016l0.157,0.146l0.011,0.011c1.642,1.563,2.536,3.656,2.649,5.78c0.11,2.1-0.543,4.248-1.979,5.971 l-0.011,0.016l-0.175,0.203l-0.035,0.035l-0.146,0.16l-0.016,0.021c-1.565,1.642-3.654,2.534-5.78,2.646 c-2.097,0.111-4.247-0.54-5.971-1.978l-0.015-0.011l-0.204-0.175l-0.029-0.024L78.761,90.865c-0.88,0.62-1.778,1.209-2.687,1.765 c-1.233,0.755-2.51,1.466-3.813,2.115c-6.699,3.342-14.269,5.222-22.272,5.222v0.007h-0.016v-0.007 c-13.799-0.004-26.296-5.601-35.338-14.645C5.605,76.291,0.016,63.805,0.007,50.021H0v-0.033v-0.016h0.007 c0.004-13.799,5.601-26.296,14.645-35.338C23.683,5.608,36.167,0.016,49.955,0.007V0H49.988L49.988,0z M50.004,11.21v0.007h-0.016 h-0.033V11.21c-10.686,0.007-20.372,4.35-27.384,11.359C15.56,29.578,11.213,39.274,11.21,49.973h0.007v0.016v0.033H11.21 c0.007,10.686,4.347,20.367,11.359,27.381c7.009,7.012,16.705,11.359,27.403,11.361v-0.007h0.016h0.033v0.007 c10.686-0.007,20.368-4.348,27.382-11.359c7.011-7.009,11.358-16.702,11.36-27.4h-0.006v-0.016v-0.033h0.006 c-0.006-10.686-4.35-20.372-11.358-27.384C70.396,15.56,60.703,11.213,50.004,11.21L50.004,11.21z"/></g></svg>
+<svg version="1.1" id="search" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" viewBox="0 0 122.879 119.799" enable-background="new 0 0 122.879 119.799" xml:space="preserve"><g><path d="M49.988,0h0.016v0.007C63.803,0.011,76.298,5.608,85.34,14.652c9.027,9.031,14.619,21.515,14.628,35.303h0.007v0.033v0.04 h-0.007c-0.005,5.557-0.917,10.905-2.594,15.892c-0.281,0.837-0.575,1.641-0.877,2.409v0.007c-1.446,3.66-3.315,7.12-5.547,10.307 l29.082,26.139l0.018,0.016l0.157,0.146l0.011,0.011c1.642,1.563,2.536,3.656,2.649,5.78c0.11,2.1-0.543,4.248-1.979,5.971 l-0.011,0.016l-0.175,0.203l-0.035,0.035l-0.146,0.16l-0.016,0.021c-1.565,1.642-3.654,2.534-5.78,2.646 c-2.097,0.111-4.247-0.54-5.971-1.978l-0.015-0.011l-0.204-0.175l-0.029-0.024L78.761,90.865c-0.88,0.62-1.778,1.209-2.687,1.765 c-1.233,0.755-2.51,1.466-3.813,2.115c-6.699,3.342-14.269,5.222-22.272,5.222v0.007h-0.016v-0.007 c-13.799-0.004-26.296-5.601-35.338-14.645C5.605,76.291,0.016,63.805,0.007,50.021H0v-0.033v-0.016h0.007 c0.004-13.799,5.601-26.296,14.645-35.338C23.683,5.608,36.167,0.016,49.955,0.007V0H49.988L49.988,0z M50.004,11.21v0.007h-0.016 h-0.033V11.21c-10.686,0.007-20.372,4.35-27.384,11.359C15.56,29.578,11.213,39.274,11.21,49.973h0.007v0.016v0.033H11.21 c0.007,10.686,4.347,20.367,11.359,27.381c7.009,7.012,16.705,11.359,27.403,11.361v-0.007h0.016h0.033v0.007 c10.686-0.007,20.368-4.348,27.382-11.359c7.011-7.009,11.358-16.702,11.36-27.4h-0.006v-0.016v-0.033h0.006 c-0.006-10.686-4.35-20.372-11.358-27.384C70.396,15.56,60.703,11.213,50.004,11.21L50.004,11.21z"/></g></svg>

changedetectionio/templates/watch-overview.html

@@ -6,7 +6,7 @@
 
 <div class="box">
 
-    <form class="pure-form" action="{{ url_for('form_quick_watch_add') }}" method="POST" id="new-watch-form">
+    <form class="pure-form" action="{{ url_for('form_quick_watch_add', tag=active_tag_uuid) }}" method="POST" id="new-watch-form">
         <input type="hidden" name="csrf_token" value="{{ csrf_token() }}" >
         <fieldset>
             <legend>Add a new change detection watch</legend>
@@ -108,7 +108,8 @@
                     {% else %}
                     <a class="state-on" href="{{url_for('index', op='pause', uuid=watch.uuid, tag=active_tag_uuid)}}"><img src="{{url_for('static_content', group='images', filename='play.svg')}}" alt="UnPause checks" title="UnPause checks" class="icon icon-unpause" ></a>
                     {% endif %}
-                    <a class="link-mute state-{{'on' if watch.notification_muted else 'off'}}" href="{{url_for('index', op='mute', uuid=watch.uuid, tag=active_tag_uuid)}}"><img src="{{url_for('static_content', group='images', filename='bell-off.svg')}}" alt="Mute notifications" title="Mute notifications" class="icon icon-mute" ></a>
+                    {% set mute_label = 'UnMute notification' if watch.notification_muted else 'Mute notification' %}
+                    <a class="link-mute state-{{'on' if watch.notification_muted else 'off'}}" href="{{url_for('index', op='mute', uuid=watch.uuid, tag=active_tag_uuid)}}"><img src="{{url_for('static_content', group='images', filename='bell-off.svg')}}" alt="{{ mute_label }}" title="{{ mute_label }}" class="icon icon-mute" ></a>
                 </td>
                 <td class="title-col inline">{{watch.title if watch.title is not none and watch.title|length > 0 else watch.url}}
                     <a class="external" target="_blank" rel="noopener" href="{{ watch.link.replace('source:','') }}"></a>
@@ -118,7 +119,7 @@
                          or (  watch.get_fetch_backend == "system" and system_default_fetcher == 'html_webdriver'  )
                          or "extra_browser_" in watch.get_fetch_backend
                     %}
-                    <img class="status-icon" src="{{url_for('static_content', group='images', filename='Google-Chrome-icon.png')}}" title="Using a Chrome browser" >
+                    <img class="status-icon" src="{{url_for('static_content', group='images', filename='Google-Chrome-icon.png')}}" alt="Using a Chrome browser" title="Using a Chrome browser" >
                     {% endif %}
 
                     {%if watch.is_pdf  %}<img class="status-icon" src="{{url_for('static_content', group='images', filename='pdf-icon.svg')}}" title="Converting PDF to text" >{% endif %}
@@ -187,11 +188,11 @@
                 <td>
                     <a {% if watch.uuid in queued_uuids %}disabled="true"{% endif %} href="{{ url_for('form_watch_checknow', uuid=watch.uuid, tag=request.args.get('tag')) }}"
                        class="recheck pure-button pure-button-primary">{% if watch.uuid in queued_uuids %}Queued{% else %}Recheck{% endif %}</a>
-                    <a href="{{ url_for('edit_page', uuid=watch.uuid)}}#general" class="pure-button pure-button-primary">Edit</a>
+                    <a href="{{ url_for('edit_page', uuid=watch.uuid, tag=active_tag_uuid)}}#general" class="pure-button pure-button-primary">Edit</a>
                     {% if watch.history_n >= 2 %}
 
                         {%  if is_unviewed %}
-                           <a href="{{ url_for('diff_history_page', uuid=watch.uuid, from_version=watch.get_next_snapshot_key_to_last_viewed) }}" target="{{watch.uuid}}" class="pure-button pure-button-primary diff-link">History</a>
+                           <a href="{{ url_for('diff_history_page', uuid=watch.uuid, from_version=watch.get_from_version_based_on_last_viewed) }}" target="{{watch.uuid}}" class="pure-button pure-button-primary diff-link">History</a>
                         {% else %}
                            <a href="{{ url_for('diff_history_page', uuid=watch.uuid)}}" target="{{watch.uuid}}" class="pure-button pure-button-primary diff-link">History</a>
                         {% endif %}

changedetectionio/tests/fetchers/test_custom_js_before_content.py

@@ -34,7 +34,7 @@ def test_execute_custom_js(client, live_server, measure_memory_usage):
     assert b"unpaused" in res.data
     wait_for_all_checks(client)
 
-    uuid = extract_UUID_from_client(client)
+    uuid = next(iter(live_server.app.config['DATASTORE'].data['watching']))
     assert live_server.app.config['DATASTORE'].data['watching'][uuid].history_n >= 1, "Watch history had atleast 1 (everything fetched OK)"
 
     assert b"This text should be removed" not in res.data

changedetectionio/tests/proxy_list/test_noproxy.py

@@ -48,7 +48,7 @@ def test_noproxy_option(client, live_server, measure_memory_usage):
         follow_redirects=True
     )
     assert b"Watch added in Paused state, saving will unpause" in res.data
-    uuid = extract_UUID_from_client(client)
+    uuid = next(iter(live_server.app.config['DATASTORE'].data['watching']))
     res = client.get(
         url_for("edit_page", uuid=uuid, unpause_on_save=1))
     assert b'No proxy' in res.data

changedetectionio/tests/proxy_socks5/test_socks5_proxy.py

@@ -81,7 +81,7 @@ def test_socks5(client, live_server, measure_memory_usage):
     assert "Awesome, you made it".encode('utf-8') in res.data
 
     # PROXY CHECKER WIDGET CHECK - this needs more checking
-    uuid = extract_UUID_from_client(client)
+    uuid = next(iter(live_server.app.config['DATASTORE'].data['watching']))
 
     res = client.get(
         url_for("check_proxies.start_check", uuid=uuid),

changedetectionio/tests/test_add_replace_remove_filter.py

@@ -1,9 +1,9 @@
 #!/usr/bin/env python3
+
 import os.path
-import time
+
 from flask import url_for
 from .util import live_server_setup, wait_for_all_checks, wait_for_notification_endpoint_output
-from changedetectionio import html_tools
 
 
 def set_original(excluding=None, add_line=None):
@@ -113,7 +113,8 @@ def test_check_add_line_contains_trigger(client, live_server, measure_memory_usa
     res = client.post(
         url_for("settings_page"),
         data={"application-notification_title": "New ChangeDetection.io Notification - {{ watch_url }}",
-              "application-notification_body": 'triggered text was -{{triggered_text}}- 网站监测 内容更新了',
+              # triggered_text will contain multiple lines
+              "application-notification_body": 'triggered text was -{{triggered_text}}- ### 网站监测 内容更新了 ####',
               # https://github.com/caronc/apprise/wiki/Notify_Custom_JSON#get-parameter-manipulation
               "application-notification_urls": test_notification_url,
               "application-minutes_between_check": 180,
@@ -164,6 +165,7 @@ def test_check_add_line_contains_trigger(client, live_server, measure_memory_usa
     client.get(url_for("form_watch_checknow"), follow_redirects=True)
     wait_for_all_checks(client)
     res = client.get(url_for("index"))
+
     assert b'unviewed' in res.data
 
     # Takes a moment for apprise to fire
@@ -171,7 +173,7 @@ def test_check_add_line_contains_trigger(client, live_server, measure_memory_usa
     assert os.path.isfile("test-datastore/notification.txt"), "Notification fired because I can see the output file"
     with open("test-datastore/notification.txt", 'rb') as f:
         response = f.read()
-        assert b'-Oh yes please-' in response
+        assert b'-Oh yes please' in response
         assert '网站监测 内容更新了'.encode('utf-8') in response
 
     res = client.get(url_for("form_delete", uuid="all"), follow_redirects=True)

changedetectionio/tests/test_api.py

@@ -44,7 +44,6 @@ def set_modified_response():
 
     return None
 
-
 def is_valid_uuid(val):
     try:
         uuid.UUID(str(val))
@@ -56,8 +55,9 @@ def is_valid_uuid(val):
 def test_setup(client, live_server, measure_memory_usage):
     live_server_setup(live_server)
 
+
 def test_api_simple(client, live_server, measure_memory_usage):
-    #live_server_setup(live_server)
+#    live_server_setup(live_server)
 
     api_key = extract_api_key_from_UI(client)
 
@@ -129,6 +129,9 @@ def test_api_simple(client, live_server, measure_memory_usage):
     assert after_recheck_info['last_checked'] != before_recheck_info['last_checked']
     assert after_recheck_info['last_changed'] != 0
 
+    # #2877 When run in a slow fetcher like playwright etc
+    assert after_recheck_info['last_changed'] ==  after_recheck_info['last_checked']
+
     # Check history index list
     res = client.get(
         url_for("watchhistory", uuid=watch_uuid),

changedetectionio/tests/test_automatic_follow_ldjson_price.py

@@ -99,7 +99,7 @@ def test_check_ldjson_price_autodetect(client, live_server, measure_memory_usage
     assert b'ldjson-price-track-offer' in res.data
 
     # Accept it
-    uuid = extract_UUID_from_client(client)
+    uuid = next(iter(live_server.app.config['DATASTORE'].data['watching']))
     #time.sleep(1)
     client.get(url_for('price_data_follower.accept', uuid=uuid, follow_redirects=True))
     client.get(url_for("form_watch_checknow"), follow_redirects=True)

changedetectionio/tests/test_backend.py

@@ -2,7 +2,6 @@
 
 import time
 from flask import url_for
-from urllib.request import urlopen
 from .util import set_original_response, set_modified_response, live_server_setup, wait_for_all_checks, extract_rss_token_from_UI, \
     extract_UUID_from_client
 
@@ -69,7 +68,7 @@ def test_check_basic_change_detection_functionality(client, live_server, measure
 
     wait_for_all_checks(client)
 
-    uuid = extract_UUID_from_client(client)
+    uuid = next(iter(live_server.app.config['DATASTORE'].data['watching']))
 
     # Check the 'get latest snapshot works'
     res = client.get(url_for("watch_get_latest_html", uuid=uuid))

changedetectionio/tests/test_conditions.py

@@ -0,0 +1,133 @@
+#!/usr/bin/env python3
+
+from flask import url_for
+from .util import live_server_setup, wait_for_all_checks
+
+def set_original_response(number="50"):
+    test_return_data = f"""<html>
+       <body>
+     <h1>Test Page for Conditions</h1>
+     <p>This page contains a number that will be tested with conditions.</p>
+     <div class="number-container">Current value: {number}</div>
+     </body>
+     </html>
+    """
+
+    with open("test-datastore/endpoint-content.txt", "w") as f:
+        f.write(test_return_data)
+
+def set_number_in_range_response(number="75"):
+    test_return_data = f"""<html>
+       <body>
+     <h1>Test Page for Conditions</h1>
+     <p>This page contains a number that will be tested with conditions.</p>
+     <div class="number-container">Current value: {number}</div>
+     </body>
+     </html>
+    """
+
+    with open("test-datastore/endpoint-content.txt", "w") as f:
+        f.write(test_return_data)
+
+def set_number_out_of_range_response(number="150"):
+    test_return_data = f"""<html>
+       <body>
+     <h1>Test Page for Conditions</h1>
+     <p>This page contains a number that will be tested with conditions.</p>
+     <div class="number-container">Current value: {number}</div>
+     </body>
+     </html>
+    """
+
+    with open("test-datastore/endpoint-content.txt", "w") as f:
+        f.write(test_return_data)
+
+
+def test_conditions_with_text_and_number(client, live_server):
+    """Test that both text and number conditions work together with AND logic."""
+    
+    set_original_response("50")
+    live_server_setup(live_server)
+
+    test_url = url_for('test_endpoint', _external=True)
+
+    # Add our URL to the import page
+    res = client.post(
+        url_for("import_page"),
+        data={"urls": test_url},
+        follow_redirects=True
+    )
+    assert b"1 Imported" in res.data
+    wait_for_all_checks(client)
+
+    # Configure the watch with two conditions connected with AND:
+    # 1. The page filtered text must contain "5" (first digit of value)
+    # 2. The extracted number should be >= 20 and <= 100
+    res = client.post(
+        url_for("edit_page", uuid="first"),
+        data={
+            "url": test_url,
+            "fetch_backend": "html_requests",
+            "include_filters": ".number-container",
+            "title": "Number AND Text Condition Test",
+            "conditions_match_logic": "ALL",  # ALL = AND logic
+            "conditions-0-operator": "in",
+            "conditions-0-field": "page_filtered_text",
+            "conditions-0-value": "5",
+
+            "conditions-1-operator": ">=",
+            "conditions-1-field": "extracted_number",
+            "conditions-1-value": "20",
+
+            "conditions-2-operator": "<=",
+            "conditions-2-field": "extracted_number",
+            "conditions-2-value": "100",
+
+            # So that 'operations' from pluggy discovery are tested
+            "conditions-3-operator": "length_min",
+            "conditions-3-field": "page_filtered_text",
+            "conditions-3-value": "1",
+
+            # So that 'operations' from pluggy discovery are tested
+            "conditions-4-operator": "length_max",
+            "conditions-4-field": "page_filtered_text",
+            "conditions-4-value": "100",
+
+            # So that 'operations' from pluggy discovery are tested
+            "conditions-5-operator": "contains_regex",
+            "conditions-5-field": "page_filtered_text",
+            "conditions-5-value": "\d",
+        },
+        follow_redirects=True
+    )
+    assert b"Updated watch." in res.data
+
+    wait_for_all_checks(client)
+    client.get(url_for("mark_all_viewed"), follow_redirects=True)
+    wait_for_all_checks(client)
+
+    # Case 1
+    set_number_in_range_response("70.5")
+    client.get(url_for("form_watch_checknow"), follow_redirects=True)
+    wait_for_all_checks(client)
+
+    # 75 is > 20 and < 100 and contains "5"
+    res = client.get(url_for("index"))
+    assert b'unviewed' in res.data
+
+
+    # Case 2: Change with one condition violated
+    # Number out of range (150) but contains '5'
+    client.get(url_for("mark_all_viewed"), follow_redirects=True)
+    set_number_out_of_range_response("150.5")
+
+
+    client.get(url_for("form_watch_checknow"), follow_redirects=True)
+    wait_for_all_checks(client)
+
+    # Should NOT be marked as having changes since not all conditions are met
+    res = client.get(url_for("index"))
+    assert b'unviewed' not in res.data
+
+    res = client.get(url_for("form_delete", uuid="all"), follow_redirects=True)
+    assert b'Deleted' in res.data

changedetectionio/tests/test_css_selector.py

@@ -125,8 +125,7 @@ def test_check_markup_include_filters_restriction(client, live_server, measure_m
 
 # Tests the whole stack works with the CSS Filter
 def test_check_multiple_filters(client, live_server, measure_memory_usage):
-    sleep_time_for_fetch_thread = 3
-
+    #live_server_setup(live_server)
     include_filters = "#blob-a\r\nxpath://*[contains(@id,'blob-b')]"
 
     with open("test-datastore/endpoint-content.txt", "w") as f:
@@ -138,9 +137,6 @@ def test_check_multiple_filters(client, live_server, measure_memory_usage):
      </html>
     """)
 
-    # Give the endpoint time to spin up
-    time.sleep(1)
-
     # Add our URL to the import page
     test_url = url_for('test_endpoint', _external=True)
     res = client.post(
@@ -149,7 +145,7 @@ def test_check_multiple_filters(client, live_server, measure_memory_usage):
         follow_redirects=True
     )
     assert b"1 Imported" in res.data
-    time.sleep(1)
+    wait_for_all_checks(client)
 
     # Goto the edit page, add our ignore text
     # Add our URL to the import page
@@ -165,7 +161,7 @@ def test_check_multiple_filters(client, live_server, measure_memory_usage):
     assert b"Updated watch." in res.data
 
     # Give the thread time to pick it up
-    time.sleep(sleep_time_for_fetch_thread)
+    wait_for_all_checks(client)
 
     res = client.get(
         url_for("preview_page", uuid="first"),

changedetectionio/tests/test_encoding.py

@@ -40,7 +40,7 @@ def test_check_encoding_detection(client, live_server, measure_memory_usage):
 
 
     # Content type recording worked
-    uuid = extract_UUID_from_client(client)
+    uuid = next(iter(live_server.app.config['DATASTORE'].data['watching']))
     assert live_server.app.config['DATASTORE'].data['watching'][uuid]['content-type'] == "text/html"
 
     res = client.get(

changedetectionio/tests/test_filter_failure_notification.py

@@ -51,7 +51,7 @@ def run_filter_test(client, live_server, content_filter):
     assert b"1 Imported" in res.data
     wait_for_all_checks(client)
 
-    uuid = extract_UUID_from_client(client)
+    uuid = next(iter(live_server.app.config['DATASTORE'].data['watching']))
 
     assert live_server.app.config['DATASTORE'].data['watching'][uuid]['consecutive_filter_failures'] == 0, "No filter = No filter failure"
 

changedetectionio/tests/test_group.py

@@ -288,7 +288,7 @@ def test_clone_tag_on_import(client, live_server, measure_memory_usage):
     assert b'test-tag' in res.data
     assert b'another-tag' in res.data
 
-    watch_uuid = extract_UUID_from_client(client)
+    watch_uuid = next(iter(live_server.app.config['DATASTORE'].data['watching']))
     res = client.get(url_for("form_clone", uuid=watch_uuid), follow_redirects=True)
 
     assert b'Cloned' in res.data
@@ -315,7 +315,7 @@ def test_clone_tag_on_quickwatchform_add(client, live_server, measure_memory_usa
     assert b'test-tag' in res.data
     assert b'another-tag' in res.data
 
-    watch_uuid = extract_UUID_from_client(client)
+    watch_uuid = next(iter(live_server.app.config['DATASTORE'].data['watching']))
     res = client.get(url_for("form_clone", uuid=watch_uuid), follow_redirects=True)
 
     assert b'Cloned' in res.data

changedetectionio/tests/test_ignore.py

@@ -36,7 +36,7 @@ def test_ignore(client, live_server, measure_memory_usage):
 
     # Give the thread time to pick it up
     wait_for_all_checks(client)
-    uuid = extract_UUID_from_client(client)
+    uuid = next(iter(live_server.app.config['DATASTORE'].data['watching']))
     # use the highlighter endpoint
     res = client.post(
         url_for("highlight_submit_ignore_url", uuid=uuid),

changedetectionio/tests/test_jsonpath_jq_selector.py

@@ -514,3 +514,15 @@ def test_check_jq_ext_filter(client, live_server, measure_memory_usage):
 def test_check_jqraw_ext_filter(client, live_server, measure_memory_usage):
     if jq_support:
         check_json_ext_filter('jq:.[] | select(.status | contains("Sold"))', client, live_server)
+
+def test_jsonpath_BOM_utf8(client, live_server, measure_memory_usage):
+    from .. import html_tools
+
+    # JSON string with BOM and correct double-quoted keys
+    json_str = '\ufeff{"name": "José", "emoji": "😊", "language": "中文", "greeting": "Привет"}'
+
+    # See that we can find the second <script> one, which is not broken, and matches our filter
+    text = html_tools.extract_json_as_string(json_str, "json:$.name")
+    assert text == '"José"'
+
+    

changedetectionio/tests/test_live_preview.py

@@ -29,7 +29,7 @@ def test_content_filter_live_preview(client, live_server, measure_memory_usage):
         data={"url": test_url, "tags": ''},
         follow_redirects=True
     )
-    uuid = extract_UUID_from_client(client)
+    uuid = next(iter(live_server.app.config['DATASTORE'].data['watching']))
     res = client.post(
         url_for("edit_page", uuid=uuid),
         data={

changedetectionio/tests/test_nonrenderable_pages.py

@@ -48,7 +48,7 @@ def test_check_basic_change_detection_functionality(client, live_server, measure
     #####################
     client.post(
         url_for("settings_page"),
-        data={"application-empty_pages_are_a_change": "",
+        data={"application-empty_pages_are_a_change": "", # default, OFF, they are NOT a change
               "requests-time_between_check-minutes": 180,
               'application-fetch_backend': "html_requests"},
         follow_redirects=True
@@ -66,6 +66,14 @@ def test_check_basic_change_detection_functionality(client, live_server, measure
     res = client.get(url_for("index"))
     assert b'unviewed' not in res.data
 
+    uuid = next(iter(live_server.app.config['DATASTORE'].data['watching']))
+    watch = live_server.app.config['DATASTORE'].data['watching'][uuid]
+
+    assert watch.last_changed == 0
+    assert watch['last_checked'] != 0
+
+
+
 
     # ok now do the opposite
 
@@ -92,6 +100,10 @@ def test_check_basic_change_detection_functionality(client, live_server, measure
     # A totally zero byte (#2528) response should also not trigger an error
     set_zero_byte_response()
     client.get(url_for("form_watch_checknow"), follow_redirects=True)
+
+    # 2877
+    assert watch.last_changed == watch['last_checked']
+
     wait_for_all_checks(client)
     res = client.get(url_for("index"))
     assert b'unviewed' in res.data # A change should have registered because empty_pages_are_a_change is ON

changedetectionio/tests/test_notification.py

@@ -6,7 +6,7 @@ from flask import url_for
 from loguru import logger
 
 from .util import set_original_response, set_modified_response, set_more_modified_response, live_server_setup, wait_for_all_checks, \
-    set_longer_modified_response
+    set_longer_modified_response, get_index
 from . util import  extract_UUID_from_client
 import logging
 import base64
@@ -29,7 +29,7 @@ def test_check_notification(client, live_server, measure_memory_usage):
 
     # Re 360 - new install should have defaults set
     res = client.get(url_for("settings_page"))
-    notification_url = url_for('test_notification_endpoint', _external=True).replace('http', 'json')
+    notification_url = url_for('test_notification_endpoint', _external=True).replace('http', 'json')+"?status_code=204"
 
     assert default_notification_body.encode() in res.data
     assert default_notification_title.encode() in res.data
@@ -76,7 +76,7 @@ def test_check_notification(client, live_server, measure_memory_usage):
     testimage_png = 'iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mNkYAAAAAYAAjCB0C8AAAAASUVORK5CYII='
 
 
-    uuid = extract_UUID_from_client(client)
+    uuid = next(iter(live_server.app.config['DATASTORE'].data['watching']))
     datastore = 'test-datastore'
     with open(os.path.join(datastore, str(uuid), 'last-screenshot.png'), 'wb') as f:
         f.write(base64.b64decode(testimage_png))
@@ -135,7 +135,14 @@ def test_check_notification(client, live_server, measure_memory_usage):
 
     # Trigger a check
     client.get(url_for("form_watch_checknow"), follow_redirects=True)
+    wait_for_all_checks(client)
     time.sleep(3)
+
+    # Check no errors were recorded
+    res = client.get(url_for("index"))
+    assert b'notification-error' not in res.data
+
+
     # Verify what was sent as a notification, this file should exist
     with open("test-datastore/notification.txt", "r") as f:
         notification_submission = f.read()
@@ -284,7 +291,7 @@ def test_notification_custom_endpoint_and_jinja2(client, live_server, measure_me
     # CUSTOM JSON BODY CHECK for POST://
     set_original_response()
     # https://github.com/caronc/apprise/wiki/Notify_Custom_JSON#header-manipulation
-    test_notification_url = url_for('test_notification_endpoint', _external=True).replace('http://', 'post://')+"?xxx={{ watch_url }}&+custom-header=123&+second=hello+world%20%22space%22"
+    test_notification_url = url_for('test_notification_endpoint', _external=True).replace('http://', 'post://')+"?status_code=204&xxx={{ watch_url }}&+custom-header=123&+second=hello+world%20%22space%22"
 
     res = client.post(
         url_for("settings_page"),
@@ -319,6 +326,11 @@ def test_notification_custom_endpoint_and_jinja2(client, live_server, measure_me
 
     time.sleep(2) # plus extra delay for notifications to fire
 
+
+    # Check no errors were recorded, because we asked for 204 which is slightly uncommon but is still OK
+    res = get_index(client)
+    assert b'notification-error' not in res.data
+
     with open("test-datastore/notification.txt", 'r') as f:
         x = f.read()
         j = json.loads(x)
@@ -360,7 +372,10 @@ def test_global_send_test_notification(client, live_server, measure_memory_usage
     #live_server_setup(live_server)
     set_original_response()
     if os.path.isfile("test-datastore/notification.txt"):
-        os.unlink("test-datastore/notification.txt")
+        os.unlink("test-datastore/notification.txt") \
+
+    # 1995 UTF-8 content should be encoded
+    test_body = 'change detection is cool 网站监测 内容更新了'
 
     # otherwise other settings would have already existed from previous tests in this file
     res = client.post(
@@ -368,8 +383,7 @@ def test_global_send_test_notification(client, live_server, measure_memory_usage
         data={
             "application-fetch_backend": "html_requests",
             "application-minutes_between_check": 180,
-            #1995 UTF-8 content should be encoded
-            "application-notification_body": 'change detection is cool 网站监测 内容更新了',
+            "application-notification_body": test_body,
             "application-notification_format": default_notification_format,
             "application-notification_urls": "",
             "application-notification_title": "New ChangeDetection.io Notification - {{ watch_url }}",
@@ -399,12 +413,10 @@ def test_global_send_test_notification(client, live_server, measure_memory_usage
     assert res.status_code != 400
     assert res.status_code != 500
 
-    # Give apprise time to fire
-    time.sleep(4)
 
     with open("test-datastore/notification.txt", 'r') as f:
         x = f.read()
-        assert 'change detection is cool 网站监测 内容更新了' in x
+        assert test_body in x
 
     os.unlink("test-datastore/notification.txt")
 
@@ -442,4 +454,67 @@ def test_global_send_test_notification(client, live_server, measure_memory_usage
     assert b"Error: You must have atleast one watch configured for 'test notification' to work" in res.data
 
 
+def _test_color_notifications(client, notification_body_token):
 
+    from changedetectionio.diff import ADDED_STYLE, REMOVED_STYLE
+
+    set_original_response()
+
+    if os.path.isfile("test-datastore/notification.txt"):
+        os.unlink("test-datastore/notification.txt")
+
+
+    test_notification_url = url_for('test_notification_endpoint', _external=True).replace('http://', 'post://')+"?xxx={{ watch_url }}&+custom-header=123"
+
+
+    # otherwise other settings would have already existed from previous tests in this file
+    res = client.post(
+        url_for("settings_page"),
+        data={
+            "application-fetch_backend": "html_requests",
+            "application-minutes_between_check": 180,
+            "application-notification_body": notification_body_token,
+            "application-notification_format": "HTML Color",
+            "application-notification_urls": test_notification_url,
+            "application-notification_title": "New ChangeDetection.io Notification - {{ watch_url }}",
+        },
+        follow_redirects=True
+    )
+    assert b'Settings updated' in res.data
+
+    test_url = url_for('test_endpoint', _external=True)
+    res = client.post(
+        url_for("form_quick_watch_add"),
+        data={"url": test_url, "tags": 'nice one'},
+        follow_redirects=True
+    )
+
+    assert b"Watch added" in res.data
+
+    wait_for_all_checks(client)
+
+    set_modified_response()
+
+
+    res = client.get(url_for("form_watch_checknow"), follow_redirects=True)
+    assert b'1 watches queued for rechecking.' in res.data
+
+    wait_for_all_checks(client)
+    time.sleep(3)
+
+    with open("test-datastore/notification.txt", 'r') as f:
+        x = f.read()
+        assert f'<span style="{REMOVED_STYLE}">Which is across multiple lines' in x
+
+
+    client.get(
+        url_for("form_delete", uuid="all"),
+        follow_redirects=True
+    )
+
+def test_html_color_notifications(client, live_server, measure_memory_usage):
+
+    #live_server_setup(live_server)
+    _test_color_notifications(client, '{{diff}}')
+    _test_color_notifications(client, '{{diff_full}}')
+    

changedetectionio/tests/test_processor_registry.py

@@ -0,0 +1,59 @@
+import pytest
+from changedetectionio.processors.processor_registry import get_processor_class, get_all_processors
+
+
+def test_get_all_processors():
+    """Test that get_all_processors returns a list of processor tuples"""
+    processors = get_all_processors()
+    assert isinstance(processors, list)
+    assert len(processors) > 0
+    
+    # Each item should be a tuple of (name, description)
+    for processor in processors:
+        assert isinstance(processor, tuple)
+        assert len(processor) == 2
+        assert isinstance(processor[0], str)
+        assert isinstance(processor[1], str)
+        
+    # Check that our WHOIS processor is included
+    whois_processor = next((p for p in processors if p[0] == "whois"), None)
+    assert whois_processor is not None
+    assert whois_processor[1] == "WHOIS Domain Information Changes"
+
+
+def test_get_processor_class():
+    """Test that get_processor_class returns the right class"""
+    # Get the WHOIS processor class
+    processor_class = get_processor_class("whois")
+    assert processor_class is not None
+    
+    # It should have perform_site_check method
+    assert hasattr(processor_class, 'perform_site_check')
+    
+    # Check for non-existent processor
+    non_existent = get_processor_class("non_existent_processor")
+    assert non_existent is None
+
+
+def test_get_processor_site_check():
+    """Test that get_processor_site_check returns a processor instance"""
+    from unittest.mock import MagicMock
+    from changedetectionio.processors.processor_registry import get_processor_site_check
+    
+    # Get a WHOIS processor instance
+    mock_datastore = MagicMock()
+    watch_uuid = "test-uuid"
+    processor = get_processor_site_check("whois", mock_datastore, watch_uuid)
+    
+    # It should be a processor instance
+    assert processor is not None
+    
+    # It should have the run_changedetection method
+    assert hasattr(processor, 'run_changedetection')
+    
+    # It should have the call_browser method
+    assert hasattr(processor, 'call_browser')
+    
+    # Check for non-existent processor
+    non_existent = get_processor_site_check("non_existent_processor", mock_datastore, watch_uuid)
+    assert non_existent is None

changedetectionio/tests/test_request.py

@@ -373,13 +373,14 @@ def test_headers_textfile_in_request(client, live_server, measure_memory_usage):
     wait_for_all_checks(client)
 
     with open('test-datastore/headers-testtag.txt', 'w') as f:
-        f.write("tag-header: test")
+        f.write("tag-header: test\r\nurl-header: http://example.com")
 
     with open('test-datastore/headers.txt', 'w') as f:
-        f.write("global-header: nice\r\nnext-global-header: nice")
+        f.write("global-header: nice\r\nnext-global-header: nice\r\nurl-header-global: http://example.com/global")
 
-    with open('test-datastore/' + extract_UUID_from_client(client) + '/headers.txt', 'w') as f:
-        f.write("watch-header: nice")
+    uuid = next(iter(live_server.app.config['DATASTORE'].data['watching']))
+    with open(f'test-datastore/{uuid}/headers.txt', 'w') as f:
+        f.write("watch-header: nice\r\nurl-header-watch: http://example.com/watch")
 
     wait_for_all_checks(client)
     client.get(url_for("form_watch_checknow"), follow_redirects=True)
@@ -410,6 +411,9 @@ def test_headers_textfile_in_request(client, live_server, measure_memory_usage):
     assert b"Xxx:ooo" in res.data
     assert b"Watch-Header:nice" in res.data
     assert b"Tag-Header:test" in res.data
+    assert b"Url-Header:http://example.com" in res.data
+    assert b"Url-Header-Global:http://example.com/global" in res.data
+    assert b"Url-Header-Watch:http://example.com/watch" in res.data
 
     # Check the custom UA from system settings page made it through
     if os.getenv('PLAYWRIGHT_DRIVER_URL'):

changedetectionio/tests/test_restock_itemprop.py

@@ -189,6 +189,17 @@ def _run_test_minmax_limit(client, extra_watch_edit_form):
 
     client.get(url_for("mark_all_viewed"))
 
+
+    # 2715 - Price detection (once it crosses the "lower" threshold) again with a lower price - should trigger again!
+    set_original_response(props_markup=instock_props[0], price='820.45')
+    res = client.get(url_for("form_watch_checknow"), follow_redirects=True)
+    assert b'1 watches queued for rechecking.' in res.data
+    wait_for_all_checks(client)
+    res = client.get(url_for("index"))
+    assert b'820.45' in res.data
+    assert b'unviewed' in res.data
+    client.get(url_for("mark_all_viewed"))
+
     # price changed to something MORE than max (1100.10), SHOULD be a change
     set_original_response(props_markup=instock_props[0], price='1890.45')
     client.get(url_for("form_watch_checknow"), follow_redirects=True)
@@ -203,7 +214,7 @@ def _run_test_minmax_limit(client, extra_watch_edit_form):
 
 
 def test_restock_itemprop_minmax(client, live_server):
-#    live_server_setup(live_server)
+    #live_server_setup(live_server)
     extras = {
         "restock_settings-follow_price_changes": "y",
         "restock_settings-price_change_min": 900.0,
@@ -369,7 +380,7 @@ def test_change_with_notification_values(client, live_server):
 
     ## Now test the "SEND TEST NOTIFICATION" is working
     os.unlink("test-datastore/notification.txt")
-    uuid = extract_UUID_from_client(client)
+    uuid = next(iter(live_server.app.config['DATASTORE'].data['watching']))
     res = client.post(url_for("ajax_callback_send_notification_test", watch_uuid=uuid), data={}, follow_redirects=True)
     time.sleep(5)
     assert os.path.isfile("test-datastore/notification.txt"), "Notification received"

changedetectionio/tests/test_rss.py

@@ -132,7 +132,7 @@ def test_rss_xpath_filtering(client, live_server, measure_memory_usage):
     )
     assert b"Watch added in Paused state, saving will unpause" in res.data
 
-    uuid = extract_UUID_from_client(client)
+    uuid = next(iter(live_server.app.config['DATASTORE'].data['watching']))
     res = client.post(
         url_for("edit_page", uuid=uuid, unpause_on_save=1),
         data={

changedetectionio/tests/test_scheduler.py

@@ -0,0 +1,179 @@
+#!/usr/bin/env python3
+
+import time
+from datetime import datetime, timezone
+from zoneinfo import ZoneInfo
+from flask import url_for
+from .util import  live_server_setup, wait_for_all_checks, extract_UUID_from_client
+
+def test_setup(client, live_server):
+    live_server_setup(live_server)
+
+def test_check_basic_scheduler_functionality(client, live_server, measure_memory_usage):
+    #live_server_setup(live_server)
+    days = ['monday', 'tuesday', 'wednesday', 'thursday', 'friday', 'saturday', 'sunday']
+    test_url = url_for('test_random_content_endpoint', _external=True)
+
+    # We use "Pacific/Kiritimati" because its the furthest +14 hours, so it might show up more interesting bugs
+    # The rest of the actual functionality should be covered in the unit-test  unit/test_scheduler.py
+    #####################
+    res = client.post(
+        url_for("settings_page"),
+        data={"application-empty_pages_are_a_change": "",
+              "requests-time_between_check-seconds": 1,
+              "application-timezone": "Pacific/Kiritimati",  # Most Forward Time Zone (UTC+14:00)
+              'application-fetch_backend': "html_requests"},
+        follow_redirects=True
+    )
+
+    assert b"Settings updated." in res.data
+
+    res = client.get(url_for("settings_page"))
+    assert b'Pacific/Kiritimati' in res.data
+
+    res = client.post(
+        url_for("import_page"),
+        data={"urls": test_url},
+        follow_redirects=True
+    )
+
+    assert b"1 Imported" in res.data
+    wait_for_all_checks(client)
+    uuid = next(iter(live_server.app.config['DATASTORE'].data['watching']))
+
+    # Setup all the days of the weeks using XXX as the placeholder for monday/tuesday/etc
+
+    tpl = {
+        "time_schedule_limit-XXX-start_time": "00:00",
+        "time_schedule_limit-XXX-duration-hours": 24,
+        "time_schedule_limit-XXX-duration-minutes": 0,
+        "time_schedule_limit-XXX-enabled": '',  # All days are turned off
+        "time_schedule_limit-enabled": 'y',  # Scheduler is enabled, all days however are off.
+    }
+
+    scheduler_data = {}
+    for day in days:
+        for key, value in tpl.items():
+            # Replace "XXX" with the current day in the key
+            new_key = key.replace("XXX", day)
+            scheduler_data[new_key] = value
+
+    last_check = live_server.app.config['DATASTORE'].data['watching'][uuid]['last_checked']
+    data = {
+        "url": test_url,
+        "fetch_backend": "html_requests"
+    }
+    data.update(scheduler_data)
+
+    res = client.post(
+        url_for("edit_page", uuid="first"),
+        data=data,
+        follow_redirects=True
+    )
+    assert b"Updated watch." in res.data
+
+    res = client.get(url_for("edit_page", uuid="first"))
+    assert b"Pacific/Kiritimati" in res.data, "Should be Pacific/Kiritimati in placeholder data"
+
+    # "Edit" should not trigger a check because it's not enabled in the schedule.
+    time.sleep(2)
+    assert live_server.app.config['DATASTORE'].data['watching'][uuid]['last_checked'] == last_check
+
+    # Enabling today in Kiritimati should work flawless
+    kiritimati_time = datetime.now(timezone.utc).astimezone(ZoneInfo("Pacific/Kiritimati"))
+    kiritimati_time_day_of_week = kiritimati_time.strftime("%A").lower()
+    live_server.app.config['DATASTORE'].data['watching'][uuid]["time_schedule_limit"][kiritimati_time_day_of_week]["enabled"] = True
+    time.sleep(3)
+    assert live_server.app.config['DATASTORE'].data['watching'][uuid]['last_checked'] != last_check
+
+    # Cleanup everything
+    res = client.get(url_for("form_delete", uuid="all"), follow_redirects=True)
+    assert b'Deleted' in res.data
+
+
+def test_check_basic_global_scheduler_functionality(client, live_server, measure_memory_usage):
+    #live_server_setup(live_server)
+    days = ['monday', 'tuesday', 'wednesday', 'thursday', 'friday', 'saturday', 'sunday']
+    test_url = url_for('test_random_content_endpoint', _external=True)
+
+    res = client.post(
+        url_for("import_page"),
+        data={"urls": test_url},
+        follow_redirects=True
+    )
+
+    assert b"1 Imported" in res.data
+    wait_for_all_checks(client)
+    uuid = next(iter(live_server.app.config['DATASTORE'].data['watching']))
+
+    # Setup all the days of the weeks using XXX as the placeholder for monday/tuesday/etc
+
+    tpl = {
+        "requests-time_schedule_limit-XXX-start_time": "00:00",
+        "requests-time_schedule_limit-XXX-duration-hours": 24,
+        "requests-time_schedule_limit-XXX-duration-minutes": 0,
+        "requests-time_schedule_limit-XXX-enabled": '',  # All days are turned off
+        "requests-time_schedule_limit-enabled": 'y',  # Scheduler is enabled, all days however are off.
+    }
+
+    scheduler_data = {}
+    for day in days:
+        for key, value in tpl.items():
+            # Replace "XXX" with the current day in the key
+            new_key = key.replace("XXX", day)
+            scheduler_data[new_key] = value
+
+    data = {
+        "application-empty_pages_are_a_change": "",
+        "application-timezone": "Pacific/Kiritimati",  # Most Forward Time Zone (UTC+14:00)
+        'application-fetch_backend': "html_requests",
+        "requests-time_between_check-hours": 0,
+        "requests-time_between_check-minutes": 0,
+        "requests-time_between_check-seconds": 1,
+    }
+    data.update(scheduler_data)
+
+    #####################
+    res = client.post(
+        url_for("settings_page"),
+        data=data,
+        follow_redirects=True
+    )
+
+    assert b"Settings updated." in res.data
+
+    res = client.get(url_for("settings_page"))
+    assert b'Pacific/Kiritimati' in res.data
+
+    wait_for_all_checks(client)
+
+    # UI Sanity check
+
+    res = client.get(url_for("edit_page", uuid="first"))
+    assert b"Pacific/Kiritimati" in res.data, "Should be Pacific/Kiritimati in placeholder data"
+
+    #### HITTING SAVE SHOULD NOT TRIGGER A CHECK
+    last_check = live_server.app.config['DATASTORE'].data['watching'][uuid]['last_checked']
+    res = client.post(
+        url_for("edit_page", uuid="first"),
+        data={
+            "url": test_url,
+            "fetch_backend": "html_requests",
+            "time_between_check_use_default": "y"},
+        follow_redirects=True
+    )
+    assert b"Updated watch." in res.data
+    time.sleep(2)
+    assert live_server.app.config['DATASTORE'].data['watching'][uuid]['last_checked'] == last_check
+
+    # Enabling "today" in Kiritimati time should make the system check that watch
+    kiritimati_time = datetime.now(timezone.utc).astimezone(ZoneInfo("Pacific/Kiritimati"))
+    kiritimati_time_day_of_week = kiritimati_time.strftime("%A").lower()
+    live_server.app.config['DATASTORE'].data['settings']['requests']['time_schedule_limit'][kiritimati_time_day_of_week]["enabled"] = True
+
+    time.sleep(3)
+    assert live_server.app.config['DATASTORE'].data['watching'][uuid]['last_checked'] != last_check
+
+    # Cleanup everything
+    res = client.get(url_for("form_delete", uuid="all"), follow_redirects=True)
+    assert b'Deleted' in res.data

changedetectionio/tests/test_security.py

@@ -1,9 +1,7 @@
 import os
 
 from flask import url_for
-from .util import set_original_response, set_modified_response, live_server_setup, wait_for_all_checks
-import time
-
+from .util import live_server_setup, wait_for_all_checks
 from .. import strtobool
 
 
@@ -61,54 +59,44 @@ def test_bad_access(client, live_server, measure_memory_usage):
     assert b'Watch protocol is not permitted by SAFE_PROTOCOL_REGEX' in res.data
 
 
-def test_file_slashslash_access(client, live_server, measure_memory_usage):
-    #live_server_setup(live_server)
+def _runner_test_various_file_slash(client, file_uri):
 
-    test_file_path = os.path.abspath(__file__)
-
-    # file:// is permitted by default, but it will be caught by ALLOW_FILE_URI
     client.post(
         url_for("form_quick_watch_add"),
-        data={"url": f"file://{test_file_path}", "tags": ''},
+        data={"url": file_uri, "tags": ''},
         follow_redirects=True
     )
     wait_for_all_checks(client)
     res = client.get(url_for("index"))
 
+    substrings = [b"URLs with hostname components are not permitted", b"No connection adapters were found for"]
+
+
     # If it is enabled at test time
     if strtobool(os.getenv('ALLOW_FILE_URI', 'false')):
-        res = client.get(
-            url_for("preview_page", uuid="first"),
-            follow_redirects=True
-        )
+        if file_uri.startswith('file:///'):
+            # This one should be the full qualified path to the file and should get the contents of this file
+            res = client.get(
+                url_for("preview_page", uuid="first"),
+                follow_redirects=True
+            )
+            assert b'_runner_test_various_file_slash' in res.data
+        else:
+            # This will give some error from requests or if it went to chrome, will give some other error :-)
+            assert any(s in res.data for s in substrings)
 
-        assert b"test_file_slashslash_access" in res.data
-    else:
-        # Default should be here
-        assert b'file:// type access is denied for security reasons.' in res.data
+    res = client.get(url_for("form_delete", uuid="all"), follow_redirects=True)
+    assert b'Deleted' in res.data
 
 def test_file_slash_access(client, live_server, measure_memory_usage):
     #live_server_setup(live_server)
 
+    # file: is NOT permitted by default, so it will be caught by ALLOW_FILE_URI check
+
     test_file_path = os.path.abspath(__file__)
-
-    # file:// is permitted by default, but it will be caught by ALLOW_FILE_URI
-    client.post(
-        url_for("form_quick_watch_add"),
-        data={"url": f"file:/{test_file_path}", "tags": ''},
-        follow_redirects=True
-    )
-    wait_for_all_checks(client)
-    res = client.get(url_for("index"))
-
-    # If it is enabled at test time
-    if strtobool(os.getenv('ALLOW_FILE_URI', 'false')):
-        # So it should permit it, but it should fall back to the 'requests' library giving an error
-        # (but means it gets passed to playwright etc)
-        assert b"URLs with hostname components are not permitted" in res.data
-    else:
-        # Default should be here
-        assert b'file:// type access is denied for security reasons.' in res.data
+    _runner_test_various_file_slash(client, file_uri=f"file://{test_file_path}")
+    _runner_test_various_file_slash(client, file_uri=f"file:/{test_file_path}")
+    _runner_test_various_file_slash(client, file_uri=f"file:{test_file_path}") # CVE-2024-56509
 
 def test_xss(client, live_server, measure_memory_usage):
     #live_server_setup(live_server)

changedetectionio/tests/test_whois_processor.py

@@ -0,0 +1,182 @@
+import pytest
+from unittest.mock import MagicMock, patch
+from changedetectionio.processors.whois_plugin import WhoisProcessor
+
+
+class MockWatch:
+    def __init__(self, url, previous_md5=None, include_filters=None, ignore_text=None):
+        self.url = url
+        self._previous_md5 = previous_md5
+        self._include_filters = include_filters or []
+        self._ignore_text = ignore_text or []
+        self.history = {}
+    
+    def get(self, key, default=None):
+        if key == 'previous_md5':
+            return self._previous_md5
+        elif key == 'include_filters':
+            return self._include_filters
+        elif key == 'ignore_text':
+            return self._ignore_text
+        elif key == 'url':
+            return self.url
+        return default
+    
+    def has_special_diff_filter_options_set(self):
+        return False
+
+
+@patch('whois.whois')
+@patch('changedetectionio.processors.difference_detection_processor.__init__')
+@patch('changedetectionio.processors.text_json_diff.processor.perform_site_check.run_changedetection')
+def test_whois_processor_basic_functionality(mock_super_run, mock_base_init, mock_whois):
+    """Test the basic functionality of the WhoisProcessor"""
+    # Mock the base class init so we don't need to set up the full watch structure
+    mock_base_init.return_value = None
+    
+    # Mock super().run_changedetection to return a simple result
+    mock_super_run.return_value = (False, {'previous_md5': 'some-md5'}, b'Some filtered text')
+    
+    # Mock the whois response
+    mock_whois_result = MagicMock()
+    mock_whois_result.text = "Domain Name: example.com\nRegistrar: Example Registrar\nCreation Date: 2020-01-01\n"
+    mock_whois.return_value = mock_whois_result
+    
+    # Create mock datastore
+    mock_datastore = MagicMock()
+    mock_datastore.proxy_list = None  # No proxies
+    mock_datastore.get_preferred_proxy_for_watch.return_value = None
+    mock_datastore.data = {
+        'settings': {
+            'application': {
+                'allow_file_uri': False
+            }
+        }
+    }
+    
+    # Create a processor instance and setup minimal required attributes
+    processor = WhoisProcessor(datastore=mock_datastore, watch_uuid='test-uuid')
+    
+    # Create a minimal watch object
+    watch = MockWatch(url="https://example.com")
+    
+    # Simulate link access in the watch
+    processor.watch = MagicMock()
+    processor.watch.link = "https://example.com"
+    processor.watch.get.return_value = "uuid-123"
+    
+    # Run the processor's run_changedetection method by first using call_browser
+    processor.call_browser()
+    
+    # Check that the fetcher was set up correctly
+    assert processor.fetcher is not None
+    assert hasattr(processor.fetcher, 'content')
+    assert hasattr(processor.fetcher, 'headers')
+    assert hasattr(processor.fetcher, 'status_code')
+    
+    # Verify that whois was called with the right domain
+    assert mock_whois.called
+    assert mock_whois.call_args[0][0] == 'example.com'
+    
+    # Now run the processor
+    result = processor.run_changedetection(watch)
+    
+    # Check that the parent run_changedetection was called
+    assert mock_super_run.called
+
+
+@patch('whois.whois')
+@patch('changedetectionio.processors.difference_detection_processor.__init__')
+def test_whois_processor_call_browser_with_proxy(mock_base_init, mock_whois):
+    """Test the call_browser method with proxy configuration"""
+    # Mock the base class init
+    mock_base_init.return_value = None
+    
+    # Mock the whois response
+    mock_whois_result = MagicMock()
+    mock_whois_result.text = "Domain Name: example.com\nRegistrar: Example Registrar\nCreation Date: 2020-01-01\n"
+    mock_whois.return_value = mock_whois_result
+    
+    # Create mock datastore
+    mock_datastore = MagicMock()
+    mock_proxy = {
+        'test-proxy': {
+            'url': 'http://proxy.example.com:8080',
+            'label': 'Test Proxy'
+        }
+    }
+    mock_datastore.proxy_list = mock_proxy
+    mock_datastore.get_preferred_proxy_for_watch.return_value = 'test-proxy'
+    mock_datastore.data = {
+        'settings': {
+            'application': {
+                'allow_file_uri': False
+            }
+        }
+    }
+    
+    # Create a processor instance with our mock datastore
+    processor = WhoisProcessor(datastore=mock_datastore, watch_uuid='test-uuid')
+    
+    # Set up watch
+    processor.watch = MagicMock()
+    processor.watch.link = "https://example.com"
+    processor.watch.get.return_value = "uuid-123"
+    
+    # Call the method with a proxy
+    processor.call_browser()
+    
+    # Verify whois was called
+    assert mock_whois.called
+    assert mock_whois.call_args[0][0] == 'example.com'
+    
+    # Check that the fetcher was set up correctly
+    assert processor.fetcher is not None
+    assert processor.fetcher.content is not None
+
+
+@patch('changedetectionio.processors.difference_detection_processor.__init__')
+def test_whois_processor_perform_site_check(mock_base_init):
+    """Test the WhoisProcessor.perform_site_check static method"""
+    mock_base_init.return_value = None
+    
+    # Test the static method
+    with patch.object(WhoisProcessor, '__init__', return_value=None) as mock_init:
+        datastore = MagicMock()
+        watch_uuid = "test-uuid"
+        
+        # Call the static method
+        processor = WhoisProcessor.perform_site_check(datastore=datastore, watch_uuid=watch_uuid)
+        
+        # Check that constructor was called with expected args
+        mock_init.assert_called_once_with(datastore=datastore, watch_uuid=watch_uuid)
+        
+        # Check it returns the right type
+        assert isinstance(processor, WhoisProcessor)
+
+
+def test_get_display_link():
+    """Test the get_display_link hook implementation"""
+    from changedetectionio.processors.whois_plugin import get_display_link
+    
+    # Test with a regular URL
+    url = "https://example.com/some/path?param=value"
+    processor_name = "whois"
+    link = get_display_link(url=url, processor_name=processor_name)
+    assert link == "WHOIS - example.com"
+    
+    # Test with a subdomain
+    url = "https://subdomain.example.com/"
+    link = get_display_link(url=url, processor_name=processor_name)
+    assert link == "WHOIS - subdomain.example.com"
+    
+    # Test with www prefix (should be removed)
+    url = "https://www.example.com/"
+    link = get_display_link(url=url, processor_name=processor_name)
+    assert link == "WHOIS - example.com"
+    
+    # Test with a different processor (should return None)
+    url = "https://example.com/"
+    processor_name = "text_json_diff"
+    link = get_display_link(url=url, processor_name=processor_name)
+    assert link is None

changedetectionio/tests/unit/test_conditions.py

@@ -0,0 +1,82 @@
+from changedetectionio.conditions import execute_ruleset_against_all_plugins
+from changedetectionio.store import ChangeDetectionStore
+import shutil
+import tempfile
+import time
+import unittest
+import uuid
+
+
+class TestTriggerConditions(unittest.TestCase):
+    def setUp(self):
+
+        # Create a temporary directory for the test datastore
+        self.test_datastore_path = tempfile.mkdtemp()
+
+        # Initialize ChangeDetectionStore with our test path and no default watches
+        self.store = ChangeDetectionStore(
+            datastore_path=self.test_datastore_path,
+            include_default_watches=False
+        )
+
+        # Add a test watch
+        watch_url = "https://example.com"
+        self.watch_uuid = self.store.add_watch(url=watch_url)
+
+    def tearDown(self):
+      # Clean up the test datastore
+      self.store.stop_thread = True
+      time.sleep(0.5)  # Give thread time to stop
+      shutil.rmtree(self.test_datastore_path)
+
+    def test_conditions_execution_pass(self):
+        # Get the watch object
+        watch = self.store.data['watching'][self.watch_uuid]
+
+        # Create and save a snapshot
+        first_content = "I saw 100 people at a rock show"
+        timestamp1 = int(time.time())
+        snapshot_id1 = str(uuid.uuid4())
+        watch.save_history_text(contents=first_content,
+                                timestamp=timestamp1,
+                                snapshot_id=snapshot_id1)
+
+        # Add another snapshot
+        second_content = "I saw 200 people at a rock show"
+        timestamp2 = int(time.time()) + 60
+        snapshot_id2 = str(uuid.uuid4())
+        watch.save_history_text(contents=second_content,
+                                timestamp=timestamp2,
+                                snapshot_id=snapshot_id2)
+
+        # Verify both snapshots are stored
+        history = watch.history
+        self.assertEqual(len(history), 2)
+
+        # Retrieve and check snapshots
+        #snapshot1 = watch.get_history_snapshot(str(timestamp1))
+        #snapshot2 = watch.get_history_snapshot(str(timestamp2))
+
+        self.store.data['watching'][self.watch_uuid].update(
+            {
+                "conditions_match_logic": "ALL",
+                "conditions": [
+                    {"operator": ">=", "field": "extracted_number", "value": "10"},
+                    {"operator": "<=", "field": "extracted_number", "value": "5000"},
+                    {"operator": "in", "field": "page_text", "value": "rock"},
+                    #{"operator": "starts_with", "field": "page_text", "value": "I saw"},
+                ]
+            }
+        )
+
+        # ephemeral_data - some data that could exist before the watch saved a new version
+        result = execute_ruleset_against_all_plugins(current_watch_uuid=self.watch_uuid,
+                                                     application_datastruct=self.store.data,
+                                                     ephemeral_data={'text': "I saw 500 people at a rock show"})
+
+        # @todo - now we can test that 'Extract number' increased more than X since last time
+        self.assertTrue(result)
+
+
+if __name__ == '__main__':
+    unittest.main()

changedetectionio/tests/unit/test_scheduler.py

@@ -0,0 +1,53 @@
+#!/usr/bin/env python3
+
+# run from dir above changedetectionio/ dir
+# python3 -m unittest changedetectionio.tests.unit.test_jinja2_security
+
+import unittest
+from datetime import datetime, timedelta
+from zoneinfo import ZoneInfo
+
+class TestScheduler(unittest.TestCase):
+
+    # UTC+14:00 (Line Islands, Kiribati) is the farthest ahead, always ahead of UTC.
+    # UTC-12:00 (Baker Island, Howland Island) is the farthest behind, always one calendar day behind UTC.
+
+    def test_timezone_basic_time_within_schedule(self):
+        from changedetectionio import time_handler
+
+        timezone_str = 'Europe/Berlin'
+        debug_datetime = datetime.now(ZoneInfo(timezone_str))
+        day_of_week = debug_datetime.strftime('%A')
+        time_str = str(debug_datetime.hour)+':00'
+        duration = 60  # minutes
+
+        # The current time should always be within 60 minutes of [time_hour]:00
+        result = time_handler.am_i_inside_time(day_of_week=day_of_week,
+                                               time_str=time_str,
+                                               timezone_str=timezone_str,
+                                               duration=duration)
+
+        self.assertEqual(result, True, f"{debug_datetime} is within time scheduler {day_of_week} {time_str} in {timezone_str} for {duration} minutes")
+
+    def test_timezone_basic_time_outside_schedule(self):
+        from changedetectionio import time_handler
+
+        timezone_str = 'Europe/Berlin'
+        # We try a date in the future..
+        debug_datetime = datetime.now(ZoneInfo(timezone_str))+ timedelta(days=-1)
+        day_of_week = debug_datetime.strftime('%A')
+        time_str = str(debug_datetime.hour) + ':00'
+        duration = 60*24  # minutes
+
+        # The current time should always be within 60 minutes of [time_hour]:00
+        result = time_handler.am_i_inside_time(day_of_week=day_of_week,
+                                               time_str=time_str,
+                                               timezone_str=timezone_str,
+                                               duration=duration)
+
+        self.assertNotEqual(result, True,
+                         f"{debug_datetime} is NOT within time scheduler {day_of_week} {time_str} in {timezone_str} for {duration} minutes")
+
+
+if __name__ == '__main__':
+    unittest.main()

changedetectionio/tests/unit/test_semver.py

@@ -0,0 +1,64 @@
+#!/usr/bin/env python3
+
+# run from dir above changedetectionio/ dir
+# python3 -m unittest changedetectionio.tests.unit.test_semver
+
+import re
+import unittest
+
+
+# The SEMVER regex
+SEMVER_REGEX = r"^(?P<major>0|[1-9]\d*)\.(?P<minor>0|[1-9]\d*)\.(?P<patch>0|[1-9]\d*)(?:-(?P<prerelease>(?:0|[1-9]\d*|\d*[a-zA-Z-][0-9a-zA-Z-]*)(?:\.(?:0|[1-9]\d*|\d*[a-zA-Z-][0-9a-zA-Z-]*))*))?(?:\+(?P<buildmetadata>[0-9a-zA-Z-]+(?:\.[0-9a-zA-Z-]+)*))?$"
+
+# Compile the regex
+semver_pattern = re.compile(SEMVER_REGEX)
+
+class TestSemver(unittest.TestCase):
+    def test_valid_versions(self):
+        """Test valid semantic version strings"""
+        valid_versions = [
+            "1.0.0",
+            "0.1.0",
+            "0.0.1",
+            "1.0.0-alpha",
+            "1.0.0-alpha.1",
+            "1.0.0-0.3.7",
+            "1.0.0-x.7.z.92",
+            "1.0.0-alpha+001",
+            "1.0.0+20130313144700",
+            "1.0.0-beta+exp.sha.5114f85"
+        ]
+        for version in valid_versions:
+            with self.subTest(version=version):
+                self.assertIsNotNone(semver_pattern.match(version), f"Version {version} should be valid")
+
+    def test_invalid_versions(self):
+        """Test invalid semantic version strings"""
+        invalid_versions = [
+            "0.48.06",
+            "1.0",
+            "1.0.0-",
+# Seems to pass the semver.org regex?
+#            "1.0.0-alpha-",
+            "1.0.0+",
+            "1.0.0-alpha+",
+            "1.0.0-",
+            "01.0.0",
+            "1.01.0",
+            "1.0.01",
+            ".1.0.0",
+            "1..0.0"
+        ]
+        for version in invalid_versions:
+            with self.subTest(version=version):
+                res = semver_pattern.match(version)
+                self.assertIsNone(res, f"Version '{version}' should be invalid")
+
+    def test_our_version(self):
+        from changedetectionio import get_version
+        our_version = get_version()
+        self.assertIsNotNone(semver_pattern.match(our_version), f"Our version '{our_version}' should be a valid SEMVER string")
+
+
+if __name__ == '__main__':
+    unittest.main()

changedetectionio/tests/unit/test_watch_model.py

@@ -16,7 +16,6 @@ class TestDiffBuilder(unittest.TestCase):
         watch = Watch.model(datastore_path='/tmp', default={})
         watch.ensure_data_dir_exists()
 
-        watch['last_viewed'] = 110
 
         # Contents from the browser are always returned from the browser/requests/etc as str, str is basically UTF-16 in python
         watch.save_history_text(contents="hello world", timestamp=100, snapshot_id=str(uuid_builder.uuid4()))
@@ -25,31 +24,59 @@ class TestDiffBuilder(unittest.TestCase):
         watch.save_history_text(contents="hello world", timestamp=112, snapshot_id=str(uuid_builder.uuid4()))
         watch.save_history_text(contents="hello world", timestamp=115, snapshot_id=str(uuid_builder.uuid4()))
         watch.save_history_text(contents="hello world", timestamp=117, snapshot_id=str(uuid_builder.uuid4()))
+    
+        p = watch.get_from_version_based_on_last_viewed
+        assert p == "100", "Correct 'last viewed' timestamp was detected"
 
-        p = watch.get_next_snapshot_key_to_last_viewed
-        assert p == "112", "Correct last-viewed timestamp was detected"
+        watch['last_viewed'] = 110
+        p = watch.get_from_version_based_on_last_viewed
+        assert p == "109", "Correct 'last viewed' timestamp was detected"
 
-        # When there is only one step of difference from the end of the list, it should return second-last change
         watch['last_viewed'] = 116
-        p = watch.get_next_snapshot_key_to_last_viewed
-        assert p == "115", "Correct 'second last' last-viewed timestamp was detected when using the last timestamp"
+        p = watch.get_from_version_based_on_last_viewed
+        assert p == "115", "Correct 'last viewed' timestamp was detected"
 
         watch['last_viewed'] = 99
-        p = watch.get_next_snapshot_key_to_last_viewed
-        assert p == "100"
+        p = watch.get_from_version_based_on_last_viewed
+        assert p == "100", "When the 'last viewed' timestamp is less than the oldest snapshot, return oldest"
 
         watch['last_viewed'] = 200
-        p = watch.get_next_snapshot_key_to_last_viewed
-        assert p == "115", "When the 'last viewed' timestamp is greater than the newest snapshot, return second last "
+        p = watch.get_from_version_based_on_last_viewed
+        assert p == "115", "When the 'last viewed' timestamp is greater than the newest snapshot, return second newest"
 
         watch['last_viewed'] = 109
-        p = watch.get_next_snapshot_key_to_last_viewed
+        p = watch.get_from_version_based_on_last_viewed
         assert p == "109", "Correct when its the same time"
 
         # new empty one
         watch = Watch.model(datastore_path='/tmp', default={})
-        p = watch.get_next_snapshot_key_to_last_viewed
+        p = watch.get_from_version_based_on_last_viewed
         assert p == None, "None when no history available"
 
+        watch.save_history_text(contents="hello world", timestamp=100, snapshot_id=str(uuid_builder.uuid4()))
+        p = watch.get_from_version_based_on_last_viewed
+        assert p == "100", "Correct with only one history snapshot"
+
+        watch['last_viewed'] = 200
+        p = watch.get_from_version_based_on_last_viewed
+        assert p == "100", "Correct with only one history snapshot"
+
+    def test_watch_link_property_with_processor(self):
+        """Test the link property with a processor that customizes the link"""
+        from unittest.mock import patch
+        
+        watch = Watch.model(datastore_path='/tmp', default={})
+        watch['url'] = 'https://example.com'
+        watch['processor'] = 'whois'
+        
+        # Mock the processor registry's get_display_link function
+        with patch('changedetectionio.processors.processor_registry.get_display_link') as mock_get_display_link:
+            mock_get_display_link.return_value = "WHOIS - example.com"
+            
+            # The link property should use the customized link from the processor
+            assert watch.link == "WHOIS - example.com"
+            mock_get_display_link.assert_called_once_with(url='https://example.com', processor_name='whois')
+
+
 if __name__ == '__main__':
     unittest.main()

changedetectionio/tests/util.py

@@ -76,6 +76,14 @@ def set_more_modified_response():
     return None
 
 
+def set_empty_text_response():
+    test_return_data = """<html><body></body></html>"""
+
+    with open("test-datastore/endpoint-content.txt", "w") as f:
+        f.write(test_return_data)
+
+    return None
+
 def wait_for_notification_endpoint_output():
     '''Apprise can take a few seconds to fire'''
     #@todo - could check the apprise object directly instead of looking for this file
@@ -215,9 +223,10 @@ def live_server_setup(live_server):
     def test_method():
         return request.method
 
-    # Where we POST to as a notification
-    @live_server.app.route('/test_notification_endpoint', methods=['POST', 'GET'])
+    # Where we POST to as a notification, also use a space here to test URL escaping is OK across all tests that use this. ( #2868 )
+    @live_server.app.route('/test_notification endpoint', methods=['POST', 'GET'])
     def test_notification_endpoint():
+
         with open("test-datastore/notification.txt", "wb") as f:
             # Debug method, dump all POST to file also, used to prove #65
             data = request.stream.read()
@@ -235,8 +244,11 @@ def live_server_setup(live_server):
                 f.write(request.content_type)
 
         print("\n>> Test notification endpoint was hit.\n", data)
-        return "Text was set"
 
+        content = "Text was set"
+        status_code = request.args.get('status_code',200)
+        resp = make_response(content, status_code)
+        return resp
 
     # Just return the verb in the request
     @live_server.app.route('/test-basicauth', methods=['GET'])
@@ -273,15 +285,43 @@ def live_server_setup(live_server):
             <p id="remove">This text should be removed</p>
               <form onsubmit="event.preventDefault();">
             <!-- obfuscated text so that we dont accidentally get a false positive due to conversion of the source :) --->
-                <button name="test-button" onclick="getElementById('remove').remove();getElementById('some-content').innerHTML = atob('SSBzbWVsbCBKYXZhU2NyaXB0IGJlY2F1c2UgdGhlIGJ1dHRvbiB3YXMgcHJlc3NlZCE=')">Click here</button>
-                <div id=some-content></div>
+                <button name="test-button" onclick="
+                getElementById('remove').remove();
+                getElementById('some-content').innerHTML = atob('SSBzbWVsbCBKYXZhU2NyaXB0IGJlY2F1c2UgdGhlIGJ1dHRvbiB3YXMgcHJlc3NlZCE=');
+                getElementById('reflect-text').innerHTML = getElementById('test-input-text').value;
+                ">Click here</button>
+                
+                <div id="some-content"></div>
+                
                 <pre>
                 {header_text.lower()}
                 </pre>
-              </body>
+                
+                <br>
+                <!-- used for testing that the jinja2 compiled here --->
+                <input type="text" value="" id="test-input-text" /><br>
+                <div id="reflect-text">Waiting to reflect text from #test-input-text here</div>
+              </form>
+                
+           </body>
          </html>""", 200)
         resp.headers['Content-Type'] = 'text/html'
         return resp
 
     live_server.start()
 
+def get_index(client):
+    import inspect
+    # Get the caller's frame (parent function)
+    frame = inspect.currentframe()
+    caller_frame = frame.f_back  # Go back to the caller's frame
+    caller_name = caller_frame.f_code.co_name
+    caller_line = caller_frame.f_lineno
+
+    print(f"Called by: {caller_name}, Line: {caller_line}")
+
+    res = client.get(url_for("index"))
+    with open(f"test-datastore/index-{caller_name}-{caller_line}.html", 'wb') as f:
+        f.write(res.data)
+
+    return res

changedetectionio/tests/visualselector/test_fetch_data.py

@@ -2,14 +2,16 @@
 
 import os
 from flask import url_for
-from ..util import live_server_setup, wait_for_all_checks, extract_UUID_from_client
+from ..util import live_server_setup, wait_for_all_checks, get_index
 
-def test_setup(client, live_server, measure_memory_usage):
+def test_setup(client, live_server):
     live_server_setup(live_server)
 
 
 # Add a site in paused mode, add an invalid filter, we should still have visual selector data ready
 def test_visual_selector_content_ready(client, live_server, measure_memory_usage):
+    live_server.stop()
+    live_server.start()
 
     import os
     import json
@@ -27,7 +29,7 @@ def test_visual_selector_content_ready(client, live_server, measure_memory_usage
         follow_redirects=True
     )
     assert b"Watch added in Paused state, saving will unpause" in res.data
-    uuid = extract_UUID_from_client(client)
+    uuid = next(iter(live_server.app.config['DATASTORE'].data['watching']))
     res = client.post(
         url_for("edit_page", uuid=uuid, unpause_on_save=1),
         data={
@@ -54,15 +56,21 @@ def test_visual_selector_content_ready(client, live_server, measure_memory_usage
 
 
     assert os.path.isfile(os.path.join('test-datastore', uuid, 'last-screenshot.png')), "last-screenshot.png should exist"
-    assert os.path.isfile(os.path.join('test-datastore', uuid, 'elements.json')), "xpath elements.json data should exist"
+    assert os.path.isfile(os.path.join('test-datastore', uuid, 'elements.deflate')), "xpath elements.deflate data should exist"
 
     # Open it and see if it roughly looks correct
-    with open(os.path.join('test-datastore', uuid, 'elements.json'), 'r') as f:
-        json.load(f)
+    with open(os.path.join('test-datastore', uuid, 'elements.deflate'), 'rb') as f:
+        import zlib
+        compressed_data = f.read()
+        decompressed_data = zlib.decompress(compressed_data)
+        # See if any error was thrown
+        json_data = json.loads(decompressed_data.decode('utf-8'))
 
     # Attempt to fetch it via the web hook that the browser would use
     res = client.get(url_for('static_content', group='visual_selector_data', filename=uuid))
-    json.loads(res.data)
+    decompressed_data = zlib.decompress(res.data)
+    json_data = json.loads(decompressed_data.decode('utf-8'))
+    
     assert res.mimetype == 'application/json'
     assert res.status_code == 200
 
@@ -81,7 +89,9 @@ def test_visual_selector_content_ready(client, live_server, measure_memory_usage
 
 def test_basic_browserstep(client, live_server, measure_memory_usage):
 
-    #live_server_setup(live_server)
+    live_server.stop()
+    live_server.start()
+
     assert os.getenv('PLAYWRIGHT_DRIVER_URL'), "Needs PLAYWRIGHT_DRIVER_URL set for this test"
 
     test_url = url_for('test_interactive_html_endpoint', _external=True)
@@ -102,9 +112,13 @@ def test_basic_browserstep(client, live_server, measure_memory_usage):
             "url": test_url,
             "tags": "",
             'fetch_backend': "html_webdriver",
-            'browser_steps-0-operation': 'Click element',
-            'browser_steps-0-selector': 'button[name=test-button]',
-            'browser_steps-0-optional_value': '',
+            'browser_steps-0-operation': 'Enter text in field',
+            'browser_steps-0-selector': '#test-input-text',
+            # Should get set to the actual text (jinja2 rendered)
+            'browser_steps-0-optional_value': "Hello-Jinja2-{% now  'Europe/Berlin', '%Y-%m-%d' %}",
+            'browser_steps-1-operation': 'Click element',
+            'browser_steps-1-selector': 'button[name=test-button]',
+            'browser_steps-1-optional_value': '',
             # For now, cookies doesnt work in headers because it must be a full cookiejar object
             'headers': "testheader: yes\buser-agent: MyCustomAgent",
         },
@@ -113,7 +127,7 @@ def test_basic_browserstep(client, live_server, measure_memory_usage):
     assert b"unpaused" in res.data
     wait_for_all_checks(client)
 
-    uuid = extract_UUID_from_client(client)
+    uuid = next(iter(live_server.app.config['DATASTORE'].data['watching']))
     assert live_server.app.config['DATASTORE'].data['watching'][uuid].history_n >= 1, "Watch history had atleast 1 (everything fetched OK)"
 
     assert b"This text should be removed" not in res.data
@@ -126,13 +140,32 @@ def test_basic_browserstep(client, live_server, measure_memory_usage):
     assert b"This text should be removed" not in res.data
     assert b"I smell JavaScript because the button was pressed" in res.data
 
+    assert b'Hello-Jinja2-20' in res.data
+
     assert b"testheader: yes" in res.data
     assert b"user-agent: mycustomagent" in res.data
+    live_server.stop()
+
+def test_non_200_errors_report_browsersteps(client, live_server):
+
+    live_server.stop()
+    live_server.start()
 
     four_o_four_url =  url_for('test_endpoint', status_code=404, _external=True)
     four_o_four_url = four_o_four_url.replace('localhost.localdomain', 'cdio')
     four_o_four_url = four_o_four_url.replace('localhost', 'cdio')
 
+    res = client.post(
+        url_for("form_quick_watch_add"),
+        data={"url": four_o_four_url, "tags": '', 'edit_and_watch_submit_button': 'Edit > Watch'},
+        follow_redirects=True
+    )
+
+    assert b"Watch added in Paused state, saving will unpause" in res.data
+    assert os.getenv('PLAYWRIGHT_DRIVER_URL'), "Needs PLAYWRIGHT_DRIVER_URL set for this test"
+
+    uuid = next(iter(live_server.app.config['DATASTORE'].data['watching']))
+
     # now test for 404 errors
     res = client.post(
         url_for("edit_page", uuid=uuid, unpause_on_save=1),
@@ -147,12 +180,14 @@ def test_basic_browserstep(client, live_server, measure_memory_usage):
         follow_redirects=True
     )
     assert b"unpaused" in res.data
+
     wait_for_all_checks(client)
 
-    res = client.get(url_for("index"))
+    res = get_index(client)
+
     assert b'Error - 404' in res.data
 
     client.get(
         url_for("form_delete", uuid="all"),
         follow_redirects=True
-    )
+    )

changedetectionio/time_handler.py

@@ -0,0 +1,105 @@
+from datetime import timedelta, datetime
+from enum import IntEnum
+from zoneinfo import ZoneInfo
+
+
+class Weekday(IntEnum):
+    """Enumeration for days of the week."""
+    Monday = 0
+    Tuesday = 1
+    Wednesday = 2
+    Thursday = 3
+    Friday = 4
+    Saturday = 5
+    Sunday = 6
+
+
+def am_i_inside_time(
+        day_of_week: str,
+        time_str: str,
+        timezone_str: str,
+        duration: int = 15,
+) -> bool:
+    """
+    Determines if the current time falls within a specified time range.
+
+    Parameters:
+        day_of_week (str): The day of the week (e.g., 'Monday').
+        time_str (str): The start time in 'HH:MM' format.
+        timezone_str (str): The timezone identifier (e.g., 'Europe/Berlin').
+        duration (int, optional): The duration of the time range in minutes. Default is 15.
+
+    Returns:
+        bool: True if the current time is within the time range, False otherwise.
+    """
+    # Parse the target day of the week
+    try:
+        target_weekday = Weekday[day_of_week.capitalize()]
+    except KeyError:
+        raise ValueError(f"Invalid day_of_week: '{day_of_week}'. Must be a valid weekday name.")
+
+    # Parse the start time
+    try:
+        target_time = datetime.strptime(time_str, '%H:%M').time()
+    except ValueError:
+        raise ValueError(f"Invalid time_str: '{time_str}'. Must be in 'HH:MM' format.")
+
+    # Define the timezone
+    try:
+        tz = ZoneInfo(timezone_str)
+    except Exception:
+        raise ValueError(f"Invalid timezone_str: '{timezone_str}'. Must be a valid timezone identifier.")
+
+    # Get the current time in the specified timezone
+    now_tz = datetime.now(tz)
+
+    # Check if the current day matches the target day or overlaps due to duration
+    current_weekday = now_tz.weekday()
+    start_datetime_tz = datetime.combine(now_tz.date(), target_time, tzinfo=tz)
+
+    # Handle previous day's overlap
+    if target_weekday == (current_weekday - 1) % 7:
+        # Calculate start and end times for the overlap from the previous day
+        start_datetime_tz -= timedelta(days=1)
+        end_datetime_tz = start_datetime_tz + timedelta(minutes=duration)
+        if start_datetime_tz <= now_tz < end_datetime_tz:
+            return True
+
+    # Handle current day's range
+    if target_weekday == current_weekday:
+        end_datetime_tz = start_datetime_tz + timedelta(minutes=duration)
+        if start_datetime_tz <= now_tz < end_datetime_tz:
+            return True
+
+    # Handle next day's overlap
+    if target_weekday == (current_weekday + 1) % 7:
+        end_datetime_tz = start_datetime_tz + timedelta(minutes=duration)
+        if now_tz < start_datetime_tz and now_tz + timedelta(days=1) < end_datetime_tz:
+            return True
+
+    return False
+
+
+def is_within_schedule(time_schedule_limit, default_tz="UTC"):
+    if time_schedule_limit and time_schedule_limit.get('enabled'):
+        # Get the timezone the time schedule is in, so we know what day it is there
+        tz_name = time_schedule_limit.get('timezone')
+        if not tz_name:
+            tz_name = default_tz
+
+        now_day_name_in_tz = datetime.now(ZoneInfo(tz_name.strip())).strftime('%A')
+        selected_day_schedule = time_schedule_limit.get(now_day_name_in_tz.lower())
+        if not selected_day_schedule.get('enabled'):
+            return False
+
+        duration = selected_day_schedule.get('duration')
+        selected_day_run_duration_m = int(duration.get('hours')) * 60 + int(duration.get('minutes'))
+
+        is_valid = am_i_inside_time(day_of_week=now_day_name_in_tz,
+                                    time_str=selected_day_schedule['start_time'],
+                                    timezone_str=tz_name,
+                                    duration=selected_day_run_duration_m)
+
+        return is_valid
+
+    return False