Adding basic TZ info output

.dockerignore

@@ -1,31 +1,18 @@
-# Git
-.git/
-.gitignore
+.git
+.github
+changedetectionio/processors/__pycache__
+changedetectionio/api/__pycache__
+changedetectionio/model/__pycache__
+changedetectionio/blueprint/price_data_follower/__pycache__
+changedetectionio/blueprint/tags/__pycache__
+changedetectionio/blueprint/__pycache__
+changedetectionio/blueprint/browser_steps/__pycache__
+changedetectionio/fetchers/__pycache__
+changedetectionio/tests/visualselector/__pycache__
+changedetectionio/tests/restock/__pycache__
+changedetectionio/tests/__pycache__
+changedetectionio/tests/fetchers/__pycache__
+changedetectionio/tests/unit/__pycache__
+changedetectionio/tests/proxy_list/__pycache__
+changedetectionio/__pycache__
 
-# GitHub
-.github/
-
-# Byte-compiled / optimized / DLL files
-**/__pycache__
-**/*.py[cod]
-
-# Caches
-.mypy_cache/
-.pytest_cache/
-.ruff_cache/
-
-# Distribution / packaging
-build/
-dist/
-*.egg-info*
-
-# Virtual environment
-.env
-.venv/
-venv/
-
-# IntelliJ IDEA
-.idea/
-
-# Visual Studio
-.vscode/

.github/test/Dockerfile-alpine

@@ -2,33 +2,32 @@
 # Test that we can still build on Alpine (musl modified libc https://musl.libc.org/)
 # Some packages wont install via pypi because they dont have a wheel available under this architecture.
 
-FROM ghcr.io/linuxserver/baseimage-alpine:3.21
+FROM ghcr.io/linuxserver/baseimage-alpine:3.18
 ENV PYTHONUNBUFFERED=1
 
 COPY requirements.txt /requirements.txt
 
 RUN \
- apk add --update --no-cache --virtual=build-dependencies \
-    build-base \
+  apk add --update --no-cache --virtual=build-dependencies \
     cargo \
-    git \
+    g++ \
+    gcc \
     jpeg-dev \
     libc-dev \
     libffi-dev \
+    libjpeg \
     libxslt-dev \
+    make \
     openssl-dev \
+    py3-wheel \
     python3-dev \
-    zip \
     zlib-dev && \
   apk add --update --no-cache \
-    libjpeg \
     libxslt \
-    nodejs \
-    poppler-utils \
-    python3 && \
+    python3 \
+    py3-pip && \
   echo "**** pip3 install test of changedetection.io ****" && \
-  python3 -m venv /lsiopy  && \
-  pip install -U pip wheel setuptools && \
-  pip install -U --no-cache-dir --find-links https://wheel-index.linuxserver.io/alpine-3.21/ -r /requirements.txt && \
+  pip3 install -U pip wheel setuptools && \
+  pip3 install -U --no-cache-dir --find-links https://wheel-index.linuxserver.io/alpine-3.18/ -r /requirements.txt && \
   apk del --purge \
     build-dependencies

.github/workflows/containers.yml

@@ -103,19 +103,6 @@ jobs:
 #          provenance: false
 
       # A new tagged release is required, which builds :tag and :latest
-      - name: Docker meta :tag
-        if: github.event_name == 'release' && startsWith(github.event.release.tag_name, '0.')
-        uses: docker/metadata-action@v5
-        id: meta
-        with:
-            images: |
-                ${{ secrets.DOCKER_HUB_USERNAME }}/changedetection.io
-                ghcr.io/dgtlmoon/changedetection.io
-            tags: |
-                type=semver,pattern={{version}}
-                type=semver,pattern={{major}}.{{minor}}
-                type=semver,pattern={{major}}
-
       - name: Build and push :tag
         id: docker_build_tag_release
         if: github.event_name == 'release' && startsWith(github.event.release.tag_name, '0.')
@@ -124,7 +111,11 @@ jobs:
           context: ./
           file: ./Dockerfile
           push: true
-          tags: ${{ steps.meta.outputs.tags }}
+          tags: |
+            ${{ secrets.DOCKER_HUB_USERNAME }}/changedetection.io:${{ github.event.release.tag_name }}
+            ghcr.io/dgtlmoon/changedetection.io:${{ github.event.release.tag_name }}
+            ${{ secrets.DOCKER_HUB_USERNAME }}/changedetection.io:latest
+            ghcr.io/dgtlmoon/changedetection.io:latest
           platforms: linux/amd64,linux/arm64,linux/arm/v7,linux/arm/v8,linux/arm64/v8
           cache-from: type=gha
           cache-to: type=gha,mode=max

.github/workflows/pypi-release.yml

@@ -45,12 +45,9 @@ jobs:
     - name: Test that the basic pip built package runs without error
       run: |
         set -ex
-        ls -alR 
-        
-        # Find and install the first .whl file
-        find dist -type f -name "*.whl" -exec pip3 install {} \; -quit
+        sudo pip3 install --upgrade pip 
+        pip3 install dist/changedetection.io*.whl
         changedetection.io -d /tmp -p 10000 &
-        
         sleep 3
         curl --retry-connrefused --retry 6 http://127.0.0.1:10000/static/styles/pure-min.css >/dev/null
         curl --retry-connrefused --retry 6 http://127.0.0.1:10000/ >/dev/null

.github/workflows/test-only.yml

@@ -37,10 +37,3 @@ jobs:
       python-version: '3.12'
       skip-pypuppeteer: true
 
-  test-application-3-13:
-    needs: lint-code
-    uses: ./.github/workflows/test-stack-reusable-workflow.yml
-    with:
-      python-version: '3.13'
-      skip-pypuppeteer: true
-      

.github/workflows/test-stack-reusable-workflow.yml

@@ -64,16 +64,14 @@ jobs:
           echo "Running processes in docker..."
           docker ps
 
-      - name: Run Unit Tests
+      - name: Test built container with Pytest (generally as requests/plaintext fetching)
         run: |
           # Unit tests
+          echo "run test with unittest"
           docker run test-changedetectionio  bash -c 'python3 -m unittest changedetectionio.tests.unit.test_notification_diff'
           docker run test-changedetectionio  bash -c 'python3 -m unittest changedetectionio.tests.unit.test_watch_model'
           docker run test-changedetectionio  bash -c 'python3 -m unittest changedetectionio.tests.unit.test_jinja2_security'
-          docker run test-changedetectionio  bash -c 'python3 -m unittest changedetectionio.tests.unit.test_semver'
-
-      - name: Test built container with Pytest (generally as requests/plaintext fetching)
-        run: |
+          
           # All tests
           echo "run test with pytest"
           # The default pytest logger_level is TRACE

.gitignore

@@ -1,29 +1,15 @@
-# Byte-compiled / optimized / DLL files
-**/__pycache__
-**/*.py[cod]
-
-# Caches
-.mypy_cache/
-.pytest_cache/
-.ruff_cache/
-
-# Distribution / packaging
-build/
-dist/
-*.egg-info*
-
-# Virtual environment
-.env
-.venv/
-venv/
-
-# IDEs
+__pycache__
 .idea
-.vscode/settings.json
-
-# Datastore files
-datastore/
-test-datastore/
-
-# Memory consumption log
+*.pyc
+datastore/url-watches.json
+datastore/*
+__pycache__
+.pytest_cache
+build
+dist
+venv
+test-datastore/*
+test-datastore
 test-memory.log
+*.egg-info*
+.vscode/settings.json

Dockerfile

@@ -32,7 +32,7 @@ RUN pip install --extra-index-url https://www.piwheels.org/simple  --target=/dep
 # Playwright is an alternative to Selenium
 # Excluded this package from requirements.txt to prevent arm/v6 and arm/v7 builds from failing
 # https://github.com/dgtlmoon/changedetection.io/pull/1067 also musl/alpine (not supported)
-RUN pip install --target=/dependencies playwright~=1.48.0 \
+RUN pip install --target=/dependencies playwright~=1.41.2 \
     || echo "WARN: Failed to install Playwright. The application can still run, but the Playwright option will be disabled."
 
 # Final image stage

README.md

@@ -105,22 +105,13 @@ We [recommend and use Bright Data](https://brightdata.grsm.io/n0r16zf7eivq) glob
 
 Please :star: star :star: this project and help it grow! https://github.com/dgtlmoon/changedetection.io/
 
-### Schedule web page watches in any timezone, limit by day of week and time.
-
-Easily set a re-check schedule, for example you could limit the web page change detection to only operate during business hours.
-Or perhaps based on a foreign timezone (for example, you want to check for the latest news-headlines in a foreign country at 0900 AM),
-
-<img src="./docs/scheduler.png" style="max-width:80%;" alt="How to monitor web page changes according to a schedule"  title="How to monitor web page changes according to a schedule"  />
-
-Includes quick short-cut buttons to setup a schedule for **business hours only**, or **weekends**.
-
 ### We have a Chrome extension!
 
 Easily add the current web page to your changedetection.io tool, simply install the extension and click "Sync" to connect it to your existing changedetection.io install.
 
 [<img src="./docs/chrome-extension-screenshot.png" style="max-width:80%;" alt="Chrome Extension to easily add the current web-page to detect a change."  title="Chrome Extension to easily add the current web-page to detect a change."  />](https://chromewebstore.google.com/detail/changedetectionio-website/kefcfmgmlhmankjmnbijimhofdjekbop)
 
-[Goto the Chrome Webstore to download the extension.](https://chromewebstore.google.com/detail/changedetectionio-website/kefcfmgmlhmankjmnbijimhofdjekbop) ( Or check out the [GitHub repo](https://github.com/dgtlmoon/changedetection.io-browser-extension) ) 
+[Goto the Chrome Webstore to download the extension.](https://chromewebstore.google.com/detail/changedetectionio-website/kefcfmgmlhmankjmnbijimhofdjekbop)
 
 ## Installation
 

changedetectionio/__init__.py

@@ -2,7 +2,7 @@
 
 # Read more https://github.com/dgtlmoon/changedetection.io/wiki
 
-__version__ = '0.49.3'
+__version__ = '0.47.06'
 
 from changedetectionio.strtobool import strtobool
 from json.decoder import JSONDecodeError
@@ -24,9 +24,6 @@ from loguru import logger
 app = None
 datastore = None
 
-def get_version():
-    return __version__
-
 # Parent wrapper or OS sends us a SIGTERM/SIGINT, do everything required for a clean shutdown
 def sigshutdown_handler(_signo, _stack_frame):
     global app
@@ -163,10 +160,11 @@ def main():
                     )
 
     # Monitored websites will not receive a Referer header when a user clicks on an outgoing link.
+    # @Note: Incompatible with password login (and maybe other features) for now, submit a PR!
     @app.after_request
     def hide_referrer(response):
         if strtobool(os.getenv("HIDE_REFERER", 'false')):
-            response.headers["Referrer-Policy"] = "same-origin"
+            response.headers["Referrer-Policy"] = "no-referrer"
 
         return response
 

changedetectionio/api/api_schema.py

@@ -112,35 +112,6 @@ def build_watch_json_schema(d):
 
     schema['properties']['time_between_check'] = build_time_between_check_json_schema()
 
-    schema['properties']['browser_steps'] = {
-        "anyOf": [
-            {
-                "type": "array",
-                "items": {
-                    "type": "object",
-                    "properties": {
-                        "operation": {
-                            "type": ["string", "null"],
-                            "maxLength": 5000  # Allows null and any string up to 5000 chars (including "")
-                        },
-                        "selector": {
-                            "type": ["string", "null"],
-                            "maxLength": 5000
-                        },
-                        "optional_value": {
-                            "type": ["string", "null"],
-                            "maxLength": 5000
-                        }
-                    },
-                    "required": ["operation", "selector", "optional_value"],
-                    "additionalProperties": False  # No extra keys allowed
-                }
-            },
-            {"type": "null"},  # Allows null for `browser_steps`
-            {"type": "array", "maxItems": 0}  # Allows empty array []
-        ]
-    }
-
     # headers ?
     return schema
 

changedetectionio/api/api_v1.py

@@ -76,7 +76,6 @@ class Watch(Resource):
         # Return without history, get that via another API call
         # Properties are not returned as a JSON, so add the required props manually
         watch['history_n'] = watch.history_n
-        # attr .last_changed will check for the last written text snapshot on change
         watch['last_changed'] = watch.last_changed
         watch['viewed'] = watch.viewed
         return watch

changedetectionio/apprise_asset.py

@@ -1,4 +1,3 @@
-from changedetectionio import apprise_plugin
 import apprise
 
 # Create our AppriseAsset and populate it with some of our new values:

changedetectionio/apprise_plugin/__init__.py

@@ -1,8 +1,6 @@
 # include the decorator
 from apprise.decorators import notify
 from loguru import logger
-from requests.structures import CaseInsensitiveDict
-
 
 @notify(on="delete")
 @notify(on="deletes")
@@ -15,84 +13,70 @@ from requests.structures import CaseInsensitiveDict
 def apprise_custom_api_call_wrapper(body, title, notify_type, *args, **kwargs):
     import requests
     import json
-    import re
-
     from urllib.parse import unquote_plus
-    from apprise.utils.parse import parse_url as apprise_parse_url
+    from apprise.utils import parse_url as apprise_parse_url
+    from apprise import URLBase
 
     url = kwargs['meta'].get('url')
-    schema = kwargs['meta'].get('schema').lower().strip()
 
-    # Choose POST, GET etc from requests
-    method =  re.sub(rf's$', '', schema)
-    requests_method = getattr(requests, method)
+    if url.startswith('post'):
+        r = requests.post
+    elif url.startswith('get'):
+        r = requests.get
+    elif url.startswith('put'):
+        r = requests.put
+    elif url.startswith('delete'):
+        r = requests.delete
 
-    params = CaseInsensitiveDict({}) # Added to requests
+    url = url.replace('post://', 'http://')
+    url = url.replace('posts://', 'https://')
+    url = url.replace('put://', 'http://')
+    url = url.replace('puts://', 'https://')
+    url = url.replace('get://', 'http://')
+    url = url.replace('gets://', 'https://')
+    url = url.replace('put://', 'http://')
+    url = url.replace('puts://', 'https://')
+    url = url.replace('delete://', 'http://')
+    url = url.replace('deletes://', 'https://')
+
+    headers = {}
+    params = {}
     auth = None
-    has_error = False
 
     # Convert /foobar?+some-header=hello to proper header dictionary
     results = apprise_parse_url(url)
+    if results:
+        # Add our headers that the user can potentially over-ride if they wish
+        # to to our returned result set and tidy entries by unquoting them
+        headers = {unquote_plus(x): unquote_plus(y)
+                   for x, y in results['qsd+'].items()}
 
-    # Add our headers that the user can potentially over-ride if they wish
-    # to to our returned result set and tidy entries by unquoting them
-    headers = CaseInsensitiveDict({unquote_plus(x): unquote_plus(y)
-               for x, y in results['qsd+'].items()})
+        # https://github.com/caronc/apprise/wiki/Notify_Custom_JSON#get-parameter-manipulation
+        # In Apprise, it relies on prefixing each request arg with "-", because it uses say &method=update as a flag for apprise
+        # but here we are making straight requests, so we need todo convert this against apprise's logic
+        for k, v in results['qsd'].items():
+            if not k.strip('+-') in results['qsd+'].keys():
+                params[unquote_plus(k)] = unquote_plus(v)
 
-    # https://github.com/caronc/apprise/wiki/Notify_Custom_JSON#get-parameter-manipulation
-    # In Apprise, it relies on prefixing each request arg with "-", because it uses say &method=update as a flag for apprise
-    # but here we are making straight requests, so we need todo convert this against apprise's logic
-    for k, v in results['qsd'].items():
-        if not k.strip('+-') in results['qsd+'].keys():
-            params[unquote_plus(k)] = unquote_plus(v)
+        # Determine Authentication
+        auth = ''
+        if results.get('user') and results.get('password'):
+            auth = (unquote_plus(results.get('user')), unquote_plus(results.get('user')))
+        elif results.get('user'):
+            auth = (unquote_plus(results.get('user')))
 
-    # Determine Authentication
-    auth = ''
-    if results.get('user') and results.get('password'):
-        auth = (unquote_plus(results.get('user')), unquote_plus(results.get('user')))
-    elif results.get('user'):
-        auth = (unquote_plus(results.get('user')))
-
-    # If it smells like it could be JSON and no content-type was already set, offer a default content type.
-    if body and '{' in body[:100] and not headers.get('Content-Type'):
-        json_header = 'application/json; charset=utf-8'
-        try:
-            # Try if it's JSON
-            json.loads(body)
-            headers['Content-Type'] = json_header
-        except ValueError as e:
-            logger.warning(f"Could not automatically add '{json_header}' header to the notification because the document failed to parse as JSON: {e}")
-            pass
-
-    # POSTS -> HTTPS etc
-    if schema.lower().endswith('s'):
-        url = re.sub(rf'^{schema}', 'https', results.get('url'))
-    else:
-        url = re.sub(rf'^{schema}', 'http', results.get('url'))
-
-    status_str = ''
+    # Try to auto-guess if it's JSON
+    h = 'application/json; charset=utf-8'
     try:
-        r = requests_method(url,
-          auth=auth,
-          data=body.encode('utf-8') if type(body) is str else body,
-          headers=headers,
-          params=params
-        )
+        json.loads(body)
+        headers['Content-Type'] = h
+    except ValueError as e:
+        logger.warning(f"Could not automatically add '{h}' header to the {kwargs['meta'].get('schema')}:// notification because the document failed to parse as JSON: {e}")
+        pass
 
-        if not (200 <= r.status_code < 300):
-            status_str = f"Error sending '{method.upper()}' request to {url} - Status: {r.status_code}: '{r.reason}'"
-            logger.error(status_str)
-            has_error = True
-        else:
-            logger.info(f"Sent '{method.upper()}' request to {url}")
-            has_error = False
-
-    except requests.RequestException as e:
-        status_str = f"Error sending '{method.upper()}' request to {url} - {str(e)}"
-        logger.error(status_str)
-        has_error = True
-
-    if has_error:
-        raise TypeError(status_str)
-
-    return True
+    r(results.get('url'),
+      auth=auth,
+      data=body.encode('utf-8') if type(body) is str else body,
+      headers=headers,
+      params=params
+      )

changedetectionio/blueprint/browser_steps/__init__.py

@@ -22,10 +22,7 @@ from loguru import logger
 
 browsersteps_sessions = {}
 io_interface_context = None
-import json
-import base64
-import hashlib
-from flask import Response
+
 
 def construct_blueprint(datastore: ChangeDetectionStore):
     browser_steps_blueprint = Blueprint('browser_steps', __name__, template_folder="templates")
@@ -163,13 +160,14 @@ def construct_blueprint(datastore: ChangeDetectionStore):
         if not browsersteps_sessions.get(browsersteps_session_id):
             return make_response('No session exists under that ID', 500)
 
-        is_last_step = False
+
         # Actions - step/apply/etc, do the thing and return state
         if request.method == 'POST':
             # @todo - should always be an existing session
             step_operation = request.form.get('operation')
             step_selector = request.form.get('selector')
             step_optional_value = request.form.get('optional_value')
+            step_n = int(request.form.get('step_n'))
             is_last_step = strtobool(request.form.get('is_last_step'))
 
             # @todo try.. accept.. nice errors not popups..
@@ -184,6 +182,16 @@ def construct_blueprint(datastore: ChangeDetectionStore):
                 # Try to find something of value to give back to the user
                 return make_response(str(e).splitlines()[0], 401)
 
+            # Get visual selector ready/update its data (also use the current filter info from the page?)
+            # When the last 'apply' button was pressed
+            # @todo this adds overhead because the xpath selection is happening twice
+            u = browsersteps_sessions[browsersteps_session_id]['browserstepper'].page.url
+            if is_last_step and u:
+                (screenshot, xpath_data) = browsersteps_sessions[browsersteps_session_id]['browserstepper'].request_visualselector_data()
+                watch = datastore.data['watching'].get(uuid)
+                if watch:
+                    watch.save_screenshot(screenshot=screenshot)
+                    watch.save_xpath_data(data=xpath_data)
 
 #        if not this_session.page:
 #            cleanup_playwright_session()
@@ -191,35 +199,31 @@ def construct_blueprint(datastore: ChangeDetectionStore):
 
         # Screenshots and other info only needed on requesting a step (POST)
         try:
-            (screenshot, xpath_data) = browsersteps_sessions[browsersteps_session_id]['browserstepper'].get_current_state()
-            if is_last_step:
-                watch = datastore.data['watching'].get(uuid)
-                u = browsersteps_sessions[browsersteps_session_id]['browserstepper'].page.url
-                if watch and u:
-                    watch.save_screenshot(screenshot=screenshot)
-                    watch.save_xpath_data(data=xpath_data)
-
+            state = browsersteps_sessions[browsersteps_session_id]['browserstepper'].get_current_state()
         except playwright._impl._api_types.Error as e:
             return make_response("Browser session ran out of time :( Please reload this page."+str(e), 401)
-        except Exception as e:
-            return make_response("Error fetching screenshot and element data - " + str(e), 401)
 
-        # SEND THIS BACK TO THE BROWSER
+        # Use send_file() which is way faster than read/write loop on bytes
+        import json
+        from tempfile import mkstemp
+        from flask import send_file
+        tmp_fd, tmp_file = mkstemp(text=True, suffix=".json", prefix="changedetectionio-")
 
-        output = {
-            "screenshot": f"data:image/jpeg;base64,{base64.b64encode(screenshot).decode('ascii')}",
-            "xpath_data": xpath_data,
-            "session_age_start": browsersteps_sessions[browsersteps_session_id]['browserstepper'].age_start,
-            "browser_time_remaining": round(remaining)
-        }
-        json_data = json.dumps(output)
+        output = json.dumps({'screenshot': "data:image/jpeg;base64,{}".format(
+            base64.b64encode(state[0]).decode('ascii')),
+            'xpath_data': state[1],
+            'session_age_start': browsersteps_sessions[browsersteps_session_id]['browserstepper'].age_start,
+            'browser_time_remaining': round(remaining)
+        })
 
-        # Generate an ETag (hash of the response body)
-        etag_hash = hashlib.md5(json_data.encode('utf-8')).hexdigest()
+        with os.fdopen(tmp_fd, 'w') as f:
+            f.write(output)
 
-        # Create the response with ETag
-        response = Response(json_data, mimetype="application/json; charset=UTF-8")
-        response.set_etag(etag_hash)
+        response = make_response(send_file(path_or_file=tmp_file,
+                                           mimetype='application/json; charset=UTF-8',
+                                           etag=True))
+        # No longer needed
+        os.unlink(tmp_file)
 
         return response
 

changedetectionio/blueprint/browser_steps/browser_steps.py

@@ -1,15 +1,14 @@
+#!/usr/bin/env python3
+
 import os
 import time
 import re
 from random import randint
 from loguru import logger
 
-from changedetectionio.content_fetchers.helpers import capture_stitched_together_full_page, SCREENSHOT_SIZE_STITCH_THRESHOLD
 from changedetectionio.content_fetchers.base import manage_user_agent
 from changedetectionio.safe_jinja import render as jinja_render
 
-
-
 # Two flags, tell the JS which of the "Selector" or "Value" field should be enabled in the front end
 # 0- off, 1- on
 browser_step_ui_config = {'Choose one': '0 0',
@@ -32,7 +31,6 @@ browser_step_ui_config = {'Choose one': '0 0',
 #                          'Extract text and use as filter': '1 0',
                           'Goto site': '0 0',
                           'Goto URL': '0 1',
-                          'Make all child elements visible': '1 0',
                           'Press Enter': '0 0',
                           'Select by label': '1 1',
                           'Scroll down': '0 0',
@@ -40,7 +38,6 @@ browser_step_ui_config = {'Choose one': '0 0',
                           'Wait for seconds': '0 1',
                           'Wait for text': '0 1',
                           'Wait for text in element': '1 1',
-                          'Remove elements': '1 0',
                           #                          'Press Page Down': '0 0',
                           #                          'Press Page Up': '0 0',
                           # weird bug, come back to it later
@@ -55,8 +52,6 @@ class steppable_browser_interface():
     page = None
     start_url = None
 
-    action_timeout = 10 * 1000
-
     def __init__(self, start_url):
         self.start_url = start_url
 
@@ -107,7 +102,7 @@ class steppable_browser_interface():
             return
         elem = self.page.get_by_text(value)
         if elem.count():
-            elem.first.click(delay=randint(200, 500), timeout=self.action_timeout)
+            elem.first.click(delay=randint(200, 500), timeout=3000)
 
     def action_click_element_containing_text_if_exists(self, selector=None, value=''):
         logger.debug("Clicking element containing text if exists")
@@ -116,7 +111,7 @@ class steppable_browser_interface():
         elem = self.page.get_by_text(value)
         logger.debug(f"Clicking element containing text - {elem.count()} elements found")
         if elem.count():
-            elem.first.click(delay=randint(200, 500), timeout=self.action_timeout)
+            elem.first.click(delay=randint(200, 500), timeout=3000)
         else:
             return
 
@@ -124,7 +119,7 @@ class steppable_browser_interface():
         if not len(selector.strip()):
             return
 
-        self.page.fill(selector, value, timeout=self.action_timeout)
+        self.page.fill(selector, value, timeout=10 * 1000)
 
     def action_execute_js(self, selector, value):
         response = self.page.evaluate(value)
@@ -135,7 +130,7 @@ class steppable_browser_interface():
         if not len(selector.strip()):
             return
 
-        self.page.click(selector=selector, timeout=self.action_timeout + 20 * 1000, delay=randint(200, 500))
+        self.page.click(selector=selector, timeout=30 * 1000, delay=randint(200, 500))
 
     def action_click_element_if_exists(self, selector, value):
         import playwright._impl._errors as _api_types
@@ -143,7 +138,7 @@ class steppable_browser_interface():
         if not len(selector.strip()):
             return
         try:
-            self.page.click(selector, timeout=self.action_timeout, delay=randint(200, 500))
+            self.page.click(selector, timeout=10 * 1000, delay=randint(200, 500))
         except _api_types.TimeoutError as e:
             return
         except _api_types.Error as e:
@@ -190,29 +185,11 @@ class steppable_browser_interface():
         self.page.keyboard.press("PageDown", delay=randint(200, 500))
 
     def action_check_checkbox(self, selector, value):
-        self.page.locator(selector).check(timeout=self.action_timeout)
+        self.page.locator(selector).check(timeout=1000)
 
     def action_uncheck_checkbox(self, selector, value):
-        self.page.locator(selector).uncheck(timeout=self.action_timeout)
+        self.page.locator(selector, timeout=1000).uncheck(timeout=1000)
 
-    def action_remove_elements(self, selector, value):
-        """Removes all elements matching the given selector from the DOM."""
-        self.page.locator(selector).evaluate_all("els => els.forEach(el => el.remove())")
-
-    def action_make_all_child_elements_visible(self, selector, value):
-        """Recursively makes all child elements inside the given selector fully visible."""
-        self.page.locator(selector).locator("*").evaluate_all("""
-            els => els.forEach(el => {
-                el.style.display = 'block';   // Forces it to be displayed
-                el.style.visibility = 'visible';   // Ensures it's not hidden
-                el.style.opacity = '1';   // Fully opaque
-                el.style.position = 'relative';   // Avoids 'absolute' hiding
-                el.style.height = 'auto';   // Expands collapsed elements
-                el.style.width = 'auto';   // Ensures full visibility
-                el.removeAttribute('hidden');   // Removes hidden attribute
-                el.classList.remove('hidden', 'd-none');  // Removes common CSS hidden classes
-            })
-        """)
 
 # Responsible for maintaining a live 'context' with the chrome CDP
 # @todo - how long do contexts live for anyway?
@@ -280,7 +257,6 @@ class browsersteps_live_ui(steppable_browser_interface):
         logger.debug(f"Time to browser setup {time.time()-now:.2f}s")
         self.page.wait_for_timeout(1 * 1000)
 
-
     def mark_as_closed(self):
         logger.debug("Page closed, cleaning up..")
 
@@ -298,30 +274,39 @@ class browsersteps_live_ui(steppable_browser_interface):
         now = time.time()
         self.page.wait_for_timeout(1 * 1000)
 
+        # The actual screenshot
+        screenshot = self.page.screenshot(type='jpeg', full_page=True, quality=40)
 
-        full_height = self.page.evaluate("document.documentElement.scrollHeight")
-
-        if full_height >= SCREENSHOT_SIZE_STITCH_THRESHOLD:
-            logger.warning(f"Page full Height: {full_height}px longer than {SCREENSHOT_SIZE_STITCH_THRESHOLD}px, using 'stitched screenshot method'.")
-            screenshot = capture_stitched_together_full_page(self.page)
-        else:
-            screenshot = self.page.screenshot(type='jpeg', full_page=True, quality=40)
-
-        logger.debug(f"Time to get screenshot from browser {time.time() - now:.2f}s")
-
-        now = time.time()
         self.page.evaluate("var include_filters=''")
         # Go find the interactive elements
         # @todo in the future, something smarter that can scan for elements with .click/focus etc event handlers?
         elements = 'a,button,input,select,textarea,i,th,td,p,li,h1,h2,h3,h4,div,span'
         xpath_element_js = xpath_element_js.replace('%ELEMENTS%', elements)
-
         xpath_data = self.page.evaluate("async () => {" + xpath_element_js + "}")
         # So the JS will find the smallest one first
         xpath_data['size_pos'] = sorted(xpath_data['size_pos'], key=lambda k: k['width'] * k['height'], reverse=True)
-        logger.debug(f"Time to scrape xpath element data in browser {time.time()-now:.2f}s")
-
+        logger.debug(f"Time to complete get_current_state of browser {time.time()-now:.2f}s")
+        # except
         # playwright._impl._api_types.Error: Browser closed.
         # @todo show some countdown timer?
         return (screenshot, xpath_data)
 
+    def request_visualselector_data(self):
+        """
+        Does the same that the playwright operation in content_fetcher does
+        This is used to just bump the VisualSelector data so it' ready to go if they click on the tab
+        @todo refactor and remove duplicate code, add include_filters
+        :param xpath_data:
+        :param screenshot:
+        :param current_include_filters:
+        :return:
+        """
+        import importlib.resources
+        self.page.evaluate("var include_filters=''")
+        xpath_element_js = importlib.resources.files("changedetectionio.content_fetchers.res").joinpath('xpath_element_scraper.js').read_text()
+        from changedetectionio.content_fetchers import visualselector_xpath_selectors
+        xpath_element_js = xpath_element_js.replace('%ELEMENTS%', visualselector_xpath_selectors)
+        xpath_data = self.page.evaluate("async () => {" + xpath_element_js + "}")
+        screenshot = self.page.screenshot(type='jpeg', full_page=True, quality=int(os.getenv("SCREENSHOT_QUALITY", 72)))
+
+        return (screenshot, xpath_data)

changedetectionio/blueprint/tags/__init__.py

@@ -13,7 +13,6 @@ def construct_blueprint(datastore: ChangeDetectionStore):
     def tags_overview_page():
         from .form import SingleTag
         add_form = SingleTag(request.form)
-
         sorted_tags = sorted(datastore.data['settings']['application'].get('tags').items(), key=lambda x: x[1]['title'])
 
         from collections import Counter
@@ -105,11 +104,9 @@ def construct_blueprint(datastore: ChangeDetectionStore):
 
         default = datastore.data['settings']['application']['tags'].get(uuid)
 
-        form = group_restock_settings_form(
-                                       formdata=request.form if request.method == 'POST' else None,
+        form = group_restock_settings_form(formdata=request.form if request.method == 'POST' else None,
                                        data=default,
-                                       extra_notification_tokens=datastore.get_unique_notification_tokens_available(),
-                                       default_system_settings = datastore.data['settings'],
+                                       extra_notification_tokens=datastore.get_unique_notification_tokens_available()
                                        )
 
         template_args = {

changedetectionio/content_fetchers/helpers.py

@@ -1,104 +0,0 @@
-
-# Pages with a vertical height longer than this will use the 'stitch together' method.
-
-# - Many GPUs have a max texture size of 16384x16384px (or lower on older devices).
-# - If a page is taller than ~8000–10000px, it risks exceeding GPU memory limits.
-# - This is especially important on headless Chromium, where Playwright may fail to allocate a massive full-page buffer.
-
-
-# The size at which we will switch to stitching method
-SCREENSHOT_SIZE_STITCH_THRESHOLD=8000
-
-from loguru import logger
-
-def capture_stitched_together_full_page(page):
-    import io
-    import os
-    import time
-    from PIL import Image, ImageDraw, ImageFont
-
-    MAX_TOTAL_HEIGHT = SCREENSHOT_SIZE_STITCH_THRESHOLD*4  # Maximum total height for the final image (When in stitch mode)
-    MAX_CHUNK_HEIGHT = 4000  # Height per screenshot chunk
-    WARNING_TEXT_HEIGHT = 20  # Height of the warning text overlay
-
-    # Save the original viewport size
-    original_viewport = page.viewport_size
-    now = time.time()
-
-    try:
-        viewport = page.viewport_size
-        page_height = page.evaluate("document.documentElement.scrollHeight")
-
-        # Limit the total capture height
-        capture_height = min(page_height, MAX_TOTAL_HEIGHT)
-
-        images = []
-        total_captured_height = 0
-
-        for offset in range(0, capture_height, MAX_CHUNK_HEIGHT):
-            # Ensure we do not exceed the total height limit
-            chunk_height = min(MAX_CHUNK_HEIGHT, MAX_TOTAL_HEIGHT - total_captured_height)
-
-            # Adjust viewport size for this chunk
-            page.set_viewport_size({"width": viewport["width"], "height": chunk_height})
-
-            # Scroll to the correct position
-            page.evaluate(f"window.scrollTo(0, {offset})")
-
-            # Capture screenshot chunk
-            screenshot_bytes = page.screenshot(type='jpeg', quality=int(os.getenv("SCREENSHOT_QUALITY", 30)))
-            images.append(Image.open(io.BytesIO(screenshot_bytes)))
-
-            total_captured_height += chunk_height
-
-            # Stop if we reached the maximum total height
-            if total_captured_height >= MAX_TOTAL_HEIGHT:
-                break
-
-        # Create the final stitched image
-        stitched_image = Image.new('RGB', (viewport["width"], total_captured_height))
-        y_offset = 0
-
-        # Stitch the screenshot chunks together
-        for img in images:
-            stitched_image.paste(img, (0, y_offset))
-            y_offset += img.height
-
-        logger.debug(f"Screenshot stitched together in {time.time()-now:.2f}s")
-
-        # Overlay warning text if the screenshot was trimmed
-        if page_height > MAX_TOTAL_HEIGHT:
-            draw = ImageDraw.Draw(stitched_image)
-            warning_text = f"WARNING: Screenshot was {page_height}px but trimmed to {MAX_TOTAL_HEIGHT}px because it was too long"
-
-            # Load font (default system font if Arial is unavailable)
-            try:
-                font = ImageFont.truetype("arial.ttf", WARNING_TEXT_HEIGHT)  # Arial (Windows/Mac)
-            except IOError:
-                font = ImageFont.load_default()  # Default font if Arial not found
-
-            # Get text bounding box (correct method for newer Pillow versions)
-            text_bbox = draw.textbbox((0, 0), warning_text, font=font)
-            text_width = text_bbox[2] - text_bbox[0]  # Calculate text width
-            text_height = text_bbox[3] - text_bbox[1]  # Calculate text height
-
-            # Define background rectangle (top of the image)
-            draw.rectangle([(0, 0), (viewport["width"], WARNING_TEXT_HEIGHT)], fill="white")
-
-            # Center text horizontally within the warning area
-            text_x = (viewport["width"] - text_width) // 2
-            text_y = (WARNING_TEXT_HEIGHT - text_height) // 2
-
-            # Draw the warning text in red
-            draw.text((text_x, text_y), warning_text, fill="red", font=font)
-
-        # Save or return the final image
-        output = io.BytesIO()
-        stitched_image.save(output, format="JPEG", quality=int(os.getenv("SCREENSHOT_QUALITY", 30)))
-        screenshot = output.getvalue()
-
-    finally:
-        # Restore the original viewport size
-        page.set_viewport_size(original_viewport)
-
-    return screenshot

changedetectionio/content_fetchers/playwright.py

@@ -4,7 +4,6 @@ from urllib.parse import urlparse
 
 from loguru import logger
 
-from changedetectionio.content_fetchers.helpers import capture_stitched_together_full_page, SCREENSHOT_SIZE_STITCH_THRESHOLD
 from changedetectionio.content_fetchers.base import Fetcher, manage_user_agent
 from changedetectionio.content_fetchers.exceptions import PageUnloadable, Non200ErrorCodeReceived, EmptyReply, ScreenshotUnavailable
 
@@ -90,7 +89,6 @@ class fetcher(Fetcher):
         from playwright.sync_api import sync_playwright
         import playwright._impl._errors
         from changedetectionio.content_fetchers import visualselector_xpath_selectors
-        import time
         self.delete_browser_steps_screenshots()
         response = None
 
@@ -181,7 +179,6 @@ class fetcher(Fetcher):
 
             self.page.wait_for_timeout(extra_wait * 1000)
 
-            now = time.time()
             # So we can find an element on the page where its selector was entered manually (maybe not xPath etc)
             if current_include_filters is not None:
                 self.page.evaluate("var include_filters={}".format(json.dumps(current_include_filters)))
@@ -193,8 +190,6 @@ class fetcher(Fetcher):
             self.instock_data = self.page.evaluate("async () => {" + self.instock_data_js + "}")
 
             self.content = self.page.content()
-            logger.debug(f"Time to scrape xpath element data in browser {time.time() - now:.2f}s")
-
             # Bug 3 in Playwright screenshot handling
             # Some bug where it gives the wrong screenshot size, but making a request with the clip set first seems to solve it
             # JPEG is better here because the screenshots can be very very large
@@ -204,15 +199,10 @@ class fetcher(Fetcher):
             # acceptable screenshot quality here
             try:
                 # The actual screenshot - this always base64 and needs decoding! horrible! huge CPU usage
-                full_height = self.page.evaluate("document.documentElement.scrollHeight")
-
-                if full_height >= SCREENSHOT_SIZE_STITCH_THRESHOLD:
-                    logger.warning(
-                        f"Page full Height: {full_height}px longer than {SCREENSHOT_SIZE_STITCH_THRESHOLD}px, using 'stitched screenshot method'.")
-                    self.screenshot = capture_stitched_together_full_page(self.page)
-                else:
-                    self.screenshot = self.page.screenshot(type='jpeg', full_page=True, quality=int(os.getenv("SCREENSHOT_QUALITY", 30)))
-
+                self.screenshot = self.page.screenshot(type='jpeg',
+                                                       full_page=True,
+                                                       quality=int(os.getenv("SCREENSHOT_QUALITY", 72)),
+                                                       )
             except Exception as e:
                 # It's likely the screenshot was too long/big and something crashed
                 raise ScreenshotUnavailable(url=url, status_code=self.status_code)

changedetectionio/content_fetchers/res/stock-not-in-stock.js

@@ -39,7 +39,6 @@ function isItemInStock() {
         'let me know when it\'s available',
         'mail me when available',
         'message if back in stock',
-        'mevcut değil',
         'nachricht bei',
         'nicht auf lager',
         'nicht lagernd',
@@ -51,8 +50,7 @@ function isItemInStock() {
         'niet beschikbaar',
         'niet leverbaar',
         'niet op voorraad',
-        'no disponible',
-        'non disponibile',
+        'no disponible temporalmente',
         'no longer in stock',
         'no tickets available',
         'not available',
@@ -69,14 +67,12 @@ function isItemInStock() {
         'produkt niedostępny',
         'sold out',
         'sold-out',
-        'stokta yok',
         'temporarily out of stock',
         'temporarily unavailable',
         'there were no search results for',
         'this item is currently unavailable',
         'tickets unavailable',
         'tijdelijk uitverkocht',
-        'tükendi',
         'unavailable nearby',
         'unavailable tickets',
         'vergriffen',

changedetectionio/content_fetchers/res/xpath_element_scraper.js

@@ -41,7 +41,7 @@ const findUpTag = (el) => {
 
     //  Strategy 1: If it's an input, with name, and there's only one, prefer that
     if (el.name !== undefined && el.name.length) {
-        var proposed = el.tagName + "[name=\"" + CSS.escape(el.name) + "\"]";
+        var proposed = el.tagName + "[name=" + el.name + "]";
         var proposed_element = window.document.querySelectorAll(proposed);
         if (proposed_element.length) {
             if (proposed_element.length === 1) {
@@ -102,15 +102,13 @@ function collectVisibleElements(parent, visibleElements) {
     const children = parent.children;
     for (let i = 0; i < children.length; i++) {
         const child = children[i];
-        const computedStyle = window.getComputedStyle(child);
-
         if (
             child.nodeType === Node.ELEMENT_NODE &&
-            computedStyle.display !== 'none' &&
-            computedStyle.visibility !== 'hidden' &&
+            window.getComputedStyle(child).display !== 'none' &&
+            window.getComputedStyle(child).visibility !== 'hidden' &&
             child.offsetWidth >= 0 &&
             child.offsetHeight >= 0 &&
-            computedStyle.contentVisibility !== 'hidden'
+            window.getComputedStyle(child).contentVisibility !== 'hidden'
         ) {
             // If the child is an element and is visible, recursively collect visible elements
             collectVisibleElements(child, visibleElements);
@@ -175,7 +173,6 @@ visibleElementsArray.forEach(function (element) {
 
     // Try to identify any possible currency amounts "Sale: 4000" or "Sale now 3000 Kc", can help with the training.
     const hasDigitCurrency = (/\d/.test(text.slice(0, 6)) || /\d/.test(text.slice(-6)) ) &&  /([€£$¥₩₹]|USD|AUD|EUR|Kč|kr|SEK|,–)/.test(text) ;
-    const computedStyle = window.getComputedStyle(element);
 
     size_pos.push({
         xpath: xpath_result,
@@ -187,10 +184,10 @@ visibleElementsArray.forEach(function (element) {
         tagName: (element.tagName) ? element.tagName.toLowerCase() : '',
         // tagtype used by Browser Steps
         tagtype: (element.tagName.toLowerCase() === 'input' && element.type) ? element.type.toLowerCase() : '',
-        isClickable: computedStyle.cursor === "pointer",
+        isClickable: window.getComputedStyle(element).cursor === "pointer",
         // Used by the keras trainer
-        fontSize: computedStyle.getPropertyValue('font-size'),
-        fontWeight: computedStyle.getPropertyValue('font-weight'),
+        fontSize: window.getComputedStyle(element).getPropertyValue('font-size'),
+        fontWeight: window.getComputedStyle(element).getPropertyValue('font-weight'),
         hasDigitCurrency: hasDigitCurrency,
         label: label,
     });

changedetectionio/diff.py

@@ -1,9 +1,6 @@
 import difflib
 from typing import List, Iterator, Union
 
-REMOVED_STYLE = "background-color: #fadad7; color: #b30000;"
-ADDED_STYLE = "background-color: #eaf2c2; color: #406619;"
-
 def same_slicer(lst: List[str], start: int, end: int) -> List[str]:
     """Return a slice of the list, or a single element if start == end."""
     return lst[start:end] if start != end else [lst[start]]
@@ -15,12 +12,11 @@ def customSequenceMatcher(
     include_removed: bool = True,
     include_added: bool = True,
     include_replaced: bool = True,
-    include_change_type_prefix: bool = True,
-    html_colour: bool = False
+    include_change_type_prefix: bool = True
 ) -> Iterator[List[str]]:
     """
     Compare two sequences and yield differences based on specified parameters.
-
+    
     Args:
         before (List[str]): Original sequence
         after (List[str]): Modified sequence
@@ -29,35 +25,26 @@ def customSequenceMatcher(
         include_added (bool): Include added parts
         include_replaced (bool): Include replaced parts
         include_change_type_prefix (bool): Add prefixes to indicate change types
-        html_colour (bool): Use HTML background colors for differences
-
+    
     Yields:
         List[str]: Differences between sequences
     """
     cruncher = difflib.SequenceMatcher(isjunk=lambda x: x in " \t", a=before, b=after)
-
-
-
+    
     for tag, alo, ahi, blo, bhi in cruncher.get_opcodes():
         if include_equal and tag == 'equal':
             yield before[alo:ahi]
         elif include_removed and tag == 'delete':
-            if html_colour:
-                yield [f'<span style="{REMOVED_STYLE}">{line}</span>' for line in same_slicer(before, alo, ahi)]
-            else:
-                yield [f"(removed) {line}" for line in same_slicer(before, alo, ahi)] if include_change_type_prefix else same_slicer(before, alo, ahi)
+            prefix = "(removed) " if include_change_type_prefix else ''
+            yield [f"{prefix}{line}" for line in same_slicer(before, alo, ahi)]
         elif include_replaced and tag == 'replace':
-            if html_colour:
-                yield [f'<span style="{REMOVED_STYLE}">{line}</span>' for line in same_slicer(before, alo, ahi)] + \
-                      [f'<span style="{ADDED_STYLE}">{line}</span>' for line in same_slicer(after, blo, bhi)]
-            else:
-                yield [f"(changed) {line}" for line in same_slicer(before, alo, ahi)] + \
-                      [f"(into) {line}" for line in same_slicer(after, blo, bhi)] if include_change_type_prefix else same_slicer(before, alo, ahi) + same_slicer(after, blo, bhi)
+            prefix_changed = "(changed) " if include_change_type_prefix else ''
+            prefix_into = "(into) " if include_change_type_prefix else ''
+            yield [f"{prefix_changed}{line}" for line in same_slicer(before, alo, ahi)] + \
+                  [f"{prefix_into}{line}" for line in same_slicer(after, blo, bhi)]
         elif include_added and tag == 'insert':
-            if html_colour:
-                yield [f'<span style="{ADDED_STYLE}">{line}</span>' for line in same_slicer(after, blo, bhi)]
-            else:
-                yield [f"(added) {line}" for line in same_slicer(after, blo, bhi)] if include_change_type_prefix else same_slicer(after, blo, bhi)
+            prefix = "(added) " if include_change_type_prefix else ''
+            yield [f"{prefix}{line}" for line in same_slicer(after, blo, bhi)]
 
 def render_diff(
     previous_version_file_contents: str,
@@ -68,12 +55,11 @@ def render_diff(
     include_replaced: bool = True,
     line_feed_sep: str = "\n",
     include_change_type_prefix: bool = True,
-    patch_format: bool = False,
-    html_colour: bool = False
+    patch_format: bool = False
 ) -> str:
     """
     Render the difference between two file contents.
-
+    
     Args:
         previous_version_file_contents (str): Original file contents
         newest_version_file_contents (str): Modified file contents
@@ -84,8 +70,7 @@ def render_diff(
         line_feed_sep (str): Separator for lines in output
         include_change_type_prefix (bool): Add prefixes to indicate change types
         patch_format (bool): Use patch format for output
-        html_colour (bool): Use HTML background colors for differences
-
+    
     Returns:
         str: Rendered difference
     """
@@ -103,11 +88,10 @@ def render_diff(
         include_removed=include_removed,
         include_added=include_added,
         include_replaced=include_replaced,
-        include_change_type_prefix=include_change_type_prefix,
-        html_colour=html_colour
+        include_change_type_prefix=include_change_type_prefix
     )
 
     def flatten(lst: List[Union[str, List[str]]]) -> str:
         return line_feed_sep.join(flatten(x) if isinstance(x, list) else x for x in lst)
 
-    return flatten(rendered_diff)
+    return flatten(rendered_diff)

changedetectionio/flask_app.py

@@ -1,7 +1,6 @@
 #!/usr/bin/env python3
 
 import datetime
-from zoneinfo import ZoneInfo
 
 import flask_login
 import locale
@@ -43,7 +42,6 @@ from loguru import logger
 from changedetectionio import html_tools, __version__
 from changedetectionio import queuedWatchMetaData
 from changedetectionio.api import api_v1
-from .time_handler import is_within_schedule
 
 datastore = None
 
@@ -161,6 +159,21 @@ def _jinja2_filter_pagination_slice(arr, skip):
 
     return arr
 
+def app_get_system_time():
+    from zoneinfo import ZoneInfo  # Built-in timezone support in Python 3.9+
+
+    system_timezone = datastore.data['settings']['application'].get('timezone')
+    if not system_timezone:
+        system_timezone = os.environ.get("TZ")
+
+    try:
+        system_zone = ZoneInfo(system_timezone)
+    except Exception as e:
+        logger.warning(f'Warning, unable to use timezone "{system_timezone}" defaulting to UTC- {str(e)}')
+        system_zone = ZoneInfo("UTC")  # Fallback to UTC if the timezone is invalid
+
+    return system_zone
+
 @app.template_filter('format_seconds_ago')
 def _jinja2_filter_seconds_precise(timestamp):
     if timestamp == False:
@@ -245,6 +258,9 @@ def changedetection_app(config=None, datastore_o=None):
     # (instead of the global var)
     app.config['DATASTORE'] = datastore_o
 
+    # Just to check (it will output some debug if not)
+    app_get_system_time()
+
     login_manager = flask_login.LoginManager(app)
     login_manager.login_view = 'login'
     app.secret_key = init_app_secret(config['datastore_path'])
@@ -598,31 +614,17 @@ def changedetection_app(config=None, datastore_o=None):
 
             if 'notification_title' in request.form and request.form['notification_title'].strip():
                 n_object['notification_title'] = request.form.get('notification_title', '').strip()
-            elif datastore.data['settings']['application'].get('notification_title'):
-                n_object['notification_title'] = datastore.data['settings']['application'].get('notification_title')
-            else:
-                n_object['notification_title'] = "Test title"
 
             if 'notification_body' in request.form and request.form['notification_body'].strip():
                 n_object['notification_body'] = request.form.get('notification_body', '').strip()
-            elif datastore.data['settings']['application'].get('notification_body'):
-                n_object['notification_body'] = datastore.data['settings']['application'].get('notification_body')
-            else:
-                n_object['notification_body'] = "Test body"
 
-            n_object['as_async'] = False
             n_object.update(watch.extra_notification_token_values())
-            from .notification import process_notification
-            sent_obj = process_notification(n_object, datastore)
 
+            from . import update_worker
+            new_worker = update_worker.update_worker(update_q, notification_q, app, datastore)
+            new_worker.queue_notification_for_watch(notification_q=notification_q, n_object=n_object, watch=watch)
         except Exception as e:
-            e_str = str(e)
-            # Remove this text which is not important and floods the container
-            e_str = e_str.replace(
-                "DEBUG - <class 'apprise.decorators.base.CustomNotifyPlugin.instantiate_plugin.<locals>.CustomNotifyPluginWrapper'>",
-                '')
-
-            return make_response(e_str, 400)
+            return make_response(f"Error: str(e)", 400)
 
         return 'OK - Sent test notifications'
 
@@ -732,8 +734,7 @@ def changedetection_app(config=None, datastore_o=None):
 
         form = form_class(formdata=request.form if request.method == 'POST' else None,
                           data=default,
-                          extra_notification_tokens=default.extra_notification_token_values(),
-                          default_system_settings=datastore.data['settings']
+                          extra_notification_tokens=default.extra_notification_token_values()
                           )
 
         # For the form widget tag UUID back to "string name" for the field
@@ -821,33 +822,7 @@ def changedetection_app(config=None, datastore_o=None):
             # But in the case something is added we should save straight away
             datastore.needs_write_urgent = True
 
-            # Do not queue on edit if its not within the time range
-
-            # @todo maybe it should never queue anyway on edit...
-            is_in_schedule = True
-            watch = datastore.data['watching'].get(uuid)
-
-            if watch.get('time_between_check_use_default'):
-                time_schedule_limit = datastore.data['settings']['requests'].get('time_schedule_limit', {})
-            else:
-                time_schedule_limit = watch.get('time_schedule_limit')
-
-            tz_name = time_schedule_limit.get('timezone')
-            if not tz_name:
-                tz_name =  datastore.data['settings']['application'].get('timezone', 'UTC')
-
-            if time_schedule_limit and time_schedule_limit.get('enabled'):
-                try:
-                    is_in_schedule = is_within_schedule(time_schedule_limit=time_schedule_limit,
-                                                        default_tz=tz_name
-                                                        )
-                except Exception as e:
-                    logger.error(
-                        f"{uuid} - Recheck scheduler, error handling timezone, check skipped - TZ name '{tz_name}' - {str(e)}")
-                    return False
-
-            #############################
-            if not datastore.data['watching'][uuid].get('paused') and is_in_schedule:
+            if not datastore.data['watching'][uuid].get('paused'):
                 # Queue the watch for immediate recheck, with a higher priority
                 update_q.put(queuedWatchMetaData.PrioritizedItem(priority=1, item={'uuid': uuid}))
 
@@ -855,7 +830,7 @@ def changedetection_app(config=None, datastore_o=None):
             if request.args.get("next") and request.args.get("next") == 'diff':
                 return redirect(url_for('diff_history_page', uuid=uuid))
 
-            return redirect(url_for('index', tag=request.args.get("tag",'')))
+            return redirect(url_for('index'))
 
         else:
             if request.method == 'POST' and not form.validate():
@@ -875,33 +850,30 @@ def changedetection_app(config=None, datastore_o=None):
 
             system_uses_webdriver = datastore.data['settings']['application']['fetch_backend'] == 'html_webdriver'
 
-            watch_uses_webdriver = False
+            is_html_webdriver = False
             if (watch.get('fetch_backend') == 'system' and system_uses_webdriver) or watch.get('fetch_backend') == 'html_webdriver' or watch.get('fetch_backend', '').startswith('extra_browser_'):
-                watch_uses_webdriver = True
-
-            from zoneinfo import available_timezones
+                is_html_webdriver = True
 
             # Only works reliably with Playwright
-
+            visualselector_enabled = os.getenv('PLAYWRIGHT_DRIVER_URL', False) and is_html_webdriver
             template_args = {
                 'available_processors': processors.available_processors(),
-                'available_timezones': sorted(available_timezones()),
                 'browser_steps_config': browser_step_ui_config,
                 'emailprefix': os.getenv('NOTIFICATION_MAIL_BUTTON_PREFIX', False),
-                'extra_notification_token_placeholder_info': datastore.get_unique_notification_token_placeholders_available(),
-                'extra_processor_config': form.extra_tab_content(),
                 'extra_title': f" - Edit - {watch.label}",
+                'extra_processor_config': form.extra_tab_content(),
+                'extra_notification_token_placeholder_info': datastore.get_unique_notification_token_placeholders_available(),
                 'form': form,
                 'has_default_notification_urls': True if len(datastore.data['settings']['application']['notification_urls']) else False,
                 'has_extra_headers_file': len(datastore.get_all_headers_in_textfile_for_watch(uuid=uuid)) > 0,
                 'has_special_tag_options': _watch_has_tag_options_set(watch=watch),
-                'watch_uses_webdriver': watch_uses_webdriver,
+                'is_html_webdriver': is_html_webdriver,
                 'jq_support': jq_support,
                 'playwright_enabled': os.getenv('PLAYWRIGHT_DRIVER_URL', False),
                 'settings_application': datastore.data['settings']['application'],
-                'timezone_default_config': datastore.data['settings']['application'].get('timezone'),
                 'using_global_webdriver_wait': not default['webdriver_delay'],
                 'uuid': uuid,
+                'visualselector_enabled': visualselector_enabled,
                 'watch': watch
             }
 
@@ -929,7 +901,6 @@ def changedetection_app(config=None, datastore_o=None):
     def settings_page():
         from changedetectionio import forms
         from datetime import datetime
-        from zoneinfo import available_timezones
 
         default = deepcopy(datastore.data['settings'])
         if datastore.proxy_list is not None:
@@ -997,20 +968,23 @@ def changedetection_app(config=None, datastore_o=None):
             else:
                 flash("An error occurred, please see below.", "error")
 
-        # Convert to ISO 8601 format, all date/time relative events stored as UTC time
-        utc_time = datetime.now(ZoneInfo("UTC")).isoformat()
+
+        system_timezone = app_get_system_time()
+        system_time = datetime.now(system_timezone)
+
+        # Fallback for locale formatting
+        formatted_system_time = system_time.strftime("%Y-%m-%d %H:%M:%S %Z%z")  # Locale-aware time
 
         output = render_template("settings.html",
                                  api_key=datastore.data['settings']['application'].get('api_access_token'),
-                                 available_timezones=sorted(available_timezones()),
                                  emailprefix=os.getenv('NOTIFICATION_MAIL_BUTTON_PREFIX', False),
                                  extra_notification_token_placeholder_info=datastore.get_unique_notification_token_placeholders_available(),
                                  form=form,
                                  hide_remove_pass=os.getenv("SALTED_PASS", False),
                                  min_system_recheck_seconds=int(os.getenv('MINIMUM_SECONDS_RECHECK_TIME', 3)),
                                  settings_application=datastore.data['settings']['application'],
-                                 timezone_default_config=datastore.data['settings']['application'].get('timezone'),
-                                 utc_time=utc_time,
+                                 system_time=formatted_system_time,
+                                 timezone_name=system_timezone
                                  )
 
         return output
@@ -1323,23 +1297,12 @@ def changedetection_app(config=None, datastore_o=None):
 
             # These files should be in our subdirectory
             try:
-                # set nocache, set content-type,
-                # `filename` is actually directory UUID of the watch
-                watch_directory = str(os.path.join(datastore_o.datastore_path, filename))
-                response = None
-                if os.path.isfile(os.path.join(watch_directory, "elements.deflate")):
-                    response = make_response(send_from_directory(watch_directory, "elements.deflate"))
-                    response.headers['Content-Type'] = 'application/json'
-                    response.headers['Content-Encoding'] = 'deflate'
-                else:
-                    logger.error(f'Request elements.deflate at "{watch_directory}" but was notfound.')
-                    abort(404)
-
-                if response:
-                    response.headers['Cache-Control'] = 'no-cache, no-store, must-revalidate'
-                    response.headers['Pragma'] = 'no-cache'
-                    response.headers['Expires'] = "0"
-
+                # set nocache, set content-type
+                response = make_response(send_from_directory(os.path.join(datastore_o.datastore_path, filename), "elements.json"))
+                response.headers['Content-type'] = 'application/json'
+                response.headers['Cache-Control'] = 'no-cache, no-store, must-revalidate'
+                response.headers['Pragma'] = 'no-cache'
+                response.headers['Expires'] = 0
                 return response
 
             except FileNotFoundError:
@@ -1408,13 +1371,13 @@ def changedetection_app(config=None, datastore_o=None):
         if new_uuid:
             if add_paused:
                 flash('Watch added in Paused state, saving will unpause.')
-                return redirect(url_for('edit_page', uuid=new_uuid, unpause_on_save=1, tag=request.args.get('tag')))
+                return redirect(url_for('edit_page', uuid=new_uuid, unpause_on_save=1))
             else:
                 # Straight into the queue.
                 update_q.put(queuedWatchMetaData.PrioritizedItem(priority=1, item={'uuid': new_uuid}))
                 flash("Watch added.")
 
-        return redirect(url_for('index', tag=request.args.get('tag','')))
+        return redirect(url_for('index'))
 
 
 
@@ -1684,6 +1647,7 @@ def changedetection_app(config=None, datastore_o=None):
     import changedetectionio.blueprint.backups as backups
     app.register_blueprint(backups.construct_blueprint(datastore), url_prefix='/backups')
 
+
     # @todo handle ctrl break
     ticker_thread = threading.Thread(target=ticker_thread_check_time_launch_checks).start()
     threading.Thread(target=notification_runner).start()
@@ -1830,28 +1794,6 @@ def ticker_thread_check_time_launch_checks():
             if watch['paused']:
                 continue
 
-            # @todo - Maybe make this a hook?
-            # Time schedule limit - Decide between watch or global settings
-            if watch.get('time_between_check_use_default'):
-                time_schedule_limit = datastore.data['settings']['requests'].get('time_schedule_limit', {})
-                logger.trace(f"{uuid} Time scheduler - Using system/global settings")
-            else:
-                time_schedule_limit = watch.get('time_schedule_limit')
-                logger.trace(f"{uuid} Time scheduler - Using watch settings (not global settings)")
-            tz_name = datastore.data['settings']['application'].get('timezone', 'UTC')
-
-            if time_schedule_limit and time_schedule_limit.get('enabled'):
-                try:
-                    result = is_within_schedule(time_schedule_limit=time_schedule_limit,
-                                                default_tz=tz_name
-                                                )
-                    if not result:
-                        logger.trace(f"{uuid} Time scheduler - not within schedule skipping.")
-                        continue
-                except Exception as e:
-                    logger.error(
-                        f"{uuid} - Recheck scheduler, error handling timezone, check skipped - TZ name '{tz_name}' - {str(e)}")
-                    return False
             # If they supplied an individual entry minutes to threshold.
             threshold = recheck_time_system_seconds if watch.get('time_between_check_use_default') else watch.threshold_seconds()
 

changedetectionio/forms.py

@@ -1,14 +1,12 @@
 import os
 import re
 from loguru import logger
-from wtforms.widgets.core import TimeInput
 
 from changedetectionio.strtobool import strtobool
 
 from wtforms import (
     BooleanField,
     Form,
-    Field,
     IntegerField,
     RadioField,
     SelectField,
@@ -127,87 +125,6 @@ class StringTagUUID(StringField):
 
         return 'error'
 
-class TimeDurationForm(Form):
-    hours = SelectField(choices=[(f"{i}", f"{i}") for i in range(0, 25)], default="24",  validators=[validators.Optional()])
-    minutes = SelectField(choices=[(f"{i}", f"{i}") for i in range(0, 60)], default="00", validators=[validators.Optional()])
-
-class TimeStringField(Field):
-    """
-    A WTForms field for time inputs (HH:MM) that stores the value as a string.
-    """
-    widget = TimeInput()  # Use the built-in time input widget
-
-    def _value(self):
-        """
-        Returns the value for rendering in the form.
-        """
-        return self.data if self.data is not None else ""
-
-    def process_formdata(self, valuelist):
-        """
-        Processes the raw input from the form and stores it as a string.
-        """
-        if valuelist:
-            time_str = valuelist[0]
-            # Simple validation for HH:MM format
-            if not time_str or len(time_str.split(":")) != 2:
-                raise ValidationError("Invalid time format. Use HH:MM.")
-            self.data = time_str
-
-
-class validateTimeZoneName(object):
-    """
-       Flask wtform validators wont work with basic auth
-    """
-
-    def __init__(self, message=None):
-        self.message = message
-
-    def __call__(self, form, field):
-        from zoneinfo import available_timezones
-        python_timezones = available_timezones()
-        if field.data and field.data not in python_timezones:
-            raise ValidationError("Not a valid timezone name")
-
-class ScheduleLimitDaySubForm(Form):
-    enabled = BooleanField("not set", default=True)
-    start_time = TimeStringField("Start At", default="00:00", validators=[validators.Optional()])
-    duration = FormField(TimeDurationForm, label="Run duration")
-
-class ScheduleLimitForm(Form):
-    enabled = BooleanField("Use time scheduler", default=False)
-    # Because the label for=""" doesnt line up/work with the actual checkbox
-    monday = FormField(ScheduleLimitDaySubForm, label="")
-    tuesday = FormField(ScheduleLimitDaySubForm, label="")
-    wednesday = FormField(ScheduleLimitDaySubForm, label="")
-    thursday = FormField(ScheduleLimitDaySubForm, label="")
-    friday = FormField(ScheduleLimitDaySubForm, label="")
-    saturday = FormField(ScheduleLimitDaySubForm, label="")
-    sunday = FormField(ScheduleLimitDaySubForm, label="")
-
-    timezone = StringField("Optional timezone to run in",
-                                  render_kw={"list": "timezones"},
-                                  validators=[validateTimeZoneName()]
-                                  )
-    def __init__(
-        self,
-        formdata=None,
-        obj=None,
-        prefix="",
-        data=None,
-        meta=None,
-        **kwargs,
-    ):
-        super().__init__(formdata, obj, prefix, data, meta, **kwargs)
-        self.monday.form.enabled.label.text="Monday"
-        self.tuesday.form.enabled.label.text = "Tuesday"
-        self.wednesday.form.enabled.label.text = "Wednesday"
-        self.thursday.form.enabled.label.text = "Thursday"
-        self.friday.form.enabled.label.text = "Friday"
-        self.saturday.form.enabled.label.text = "Saturday"
-        self.sunday.form.enabled.label.text = "Sunday"
-
-
 class TimeBetweenCheckForm(Form):
     weeks = IntegerField('Weeks', validators=[validators.Optional(), validators.NumberRange(min=0, message="Should contain zero or more seconds")])
     days = IntegerField('Days', validators=[validators.Optional(), validators.NumberRange(min=0, message="Should contain zero or more seconds")])
@@ -308,12 +225,8 @@ class ValidateAppRiseServers(object):
         # so that the custom endpoints are registered
         from changedetectionio.apprise_plugin import apprise_custom_api_call_wrapper
         for server_url in field.data:
-            url = server_url.strip()
-            if url.startswith("#"):
-                continue
-
-            if not apobj.add(url):
-                message = field.gettext('\'%s\' is not a valid AppRise URL.' % (url))
+            if not apobj.add(server_url):
+                message = field.gettext('\'%s\' is not a valid AppRise URL.' % (server_url))
                 raise ValidationError(message)
 
 class ValidateJinja2Template(object):
@@ -366,7 +279,6 @@ class validateURL(object):
         # This should raise a ValidationError() or not
         validate_url(field.data)
 
-
 def validate_url(test_url):
     # If hosts that only contain alphanumerics are allowed ("localhost" for example)
     try:
@@ -526,7 +438,6 @@ class commonSettingsForm(Form):
     notification_title = StringField('Notification Title', default='ChangeDetection.io Notification - {{ watch_url }}', validators=[validators.Optional(), ValidateJinja2Template()])
     notification_urls = StringListField('Notification URL List', validators=[validators.Optional(), ValidateAppRiseServers(), ValidateJinja2Template()])
     processor = RadioField( label=u"Processor - What do you want to achieve?", choices=processors.available_processors(), default="text_json_diff")
-    timezone = StringField("Timezone for watch schedule", render_kw={"list": "timezones"}, validators=[validateTimeZoneName()])
     webdriver_delay = IntegerField('Wait seconds before extracting text', validators=[validators.Optional(), validators.NumberRange(min=1, message="Should contain one or more seconds")])
 
 
@@ -537,6 +448,7 @@ class importForm(Form):
     xlsx_file = FileField('Upload .xlsx file', validators=[FileAllowed(['xlsx'], 'Must be .xlsx file!')])
     file_mapping = SelectField('File mapping', [validators.DataRequired()], choices={('wachete', 'Wachete mapping'), ('custom','Custom mapping')})
 
+
 class SingleBrowserStep(Form):
 
     operation = SelectField('Operation', [validators.Optional()], choices=browser_step_ui_config.keys())
@@ -554,9 +466,6 @@ class processor_text_json_diff_form(commonSettingsForm):
     tags = StringTagUUID('Group tag', [validators.Optional()], default='')
 
     time_between_check = FormField(TimeBetweenCheckForm)
-
-    time_schedule_limit = FormField(ScheduleLimitForm)
-
     time_between_check_use_default = BooleanField('Use global settings for time between check', default=False)
 
     include_filters = StringListField('CSS/JSONPath/JQ/XPath Filters', [ValidateCSSJSONXPATHInput()], default='')
@@ -658,23 +567,6 @@ class processor_text_json_diff_form(commonSettingsForm):
 
         return result
 
-    def __init__(
-            self,
-            formdata=None,
-            obj=None,
-            prefix="",
-            data=None,
-            meta=None,
-            **kwargs,
-    ):
-        super().__init__(formdata, obj, prefix, data, meta, **kwargs)
-        if kwargs and kwargs.get('default_system_settings'):
-            default_tz = kwargs.get('default_system_settings').get('application', {}).get('timezone')
-            if default_tz:
-                self.time_schedule_limit.form.timezone.render_kw['placeholder'] = default_tz
-
-
-
 class SingleExtraProxy(Form):
 
     # maybe better to set some <script>var..
@@ -695,7 +587,6 @@ class DefaultUAInputForm(Form):
 # datastore.data['settings']['requests']..
 class globalSettingsRequestForm(Form):
     time_between_check = FormField(TimeBetweenCheckForm)
-    time_schedule_limit = FormField(ScheduleLimitForm)
     proxy = RadioField('Proxy')
     jitter_seconds = IntegerField('Random jitter seconds ± check',
                                   render_kw={"style": "width: 5em;"},

changedetectionio/html_tools.py

@@ -1,6 +1,5 @@
-from loguru import logger
-from lxml import etree
 from typing import List
+from lxml import etree
 import json
 import re
 
@@ -299,10 +298,8 @@ def extract_json_as_string(content, json_filter, ensure_is_ldjson_info_type=None
 # https://github.com/dgtlmoon/changedetection.io/pull/2041#issuecomment-1848397161w
     # Try to parse/filter out the JSON, if we get some parser error, then maybe it's embedded within HTML tags
     try:
-        # .lstrip("\ufeff") strings ByteOrderMark from UTF8 and still lets the UTF work
-        stripped_text_from_html = _parse_json(json.loads(content.lstrip("\ufeff") ), json_filter)
-    except json.JSONDecodeError as e:
-        logger.warning(str(e))
+        stripped_text_from_html = _parse_json(json.loads(content), json_filter)
+    except json.JSONDecodeError:
 
         # Foreach <script json></script> blob.. just return the first that matches json_filter
         # As a last resort, try to parse the whole <body>

changedetectionio/model/App.py

@@ -53,7 +53,7 @@ class model(dict):
                     'shared_diff_access': False,
                     'webdriver_delay': None , # Extra delay in seconds before extracting text
                     'tags': {}, #@todo use Tag.model initialisers
-                    'timezone': None, # Default IANA timezone name
+                    'timezone': None,
                 }
             }
         }
@@ -69,7 +69,7 @@ def parse_headers_from_text_file(filepath):
         for l in f.readlines():
             l = l.strip()
             if not l.startswith('#') and ':' in l:
-                (k, v) = l.split(':', 1)  # Split only on the first colon
+                (k, v) = l.split(':')
                 headers[k.strip()] = v.strip()
 
     return headers

changedetectionio/model/Watch.py

@@ -247,32 +247,37 @@ class model(watch_base):
         bump = self.history
         return self.__newest_history_key
 
-    # Given an arbitrary timestamp, find the best history key for the [diff] button so it can preset a smarter from_version
+    # Given an arbitrary timestamp, find the closest next key
+    # For example, last_viewed = 1000 so it should return the next 1001 timestamp
+    #
+    # used for the [diff] button so it can preset a smarter from_version
     @property
-    def get_from_version_based_on_last_viewed(self):
+    def get_next_snapshot_key_to_last_viewed(self):
 
         """Unfortunately for now timestamp is stored as string key"""
         keys = list(self.history.keys())
         if not keys:
             return None
-        if len(keys) == 1:
-            return keys[0]
 
         last_viewed = int(self.get('last_viewed'))
+        prev_k = keys[0]
         sorted_keys = sorted(keys, key=lambda x: int(x))
         sorted_keys.reverse()
 
-        # When the 'last viewed' timestamp is greater than or equal the newest snapshot, return second newest
-        if last_viewed >= int(sorted_keys[0]):
+        # When the 'last viewed' timestamp is greater than the newest snapshot, return second last
+        if last_viewed > int(sorted_keys[0]):
             return sorted_keys[1]
-        
-        # When the 'last viewed' timestamp is between snapshots, return the older snapshot
-        for newer, older in list(zip(sorted_keys[0:], sorted_keys[1:])):
-            if last_viewed < int(newer) and last_viewed >= int(older):
-                return older
 
-        # When the 'last viewed' timestamp is less than the oldest snapshot, return oldest
-        return sorted_keys[-1]
+        for k in sorted_keys:
+            if int(k) < last_viewed:
+                if prev_k == sorted_keys[0]:
+                    # Return the second last one so we dont recommend the same version compares itself
+                    return sorted_keys[1]
+
+                return prev_k
+            prev_k = k
+
+        return keys[0]
 
     def get_history_snapshot(self, timestamp):
         import brotli
@@ -334,6 +339,7 @@ class model(watch_base):
         # @todo bump static cache of the last timestamp so we dont need to examine the file to set a proper ''viewed'' status
         return snapshot_fname
 
+    @property
     @property
     def has_empty_checktime(self):
         # using all() + dictionary comprehension
@@ -352,7 +358,7 @@ class model(watch_base):
     # Iterate over all history texts and see if something new exists
     # Always applying .strip() to start/end but optionally replace any other whitespace
     def lines_contain_something_unique_compared_to_history(self, lines: list, ignore_whitespace=False):
-        local_lines = set([])
+        local_lines = []
         if lines:
             if ignore_whitespace:
                 if isinstance(lines[0], str): # Can be either str or bytes depending on what was on the disk
@@ -532,17 +538,16 @@ class model(watch_base):
 
     def save_xpath_data(self, data, as_error=False):
         import json
-        import zlib
 
         if as_error:
-            target_path = os.path.join(str(self.watch_data_dir), "elements-error.deflate")
+            target_path = os.path.join(self.watch_data_dir, "elements-error.json")
         else:
-            target_path = os.path.join(str(self.watch_data_dir), "elements.deflate")
+            target_path = os.path.join(self.watch_data_dir, "elements.json")
 
         self.ensure_data_dir_exists()
 
-        with open(target_path, 'wb') as f:
-            f.write(zlib.compress(json.dumps(data).encode()))
+        with open(target_path, 'w') as f:
+            f.write(json.dumps(data))
             f.close()
 
     # Save as PNG, PNG is larger but better for doing visual diff in the future

changedetectionio/model/__init__.py

@@ -59,65 +59,6 @@ class watch_base(dict):
             'text_should_not_be_present': [],  # Text that should not present
             'time_between_check': {'weeks': None, 'days': None, 'hours': None, 'minutes': None, 'seconds': None},
             'time_between_check_use_default': True,
-            "time_schedule_limit": {
-                "enabled": False,
-                "monday": {
-                    "enabled": True,
-                    "start_time": "00:00",
-                    "duration": {
-                        "hours": "24",
-                        "minutes": "00"
-                    }
-                },
-                "tuesday": {
-                    "enabled": True,
-                    "start_time": "00:00",
-                    "duration": {
-                        "hours": "24",
-                        "minutes": "00"
-                    }
-                },
-                "wednesday": {
-                    "enabled": True,
-                    "start_time": "00:00",
-                    "duration": {
-                        "hours": "24",
-                        "minutes": "00"
-                    }
-                },
-                "thursday": {
-                    "enabled": True,
-                    "start_time": "00:00",
-                    "duration": {
-                        "hours": "24",
-                        "minutes": "00"
-                    }
-                },
-                "friday": {
-                    "enabled": True,
-                    "start_time": "00:00",
-                    "duration": {
-                        "hours": "24",
-                        "minutes": "00"
-                    }
-                },
-                "saturday": {
-                    "enabled": True,
-                    "start_time": "00:00",
-                    "duration": {
-                        "hours": "24",
-                        "minutes": "00"
-                    }
-                },
-                "sunday": {
-                    "enabled": True,
-                    "start_time": "00:00",
-                    "duration": {
-                        "hours": "24",
-                        "minutes": "00"
-                    }
-                },
-            },
             'title': None,
             'track_ldjson_price_data': None,
             'trim_text_whitespace': False,

changedetectionio/notification.py

@@ -23,7 +23,7 @@ valid_tokens = {
 }
 
 default_notification_format_for_watch = 'System default'
-default_notification_format = 'HTML Color'
+default_notification_format = 'Text'
 default_notification_body = '{{watch_url}} had a change.\n---\n{{diff}}\n---\n'
 default_notification_title = 'ChangeDetection.io Notification - {{watch_url}}'
 
@@ -31,7 +31,6 @@ valid_notification_formats = {
     'Text': NotifyFormat.TEXT,
     'Markdown': NotifyFormat.MARKDOWN,
     'HTML': NotifyFormat.HTML,
-    'HTML Color': 'htmlcolor',
     # Used only for editing a watch (not for global)
     default_notification_format_for_watch: default_notification_format_for_watch
 }
@@ -67,10 +66,6 @@ def process_notification(n_object, datastore):
 
     sent_objs = []
     from .apprise_asset import asset
-
-    if 'as_async' in n_object:
-        asset.async_mode = n_object.get('as_async')
-
     apobj = apprise.Apprise(debug=True, asset=asset)
 
     if not n_object.get('notification_urls'):
@@ -81,16 +76,9 @@ def process_notification(n_object, datastore):
 
             # Get the notification body from datastore
             n_body = jinja_render(template_str=n_object.get('notification_body', ''), **notification_parameters)
-            if n_object.get('notification_format', '').startswith('HTML'):
-                n_body = n_body.replace("\n", '<br>')
-
             n_title = jinja_render(template_str=n_object.get('notification_title', ''), **notification_parameters)
 
             url = url.strip()
-            if url.startswith('#'):
-                logger.trace(f"Skipping commented out notification URL - {url}")
-                continue
-
             if not url:
                 logger.warning(f"Process Notification: skipping empty notification URL.")
                 continue
@@ -161,6 +149,8 @@ def process_notification(n_object, datastore):
             attach=n_object.get('screenshot', None)
         )
 
+        # Give apprise time to register an error
+        time.sleep(3)
 
         # Returns empty string if nothing found, multi-line string otherwise
         log_value = logs.getvalue()

changedetectionio/processors/__init__.py

@@ -33,8 +33,8 @@ class difference_detection_processor():
 
         url = self.watch.link
 
-        # Protect against file:, file:/, file:// access, check the real "link" without any meta "source:" etc prepended.
-        if re.search(r'^file:', url.strip(), re.IGNORECASE):
+        # Protect against file://, file:/ access, check the real "link" without any meta "source:" etc prepended.
+        if re.search(r'^file:/', url.strip(), re.IGNORECASE):
             if not strtobool(os.getenv('ALLOW_FILE_URI', 'false')):
                 raise Exception(
                     "file:// type access is denied for security reasons."

changedetectionio/static/images/copy.svg

@@ -1,7 +1,7 @@
 <?xml version="1.0" encoding="UTF-8" standalone="no"?>
 <svg
    version="1.1"
-   id="copy"
+   id="Layer_1"
    x="0px"
    y="0px"
    viewBox="0 0 115.77 122.88"

changedetectionio/static/images/email.svg

@@ -6,7 +6,7 @@
    height="7.5005589"
    width="11.248507"
    version="1.1"
-   id="email"
+   id="Layer_1"
    viewBox="0 0 7.1975545 4.7993639"
    xml:space="preserve"
    xmlns="http://www.w3.org/2000/svg"

changedetectionio/static/images/schedule.svg

@@ -1,225 +0,0 @@
-<?xml version="1.0" encoding="UTF-8" standalone="no"?>
-<svg
-   version="1.1"
-   id="schedule"
-   x="0px"
-   y="0px"
-   viewBox="0 0 661.20001 665.40002"
-   xml:space="preserve"
-   width="661.20001"
-   height="665.40002"
-   inkscape:version="1.1.2 (0a00cf5339, 2022-02-04)"
-   sodipodi:docname="schedule.svg"
-   xmlns:inkscape="http://www.inkscape.org/namespaces/inkscape"
-   xmlns:sodipodi="http://sodipodi.sourceforge.net/DTD/sodipodi-0.dtd"
-   xmlns="http://www.w3.org/2000/svg"
-   xmlns:svg="http://www.w3.org/2000/svg"><defs
-   id="defs77" /><sodipodi:namedview
-   id="namedview75"
-   pagecolor="#ffffff"
-   bordercolor="#666666"
-   borderopacity="1.0"
-   inkscape:pageshadow="2"
-   inkscape:pageopacity="0.0"
-   inkscape:pagecheckerboard="0"
-   showgrid="false"
-   fit-margin-top="0"
-   fit-margin-left="0"
-   fit-margin-right="0"
-   fit-margin-bottom="0"
-   inkscape:zoom="1.2458671"
-   inkscape:cx="300.59386"
-   inkscape:cy="332.29869"
-   inkscape:window-width="1920"
-   inkscape:window-height="1051"
-   inkscape:window-x="1920"
-   inkscape:window-y="0"
-   inkscape:window-maximized="1"
-   inkscape:current-layer="g72" /> <style
-   type="text/css"
-   id="style2"> .st0{fill:#FFFFFF;} .st1{fill:#C1272D;} .st2{fill:#991D26;} .st3{fill:#CCCCCC;} .st4{fill:#E6E6E6;} .st5{fill:#F7931E;} .st6{fill:#F2F2F2;} .st7{fill:none;stroke:#999999;stroke-width:17.9763;stroke-linecap:round;stroke-miterlimit:10;} .st8{fill:none;stroke:#333333;stroke-width:8.9882;stroke-linecap:round;stroke-miterlimit:10;} .st9{fill:none;stroke:#C1272D;stroke-width:5.9921;stroke-linecap:round;stroke-miterlimit:10;} .st10{fill:#245F7F;} </style> <g
-   id="g72"
-   transform="translate(-149.4,-147.3)"> <path
-   class="st0"
-   d="M 601.2,699.8 H 205 c -30.7,0 -55.6,-24.9 -55.6,-55.6 V 248 c 0,-30.7 24.9,-55.6 55.6,-55.6 h 396.2 c 30.7,0 55.6,24.9 55.6,55.6 v 396.2 c 0,30.7 -24.9,55.6 -55.6,55.6 z"
-   id="path4"
-   style="fill:#dfdfdf;fill-opacity:1" /> <path
-   class="st1"
-   d="M 601.2,192.4 H 205 c -30.7,0 -55.6,24.9 -55.6,55.6 v 88.5 H 656.8 V 248 c 0,-30.7 -24.9,-55.6 -55.6,-55.6 z"
-   id="path6"
-   style="fill:#d62128;fill-opacity:1" /> <circle
-   class="st2"
-   cx="253.3"
-   cy="264.5"
-   r="36.700001"
-   id="circle8" /> <circle
-   class="st2"
-   cx="551.59998"
-   cy="264.5"
-   r="36.700001"
-   id="circle10" /> <path
-   class="st3"
-   d="m 253.3,275.7 v 0 c -11.8,0 -21.3,-9.6 -21.3,-21.3 v -85.8 c 0,-11.8 9.6,-21.3 21.3,-21.3 v 0 c 11.8,0 21.3,9.6 21.3,21.3 v 85.8 c 0,11.8 -9.5,21.3 -21.3,21.3 z"
-   id="path12" /> <path
-   class="st3"
-   d="m 551.6,275.7 v 0 c -11.8,0 -21.3,-9.6 -21.3,-21.3 v -85.8 c 0,-11.8 9.6,-21.3 21.3,-21.3 v 0 c 11.8,0 21.3,9.6 21.3,21.3 v 85.8 c 0.1,11.8 -9.5,21.3 -21.3,21.3 z"
-   id="path14" /> <rect
-   x="215.7"
-   y="370.89999"
-   class="st4"
-   width="75.199997"
-   height="75.199997"
-   id="rect16"
-   style="fill:#ffffff;fill-opacity:1" /> <rect
-   x="313"
-   y="370.89999"
-   class="st4"
-   width="75.199997"
-   height="75.199997"
-   id="rect18"
-   style="fill:#ffffff;fill-opacity:1" /> <rect
-   x="410.20001"
-   y="370.89999"
-   class="st4"
-   width="75.199997"
-   height="75.199997"
-   id="rect20"
-   style="fill:#ffffff;fill-opacity:1" /> <rect
-   x="507.5"
-   y="370.89999"
-   class="st4"
-   width="75.199997"
-   height="75.199997"
-   id="rect22"
-   style="fill:#ffffff;fill-opacity:1" /> <rect
-   x="215.7"
-   y="465"
-   class="st4"
-   width="75.199997"
-   height="75.199997"
-   id="rect24"
-   style="fill:#ffffff;fill-opacity:1" /> <rect
-   x="313"
-   y="465"
-   class="st1"
-   width="75.199997"
-   height="75.199997"
-   id="rect26"
-   style="fill:#27c12b;fill-opacity:1" /> <rect
-   x="410.20001"
-   y="465"
-   class="st4"
-   width="75.199997"
-   height="75.199997"
-   id="rect28"
-   style="fill:#ffffff;fill-opacity:1" /> <rect
-   x="507.5"
-   y="465"
-   class="st4"
-   width="75.199997"
-   height="75.199997"
-   id="rect30" /> <rect
-   x="215.7"
-   y="559.09998"
-   class="st4"
-   width="75.199997"
-   height="75.199997"
-   id="rect32"
-   style="fill:#ffffff;fill-opacity:1" /> <rect
-   x="313"
-   y="559.09998"
-   class="st4"
-   width="75.199997"
-   height="75.199997"
-   id="rect34"
-   style="fill:#ffffff;fill-opacity:1" /> <rect
-   x="410.20001"
-   y="559.09998"
-   class="st4"
-   width="75.199997"
-   height="75.199997"
-   id="rect36"
-   style="fill:#ffffff;fill-opacity:1" /> <rect
-   x="507.5"
-   y="559.09998"
-   class="st4"
-   width="75.199997"
-   height="75.199997"
-   id="rect38" /> <g
-   id="g70"> <circle
-   class="st5"
-   cx="621.90002"
-   cy="624"
-   r="188.7"
-   id="circle40" /> <circle
-   class="st0"
-   cx="621.90002"
-   cy="624"
-   r="148"
-   id="circle42" /> <path
-   class="st6"
-   d="m 486.6,636.8 c 0,-81.7 66.3,-148 148,-148 37.6,0 72,14.1 98.1,37.2 -27.1,-30.6 -66.7,-49.9 -110.8,-49.9 -81.7,0 -148,66.3 -148,148 0,44.1 19.3,83.7 49.9,110.8 -23.1,-26.2 -37.2,-60.5 -37.2,-98.1 z"
-   id="path44" /> <polyline
-   class="st7"
-   points="621.9,530.4 621.9,624 559,624  "
-   id="polyline46" /> <g
-   id="g64"> <line
-   class="st8"
-   x1="621.90002"
-   y1="508.29999"
-   x2="621.90002"
-   y2="497.10001"
-   id="line48" /> <line
-   class="st8"
-   x1="621.90002"
-   y1="756.29999"
-   x2="621.90002"
-   y2="745.09998"
-   id="line50" /> <line
-   class="st8"
-   x1="740.29999"
-   y1="626.70001"
-   x2="751.5"
-   y2="626.70001"
-   id="line52" /> <line
-   class="st8"
-   x1="492.29999"
-   y1="626.70001"
-   x2="503.5"
-   y2="626.70001"
-   id="line54" /> <line
-   class="st8"
-   x1="705.59998"
-   y1="710.40002"
-   x2="713.5"
-   y2="718.29999"
-   id="line56" /> <line
-   class="st8"
-   x1="530.29999"
-   y1="535.09998"
-   x2="538.20001"
-   y2="543"
-   id="line58" /> <line
-   class="st8"
-   x1="538.20001"
-   y1="710.40002"
-   x2="530.29999"
-   y2="718.29999"
-   id="line60" /> <line
-   class="st8"
-   x1="713.5"
-   y1="535.09998"
-   x2="705.59998"
-   y2="543"
-   id="line62" /> </g> <line
-   class="st9"
-   x1="604.40002"
-   y1="606.29999"
-   x2="684.5"
-   y2="687.40002"
-   id="line66" /> <circle
-   class="st10"
-   cx="621.90002"
-   cy="624"
-   r="16.1"
-   id="circle68" /> </g> </g> </svg>

changedetectionio/static/js/browser-steps.js

@@ -221,7 +221,7 @@ $(document).ready(function () {
                     // If you switch to "Click X,y" after an element here is setup, it will give the last co-ords anyway
                     //if (x['isClickable'] || x['tagName'].startsWith('h') || x['tagName'] === 'a' || x['tagName'] === 'button' || x['tagtype'] === 'submit' || x['tagtype'] === 'checkbox' || x['tagtype'] === 'radio' || x['tagtype'] === 'li') {
                         $('select', first_available).val('Click element').change();
-                        $('input[type=text]', first_available).first().val(x['xpath']).focus();
+                        $('input[type=text]', first_available).first().val(x['xpath']);
                         found_something = true;
                     //}
                 }
@@ -305,7 +305,7 @@ $(document).ready(function () {
 
         if ($(this).val() === 'Click X,Y' && last_click_xy['x'] > 0 && $(elem_value).val().length === 0) {
             // @todo handle scale
-            $(elem_value).val(last_click_xy['x'] + ',' + last_click_xy['y']).focus();
+            $(elem_value).val(last_click_xy['x'] + ',' + last_click_xy['y']);
         }
     }).change();
 

changedetectionio/static/js/global-settings.js

@@ -24,19 +24,5 @@ $(document).ready(function () {
         $(target).toggle();
     });
 
-    // Time zone config related
-    $(".local-time").each(function (e) {
-        $(this).text(new Date($(this).data("utc")).toLocaleString());
-    })
-
-    const timezoneInput = $('#application-timezone');
-    if(timezoneInput.length) {
-        const timezone = Intl.DateTimeFormat().resolvedOptions().timeZone;
-        if (!timezoneInput.val().trim()) {
-            timezoneInput.val(timezone);
-            timezoneInput.after('<div class="timezone-message">The timezone was set from your browser, <strong>be sure to press save!</strong></div>');
-        }
-    }
-
 });
 

changedetectionio/static/js/notifications.js

@@ -1,52 +1,42 @@
-$(document).ready(function () {
+$(document).ready(function() {
 
-    $('#add-email-helper').click(function (e) {
-        e.preventDefault();
-        email = prompt("Destination email");
-        if (email) {
-            var n = $(".notification-urls");
-            var p = email_notification_prefix;
-            $(n).val($.trim($(n).val()) + "\n" + email_notification_prefix + email);
+  $('#add-email-helper').click(function (e) {
+    e.preventDefault();
+    email = prompt("Destination email");
+    if(email) {
+      var n = $(".notification-urls");
+      var p=email_notification_prefix;
+      $(n).val( $.trim( $(n).val() )+"\n"+email_notification_prefix+email );
+    }
+  });
+
+  $('#send-test-notification').click(function (e) {
+    e.preventDefault();
+
+    data = {
+      notification_body: $('#notification_body').val(),
+      notification_format: $('#notification_format').val(),
+      notification_title: $('#notification_title').val(),
+      notification_urls: $('.notification-urls').val(),
+      tags: $('#tags').val(),
+      window_url: window.location.href,
+    }
+
+
+    $.ajax({
+      type: "POST",
+      url: notification_base_url,
+      data : data,
+        statusCode: {
+        400: function(data) {
+          // More than likely the CSRF token was lost when the server restarted
+          alert(data.responseText);
         }
-    });
-
-    $('#send-test-notification').click(function (e) {
-        e.preventDefault();
-
-        data = {
-            notification_body: $('#notification_body').val(),
-            notification_format: $('#notification_format').val(),
-            notification_title: $('#notification_title').val(),
-            notification_urls: $('.notification-urls').val(),
-            tags: $('#tags').val(),
-            window_url: window.location.href,
-        }
-
-        $('.notifications-wrapper .spinner').fadeIn();
-        $('#notification-test-log').show();
-        $.ajax({
-            type: "POST",
-            url: notification_base_url,
-            data: data,
-            statusCode: {
-                400: function (data) {
-                    $("#notification-test-log>span").text(data.responseText);
-                },
-            }
-        }).done(function (data) {
-            $("#notification-test-log>span").text(data);
-        }).fail(function (jqXHR, textStatus, errorThrown) {
-            // Handle connection refused or other errors
-            if (textStatus === "error" && errorThrown === "") {
-                console.error("Connection refused or server unreachable");
-                $("#notification-test-log>span").text("Error: Connection refused or server is unreachable.");
-            } else {
-                console.error("Error:", textStatus, errorThrown);
-                $("#notification-test-log>span").text("An error occurred: " + textStatus);
-            }
-        }).always(function () {
-            $('.notifications-wrapper .spinner').hide();
-        })
-    });
+      }
+    }).done(function(data){
+      console.log(data);
+      alert(data);
+    })
+  });
 });
 

changedetectionio/static/js/plugins.js

@@ -159,38 +159,4 @@
         // Return the current request in case it's needed
         return requests[namespace];
     };
-})(jQuery);
-
-
-
-function toggleOpacity(checkboxSelector, fieldSelector, inverted) {
-    const checkbox = document.querySelector(checkboxSelector);
-    const fields = document.querySelectorAll(fieldSelector);
-
-    function updateOpacity() {
-        const opacityValue = !checkbox.checked ? (inverted ? 0.6 : 1) : (inverted ? 1 : 0.6);
-        fields.forEach(field => {
-            field.style.opacity = opacityValue;
-        });
-    }
-
-    // Initial setup
-    updateOpacity();
-    checkbox.addEventListener('change', updateOpacity);
-}
-
-function toggleVisibility(checkboxSelector, fieldSelector, inverted) {
-    const checkbox = document.querySelector(checkboxSelector);
-    const fields = document.querySelectorAll(fieldSelector);
-
-    function updateOpacity() {
-        const opacityValue = !checkbox.checked ? (inverted ? 'none' : 'block') : (inverted ? 'block' : 'none');
-        fields.forEach(field => {
-            field.style.display = opacityValue;
-        });
-    }
-
-    // Initial setup
-    updateOpacity();
-    checkbox.addEventListener('change', updateOpacity);
-}
+})(jQuery);

changedetectionio/static/js/scheduler.js

@@ -1,109 +0,0 @@
-function getTimeInTimezone(timezone) {
-    const now = new Date();
-    const options = {
-        timeZone: timezone,
-        weekday: 'long',
-        year: 'numeric',
-        hour12: false,
-        month: '2-digit',
-        day: '2-digit',
-        hour: '2-digit',
-        minute: '2-digit',
-        second: '2-digit',
-    };
-
-    const formatter = new Intl.DateTimeFormat('en-US', options);
-    return formatter.format(now);
-}
-
-$(document).ready(function () {
-
-    let exceedsLimit = false;
-    const warning_text = $("#timespan-warning")
-    const timezone_text_widget = $("input[id*='time_schedule_limit-timezone']")
-
-    toggleVisibility('#time_schedule_limit-enabled, #requests-time_schedule_limit-enabled', '#schedule-day-limits-wrapper', true)
-
-    setInterval(() => {
-        let success = true;
-        try {
-            // Show the current local time according to either placeholder or entered TZ name
-            if (timezone_text_widget.val().length) {
-                $('#local-time-in-tz').text(getTimeInTimezone(timezone_text_widget.val()));
-            } else {
-                // So maybe use what is in the placeholder (which will be the default settings)
-                $('#local-time-in-tz').text(getTimeInTimezone(timezone_text_widget.attr('placeholder')));
-            }
-        } catch (error) {
-            success = false;
-            $('#local-time-in-tz').text("");
-            console.error(timezone_text_widget.val())
-        }
-
-        $(timezone_text_widget).toggleClass('error', !success);
-
-    }, 500);
-
-    $('#schedule-day-limits-wrapper').on('change click blur', 'input, checkbox, select', function() {
-
-        let allOk = true;
-
-        // Controls setting the warning that the time could overlap into the next day
-        $("li.day-schedule").each(function () {
-            const $schedule = $(this);
-            const $checkbox = $schedule.find("input[type='checkbox']");
-
-            if ($checkbox.is(":checked")) {
-                const timeValue = $schedule.find("input[type='time']").val();
-                const durationHours = parseInt($schedule.find("select[name*='-duration-hours']").val(), 10) || 0;
-                const durationMinutes = parseInt($schedule.find("select[name*='-duration-minutes']").val(), 10) || 0;
-
-                if (timeValue) {
-                    const [startHours, startMinutes] = timeValue.split(":").map(Number);
-                    const totalMinutes = (startHours * 60 + startMinutes) + (durationHours * 60 + durationMinutes);
-
-                    exceedsLimit = totalMinutes > 1440
-                    if (exceedsLimit) {
-                        allOk = false
-                    }
-                    // Set the row/day-of-week highlight
-                    $schedule.toggleClass("warning", exceedsLimit);
-                }
-            } else {
-                $schedule.toggleClass("warning", false);
-            }
-        });
-
-        warning_text.toggle(!allOk)
-    });
-
-    $('table[id*="time_schedule_limit-saturday"], table[id*="time_schedule_limit-sunday"]').addClass("weekend-day")
-
-    // Presets [weekend] [business hours] etc
-    $(document).on('click', '[data-template].set-schedule', function () {
-        // Get the value of the 'data-template' attribute
-        switch ($(this).attr('data-template')) {
-            case 'business-hours':
-                $('.day-schedule table:not(.weekend-day) input[type="time"]').val('09:00')
-                $('.day-schedule table:not(.weekend-day) select[id*="-duration-hours"]').val('8');
-                $('.day-schedule table:not(.weekend-day) select[id*="-duration-minutes"]').val('0');
-                $('.day-schedule input[id*="-enabled"]').prop('checked', true);
-                $('.day-schedule .weekend-day input[id*="-enabled"]').prop('checked', false);
-                break;
-            case 'weekend':
-                $('.day-schedule .weekend-day input[type="time"][id$="start-time"]').val('00:00')
-                $('.day-schedule .weekend-day select[id*="-duration-hours"]').val('24');
-                $('.day-schedule .weekend-day select[id*="-duration-minutes"]').val('0');
-                $('.day-schedule input[id*="-enabled"]').prop('checked', false);
-                $('.day-schedule .weekend-day input[id*="-enabled"]').prop('checked', true);
-                break;
-            case 'reset':
-
-                $('.day-schedule input[type="time"]').val('00:00')
-                $('.day-schedule select[id*="-duration-hours"]').val('24');
-                $('.day-schedule select[id*="-duration-minutes"]').val('0');
-                $('.day-schedule input[id*="-enabled"]').prop('checked', true);
-                break;
-        }
-    });
-});

changedetectionio/static/js/visual-selector.js

@@ -132,7 +132,6 @@ $(document).ready(() => {
         }).done((data) => {
             $fetchingUpdateNoticeElem.html("Rendering..");
             selectorData = data;
-
             sortScrapedElementsBySize();
             console.log(`Reported browser width from backend: ${data['browser_width']}`);
 

changedetectionio/static/js/watch-settings.js

@@ -1,3 +1,17 @@
+function toggleOpacity(checkboxSelector, fieldSelector, inverted) {
+    const checkbox = document.querySelector(checkboxSelector);
+    const fields = document.querySelectorAll(fieldSelector);
+    function updateOpacity() {
+        const opacityValue = !checkbox.checked ? (inverted ? 0.6 : 1) : (inverted ? 1 : 0.6);
+        fields.forEach(field => {
+            field.style.opacity = opacityValue;
+        });
+    }
+    // Initial setup
+    updateOpacity();
+    checkbox.addEventListener('change', updateOpacity);
+}
+
 
 function request_textpreview_update() {
     if (!$('body').hasClass('preview-text-enabled')) {
@@ -43,9 +57,7 @@ function request_textpreview_update() {
     })
 }
 
-
 $(document).ready(function () {
-
     $('#notification-setting-reset-to-default').click(function (e) {
         $('#notification_title').val('');
         $('#notification_body').val('');
@@ -58,12 +70,11 @@ $(document).ready(function () {
         $('#notification-tokens-info').toggle();
     });
 
-    toggleOpacity('#time_between_check_use_default', '#time-check-widget-wrapper, #time-between-check-schedule', false);
-
+    toggleOpacity('#time_between_check_use_default', '#time_between_check', false);
 
     const vh = Math.max(document.documentElement.clientHeight || 0, window.innerHeight || 0);
-    $("#text-preview-inner").css('max-height', (vh - 300) + "px");
-    $("#text-preview-before-inner").css('max-height', (vh - 300) + "px");
+    $("#text-preview-inner").css('max-height', (vh-300)+"px");
+    $("#text-preview-before-inner").css('max-height', (vh-300)+"px");
 
     $("#activate-text-preview").click(function (e) {
         $('body').toggleClass('preview-text-enabled')

changedetectionio/static/styles/scss/parts/_browser-steps.scss

@@ -40,22 +40,19 @@
   }
 }
 
-@media only screen and (min-width: 760px) {
 
-  #browser-steps .flex-wrapper {
-    display: flex;
-    flex-flow: row;
-    height: 70vh;
-    font-size: 80%;
+#browser-steps .flex-wrapper {
+  display: flex;
+  flex-flow: row;
+  height: 70vh;
+  font-size: 80%;
+  #browser-steps-ui {
+    flex-grow: 1;      /* Allow it to grow and fill the available space */
+    flex-shrink: 1;    /* Allow it to shrink if needed */
+    flex-basis: 0;     /* Start with 0 base width so it stretches as much as possible */
+    background-color: #eee;
+    border-radius: 5px;
 
-    #browser-steps-ui {
-      flex-grow: 1; /* Allow it to grow and fill the available space */
-      flex-shrink: 1; /* Allow it to shrink if needed */
-      flex-basis: 0; /* Start with 0 base width so it stretches as much as possible */
-      background-color: #eee;
-      border-radius: 5px;
-
-    }
   }
 
   #browser-steps-fieldlist {
@@ -66,21 +63,15 @@
     padding-left: 1rem;
     overflow-y: scroll;
   }
-
-  /*  this is duplicate :( */
-  #browsersteps-selector-wrapper {
-    height: 100% !important;
-  }
 }
 
 /*  this is duplicate :( */
 #browsersteps-selector-wrapper {
-
+  height: 100%;
   width: 100%;
   overflow-y: scroll;
   position: relative;
-  height: 80vh;
-
+  //width: 100%;
   > img {
     position: absolute;
     max-width: 100%;
@@ -100,6 +91,7 @@
     left: 50%;
     top: 50%;
     transform: translate(-50%, -50%);
+    margin-left: -40px;
     z-index: 100;
     max-width: 350px;
     text-align: center;

changedetectionio/static/styles/scss/styles.scss

@@ -380,15 +380,7 @@ a.pure-button-selected {
 }
 
 .notifications-wrapper {
-  padding-top: 0.5rem;
-  #notification-test-log {
-    padding-top: 1rem;
-    white-space: pre-wrap;
-    word-break: break-word;
-    overflow-wrap: break-word;
-    max-width: 100%;
-    box-sizing: border-box;
-  }
+  padding: 0.5rem 0 1rem 0;
 }
 
 label {

changedetectionio/static/styles/styles.css

@@ -46,22 +46,21 @@
     #browser_steps li > label {
       display: none; }
 
-@media only screen and (min-width: 760px) {
-  #browser-steps .flex-wrapper {
-    display: flex;
-    flex-flow: row;
-    height: 70vh;
-    font-size: 80%; }
-    #browser-steps .flex-wrapper #browser-steps-ui {
-      flex-grow: 1;
-      /* Allow it to grow and fill the available space */
-      flex-shrink: 1;
-      /* Allow it to shrink if needed */
-      flex-basis: 0;
-      /* Start with 0 base width so it stretches as much as possible */
-      background-color: #eee;
-      border-radius: 5px; }
-  #browser-steps-fieldlist {
+#browser-steps .flex-wrapper {
+  display: flex;
+  flex-flow: row;
+  height: 70vh;
+  font-size: 80%; }
+  #browser-steps .flex-wrapper #browser-steps-ui {
+    flex-grow: 1;
+    /* Allow it to grow and fill the available space */
+    flex-shrink: 1;
+    /* Allow it to shrink if needed */
+    flex-basis: 0;
+    /* Start with 0 base width so it stretches as much as possible */
+    background-color: #eee;
+    border-radius: 5px; }
+  #browser-steps .flex-wrapper #browser-steps-fieldlist {
     flex-grow: 0;
     /* Don't allow it to grow */
     flex-shrink: 0;
@@ -72,16 +71,13 @@
     /* Set a max width to prevent overflow */
     padding-left: 1rem;
     overflow-y: scroll; }
-  /*  this is duplicate :( */
-  #browsersteps-selector-wrapper {
-    height: 100% !important; } }
 
 /*  this is duplicate :( */
 #browsersteps-selector-wrapper {
+  height: 100%;
   width: 100%;
   overflow-y: scroll;
   position: relative;
-  height: 80vh;
   /* nice tall skinny one */ }
   #browsersteps-selector-wrapper > img {
     position: absolute;
@@ -96,6 +92,7 @@
     left: 50%;
     top: 50%;
     transform: translate(-50%, -50%);
+    margin-left: -40px;
     z-index: 100;
     max-width: 350px;
     text-align: center; }
@@ -783,14 +780,7 @@ a.pure-button-selected {
   cursor: pointer; }
 
 .notifications-wrapper {
-  padding-top: 0.5rem; }
-  .notifications-wrapper #notification-test-log {
-    padding-top: 1rem;
-    white-space: pre-wrap;
-    word-break: break-word;
-    overflow-wrap: break-word;
-    max-width: 100%;
-    box-sizing: border-box; }
+  padding: 0.5rem 0 1rem 0; }
 
 label:hover {
   cursor: pointer; }

changedetectionio/store.py

@@ -374,7 +374,7 @@ class ChangeDetectionStore:
     def visualselector_data_is_ready(self, watch_uuid):
         output_path = "{}/{}".format(self.datastore_path, watch_uuid)
         screenshot_filename = "{}/last-screenshot.png".format(output_path)
-        elements_index_filename = "{}/elements.deflate".format(output_path)
+        elements_index_filename = "{}/elements.json".format(output_path)
         if path.isfile(screenshot_filename) and  path.isfile(elements_index_filename) :
             return True
 
@@ -909,18 +909,3 @@ class ChangeDetectionStore:
             if self.data['watching'][uuid].get('in_stock_only'):
                 del (self.data['watching'][uuid]['in_stock_only'])
 
-    # Compress old elements.json to elements.deflate, saving disk, this compression is pretty fast.
-    def update_19(self):
-        import zlib
-
-        for uuid, watch in self.data['watching'].items():
-            json_path = os.path.join(self.datastore_path, uuid, "elements.json")
-            deflate_path = os.path.join(self.datastore_path, uuid, "elements.deflate")
-
-            if os.path.exists(json_path):
-                with open(json_path, "rb") as f_j:
-                    with open(deflate_path, "wb") as f_d:
-                        logger.debug(f"Compressing {str(json_path)} to {str(deflate_path)}..")
-                        f_d.write(zlib.compress(f_j.read()))
-                        os.unlink(json_path)
-

changedetectionio/templates/_common_fields.html

@@ -12,25 +12,23 @@
                             }}
                             <div class="pure-form-message-inline">
                                 <p>
-                                <strong>Tip:</strong> Use <a target="newwindow" href="https://github.com/caronc/apprise">AppRise Notification URLs</a> for notification to just about any service! <i><a target="newwindow" href="https://github.com/dgtlmoon/changedetection.io/wiki/Notification-configuration-notes">Please read the notification services wiki here for important configuration notes</a></i>.<br>
+                                <strong>Tip:</strong> Use <a target=_new href="https://github.com/caronc/apprise">AppRise Notification URLs</a> for notification to just about any service! <i><a target=_new href="https://github.com/dgtlmoon/changedetection.io/wiki/Notification-configuration-notes">Please read the notification services wiki here for important configuration notes</a></i>.<br>
 </p>
                                 <div data-target="#advanced-help-notifications" class="toggle-show pure-button button-tag button-xsmall">Show advanced help and tips</div>
                                 <ul style="display: none" id="advanced-help-notifications">
-                                <li><code><a target="newwindow" href="https://github.com/caronc/apprise/wiki/Notify_discord">discord://</a></code> (or <code>https://discord.com/api/webhooks...</code>)) only supports a maximum <strong>2,000 characters</strong> of notification text, including the title.</li>
-                                <li><code><a target="newwindow" href="https://github.com/caronc/apprise/wiki/Notify_telegram">tgram://</a></code> bots can't send messages to other bots, so you should specify chat ID of non-bot user.</li>
-                                <li><code><a target="newwindow" href="https://github.com/caronc/apprise/wiki/Notify_telegram">tgram://</a></code> only supports very limited HTML and can fail when extra tags are sent, <a href="https://core.telegram.org/bots/api#html-style">read more here</a> (or use plaintext/markdown format)</li>
+                                <li><code><a target=_new href="https://github.com/caronc/apprise/wiki/Notify_discord">discord://</a></code> (or <code>https://discord.com/api/webhooks...</code>)) only supports a maximum <strong>2,000 characters</strong> of notification text, including the title.</li>
+                                <li><code><a target=_new href="https://github.com/caronc/apprise/wiki/Notify_telegram">tgram://</a></code> bots can't send messages to other bots, so you should specify chat ID of non-bot user.</li>
+                                <li><code><a target=_new href="https://github.com/caronc/apprise/wiki/Notify_telegram">tgram://</a></code> only supports very limited HTML and can fail when extra tags are sent, <a href="https://core.telegram.org/bots/api#html-style">read more here</a> (or use plaintext/markdown format)</li>
                                 <li><code>gets://</code>, <code>posts://</code>, <code>puts://</code>, <code>deletes://</code> for direct API calls (or omit the "<code>s</code>" for non-SSL ie <code>get://</code>) <a href="https://github.com/dgtlmoon/changedetection.io/wiki/Notification-configuration-notes#postposts">more help here</a></li>
                                   <li>Accepts the <code>{{ '{{token}}' }}</code> placeholders listed below</li>
                               </ul>
                             </div>
                             <div class="notifications-wrapper">
-                              <a id="send-test-notification" class="pure-button button-secondary button-xsmall" >Send test notification</a> <div class="spinner"  style="display: none;"></div>
+                              <a id="send-test-notification" class="pure-button button-secondary button-xsmall" >Send test notification</a>
                             {% if emailprefix %}
                               <a id="add-email-helper" class="pure-button button-secondary button-xsmall" >Add email <img style="height: 1em; display: inline-block" src="{{url_for('static_content', group='images', filename='email.svg')}}" alt="Add an email address"> </a>
                             {% endif %}
                               <a href="{{url_for('notification_logs')}}" class="pure-button button-secondary button-xsmall" >Notification debug logs</a>
-                              <br>
-                                <div id="notification-test-log" style="display: none;"><span class="pure-form-message-inline">Processing..</span></div>
                             </div>
                         </div>
                         <div id="notification-customisation" class="pure-control-group">
@@ -40,7 +38,7 @@
                             </div>
                             <div class="pure-control-group">
                                 {{ render_field(form.notification_body , rows=5, class="notification-body", placeholder=settings_application['notification_body']) }}
-                                <span class="pure-form-message-inline">Body for all notifications &dash; You can use <a target="newwindow" href="https://jinja.palletsprojects.com/en/3.0.x/templates/">Jinja2</a> templating in the notification title, body and URL, and tokens from below.
+                                <span class="pure-form-message-inline">Body for all notifications &dash; You can use <a target="_new" href="https://jinja.palletsprojects.com/en/3.0.x/templates/">Jinja2</a> templating in the notification title, body and URL, and tokens from below.
                                 </span>
 
                             </div>
@@ -126,7 +124,7 @@
                                 <div class="pure-form-message-inline">
                                     <p>
 									Warning: Contents of <code>{{ '{{diff}}' }}</code>, <code>{{ '{{diff_removed}}' }}</code>, and <code>{{ '{{diff_added}}' }}</code> depend on how the difference algorithm perceives the change. <br>
-                                    For example, an addition or removal could be perceived as a change in some cases. <a target="newwindow" href="https://github.com/dgtlmoon/changedetection.io/wiki/Using-the-%7B%7Bdiff%7D%7D,-%7B%7Bdiff_added%7D%7D,-and-%7B%7Bdiff_removed%7D%7D-notification-tokens">More Here</a> <br>
+                                    For example, an addition or removal could be perceived as a change in some cases. <a target="_new" href="https://github.com/dgtlmoon/changedetection.io/wiki/Using-the-%7B%7Bdiff%7D%7D,-%7B%7Bdiff_added%7D%7D,-and-%7B%7Bdiff_removed%7D%7D-notification-tokens">More Here</a> <br>
                                     </p>
                                     <p>
                                         For JSON payloads, use <strong>|tojson</strong> without quotes for automatic escaping, for example - <code>{ "name": {{ '{{ watch_title|tojson }}' }} }</code>

changedetectionio/templates/_helpers.html

@@ -59,112 +59,4 @@
 
 {% macro render_button(field) %}
   {{ field(**kwargs)|safe }}
-{% endmacro %}
-
-{% macro playwright_warning() %}
-    <p><strong>Error - Playwright support for Chrome based fetching is not enabled.</strong> Alternatively try our <a href="https://changedetection.io">very affordable subscription based service which has all this setup for you</a>.</p>
-    <p>You may need to <a href="https://github.com/dgtlmoon/changedetection.io/blob/09ebc6ec6338545bdd694dc6eee57f2e9d2b8075/docker-compose.yml#L31">Enable playwright environment variable</a> and uncomment the <strong>sockpuppetbrowser</strong> in the <a href="https://github.com/dgtlmoon/changedetection.io/blob/master/docker-compose.yml">docker-compose.yml</a> file.</p>
-    <br>
-    <p>(Also Selenium/WebDriver can not extract full page screenshots reliably so Playwright is recommended here)</p>
-
-{% endmacro %}
-
-{% macro only_webdriver_type_watches_warning() %}
-    <p><strong>Sorry, this functionality only works with Playwright/Chrome enabled watches.<br>You need to <a href="#request">Set the fetch method to Playwright/Chrome mode and resave</a> and have the Playwright connection enabled.</strong></p><br>
-{% endmacro %}
-
-{% macro render_time_schedule_form(form, available_timezones, timezone_default_config) %}
-    <style>
-    .day-schedule *, .day-schedule select {
-        display: inline-block;
-    }
-
-    .day-schedule label[for*="time_schedule_limit-"][for$="-enabled"] {
-        min-width: 6rem;
-        font-weight: bold;
-    }
-    .day-schedule label {
-        font-weight: normal;
-    }
-
-    .day-schedule table label {
-        padding-left: 0.5rem;
-        padding-right: 0.5rem;
-    }
-    #timespan-warning, input[id*='time_schedule_limit-timezone'].error {
-        color: #ff0000;
-    }
-    .day-schedule.warning table {
-        background-color: #ffbbc2;
-    }
-    ul#day-wrapper {
-        list-style: none;
-    }
-    #timezone-info > * {
-        display: inline-block;
-    }
-
-    #scheduler-icon-label {
-        background-position: left center;
-        background-repeat: no-repeat;
-        background-size: contain;
-        display: inline-block;
-        vertical-align: middle;
-        padding-left: 50px;
-        background-image: url({{ url_for('static_content', group='images', filename='schedule.svg') }});
-    }
-    #timespan-warning {
-        display: none;
-    }
-    </style>
-    <br>
-
-    {% if timezone_default_config %}
-    <div>
-        <span id="scheduler-icon-label" style="">
-            {{ render_checkbox_field(form.time_schedule_limit.enabled) }}
-            <div class="pure-form-message-inline">
-                Set a hourly/week day schedule
-            </div>
-        </span>
-
-    </div>
-    <br>
-    <div id="schedule-day-limits-wrapper">
-        <label>Schedule time limits</label><a data-template="business-hours"
-                                              class="set-schedule pure-button button-secondary button-xsmall">Business
-        hours</a>
-        <a data-template="weekend" class="set-schedule pure-button button-secondary button-xsmall">Weekends</a>
-        <a data-template="reset" class="set-schedule pure-button button-xsmall">Reset</a><br>
-        <br>
-
-        <ul id="day-wrapper">
-            {% for day in ['monday', 'tuesday', 'wednesday', 'thursday', 'friday', 'saturday', 'sunday'] %}
-                <li class="day-schedule" id="schedule-{{ day }}">
-                    {{ render_nolabel_field(form.time_schedule_limit[day]) }}
-                </li>
-            {% endfor %}
-            <li id="timespan-warning">Warning, one or more of your 'days' has a duration that would extend into the next day.<br>
-            This could have unintended consequences.</li>
-            <li id="timezone-info">
-                {{ render_field(form.time_schedule_limit.timezone, placeholder=timezone_default_config) }} <span id="local-time-in-tz"></span>
-                <datalist id="timezones" style="display: none;">
-                    {% for timezone in available_timezones %}
-                        <option value="{{ timezone }}">{{ timezone }}</option>
-                    {% endfor %}
-                </datalist>
-            </li>
-        </ul>
-    <br>
-        <span class="pure-form-message-inline">
-         <a href="https://changedetection.io/tutorials">More help and examples about using the scheduler</a>
-        </span>
-    </div>
-    {% else %}
-        <span class="pure-form-message-inline">
-            Want to use a time schedule? <a href="{{url_for('settings_page')}}#timedate">First confirm/save your Time Zone Settings</a>
-        </span>
-        <br>
-    {% endif %}
-
 {% endmacro %}

changedetectionio/templates/base.html

@@ -159,7 +159,7 @@
                     <a id="chrome-extension-link"
                        title="Try our new Chrome Extension!"
                        href="https://chromewebstore.google.com/detail/changedetectionio-website/kefcfmgmlhmankjmnbijimhofdjekbop">
-                        <img alt="Chrome store icon" src="{{url_for('static_content', group='images', filename='Google-Chrome-icon.png')}}">
+                        <img src="{{url_for('static_content', group='images', filename='Google-Chrome-icon.png')}}">
                         Chrome Webstore
                     </a>
                 </p>

changedetectionio/templates/edit.html

@@ -1,11 +1,10 @@
 {% extends 'base.html' %}
 {% block content %}
-{% from '_helpers.html' import render_field, render_checkbox_field, render_button, render_time_schedule_form, playwright_warning, only_webdriver_type_watches_warning %}
+{% from '_helpers.html' import render_field, render_checkbox_field, render_button %}
 {% from '_common_fields.html' import render_common_settings_form %}
 <script src="{{url_for('static_content', group='js', filename='tabs.js')}}" defer></script>
 <script src="{{url_for('static_content', group='js', filename='vis.js')}}" defer></script>
 <script src="{{url_for('static_content', group='js', filename='global-settings.js')}}" defer></script>
-<script src="{{url_for('static_content', group='js', filename='scheduler.js')}}" defer></script>
 <script>
     const browser_steps_available_screenshots=JSON.parse('{{ watch.get_browsersteps_available_screenshots|tojson }}');
     const browser_steps_config=JSON.parse('{{ browser_steps_config|tojson }}');
@@ -40,13 +39,14 @@
 
     <div class="tabs collapsable">
         <ul>
-            <li class="tab"><a href="#general">General</a></li>
+            <li class="tab" id=""><a href="#general">General</a></li>
             <li class="tab"><a href="#request">Request</a></li>
             {% if extra_tab_content %}
             <li class="tab"><a href="#extras_tab">{{ extra_tab_content }}</a></li>
             {% endif %}
+            {% if playwright_enabled %}
             <li class="tab"><a id="browsersteps-tab" href="#browser-steps">Browser Steps</a></li>
-        <!-- should goto extra forms? -->
+            {% endif %}
             {% if watch['processor'] == 'text_json_diff' %}
             <li class="tab"><a id="visualselector-tab" href="#visualselector">Visual Filter Selector</a></li>
             <li class="tab" id="filters-and-triggers-tab"><a href="#filters-and-triggers">Filters &amp; Triggers</a></li>
@@ -58,7 +58,7 @@
 
     <div class="box-wrap inner">
         <form class="pure-form pure-form-stacked"
-              action="{{ url_for('edit_page', uuid=uuid, next = request.args.get('next'), unpause_on_save = request.args.get('unpause_on_save'), tag = request.args.get('tag')) }}" method="POST">
+              action="{{ url_for('edit_page', uuid=uuid, next = request.args.get('next'), unpause_on_save = request.args.get('unpause_on_save')) }}" method="POST">
              <input type="hidden" name="csrf_token" value="{{ csrf_token() }}">
 
             <div class="tab-pane-inner" id="general">
@@ -79,24 +79,9 @@
                         <span class="pure-form-message-inline">Organisational tag/group name used in the main listing page</span>
                     </div>
                     <div class="pure-control-group time-between-check border-fieldset">
-
+                        {{ render_field(form.time_between_check, class="time-check-widget") }}
                         {{ render_checkbox_field(form.time_between_check_use_default, class="use-default-timecheck") }}
-                        <br>
-                        <div id="time-check-widget-wrapper">
-                            {{ render_field(form.time_between_check, class="time-check-widget") }}
-
-                            <span class="pure-form-message-inline">
-                             The interval/amount of time between each check.
-                            </span>
-                        </div>
-                        <div id="time-between-check-schedule">
-                            <!-- Start Time and End Time -->
-                            <div id="limit-between-time">
-                                {{ render_time_schedule_form(form, available_timezones, timezone_default_config) }}
-                            </div>
-                        </div>
-<br>
-              </div>
+                    </div>
                     <div class="pure-control-group">
                         {{ render_checkbox_field(form.extract_title_as_title) }}
                     </div>
@@ -198,9 +183,8 @@ Math: {{ 1 + 1 }}") }}
                     </div>
             </fieldset>
             </div>
-
+            {% if playwright_enabled %}
             <div class="tab-pane-inner" id="browser-steps">
-            {% if playwright_enabled and watch_uses_webdriver %}
                 <img class="beta-logo" src="{{url_for('static_content', group='images', filename='beta-logo.png')}}" alt="New beta functionality">
                 <fieldset>
                     <div class="pure-control-group">
@@ -224,7 +208,7 @@ Math: {{ 1 + 1 }}") }}
                                     <span class="loader" >
                                         <span id="browsersteps-click-start">
                                             <h2 >Click here to Start</h2>
-                                            <svg style="height: 3.5rem;" version="1.1" viewBox="0 0 32 32"  xml:space="preserve" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink"><g id="start"/><g id="play_x5F_alt"><path d="M16,0C7.164,0,0,7.164,0,16s7.164,16,16,16s16-7.164,16-16S24.836,0,16,0z M10,24V8l16.008,8L10,24z" style="fill: var(--color-grey-400);"/></g></svg><br>
+                                            <svg style="height: 3.5rem;" version="1.1" viewBox="0 0 32 32"  xml:space="preserve" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink"><g id="Layer_1"/><g id="play_x5F_alt"><path d="M16,0C7.164,0,0,7.164,0,16s7.164,16,16,16s16-7.164,16-16S24.836,0,16,0z M10,24V8l16.008,8L10,24z" style="fill: var(--color-grey-400);"/></g></svg><br>
                                             Please allow 10-15 seconds for the browser to connect.<br>
                                         </span>
                                         <div class="spinner"  style="display: none;"></div>
@@ -234,31 +218,21 @@ Math: {{ 1 + 1 }}") }}
                                 </div>
                             </div>
                             <div id="browser-steps-fieldlist" >
-                                <span id="browser-seconds-remaining">Loading</span> <span style="font-size: 80%;"> (<a target="newwindow" href="https://github.com/dgtlmoon/changedetection.io/pull/478/files#diff-1a79d924d1840c485238e66772391268a89c95b781d69091384cf1ea1ac146c9R4">?</a>) </span>
+                                <span id="browser-seconds-remaining">Loading</span> <span style="font-size: 80%;"> (<a target=_new href="https://github.com/dgtlmoon/changedetection.io/pull/478/files#diff-1a79d924d1840c485238e66772391268a89c95b781d69091384cf1ea1ac146c9R4">?</a>) </span>
                                 {{ render_field(form.browser_steps) }}
                             </div>
                         </div>
                     </div>
                 </fieldset>
-                {% else %}
-                    <span class="pure-form-message-inline">
-                        {% if not watch_uses_webdriver %}
-                            {{ only_webdriver_type_watches_warning() }}
-                        {% endif %}
-                        {%  if not playwright_enabled %}
-                            {{ playwright_warning() }}
-                        {% endif %}
-                    </span>
-                {% endif %}
             </div>
-
+            {% endif %}
 
             <div class="tab-pane-inner" id="notifications">
                 <fieldset>
                     <div  class="pure-control-group inline-radio">
                       {{ render_checkbox_field(form.notification_muted) }}
                     </div>
-                    {% if watch_uses_webdriver %}
+                    {% if is_html_webdriver %}
                     <div class="pure-control-group inline-radio">
                       {{ render_checkbox_field(form.notification_screenshot) }}
                         <span class="pure-form-message-inline">
@@ -308,7 +282,7 @@ xpath://body/div/span[contains(@class, 'example-class')]",
                           <span class="pure-form-message-inline"><strong>Note!: //text() function does not work where the &lt;element&gt; contains &lt;![CDATA[]]&gt;</strong></span><br>
                         {% endif %}
                         <span class="pure-form-message-inline">One CSS, xPath, JSON Path/JQ selector per line, <i>any</i> rules that matches will be used.<br>
-                        <span data-target="#advanced-help-selectors" class="toggle-show pure-button button-tag button-xsmall">Show advanced help and tips</span><br>
+<p><div data-target="#advanced-help-selectors" class="toggle-show pure-button button-tag button-xsmall">Show advanced help and tips</div><br></p>
                     <ul id="advanced-help-selectors" style="display: none;">
                         <li>CSS - Limit text to this CSS rule, only text matching this CSS rule is included.</li>
                         <li>JSON - Limit text to this JSON rule, using either <a href="https://pypi.org/project/jsonpath-ng/" target="new">JSONPath</a> or <a href="https://stedolan.github.io/jq/" target="new">jq</a> (if installed).
@@ -481,7 +455,7 @@ keyword") }}
 
                 <fieldset>
                     <div class="pure-control-group">
-                        {% if playwright_enabled and watch_uses_webdriver %}
+                        {% if visualselector_enabled %}
                             <span class="pure-form-message-inline" id="visual-selector-heading">
                                 The Visual Selector tool lets you select the <i>text</i> elements that will be used for the change detection. It automatically fills-in the filters in the "CSS/JSONPath/JQ/XPath Filters" box of the <a href="#filters-and-triggers">Filters & Triggers</a> tab. Use <strong>Shift+Click</strong> to select multiple items.
                             </span>
@@ -499,12 +473,11 @@ keyword") }}
                             </div>
                             <div id="selector-current-xpath" style="overflow-x: hidden"><strong>Currently:</strong>&nbsp;<span class="text">Loading...</span></div>
                         {% else %}
-                            {% if not watch_uses_webdriver %}
-                                {{ only_webdriver_type_watches_warning() }}
-                            {% endif %}
-                            {% if not playwright_enabled %}
-                                {{ playwright_warning() }}
-                            {% endif %}
+                            <span class="pure-form-message-inline">
+                                <p>Sorry, this functionality only works with Playwright/Chrome enabled watches.</p>
+                                <p>Enable the Playwright Chrome fetcher, or alternatively try our <a href="https://lemonade.changedetection.io/start">very affordable subscription based service</a>.</p>
+                                <p>This is because Selenium/WebDriver can not extract full page screenshots reliably.</p>
+                            </span>
                         {% endif %}
                     </div>
                 </fieldset>

changedetectionio/templates/settings.html

@@ -1,7 +1,7 @@
 {% extends 'base.html' %}
 
 {% block content %}
-{% from '_helpers.html' import render_field, render_checkbox_field, render_button, render_time_schedule_form %}
+{% from '_helpers.html' import render_field, render_checkbox_field, render_button %}
 {% from '_common_fields.html' import render_common_settings_form %}
 <script>
     const notification_base_url="{{url_for('ajax_callback_send_notification_test', mode="global-settings")}}";
@@ -10,11 +10,9 @@
 {% endif %}
 </script>
 <script src="{{url_for('static_content', group='js', filename='tabs.js')}}" defer></script>
-<script src="{{url_for('static_content', group='js', filename='plugins.js')}}" defer></script>
 <script src="{{url_for('static_content', group='js', filename='notifications.js')}}" defer></script>
 <script src="{{url_for('static_content', group='js', filename='vis.js')}}" defer></script>
 <script src="{{url_for('static_content', group='js', filename='global-settings.js')}}" defer></script>
-<script src="{{url_for('static_content', group='js', filename='scheduler.js')}}" defer></script>
 <div class="edit-form">
     <div class="tabs collapsable">
         <ul>
@@ -23,7 +21,6 @@
             <li class="tab"><a href="#fetching">Fetching</a></li>
             <li class="tab"><a href="#filters">Global Filters</a></li>
             <li class="tab"><a href="#api">API</a></li>
-            <li class="tab"><a href="#timedate">Time &amp Date</a></li>
             <li class="tab"><a href="#proxies">CAPTCHA &amp; Proxies</a></li>
         </ul>
     </div>
@@ -35,12 +32,6 @@
                     <div class="pure-control-group">
                         {{ render_field(form.requests.form.time_between_check, class="time-check-widget") }}
                         <span class="pure-form-message-inline">Default recheck time for all watches, current system minimum is <i>{{min_system_recheck_seconds}}</i> seconds (<a href="https://github.com/dgtlmoon/changedetection.io/wiki/Misc-system-settings#enviroment-variables">more info</a>).</span>
-                            <div id="time-between-check-schedule">
-                                <!-- Start Time and End Time -->
-                                <div id="limit-between-time">
-                                    {{ render_time_schedule_form(form.requests, available_timezones, timezone_default_config) }}
-                                </div>
-                        </div>
                     </div>
                     <div class="pure-control-group">
                         {{ render_field(form.requests.form.jitter_seconds, class="jitter_seconds") }}
@@ -87,6 +78,10 @@
                         {{ render_checkbox_field(form.application.form.empty_pages_are_a_change) }}
                         <span class="pure-form-message-inline">When a request returns no content, or the HTML does not contain any text, is this considered a change?</span>
                     </div>
+                    <div class="pure-control-group">
+                        <p><strong>Local Time:</strong> {{ system_time }}</p>
+                        <p><strong>Timezone:</strong> {{ timezone_name }}</p>
+                    </div>
                 {% if form.requests.proxy %}
                     <div class="pure-control-group inline-radio">
                         {{ render_field(form.requests.form.proxy, class="fetch-backend-proxy") }}
@@ -214,29 +209,12 @@ nav
                         <a id="chrome-extension-link"
                            title="Try our new Chrome Extension!"
                            href="https://chromewebstore.google.com/detail/changedetectionio-website/kefcfmgmlhmankjmnbijimhofdjekbop">
-                            <img alt="Chrome store icon" src="{{ url_for('static_content', group='images', filename='Google-Chrome-icon.png') }}" alt="Chrome">
+                            <img src="{{ url_for('static_content', group='images', filename='Google-Chrome-icon.png') }}" alt="Chrome">
                             Chrome Webstore
                         </a>
                     </p>
                 </div>
             </div>
-            <div class="tab-pane-inner" id="timedate">
-                <div class="pure-control-group">
-                    Ensure the settings below are correct, they are used to manage the time schedule for checking your web page watches.
-                </div>
-                <div class="pure-control-group">
-                    <p><strong>UTC Time &amp Date from Server:</strong> <span id="utc-time" >{{ utc_time }}</span></p>
-                    <p><strong>Local Time &amp Date in Browser:</strong> <span class="local-time" data-utc="{{ utc_time }}"></span></p>
-                    <p>
-                       {{ render_field(form.application.form.timezone) }}
-                        <datalist id="timezones" style="display: none;">
-                            {% for tz_name in available_timezones %}
-                                <option value="{{ tz_name }}">{{ tz_name }}</option>
-                            {% endfor %}
-                        </datalist>
-                    </p>
-                </div>
-            </div>
             <div class="tab-pane-inner" id="proxies">
                 <div id="recommended-proxy">
                     <div>
@@ -280,7 +258,9 @@ nav
                         
                     </div>
                 </div>
-
+                <p>
+                    Your proxy provider may need to whitelist our IP of <code>204.15.192.195</code>
+                </p>
                <p><strong>Tip</strong>: "Residential" and "Mobile" proxy type can be more successfull than "Data Center" for blocked websites.
 
                 <div class="pure-control-group" id="extra-proxies-setting">

changedetectionio/templates/svgs/search-icon.svg

@@ -1 +1 @@
-<svg version="1.1" id="search" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" viewBox="0 0 122.879 119.799" enable-background="new 0 0 122.879 119.799" xml:space="preserve"><g><path d="M49.988,0h0.016v0.007C63.803,0.011,76.298,5.608,85.34,14.652c9.027,9.031,14.619,21.515,14.628,35.303h0.007v0.033v0.04 h-0.007c-0.005,5.557-0.917,10.905-2.594,15.892c-0.281,0.837-0.575,1.641-0.877,2.409v0.007c-1.446,3.66-3.315,7.12-5.547,10.307 l29.082,26.139l0.018,0.016l0.157,0.146l0.011,0.011c1.642,1.563,2.536,3.656,2.649,5.78c0.11,2.1-0.543,4.248-1.979,5.971 l-0.011,0.016l-0.175,0.203l-0.035,0.035l-0.146,0.16l-0.016,0.021c-1.565,1.642-3.654,2.534-5.78,2.646 c-2.097,0.111-4.247-0.54-5.971-1.978l-0.015-0.011l-0.204-0.175l-0.029-0.024L78.761,90.865c-0.88,0.62-1.778,1.209-2.687,1.765 c-1.233,0.755-2.51,1.466-3.813,2.115c-6.699,3.342-14.269,5.222-22.272,5.222v0.007h-0.016v-0.007 c-13.799-0.004-26.296-5.601-35.338-14.645C5.605,76.291,0.016,63.805,0.007,50.021H0v-0.033v-0.016h0.007 c0.004-13.799,5.601-26.296,14.645-35.338C23.683,5.608,36.167,0.016,49.955,0.007V0H49.988L49.988,0z M50.004,11.21v0.007h-0.016 h-0.033V11.21c-10.686,0.007-20.372,4.35-27.384,11.359C15.56,29.578,11.213,39.274,11.21,49.973h0.007v0.016v0.033H11.21 c0.007,10.686,4.347,20.367,11.359,27.381c7.009,7.012,16.705,11.359,27.403,11.361v-0.007h0.016h0.033v0.007 c10.686-0.007,20.368-4.348,27.382-11.359c7.011-7.009,11.358-16.702,11.36-27.4h-0.006v-0.016v-0.033h0.006 c-0.006-10.686-4.35-20.372-11.358-27.384C70.396,15.56,60.703,11.213,50.004,11.21L50.004,11.21z"/></g></svg>
+<svg version="1.1" id="Layer_1" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" viewBox="0 0 122.879 119.799" enable-background="new 0 0 122.879 119.799" xml:space="preserve"><g><path d="M49.988,0h0.016v0.007C63.803,0.011,76.298,5.608,85.34,14.652c9.027,9.031,14.619,21.515,14.628,35.303h0.007v0.033v0.04 h-0.007c-0.005,5.557-0.917,10.905-2.594,15.892c-0.281,0.837-0.575,1.641-0.877,2.409v0.007c-1.446,3.66-3.315,7.12-5.547,10.307 l29.082,26.139l0.018,0.016l0.157,0.146l0.011,0.011c1.642,1.563,2.536,3.656,2.649,5.78c0.11,2.1-0.543,4.248-1.979,5.971 l-0.011,0.016l-0.175,0.203l-0.035,0.035l-0.146,0.16l-0.016,0.021c-1.565,1.642-3.654,2.534-5.78,2.646 c-2.097,0.111-4.247-0.54-5.971-1.978l-0.015-0.011l-0.204-0.175l-0.029-0.024L78.761,90.865c-0.88,0.62-1.778,1.209-2.687,1.765 c-1.233,0.755-2.51,1.466-3.813,2.115c-6.699,3.342-14.269,5.222-22.272,5.222v0.007h-0.016v-0.007 c-13.799-0.004-26.296-5.601-35.338-14.645C5.605,76.291,0.016,63.805,0.007,50.021H0v-0.033v-0.016h0.007 c0.004-13.799,5.601-26.296,14.645-35.338C23.683,5.608,36.167,0.016,49.955,0.007V0H49.988L49.988,0z M50.004,11.21v0.007h-0.016 h-0.033V11.21c-10.686,0.007-20.372,4.35-27.384,11.359C15.56,29.578,11.213,39.274,11.21,49.973h0.007v0.016v0.033H11.21 c0.007,10.686,4.347,20.367,11.359,27.381c7.009,7.012,16.705,11.359,27.403,11.361v-0.007h0.016h0.033v0.007 c10.686-0.007,20.368-4.348,27.382-11.359c7.011-7.009,11.358-16.702,11.36-27.4h-0.006v-0.016v-0.033h0.006 c-0.006-10.686-4.35-20.372-11.358-27.384C70.396,15.56,60.703,11.213,50.004,11.21L50.004,11.21z"/></g></svg>

changedetectionio/templates/watch-overview.html

@@ -6,7 +6,7 @@
 
 <div class="box">
 
-    <form class="pure-form" action="{{ url_for('form_quick_watch_add', tag=active_tag_uuid) }}" method="POST" id="new-watch-form">
+    <form class="pure-form" action="{{ url_for('form_quick_watch_add') }}" method="POST" id="new-watch-form">
         <input type="hidden" name="csrf_token" value="{{ csrf_token() }}" >
         <fieldset>
             <legend>Add a new change detection watch</legend>
@@ -108,8 +108,7 @@
                     {% else %}
                     <a class="state-on" href="{{url_for('index', op='pause', uuid=watch.uuid, tag=active_tag_uuid)}}"><img src="{{url_for('static_content', group='images', filename='play.svg')}}" alt="UnPause checks" title="UnPause checks" class="icon icon-unpause" ></a>
                     {% endif %}
-                    {% set mute_label = 'UnMute notification' if watch.notification_muted else 'Mute notification' %}
-                    <a class="link-mute state-{{'on' if watch.notification_muted else 'off'}}" href="{{url_for('index', op='mute', uuid=watch.uuid, tag=active_tag_uuid)}}"><img src="{{url_for('static_content', group='images', filename='bell-off.svg')}}" alt="{{ mute_label }}" title="{{ mute_label }}" class="icon icon-mute" ></a>
+                    <a class="link-mute state-{{'on' if watch.notification_muted else 'off'}}" href="{{url_for('index', op='mute', uuid=watch.uuid, tag=active_tag_uuid)}}"><img src="{{url_for('static_content', group='images', filename='bell-off.svg')}}" alt="Mute notifications" title="Mute notifications" class="icon icon-mute" ></a>
                 </td>
                 <td class="title-col inline">{{watch.title if watch.title is not none and watch.title|length > 0 else watch.url}}
                     <a class="external" target="_blank" rel="noopener" href="{{ watch.link.replace('source:','') }}"></a>
@@ -119,7 +118,7 @@
                          or (  watch.get_fetch_backend == "system" and system_default_fetcher == 'html_webdriver'  )
                          or "extra_browser_" in watch.get_fetch_backend
                     %}
-                    <img class="status-icon" src="{{url_for('static_content', group='images', filename='Google-Chrome-icon.png')}}" alt="Using a Chrome browser" title="Using a Chrome browser" >
+                    <img class="status-icon" src="{{url_for('static_content', group='images', filename='Google-Chrome-icon.png')}}" title="Using a Chrome browser" >
                     {% endif %}
 
                     {%if watch.is_pdf  %}<img class="status-icon" src="{{url_for('static_content', group='images', filename='pdf-icon.svg')}}" title="Converting PDF to text" >{% endif %}
@@ -188,11 +187,11 @@
                 <td>
                     <a {% if watch.uuid in queued_uuids %}disabled="true"{% endif %} href="{{ url_for('form_watch_checknow', uuid=watch.uuid, tag=request.args.get('tag')) }}"
                        class="recheck pure-button pure-button-primary">{% if watch.uuid in queued_uuids %}Queued{% else %}Recheck{% endif %}</a>
-                    <a href="{{ url_for('edit_page', uuid=watch.uuid, tag=active_tag_uuid)}}#general" class="pure-button pure-button-primary">Edit</a>
+                    <a href="{{ url_for('edit_page', uuid=watch.uuid)}}#general" class="pure-button pure-button-primary">Edit</a>
                     {% if watch.history_n >= 2 %}
 
                         {%  if is_unviewed %}
-                           <a href="{{ url_for('diff_history_page', uuid=watch.uuid, from_version=watch.get_from_version_based_on_last_viewed) }}" target="{{watch.uuid}}" class="pure-button pure-button-primary diff-link">History</a>
+                           <a href="{{ url_for('diff_history_page', uuid=watch.uuid, from_version=watch.get_next_snapshot_key_to_last_viewed) }}" target="{{watch.uuid}}" class="pure-button pure-button-primary diff-link">History</a>
                         {% else %}
                            <a href="{{ url_for('diff_history_page', uuid=watch.uuid)}}" target="{{watch.uuid}}" class="pure-button pure-button-primary diff-link">History</a>
                         {% endif %}

changedetectionio/tests/fetchers/test_custom_js_before_content.py

@@ -34,7 +34,7 @@ def test_execute_custom_js(client, live_server, measure_memory_usage):
     assert b"unpaused" in res.data
     wait_for_all_checks(client)
 
-    uuid = next(iter(live_server.app.config['DATASTORE'].data['watching']))
+    uuid = extract_UUID_from_client(client)
     assert live_server.app.config['DATASTORE'].data['watching'][uuid].history_n >= 1, "Watch history had atleast 1 (everything fetched OK)"
 
     assert b"This text should be removed" not in res.data

changedetectionio/tests/proxy_list/test_noproxy.py

@@ -48,7 +48,7 @@ def test_noproxy_option(client, live_server, measure_memory_usage):
         follow_redirects=True
     )
     assert b"Watch added in Paused state, saving will unpause" in res.data
-    uuid = next(iter(live_server.app.config['DATASTORE'].data['watching']))
+    uuid = extract_UUID_from_client(client)
     res = client.get(
         url_for("edit_page", uuid=uuid, unpause_on_save=1))
     assert b'No proxy' in res.data

changedetectionio/tests/proxy_socks5/test_socks5_proxy.py

@@ -81,7 +81,7 @@ def test_socks5(client, live_server, measure_memory_usage):
     assert "Awesome, you made it".encode('utf-8') in res.data
 
     # PROXY CHECKER WIDGET CHECK - this needs more checking
-    uuid = next(iter(live_server.app.config['DATASTORE'].data['watching']))
+    uuid = extract_UUID_from_client(client)
 
     res = client.get(
         url_for("check_proxies.start_check", uuid=uuid),

changedetectionio/tests/test_add_replace_remove_filter.py

@@ -1,9 +1,9 @@
 #!/usr/bin/env python3
-
 import os.path
-
+import time
 from flask import url_for
 from .util import live_server_setup, wait_for_all_checks, wait_for_notification_endpoint_output
+from changedetectionio import html_tools
 
 
 def set_original(excluding=None, add_line=None):
@@ -113,8 +113,7 @@ def test_check_add_line_contains_trigger(client, live_server, measure_memory_usa
     res = client.post(
         url_for("settings_page"),
         data={"application-notification_title": "New ChangeDetection.io Notification - {{ watch_url }}",
-              # triggered_text will contain multiple lines
-              "application-notification_body": 'triggered text was -{{triggered_text}}- ### 网站监测 内容更新了 ####',
+              "application-notification_body": 'triggered text was -{{triggered_text}}- 网站监测 内容更新了',
               # https://github.com/caronc/apprise/wiki/Notify_Custom_JSON#get-parameter-manipulation
               "application-notification_urls": test_notification_url,
               "application-minutes_between_check": 180,
@@ -172,7 +171,7 @@ def test_check_add_line_contains_trigger(client, live_server, measure_memory_usa
     assert os.path.isfile("test-datastore/notification.txt"), "Notification fired because I can see the output file"
     with open("test-datastore/notification.txt", 'rb') as f:
         response = f.read()
-        assert b'-Oh yes please' in response
+        assert b'-Oh yes please-' in response
         assert '网站监测 内容更新了'.encode('utf-8') in response
 
     res = client.get(url_for("form_delete", uuid="all"), follow_redirects=True)

changedetectionio/tests/test_api.py

@@ -44,6 +44,7 @@ def set_modified_response():
 
     return None
 
+
 def is_valid_uuid(val):
     try:
         uuid.UUID(str(val))
@@ -55,9 +56,8 @@ def is_valid_uuid(val):
 def test_setup(client, live_server, measure_memory_usage):
     live_server_setup(live_server)
 
-
 def test_api_simple(client, live_server, measure_memory_usage):
-#    live_server_setup(live_server)
+    #live_server_setup(live_server)
 
     api_key = extract_api_key_from_UI(client)
 
@@ -129,9 +129,6 @@ def test_api_simple(client, live_server, measure_memory_usage):
     assert after_recheck_info['last_checked'] != before_recheck_info['last_checked']
     assert after_recheck_info['last_changed'] != 0
 
-    # #2877 When run in a slow fetcher like playwright etc
-    assert after_recheck_info['last_changed'] ==  after_recheck_info['last_checked']
-
     # Check history index list
     res = client.get(
         url_for("watchhistory", uuid=watch_uuid),

changedetectionio/tests/test_automatic_follow_ldjson_price.py

@@ -99,7 +99,7 @@ def test_check_ldjson_price_autodetect(client, live_server, measure_memory_usage
     assert b'ldjson-price-track-offer' in res.data
 
     # Accept it
-    uuid = next(iter(live_server.app.config['DATASTORE'].data['watching']))
+    uuid = extract_UUID_from_client(client)
     #time.sleep(1)
     client.get(url_for('price_data_follower.accept', uuid=uuid, follow_redirects=True))
     client.get(url_for("form_watch_checknow"), follow_redirects=True)

changedetectionio/tests/test_backend.py

@@ -2,6 +2,7 @@
 
 import time
 from flask import url_for
+from urllib.request import urlopen
 from .util import set_original_response, set_modified_response, live_server_setup, wait_for_all_checks, extract_rss_token_from_UI, \
     extract_UUID_from_client
 
@@ -68,7 +69,7 @@ def test_check_basic_change_detection_functionality(client, live_server, measure
 
     wait_for_all_checks(client)
 
-    uuid = next(iter(live_server.app.config['DATASTORE'].data['watching']))
+    uuid = extract_UUID_from_client(client)
 
     # Check the 'get latest snapshot works'
     res = client.get(url_for("watch_get_latest_html", uuid=uuid))

changedetectionio/tests/test_css_selector.py

@@ -125,7 +125,8 @@ def test_check_markup_include_filters_restriction(client, live_server, measure_m
 
 # Tests the whole stack works with the CSS Filter
 def test_check_multiple_filters(client, live_server, measure_memory_usage):
-    #live_server_setup(live_server)
+    sleep_time_for_fetch_thread = 3
+
     include_filters = "#blob-a\r\nxpath://*[contains(@id,'blob-b')]"
 
     with open("test-datastore/endpoint-content.txt", "w") as f:
@@ -137,6 +138,9 @@ def test_check_multiple_filters(client, live_server, measure_memory_usage):
      </html>
     """)
 
+    # Give the endpoint time to spin up
+    time.sleep(1)
+
     # Add our URL to the import page
     test_url = url_for('test_endpoint', _external=True)
     res = client.post(
@@ -145,7 +149,7 @@ def test_check_multiple_filters(client, live_server, measure_memory_usage):
         follow_redirects=True
     )
     assert b"1 Imported" in res.data
-    wait_for_all_checks(client)
+    time.sleep(1)
 
     # Goto the edit page, add our ignore text
     # Add our URL to the import page
@@ -161,7 +165,7 @@ def test_check_multiple_filters(client, live_server, measure_memory_usage):
     assert b"Updated watch." in res.data
 
     # Give the thread time to pick it up
-    wait_for_all_checks(client)
+    time.sleep(sleep_time_for_fetch_thread)
 
     res = client.get(
         url_for("preview_page", uuid="first"),

changedetectionio/tests/test_encoding.py

@@ -40,7 +40,7 @@ def test_check_encoding_detection(client, live_server, measure_memory_usage):
 
 
     # Content type recording worked
-    uuid = next(iter(live_server.app.config['DATASTORE'].data['watching']))
+    uuid = extract_UUID_from_client(client)
     assert live_server.app.config['DATASTORE'].data['watching'][uuid]['content-type'] == "text/html"
 
     res = client.get(

changedetectionio/tests/test_filter_failure_notification.py

@@ -51,7 +51,7 @@ def run_filter_test(client, live_server, content_filter):
     assert b"1 Imported" in res.data
     wait_for_all_checks(client)
 
-    uuid = next(iter(live_server.app.config['DATASTORE'].data['watching']))
+    uuid = extract_UUID_from_client(client)
 
     assert live_server.app.config['DATASTORE'].data['watching'][uuid]['consecutive_filter_failures'] == 0, "No filter = No filter failure"
 

changedetectionio/tests/test_group.py

@@ -288,7 +288,7 @@ def test_clone_tag_on_import(client, live_server, measure_memory_usage):
     assert b'test-tag' in res.data
     assert b'another-tag' in res.data
 
-    watch_uuid = next(iter(live_server.app.config['DATASTORE'].data['watching']))
+    watch_uuid = extract_UUID_from_client(client)
     res = client.get(url_for("form_clone", uuid=watch_uuid), follow_redirects=True)
 
     assert b'Cloned' in res.data
@@ -315,7 +315,7 @@ def test_clone_tag_on_quickwatchform_add(client, live_server, measure_memory_usa
     assert b'test-tag' in res.data
     assert b'another-tag' in res.data
 
-    watch_uuid = next(iter(live_server.app.config['DATASTORE'].data['watching']))
+    watch_uuid = extract_UUID_from_client(client)
     res = client.get(url_for("form_clone", uuid=watch_uuid), follow_redirects=True)
 
     assert b'Cloned' in res.data

changedetectionio/tests/test_ignore.py

@@ -36,7 +36,7 @@ def test_ignore(client, live_server, measure_memory_usage):
 
     # Give the thread time to pick it up
     wait_for_all_checks(client)
-    uuid = next(iter(live_server.app.config['DATASTORE'].data['watching']))
+    uuid = extract_UUID_from_client(client)
     # use the highlighter endpoint
     res = client.post(
         url_for("highlight_submit_ignore_url", uuid=uuid),

changedetectionio/tests/test_jsonpath_jq_selector.py

@@ -514,15 +514,3 @@ def test_check_jq_ext_filter(client, live_server, measure_memory_usage):
 def test_check_jqraw_ext_filter(client, live_server, measure_memory_usage):
     if jq_support:
         check_json_ext_filter('jq:.[] | select(.status | contains("Sold"))', client, live_server)
-
-def test_jsonpath_BOM_utf8(client, live_server, measure_memory_usage):
-    from .. import html_tools
-
-    # JSON string with BOM and correct double-quoted keys
-    json_str = '\ufeff{"name": "José", "emoji": "😊", "language": "中文", "greeting": "Привет"}'
-
-    # See that we can find the second <script> one, which is not broken, and matches our filter
-    text = html_tools.extract_json_as_string(json_str, "json:$.name")
-    assert text == '"José"'
-
-    

changedetectionio/tests/test_live_preview.py

@@ -29,7 +29,7 @@ def test_content_filter_live_preview(client, live_server, measure_memory_usage):
         data={"url": test_url, "tags": ''},
         follow_redirects=True
     )
-    uuid = next(iter(live_server.app.config['DATASTORE'].data['watching']))
+    uuid = extract_UUID_from_client(client)
     res = client.post(
         url_for("edit_page", uuid=uuid),
         data={

changedetectionio/tests/test_nonrenderable_pages.py

@@ -48,7 +48,7 @@ def test_check_basic_change_detection_functionality(client, live_server, measure
     #####################
     client.post(
         url_for("settings_page"),
-        data={"application-empty_pages_are_a_change": "", # default, OFF, they are NOT a change
+        data={"application-empty_pages_are_a_change": "",
               "requests-time_between_check-minutes": 180,
               'application-fetch_backend': "html_requests"},
         follow_redirects=True
@@ -66,14 +66,6 @@ def test_check_basic_change_detection_functionality(client, live_server, measure
     res = client.get(url_for("index"))
     assert b'unviewed' not in res.data
 
-    uuid = next(iter(live_server.app.config['DATASTORE'].data['watching']))
-    watch = live_server.app.config['DATASTORE'].data['watching'][uuid]
-
-    assert watch.last_changed == 0
-    assert watch['last_checked'] != 0
-
-
-
 
     # ok now do the opposite
 
@@ -100,10 +92,6 @@ def test_check_basic_change_detection_functionality(client, live_server, measure
     # A totally zero byte (#2528) response should also not trigger an error
     set_zero_byte_response()
     client.get(url_for("form_watch_checknow"), follow_redirects=True)
-
-    # 2877
-    assert watch.last_changed == watch['last_checked']
-
     wait_for_all_checks(client)
     res = client.get(url_for("index"))
     assert b'unviewed' in res.data # A change should have registered because empty_pages_are_a_change is ON

changedetectionio/tests/test_notification.py

@@ -6,7 +6,7 @@ from flask import url_for
 from loguru import logger
 
 from .util import set_original_response, set_modified_response, set_more_modified_response, live_server_setup, wait_for_all_checks, \
-    set_longer_modified_response, get_index
+    set_longer_modified_response
 from . util import  extract_UUID_from_client
 import logging
 import base64
@@ -29,7 +29,7 @@ def test_check_notification(client, live_server, measure_memory_usage):
 
     # Re 360 - new install should have defaults set
     res = client.get(url_for("settings_page"))
-    notification_url = url_for('test_notification_endpoint', _external=True).replace('http', 'json')+"?status_code=204"
+    notification_url = url_for('test_notification_endpoint', _external=True).replace('http', 'json')
 
     assert default_notification_body.encode() in res.data
     assert default_notification_title.encode() in res.data
@@ -76,7 +76,7 @@ def test_check_notification(client, live_server, measure_memory_usage):
     testimage_png = 'iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mNkYAAAAAYAAjCB0C8AAAAASUVORK5CYII='
 
 
-    uuid = next(iter(live_server.app.config['DATASTORE'].data['watching']))
+    uuid = extract_UUID_from_client(client)
     datastore = 'test-datastore'
     with open(os.path.join(datastore, str(uuid), 'last-screenshot.png'), 'wb') as f:
         f.write(base64.b64decode(testimage_png))
@@ -135,14 +135,7 @@ def test_check_notification(client, live_server, measure_memory_usage):
 
     # Trigger a check
     client.get(url_for("form_watch_checknow"), follow_redirects=True)
-    wait_for_all_checks(client)
     time.sleep(3)
-
-    # Check no errors were recorded
-    res = client.get(url_for("index"))
-    assert b'notification-error' not in res.data
-
-
     # Verify what was sent as a notification, this file should exist
     with open("test-datastore/notification.txt", "r") as f:
         notification_submission = f.read()
@@ -291,7 +284,7 @@ def test_notification_custom_endpoint_and_jinja2(client, live_server, measure_me
     # CUSTOM JSON BODY CHECK for POST://
     set_original_response()
     # https://github.com/caronc/apprise/wiki/Notify_Custom_JSON#header-manipulation
-    test_notification_url = url_for('test_notification_endpoint', _external=True).replace('http://', 'post://')+"?status_code=204&xxx={{ watch_url }}&+custom-header=123&+second=hello+world%20%22space%22"
+    test_notification_url = url_for('test_notification_endpoint', _external=True).replace('http://', 'post://')+"?xxx={{ watch_url }}&+custom-header=123&+second=hello+world%20%22space%22"
 
     res = client.post(
         url_for("settings_page"),
@@ -326,11 +319,6 @@ def test_notification_custom_endpoint_and_jinja2(client, live_server, measure_me
 
     time.sleep(2) # plus extra delay for notifications to fire
 
-
-    # Check no errors were recorded, because we asked for 204 which is slightly uncommon but is still OK
-    res = get_index(client)
-    assert b'notification-error' not in res.data
-
     with open("test-datastore/notification.txt", 'r') as f:
         x = f.read()
         j = json.loads(x)
@@ -372,10 +360,7 @@ def test_global_send_test_notification(client, live_server, measure_memory_usage
     #live_server_setup(live_server)
     set_original_response()
     if os.path.isfile("test-datastore/notification.txt"):
-        os.unlink("test-datastore/notification.txt") \
-
-    # 1995 UTF-8 content should be encoded
-    test_body = 'change detection is cool 网站监测 内容更新了'
+        os.unlink("test-datastore/notification.txt")
 
     # otherwise other settings would have already existed from previous tests in this file
     res = client.post(
@@ -383,7 +368,8 @@ def test_global_send_test_notification(client, live_server, measure_memory_usage
         data={
             "application-fetch_backend": "html_requests",
             "application-minutes_between_check": 180,
-            "application-notification_body": test_body,
+            #1995 UTF-8 content should be encoded
+            "application-notification_body": 'change detection is cool 网站监测 内容更新了',
             "application-notification_format": default_notification_format,
             "application-notification_urls": "",
             "application-notification_title": "New ChangeDetection.io Notification - {{ watch_url }}",
@@ -413,10 +399,12 @@ def test_global_send_test_notification(client, live_server, measure_memory_usage
     assert res.status_code != 400
     assert res.status_code != 500
 
+    # Give apprise time to fire
+    time.sleep(4)
 
     with open("test-datastore/notification.txt", 'r') as f:
         x = f.read()
-        assert test_body in x
+        assert 'change detection is cool 网站监测 内容更新了' in x
 
     os.unlink("test-datastore/notification.txt")
 
@@ -454,67 +442,4 @@ def test_global_send_test_notification(client, live_server, measure_memory_usage
     assert b"Error: You must have atleast one watch configured for 'test notification' to work" in res.data
 
 
-def _test_color_notifications(client, notification_body_token):
 
-    from changedetectionio.diff import ADDED_STYLE, REMOVED_STYLE
-
-    set_original_response()
-
-    if os.path.isfile("test-datastore/notification.txt"):
-        os.unlink("test-datastore/notification.txt")
-
-
-    test_notification_url = url_for('test_notification_endpoint', _external=True).replace('http://', 'post://')+"?xxx={{ watch_url }}&+custom-header=123"
-
-
-    # otherwise other settings would have already existed from previous tests in this file
-    res = client.post(
-        url_for("settings_page"),
-        data={
-            "application-fetch_backend": "html_requests",
-            "application-minutes_between_check": 180,
-            "application-notification_body": notification_body_token,
-            "application-notification_format": "HTML Color",
-            "application-notification_urls": test_notification_url,
-            "application-notification_title": "New ChangeDetection.io Notification - {{ watch_url }}",
-        },
-        follow_redirects=True
-    )
-    assert b'Settings updated' in res.data
-
-    test_url = url_for('test_endpoint', _external=True)
-    res = client.post(
-        url_for("form_quick_watch_add"),
-        data={"url": test_url, "tags": 'nice one'},
-        follow_redirects=True
-    )
-
-    assert b"Watch added" in res.data
-
-    wait_for_all_checks(client)
-
-    set_modified_response()
-
-
-    res = client.get(url_for("form_watch_checknow"), follow_redirects=True)
-    assert b'1 watches queued for rechecking.' in res.data
-
-    wait_for_all_checks(client)
-    time.sleep(3)
-
-    with open("test-datastore/notification.txt", 'r') as f:
-        x = f.read()
-        assert f'<span style="{REMOVED_STYLE}">Which is across multiple lines' in x
-
-
-    client.get(
-        url_for("form_delete", uuid="all"),
-        follow_redirects=True
-    )
-
-def test_html_color_notifications(client, live_server, measure_memory_usage):
-
-    #live_server_setup(live_server)
-    _test_color_notifications(client, '{{diff}}')
-    _test_color_notifications(client, '{{diff_full}}')
-    

changedetectionio/tests/test_request.py

@@ -373,14 +373,13 @@ def test_headers_textfile_in_request(client, live_server, measure_memory_usage):
     wait_for_all_checks(client)
 
     with open('test-datastore/headers-testtag.txt', 'w') as f:
-        f.write("tag-header: test\r\nurl-header: http://example.com")
+        f.write("tag-header: test")
 
     with open('test-datastore/headers.txt', 'w') as f:
-        f.write("global-header: nice\r\nnext-global-header: nice\r\nurl-header-global: http://example.com/global")
+        f.write("global-header: nice\r\nnext-global-header: nice")
 
-    uuid = next(iter(live_server.app.config['DATASTORE'].data['watching']))
-    with open(f'test-datastore/{uuid}/headers.txt', 'w') as f:
-        f.write("watch-header: nice\r\nurl-header-watch: http://example.com/watch")
+    with open('test-datastore/' + extract_UUID_from_client(client) + '/headers.txt', 'w') as f:
+        f.write("watch-header: nice")
 
     wait_for_all_checks(client)
     client.get(url_for("form_watch_checknow"), follow_redirects=True)
@@ -411,9 +410,6 @@ def test_headers_textfile_in_request(client, live_server, measure_memory_usage):
     assert b"Xxx:ooo" in res.data
     assert b"Watch-Header:nice" in res.data
     assert b"Tag-Header:test" in res.data
-    assert b"Url-Header:http://example.com" in res.data
-    assert b"Url-Header-Global:http://example.com/global" in res.data
-    assert b"Url-Header-Watch:http://example.com/watch" in res.data
 
     # Check the custom UA from system settings page made it through
     if os.getenv('PLAYWRIGHT_DRIVER_URL'):

changedetectionio/tests/test_restock_itemprop.py

@@ -189,17 +189,6 @@ def _run_test_minmax_limit(client, extra_watch_edit_form):
 
     client.get(url_for("mark_all_viewed"))
 
-
-    # 2715 - Price detection (once it crosses the "lower" threshold) again with a lower price - should trigger again!
-    set_original_response(props_markup=instock_props[0], price='820.45')
-    res = client.get(url_for("form_watch_checknow"), follow_redirects=True)
-    assert b'1 watches queued for rechecking.' in res.data
-    wait_for_all_checks(client)
-    res = client.get(url_for("index"))
-    assert b'820.45' in res.data
-    assert b'unviewed' in res.data
-    client.get(url_for("mark_all_viewed"))
-
     # price changed to something MORE than max (1100.10), SHOULD be a change
     set_original_response(props_markup=instock_props[0], price='1890.45')
     client.get(url_for("form_watch_checknow"), follow_redirects=True)
@@ -214,7 +203,7 @@ def _run_test_minmax_limit(client, extra_watch_edit_form):
 
 
 def test_restock_itemprop_minmax(client, live_server):
-    #live_server_setup(live_server)
+#    live_server_setup(live_server)
     extras = {
         "restock_settings-follow_price_changes": "y",
         "restock_settings-price_change_min": 900.0,
@@ -380,7 +369,7 @@ def test_change_with_notification_values(client, live_server):
 
     ## Now test the "SEND TEST NOTIFICATION" is working
     os.unlink("test-datastore/notification.txt")
-    uuid = next(iter(live_server.app.config['DATASTORE'].data['watching']))
+    uuid = extract_UUID_from_client(client)
     res = client.post(url_for("ajax_callback_send_notification_test", watch_uuid=uuid), data={}, follow_redirects=True)
     time.sleep(5)
     assert os.path.isfile("test-datastore/notification.txt"), "Notification received"

changedetectionio/tests/test_rss.py

@@ -132,7 +132,7 @@ def test_rss_xpath_filtering(client, live_server, measure_memory_usage):
     )
     assert b"Watch added in Paused state, saving will unpause" in res.data
 
-    uuid = next(iter(live_server.app.config['DATASTORE'].data['watching']))
+    uuid = extract_UUID_from_client(client)
     res = client.post(
         url_for("edit_page", uuid=uuid, unpause_on_save=1),
         data={

changedetectionio/tests/test_scheduler.py

@@ -1,179 +0,0 @@
-#!/usr/bin/env python3
-
-import time
-from datetime import datetime, timezone
-from zoneinfo import ZoneInfo
-from flask import url_for
-from .util import  live_server_setup, wait_for_all_checks, extract_UUID_from_client
-
-def test_setup(client, live_server):
-    live_server_setup(live_server)
-
-def test_check_basic_scheduler_functionality(client, live_server, measure_memory_usage):
-    #live_server_setup(live_server)
-    days = ['monday', 'tuesday', 'wednesday', 'thursday', 'friday', 'saturday', 'sunday']
-    test_url = url_for('test_random_content_endpoint', _external=True)
-
-    # We use "Pacific/Kiritimati" because its the furthest +14 hours, so it might show up more interesting bugs
-    # The rest of the actual functionality should be covered in the unit-test  unit/test_scheduler.py
-    #####################
-    res = client.post(
-        url_for("settings_page"),
-        data={"application-empty_pages_are_a_change": "",
-              "requests-time_between_check-seconds": 1,
-              "application-timezone": "Pacific/Kiritimati",  # Most Forward Time Zone (UTC+14:00)
-              'application-fetch_backend': "html_requests"},
-        follow_redirects=True
-    )
-
-    assert b"Settings updated." in res.data
-
-    res = client.get(url_for("settings_page"))
-    assert b'Pacific/Kiritimati' in res.data
-
-    res = client.post(
-        url_for("import_page"),
-        data={"urls": test_url},
-        follow_redirects=True
-    )
-
-    assert b"1 Imported" in res.data
-    wait_for_all_checks(client)
-    uuid = next(iter(live_server.app.config['DATASTORE'].data['watching']))
-
-    # Setup all the days of the weeks using XXX as the placeholder for monday/tuesday/etc
-
-    tpl = {
-        "time_schedule_limit-XXX-start_time": "00:00",
-        "time_schedule_limit-XXX-duration-hours": 24,
-        "time_schedule_limit-XXX-duration-minutes": 0,
-        "time_schedule_limit-XXX-enabled": '',  # All days are turned off
-        "time_schedule_limit-enabled": 'y',  # Scheduler is enabled, all days however are off.
-    }
-
-    scheduler_data = {}
-    for day in days:
-        for key, value in tpl.items():
-            # Replace "XXX" with the current day in the key
-            new_key = key.replace("XXX", day)
-            scheduler_data[new_key] = value
-
-    last_check = live_server.app.config['DATASTORE'].data['watching'][uuid]['last_checked']
-    data = {
-        "url": test_url,
-        "fetch_backend": "html_requests"
-    }
-    data.update(scheduler_data)
-
-    res = client.post(
-        url_for("edit_page", uuid="first"),
-        data=data,
-        follow_redirects=True
-    )
-    assert b"Updated watch." in res.data
-
-    res = client.get(url_for("edit_page", uuid="first"))
-    assert b"Pacific/Kiritimati" in res.data, "Should be Pacific/Kiritimati in placeholder data"
-
-    # "Edit" should not trigger a check because it's not enabled in the schedule.
-    time.sleep(2)
-    assert live_server.app.config['DATASTORE'].data['watching'][uuid]['last_checked'] == last_check
-
-    # Enabling today in Kiritimati should work flawless
-    kiritimati_time = datetime.now(timezone.utc).astimezone(ZoneInfo("Pacific/Kiritimati"))
-    kiritimati_time_day_of_week = kiritimati_time.strftime("%A").lower()
-    live_server.app.config['DATASTORE'].data['watching'][uuid]["time_schedule_limit"][kiritimati_time_day_of_week]["enabled"] = True
-    time.sleep(3)
-    assert live_server.app.config['DATASTORE'].data['watching'][uuid]['last_checked'] != last_check
-
-    # Cleanup everything
-    res = client.get(url_for("form_delete", uuid="all"), follow_redirects=True)
-    assert b'Deleted' in res.data
-
-
-def test_check_basic_global_scheduler_functionality(client, live_server, measure_memory_usage):
-    #live_server_setup(live_server)
-    days = ['monday', 'tuesday', 'wednesday', 'thursday', 'friday', 'saturday', 'sunday']
-    test_url = url_for('test_random_content_endpoint', _external=True)
-
-    res = client.post(
-        url_for("import_page"),
-        data={"urls": test_url},
-        follow_redirects=True
-    )
-
-    assert b"1 Imported" in res.data
-    wait_for_all_checks(client)
-    uuid = next(iter(live_server.app.config['DATASTORE'].data['watching']))
-
-    # Setup all the days of the weeks using XXX as the placeholder for monday/tuesday/etc
-
-    tpl = {
-        "requests-time_schedule_limit-XXX-start_time": "00:00",
-        "requests-time_schedule_limit-XXX-duration-hours": 24,
-        "requests-time_schedule_limit-XXX-duration-minutes": 0,
-        "requests-time_schedule_limit-XXX-enabled": '',  # All days are turned off
-        "requests-time_schedule_limit-enabled": 'y',  # Scheduler is enabled, all days however are off.
-    }
-
-    scheduler_data = {}
-    for day in days:
-        for key, value in tpl.items():
-            # Replace "XXX" with the current day in the key
-            new_key = key.replace("XXX", day)
-            scheduler_data[new_key] = value
-
-    data = {
-        "application-empty_pages_are_a_change": "",
-        "application-timezone": "Pacific/Kiritimati",  # Most Forward Time Zone (UTC+14:00)
-        'application-fetch_backend': "html_requests",
-        "requests-time_between_check-hours": 0,
-        "requests-time_between_check-minutes": 0,
-        "requests-time_between_check-seconds": 1,
-    }
-    data.update(scheduler_data)
-
-    #####################
-    res = client.post(
-        url_for("settings_page"),
-        data=data,
-        follow_redirects=True
-    )
-
-    assert b"Settings updated." in res.data
-
-    res = client.get(url_for("settings_page"))
-    assert b'Pacific/Kiritimati' in res.data
-
-    wait_for_all_checks(client)
-
-    # UI Sanity check
-
-    res = client.get(url_for("edit_page", uuid="first"))
-    assert b"Pacific/Kiritimati" in res.data, "Should be Pacific/Kiritimati in placeholder data"
-
-    #### HITTING SAVE SHOULD NOT TRIGGER A CHECK
-    last_check = live_server.app.config['DATASTORE'].data['watching'][uuid]['last_checked']
-    res = client.post(
-        url_for("edit_page", uuid="first"),
-        data={
-            "url": test_url,
-            "fetch_backend": "html_requests",
-            "time_between_check_use_default": "y"},
-        follow_redirects=True
-    )
-    assert b"Updated watch." in res.data
-    time.sleep(2)
-    assert live_server.app.config['DATASTORE'].data['watching'][uuid]['last_checked'] == last_check
-
-    # Enabling "today" in Kiritimati time should make the system check that watch
-    kiritimati_time = datetime.now(timezone.utc).astimezone(ZoneInfo("Pacific/Kiritimati"))
-    kiritimati_time_day_of_week = kiritimati_time.strftime("%A").lower()
-    live_server.app.config['DATASTORE'].data['settings']['requests']['time_schedule_limit'][kiritimati_time_day_of_week]["enabled"] = True
-
-    time.sleep(3)
-    assert live_server.app.config['DATASTORE'].data['watching'][uuid]['last_checked'] != last_check
-
-    # Cleanup everything
-    res = client.get(url_for("form_delete", uuid="all"), follow_redirects=True)
-    assert b'Deleted' in res.data

changedetectionio/tests/test_security.py

@@ -1,7 +1,9 @@
 import os
 
 from flask import url_for
-from .util import live_server_setup, wait_for_all_checks
+from .util import set_original_response, set_modified_response, live_server_setup, wait_for_all_checks
+import time
+
 from .. import strtobool
 
 
@@ -59,44 +61,54 @@ def test_bad_access(client, live_server, measure_memory_usage):
     assert b'Watch protocol is not permitted by SAFE_PROTOCOL_REGEX' in res.data
 
 
-def _runner_test_various_file_slash(client, file_uri):
+def test_file_slashslash_access(client, live_server, measure_memory_usage):
+    #live_server_setup(live_server)
 
+    test_file_path = os.path.abspath(__file__)
+
+    # file:// is permitted by default, but it will be caught by ALLOW_FILE_URI
     client.post(
         url_for("form_quick_watch_add"),
-        data={"url": file_uri, "tags": ''},
+        data={"url": f"file://{test_file_path}", "tags": ''},
         follow_redirects=True
     )
     wait_for_all_checks(client)
     res = client.get(url_for("index"))
 
-    substrings = [b"URLs with hostname components are not permitted", b"No connection adapters were found for"]
-
-
     # If it is enabled at test time
     if strtobool(os.getenv('ALLOW_FILE_URI', 'false')):
-        if file_uri.startswith('file:///'):
-            # This one should be the full qualified path to the file and should get the contents of this file
-            res = client.get(
-                url_for("preview_page", uuid="first"),
-                follow_redirects=True
-            )
-            assert b'_runner_test_various_file_slash' in res.data
-        else:
-            # This will give some error from requests or if it went to chrome, will give some other error :-)
-            assert any(s in res.data for s in substrings)
+        res = client.get(
+            url_for("preview_page", uuid="first"),
+            follow_redirects=True
+        )
 
-    res = client.get(url_for("form_delete", uuid="all"), follow_redirects=True)
-    assert b'Deleted' in res.data
+        assert b"test_file_slashslash_access" in res.data
+    else:
+        # Default should be here
+        assert b'file:// type access is denied for security reasons.' in res.data
 
 def test_file_slash_access(client, live_server, measure_memory_usage):
     #live_server_setup(live_server)
 
-    # file: is NOT permitted by default, so it will be caught by ALLOW_FILE_URI check
-
     test_file_path = os.path.abspath(__file__)
-    _runner_test_various_file_slash(client, file_uri=f"file://{test_file_path}")
-    _runner_test_various_file_slash(client, file_uri=f"file:/{test_file_path}")
-    _runner_test_various_file_slash(client, file_uri=f"file:{test_file_path}") # CVE-2024-56509
+
+    # file:// is permitted by default, but it will be caught by ALLOW_FILE_URI
+    client.post(
+        url_for("form_quick_watch_add"),
+        data={"url": f"file:/{test_file_path}", "tags": ''},
+        follow_redirects=True
+    )
+    wait_for_all_checks(client)
+    res = client.get(url_for("index"))
+
+    # If it is enabled at test time
+    if strtobool(os.getenv('ALLOW_FILE_URI', 'false')):
+        # So it should permit it, but it should fall back to the 'requests' library giving an error
+        # (but means it gets passed to playwright etc)
+        assert b"URLs with hostname components are not permitted" in res.data
+    else:
+        # Default should be here
+        assert b'file:// type access is denied for security reasons.' in res.data
 
 def test_xss(client, live_server, measure_memory_usage):
     #live_server_setup(live_server)

changedetectionio/tests/unit/test_scheduler.py

@@ -1,53 +0,0 @@
-#!/usr/bin/env python3
-
-# run from dir above changedetectionio/ dir
-# python3 -m unittest changedetectionio.tests.unit.test_jinja2_security
-
-import unittest
-from datetime import datetime, timedelta
-from zoneinfo import ZoneInfo
-
-class TestScheduler(unittest.TestCase):
-
-    # UTC+14:00 (Line Islands, Kiribati) is the farthest ahead, always ahead of UTC.
-    # UTC-12:00 (Baker Island, Howland Island) is the farthest behind, always one calendar day behind UTC.
-
-    def test_timezone_basic_time_within_schedule(self):
-        from changedetectionio import time_handler
-
-        timezone_str = 'Europe/Berlin'
-        debug_datetime = datetime.now(ZoneInfo(timezone_str))
-        day_of_week = debug_datetime.strftime('%A')
-        time_str = str(debug_datetime.hour)+':00'
-        duration = 60  # minutes
-
-        # The current time should always be within 60 minutes of [time_hour]:00
-        result = time_handler.am_i_inside_time(day_of_week=day_of_week,
-                                               time_str=time_str,
-                                               timezone_str=timezone_str,
-                                               duration=duration)
-
-        self.assertEqual(result, True, f"{debug_datetime} is within time scheduler {day_of_week} {time_str} in {timezone_str} for {duration} minutes")
-
-    def test_timezone_basic_time_outside_schedule(self):
-        from changedetectionio import time_handler
-
-        timezone_str = 'Europe/Berlin'
-        # We try a date in the future..
-        debug_datetime = datetime.now(ZoneInfo(timezone_str))+ timedelta(days=-1)
-        day_of_week = debug_datetime.strftime('%A')
-        time_str = str(debug_datetime.hour) + ':00'
-        duration = 60*24  # minutes
-
-        # The current time should always be within 60 minutes of [time_hour]:00
-        result = time_handler.am_i_inside_time(day_of_week=day_of_week,
-                                               time_str=time_str,
-                                               timezone_str=timezone_str,
-                                               duration=duration)
-
-        self.assertNotEqual(result, True,
-                         f"{debug_datetime} is NOT within time scheduler {day_of_week} {time_str} in {timezone_str} for {duration} minutes")
-
-
-if __name__ == '__main__':
-    unittest.main()

changedetectionio/tests/unit/test_semver.py

@@ -1,64 +0,0 @@
-#!/usr/bin/env python3
-
-# run from dir above changedetectionio/ dir
-# python3 -m unittest changedetectionio.tests.unit.test_semver
-
-import re
-import unittest
-
-
-# The SEMVER regex
-SEMVER_REGEX = r"^(?P<major>0|[1-9]\d*)\.(?P<minor>0|[1-9]\d*)\.(?P<patch>0|[1-9]\d*)(?:-(?P<prerelease>(?:0|[1-9]\d*|\d*[a-zA-Z-][0-9a-zA-Z-]*)(?:\.(?:0|[1-9]\d*|\d*[a-zA-Z-][0-9a-zA-Z-]*))*))?(?:\+(?P<buildmetadata>[0-9a-zA-Z-]+(?:\.[0-9a-zA-Z-]+)*))?$"
-
-# Compile the regex
-semver_pattern = re.compile(SEMVER_REGEX)
-
-class TestSemver(unittest.TestCase):
-    def test_valid_versions(self):
-        """Test valid semantic version strings"""
-        valid_versions = [
-            "1.0.0",
-            "0.1.0",
-            "0.0.1",
-            "1.0.0-alpha",
-            "1.0.0-alpha.1",
-            "1.0.0-0.3.7",
-            "1.0.0-x.7.z.92",
-            "1.0.0-alpha+001",
-            "1.0.0+20130313144700",
-            "1.0.0-beta+exp.sha.5114f85"
-        ]
-        for version in valid_versions:
-            with self.subTest(version=version):
-                self.assertIsNotNone(semver_pattern.match(version), f"Version {version} should be valid")
-
-    def test_invalid_versions(self):
-        """Test invalid semantic version strings"""
-        invalid_versions = [
-            "0.48.06",
-            "1.0",
-            "1.0.0-",
-# Seems to pass the semver.org regex?
-#            "1.0.0-alpha-",
-            "1.0.0+",
-            "1.0.0-alpha+",
-            "1.0.0-",
-            "01.0.0",
-            "1.01.0",
-            "1.0.01",
-            ".1.0.0",
-            "1..0.0"
-        ]
-        for version in invalid_versions:
-            with self.subTest(version=version):
-                res = semver_pattern.match(version)
-                self.assertIsNone(res, f"Version '{version}' should be invalid")
-
-    def test_our_version(self):
-        from changedetectionio import get_version
-        our_version = get_version()
-        self.assertIsNotNone(semver_pattern.match(our_version), f"Our version '{our_version}' should be a valid SEMVER string")
-
-
-if __name__ == '__main__':
-    unittest.main()

changedetectionio/tests/unit/test_watch_model.py

@@ -16,6 +16,7 @@ class TestDiffBuilder(unittest.TestCase):
         watch = Watch.model(datastore_path='/tmp', default={})
         watch.ensure_data_dir_exists()
 
+        watch['last_viewed'] = 110
 
         # Contents from the browser are always returned from the browser/requests/etc as str, str is basically UTF-16 in python
         watch.save_history_text(contents="hello world", timestamp=100, snapshot_id=str(uuid_builder.uuid4()))
@@ -24,42 +25,31 @@ class TestDiffBuilder(unittest.TestCase):
         watch.save_history_text(contents="hello world", timestamp=112, snapshot_id=str(uuid_builder.uuid4()))
         watch.save_history_text(contents="hello world", timestamp=115, snapshot_id=str(uuid_builder.uuid4()))
         watch.save_history_text(contents="hello world", timestamp=117, snapshot_id=str(uuid_builder.uuid4()))
-    
-        p = watch.get_from_version_based_on_last_viewed
-        assert p == "100", "Correct 'last viewed' timestamp was detected"
 
-        watch['last_viewed'] = 110
-        p = watch.get_from_version_based_on_last_viewed
-        assert p == "109", "Correct 'last viewed' timestamp was detected"
+        p = watch.get_next_snapshot_key_to_last_viewed
+        assert p == "112", "Correct last-viewed timestamp was detected"
 
+        # When there is only one step of difference from the end of the list, it should return second-last change
         watch['last_viewed'] = 116
-        p = watch.get_from_version_based_on_last_viewed
-        assert p == "115", "Correct 'last viewed' timestamp was detected"
+        p = watch.get_next_snapshot_key_to_last_viewed
+        assert p == "115", "Correct 'second last' last-viewed timestamp was detected when using the last timestamp"
 
         watch['last_viewed'] = 99
-        p = watch.get_from_version_based_on_last_viewed
-        assert p == "100", "When the 'last viewed' timestamp is less than the oldest snapshot, return oldest"
+        p = watch.get_next_snapshot_key_to_last_viewed
+        assert p == "100"
 
         watch['last_viewed'] = 200
-        p = watch.get_from_version_based_on_last_viewed
-        assert p == "115", "When the 'last viewed' timestamp is greater than the newest snapshot, return second newest"
+        p = watch.get_next_snapshot_key_to_last_viewed
+        assert p == "115", "When the 'last viewed' timestamp is greater than the newest snapshot, return second last "
 
         watch['last_viewed'] = 109
-        p = watch.get_from_version_based_on_last_viewed
+        p = watch.get_next_snapshot_key_to_last_viewed
         assert p == "109", "Correct when its the same time"
 
         # new empty one
         watch = Watch.model(datastore_path='/tmp', default={})
-        p = watch.get_from_version_based_on_last_viewed
+        p = watch.get_next_snapshot_key_to_last_viewed
         assert p == None, "None when no history available"
 
-        watch.save_history_text(contents="hello world", timestamp=100, snapshot_id=str(uuid_builder.uuid4()))
-        p = watch.get_from_version_based_on_last_viewed
-        assert p == "100", "Correct with only one history snapshot"
-
-        watch['last_viewed'] = 200
-        p = watch.get_from_version_based_on_last_viewed
-        assert p == "100", "Correct with only one history snapshot"
-
 if __name__ == '__main__':
     unittest.main()

changedetectionio/tests/util.py

@@ -76,14 +76,6 @@ def set_more_modified_response():
     return None
 
 
-def set_empty_text_response():
-    test_return_data = """<html><body></body></html>"""
-
-    with open("test-datastore/endpoint-content.txt", "w") as f:
-        f.write(test_return_data)
-
-    return None
-
 def wait_for_notification_endpoint_output():
     '''Apprise can take a few seconds to fire'''
     #@todo - could check the apprise object directly instead of looking for this file
@@ -223,10 +215,9 @@ def live_server_setup(live_server):
     def test_method():
         return request.method
 
-    # Where we POST to as a notification, also use a space here to test URL escaping is OK across all tests that use this. ( #2868 )
-    @live_server.app.route('/test_notification endpoint', methods=['POST', 'GET'])
+    # Where we POST to as a notification
+    @live_server.app.route('/test_notification_endpoint', methods=['POST', 'GET'])
     def test_notification_endpoint():
-
         with open("test-datastore/notification.txt", "wb") as f:
             # Debug method, dump all POST to file also, used to prove #65
             data = request.stream.read()
@@ -244,11 +235,8 @@ def live_server_setup(live_server):
                 f.write(request.content_type)
 
         print("\n>> Test notification endpoint was hit.\n", data)
+        return "Text was set"
 
-        content = "Text was set"
-        status_code = request.args.get('status_code',200)
-        resp = make_response(content, status_code)
-        return resp
 
     # Just return the verb in the request
     @live_server.app.route('/test-basicauth', methods=['GET'])
@@ -285,43 +273,15 @@ def live_server_setup(live_server):
             <p id="remove">This text should be removed</p>
               <form onsubmit="event.preventDefault();">
             <!-- obfuscated text so that we dont accidentally get a false positive due to conversion of the source :) --->
-                <button name="test-button" onclick="
-                getElementById('remove').remove();
-                getElementById('some-content').innerHTML = atob('SSBzbWVsbCBKYXZhU2NyaXB0IGJlY2F1c2UgdGhlIGJ1dHRvbiB3YXMgcHJlc3NlZCE=');
-                getElementById('reflect-text').innerHTML = getElementById('test-input-text').value;
-                ">Click here</button>
-                
-                <div id="some-content"></div>
-                
+                <button name="test-button" onclick="getElementById('remove').remove();getElementById('some-content').innerHTML = atob('SSBzbWVsbCBKYXZhU2NyaXB0IGJlY2F1c2UgdGhlIGJ1dHRvbiB3YXMgcHJlc3NlZCE=')">Click here</button>
+                <div id=some-content></div>
                 <pre>
                 {header_text.lower()}
                 </pre>
-                
-                <br>
-                <!-- used for testing that the jinja2 compiled here --->
-                <input type="text" value="" id="test-input-text" /><br>
-                <div id="reflect-text">Waiting to reflect text from #test-input-text here</div>
-              </form>
-                
-           </body>
+              </body>
          </html>""", 200)
         resp.headers['Content-Type'] = 'text/html'
         return resp
 
     live_server.start()
 
-def get_index(client):
-    import inspect
-    # Get the caller's frame (parent function)
-    frame = inspect.currentframe()
-    caller_frame = frame.f_back  # Go back to the caller's frame
-    caller_name = caller_frame.f_code.co_name
-    caller_line = caller_frame.f_lineno
-
-    print(f"Called by: {caller_name}, Line: {caller_line}")
-
-    res = client.get(url_for("index"))
-    with open(f"test-datastore/index-{caller_name}-{caller_line}.html", 'wb') as f:
-        f.write(res.data)
-
-    return res

changedetectionio/tests/visualselector/test_fetch_data.py

@@ -2,16 +2,14 @@
 
 import os
 from flask import url_for
-from ..util import live_server_setup, wait_for_all_checks, get_index
+from ..util import live_server_setup, wait_for_all_checks, extract_UUID_from_client
 
-def test_setup(client, live_server):
+def test_setup(client, live_server, measure_memory_usage):
     live_server_setup(live_server)
 
 
 # Add a site in paused mode, add an invalid filter, we should still have visual selector data ready
 def test_visual_selector_content_ready(client, live_server, measure_memory_usage):
-    live_server.stop()
-    live_server.start()
 
     import os
     import json
@@ -29,7 +27,7 @@ def test_visual_selector_content_ready(client, live_server, measure_memory_usage
         follow_redirects=True
     )
     assert b"Watch added in Paused state, saving will unpause" in res.data
-    uuid = next(iter(live_server.app.config['DATASTORE'].data['watching']))
+    uuid = extract_UUID_from_client(client)
     res = client.post(
         url_for("edit_page", uuid=uuid, unpause_on_save=1),
         data={
@@ -56,21 +54,15 @@ def test_visual_selector_content_ready(client, live_server, measure_memory_usage
 
 
     assert os.path.isfile(os.path.join('test-datastore', uuid, 'last-screenshot.png')), "last-screenshot.png should exist"
-    assert os.path.isfile(os.path.join('test-datastore', uuid, 'elements.deflate')), "xpath elements.deflate data should exist"
+    assert os.path.isfile(os.path.join('test-datastore', uuid, 'elements.json')), "xpath elements.json data should exist"
 
     # Open it and see if it roughly looks correct
-    with open(os.path.join('test-datastore', uuid, 'elements.deflate'), 'rb') as f:
-        import zlib
-        compressed_data = f.read()
-        decompressed_data = zlib.decompress(compressed_data)
-        # See if any error was thrown
-        json_data = json.loads(decompressed_data.decode('utf-8'))
+    with open(os.path.join('test-datastore', uuid, 'elements.json'), 'r') as f:
+        json.load(f)
 
     # Attempt to fetch it via the web hook that the browser would use
     res = client.get(url_for('static_content', group='visual_selector_data', filename=uuid))
-    decompressed_data = zlib.decompress(res.data)
-    json_data = json.loads(decompressed_data.decode('utf-8'))
-    
+    json.loads(res.data)
     assert res.mimetype == 'application/json'
     assert res.status_code == 200
 
@@ -89,9 +81,7 @@ def test_visual_selector_content_ready(client, live_server, measure_memory_usage
 
 def test_basic_browserstep(client, live_server, measure_memory_usage):
 
-    live_server.stop()
-    live_server.start()
-
+    #live_server_setup(live_server)
     assert os.getenv('PLAYWRIGHT_DRIVER_URL'), "Needs PLAYWRIGHT_DRIVER_URL set for this test"
 
     test_url = url_for('test_interactive_html_endpoint', _external=True)
@@ -112,13 +102,9 @@ def test_basic_browserstep(client, live_server, measure_memory_usage):
             "url": test_url,
             "tags": "",
             'fetch_backend': "html_webdriver",
-            'browser_steps-0-operation': 'Enter text in field',
-            'browser_steps-0-selector': '#test-input-text',
-            # Should get set to the actual text (jinja2 rendered)
-            'browser_steps-0-optional_value': "Hello-Jinja2-{% now  'Europe/Berlin', '%Y-%m-%d' %}",
-            'browser_steps-1-operation': 'Click element',
-            'browser_steps-1-selector': 'button[name=test-button]',
-            'browser_steps-1-optional_value': '',
+            'browser_steps-0-operation': 'Click element',
+            'browser_steps-0-selector': 'button[name=test-button]',
+            'browser_steps-0-optional_value': '',
             # For now, cookies doesnt work in headers because it must be a full cookiejar object
             'headers': "testheader: yes\buser-agent: MyCustomAgent",
         },
@@ -127,7 +113,7 @@ def test_basic_browserstep(client, live_server, measure_memory_usage):
     assert b"unpaused" in res.data
     wait_for_all_checks(client)
 
-    uuid = next(iter(live_server.app.config['DATASTORE'].data['watching']))
+    uuid = extract_UUID_from_client(client)
     assert live_server.app.config['DATASTORE'].data['watching'][uuid].history_n >= 1, "Watch history had atleast 1 (everything fetched OK)"
 
     assert b"This text should be removed" not in res.data
@@ -140,32 +126,13 @@ def test_basic_browserstep(client, live_server, measure_memory_usage):
     assert b"This text should be removed" not in res.data
     assert b"I smell JavaScript because the button was pressed" in res.data
 
-    assert b'Hello-Jinja2-20' in res.data
-
     assert b"testheader: yes" in res.data
     assert b"user-agent: mycustomagent" in res.data
-    live_server.stop()
-
-def test_non_200_errors_report_browsersteps(client, live_server):
-
-    live_server.stop()
-    live_server.start()
 
     four_o_four_url =  url_for('test_endpoint', status_code=404, _external=True)
     four_o_four_url = four_o_four_url.replace('localhost.localdomain', 'cdio')
     four_o_four_url = four_o_four_url.replace('localhost', 'cdio')
 
-    res = client.post(
-        url_for("form_quick_watch_add"),
-        data={"url": four_o_four_url, "tags": '', 'edit_and_watch_submit_button': 'Edit > Watch'},
-        follow_redirects=True
-    )
-
-    assert b"Watch added in Paused state, saving will unpause" in res.data
-    assert os.getenv('PLAYWRIGHT_DRIVER_URL'), "Needs PLAYWRIGHT_DRIVER_URL set for this test"
-
-    uuid = next(iter(live_server.app.config['DATASTORE'].data['watching']))
-
     # now test for 404 errors
     res = client.post(
         url_for("edit_page", uuid=uuid, unpause_on_save=1),
@@ -180,14 +147,12 @@ def test_non_200_errors_report_browsersteps(client, live_server):
         follow_redirects=True
     )
     assert b"unpaused" in res.data
-
     wait_for_all_checks(client)
 
-    res = get_index(client)
-
+    res = client.get(url_for("index"))
     assert b'Error - 404' in res.data
 
     client.get(
         url_for("form_delete", uuid="all"),
         follow_redirects=True
-    )
+    )

changedetectionio/time_handler.py

@@ -1,105 +0,0 @@
-from datetime import timedelta, datetime
-from enum import IntEnum
-from zoneinfo import ZoneInfo
-
-
-class Weekday(IntEnum):
-    """Enumeration for days of the week."""
-    Monday = 0
-    Tuesday = 1
-    Wednesday = 2
-    Thursday = 3
-    Friday = 4
-    Saturday = 5
-    Sunday = 6
-
-
-def am_i_inside_time(
-        day_of_week: str,
-        time_str: str,
-        timezone_str: str,
-        duration: int = 15,
-) -> bool:
-    """
-    Determines if the current time falls within a specified time range.
-
-    Parameters:
-        day_of_week (str): The day of the week (e.g., 'Monday').
-        time_str (str): The start time in 'HH:MM' format.
-        timezone_str (str): The timezone identifier (e.g., 'Europe/Berlin').
-        duration (int, optional): The duration of the time range in minutes. Default is 15.
-
-    Returns:
-        bool: True if the current time is within the time range, False otherwise.
-    """
-    # Parse the target day of the week
-    try:
-        target_weekday = Weekday[day_of_week.capitalize()]
-    except KeyError:
-        raise ValueError(f"Invalid day_of_week: '{day_of_week}'. Must be a valid weekday name.")
-
-    # Parse the start time
-    try:
-        target_time = datetime.strptime(time_str, '%H:%M').time()
-    except ValueError:
-        raise ValueError(f"Invalid time_str: '{time_str}'. Must be in 'HH:MM' format.")
-
-    # Define the timezone
-    try:
-        tz = ZoneInfo(timezone_str)
-    except Exception:
-        raise ValueError(f"Invalid timezone_str: '{timezone_str}'. Must be a valid timezone identifier.")
-
-    # Get the current time in the specified timezone
-    now_tz = datetime.now(tz)
-
-    # Check if the current day matches the target day or overlaps due to duration
-    current_weekday = now_tz.weekday()
-    start_datetime_tz = datetime.combine(now_tz.date(), target_time, tzinfo=tz)
-
-    # Handle previous day's overlap
-    if target_weekday == (current_weekday - 1) % 7:
-        # Calculate start and end times for the overlap from the previous day
-        start_datetime_tz -= timedelta(days=1)
-        end_datetime_tz = start_datetime_tz + timedelta(minutes=duration)
-        if start_datetime_tz <= now_tz < end_datetime_tz:
-            return True
-
-    # Handle current day's range
-    if target_weekday == current_weekday:
-        end_datetime_tz = start_datetime_tz + timedelta(minutes=duration)
-        if start_datetime_tz <= now_tz < end_datetime_tz:
-            return True
-
-    # Handle next day's overlap
-    if target_weekday == (current_weekday + 1) % 7:
-        end_datetime_tz = start_datetime_tz + timedelta(minutes=duration)
-        if now_tz < start_datetime_tz and now_tz + timedelta(days=1) < end_datetime_tz:
-            return True
-
-    return False
-
-
-def is_within_schedule(time_schedule_limit, default_tz="UTC"):
-    if time_schedule_limit and time_schedule_limit.get('enabled'):
-        # Get the timezone the time schedule is in, so we know what day it is there
-        tz_name = time_schedule_limit.get('timezone')
-        if not tz_name:
-            tz_name = default_tz
-
-        now_day_name_in_tz = datetime.now(ZoneInfo(tz_name.strip())).strftime('%A')
-        selected_day_schedule = time_schedule_limit.get(now_day_name_in_tz.lower())
-        if not selected_day_schedule.get('enabled'):
-            return False
-
-        duration = selected_day_schedule.get('duration')
-        selected_day_run_duration_m = int(duration.get('hours')) * 60 + int(duration.get('minutes'))
-
-        is_valid = am_i_inside_time(day_of_week=now_day_name_in_tz,
-                                    time_str=selected_day_schedule['start_time'],
-                                    timezone_str=tz_name,
-                                    duration=selected_day_run_duration_m)
-
-        return is_valid
-
-    return False

changedetectionio/update_worker.py

@@ -28,8 +28,6 @@ class update_worker(threading.Thread):
 
     def queue_notification_for_watch(self, notification_q, n_object, watch):
         from changedetectionio import diff
-        from changedetectionio.notification import default_notification_format_for_watch
-
         dates = []
         trigger_text = ''
 
@@ -46,21 +44,11 @@ class update_worker(threading.Thread):
         else:
             snapshot_contents = "No snapshot/history available, the watch should fetch atleast once."
 
-        # If we ended up here with "System default"
-        if n_object.get('notification_format') == default_notification_format_for_watch:
-            n_object['notification_format'] = self.datastore.data['settings']['application'].get('notification_format')
-
-        html_colour_enable = False
         # HTML needs linebreak, but MarkDown and Text can use a linefeed
         if n_object.get('notification_format') == 'HTML':
             line_feed_sep = "<br>"
             # Snapshot will be plaintext on the disk, convert to some kind of HTML
             snapshot_contents = snapshot_contents.replace('\n', line_feed_sep)
-        elif n_object.get('notification_format') == 'HTML Color':
-            line_feed_sep = "<br>"
-            # Snapshot will be plaintext on the disk, convert to some kind of HTML
-            snapshot_contents = snapshot_contents.replace('\n', line_feed_sep)
-            html_colour_enable = True
         else:
             line_feed_sep = "\n"
 
@@ -81,9 +69,9 @@ class update_worker(threading.Thread):
 
         n_object.update({
             'current_snapshot': snapshot_contents,
-            'diff': diff.render_diff(prev_snapshot, current_snapshot, line_feed_sep=line_feed_sep, html_colour=html_colour_enable),
+            'diff': diff.render_diff(prev_snapshot, current_snapshot, line_feed_sep=line_feed_sep),
             'diff_added': diff.render_diff(prev_snapshot, current_snapshot, include_removed=False, line_feed_sep=line_feed_sep),
-            'diff_full': diff.render_diff(prev_snapshot, current_snapshot, include_equal=True, line_feed_sep=line_feed_sep, html_colour=html_colour_enable),
+            'diff_full': diff.render_diff(prev_snapshot, current_snapshot, include_equal=True, line_feed_sep=line_feed_sep),
             'diff_patch': diff.render_diff(prev_snapshot, current_snapshot, line_feed_sep=line_feed_sep, patch_format=True),
             'diff_removed': diff.render_diff(prev_snapshot, current_snapshot, include_added=False, line_feed_sep=line_feed_sep),
             'notification_timestamp': now,
@@ -243,6 +231,7 @@ class update_worker(threading.Thread):
                 os.unlink(full_path)
 
     def run(self):
+        now = time.time()
         
         while not self.app.config.exit.is_set():
             update_handler = None
@@ -253,7 +242,6 @@ class update_worker(threading.Thread):
                 pass
 
             else:
-                fetch_start_time = time.time()
                 uuid = queued_item_data.item.get('uuid')
                 self.current_uuid = uuid
                 if uuid in list(self.datastore.data['watching'].keys()) and self.datastore.data['watching'][uuid].get('url'):
@@ -268,6 +256,7 @@ class update_worker(threading.Thread):
                     watch = self.datastore.data['watching'].get(uuid)
 
                     logger.info(f"Processing watch UUID {uuid} Priority {queued_item_data.priority} URL {watch['url']}")
+                    now = time.time()
 
                     try:
                         # Processor is what we are using for detecting the "Change"
@@ -287,10 +276,6 @@ class update_worker(threading.Thread):
 
                         update_handler.call_browser()
 
-                        # In reality, the actual time of when the change was detected could be a few seconds after this
-                        # For example it should include when the page stopped rendering if using a playwright/chrome type fetch
-                        fetch_start_time = time.time()
-
                         changed_detected, update_obj, contents = update_handler.run_changedetection(watch=watch)
 
                         # Re #342
@@ -515,7 +500,7 @@ class update_worker(threading.Thread):
 
                     if not self.datastore.data['watching'].get(uuid):
                         continue
-
+                    #
                     # Different exceptions mean that we may or may not want to bump the snapshot, trigger notifications etc
                     if process_changedetection_results:
 
@@ -528,6 +513,8 @@ class update_worker(threading.Thread):
                                 except Exception as e:
                                     logger.warning(f"UUID: {uuid} Extract <title> as watch title was enabled, but couldn't find a <title>.")
 
+                        # Now update after running everything
+                        timestamp = round(time.time())
                         try:
                             self.datastore.update_watch(uuid=uuid, update_obj=update_obj)
 
@@ -543,28 +530,24 @@ class update_worker(threading.Thread):
 
                                 # Small hack so that we sleep just enough to allow 1 second  between history snapshots
                                 # this is because history.txt indexes/keys snapshots by epoch seconds and we dont want dupe keys
-                                # @also - the keys are one per second at the most (for now)
-                                if watch.newest_history_key and int(fetch_start_time) == int(watch.newest_history_key):
+
+                                if watch.newest_history_key and int(timestamp) == int(watch.newest_history_key):
                                     logger.warning(
-                                        f"Timestamp {fetch_start_time} already exists, waiting 1 seconds so we have a unique key in history.txt")
-                                    fetch_start_time += 1
+                                        f"Timestamp {timestamp} already exists, waiting 1 seconds so we have a unique key in history.txt")
+                                    timestamp = str(int(timestamp) + 1)
                                     time.sleep(1)
 
                                 watch.save_history_text(contents=contents,
-                                                        timestamp=int(fetch_start_time),
+                                                        timestamp=timestamp,
                                                         snapshot_id=update_obj.get('previous_md5', 'none'))
 
-
-                                empty_pages_are_a_change = self.datastore.data['settings']['application'].get('empty_pages_are_a_change', False)
-                                if update_handler.fetcher.content or (not update_handler.fetcher.content and empty_pages_are_a_change):
-                                    # attribute .last_changed is then based on this data
-                                    watch.save_last_fetched_html(contents=update_handler.fetcher.content, timestamp=int(fetch_start_time))
+                                if update_handler.fetcher.content:
+                                    watch.save_last_fetched_html(contents=update_handler.fetcher.content, timestamp=timestamp)
 
                                 # Notifications should only trigger on the second time (first time, we gather the initial snapshot)
                                 if watch.history_n >= 2:
                                     logger.info(f"Change detected in UUID {uuid} - {watch['url']}")
                                     if not watch.get('notification_muted'):
-                                        # @todo only run this if notifications exist
                                         self.send_content_changed_notification(watch_uuid=uuid)
 
                         except Exception as e:
@@ -586,15 +569,15 @@ class update_worker(threading.Thread):
                     except Exception as e:
                         pass
 
-                    self.datastore.update_watch(uuid=uuid, update_obj={'fetch_time': round(time.time() - fetch_start_time, 3),
-                                                                       'last_checked': int(fetch_start_time),
+                    self.datastore.update_watch(uuid=uuid, update_obj={'fetch_time': round(time.time() - now, 3),
+                                                                       'last_checked': round(time.time()),
                                                                        'check_count': count
                                                                        })
 
 
                 self.current_uuid = None  # Done
                 self.q.task_done()
-                logger.debug(f"Watch {uuid} done in {time.time()-fetch_start_time:.2f}s")
+                logger.debug(f"Watch {uuid} done in {time.time()-now:.2f}s")
 
                 # Give the CPU time to interrupt
                 time.sleep(0.1)

docker-compose.yml

@@ -1,3 +1,4 @@
+version: '3.2'
 services:
     changedetection:
       image: ghcr.io/dgtlmoon/changedetection.io
@@ -11,6 +12,9 @@ services:
   #    environment:
   #        Default listening port, can also be changed with the -p option
   #      - PORT=5000
+
+  #      - PUID=1000
+  #      - PGID=1000
   #
   #        Log levels are in descending order. (TRACE is the most detailed one)
   #        Log output levels: TRACE, DEBUG(default), INFO, SUCCESS, WARNING, ERROR, CRITICAL
@@ -81,7 +85,7 @@ services:
 
 
      # Sockpuppetbrowser is basically chrome wrapped in an API for allowing fast fetching of web-pages.
-     # RECOMMENDED FOR FETCHING PAGES WITH CHROME, be sure to enable the "PLAYWRIGHT_DRIVER_URL" env variable in the main changedetection container
+     # RECOMMENDED FOR FETCHING PAGES WITH CHROME
 #    sockpuppetbrowser:
 #        hostname: sockpuppetbrowser
 #        image: dgtlmoon/sockpuppetbrowser:latest

requirements.txt

@@ -1,7 +1,7 @@
 # Used by Pyppeteer
 pyee
 
-eventlet>=0.38.0
+eventlet>=0.36.1 # fixes SSL error on Python 3.12
 feedgen~=0.9
 flask-compress
 # 0.6.3 included compatibility fix for werkzeug 3.x (2.x had deprecation of url handlers)
@@ -35,11 +35,12 @@ dnspython==2.6.1 # related to eventlet fixes
 # jq not available on Windows so must be installed manually
 
 # Notification library
-apprise==1.9.2
+apprise==1.9.0
 
 # apprise mqtt https://github.com/dgtlmoon/changedetection.io/issues/315
-# use any version other than 2.0.x due to https://github.com/eclipse/paho.mqtt.python/issues/814
-paho-mqtt!=2.0.*
+# and 2.0.0 https://github.com/dgtlmoon/changedetection.io/issues/2241 not yet compatible
+# use v1.x due to https://github.com/eclipse/paho.mqtt.python/issues/814
+paho-mqtt>=1.6.1,<2.0.0
 
 # Requires extra wheel for rPi
 cryptography~=42.0.8
@@ -95,8 +96,3 @@ babel
 # Needed for > 3.10, https://github.com/microsoft/playwright-python/issues/2096
 greenlet >= 3.0.3
 
-# Pinned or it causes problems with flask_expects_json which seems unmaintained
-referencing==0.35.1
-
-# Scheduler - Windows seemed to miss a lot of default timezone info (even "UTC" !)
-tzdata