bump JS

Co-authored-by: silversub <silversub@gmail.com>

.dockerignore

@@ -1,2 +1,62 @@
-.git
-.github
+# Git
+.git/
+.gitignore
+
+# GitHub
+.github/
+
+# Byte-compiled / optimized / DLL files
+**/__pycache__
+**/*.py[cod]
+
+# Caches
+.mypy_cache/
+.pytest_cache/
+.ruff_cache/
+
+# Distribution / packaging
+build/
+dist/
+*.egg-info*
+
+# Virtual environment
+.env
+.venv/
+venv/
+
+# IntelliJ IDEA
+.idea/
+
+# Visual Studio
+.vscode/
+
+# Test and development files
+test-datastore/
+tests/
+*.md
+!README.md
+
+# Temporary and log files
+*.log
+*.tmp
+tmp/
+temp/
+
+# Training data and large files
+train-data/
+works-data/
+
+# Container files
+Dockerfile*
+docker-compose*.yml
+.dockerignore
+
+# Development certificates and keys
+*.pem
+*.key
+*.crt
+profile_output.prof
+
+# Large binary files that shouldn't be in container
+*.pdf
+chrome.json

.github/ISSUE_TEMPLATE/bug_report.md

@@ -7,12 +7,30 @@ assignees: 'dgtlmoon'
 
 ---
 
+**DO NOT USE THIS FORM TO REPORT THAT A PARTICULAR WEBSITE IS NOT SCRAPING/WATCHING AS EXPECTED**
+
+This form is only for direct bugs and feature requests todo directly with the software.
+
+Please report watched websites (full URL and _any_ settings) that do not work with changedetection.io as expected [**IN THE DISCUSSION FORUMS**](https://github.com/dgtlmoon/changedetection.io/discussions) or your report will be deleted
+
+CONSIDER TAKING OUT A SUBSCRIPTION FOR A SMALL PRICE PER MONTH, YOU GET THE BENEFIT OF USING OUR PAID PROXIES AND FURTHERING THE DEVELOPMENT OF CHANGEDETECTION.IO
+
+THANK YOU
+
+
+
+
+
 **Describe the bug**
 A clear and concise description of what the bug is.
 
 **Version**
 *Exact version* in the top right area: 0....
 
+**How did you install?**
+
+Docker, Pip, from source directly etc
+
 **To Reproduce**
 
 Steps to reproduce the behavior:

.github/dependabot.yml

@@ -0,0 +1,14 @@
+version: 2
+updates:
+  - package-ecosystem: github-actions
+    directory: /
+    schedule:
+      interval: "weekly"
+    "caronc/apprise":
+      versioning-strategy: "increase"
+      schedule:
+        interval: "daily"
+    groups:
+      all:
+        patterns:
+        - "*"

.github/test/Dockerfile-alpine

@@ -0,0 +1,36 @@
+# Taken from https://github.com/linuxserver/docker-changedetection.io/blob/main/Dockerfile
+# Test that we can still build on Alpine (musl modified libc https://musl.libc.org/)
+# Some packages wont install via pypi because they dont have a wheel available under this architecture.
+
+FROM ghcr.io/linuxserver/baseimage-alpine:3.22
+ENV PYTHONUNBUFFERED=1
+
+COPY requirements.txt /requirements.txt
+
+RUN \
+ apk add --update --no-cache --virtual=build-dependencies \
+    build-base \
+    cargo \
+    git \
+    jpeg-dev \
+    libc-dev \
+    libffi-dev \
+    libxslt-dev \
+    openssl-dev \
+    python3-dev \
+    file \
+    zip \
+    zlib-dev && \
+  apk add --update --no-cache \
+    libjpeg \
+    libxslt \
+    file \
+    nodejs \
+    poppler-utils \
+    python3 && \
+  echo "**** pip3 install test of changedetection.io ****" && \
+  python3 -m venv /lsiopy  && \
+  pip install -U pip wheel setuptools && \
+  pip install -U --no-cache-dir --find-links https://wheel-index.linuxserver.io/alpine-3.22/ -r /requirements.txt && \
+  apk del --purge \
+    build-dependencies

.github/workflows/codeql-analysis.yml

@@ -30,11 +30,11 @@ jobs:
 
     steps:
     - name: Checkout repository
-      uses: actions/checkout@v2
+      uses: actions/checkout@v5
 
     # Initializes the CodeQL tools for scanning.
     - name: Initialize CodeQL
-      uses: github/codeql-action/init@v1
+      uses: github/codeql-action/init@v3
       with:
         languages: ${{ matrix.language }}
         # If you wish to specify custom queries, you can do so here or in a config file.
@@ -45,7 +45,7 @@ jobs:
     # Autobuild attempts to build any compiled languages  (C/C++, C#, or Java).
     # If this step fails, then you should remove it and run the build manually (see below)
     - name: Autobuild
-      uses: github/codeql-action/autobuild@v1
+      uses: github/codeql-action/autobuild@v3
 
     # ℹ️ Command-line programs to run using the OS shell.
     # 📚 https://git.io/JvXDl
@@ -59,4 +59,4 @@ jobs:
     #   make release
 
     - name: Perform CodeQL Analysis
-      uses: github/codeql-action/analyze@v1
+      uses: github/codeql-action/analyze@v3

.github/workflows/containers.yml

@@ -39,18 +39,17 @@ jobs:
     # Or if we are in a tagged release scenario.
     if: ${{ github.event.workflow_run.conclusion == 'success' }} || ${{ github.event.release.tag_name }} != ''
     steps:
-      - uses: actions/checkout@v2
-      - name: Set up Python 3.9
-        uses: actions/setup-python@v2
+      - uses: actions/checkout@v5
+      - name: Set up Python 3.11
+        uses: actions/setup-python@v6
         with:
-          python-version: 3.9
+          python-version: 3.11
 
       - name: Install dependencies
         run: |
           python -m pip install --upgrade pip
           pip install flake8 pytest
           if [ -f requirements.txt ]; then pip install -r requirements.txt; fi
-          if [ -f requirements-dev.txt ]; then pip install -r requirements-dev.txt; fi
 
       - name: Create release metadata
         run: |
@@ -59,27 +58,27 @@ jobs:
           echo ${{ github.ref }} > changedetectionio/tag.txt
 
       - name: Set up QEMU
-        uses: docker/setup-qemu-action@v1
+        uses: docker/setup-qemu-action@v3
         with:
           image: tonistiigi/binfmt:latest
           platforms: all
 
       - name: Login to GitHub Container Registry
-        uses: docker/login-action@v1
+        uses: docker/login-action@v3
         with:
           registry: ghcr.io
           username: ${{ github.actor }}
           password: ${{ secrets.GITHUB_TOKEN }}
 
       - name: Login to Docker Hub Container Registry
-        uses: docker/login-action@v1
+        uses: docker/login-action@v3
         with:
           username: ${{ secrets.DOCKER_HUB_USERNAME }}
           password: ${{ secrets.DOCKER_HUB_ACCESS_TOKEN }}
 
       - name: Set up Docker Buildx
         id: buildx
-        uses: docker/setup-buildx-action@v1
+        uses: docker/setup-buildx-action@v3
         with:
           install: true
           version: latest
@@ -89,42 +88,57 @@ jobs:
       - name: Build and push :dev
         id: docker_build
         if: ${{ github.ref }} == "refs/heads/master"
-        uses: docker/build-push-action@v2
+        uses: docker/build-push-action@v6
         with:
           context: ./
           file: ./Dockerfile
           push: true
           tags: |
             ${{ secrets.DOCKER_HUB_USERNAME }}/changedetection.io:dev,ghcr.io/${{ github.repository }}:dev
-          platforms: linux/amd64,linux/arm64,linux/arm/v6,linux/arm/v7
-          cache-from: type=local,src=/tmp/.buildx-cache
-          cache-to: type=local,dest=/tmp/.buildx-cache
+          platforms: linux/amd64,linux/arm64,linux/arm/v7,linux/arm/v8,linux/arm64/v8
+          cache-from: type=gha
+          cache-to: type=gha,mode=max
+
+# Looks like this was disabled
+#          provenance: false
 
       # A new tagged release is required, which builds :tag and :latest
+      - name: Debug release info
+        if: github.event_name == 'release' && startsWith(github.event.release.tag_name, '0.')
+        run: |
+          echo "Release tag: ${{ github.event.release.tag_name }}"
+          echo "Github ref: ${{ github.ref }}"
+          echo "Github ref name: ${{ github.ref_name }}"
+          
+      - name: Docker meta :tag
+        if: github.event_name == 'release' && startsWith(github.event.release.tag_name, '0.')
+        uses: docker/metadata-action@v5
+        id: meta
+        with:
+            images: |
+                ${{ secrets.DOCKER_HUB_USERNAME }}/changedetection.io
+                ghcr.io/dgtlmoon/changedetection.io
+            tags: |
+                type=semver,pattern={{version}},value=${{ github.event.release.tag_name }}
+                type=semver,pattern={{major}}.{{minor}},value=${{ github.event.release.tag_name }}
+                type=semver,pattern={{major}},value=${{ github.event.release.tag_name }}
+                type=raw,value=latest
+
       - name: Build and push :tag
         id: docker_build_tag_release
         if: github.event_name == 'release' && startsWith(github.event.release.tag_name, '0.')
-        uses: docker/build-push-action@v2
+        uses: docker/build-push-action@v6
         with:
           context: ./
           file: ./Dockerfile
           push: true
-          tags: |
-            ${{ secrets.DOCKER_HUB_USERNAME }}/changedetection.io:${{ github.event.release.tag_name }}
-            ghcr.io/dgtlmoon/changedetection.io:${{ github.event.release.tag_name }}
-            ${{ secrets.DOCKER_HUB_USERNAME }}/changedetection.io:latest
-            ghcr.io/dgtlmoon/changedetection.io:latest
-          platforms: linux/amd64,linux/arm64,linux/arm/v6,linux/arm/v7
-          cache-from: type=local,src=/tmp/.buildx-cache
-          cache-to: type=local,dest=/tmp/.buildx-cache
+          tags: ${{ steps.meta.outputs.tags }}
+          platforms: linux/amd64,linux/arm64,linux/arm/v7,linux/arm/v8,linux/arm64/v8
+          cache-from: type=gha
+          cache-to: type=gha,mode=max
+# Looks like this was disabled
+#          provenance: false
 
       - name: Image digest
         run: echo step SHA ${{ steps.vars.outputs.sha_short }} tag ${{steps.vars.outputs.tag}} branch ${{steps.vars.outputs.branch}} digest ${{ steps.docker_build.outputs.digest }}
 
-      - name: Cache Docker layers
-        uses: actions/cache@v2
-        with:
-          path: /tmp/.buildx-cache
-          key: ${{ runner.os }}-buildx-${{ github.sha }}
-          restore-keys: |
-            ${{ runner.os }}-buildx-

.github/workflows/pypi-release.yml

@@ -0,0 +1,80 @@
+name: Publish Python 🐍distribution 📦 to PyPI and TestPyPI
+
+on: push
+jobs:
+  build:
+    name: Build distribution 📦
+    runs-on: ubuntu-latest
+
+    steps:
+    - uses: actions/checkout@v5
+    - name: Set up Python
+      uses: actions/setup-python@v6
+      with:
+        python-version: "3.11"
+    - name: Install pypa/build
+      run: >-
+        python3 -m
+        pip install
+        build
+        --user
+    - name: Build a binary wheel and a source tarball
+      run: python3 -m build
+    - name: Store the distribution packages
+      uses: actions/upload-artifact@v4
+      with:
+        name: python-package-distributions
+        path: dist/
+
+
+  test-pypi-package:
+    name: Test the built 📦 package works basically.
+    runs-on: ubuntu-latest
+    needs:
+    - build
+    steps:
+    - name: Download all the dists
+      uses: actions/download-artifact@v5
+      with:
+        name: python-package-distributions
+        path: dist/
+    - name: Set up Python 3.11
+      uses: actions/setup-python@v6
+      with:
+        python-version: '3.11'
+    - name: Test that the basic pip built package runs without error
+      run: |
+        set -ex
+        ls -alR 
+        
+        # Find and install the first .whl file
+        find dist -type f -name "*.whl" -exec pip3 install {} \; -quit
+        changedetection.io -d /tmp -p 10000 &
+        
+        sleep 3
+        curl --retry-connrefused --retry 6 http://127.0.0.1:10000/static/styles/pure-min.css >/dev/null
+        curl --retry-connrefused --retry 6 http://127.0.0.1:10000/ >/dev/null
+        killall changedetection.io
+
+
+  publish-to-pypi:
+    name: >-
+      Publish Python 🐍 distribution 📦 to PyPI
+    if: startsWith(github.ref, 'refs/tags/')  # only publish to PyPI on tag pushes
+    needs:
+    - test-pypi-package
+    runs-on: ubuntu-latest
+    environment:
+      name: release
+      url: https://pypi.org/p/changedetection.io
+    permissions:
+      id-token: write  # IMPORTANT: mandatory for trusted publishing
+
+    steps:
+    - name: Download all the dists
+      uses: actions/download-artifact@v5
+      with:
+        name: python-package-distributions
+        path: dist/
+    - name: Publish distribution 📦 to PyPI
+      uses: pypa/gh-action-pypi-publish@release/v1

.github/workflows/pypi.yml

@@ -1,44 +0,0 @@
-name: PyPi Test and Push tagged release
-
-# Triggers the workflow on push or pull request events
-on:
-  workflow_run:
-    workflows: ["ChangeDetection.io Test"]
-    tags: '*.*'
-    types: [completed]
-
-
-jobs:
-  test-build:
-    runs-on: ubuntu-latest
-    steps:
-
-      - uses: actions/checkout@v2
-      - name: Set up Python 3.9
-        uses: actions/setup-python@v2
-        with:
-          python-version: 3.9
-
-#      - name: Install dependencies
-#        run: |
-#          python -m pip install --upgrade pip
-#          pip install flake8 pytest
-#          if [ -f requirements.txt ]; then pip install -r requirements.txt; fi
-#          if [ -f requirements-dev.txt ]; then pip install -r requirements-dev.txt; fi
-
-      - name: Test that pip builds without error
-        run: |
-          pip3 --version
-          python3 -m pip install wheel
-          python3 setup.py bdist_wheel
-          python3 -m pip install dist/changedetection.io-*-none-any.whl --force
-          changedetection.io -d /tmp -p 10000 &
-          sleep 3
-          curl http://127.0.0.1:10000/static/styles/pure-min.css >/dev/null
-          killall -9 changedetection.io
-
-      # https://github.com/docker/build-push-action/blob/master/docs/advanced/test-before-push.md ?
-      # https://github.com/docker/buildx/issues/59 ? Needs to be one platform?
-
-      # https://github.com/docker/buildx/issues/495#issuecomment-918925854
-#if: ${{ github.event_name == 'release'}}

.github/workflows/test-container-build.yml

@@ -0,0 +1,80 @@
+name: ChangeDetection.io Container Build Test
+
+# Triggers the workflow on push or pull request events
+
+# This line doesnt work, even tho it is the documented one
+#on: [push, pull_request]
+
+on:
+  push:
+    paths:
+      - requirements.txt
+      - Dockerfile
+      - .github/workflows/*
+      - .github/test/Dockerfile*
+
+  pull_request:
+    paths:
+      - requirements.txt
+      - Dockerfile
+      - .github/workflows/*
+      - .github/test/Dockerfile*
+
+  # Changes to requirements.txt packages and Dockerfile may or may not always be compatible with arm etc, so worth testing
+  # @todo: some kind of path filter for requirements.txt and Dockerfile
+jobs:
+  builder:
+    name: Build ${{ matrix.platform }} (${{ matrix.dockerfile == './Dockerfile' && 'main' || 'alpine' }})
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        include:
+          # Main Dockerfile platforms
+          - platform: linux/amd64
+            dockerfile: ./Dockerfile
+          - platform: linux/arm64
+            dockerfile: ./Dockerfile
+          - platform: linux/arm/v7
+            dockerfile: ./Dockerfile
+          - platform: linux/arm/v8
+            dockerfile: ./Dockerfile
+          - platform: linux/arm64/v8
+            dockerfile: ./Dockerfile
+          # Alpine Dockerfile platforms (musl via alpine check)
+          - platform: linux/amd64
+            dockerfile: ./.github/test/Dockerfile-alpine
+          - platform: linux/arm64
+            dockerfile: ./.github/test/Dockerfile-alpine
+    steps:
+        - uses: actions/checkout@v5
+        - name: Set up Python 3.11
+          uses: actions/setup-python@v6
+          with:
+            python-version: 3.11
+
+        # Just test that the build works, some libraries won't compile on ARM/rPi etc
+        - name: Set up QEMU
+          uses: docker/setup-qemu-action@v3
+          with:
+            image: tonistiigi/binfmt:latest
+            platforms: all
+
+        - name: Set up Docker Buildx
+          id: buildx
+          uses: docker/setup-buildx-action@v3
+          with:
+            install: true
+            version: latest
+            driver-opts: image=moby/buildkit:master
+
+        - name: Test that the docker containers can build (${{ matrix.platform }} - ${{ matrix.dockerfile }})
+          id: docker_build
+          uses: docker/build-push-action@v6
+          # https://github.com/docker/build-push-action#customizing
+          with:
+            context: ./
+            file: ${{ matrix.dockerfile }}
+            platforms: ${{ matrix.platform }}
+            cache-from: type=gha
+            cache-to: type=gha,mode=max
+

.github/workflows/test-only.yml

@@ -1,45 +1,48 @@
-name: ChangeDetection.io Test
+name: ChangeDetection.io App Test
 
 # Triggers the workflow on push or pull request events
 on: [push, pull_request]
 
 jobs:
-  test-build:
+  lint-code:
     runs-on: ubuntu-latest
     steps:
-
-      - uses: actions/checkout@v2
-      - name: Set up Python 3.9
-        uses: actions/setup-python@v2
-        with:
-          python-version: 3.9
-
-      - name: Show env vars
-        run: set
-
-      - name: Install dependencies
+      - uses: actions/checkout@v5
+      - name: Lint with Ruff
         run: |
-          python -m pip install --upgrade pip
-          pip install flake8 pytest
-          if [ -f requirements.txt ]; then pip install -r requirements.txt; fi
-          if [ -f requirements-dev.txt ]; then pip install -r requirements-dev.txt; fi
-      - name: Lint with flake8
+          pip install ruff
+          # Check for syntax errors and undefined names
+          ruff check . --select E9,F63,F7,F82
+          # Complete check with errors treated as warnings
+          ruff check . --exit-zero
+      - name: Validate OpenAPI spec
         run: |
-          # stop the build if there are Python syntax errors or undefined names
-          flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics
-          # exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide
-          flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics
+          pip install openapi-spec-validator
+          python3 -c "from openapi_spec_validator import validate_spec; import yaml; validate_spec(yaml.safe_load(open('docs/api-spec.yaml')))"
 
-      - name: Unit tests
-        run: |
-          python3 -m unittest changedetectionio.tests.unit.test_notification_diff
+  test-application-3-10:
+    needs: lint-code
+    uses: ./.github/workflows/test-stack-reusable-workflow.yml
+    with:
+      python-version: '3.10'
 
-      - name: Test with pytest
-        run: |
-          # Each test is totally isolated and performs its own cleanup/reset
-          cd changedetectionio; ./run_all_tests.sh
 
-      # https://github.com/docker/build-push-action/blob/master/docs/advanced/test-before-push.md ?
-      # https://github.com/docker/buildx/issues/59 ? Needs to be one platform?
+  test-application-3-11:
+    needs: lint-code
+    uses: ./.github/workflows/test-stack-reusable-workflow.yml
+    with:
+      python-version: '3.11'
 
-      # https://github.com/docker/buildx/issues/495#issuecomment-918925854
+  test-application-3-12:
+    needs: lint-code
+    uses: ./.github/workflows/test-stack-reusable-workflow.yml
+    with:
+      python-version: '3.12'
+      skip-pypuppeteer: true
+
+  test-application-3-13:
+    needs: lint-code
+    uses: ./.github/workflows/test-stack-reusable-workflow.yml
+    with:
+      python-version: '3.13'
+      skip-pypuppeteer: true

.github/workflows/test-stack-reusable-workflow.yml

@@ -0,0 +1,261 @@
+name: ChangeDetection.io App Test
+
+on:
+  workflow_call:
+    inputs:
+      python-version:
+        description: 'Python version to use'
+        required: true
+        type: string
+        default: '3.11'
+      skip-pypuppeteer:
+        description: 'Skip PyPuppeteer (not supported in 3.11/3.12)'
+        required: false
+        type: boolean
+        default: false
+
+jobs:
+  test-application:
+    runs-on: ubuntu-latest
+    env:
+      PYTHON_VERSION: ${{ inputs.python-version }}
+    steps:
+      - uses: actions/checkout@v5
+
+      # Mainly just for link/flake8
+      - name: Set up Python ${{ env.PYTHON_VERSION }}
+        uses: actions/setup-python@v6
+        with:
+          python-version: ${{ env.PYTHON_VERSION }}
+
+      - name: Build changedetection.io container for testing under Python ${{ env.PYTHON_VERSION }}
+        run: |
+          echo "---- Building for Python ${{ env.PYTHON_VERSION }} -----"
+          # Build a changedetection.io container and start testing inside
+          docker build --build-arg PYTHON_VERSION=${{ env.PYTHON_VERSION }} --build-arg LOGGER_LEVEL=TRACE -t test-changedetectionio .
+          # Debug info
+          docker run test-changedetectionio  bash -c 'pip list'         
+
+      - name: We should be Python ${{ env.PYTHON_VERSION }} ...
+        run: |         
+          docker run test-changedetectionio  bash -c 'python3 --version'
+
+      - name: Spin up ancillary testable services
+        run: |
+          
+          docker network create changedet-network
+          
+          # Selenium
+          docker run --network changedet-network -d --hostname selenium  -p 4444:4444 --rm --shm-size="2g"  selenium/standalone-chrome:4
+          
+          # SocketPuppetBrowser + Extra for custom browser test
+          docker run --network changedet-network -d -e "LOG_LEVEL=TRACE" --cap-add=SYS_ADMIN --name sockpuppetbrowser --hostname sockpuppetbrowser --rm -p 3000:3000 dgtlmoon/sockpuppetbrowser:latest                    
+          docker run --network changedet-network -d -e "LOG_LEVEL=TRACE" --cap-add=SYS_ADMIN --name sockpuppetbrowser-custom-url --hostname sockpuppetbrowser-custom-url  -p 3001:3000 --rm dgtlmoon/sockpuppetbrowser:latest
+
+      - name: Spin up ancillary SMTP+Echo message test server
+        run: |
+          # Debug SMTP server/echo message back server
+          docker run --network changedet-network -d -p 11025:11025 -p 11080:11080  --hostname mailserver test-changedetectionio  bash -c 'pip3 install aiosmtpd && python changedetectionio/tests/smtp/smtp-test-server.py'
+          docker ps
+
+      - name: Show docker container state and other debug info
+        run: |
+          set -x
+          echo "Running processes in docker..."
+          docker ps
+
+      - name: Run Unit Tests
+        run: |
+          # Unit tests
+          docker run test-changedetectionio  bash -c 'python3 -m unittest changedetectionio.tests.unit.test_notification_diff'
+          docker run test-changedetectionio  bash -c 'python3 -m unittest changedetectionio.tests.unit.test_watch_model'
+          docker run test-changedetectionio  bash -c 'python3 -m unittest changedetectionio.tests.unit.test_jinja2_security'
+          docker run test-changedetectionio  bash -c 'python3 -m unittest changedetectionio.tests.unit.test_semver'
+
+      - name: Test built container with Pytest (generally as requests/plaintext fetching)
+        run: |
+          # All tests
+          echo "run test with pytest"
+          # The default pytest logger_level is TRACE
+          # To change logger_level for pytest(test/conftest.py),
+          # append the docker option. e.g. '-e LOGGER_LEVEL=DEBUG'
+          docker run --name test-cdio-basic-tests --network changedet-network  test-changedetectionio  bash -c 'cd changedetectionio && ./run_basic_tests.sh'
+
+# PLAYWRIGHT/NODE-> CDP
+      - name: Playwright and SocketPuppetBrowser - Specific tests in built container
+        run: |
+          # Playwright via Sockpuppetbrowser fetch
+          # tests/visualselector/test_fetch_data.py will do browser steps  
+          docker run --rm -e "FLASK_SERVER_NAME=cdio" -e "PLAYWRIGHT_DRIVER_URL=ws://sockpuppetbrowser:3000" --network changedet-network --hostname=cdio test-changedetectionio  bash -c 'cd changedetectionio;pytest  -vv --capture=tee-sys --showlocals --tb=long --live-server-host=0.0.0.0 --live-server-port=5004 tests/fetchers/test_content.py'
+          docker run --rm -e "FLASK_SERVER_NAME=cdio" -e "PLAYWRIGHT_DRIVER_URL=ws://sockpuppetbrowser:3000" --network changedet-network --hostname=cdio test-changedetectionio  bash -c 'cd changedetectionio;pytest  -vv --capture=tee-sys --showlocals --tb=long --live-server-host=0.0.0.0 --live-server-port=5004 tests/test_errorhandling.py'
+          docker run --rm -e "FLASK_SERVER_NAME=cdio" -e "PLAYWRIGHT_DRIVER_URL=ws://sockpuppetbrowser:3000" --network changedet-network --hostname=cdio test-changedetectionio  bash -c 'cd changedetectionio;pytest  -vv --capture=tee-sys --showlocals --tb=long --live-server-host=0.0.0.0 --live-server-port=5004 tests/visualselector/test_fetch_data.py'
+          docker run --rm -e "FLASK_SERVER_NAME=cdio" -e "PLAYWRIGHT_DRIVER_URL=ws://sockpuppetbrowser:3000" --network changedet-network --hostname=cdio test-changedetectionio  bash -c 'cd changedetectionio;pytest  -vv --capture=tee-sys --showlocals --tb=long --live-server-host=0.0.0.0 --live-server-port=5004 tests/fetchers/test_custom_js_before_content.py'
+
+
+      - name: Playwright and SocketPuppetBrowser - Headers and requests
+        run: |       
+          # Settings headers playwright tests - Call back in from Sockpuppetbrowser, check headers
+          docker run --name "changedet" --hostname changedet --rm -e "FLASK_SERVER_NAME=changedet" -e "PLAYWRIGHT_DRIVER_URL=ws://sockpuppetbrowser:3000?dumpio=true" --network changedet-network test-changedetectionio  bash -c 'find .; cd changedetectionio; pytest --live-server-host=0.0.0.0  --live-server-port=5004 tests/test_request.py; pwd;find .'
+
+      - name: Playwright and SocketPuppetBrowser - Restock detection
+        run: |                            
+          # restock detection via playwright - added name=changedet here so that playwright and sockpuppetbrowser can connect to it
+          docker run --rm --name "changedet" -e "FLASK_SERVER_NAME=changedet" -e "PLAYWRIGHT_DRIVER_URL=ws://sockpuppetbrowser:3000" --network changedet-network test-changedetectionio  bash -c 'cd changedetectionio;pytest --live-server-port=5004 --live-server-host=0.0.0.0 tests/restock/test_restock.py'
+
+# STRAIGHT TO CDP
+      - name: Pyppeteer and SocketPuppetBrowser - Specific tests in built container
+        if: ${{ inputs.skip-pypuppeteer == false }}
+        run: |
+          # Playwright via Sockpuppetbrowser fetch 
+          docker run --rm -e "FLASK_SERVER_NAME=cdio" -e "FAST_PUPPETEER_CHROME_FETCHER=True" -e "PLAYWRIGHT_DRIVER_URL=ws://sockpuppetbrowser:3000" --network changedet-network --hostname=cdio test-changedetectionio  bash -c 'cd changedetectionio;pytest --live-server-host=0.0.0.0 --live-server-port=5004 tests/fetchers/test_content.py'
+          docker run --rm -e "FLASK_SERVER_NAME=cdio" -e "FAST_PUPPETEER_CHROME_FETCHER=True" -e "PLAYWRIGHT_DRIVER_URL=ws://sockpuppetbrowser:3000" --network changedet-network --hostname=cdio test-changedetectionio  bash -c 'cd changedetectionio;pytest --live-server-host=0.0.0.0 --live-server-port=5004 tests/test_errorhandling.py'
+          docker run --rm -e "FLASK_SERVER_NAME=cdio" -e "FAST_PUPPETEER_CHROME_FETCHER=True" -e "PLAYWRIGHT_DRIVER_URL=ws://sockpuppetbrowser:3000" --network changedet-network --hostname=cdio test-changedetectionio  bash -c 'cd changedetectionio;pytest --live-server-host=0.0.0.0 --live-server-port=5004 tests/visualselector/test_fetch_data.py'
+          docker run --rm -e "FLASK_SERVER_NAME=cdio" -e "FAST_PUPPETEER_CHROME_FETCHER=True" -e "PLAYWRIGHT_DRIVER_URL=ws://sockpuppetbrowser:3000" --network changedet-network --hostname=cdio test-changedetectionio  bash -c 'cd changedetectionio;pytest --live-server-host=0.0.0.0 --live-server-port=5004 tests/fetchers/test_custom_js_before_content.py'
+
+      - name: Pyppeteer and SocketPuppetBrowser - Headers and requests checks
+        if: ${{ inputs.skip-pypuppeteer == false }}
+        run: |
+          # Settings headers playwright tests - Call back in from Sockpuppetbrowser, check headers
+          docker run --name "changedet" --hostname changedet --rm  -e "FAST_PUPPETEER_CHROME_FETCHER=True" -e "FLASK_SERVER_NAME=changedet" -e "PLAYWRIGHT_DRIVER_URL=ws://sockpuppetbrowser:3000?dumpio=true" --network changedet-network test-changedetectionio  bash -c 'cd changedetectionio; pytest --live-server-host=0.0.0.0  --live-server-port=5004 tests/test_request.py'
+
+      - name: Pyppeteer and SocketPuppetBrowser - Restock detection
+        if: ${{ inputs.skip-pypuppeteer == false }}
+        run: |                            
+          # restock detection via playwright - added name=changedet here so that playwright and sockpuppetbrowser can connect to it
+          docker run --rm --name "changedet" -e "FLASK_SERVER_NAME=changedet"  -e "FAST_PUPPETEER_CHROME_FETCHER=True"  -e "PLAYWRIGHT_DRIVER_URL=ws://sockpuppetbrowser:3000" --network changedet-network test-changedetectionio  bash -c 'cd changedetectionio;pytest --live-server-port=5004 --live-server-host=0.0.0.0 tests/restock/test_restock.py'
+
+# SELENIUM
+      - name: Specific tests in built container for Selenium
+        run: |
+          # Selenium fetch
+          docker run --rm -e "WEBDRIVER_URL=http://selenium:4444/wd/hub" --network changedet-network test-changedetectionio  bash -c 'cd changedetectionio;pytest tests/fetchers/test_content.py && pytest tests/test_errorhandling.py'
+
+      - name: Specific tests in built container for headers and requests checks with Selenium
+        run: |
+          docker run --name "changedet" --hostname changedet --rm -e "FLASK_SERVER_NAME=changedet" -e "WEBDRIVER_URL=http://selenium:4444/wd/hub" --network changedet-network test-changedetectionio  bash -c 'cd changedetectionio; pytest --live-server-host=0.0.0.0  --live-server-port=5004 tests/test_request.py'
+
+# OTHER STUFF
+      - name: Test SMTP notification mime types
+        run: |
+          # SMTP content types - needs the 'Debug SMTP server/echo message back server' container from above
+          # "mailserver" hostname defined above
+          docker run --rm --network changedet-network test-changedetectionio bash -c 'cd changedetectionio;pytest tests/smtp/test_notification_smtp.py'
+
+      # @todo Add a test via playwright/puppeteer
+      # squid with auth is tested in run_proxy_tests.sh -> tests/proxy_list/test_select_custom_proxy.py
+      - name: Test proxy squid style interaction
+        run: |
+          cd changedetectionio
+          ./run_proxy_tests.sh
+          cd ..
+
+      - name: Test proxy SOCKS5 style interaction
+        run: |
+          cd changedetectionio
+          ./run_socks_proxy_tests.sh
+          cd ..
+
+      - name: Test custom browser URL
+        run: |
+          cd changedetectionio
+          ./run_custom_browser_url_tests.sh
+          cd ..
+
+      - name: Test changedetection.io container starts+runs basically without error
+        run: |
+          docker run --name test-changedetectionio -p 5556:5000  -d test-changedetectionio
+          sleep 3
+          # Should return 0 (no error) when grep finds it
+          curl --retry-connrefused --retry 6  -s http://localhost:5556 |grep -q checkbox-uuid
+          
+          # and IPv6
+          curl --retry-connrefused --retry 6  -s -g -6 "http://[::1]:5556"|grep -q checkbox-uuid
+
+          # Check whether TRACE log is enabled.
+          # Also, check whether TRACE came from STDOUT
+          docker logs test-changedetectionio 2>/dev/null | grep 'TRACE log is enabled' || exit 1
+          # Check whether DEBUG is came from STDOUT
+          docker logs test-changedetectionio 2>/dev/null | grep 'DEBUG' || exit 1
+
+          docker kill test-changedetectionio
+
+      - name: Test HTTPS SSL mode
+        run: |
+          openssl req -x509 -newkey rsa:4096 -keyout privkey.pem -out cert.pem -days 365 -nodes -subj "/CN=localhost"
+          docker run --name test-changedetectionio-ssl --rm -e SSL_CERT_FILE=cert.pem -e SSL_PRIVKEY_FILE=privkey.pem -p 5000:5000 -v ./cert.pem:/app/cert.pem -v ./privkey.pem:/app/privkey.pem -d test-changedetectionio
+          sleep 3
+          # Should return 0 (no error) when grep finds it
+          # -k because its self-signed
+          curl --retry-connrefused --retry 6 -k https://localhost:5000 -v|grep -q checkbox-uuid
+      
+          docker kill test-changedetectionio-ssl
+
+      - name: Test IPv6 Mode
+        run: |
+          # IPv6 - :: bind to all interfaces inside container (like 0.0.0.0), ::1 would be localhost only
+          docker run --name test-changedetectionio-ipv6 --rm -p 5000:5000 -e LISTEN_HOST=:: -d test-changedetectionio
+          sleep 3
+          # Should return 0 (no error) when grep finds it on localhost
+          curl --retry-connrefused --retry 6 http://[::1]:5000 -v|grep -q checkbox-uuid
+          docker kill test-changedetectionio-ipv6
+
+      - name: Test changedetection.io SIGTERM and SIGINT signal shutdown
+        run: |
+          
+          echo SIGINT Shutdown request test
+          docker run --name sig-test -d test-changedetectionio
+          sleep 3
+          echo ">>> Sending SIGINT to sig-test container"
+          docker kill --signal=SIGINT sig-test
+          sleep 3
+          # invert the check (it should be not 0/not running)
+          docker ps
+          # check signal catch(STDERR) log. Because of
+          # changedetectionio/__init__.py: logger.add(sys.stderr, level=logger_level)
+          docker logs sig-test 2>&1 | grep 'Shutdown: Got Signal - SIGINT' || exit 1
+          test -z "`docker ps|grep sig-test`"
+          if [ $? -ne 0 ]
+          then
+            echo "Looks like container was running when it shouldnt be"
+            docker ps
+            exit 1
+          fi
+          
+          # @todo - scan the container log to see the right "graceful shutdown" text exists 
+          docker rm sig-test
+          
+          echo SIGTERM Shutdown request test
+          docker run --name sig-test -d test-changedetectionio
+          sleep 3
+          echo ">>> Sending SIGTERM to sig-test container"
+          docker kill --signal=SIGTERM sig-test
+          sleep 3
+          # invert the check (it should be not 0/not running)
+          docker ps
+          # check signal catch(STDERR) log. Because of
+          # changedetectionio/__init__.py: logger.add(sys.stderr, level=logger_level)
+          docker logs sig-test 2>&1 | grep 'Shutdown: Got Signal - SIGTERM' || exit 1
+          test -z "`docker ps|grep sig-test`"
+          if [ $? -ne 0 ]
+          then
+            echo "Looks like container was running when it shouldnt be"
+            docker ps
+            exit 1
+          fi
+          
+          # @todo - scan the container log to see the right "graceful shutdown" text exists           
+          docker rm sig-test
+
+      - name: Dump container log
+        if: always()
+        run: |
+          mkdir output-logs
+          docker logs test-cdio-basic-tests > output-logs/test-cdio-basic-tests-stdout-${{ env.PYTHON_VERSION }}.txt
+          docker logs test-cdio-basic-tests 2> output-logs/test-cdio-basic-tests-stderr-${{ env.PYTHON_VERSION }}.txt
+
+      - name: Store everything including test-datastore
+        if: always()
+        uses: actions/upload-artifact@v4
+        with:
+          name: test-cdio-basic-tests-output-py${{ env.PYTHON_VERSION }}
+          path: .

.gitignore

@@ -1,13 +1,30 @@
-__pycache__
-.idea
-*.pyc
-datastore/url-watches.json
-datastore/*
-__pycache__
-.pytest_cache
-build
-dist
-venv
-test-datastore
+# Byte-compiled / optimized / DLL files
+**/__pycache__
+**/*.py[cod]
+
+# Caches
+.mypy_cache/
+.pytest_cache/
+.ruff_cache/
+
+# Distribution / packaging
+build/
+dist/
 *.egg-info*
+
+# Virtual environment
+.env
+.venv/
+venv/
+.python-version
+
+# IDEs
+.idea
 .vscode/settings.json
+
+# Datastore files
+datastore/
+test-datastore/
+
+# Memory consumption log
+test-memory.log

.pre-commit-config.yaml

@@ -0,0 +1,9 @@
+repos:
+  - repo: https://github.com/astral-sh/ruff-pre-commit
+    rev: v0.11.2
+    hooks:
+      # Lint (and apply safe fixes)
+      - id: ruff
+        args: [--fix]
+      # Fomrat
+      - id: ruff-format

.ruff.toml

@@ -0,0 +1,48 @@
+# Minimum supported version
+target-version = "py310"
+
+# Formatting options
+line-length = 100
+indent-width = 4
+
+exclude = [
+    "__pycache__",
+    ".eggs",
+    ".git",
+    ".tox",
+    ".venv",
+    "*.egg-info",
+    "*.pyc",
+]
+
+[lint]
+# https://docs.astral.sh/ruff/rules/
+select = [
+    "B", # flake8-bugbear
+    "B9",
+    "C", 
+    "E", # pycodestyle
+    "F", # Pyflakes
+    "I", # isort
+    "N", # pep8-naming
+    "UP", # pyupgrade
+    "W", # pycodestyle
+]
+ignore = [
+    "B007", # unused-loop-control-variable
+    "B909", # loop-iterator-mutation
+    "E203", # whitespace-before-punctuation
+    "E266", # multiple-leading-hashes-for-block-comment
+    "E501", # redundant-backslash
+    "F403", # undefined-local-with-import-star
+    "N802", # invalid-function-name
+    "N806", # non-lowercase-variable-in-function
+    "N815", # mixed-case-variable-in-class-scope
+]
+
+[lint.mccabe]
+max-complexity = 12
+
+[format]
+indent-style = "space"
+quote-style = "preserve"

COMMERCIAL_LICENCE.md

@@ -0,0 +1,54 @@
+# Generally
+
+In any commercial activity involving 'Hosting' (as defined herein), whether in part or in full, this license must be executed and adhered to.
+
+# Commercial License Agreement
+
+This Commercial License Agreement ("Agreement") is entered into by and between Web Technologies s.r.o. here-in ("Licensor") and (your company or personal name) _____________ ("Licensee"). This Agreement sets forth the terms and conditions under which Licensor provides its software ("Software") and services to Licensee for the purpose of reselling the software either in part or full, as part of any commercial activity where the activity involves a third party.
+
+### Definition of Hosting
+
+For the purposes of this Agreement, "hosting" means making the functionality of the Program or modified version available to third parties as a service. This includes, without limitation:
+- Enabling third parties to interact with the functionality of the Program or modified version remotely through a computer network.
+- Offering a service the value of which entirely or primarily derives from the value of the Program or modified version.
+- Offering a service that accomplishes for users the primary purpose of the Program or modified version.
+
+## 1. Grant of License
+Subject to the terms and conditions of this Agreement, Licensor grants Licensee a non-exclusive, non-transferable license to install, use, and resell the Software. Licensee may:
+- Resell the Software as part of a service offering or as a standalone product.
+- Host the Software on a server and provide it as a hosted service (e.g., Software as a Service - SaaS).
+- Integrate the Software into a larger product or service that is then sold or provided for commercial purposes, where the software is used either in part or full.
+
+## 2. License Fees
+Licensee agrees to pay Licensor the license fees specified in the ordering document. License fees are due and payable as specified in the ordering document. The fees may include initial licensing costs and recurring fees based on the number of end users, instances of the Software resold, or revenue generated from the resale activities.
+
+## 3. Resale Conditions
+Licensee must comply with the following conditions when reselling the Software, whether the software is resold in part or full:
+- Provide end users with access to the source code under the same open-source license conditions as provided by Licensor.
+- Clearly state in all marketing and sales materials that the Software is provided under a commercial license from Licensor, and provide a link back to https://changedetection.io.
+- Ensure end users are aware of and agree to the terms of the commercial license prior to resale.
+- Do not sublicense or transfer the Software to third parties except as part of an authorized resale activity.
+
+## 4. Hosting and Provision of Services
+Licensee may host the Software (either in part or full) on its servers and provide it as a hosted service to end users. The following conditions apply:
+- Licensee must ensure that all hosted versions of the Software comply with the terms of this Agreement.
+- Licensee must provide Licensor with regular reports detailing the number of end users and instances of the hosted service.
+- Any modifications to the Software made by Licensee for hosting purposes must be made available to end users under the same open-source license conditions, unless agreed otherwise.
+
+## 5. Services
+Licensor will provide support and maintenance services as described in the support policy referenced in the ordering document should such an agreement be signed by all parties. Additional fees may apply for support services provided to end users resold by Licensee.
+
+## 6. Reporting and Audits
+Licensee agrees to provide Licensor with regular reports detailing the number of instances, end users, and revenue generated from the resale of the Software. Licensor reserves the right to audit Licensee’s records to ensure compliance with this Agreement.
+
+## 7. Term and Termination
+This Agreement shall commence on the effective date and continue for the period set forth in the ordering document unless terminated earlier in accordance with this Agreement. Either party may terminate this Agreement if the other party breaches any material term and fails to cure such breach within thirty (30) days after receipt of written notice.
+
+## 8. Limitation of Liability and Disclaimer of Warranty
+Executing this commercial license does not waive the Limitation of Liability or Disclaimer of Warranty as stated in the open-source LICENSE provided with the Software. The Software is provided "as is," without warranty of any kind, express or implied, including but not limited to the warranties of merchantability, fitness for a particular purpose, and noninfringement. In no event shall the authors or copyright holders be liable for any claim, damages, or other liability, whether in an action of contract, tort, or otherwise, arising from, out of, or in connection with the Software or the use or other dealings in the Software.
+
+## 9. Governing Law
+This Agreement shall be governed by and construed in accordance with the laws of the Czech Republic.
+
+## Contact Information
+For commercial licensing inquiries, please contact contact@changedetection.io and dgtlmoon@gmail.com.

CONTRIBUTING.md

@@ -2,14 +2,8 @@ Contributing is always welcome!
 
 I am no professional flask developer, if you know a better way that something can be done, please let me know!
 
-Otherwise, it's always best to PR into the `dev` branch.
+Otherwise, it's always best to PR into the `master` branch.
 
 Please be sure that all new functionality has a matching test!
 
-Use `pytest` to validate/test, you can run the existing tests as `pytest tests/test_notifications.py` for example
-
-```
-pip3 install -r requirements-dev
-```
-
-this is from https://github.com/dgtlmoon/changedetection.io/blob/master/requirements-dev.txt
+Use `pytest` to validate/test, you can run the existing tests as `pytest tests/test_notification.py` for example

Dockerfile

@@ -1,46 +1,72 @@
 # pip dependencies install stage
-FROM python:3.8-slim as builder
 
-# rustc compiler would be needed on ARM type devices but theres an issue with some deps not building..
-ARG CRYPTOGRAPHY_DONT_BUILD_RUST=1
+ARG PYTHON_VERSION=3.11
+
+FROM python:${PYTHON_VERSION}-slim-bookworm AS builder
+
+# See `cryptography` pin comment in requirements.txt
 
 RUN apt-get update && apt-get install -y --no-install-recommends \
-    libssl-dev \
-    libffi-dev \
+    g++ \
     gcc \
     libc-dev \
+    libffi-dev \
+    libjpeg-dev \
+    libssl-dev \
     libxslt-dev \
-    zlib1g-dev \
-    g++
+    make \
+    patch \
+    pkg-config \
+    zlib1g-dev
 
 RUN mkdir /install
 WORKDIR /install
 
 COPY requirements.txt /requirements.txt
 
-RUN pip install --target=/dependencies -r /requirements.txt
+# Use cache mounts and multiple wheel sources for faster ARM builds
+ENV PIP_CACHE_DIR=/tmp/pip-cache
+# Help Rust find OpenSSL for cryptography package compilation on ARM
+ENV PKG_CONFIG_PATH="/usr/lib/pkgconfig:/usr/lib/arm-linux-gnueabihf/pkgconfig:/usr/lib/aarch64-linux-gnu/pkgconfig"
+ENV PKG_CONFIG_ALLOW_SYSTEM_CFLAGS=1
+ENV OPENSSL_DIR="/usr"
+ENV OPENSSL_LIB_DIR="/usr/lib/arm-linux-gnueabihf"
+ENV OPENSSL_INCLUDE_DIR="/usr/include/openssl"
+# Additional environment variables for cryptography Rust build
+ENV CRYPTOGRAPHY_DONT_BUILD_RUST=1
+RUN --mount=type=cache,target=/tmp/pip-cache \
+    pip install \
+    --extra-index-url https://www.piwheels.org/simple \
+    --extra-index-url https://pypi.anaconda.org/ARM-software/simple \
+    --cache-dir=/tmp/pip-cache \
+    --target=/dependencies \
+    -r /requirements.txt
 
 # Playwright is an alternative to Selenium
 # Excluded this package from requirements.txt to prevent arm/v6 and arm/v7 builds from failing
-RUN pip install --target=/dependencies playwright~=1.20 \
+# https://github.com/dgtlmoon/changedetection.io/pull/1067 also musl/alpine (not supported)
+RUN --mount=type=cache,target=/tmp/pip-cache \
+    pip install \
+    --cache-dir=/tmp/pip-cache \
+    --target=/dependencies \
+    playwright~=1.48.0 \
     || echo "WARN: Failed to install Playwright. The application can still run, but the Playwright option will be disabled."
 
 # Final image stage
-FROM python:3.8-slim
+FROM python:${PYTHON_VERSION}-slim-bookworm
+LABEL org.opencontainers.image.source="https://github.com/dgtlmoon/changedetection.io"
 
-# Actual packages needed at runtime, usually due to the notification (apprise) backend
-# rustc compiler would be needed on ARM type devices but theres an issue with some deps not building..
-ARG CRYPTOGRAPHY_DONT_BUILD_RUST=1
-
-# Re #93, #73, excluding rustc (adds another 430Mb~)
 RUN apt-get update && apt-get install -y --no-install-recommends \
-    libssl-dev \
-    libffi-dev \
-    gcc \
-    libc-dev \
-    libxslt-dev \
-    zlib1g-dev \
-    g++
+    libxslt1.1 \
+    # For presenting price amounts correctly in the restock/price detection overview
+    locales \
+    # For pdftohtml
+    poppler-utils \
+    # favicon type detection and other uses
+    file \
+    zlib1g \
+    && apt-get clean && rm -rf /var/lib/apt/lists/*
+
 
 # https://stackoverflow.com/questions/58701233/docker-logs-erroneously-appears-empty-until-container-stops
 ENV PYTHONUNBUFFERED=1
@@ -56,11 +82,25 @@ ENV PYTHONPATH=/usr/local
 
 EXPOSE 5000
 
-# The actual flask app
+# The actual flask app module
 COPY changedetectionio /app/changedetectionio
-# The eventlet server wrapper
+
+# Also for OpenAPI validation wrapper - needs the YML
+RUN [ ! -d "/app/docs" ] && mkdir /app/docs
+COPY docs/api-spec.yaml /app/docs/api-spec.yaml
+
+# Starting wrapper
 COPY changedetection.py /app/changedetection.py
 
-WORKDIR /app
+# Github Action test purpose(test-only.yml).
+# On production, it is effectively LOGGER_LEVEL=''.
+ARG LOGGER_LEVEL=''
+ENV LOGGER_LEVEL="$LOGGER_LEVEL"
+
+# Default
+ENV LC_ALL=en_US.UTF-8
+
+WORKDIR /app
+CMD ["python", "./changedetection.py", "-d", "/datastore"]
+
 
-CMD [ "python", "./changedetection.py" , "-d", "/datastore"]

LICENSE

@@ -186,7 +186,7 @@
       same "printed page" as the copyright notice for easier
       identification within third-party archives.
 
-   Copyright [yyyy] [name of copyright owner]
+   Copyright 2025 Web Technologies s.r.o.
 
    Licensed under the Apache License, Version 2.0 (the "License");
    you may not use this file except in compliance with the License.

MANIFEST.in

@@ -1,8 +1,25 @@
 recursive-include changedetectionio/api *
-recursive-include changedetectionio/templates *
-recursive-include changedetectionio/static *
+recursive-include changedetectionio/blueprint *
+recursive-include changedetectionio/conditions *
+recursive-include changedetectionio/content_fetchers *
 recursive-include changedetectionio/model *
+recursive-include changedetectionio/notification *
+recursive-include changedetectionio/processors *
+recursive-include changedetectionio/realtime *
+recursive-include changedetectionio/static *
+recursive-include changedetectionio/templates *
+recursive-include changedetectionio/tests *
+recursive-include changedetectionio/widgets *
+prune changedetectionio/static/package-lock.json
+prune changedetectionio/static/styles/node_modules
+prune changedetectionio/static/styles/package-lock.json
 include changedetection.py
+include requirements.txt
+include README-pip.md
 global-exclude *.pyc
 global-exclude node_modules
-global-exclude venv
+global-exclude venv
+
+global-exclude test-datastore
+global-exclude changedetection.io*dist-info
+global-exclude changedetectionio/tests/proxy_socks5/test-datastore

Procfile

@@ -1 +0,0 @@
-web: python3 ./changedetection.py -C -d ./datastore -p $PORT

README-pip.md

@@ -1,45 +1,99 @@
-#  changedetection.io
-![changedetection.io](https://github.com/dgtlmoon/changedetection.io/actions/workflows/test-only.yml/badge.svg?branch=master)
-<a href="https://hub.docker.com/r/dgtlmoon/changedetection.io" target="_blank" title="Change detection docker hub">
-  <img src="https://img.shields.io/docker/pulls/dgtlmoon/changedetection.io" alt="Docker Pulls"/>
-</a>
-<a href="https://hub.docker.com/r/dgtlmoon/changedetection.io" target="_blank" title="Change detection docker hub">
-  <img src="https://img.shields.io/github/v/release/dgtlmoon/changedetection.io" alt="Change detection latest tag version"/> 
-</a>
+# Monitor website changes
 
-## Self-hosted open source change monitoring of web pages.
+Detect WebPage Changes Automatically — Monitor Web Page Changes in Real Time
 
-_Know when web pages change! Stay ontop of new information!_ 
+Monitor websites for updates — get notified via Discord, Email, Slack, Telegram, Webhook and many more.
 
-Live your data-life *pro-actively* instead of *re-actively*, do not rely on manipulative social media for consuming important information.
+Detect web page content changes and get instant alerts.
 
 
-<img src="https://raw.githubusercontent.com/dgtlmoon/changedetection.io/master/screenshot.png" style="max-width:100%;" alt="Self-hosted web page change monitoring"  title="Self-hosted web page change monitoring"  />
+[Changedetection.io is the best tool to monitor web-pages for changes](https://changedetection.io) Track website content changes and receive notifications via Discord, Email, Slack, Telegram and 90+ more
+
+Ideal for monitoring price changes, content edits, conditional changes and more.
+
+[<img src="https://raw.githubusercontent.com/dgtlmoon/changedetection.io/master/docs/screenshot.png" style="max-width:100%;" alt="Self-hosted web page change monitoring, list of websites with changes"  title="Self-hosted web page change monitoring, list of websites with changes"  />](https://changedetection.io)
 
 
-**Get your own private instance now! Let us host it for you!**
-
-[**Try our $6.99/month subscription - unlimited checks, watches and notifications!**](https://lemonade.changedetection.io/start), choose from different geographical locations, let us handle everything for you. 
+[**Don't have time? Try our extremely affordable subscription use our proxies and support!**](https://changedetection.io) 
 
 
 
-#### Example use cases
+### Target specific parts of the webpage using the Visual Selector tool.
 
-Know when ...
+Available when connected to a <a href="https://github.com/dgtlmoon/changedetection.io/wiki/Playwright-content-fetcher">playwright content fetcher</a> (included as part of our subscription service)
 
-- Government department updates (changes are often only on their websites)
-- Local government news (changes are often only on their websites)
+[<img src="https://raw.githubusercontent.com/dgtlmoon/changedetection.io/master/docs/visualselector-anim.gif" style="max-width:100%;" alt="Select parts and elements of a web page to monitor for changes"  title="Select parts and elements of a web page to monitor for changes" />](https://changedetection.io?src=pip)
+
+### Easily see what changed, examine by word, line, or individual character.
+
+[<img src="https://raw.githubusercontent.com/dgtlmoon/changedetection.io/master/docs/screenshot-diff.png" style="max-width:100%;" alt="Self-hosted web page change monitoring context difference "  title="Self-hosted web page change monitoring context difference " />](https://changedetection.io?src=pip)
+
+
+### Perform interactive browser steps
+
+Fill in text boxes, click buttons and more, setup your changedetection scenario. 
+
+Using the **Browser Steps** configuration, add basic steps before performing change detection, such as logging into websites, adding a product to a cart, accept cookie logins, entering dates and refining searches.
+
+[<img src="docs/browsersteps-anim.gif" style="max-width:100%;" alt="Website change detection with interactive browser steps, detect changes behind login and password, search queries and more"  title="Website change detection with interactive browser steps, detect changes behind login and password, search queries and more" />](https://changedetection.io?src=pip)
+
+After **Browser Steps** have been run, then visit the **Visual Selector** tab to refine the content you're interested in.
+Requires Playwright to be enabled.
+
+
+### Example use cases
+
+- Products and services have a change in pricing
+- _Out of stock notification_ and _Back In stock notification_
+- Monitor and track PDF file changes, know when a PDF file has text changes.
+- Governmental department updates (changes are often only on their websites)
 - New software releases, security advisories when you're not on their mailing list.
 - Festivals with changes
+- Discogs restock alerts and monitoring
 - Realestate listing changes
+- Know when your favourite whiskey is on sale, or other special deals are announced before anyone else
 - COVID related news from government websites
+- University/organisation news from their website
 - Detect and monitor changes in JSON API responses 
-- API monitoring and alerting
+- JSON API monitoring and alerting
+- Changes in legal and other documents
+- Trigger API calls via notifications when text appears on a website
+- Glue together APIs using the JSON filter and JSON notifications
+- Create RSS feeds based on changes in web content
+- Monitor HTML source code for unexpected changes, strengthen your PCI compliance
+- You have a very sensitive list of URLs to watch and you do _not_ want to use the paid alternatives. (Remember, _you_ are the product)
+- Get notified when certain keywords appear in Twitter search results
+- Proactively search for jobs, get notified when companies update their careers page, search job portals for keywords.
+- Get alerts when new job positions are open on Bamboo HR and other job platforms
+- Website defacement monitoring
+- Pokémon Card Restock Tracker / Pokémon TCG Tracker
+- RegTech - stay ahead of regulatory changes, regulatory compliance
+
+_Need an actual Chrome runner with Javascript support? We support fetching via WebDriver and Playwright!</a>_
+
+#### Key Features
+
+- Lots of trigger filters, such as "Trigger on text", "Remove text by selector", "Ignore text", "Extract text", also using regular-expressions!
+- Target elements with xPath(1.0) and CSS Selectors, Easily monitor complex JSON with JSONPath or jq
+- Switch between fast non-JS and Chrome JS based "fetchers"
+- Track changes in PDF files (Monitor text changed in the PDF, Also monitor PDF filesize and checksums)
+- Easily specify how often a site should be checked
+- Execute JS before extracting text (Good for logging in, see examples in the UI!)
+- Override Request Headers, Specify `POST` or `GET` and other methods
+- Use the "Visual Selector" to help target specific elements
+- Configurable [proxy per watch](https://github.com/dgtlmoon/changedetection.io/wiki/Proxy-configuration)
+- Send a screenshot with the notification when a change is detected in the web page
+
+We [recommend and use Bright Data](https://brightdata.grsm.io/n0r16zf7eivq) global proxy services, Bright Data will match any first deposit up to $100 using our signup link.
+
+[Oxylabs](https://oxylabs.go2cloud.org/SH2d) is also an excellent proxy provider and well worth using, they offer Residental, ISP, Rotating and many other proxy types to suit your project. 
+
+Please :star: star :star: this project and help it grow! https://github.com/dgtlmoon/changedetection.io/
+
 
-**Get monitoring now!**
 
 ```bash
-$ pip3 install changedetection.io   
+$ pip3 install changedetection.io
 ```
 
 Specify a target for the *datastore path* with `-d` (required) and a *listening port* with `-p` (defaults to `5000`)
@@ -51,17 +105,5 @@ $ changedetection.io -d /path/to/empty/data/dir -p 5000
 
 Then visit http://127.0.0.1:5000 , You should now be able to access the UI.
 
-### Features
-- Website monitoring
-- Change detection of content and analyses
-- Filters on change (Select by CSS or JSON)
-- Triggers (Wait for text, wait for regex)
-- Notification support
-- JSON API Monitoring
-- Parse JSON embedded in HTML
-- (Reverse) Proxy support
-- Javascript support via WebDriver
-- RaspberriPi (arm v6/v7/64 support)
-
-See https://github.com/dgtlmoon/changedetection.io for more information.
+See https://changedetection.io for more information.
 

README.md

@@ -1,39 +1,73 @@
-#  changedetection.io
+# Detect Website Changes Automatically — Monitor Web Page Changes in Real Time
+
+Monitor websites for updates — get notified via Discord, Email, Slack, Telegram, Webhook and many more.
+
+**Detect web page content changes and get instant alerts.**  
+
+Ideal for monitoring price changes, content edits, conditional changes and more.
+
+
+[<img src="https://raw.githubusercontent.com/dgtlmoon/changedetection.io/master/docs/screenshot.png" style="max-width:100%;" alt="Web site page change monitoring"  title="Web site page change monitoring"  />](https://changedetection.io?src=github)
+
 [![Release Version][release-shield]][release-link] [![Docker Pulls][docker-pulls]][docker-link] [![License][license-shield]](LICENSE.md)
 
 ![changedetection.io](https://github.com/dgtlmoon/changedetection.io/actions/workflows/test-only.yml/badge.svg?branch=master)
 
-## Web Site Change Detection, Monitoring and Notification - Self-Hosted or SaaS.
-
-_Know when web pages change! Stay ontop of new information! get notifications when important website content changes_ 
-
-Live your data-life *pro-actively* instead of *re-actively*.
-
-Free, Open-source web page monitoring, notification and change detection. Don't have time? [**Try our $6.99/month subscription - unlimited checks and watches!**](https://lemonade.changedetection.io/start)
-
-[![Discord](https://img.shields.io/badge/DISCORD-%237289DA.svg?style=for-the-badge&logo=discord&logoColor=white)](https://discord.gg/vUNt4EtWMF) [ ![YouTube](https://img.shields.io/badge/YouTube-%23FF0000.svg?style=for-the-badge&logo=YouTube&logoColor=white)](https://www.youtube.com/channel/UCbS09q1TRf0o4N2t-WA3emQ) [![LinkedIn](https://img.shields.io/badge/linkedin-%230077B5.svg?style=for-the-badge&logo=linkedin&logoColor=white)](https://www.linkedin.com/company/changedetection-io/)
+[**Get started with website page change monitoring straight away. Don't have time? Try our $8.99/month subscription, use our proxies and support!**](https://changedetection.io) , _half the price of other website change monitoring services!_
 
 
-[<img src="https://raw.githubusercontent.com/dgtlmoon/changedetection.io/master/docs/screenshot.png" style="max-width:100%;" alt="Self-hosted web page change monitoring"  title="Self-hosted web page change monitoring"  />](https://lemonade.changedetection.io/start)
+- Chrome browser included.
+- Nothing to install, access via browser login after signup.
+- Super fast, no registration needed setup.
+- Get started watching and receiving website change notifications straight away.
+- See our [tutorials and how-to page for more inspiration](https://changedetection.io/tutorials) 
+
+### Target specific parts of the webpage using the Visual Selector tool.
+
+Available when connected to a <a href="https://github.com/dgtlmoon/changedetection.io/wiki/Playwright-content-fetcher">playwright content fetcher</a> (included as part of our subscription service)
+
+[<img src="https://raw.githubusercontent.com/dgtlmoon/changedetection.io/master/docs/visualselector-anim.gif" style="max-width:100%;" alt="Select parts and elements of a web page to monitor for changes"  title="Select parts and elements of a web page to monitor for changes" />](https://changedetection.io?src=github)
+
+### Easily see what changed, examine by word, line, or individual character.
+
+[<img src="https://raw.githubusercontent.com/dgtlmoon/changedetection.io/master/docs/screenshot-diff.png" style="max-width:100%;" alt="Self-hosted web page change monitoring context difference "  title="Self-hosted web page change monitoring context difference " />](https://changedetection.io?src=github)
 
 
-**Get your own private instance now! Let us host it for you!**
+### Perform interactive browser steps
 
-[**Try our $6.99/month subscription - unlimited checks and watches!**](https://lemonade.changedetection.io/start) , _half the price of other website change monitoring services and comes with unlimited watches & checks!_
+Fill in text boxes, click buttons and more, setup your changedetection scenario. 
+
+Using the **Browser Steps** configuration, add basic steps before performing change detection, such as logging into websites, adding a product to a cart, accept cookie logins, entering dates and refining searches.
+
+[<img src="docs/browsersteps-anim.gif" style="max-width:100%;" alt="Website change detection with interactive browser steps, detect changes behind login and password, search queries and more"  title="Website change detection with interactive browser steps, detect changes behind login and password, search queries and more" />](https://changedetection.io?src=github)
+
+After **Browser Steps** have been run, then visit the **Visual Selector** tab to refine the content you're interested in.
+Requires Playwright to be enabled.
+
+### Awesome restock and price change notifications
+
+Enable the _"Re-stock & Price detection for single product pages"_ option to activate the best way to monitor product pricing, this will extract any meta-data in the HTML page and give you many options to follow the pricing of the product.
+
+Easily organise and monitor prices for products from the dashboard, get alerts and notifications when the price of a product changes or comes back in stock again!
+
+[<img src="docs/restock-overview.png" style="max-width:100%;" alt="Easily keep an eye on product price changes directly from the UI"  title="Easily keep an eye on product price changes directly from the UI" />](https://changedetection.io?src=github)
+
+Set price change notification parameters, upper and lower price, price change percentage and more.
+Always know when a product for sale drops in price.
+
+[<img src="docs/restock-settings.png" style="max-width:100%;" alt="Set upper lower and percentage price change notification values"  title="Set upper lower and percentage price change notification values" />](https://changedetection.io?src=github)
 
 
 
-- Automatic Updates, Automatic Backups, No Heroku "paused application", don't miss a change!
-- Javascript browser included
-- Unlimited checks and watches!
-
-
-#### Example use cases
+### Example use cases
 
 - Products and services have a change in pricing
+- _Out of stock notification_ and _Back In stock notification_
+- Monitor and track PDF file changes, know when a PDF file has text changes.
 - Governmental department updates (changes are often only on their websites)
 - New software releases, security advisories when you're not on their mailing list.
 - Festivals with changes
+- Discogs restock alerts and monitoring
 - Realestate listing changes
 - Know when your favourite whiskey is on sale, or other special deals are announced before anyone else
 - COVID related news from government websites
@@ -46,24 +80,54 @@ Free, Open-source web page monitoring, notification and change detection. Don't
 - Create RSS feeds based on changes in web content
 - Monitor HTML source code for unexpected changes, strengthen your PCI compliance
 - You have a very sensitive list of URLs to watch and you do _not_ want to use the paid alternatives. (Remember, _you_ are the product)
+- Get notified when certain keywords appear in Twitter search results
+- Proactively search for jobs, get notified when companies update their careers page, search job portals for keywords.
+- Get alerts when new job positions are open on Bamboo HR and other job platforms
+- Website defacement monitoring
+- Pokémon Card Restock Tracker / Pokémon TCG Tracker
+- RegTech - stay ahead of regulatory changes, regulatory compliance
 
-_Need an actual Chrome runner with Javascript support? We support fetching via WebDriver!</a>_
+_Need an actual Chrome runner with Javascript support? We support fetching via WebDriver and Playwright!</a>_
 
-## Screenshots
+#### Key Features
 
-### Examine differences in content.
+- Lots of trigger filters, such as "Trigger on text", "Remove text by selector", "Ignore text", "Extract text", also using regular-expressions!
+- Target elements with xPath 1 and xPath 2, CSS Selectors, Easily monitor complex JSON with JSONPath or jq
+- Switch between fast non-JS and Chrome JS based "fetchers"
+- Track changes in PDF files (Monitor text changed in the PDF, Also monitor PDF filesize and checksums)
+- Easily specify how often a site should be checked
+- Execute JS before extracting text (Good for logging in, see examples in the UI!)
+- Override Request Headers, Specify `POST` or `GET` and other methods
+- Use the "Visual Selector" to help target specific elements
+- Configurable [proxy per watch](https://github.com/dgtlmoon/changedetection.io/wiki/Proxy-configuration)
+- Send a screenshot with the notification when a change is detected in the web page
 
-Easily see what changed, examine by word, line, or individual character.
-
-<img src="https://raw.githubusercontent.com/dgtlmoon/changedetection.io/master/docs/screenshot-diff.png" style="max-width:100%;" alt="Self-hosted web page change monitoring context difference "  title="Self-hosted web page change monitoring context difference " />
+We [recommend and use Bright Data](https://brightdata.grsm.io/n0r16zf7eivq) global proxy services, Bright Data will match any first deposit up to $150 using our signup link.
 
 Please :star: star :star: this project and help it grow! https://github.com/dgtlmoon/changedetection.io/
 
-### Filter by elements using the Visual Selector tool.
+### Conditional web page changes
 
-Available when connected to a <a href="https://github.com/dgtlmoon/changedetection.io/wiki/Playwright-content-fetcher">playwright content fetcher</a> (included as part of our subscription service)
+Easily [configure conditional actions](https://changedetection.io/tutorial/conditional-actions-web-page-changes), for example, only trigger when a price is above or below a preset amount, or [when a web page includes (or does not include) a keyword](https://changedetection.io/tutorial/how-monitor-keywords-any-website)
 
-<img src="https://raw.githubusercontent.com/dgtlmoon/changedetection.io/master/docs/visualselector-anim.gif" style="max-width:100%;" alt="Self-hosted web page change monitoring context difference "  title="Self-hosted web page change monitoring context difference " />
+<img src="./docs/web-page-change-conditions.png" style="max-width:80%;" alt="Conditional web page changes"  title="Conditional web page changes"  />
+
+### Schedule web page watches in any timezone, limit by day of week and time.
+
+Easily set a re-check schedule, for example you could limit the web page change detection to only operate during business hours.
+Or perhaps based on a foreign timezone (for example, you want to check for the latest news-headlines in a foreign country at 0900 AM),
+
+<img src="./docs/scheduler.png" style="max-width:80%;" alt="How to monitor web page changes according to a schedule"  title="How to monitor web page changes according to a schedule"  />
+
+Includes quick short-cut buttons to setup a schedule for **business hours only**, or **weekends**.
+
+### We have a Chrome extension!
+
+Easily add the current web page to your changedetection.io tool, simply install the extension and click "Sync" to connect it to your existing changedetection.io install.
+
+[<img src="./docs/chrome-extension-screenshot.png" style="max-width:80%;" alt="Chrome Extension to easily add the current web-page to detect a change."  title="Chrome Extension to easily add the current web-page to detect a change."  />](https://chromewebstore.google.com/detail/changedetectionio-website/kefcfmgmlhmankjmnbijimhofdjekbop)
+
+[Goto the Chrome Webstore to download the extension.](https://chromewebstore.google.com/detail/changedetectionio-website/kefcfmgmlhmankjmnbijimhofdjekbop) ( Or check out the [GitHub repo](https://github.com/dgtlmoon/changedetection.io-browser-extension) ) 
 
 ## Installation
 
@@ -72,7 +136,7 @@ Available when connected to a <a href="https://github.com/dgtlmoon/changedetecti
 With Docker composer, just clone this repository and..
 
 ```bash
-$ docker-compose up -d
+$ docker compose up -d
 ```
 
 Docker standalone
@@ -82,6 +146,8 @@ $ docker run -d --restart always -p "127.0.0.1:5000:5000" -v datastore-volume:/d
 
 `:latest` tag is our latest stable release, `:dev` tag is our bleeding edge `master` branch.
 
+Alternative docker repository over at ghcr - [ghcr.io/dgtlmoon/changedetection.io](https://ghcr.io/dgtlmoon/changedetection.io)
+
 ### Windows
 
 See the install instructions at the wiki https://github.com/dgtlmoon/changedetection.io/wiki/Microsoft-Windows
@@ -104,24 +170,24 @@ _Now with per-site configurable support for using a fast built in HTTP fetcher o
 ### Docker
 ```
 docker pull dgtlmoon/changedetection.io
-docker kill $(docker ps -a|grep changedetection.io|awk '{print $1}')
-docker rm $(docker ps -a|grep changedetection.io|awk '{print $1}')
+docker kill $(docker ps -a -f name=changedetection.io -q)
+docker rm $(docker ps -a -f name=changedetection.io -q)
 docker run -d --restart always -p "127.0.0.1:5000:5000" -v datastore-volume:/datastore --name changedetection.io dgtlmoon/changedetection.io
 ```
 
-### docker-compose
+### docker compose
 
 ```bash
-docker-compose pull && docker-compose up -d
+docker compose pull && docker compose up -d
 ```
 
 See the wiki for more information https://github.com/dgtlmoon/changedetection.io/wiki
 
 
 ## Filters
-XPath, JSONPath and CSS support comes baked in! You can be as specific as you need, use XPath exported from various XPath element query creation tools.
 
-(We support LXML `re:test`, `re:math` and `re:replace`.)
+XPath(1.0), JSONPath, jq, and CSS support comes baked in! You can be as specific as you need, use XPath exported from various XPath element query creation tools. 
+(We support LXML `re:test`, `re:match` and `re:replace`.)
 
 ## Notifications
 
@@ -145,11 +211,11 @@ Just some examples
 
 <img src="https://raw.githubusercontent.com/dgtlmoon/changedetection.io/master/docs/screenshot-notifications.png" style="max-width:100%;" alt="Self-hosted web page change monitoring notifications"  title="Self-hosted web page change monitoring notifications"  />
 
-Now you can also customise your notification content!
+Now you can also customise your notification content and use <a target="_new" href="https://jinja.palletsprojects.com/en/3.0.x/templates/">Jinja2 templating</a> for their title and body!
 
 ## JSON API Monitoring
 
-Detect changes and monitor data in JSON API's by using the built-in JSONPath selectors as a filter / selector.
+Detect changes and monitor data in JSON API's by using either JSONPath or jq to filter, parse, and restructure JSON as needed.
 
 ![image](https://raw.githubusercontent.com/dgtlmoon/changedetection.io/master/docs/json-filter-field-example.png)
 
@@ -157,41 +223,74 @@ This will re-parse the JSON and apply formatting to the text, making it super ea
 
 ![image](https://raw.githubusercontent.com/dgtlmoon/changedetection.io/master/docs/json-diff-example.png)
 
+### JSONPath or jq?
+
+For more complex parsing, filtering, and modifying of JSON data, jq is recommended due to the built-in operators and functions. Refer to the [documentation](https://stedolan.github.io/jq/manual/) for more specific information on jq.
+
+One big advantage of `jq` is that you can use logic in your JSON filter, such as filters to only show items that have a value greater than/less than etc.
+
+See the wiki https://github.com/dgtlmoon/changedetection.io/wiki/JSON-Selector-Filter-help for more information and examples
+
 ### Parse JSON embedded in HTML!
 
-When you enable a `json:` filter, you can even automatically extract and parse embedded JSON inside a HTML page! Amazingly handy for sites that build content based on JSON, such as many e-commerce websites. 
+When you enable a `json:` or `jq:` filter, you can even automatically extract and parse embedded JSON inside a HTML page! Amazingly handy for sites that build content based on JSON, such as many e-commerce websites. 
 
 ```
 <html>
 ...
 <script type="application/ld+json">
-  {"@context":"http://schema.org","@type":"Product","name":"Nan Optipro Stage 1 Baby Formula  800g","price": 23.50 }
+
+{
+   "@context":"http://schema.org/",
+   "@type":"Product",
+   "offers":{
+      "@type":"Offer",
+      "availability":"http://schema.org/InStock",
+      "price":"3949.99",
+      "priceCurrency":"USD",
+      "url":"https://www.newegg.com/p/3D5-000D-001T1"
+   },
+   "description":"Cobratype King Cobra Hero Desktop Gaming PC",
+   "name":"Cobratype King Cobra Hero Desktop Gaming PC",
+   "sku":"3D5-000D-001T1",
+   "itemCondition":"NewCondition"
+}
 </script>
 ```  
 
-`json:$.price` would give `23.50`, or you can extract the whole structure
+`json:$..price` or `jq:..price` would give `3949.99`, or you can extract the whole structure (use a JSONpath test website to validate with)
 
-## Proxy configuration
+The application also supports notifying you that it can follow this information automatically
 
-See the wiki https://github.com/dgtlmoon/changedetection.io/wiki/Proxy-configuration
+
+## Proxy Configuration
+
+See the wiki https://github.com/dgtlmoon/changedetection.io/wiki/Proxy-configuration , we also support using [Bright Data proxy services where possible](https://github.com/dgtlmoon/changedetection.io/wiki/Proxy-configuration#brightdata-proxy-support) and [Oxylabs](https://oxylabs.go2cloud.org/SH2d) proxy services.
 
 ## Raspberry Pi support?
 
 Raspberry Pi and linux/arm/v6 linux/arm/v7 arm64 devices are supported! See the wiki for [details](https://github.com/dgtlmoon/changedetection.io/wiki/Fetching-pages-with-WebDriver)
 
+## Import support
+
+Easily [import your list of websites to watch for changes in Excel .xslx file format](https://changedetection.io/tutorial/how-import-your-website-change-detection-lists-excel), or paste in lists of website URLs as plaintext. 
+
+Excel import is recommended - that way you can better organise tags/groups of websites and other features.
+
+
+## API Support
+
+Full REST API for programmatic management of watches, tags, notifications and more. 
+
+- **[Interactive API Documentation](https://changedetection.io/docs/api_v1/index.html)** - Complete API reference with live testing
+- **[OpenAPI Specification](docs/api-spec.yaml)** - Generate SDKs for any programming language
 
 ## Support us
 
 Do you use changedetection.io to make money? does it save you time or money? Does it make your life easier? less stressful? Remember, we write this software when we should be doing actual paid work, we have to buy food and pay rent just like you.
 
 
-Firstly, consider taking out a [change detection monthly subscription - unlimited checks and watches](https://lemonade.changedetection.io/start) , even if you don't use it, you still get the warm fuzzy feeling of helping out the project. (And who knows, you might just use it!)
-
-Or directly donate an amount PayPal [![Donate](https://img.shields.io/badge/Donate-PayPal-green.svg)](https://www.paypal.com/donate/?hosted_button_id=7CP6HR9ZCNDYJ)
-
-Or BTC `1PLFN327GyUarpJd7nVe7Reqg9qHx5frNn`
-
-<img src="https://raw.githubusercontent.com/dgtlmoon/changedetection.io/master/docs/btc-support.png" style="max-width:50%;" alt="Support us!"  />
+Consider taking out an officially supported [website change detection subscription](https://changedetection.io?src=github) , even if you don't use it, you still get the warm fuzzy feeling of helping out the project. (And who knows, you might just use it!)
 
 ## Commercial Support
 
@@ -203,5 +302,19 @@ I offer commercial support, this software is depended on by network security, ae
 [test-shield]: https://github.com/dgtlmoon/changedetection.io/actions/workflows/test-only.yml/badge.svg?branch=master
 
 [license-shield]: https://img.shields.io/github/license/dgtlmoon/changedetection.io.svg?style=for-the-badge
-[release-link]: https://github.com/dgtlmoon.com/changedetection.io/releases
+[release-link]: https://github.com/dgtlmoon/changedetection.io/releases
 [docker-link]: https://hub.docker.com/r/dgtlmoon/changedetection.io
+
+## Commercial Licencing
+
+If you are reselling this software either in part or full as part of any commercial arrangement, you must abide by our COMMERCIAL_LICENCE.md found in our code repository, please contact dgtlmoon@gmail.com and contact@changedetection.io .
+
+## Third-party licenses
+
+changedetectionio.html_tools.elementpath_tostring: Copyright (c), 2018-2021, SISSA (Scuola Internazionale Superiore di Studi Avanzati), Licensed under [MIT license](https://github.com/sissaschool/elementpath/blob/master/LICENSE)
+
+## Contributors
+
+Recognition of fantastic contributors to the project
+
+- Constantin Hong https://github.com/Constantin1489

app.json

@@ -1,21 +0,0 @@
-{
-  "name": "ChangeDetection.io",
-  "description": "The best and simplest self-hosted open source website change detection monitoring and notification service.",
-  "keywords": [
-    "changedetection",
-    "website monitoring"
-  ],
-  "repository": "https://github.com/dgtlmoon/changedetection.io",
-  "success_url": "/",
-  "scripts": {
-  },
-  "env": {
-  },
-  "formation": {
-    "web": {
-      "quantity": 1,
-      "size": "free"
-    }
-  },
-  "image": "heroku/python"
-}

changedetection.py

@@ -1,11 +1,8 @@
-#!/usr/bin/python3
+#!/usr/bin/env python3
 
-# Entry-point for running from the CLI when not installed via Pip, Pip will handle the console_scripts entry_points's from setup.py
-# It's recommended to use `pip3 install changedetection.io` and start with `changedetection.py` instead, it will be linkd to your global path.
-# or Docker.
-# Read more https://github.com/dgtlmoon/changedetection.io/wiki
+# Only exists for direct CLI usage
 
-from changedetectionio import changedetection
+import changedetectionio
 
 if __name__ == '__main__':
-    changedetection.main()
+    changedetectionio.main()

changedetectionio/.gitignore

@@ -1 +1,2 @@
 test-datastore
+package-lock.json

changedetectionio/PLUGIN_README.md

@@ -0,0 +1,98 @@
+# Creating Plugins for changedetection.io
+
+This document describes how to create plugins for changedetection.io. Plugins can be used to extend the functionality of the application in various ways.
+
+## Plugin Types
+
+### UI Stats Tab Plugins
+
+These plugins can add content to the Stats tab in the Edit page. This is useful for adding custom statistics or visualizations about a watch.
+
+#### Creating a UI Stats Tab Plugin
+
+1. Create a Python file in a directory that will be loaded by the plugin system.
+
+2. Use the `global_hookimpl` decorator to implement the `ui_edit_stats_extras` hook:
+
+```python
+import pluggy
+from loguru import logger
+
+global_hookimpl = pluggy.HookimplMarker("changedetectionio")
+
+@global_hookimpl
+def ui_edit_stats_extras(watch):
+    """Add custom content to the stats tab"""
+    # Calculate or retrieve your stats
+    my_stat = calculate_something(watch)
+    
+    # Return HTML content as a string
+    html = f"""
+    <div class="my-plugin-stats">
+        <h4>My Plugin Statistics</h4>
+        <p>My statistic: {my_stat}</p>
+    </div>
+    """
+    return html
+```
+
+3. The HTML you return will be included in the Stats tab.
+
+## Plugin Loading
+
+Plugins can be loaded from:
+
+1. Built-in plugin directories in the codebase
+2. External packages using setuptools entry points
+
+To add a new plugin directory, modify the `plugin_dirs` dictionary in `pluggy_interface.py`.
+
+## Example Plugin
+
+Here's a simple example of a plugin that adds a word count statistic to the Stats tab:
+
+```python
+import pluggy
+from loguru import logger
+
+global_hookimpl = pluggy.HookimplMarker("changedetectionio")
+
+def count_words_in_history(watch):
+    """Count words in the latest snapshot"""
+    try:
+        if not watch.history.keys():
+            return 0
+            
+        latest_key = list(watch.history.keys())[-1]
+        latest_content = watch.get_history_snapshot(latest_key)
+        return len(latest_content.split())
+    except Exception as e:
+        logger.error(f"Error counting words: {str(e)}")
+        return 0
+
+@global_hookimpl
+def ui_edit_stats_extras(watch):
+    """Add word count to the Stats tab"""
+    word_count = count_words_in_history(watch)
+    
+    html = f"""
+    <div class="word-count-stats">
+        <h4>Content Analysis</h4>
+        <table class="pure-table">
+            <tbody>
+                <tr>
+                    <td>Word count (latest snapshot)</td>
+                    <td>{word_count}</td>
+                </tr>
+            </tbody>
+        </table>
+    </div>
+    """
+    return html
+```
+
+## Testing Your Plugin
+
+1. Place your plugin in one of the directories scanned by the plugin system
+2. Restart changedetection.io
+3. Go to the Edit page of a watch and check the Stats tab to see your content

changedetectionio/api/Import.py

@@ -0,0 +1,54 @@
+import os
+from changedetectionio.strtobool import strtobool
+from flask_restful import abort, Resource
+from flask import request
+import validators
+from . import auth, validate_openapi_request
+
+
+class Import(Resource):
+    def __init__(self, **kwargs):
+        # datastore is a black box dependency
+        self.datastore = kwargs['datastore']
+
+    @auth.check_token
+    @validate_openapi_request('importWatches')
+    def post(self):
+        """Import a list of watched URLs."""
+
+        extras = {}
+
+        if request.args.get('proxy'):
+            plist = self.datastore.proxy_list
+            if not request.args.get('proxy') in plist:
+                return "Invalid proxy choice, currently supported proxies are '{}'".format(', '.join(plist)), 400
+            else:
+                extras['proxy'] = request.args.get('proxy')
+
+        dedupe = strtobool(request.args.get('dedupe', 'true'))
+
+        tags = request.args.get('tag')
+        tag_uuids = request.args.get('tag_uuids')
+
+        if tag_uuids:
+            tag_uuids = tag_uuids.split(',')
+
+        urls = request.get_data().decode('utf8').splitlines()
+        added = []
+        allow_simplehost = not strtobool(os.getenv('BLOCK_SIMPLEHOSTS', 'False'))
+        for url in urls:
+            url = url.strip()
+            if not len(url):
+                continue
+
+            # If hosts that only contain alphanumerics are allowed ("localhost" for example)
+            if not validators.url(url, simple_host=allow_simplehost):
+                return f"Invalid or unsupported URL - {url}", 400
+
+            if dedupe and self.datastore.url_exists(url):
+                continue
+
+            new_uuid = self.datastore.add_watch(url=url, extras=extras, tag=tags, tag_uuids=tag_uuids)
+            added.append(new_uuid)
+
+        return added

changedetectionio/api/Notifications.py

@@ -0,0 +1,108 @@
+from flask_expects_json import expects_json
+from flask_restful import Resource, abort
+from flask import request
+from . import auth, validate_openapi_request
+from . import schema_create_notification_urls, schema_delete_notification_urls
+
+class Notifications(Resource):
+    def __init__(self, **kwargs):
+        # datastore is a black box dependency
+        self.datastore = kwargs['datastore']
+
+    @auth.check_token
+    @validate_openapi_request('getNotifications')
+    def get(self):
+        """Return Notification URL List."""
+
+        notification_urls = self.datastore.data.get('settings', {}).get('application', {}).get('notification_urls', [])        
+
+        return {
+                'notification_urls': notification_urls,
+               }, 200
+    
+    @auth.check_token
+    @validate_openapi_request('addNotifications')
+    @expects_json(schema_create_notification_urls)
+    def post(self):
+        """Create Notification URLs."""
+
+        json_data = request.get_json()
+        notification_urls = json_data.get("notification_urls", [])
+
+        from wtforms import ValidationError
+        try:
+            validate_notification_urls(notification_urls)
+        except ValidationError as e:
+            return str(e), 400
+
+        added_urls = []
+
+        for url in notification_urls:
+            clean_url = url.strip()
+            added_url = self.datastore.add_notification_url(clean_url)
+            if added_url:
+                added_urls.append(added_url)
+
+        if not added_urls:
+            return "No valid notification URLs were added", 400
+
+        return {'notification_urls': added_urls}, 201
+    
+    @auth.check_token
+    @validate_openapi_request('replaceNotifications')
+    @expects_json(schema_create_notification_urls)
+    def put(self):
+        """Replace Notification URLs."""
+        json_data = request.get_json()
+        notification_urls = json_data.get("notification_urls", [])
+
+        from wtforms import ValidationError
+        try:
+            validate_notification_urls(notification_urls)
+        except ValidationError as e:
+            return str(e), 400
+        
+        if not isinstance(notification_urls, list):
+            return "Invalid input format", 400
+
+        clean_urls = [url.strip() for url in notification_urls if isinstance(url, str)]
+        self.datastore.data['settings']['application']['notification_urls'] = clean_urls
+        self.datastore.needs_write = True
+
+        return {'notification_urls': clean_urls}, 200
+        
+    @auth.check_token
+    @validate_openapi_request('deleteNotifications')
+    @expects_json(schema_delete_notification_urls)
+    def delete(self):
+        """Delete Notification URLs."""
+
+        json_data = request.get_json()
+        urls_to_delete = json_data.get("notification_urls", [])
+        if not isinstance(urls_to_delete, list):
+            abort(400, message="Expected a list of notification URLs.")
+
+        notification_urls = self.datastore.data['settings']['application'].get('notification_urls', [])
+        deleted = []
+
+        for url in urls_to_delete:
+            clean_url = url.strip()
+            if clean_url in notification_urls:
+                notification_urls.remove(clean_url)
+                deleted.append(clean_url)
+
+        if not deleted:
+            abort(400, message="No matching notification URLs found.")
+
+        self.datastore.data['settings']['application']['notification_urls'] = notification_urls
+        self.datastore.needs_write = True
+
+        return 'OK', 204
+    
+def validate_notification_urls(notification_urls):
+    from changedetectionio.forms import ValidateAppRiseServers
+    validator = ValidateAppRiseServers()
+    class DummyForm: pass
+    dummy_form = DummyForm()
+    field = type("Field", (object,), {"data": notification_urls, "gettext": lambda self, x: x})()
+    validator(dummy_form, field)

changedetectionio/api/Search.py

@@ -0,0 +1,39 @@
+from flask_restful import Resource, abort
+from flask import request
+from . import auth, validate_openapi_request
+
+class Search(Resource):
+    def __init__(self, **kwargs):
+        # datastore is a black box dependency
+        self.datastore = kwargs['datastore']
+
+    @auth.check_token
+    @validate_openapi_request('searchWatches')
+    def get(self):
+        """Search for watches by URL or title text."""
+        query = request.args.get('q', '').strip()
+        tag_limit = request.args.get('tag', '').strip()
+        from changedetectionio.strtobool import strtobool
+        partial = bool(strtobool(request.args.get('partial', '0'))) if 'partial' in request.args else False
+
+        # Require a search query
+        if not query:
+            abort(400, message="Search query 'q' parameter is required")
+
+        # Use the search function from the datastore
+        matching_uuids = self.datastore.search_watches_for_url(query=query, tag_limit=tag_limit, partial=partial)
+
+        # Build the response with watch details
+        results = {}
+        for uuid in matching_uuids:
+            watch = self.datastore.data['watching'].get(uuid)
+            results[uuid] = {
+                'last_changed': watch.last_changed,
+                'last_checked': watch['last_checked'],
+                'last_error': watch['last_error'],
+                'title': watch['title'],
+                'url': watch['url'],
+                'viewed': watch.viewed
+            }
+
+        return results, 200

changedetectionio/api/SystemInfo.py

@@ -0,0 +1,40 @@
+from flask_restful import Resource
+from . import auth, validate_openapi_request
+
+
+class SystemInfo(Resource):
+    def __init__(self, **kwargs):
+        # datastore is a black box dependency
+        self.datastore = kwargs['datastore']
+        self.update_q = kwargs['update_q']
+
+    @auth.check_token
+    @validate_openapi_request('getSystemInfo')
+    def get(self):
+        """Return system info."""
+        import time
+        overdue_watches = []
+
+        # Check all watches and report which have not been checked but should have been
+
+        for uuid, watch in self.datastore.data.get('watching', {}).items():
+            # see if now - last_checked is greater than the time that should have been
+            # this is not super accurate (maybe they just edited it) but better than nothing
+            t = watch.threshold_seconds()
+            if not t:
+                # Use the system wide default
+                t = self.datastore.threshold_seconds
+
+            time_since_check = time.time() - watch.get('last_checked')
+
+            # Allow 5 minutes of grace time before we decide it's overdue
+            if time_since_check - (5 * 60) > t:
+                overdue_watches.append(uuid)
+        from changedetectionio import __version__ as main_version
+        return {
+                   'queue_size': self.update_q.qsize(),
+                   'overdue_watches': overdue_watches,
+                   'uptime': round(time.time() - self.datastore.start_time, 2),
+                   'watch_count': len(self.datastore.data.get('watching', {})),
+                   'version': main_version
+               }, 200

changedetectionio/api/Tags.py

@@ -0,0 +1,119 @@
+from changedetectionio import queuedWatchMetaData
+from changedetectionio import worker_handler
+from flask_expects_json import expects_json
+from flask_restful import abort, Resource
+
+from flask import request
+from . import auth
+
+# Import schemas from __init__.py
+from . import schema_tag, schema_create_tag, schema_update_tag, validate_openapi_request
+
+
+class Tag(Resource):
+    def __init__(self, **kwargs):
+        # datastore is a black box dependency
+        self.datastore = kwargs['datastore']
+        self.update_q = kwargs['update_q']
+
+    # Get information about a single tag
+    # curl http://localhost:5000/api/v1/tag/<string:uuid>
+    @auth.check_token
+    @validate_openapi_request('getTag')
+    def get(self, uuid):
+        """Get data for a single tag/group, toggle notification muting, or recheck all."""
+        from copy import deepcopy
+        tag = deepcopy(self.datastore.data['settings']['application']['tags'].get(uuid))
+        if not tag:
+            abort(404, message=f'No tag exists with the UUID of {uuid}')
+
+        if request.args.get('recheck'):
+            # Recheck all, including muted
+            # Get most overdue first
+            i=0
+            for k in sorted(self.datastore.data['watching'].items(), key=lambda item: item[1].get('last_checked', 0)):
+                watch_uuid = k[0]
+                watch = k[1]
+                if not watch['paused'] and tag['uuid'] not in watch['tags']:
+                    continue
+                worker_handler.queue_item_async_safe(self.update_q, queuedWatchMetaData.PrioritizedItem(priority=1, item={'uuid': watch_uuid}))
+                i+=1
+
+            return f"OK, {i} watches queued", 200
+
+        if request.args.get('muted', '') == 'muted':
+            self.datastore.data['settings']['application']['tags'][uuid]['notification_muted'] = True
+            return "OK", 200
+        elif request.args.get('muted', '') == 'unmuted':
+            self.datastore.data['settings']['application']['tags'][uuid]['notification_muted'] = False
+            return "OK", 200
+
+        return tag
+
+    @auth.check_token
+    @validate_openapi_request('deleteTag')
+    def delete(self, uuid):
+        """Delete a tag/group and remove it from all watches."""
+        if not self.datastore.data['settings']['application']['tags'].get(uuid):
+            abort(400, message='No tag exists with the UUID of {}'.format(uuid))
+
+        # Delete the tag, and any tag reference
+        del self.datastore.data['settings']['application']['tags'][uuid]
+        
+        # Remove tag from all watches
+        for watch_uuid, watch in self.datastore.data['watching'].items():
+            if watch.get('tags') and uuid in watch['tags']:
+                watch['tags'].remove(uuid)
+
+        return 'OK', 204
+
+    @auth.check_token
+    @validate_openapi_request('updateTag')
+    @expects_json(schema_update_tag)
+    def put(self, uuid):
+        """Update tag information."""
+        tag = self.datastore.data['settings']['application']['tags'].get(uuid)
+        if not tag:
+            abort(404, message='No tag exists with the UUID of {}'.format(uuid))
+
+        tag.update(request.json)
+        self.datastore.needs_write_urgent = True
+
+        return "OK", 200
+
+
+    @auth.check_token
+    @validate_openapi_request('createTag')
+    # Only cares for {'title': 'xxxx'}
+    def post(self):
+        """Create a single tag/group."""
+
+        json_data = request.get_json()
+        title = json_data.get("title",'').strip()
+
+
+        new_uuid = self.datastore.add_tag(title=title)
+        if new_uuid:
+            return {'uuid': new_uuid}, 201
+        else:
+            return "Invalid or unsupported tag", 400
+
+class Tags(Resource):
+    def __init__(self, **kwargs):
+        # datastore is a black box dependency
+        self.datastore = kwargs['datastore']
+
+    @auth.check_token
+    @validate_openapi_request('listTags')
+    def get(self):
+        """List tags/groups."""
+        result = {}
+        for uuid, tag in self.datastore.data['settings']['application']['tags'].items():
+            result[uuid] = {
+                'date_created': tag.get('date_created', 0),
+                'notification_muted': tag.get('notification_muted', False),
+                'title': tag.get('title', ''),
+                'uuid': tag.get('uuid')
+            }
+
+        return result, 200

changedetectionio/api/Watch.py

@@ -0,0 +1,290 @@
+import os
+from changedetectionio.strtobool import strtobool
+
+from flask_expects_json import expects_json
+from changedetectionio import queuedWatchMetaData
+from changedetectionio import worker_handler
+from flask_restful import abort, Resource
+from flask import request, make_response, send_from_directory
+import validators
+from . import auth
+import copy
+
+# Import schemas from __init__.py
+from . import schema, schema_create_watch, schema_update_watch, validate_openapi_request
+
+
+def validate_time_between_check_required(json_data):
+    """
+    Validate that at least one time interval is specified when not using default settings.
+    Returns None if valid, or error message string if invalid.
+    Defaults to using global settings if time_between_check_use_default is not provided.
+    """
+    # Default to using global settings if not specified
+    use_default = json_data.get('time_between_check_use_default', True)
+
+    # If using default settings, no validation needed
+    if use_default:
+        return None
+
+    # If not using defaults, check if time_between_check exists and has at least one non-zero value
+    time_check = json_data.get('time_between_check')
+    if not time_check:
+        # No time_between_check provided and not using defaults - this is an error
+        return "At least one time interval (weeks, days, hours, minutes, or seconds) must be specified when not using global settings."
+
+    # time_between_check exists, check if it has at least one non-zero value
+    if any([
+        (time_check.get('weeks') or 0) > 0,
+        (time_check.get('days') or 0) > 0,
+        (time_check.get('hours') or 0) > 0,
+        (time_check.get('minutes') or 0) > 0,
+        (time_check.get('seconds') or 0) > 0
+    ]):
+        return None
+
+    # time_between_check exists but all values are 0 or empty - this is an error
+    return "At least one time interval (weeks, days, hours, minutes, or seconds) must be specified when not using global settings."
+
+
+class Watch(Resource):
+    def __init__(self, **kwargs):
+        # datastore is a black box dependency
+        self.datastore = kwargs['datastore']
+        self.update_q = kwargs['update_q']
+
+    # Get information about a single watch, excluding the history list (can be large)
+    # curl http://localhost:5000/api/v1/watch/<string:uuid>
+    # @todo - version2 - ?muted and ?paused should be able to be called together, return the watch struct not "OK"
+    # ?recheck=true
+    @auth.check_token
+    @validate_openapi_request('getWatch')
+    def get(self, uuid):
+        """Get information about a single watch, recheck, pause, or mute."""
+        from copy import deepcopy
+        watch = deepcopy(self.datastore.data['watching'].get(uuid))
+        if not watch:
+            abort(404, message='No watch exists with the UUID of {}'.format(uuid))
+
+        if request.args.get('recheck'):
+            worker_handler.queue_item_async_safe(self.update_q, queuedWatchMetaData.PrioritizedItem(priority=1, item={'uuid': uuid}))
+            return "OK", 200
+        if request.args.get('paused', '') == 'paused':
+            self.datastore.data['watching'].get(uuid).pause()
+            return "OK", 200
+        elif request.args.get('paused', '') == 'unpaused':
+            self.datastore.data['watching'].get(uuid).unpause()
+            return "OK", 200
+        if request.args.get('muted', '') == 'muted':
+            self.datastore.data['watching'].get(uuid).mute()
+            return "OK", 200
+        elif request.args.get('muted', '') == 'unmuted':
+            self.datastore.data['watching'].get(uuid).unmute()
+            return "OK", 200
+
+        # Return without history, get that via another API call
+        # Properties are not returned as a JSON, so add the required props manually
+        watch['history_n'] = watch.history_n
+        # attr .last_changed will check for the last written text snapshot on change
+        watch['last_changed'] = watch.last_changed
+        watch['viewed'] = watch.viewed
+        watch['link'] = watch.link,
+
+        return watch
+
+    @auth.check_token
+    @validate_openapi_request('deleteWatch')
+    def delete(self, uuid):
+        """Delete a watch and related history."""
+        if not self.datastore.data['watching'].get(uuid):
+            abort(400, message='No watch exists with the UUID of {}'.format(uuid))
+
+        self.datastore.delete(uuid)
+        return 'OK', 204
+
+    @auth.check_token
+    @validate_openapi_request('updateWatch')
+    @expects_json(schema_update_watch)
+    def put(self, uuid):
+        """Update watch information."""
+        watch = self.datastore.data['watching'].get(uuid)
+        if not watch:
+            abort(404, message='No watch exists with the UUID of {}'.format(uuid))
+
+        if request.json.get('proxy'):
+            plist = self.datastore.proxy_list
+            if not request.json.get('proxy') in plist:
+                return "Invalid proxy choice, currently supported proxies are '{}'".format(', '.join(plist)), 400
+
+        # Validate time_between_check when not using defaults
+        validation_error = validate_time_between_check_required(request.json)
+        if validation_error:
+            return validation_error, 400
+
+        watch.update(request.json)
+
+        return "OK", 200
+
+
+class WatchHistory(Resource):
+    def __init__(self, **kwargs):
+        # datastore is a black box dependency
+        self.datastore = kwargs['datastore']
+
+    # Get a list of available history for a watch by UUID
+    # curl http://localhost:5000/api/v1/watch/<string:uuid>/history
+    @auth.check_token
+    @validate_openapi_request('getWatchHistory')
+    def get(self, uuid):
+        """Get a list of all historical snapshots available for a watch."""
+        watch = self.datastore.data['watching'].get(uuid)
+        if not watch:
+            abort(404, message='No watch exists with the UUID of {}'.format(uuid))
+        return watch.history, 200
+
+
+class WatchSingleHistory(Resource):
+    def __init__(self, **kwargs):
+        # datastore is a black box dependency
+        self.datastore = kwargs['datastore']
+
+    @auth.check_token
+    @validate_openapi_request('getWatchSnapshot')
+    def get(self, uuid, timestamp):
+        """Get single snapshot from watch."""
+        watch = self.datastore.data['watching'].get(uuid)
+        if not watch:
+            abort(404, message=f"No watch exists with the UUID of {uuid}")
+
+        if not len(watch.history):
+            abort(404, message=f"Watch found but no history exists for the UUID {uuid}")
+
+        if timestamp == 'latest':
+            timestamp = list(watch.history.keys())[-1]
+
+        if request.args.get('html'):
+            content = watch.get_fetched_html(timestamp)
+            if content:
+                response = make_response(content, 200)
+                response.mimetype = "text/html"
+            else:
+                response = make_response("No content found", 404)
+                response.mimetype = "text/plain"
+        else:
+            content = watch.get_history_snapshot(timestamp)
+            response = make_response(content, 200)
+            response.mimetype = "text/plain"
+
+        return response
+
+class WatchFavicon(Resource):
+    def __init__(self, **kwargs):
+        # datastore is a black box dependency
+        self.datastore = kwargs['datastore']
+
+    @auth.check_token
+    @validate_openapi_request('getWatchFavicon')
+    def get(self, uuid):
+        """Get favicon for a watch."""
+        watch = self.datastore.data['watching'].get(uuid)
+        if not watch:
+            abort(404, message=f"No watch exists with the UUID of {uuid}")
+
+        favicon_filename = watch.get_favicon_filename()
+        if favicon_filename:
+            try:
+                import magic
+                mime = magic.from_file(
+                    os.path.join(watch.watch_data_dir, favicon_filename),
+                    mime=True
+                )
+            except ImportError:
+                # Fallback, no python-magic
+                import mimetypes
+                mime, encoding = mimetypes.guess_type(favicon_filename)
+
+            response = make_response(send_from_directory(watch.watch_data_dir, favicon_filename))
+            response.headers['Content-type'] = mime
+            response.headers['Cache-Control'] = 'max-age=300, must-revalidate'  # Cache for 5 minutes, then revalidate
+            return response
+
+        abort(404, message=f'No Favicon available for {uuid}')
+
+
+class CreateWatch(Resource):
+    def __init__(self, **kwargs):
+        # datastore is a black box dependency
+        self.datastore = kwargs['datastore']
+        self.update_q = kwargs['update_q']
+
+    @auth.check_token
+    @validate_openapi_request('createWatch')
+    @expects_json(schema_create_watch)
+    def post(self):
+        """Create a single watch."""
+
+        json_data = request.get_json()
+        url = json_data['url'].strip()
+
+        # If hosts that only contain alphanumerics are allowed ("localhost" for example)
+        allow_simplehost = not strtobool(os.getenv('BLOCK_SIMPLEHOSTS', 'False'))
+        if not validators.url(url, simple_host=allow_simplehost):
+            return "Invalid or unsupported URL", 400
+
+        if json_data.get('proxy'):
+            plist = self.datastore.proxy_list
+            if not json_data.get('proxy') in plist:
+                return "Invalid proxy choice, currently supported proxies are '{}'".format(', '.join(plist)), 400
+
+        # Validate time_between_check when not using defaults
+        validation_error = validate_time_between_check_required(json_data)
+        if validation_error:
+            return validation_error, 400
+
+        extras = copy.deepcopy(json_data)
+
+        # Because we renamed 'tag' to 'tags' but don't want to change the API (can do this in v2 of the API)
+        tags = None
+        if extras.get('tag'):
+            tags = extras.get('tag')
+            del extras['tag']
+
+        del extras['url']
+
+        new_uuid = self.datastore.add_watch(url=url, extras=extras, tag=tags)
+        if new_uuid:
+            worker_handler.queue_item_async_safe(self.update_q, queuedWatchMetaData.PrioritizedItem(priority=1, item={'uuid': new_uuid}))
+            return {'uuid': new_uuid}, 201
+        else:
+            return "Invalid or unsupported URL", 400
+
+    @auth.check_token
+    @validate_openapi_request('listWatches')
+    def get(self):
+        """List watches."""
+        list = {}
+
+        tag_limit = request.args.get('tag', '').lower()
+        for uuid, watch in self.datastore.data['watching'].items():
+            # Watch tags by name (replace the other calls?)
+            tags = self.datastore.get_all_tags_for_watch(uuid=uuid)
+            if tag_limit and not any(v.get('title').lower() == tag_limit for k, v in tags.items()):
+                continue
+
+            list[uuid] = {
+                'last_changed': watch.last_changed,
+                'last_checked': watch['last_checked'],
+                'last_error': watch['last_error'],
+                'link': watch.link,
+                'page_title': watch['page_title'],
+                'title': watch['title'],
+                'url': watch['url'],
+                'viewed': watch.viewed
+            }
+
+        if request.args.get('recheck_all'):
+            for uuid in self.datastore.data['watching'].keys():
+                worker_handler.queue_item_async_safe(self.update_q, queuedWatchMetaData.PrioritizedItem(priority=1, item={'uuid': uuid}))
+            return {'status': "OK"}, 200
+
+        return list, 200

changedetectionio/api/__init__.py

@@ -0,0 +1,78 @@
+import copy
+import yaml
+import functools
+from flask import request, abort
+from loguru import logger
+from openapi_core import OpenAPI
+from openapi_core.contrib.flask import FlaskOpenAPIRequest
+from . import api_schema
+from ..model import watch_base
+
+# Build a JSON Schema atleast partially based on our Watch model
+watch_base_config = watch_base()
+schema = api_schema.build_watch_json_schema(watch_base_config)
+
+schema_create_watch = copy.deepcopy(schema)
+schema_create_watch['required'] = ['url']
+del schema_create_watch['properties']['last_viewed']
+
+schema_update_watch = copy.deepcopy(schema)
+schema_update_watch['additionalProperties'] = False
+
+# Tag schema is also based on watch_base since Tag inherits from it
+schema_tag = copy.deepcopy(schema)
+schema_create_tag = copy.deepcopy(schema_tag)
+schema_create_tag['required'] = ['title']
+schema_update_tag = copy.deepcopy(schema_tag)
+schema_update_tag['additionalProperties'] = False
+
+schema_notification_urls = copy.deepcopy(schema)
+schema_create_notification_urls = copy.deepcopy(schema_notification_urls)
+schema_create_notification_urls['required'] = ['notification_urls']
+schema_delete_notification_urls = copy.deepcopy(schema_notification_urls)
+schema_delete_notification_urls['required'] = ['notification_urls']
+
+@functools.cache
+def get_openapi_spec():
+    import os
+    spec_path = os.path.join(os.path.dirname(__file__), '../../docs/api-spec.yaml')
+    with open(spec_path, 'r') as f:
+        spec_dict = yaml.safe_load(f)
+    _openapi_spec = OpenAPI.from_dict(spec_dict)
+    return _openapi_spec
+
+def validate_openapi_request(operation_id):
+    """Decorator to validate incoming requests against OpenAPI spec."""
+    def decorator(f):
+        @functools.wraps(f)
+        def wrapper(*args, **kwargs):
+            try:
+                # Skip OpenAPI validation for GET requests since they don't have request bodies
+                if request.method.upper() != 'GET':
+                    spec = get_openapi_spec()
+                    openapi_request = FlaskOpenAPIRequest(request)
+                    result = spec.unmarshal_request(openapi_request)
+                    if result.errors:
+                        from werkzeug.exceptions import BadRequest
+                        error_details = []
+                        for error in result.errors:
+                            error_details.append(str(error))
+                        raise BadRequest(f"OpenAPI validation failed: {error_details}")
+            except BadRequest:
+                # Re-raise BadRequest exceptions (validation failures)
+                raise
+            except Exception as e:
+                # If OpenAPI spec loading fails, log but don't break existing functionality
+                logger.critical(f"OpenAPI validation warning for {operation_id}: {e}")
+                abort(500)
+            return f(*args, **kwargs)
+        return wrapper
+    return decorator
+
+# Import all API resources
+from .Watch import Watch, WatchHistory, WatchSingleHistory, CreateWatch, WatchFavicon
+from .Tags import Tags, Tag
+from .Import import Import
+from .SystemInfo import SystemInfo
+from .Notifications import Notifications
+

changedetectionio/api/api_schema.py

@@ -0,0 +1,159 @@
+# Responsible for building the storage dict into a set of rules ("JSON Schema") acceptable via the API
+# Probably other ways to solve this when the backend switches to some ORM
+from changedetectionio.notification import valid_notification_formats
+
+
+def build_time_between_check_json_schema():
+    # Setup time between check schema
+    schema_properties_time_between_check = {
+        "type": "object",
+        "additionalProperties": False,
+        "properties": {}
+    }
+    for p in ['weeks', 'days', 'hours', 'minutes', 'seconds']:
+        schema_properties_time_between_check['properties'][p] = {
+            "anyOf": [
+                {
+                    "type": "integer"
+                },
+                {
+                    "type": "null"
+                }
+            ]
+        }
+
+    return schema_properties_time_between_check
+
+def build_watch_json_schema(d):
+    # Base JSON schema
+    schema = {
+        'type': 'object',
+        'properties': {},
+    }
+
+    for k, v in d.items():
+        # @todo 'integer' is not covered here because its almost always for internal usage
+
+        if isinstance(v, type(None)):
+            schema['properties'][k] = {
+                "anyOf": [
+                    {"type": "null"},
+                ]
+            }
+        elif isinstance(v, list):
+            schema['properties'][k] = {
+                "anyOf": [
+                    {"type": "array",
+                     # Always is an array of strings, like text or regex or something
+                     "items": {
+                         "type": "string",
+                         "maxLength": 5000
+                     }
+                     },
+                ]
+            }
+        elif isinstance(v, bool):
+            schema['properties'][k] = {
+                "anyOf": [
+                    {"type": "boolean"},
+                ]
+            }
+        elif isinstance(v, str):
+            schema['properties'][k] = {
+                "anyOf": [
+                    {"type": "string",
+                     "maxLength": 5000},
+                ]
+            }
+
+    # Can also be a string (or None by default above)
+    for v in ['body',
+              'notification_body',
+              'notification_format',
+              'notification_title',
+              'proxy',
+              'tag',
+              'title',
+              'webdriver_js_execute_code'
+              ]:
+        schema['properties'][v]['anyOf'].append({'type': 'string', "maxLength": 5000})
+
+    for v in ['last_viewed']:
+        schema['properties'][v] = {
+            "type": "integer",
+            "description": "Unix timestamp in seconds of the last time the watch was viewed.",
+            "minimum": 0
+        }
+
+    # None or Boolean
+    schema['properties']['track_ldjson_price_data']['anyOf'].append({'type': 'boolean'})
+
+    schema['properties']['method'] = {"type": "string",
+                                      "enum": ["GET", "POST", "DELETE", "PUT"]
+                                      }
+
+    schema['properties']['fetch_backend']['anyOf'].append({"type": "string",
+                                                           "enum": ["html_requests", "html_webdriver"]
+                                                           })
+
+
+
+    # All headers must be key/value type dict
+    schema['properties']['headers'] = {
+        "type": "object",
+        "patternProperties": {
+            # Should always be a string:string type value
+            ".*": {"type": "string"},
+        }
+    }
+
+    schema['properties']['notification_format'] = {'type': 'string',
+                                                   'enum': list(valid_notification_formats.keys())
+                                                   }
+
+    # Stuff that shouldn't be available but is just state-storage
+    for v in ['previous_md5', 'last_error', 'has_ldjson_price_data', 'previous_md5_before_filters', 'uuid']:
+        del schema['properties'][v]
+
+    schema['properties']['webdriver_delay']['anyOf'].append({'type': 'integer'})
+
+    schema['properties']['time_between_check'] = build_time_between_check_json_schema()
+
+    schema['properties']['time_between_check_use_default'] = {
+        "type": "boolean",
+        "default": True,
+        "description": "Whether to use global settings for time between checks - defaults to true if not set"
+    }
+
+    schema['properties']['browser_steps'] = {
+        "anyOf": [
+            {
+                "type": "array",
+                "items": {
+                    "type": "object",
+                    "properties": {
+                        "operation": {
+                            "type": ["string", "null"],
+                            "maxLength": 5000  # Allows null and any string up to 5000 chars (including "")
+                        },
+                        "selector": {
+                            "type": ["string", "null"],
+                            "maxLength": 5000
+                        },
+                        "optional_value": {
+                            "type": ["string", "null"],
+                            "maxLength": 5000
+                        }
+                    },
+                    "required": ["operation", "selector", "optional_value"],
+                    "additionalProperties": False  # No extra keys allowed
+                }
+            },
+            {"type": "null"},  # Allows null for `browser_steps`
+            {"type": "array", "maxItems": 0}  # Allows empty array []
+        ]
+    }
+
+    # headers ?
+    return schema
+

changedetectionio/api/api_v1.py

@@ -1,124 +0,0 @@
-from flask_restful import abort, Resource
-from flask import request, make_response
-import validators
-from . import auth
-
-
-
-# https://www.w3.org/Protocols/rfc2616/rfc2616-sec10.html
-
-class Watch(Resource):
-    def __init__(self, **kwargs):
-        # datastore is a black box dependency
-        self.datastore = kwargs['datastore']
-        self.update_q = kwargs['update_q']
-
-    # Get information about a single watch, excluding the history list (can be large)
-    # curl http://localhost:4000/api/v1/watch/<string:uuid>
-    # ?recheck=true
-    @auth.check_token
-    def get(self, uuid):
-        from copy import deepcopy
-        watch = deepcopy(self.datastore.data['watching'].get(uuid))
-        if not watch:
-            abort(404, message='No watch exists with the UUID of {}'.format(uuid))
-
-        if request.args.get('recheck'):
-            self.update_q.put(uuid)
-            return "OK", 200
-
-        # Return without history, get that via another API call
-        watch['history_n'] = watch.history_n
-        return watch
-
-    @auth.check_token
-    def delete(self, uuid):
-        if not self.datastore.data['watching'].get(uuid):
-            abort(400, message='No watch exists with the UUID of {}'.format(uuid))
-
-        self.datastore.delete(uuid)
-        return 'OK', 204
-
-
-class WatchHistory(Resource):
-    def __init__(self, **kwargs):
-        # datastore is a black box dependency
-        self.datastore = kwargs['datastore']
-
-    # Get a list of available history for a watch by UUID
-    # curl http://localhost:4000/api/v1/watch/<string:uuid>/history
-    def get(self, uuid):
-        watch = self.datastore.data['watching'].get(uuid)
-        if not watch:
-            abort(404, message='No watch exists with the UUID of {}'.format(uuid))
-        return watch.history, 200
-
-
-class WatchSingleHistory(Resource):
-    def __init__(self, **kwargs):
-        # datastore is a black box dependency
-        self.datastore = kwargs['datastore']
-
-    # Read a given history snapshot and return its content
-    # <string:timestamp> or "latest"
-    # curl http://localhost:4000/api/v1/watch/<string:uuid>/history/<int:timestamp>
-    @auth.check_token
-    def get(self, uuid, timestamp):
-        watch = self.datastore.data['watching'].get(uuid)
-        if not watch:
-            abort(404, message='No watch exists with the UUID of {}'.format(uuid))
-
-        if not len(watch.history):
-            abort(404, message='Watch found but no history exists for the UUID {}'.format(uuid))
-
-        if timestamp == 'latest':
-            timestamp = list(watch.history.keys())[-1]
-
-        with open(watch.history[timestamp], 'r') as f:
-            content = f.read()
-
-        response = make_response(content, 200)
-        response.mimetype = "text/plain"
-        return response
-
-
-class CreateWatch(Resource):
-    def __init__(self, **kwargs):
-        # datastore is a black box dependency
-        self.datastore = kwargs['datastore']
-        self.update_q = kwargs['update_q']
-
-    @auth.check_token
-    def post(self):
-        # curl http://localhost:4000/api/v1/watch -H "Content-Type: application/json" -d '{"url": "https://my-nice.com", "tag": "one, two" }'
-        json_data = request.get_json()
-        tag = json_data['tag'].strip() if json_data.get('tag') else ''
-
-        if not validators.url(json_data['url'].strip()):
-            return "Invalid or unsupported URL", 400
-
-        extras = {'title': json_data['title'].strip()} if json_data.get('title') else {}
-
-        new_uuid = self.datastore.add_watch(url=json_data['url'].strip(), tag=tag, extras=extras)
-        self.update_q.put(new_uuid)
-        return {'uuid': new_uuid}, 201
-
-    # Return concise list of available watches and some very basic info
-    # curl http://localhost:4000/api/v1/watch|python -mjson.tool
-    # ?recheck_all=1 to recheck all
-    @auth.check_token
-    def get(self):
-        list = {}
-        for k, v in self.datastore.data['watching'].items():
-            list[k] = {'url': v['url'],
-                       'title': v['title'],
-                       'last_checked': v['last_checked'],
-                       'last_changed': v['last_changed'],
-                       'last_error': v['last_error']}
-
-        if request.args.get('recheck_all'):
-            for uuid in self.datastore.data['watching'].keys():
-                self.update_q.put(uuid)
-            return {'status': "OK"}, 200
-
-        return list, 200

changedetectionio/api/auth.py

@@ -11,22 +11,14 @@ def check_token(f):
         datastore = args[0].datastore
 
         config_api_token_enabled = datastore.data['settings']['application'].get('api_access_token_enabled')
-        if not config_api_token_enabled:
-            return
-
-        try:
-            api_key_header = request.headers['x-api-key']
-        except KeyError:
-            return make_response(
-                jsonify("No authorization x-api-key header."), 403
-            )
-
         config_api_token = datastore.data['settings']['application'].get('api_access_token')
 
-        if api_key_header != config_api_token:
-            return make_response(
-                jsonify("Invalid access - API key invalid."), 403
-            )
+        # config_api_token_enabled - a UI option in settings if access should obey the key or not
+        if config_api_token_enabled:
+            if request.headers.get('x-api-key') != config_api_token:
+                return make_response(
+                    jsonify("Invalid access - API key invalid."), 403
+                )
 
         return f(*args, **kwargs)
 

changedetectionio/apprise_asset.py

@@ -1,11 +0,0 @@
-import apprise
-
-# Create our AppriseAsset and populate it with some of our new values:
-# https://github.com/caronc/apprise/wiki/Development_API#the-apprise-asset-object
-asset = apprise.AppriseAsset(
-   image_url_logo='https://raw.githubusercontent.com/dgtlmoon/changedetection.io/master/changedetectionio/static/images/avatar-256x256.png'
-)
-
-asset.app_id = "changedetection.io"
-asset.app_desc = "ChangeDetection.io best and simplest website monitoring and change detection"
-asset.app_url = "https://changedetection.io"

changedetectionio/async_update_worker.py

@@ -0,0 +1,465 @@
+from .processors.exceptions import ProcessorException
+import changedetectionio.content_fetchers.exceptions as content_fetchers_exceptions
+from changedetectionio.processors.text_json_diff.processor import FilterNotFoundInResponse
+from changedetectionio import html_tools
+from changedetectionio.flask_app import watch_check_update
+
+import asyncio
+import importlib
+import os
+import queue
+import time
+
+from loguru import logger
+
+# Async version of update_worker
+# Processes jobs from AsyncSignalPriorityQueue instead of threaded queue
+
+async def async_update_worker(worker_id, q, notification_q, app, datastore):
+    """
+    Async worker function that processes watch check jobs from the queue.
+    
+    Args:
+        worker_id: Unique identifier for this worker
+        q: AsyncSignalPriorityQueue containing jobs to process
+        notification_q: Standard queue for notifications
+        app: Flask application instance
+        datastore: Application datastore
+    """
+    # Set a descriptive name for this task
+    task = asyncio.current_task()
+    if task:
+        task.set_name(f"async-worker-{worker_id}")
+    
+    logger.info(f"Starting async worker {worker_id}")
+    
+    while not app.config.exit.is_set():
+        update_handler = None
+        watch = None
+
+        try:
+            # Use native janus async interface - no threads needed!
+            queued_item_data = await asyncio.wait_for(q.async_get(), timeout=1.0)
+            
+        except asyncio.TimeoutError:
+            # No jobs available, continue loop
+            continue
+        except Exception as e:
+            logger.critical(f"CRITICAL: Worker {worker_id} failed to get queue item: {type(e).__name__}: {e}")
+            
+            # Log queue health for debugging
+            try:
+                queue_size = q.qsize()
+                is_empty = q.empty()
+                logger.critical(f"CRITICAL: Worker {worker_id} queue health - size: {queue_size}, empty: {is_empty}")
+            except Exception as health_e:
+                logger.critical(f"CRITICAL: Worker {worker_id} queue health check failed: {health_e}")
+            
+            await asyncio.sleep(0.1)
+            continue
+        
+        uuid = queued_item_data.item.get('uuid')
+        fetch_start_time = round(time.time())
+        
+        # Mark this UUID as being processed
+        from changedetectionio import worker_handler
+        worker_handler.set_uuid_processing(uuid, processing=True)
+        
+        try:
+            if uuid in list(datastore.data['watching'].keys()) and datastore.data['watching'][uuid].get('url'):
+                changed_detected = False
+                contents = b''
+                process_changedetection_results = True
+                update_obj = {}
+
+                # Clear last errors
+                datastore.data['watching'][uuid]['browser_steps_last_error_step'] = None
+                datastore.data['watching'][uuid]['last_checked'] = fetch_start_time
+
+                watch = datastore.data['watching'].get(uuid)
+
+                logger.info(f"Worker {worker_id} processing watch UUID {uuid} Priority {queued_item_data.priority} URL {watch['url']}")
+
+                try:
+                    watch_check_update.send(watch_uuid=uuid)
+
+                    # Processor is what we are using for detecting the "Change"
+                    processor = watch.get('processor', 'text_json_diff')
+
+                    # Init a new 'difference_detection_processor'
+                    processor_module_name = f"changedetectionio.processors.{processor}.processor"
+                    try:
+                        processor_module = importlib.import_module(processor_module_name)
+                    except ModuleNotFoundError as e:
+                        print(f"Processor module '{processor}' not found.")
+                        raise e
+
+                    update_handler = processor_module.perform_site_check(datastore=datastore,
+                                                                         watch_uuid=uuid)
+
+                    # All fetchers are now async, so call directly
+                    await update_handler.call_browser()
+
+                    # Run change detection (this is synchronous)
+                    changed_detected, update_obj, contents = update_handler.run_changedetection(watch=watch)
+
+                except PermissionError as e:
+                    logger.critical(f"File permission error updating file, watch: {uuid}")
+                    logger.critical(str(e))
+                    process_changedetection_results = False
+
+                except ProcessorException as e:
+                    if e.screenshot:
+                        watch.save_screenshot(screenshot=e.screenshot)
+                    if e.xpath_data:
+                        watch.save_xpath_data(data=e.xpath_data)
+                    datastore.update_watch(uuid=uuid, update_obj={'last_error': e.message})
+                    process_changedetection_results = False
+
+                except content_fetchers_exceptions.ReplyWithContentButNoText as e:
+                    extra_help = ""
+                    if e.has_filters:
+                        has_img = html_tools.include_filters(include_filters='img',
+                                                             html_content=e.html_content)
+                        if has_img:
+                            extra_help = ", it's possible that the filters you have give an empty result or contain only an image."
+                        else:
+                            extra_help = ", it's possible that the filters were found, but contained no usable text."
+
+                    datastore.update_watch(uuid=uuid, update_obj={
+                        'last_error': f"Got HTML content but no text found (With {e.status_code} reply code){extra_help}"
+                    })
+
+                    if e.screenshot:
+                        watch.save_screenshot(screenshot=e.screenshot, as_error=True)
+
+                    if e.xpath_data:
+                        watch.save_xpath_data(data=e.xpath_data)
+                        
+                    process_changedetection_results = False
+
+                except content_fetchers_exceptions.Non200ErrorCodeReceived as e:
+                    if e.status_code == 403:
+                        err_text = "Error - 403 (Access denied) received"
+                    elif e.status_code == 404:
+                        err_text = "Error - 404 (Page not found) received"
+                    elif e.status_code == 407:
+                        err_text = "Error - 407 (Proxy authentication required) received, did you need a username and password for the proxy?"
+                    elif e.status_code == 500:
+                        err_text = "Error - 500 (Internal server error) received from the web site"
+                    else:
+                        extra = ' (Access denied or blocked)' if str(e.status_code).startswith('4') else ''
+                        err_text = f"Error - Request returned a HTTP error code {e.status_code}{extra}"
+
+                    if e.screenshot:
+                        watch.save_screenshot(screenshot=e.screenshot, as_error=True)
+                    if e.xpath_data:
+                        watch.save_xpath_data(data=e.xpath_data, as_error=True)
+                    if e.page_text:
+                        watch.save_error_text(contents=e.page_text)
+
+                    datastore.update_watch(uuid=uuid, update_obj={'last_error': err_text})
+                    process_changedetection_results = False
+
+                except FilterNotFoundInResponse as e:
+                    if not datastore.data['watching'].get(uuid):
+                        continue
+
+                    err_text = "Warning, no filters were found, no change detection ran - Did the page change layout? update your Visual Filter if necessary."
+                    datastore.update_watch(uuid=uuid, update_obj={'last_error': err_text})
+
+                    # Filter wasnt found, but we should still update the visual selector so that they can have a chance to set it up again
+                    if e.screenshot:
+                        watch.save_screenshot(screenshot=e.screenshot)
+
+                    if e.xpath_data:
+                        watch.save_xpath_data(data=e.xpath_data)
+
+                    # Only when enabled, send the notification
+                    if watch.get('filter_failure_notification_send', False):
+                        c = watch.get('consecutive_filter_failures', 0)
+                        c += 1
+                        # Send notification if we reached the threshold?
+                        threshold = datastore.data['settings']['application'].get('filter_failure_notification_threshold_attempts', 0)
+                        logger.debug(f"Filter for {uuid} not found, consecutive_filter_failures: {c} of threshold {threshold}")
+                        if c >= threshold:
+                            if not watch.get('notification_muted'):
+                                logger.debug(f"Sending filter failed notification for {uuid}")
+                                await send_filter_failure_notification(uuid, notification_q, datastore)
+                            c = 0
+                            logger.debug(f"Reset filter failure count back to zero")
+
+                        datastore.update_watch(uuid=uuid, update_obj={'consecutive_filter_failures': c})
+                    else:
+                        logger.trace(f"{uuid} - filter_failure_notification_send not enabled, skipping")
+
+                    process_changedetection_results = False
+
+                except content_fetchers_exceptions.checksumFromPreviousCheckWasTheSame as e:
+                    # Yes fine, so nothing todo, don't continue to process.
+                    process_changedetection_results = False
+                    changed_detected = False
+                    
+                except content_fetchers_exceptions.BrowserConnectError as e:
+                    datastore.update_watch(uuid=uuid,
+                                         update_obj={'last_error': e.msg})
+                    process_changedetection_results = False
+                    
+                except content_fetchers_exceptions.BrowserFetchTimedOut as e:
+                    datastore.update_watch(uuid=uuid,
+                                         update_obj={'last_error': e.msg})
+                    process_changedetection_results = False
+                    
+                except content_fetchers_exceptions.BrowserStepsStepException as e:
+                    if not datastore.data['watching'].get(uuid):
+                        continue
+
+                    error_step = e.step_n + 1
+                    from playwright._impl._errors import TimeoutError, Error
+
+                    # Generally enough info for TimeoutError (couldnt locate the element after default seconds)
+                    err_text = f"Browser step at position {error_step} could not run, check the watch, add a delay if necessary, view Browser Steps to see screenshot at that step."
+
+                    if e.original_e.name == "TimeoutError":
+                        # Just the first line is enough, the rest is the stack trace
+                        err_text += " Could not find the target."
+                    else:
+                        # Other Error, more info is good.
+                        err_text += " " + str(e.original_e).splitlines()[0]
+
+                    logger.debug(f"BrowserSteps exception at step {error_step} {str(e.original_e)}")
+
+                    datastore.update_watch(uuid=uuid,
+                                         update_obj={'last_error': err_text,
+                                                   'browser_steps_last_error_step': error_step})
+
+                    if watch.get('filter_failure_notification_send', False):
+                        c = watch.get('consecutive_filter_failures', 0)
+                        c += 1
+                        # Send notification if we reached the threshold?
+                        threshold = datastore.data['settings']['application'].get('filter_failure_notification_threshold_attempts', 0)
+                        logger.error(f"Step for {uuid} not found, consecutive_filter_failures: {c}")
+                        if threshold > 0 and c >= threshold:
+                            if not watch.get('notification_muted'):
+                                await send_step_failure_notification(watch_uuid=uuid, step_n=e.step_n, notification_q=notification_q, datastore=datastore)
+                            c = 0
+
+                        datastore.update_watch(uuid=uuid, update_obj={'consecutive_filter_failures': c})
+
+                    process_changedetection_results = False
+
+                except content_fetchers_exceptions.EmptyReply as e:
+                    # Some kind of custom to-str handler in the exception handler that does this?
+                    err_text = "EmptyReply - try increasing 'Wait seconds before extracting text', Status Code {}".format(e.status_code)
+                    datastore.update_watch(uuid=uuid, update_obj={'last_error': err_text,
+                                                                'last_check_status': e.status_code})
+                    process_changedetection_results = False
+                    
+                except content_fetchers_exceptions.ScreenshotUnavailable as e:
+                    err_text = "Screenshot unavailable, page did not render fully in the expected time or page was too long - try increasing 'Wait seconds before extracting text'"
+                    datastore.update_watch(uuid=uuid, update_obj={'last_error': err_text,
+                                                                'last_check_status': e.status_code})
+                    process_changedetection_results = False
+                    
+                except content_fetchers_exceptions.JSActionExceptions as e:
+                    err_text = "Error running JS Actions - Page request - "+e.message
+                    if e.screenshot:
+                        watch.save_screenshot(screenshot=e.screenshot, as_error=True)
+                    datastore.update_watch(uuid=uuid, update_obj={'last_error': err_text,
+                                                                'last_check_status': e.status_code})
+                    process_changedetection_results = False
+                    
+                except content_fetchers_exceptions.PageUnloadable as e:
+                    err_text = "Page request from server didnt respond correctly"
+                    if e.message:
+                        err_text = "{} - {}".format(err_text, e.message)
+
+                    if e.screenshot:
+                        watch.save_screenshot(screenshot=e.screenshot, as_error=True)
+
+                    datastore.update_watch(uuid=uuid, update_obj={'last_error': err_text,
+                                                                'last_check_status': e.status_code,
+                                                                'has_ldjson_price_data': None})
+                    process_changedetection_results = False
+                    
+                except content_fetchers_exceptions.BrowserStepsInUnsupportedFetcher as e:
+                    err_text = "This watch has Browser Steps configured and so it cannot run with the 'Basic fast Plaintext/HTTP Client', either remove the Browser Steps or select a Chrome fetcher."
+                    datastore.update_watch(uuid=uuid, update_obj={'last_error': err_text})
+                    process_changedetection_results = False
+                    logger.error(f"Exception (BrowserStepsInUnsupportedFetcher) reached processing watch UUID: {uuid}")
+
+                except Exception as e:
+                    logger.error(f"Worker {worker_id} exception processing watch UUID: {uuid}")
+                    logger.error(str(e))
+                    datastore.update_watch(uuid=uuid, update_obj={'last_error': "Exception: " + str(e)})
+                    process_changedetection_results = False
+
+                else:
+                    if not datastore.data['watching'].get(uuid):
+                        continue
+
+                    update_obj['content-type'] = update_handler.fetcher.get_all_headers().get('content-type', '').lower()
+
+                    if not watch.get('ignore_status_codes'):
+                        update_obj['consecutive_filter_failures'] = 0
+
+                    update_obj['last_error'] = False
+                    cleanup_error_artifacts(uuid, datastore)
+
+                if not datastore.data['watching'].get(uuid):
+                    continue
+
+                if process_changedetection_results:
+                    try:
+                        datastore.update_watch(uuid=uuid, update_obj=update_obj)
+
+                        if changed_detected or not watch.history_n:
+                            if update_handler.screenshot:
+                                watch.save_screenshot(screenshot=update_handler.screenshot)
+
+                            if update_handler.xpath_data:
+                                watch.save_xpath_data(data=update_handler.xpath_data)
+
+                            # Ensure unique timestamp for history
+                            if watch.newest_history_key and int(fetch_start_time) == int(watch.newest_history_key):
+                                logger.warning(f"Timestamp {fetch_start_time} already exists, waiting 1 seconds")
+                                fetch_start_time += 1
+                                await asyncio.sleep(1)
+
+                            watch.save_history_text(contents=contents,
+                                                    timestamp=int(fetch_start_time),
+                                                    snapshot_id=update_obj.get('previous_md5', 'none'))
+
+                            empty_pages_are_a_change = datastore.data['settings']['application'].get('empty_pages_are_a_change', False)
+                            if update_handler.fetcher.content or (not update_handler.fetcher.content and empty_pages_are_a_change):
+                                watch.save_last_fetched_html(contents=update_handler.fetcher.content, timestamp=int(fetch_start_time))
+
+                            # Send notifications on second+ check
+                            if watch.history_n >= 2:
+                                logger.info(f"Change detected in UUID {uuid} - {watch['url']}")
+                                if not watch.get('notification_muted'):
+                                    await send_content_changed_notification(uuid, notification_q, datastore)
+
+                    except Exception as e:
+                        logger.critical(f"Worker {worker_id} exception in process_changedetection_results")
+                        logger.critical(str(e))
+                        datastore.update_watch(uuid=uuid, update_obj={'last_error': str(e)})
+
+                # Always record attempt count
+                count = watch.get('check_count', 0) + 1
+
+                # Always record page title (used in notifications, and can change even when the content is the same)
+                try:
+                    page_title = html_tools.extract_title(data=update_handler.fetcher.content)
+                    logger.debug(f"UUID: {uuid} Page <title> is '{page_title}'")
+                    datastore.update_watch(uuid=uuid, update_obj={'page_title': page_title})
+                except Exception as e:
+                    logger.warning(f"UUID: {uuid} Exception when extracting <title> - {str(e)}")
+
+                # Record server header
+                try:
+                    server_header = update_handler.fetcher.headers.get('server', '').strip().lower()[:255]
+                    datastore.update_watch(uuid=uuid, update_obj={'remote_server_reply': server_header})
+                except Exception as e:
+                    pass
+
+                # Store favicon if necessary
+                if update_handler.fetcher.favicon_blob and update_handler.fetcher.favicon_blob.get('base64'):
+                    watch.bump_favicon(url=update_handler.fetcher.favicon_blob.get('url'),
+                                       favicon_base_64=update_handler.fetcher.favicon_blob.get('base64')
+                                       )
+
+                datastore.update_watch(uuid=uuid, update_obj={'fetch_time': round(time.time() - fetch_start_time, 3),
+                                                               'check_count': count})
+
+        except Exception as e:
+            logger.error(f"Worker {worker_id} unexpected error processing {uuid}: {e}")
+            logger.error(f"Worker {worker_id} traceback:", exc_info=True)
+            
+            # Also update the watch with error information
+            if datastore and uuid in datastore.data['watching']:
+                datastore.update_watch(uuid=uuid, update_obj={'last_error': f"Worker error: {str(e)}"})
+        
+        finally:
+            # Always cleanup - this runs whether there was an exception or not
+            if uuid:
+                try:
+                    # Mark UUID as no longer being processed
+                    worker_handler.set_uuid_processing(uuid, processing=False)
+                    
+                    # Send completion signal
+                    if watch:
+                        #logger.info(f"Worker {worker_id} sending completion signal for UUID {watch['uuid']}")
+                        watch_check_update.send(watch_uuid=watch['uuid'])
+
+                    update_handler = None
+                    logger.debug(f"Worker {worker_id} completed watch {uuid} in {time.time()-fetch_start_time:.2f}s")
+                except Exception as cleanup_error:
+                    logger.error(f"Worker {worker_id} error during cleanup: {cleanup_error}")
+            
+            # Brief pause before continuing to avoid tight error loops (only on error)
+            if 'e' in locals():
+                await asyncio.sleep(1.0)
+            else:
+                # Small yield for normal completion
+                await asyncio.sleep(0.01)
+
+        # Check if we should exit
+        if app.config.exit.is_set():
+            break
+
+    # Check if we're in pytest environment - if so, be more gentle with logging
+    import sys
+    in_pytest = "pytest" in sys.modules or "PYTEST_CURRENT_TEST" in os.environ
+    
+    if not in_pytest:
+        logger.info(f"Worker {worker_id} shutting down")
+
+
+def cleanup_error_artifacts(uuid, datastore):
+    """Helper function to clean up error artifacts"""
+    cleanup_files = ["last-error-screenshot.png", "last-error.txt"]
+    for f in cleanup_files:
+        full_path = os.path.join(datastore.datastore_path, uuid, f)
+        if os.path.isfile(full_path):
+            os.unlink(full_path)
+
+
+
+async def send_content_changed_notification(watch_uuid, notification_q, datastore):
+    """Helper function to queue notifications using the new notification service"""
+    try:
+        from changedetectionio.notification_service import create_notification_service
+        
+        # Create notification service instance
+        notification_service = create_notification_service(datastore, notification_q)
+        
+        notification_service.send_content_changed_notification(watch_uuid)
+    except Exception as e:
+        logger.error(f"Error sending notification for {watch_uuid}: {e}")
+
+
+async def send_filter_failure_notification(watch_uuid, notification_q, datastore):
+    """Helper function to send filter failure notifications using the new notification service"""
+    try:
+        from changedetectionio.notification_service import create_notification_service
+        
+        # Create notification service instance
+        notification_service = create_notification_service(datastore, notification_q)
+        
+        notification_service.send_filter_failure_notification(watch_uuid)
+    except Exception as e:
+        logger.error(f"Error sending filter failure notification for {watch_uuid}: {e}")
+
+
+async def send_step_failure_notification(watch_uuid, step_n, notification_q, datastore):
+    """Helper function to send step failure notifications using the new notification service"""
+    try:
+        from changedetectionio.notification_service import create_notification_service
+        
+        # Create notification service instance
+        notification_service = create_notification_service(datastore, notification_q)
+        
+        notification_service.send_step_failure_notification(watch_uuid, step_n)
+    except Exception as e:
+        logger.error(f"Error sending step failure notification for {watch_uuid}: {e}")

changedetectionio/auth_decorator.py

@@ -0,0 +1,33 @@
+import os
+from functools import wraps
+from flask import current_app, redirect, request
+from loguru import logger
+
+def login_optionally_required(func):
+    """
+    If password authentication is enabled, verify the user is logged in.
+    To be used as a decorator for routes that should optionally require login.
+    This version is blueprint-friendly as it uses current_app instead of directly accessing app.
+    """
+    @wraps(func)
+    def decorated_view(*args, **kwargs):
+        from flask import current_app
+        import flask_login
+        from flask_login import current_user
+
+        # Access datastore through the app config
+        datastore = current_app.config['DATASTORE']
+        has_password_enabled = datastore.data['settings']['application'].get('password') or os.getenv("SALTED_PASS", False)
+
+        # Permitted
+        if request.endpoint and 'diff_history_page' in request.endpoint and datastore.data['settings']['application'].get('shared_diff_access'):
+            return func(*args, **kwargs)
+        elif request.method in flask_login.config.EXEMPT_METHODS:
+            return func(*args, **kwargs)
+        elif current_app.config.get('LOGIN_DISABLED'):
+            return func(*args, **kwargs)
+        elif has_password_enabled and not current_user.is_authenticated:
+            return current_app.login_manager.unauthorized()
+
+        return func(*args, **kwargs)
+    return decorated_view

changedetectionio/blueprint/backups/__init__.py

@@ -0,0 +1,164 @@
+import datetime
+import glob
+import threading
+
+from flask import Blueprint, render_template, send_from_directory, flash, url_for, redirect, abort
+import os
+
+from changedetectionio.store import ChangeDetectionStore
+from changedetectionio.flask_app import login_optionally_required
+from loguru import logger
+
+BACKUP_FILENAME_FORMAT = "changedetection-backup-{}.zip"
+
+
+def create_backup(datastore_path, watches: dict):
+    logger.debug("Creating backup...")
+    import zipfile
+    from pathlib import Path
+
+    # create a ZipFile object
+    timestamp = datetime.datetime.now().strftime("%Y%m%d%H%M%S")
+    backupname = BACKUP_FILENAME_FORMAT.format(timestamp)
+    backup_filepath = os.path.join(datastore_path, backupname)
+
+    with zipfile.ZipFile(backup_filepath.replace('.zip', '.tmp'), "w",
+                         compression=zipfile.ZIP_DEFLATED,
+                         compresslevel=8) as zipObj:
+
+        # Add the index
+        zipObj.write(os.path.join(datastore_path, "url-watches.json"), arcname="url-watches.json")
+
+        # Add the flask app secret
+        zipObj.write(os.path.join(datastore_path, "secret.txt"), arcname="secret.txt")
+
+        # Add any data in the watch data directory.
+        for uuid, w in watches.items():
+            for f in Path(w.watch_data_dir).glob('*'):
+                zipObj.write(f,
+                             # Use the full path to access the file, but make the file 'relative' in the Zip.
+                             arcname=os.path.join(f.parts[-2], f.parts[-1]),
+                             compress_type=zipfile.ZIP_DEFLATED,
+                             compresslevel=8)
+
+        # Create a list file with just the URLs, so it's easier to port somewhere else in the future
+        list_file = "url-list.txt"
+        with open(os.path.join(datastore_path, list_file), "w") as f:
+            for uuid in watches:
+                url = watches[uuid]["url"]
+                f.write("{}\r\n".format(url))
+        list_with_tags_file = "url-list-with-tags.txt"
+        with open(
+                os.path.join(datastore_path, list_with_tags_file), "w"
+        ) as f:
+            for uuid in watches:
+                url = watches[uuid].get('url')
+                tag = watches[uuid].get('tags', {})
+                f.write("{} {}\r\n".format(url, tag))
+
+        # Add it to the Zip
+        zipObj.write(
+            os.path.join(datastore_path, list_file),
+            arcname=list_file,
+            compress_type=zipfile.ZIP_DEFLATED,
+            compresslevel=8,
+        )
+        zipObj.write(
+            os.path.join(datastore_path, list_with_tags_file),
+            arcname=list_with_tags_file,
+            compress_type=zipfile.ZIP_DEFLATED,
+            compresslevel=8,
+        )
+
+    # Now it's done, rename it so it shows up finally and its completed being written.
+    os.rename(backup_filepath.replace('.zip', '.tmp'), backup_filepath.replace('.tmp', '.zip'))
+
+
+def construct_blueprint(datastore: ChangeDetectionStore):
+    backups_blueprint = Blueprint('backups', __name__, template_folder="templates")
+    backup_threads = []
+
+    @login_optionally_required
+    @backups_blueprint.route("/request-backup", methods=['GET'])
+    def request_backup():
+        if any(thread.is_alive() for thread in backup_threads):
+            flash("A backup is already running, check back in a few minutes", "error")
+            return redirect(url_for('backups.index'))
+
+        if len(find_backups()) > int(os.getenv("MAX_NUMBER_BACKUPS", 100)):
+            flash("Maximum number of backups reached, please remove some", "error")
+            return redirect(url_for('backups.index'))
+
+        # Be sure we're written fresh
+        datastore.sync_to_json()
+        zip_thread = threading.Thread(target=create_backup, args=(datastore.datastore_path, datastore.data.get("watching")))
+        zip_thread.start()
+        backup_threads.append(zip_thread)
+        flash("Backup building in background, check back in a few minutes.")
+
+        return redirect(url_for('backups.index'))
+
+    def find_backups():
+        backup_filepath = os.path.join(datastore.datastore_path, BACKUP_FILENAME_FORMAT.format("*"))
+        backups = glob.glob(backup_filepath)
+        backup_info = []
+
+        for backup in backups:
+            size = os.path.getsize(backup) / (1024 * 1024)
+            creation_time = os.path.getctime(backup)
+            backup_info.append({
+                'filename': os.path.basename(backup),
+                'filesize': f"{size:.2f}",
+                'creation_time': creation_time
+            })
+
+        backup_info.sort(key=lambda x: x['creation_time'], reverse=True)
+
+        return backup_info
+
+    @login_optionally_required
+    @backups_blueprint.route("/download/<string:filename>", methods=['GET'])
+    def download_backup(filename):
+        import re
+        filename = filename.strip()
+        backup_filename_regex = BACKUP_FILENAME_FORMAT.format("\d+")
+
+        full_path = os.path.join(os.path.abspath(datastore.datastore_path), filename)
+        if not full_path.startswith(os.path.abspath(datastore.datastore_path)):
+            abort(404)
+
+        if filename == 'latest':
+            backups = find_backups()
+            filename = backups[0]['filename']
+
+        if not re.match(r"^" + backup_filename_regex + "$", filename):
+            abort(400)  # Bad Request if the filename doesn't match the pattern
+
+        logger.debug(f"Backup download request for '{full_path}'")
+        return send_from_directory(os.path.abspath(datastore.datastore_path), filename, as_attachment=True)
+
+    @login_optionally_required
+    @backups_blueprint.route("", methods=['GET'])
+    def index():
+        backups = find_backups()
+        output = render_template("overview.html",
+                                 available_backups=backups,
+                                 backup_running=any(thread.is_alive() for thread in backup_threads)
+                                 )
+
+        return output
+
+    @login_optionally_required
+    @backups_blueprint.route("/remove-backups", methods=['GET'])
+    def remove_backups():
+
+        backup_filepath = os.path.join(datastore.datastore_path, BACKUP_FILENAME_FORMAT.format("*"))
+        backups = glob.glob(backup_filepath)
+        for backup in backups:
+            os.unlink(backup)
+
+        flash("Backups were deleted.")
+
+        return redirect(url_for('backups.index'))
+
+    return backups_blueprint

changedetectionio/blueprint/backups/templates/overview.html

@@ -0,0 +1,36 @@
+{% extends 'base.html' %}
+{% block content %}
+    {% from '_helpers.html' import render_simple_field, render_field %}
+    <div class="edit-form">
+        <div class="box-wrap inner">
+            <h4>Backups</h4>
+            {% if backup_running %}
+                <p>
+                    <strong>A backup is running!</strong>
+                </p>
+            {% endif %}
+            <p>
+                Here you can download and request a new backup, when a backup is completed you will see it listed below.
+            </p>
+            <br>
+                {% if available_backups %}
+                    <ul>
+                    {% for backup in available_backups %}
+                        <li><a href="{{ url_for('backups.download_backup', filename=backup["filename"]) }}">{{ backup["filename"] }}</a> {{  backup["filesize"] }} Mb</li>
+                    {% endfor %}
+                    </ul>
+                {% else %}
+                    <p>
+                    <strong>No backups found.</strong>
+                    </p>
+                {% endif %}
+
+            <a class="pure-button pure-button-primary" href="{{ url_for('backups.request_backup') }}">Create backup</a>
+            {% if available_backups %}
+                <a class="pure-button button-small button-error " href="{{ url_for('backups.remove_backups') }}">Remove backups</a>
+            {% endif %}
+        </div>
+    </div>
+
+
+{% endblock %}

changedetectionio/blueprint/browser_steps/TODO.txt

@@ -0,0 +1,7 @@
+- This needs an abstraction to directly handle the puppeteer connection methods
+- Then remove the playwright stuff
+- Remove hack redirect at line 65 changedetectionio/processors/__init__.py
+
+The screenshots are base64 encoded/decoded which is very CPU intensive for large screenshots (in playwright) but not
+in the direct puppeteer connection (they are binary end to end)
+

changedetectionio/blueprint/browser_steps/__init__.py

@@ -0,0 +1,254 @@
+
+# HORRIBLE HACK BUT WORKS :-) PR anyone?
+#
+# Why?
+# `browsersteps_playwright_browser_interface.chromium.connect_over_cdp()` will only run once without async()
+# - this flask app is not async()
+# - A single timeout/keepalive which applies to the session made at .connect_over_cdp()
+#
+# So it means that we must unfortunately for now just keep a single timer since .connect_over_cdp() was run
+# and know when that reaches timeout/keepalive :( when that time is up, restart the connection and tell the user
+# that their time is up, insert another coin. (reload)
+#
+#
+
+from changedetectionio.strtobool import strtobool
+from flask import Blueprint, request, make_response
+import os
+
+from changedetectionio.store import ChangeDetectionStore
+from changedetectionio.flask_app import login_optionally_required
+from loguru import logger
+
+browsersteps_sessions = {}
+io_interface_context = None
+import json
+import hashlib
+from flask import Response
+import asyncio
+import threading
+
+def run_async_in_browser_loop(coro):
+    """Run async coroutine using the existing async worker event loop"""
+    from changedetectionio import worker_handler
+    
+    # Use the existing async worker event loop instead of creating a new one
+    if worker_handler.USE_ASYNC_WORKERS and worker_handler.async_loop and not worker_handler.async_loop.is_closed():
+        logger.debug("Browser steps using existing async worker event loop")
+        future = asyncio.run_coroutine_threadsafe(coro, worker_handler.async_loop)
+        return future.result()
+    else:
+        # Fallback: create a new event loop (for sync workers or if async loop not available)
+        logger.debug("Browser steps creating temporary event loop")
+        loop = asyncio.new_event_loop()
+        asyncio.set_event_loop(loop)
+        try:
+            return loop.run_until_complete(coro)
+        finally:
+            loop.close()
+
+def construct_blueprint(datastore: ChangeDetectionStore):
+    browser_steps_blueprint = Blueprint('browser_steps', __name__, template_folder="templates")
+
+    async def start_browsersteps_session(watch_uuid):
+        from . import browser_steps
+        import time
+        from playwright.async_api import async_playwright
+
+        # We keep the playwright session open for many minutes
+        keepalive_seconds = int(os.getenv('BROWSERSTEPS_MINUTES_KEEPALIVE', 10)) * 60
+
+        browsersteps_start_session = {'start_time': time.time()}
+
+        # Create a new async playwright instance for browser steps
+        playwright_instance = async_playwright()
+        playwright_context = await playwright_instance.start()
+
+        keepalive_ms = ((keepalive_seconds + 3) * 1000)
+        base_url = os.getenv('PLAYWRIGHT_DRIVER_URL', '').strip('"')
+        a = "?" if not '?' in base_url else '&'
+        base_url += a + f"timeout={keepalive_ms}"
+
+        browser = await playwright_context.chromium.connect_over_cdp(base_url, timeout=keepalive_ms)
+        browsersteps_start_session['browser'] = browser
+        browsersteps_start_session['playwright_context'] = playwright_context
+
+        proxy_id = datastore.get_preferred_proxy_for_watch(uuid=watch_uuid)
+        proxy = None
+        if proxy_id:
+            proxy_url = datastore.proxy_list.get(proxy_id).get('url')
+            if proxy_url:
+
+                # Playwright needs separate username and password values
+                from urllib.parse import urlparse
+                parsed = urlparse(proxy_url)
+                proxy = {'server': proxy_url}
+
+                if parsed.username:
+                    proxy['username'] = parsed.username
+
+                if parsed.password:
+                    proxy['password'] = parsed.password
+
+                logger.debug(f"Browser Steps: UUID {watch_uuid} selected proxy {proxy_url}")
+
+        # Tell Playwright to connect to Chrome and setup a new session via our stepper interface
+        browserstepper = browser_steps.browsersteps_live_ui(
+            playwright_browser=browser,
+            proxy=proxy,
+            start_url=datastore.data['watching'][watch_uuid].link,
+            headers=datastore.data['watching'][watch_uuid].get('headers')
+        )
+        
+        # Initialize the async connection
+        await browserstepper.connect(proxy=proxy)
+        
+        browsersteps_start_session['browserstepper'] = browserstepper
+
+        # For test
+        #await browsersteps_start_session['browserstepper'].action_goto_url(value="http://example.com?time="+str(time.time()))
+
+        return browsersteps_start_session
+
+
+    @login_optionally_required
+    @browser_steps_blueprint.route("/browsersteps_start_session", methods=['GET'])
+    def browsersteps_start_session():
+        # A new session was requested, return sessionID
+        import asyncio
+        import uuid
+        browsersteps_session_id = str(uuid.uuid4())
+        watch_uuid = request.args.get('uuid')
+
+        if not watch_uuid:
+            return make_response('No Watch UUID specified', 500)
+
+        logger.debug("Starting connection with playwright")
+        logger.debug("browser_steps.py connecting")
+
+        try:
+            # Run the async function in the dedicated browser steps event loop
+            browsersteps_sessions[browsersteps_session_id] = run_async_in_browser_loop(
+                start_browsersteps_session(watch_uuid)
+            )
+        except Exception as e:
+            if 'ECONNREFUSED' in str(e):
+                return make_response('Unable to start the Playwright Browser session, is sockpuppetbrowser running? Network configuration is OK?', 401)
+            else:
+                # Other errors, bad URL syntax, bad reply etc
+                return make_response(str(e), 401)
+
+        logger.debug("Starting connection with playwright - done")
+        return {'browsersteps_session_id': browsersteps_session_id}
+
+    @login_optionally_required
+    @browser_steps_blueprint.route("/browsersteps_image", methods=['GET'])
+    def browser_steps_fetch_screenshot_image():
+        from flask import (
+            make_response,
+            request,
+            send_from_directory,
+        )
+        uuid = request.args.get('uuid')
+        step_n = int(request.args.get('step_n'))
+
+        watch = datastore.data['watching'].get(uuid)
+        filename = f"step_before-{step_n}.jpeg" if request.args.get('type', '') == 'before' else f"step_{step_n}.jpeg"
+
+        if step_n and watch and os.path.isfile(os.path.join(watch.watch_data_dir, filename)):
+            response = make_response(send_from_directory(directory=watch.watch_data_dir, path=filename))
+            response.headers['Content-type'] = 'image/jpeg'
+            response.headers['Cache-Control'] = 'no-cache, no-store, must-revalidate'
+            response.headers['Pragma'] = 'no-cache'
+            response.headers['Expires'] = 0
+            return response
+
+        else:
+            return make_response('Unable to fetch image, is the URL correct? does the watch exist? does the step_type-n.jpeg exist?', 401)
+
+    # A request for an action was received
+    @login_optionally_required
+    @browser_steps_blueprint.route("/browsersteps_update", methods=['POST'])
+    def browsersteps_ui_update():
+        import base64
+        import playwright._impl._errors
+        from changedetectionio.blueprint.browser_steps import browser_steps
+
+        remaining =0
+        uuid = request.args.get('uuid')
+
+        browsersteps_session_id = request.args.get('browsersteps_session_id')
+
+        if not browsersteps_session_id:
+            return make_response('No browsersteps_session_id specified', 500)
+
+        if not browsersteps_sessions.get(browsersteps_session_id):
+            return make_response('No session exists under that ID', 500)
+
+        is_last_step = False
+        # Actions - step/apply/etc, do the thing and return state
+        if request.method == 'POST':
+            # @todo - should always be an existing session
+            step_operation = request.form.get('operation')
+            step_selector = request.form.get('selector')
+            step_optional_value = request.form.get('optional_value')
+            is_last_step = strtobool(request.form.get('is_last_step'))
+
+            try:
+                # Run the async call_action method in the dedicated browser steps event loop
+                run_async_in_browser_loop(
+                    browsersteps_sessions[browsersteps_session_id]['browserstepper'].call_action(
+                        action_name=step_operation,
+                        selector=step_selector,
+                        optional_value=step_optional_value
+                    )
+                )
+
+            except Exception as e:
+                logger.error(f"Exception when calling step operation {step_operation} {str(e)}")
+                # Try to find something of value to give back to the user
+                return make_response(str(e).splitlines()[0], 401)
+
+
+#        if not this_session.page:
+#            cleanup_playwright_session()
+#            return make_response('Browser session ran out of time :( Please reload this page.', 401)
+
+        # Screenshots and other info only needed on requesting a step (POST)
+        try:
+            # Run the async get_current_state method in the dedicated browser steps event loop
+            (screenshot, xpath_data) = run_async_in_browser_loop(
+                browsersteps_sessions[browsersteps_session_id]['browserstepper'].get_current_state()
+            )
+                
+            if is_last_step:
+                watch = datastore.data['watching'].get(uuid)
+                u = browsersteps_sessions[browsersteps_session_id]['browserstepper'].page.url
+                if watch and u:
+                    watch.save_screenshot(screenshot=screenshot)
+                    watch.save_xpath_data(data=xpath_data)
+
+        except Exception as e:
+            return make_response(f"Error fetching screenshot and element data - {str(e)}", 401)
+
+        # SEND THIS BACK TO THE BROWSER
+        output = {
+            "screenshot": f"data:image/jpeg;base64,{base64.b64encode(screenshot).decode('ascii')}",
+            "xpath_data": xpath_data,
+            "session_age_start": browsersteps_sessions[browsersteps_session_id]['browserstepper'].age_start,
+            "browser_time_remaining": round(remaining)
+        }
+        json_data = json.dumps(output)
+
+        # Generate an ETag (hash of the response body)
+        etag_hash = hashlib.md5(json_data.encode('utf-8')).hexdigest()
+
+        # Create the response with ETag
+        response = Response(json_data, mimetype="application/json; charset=UTF-8")
+        response.set_etag(etag_hash)
+
+        return response
+
+    return browser_steps_blueprint
+
+

changedetectionio/blueprint/browser_steps/browser_steps.py

@@ -0,0 +1,497 @@
+import os
+import time
+import re
+from random import randint
+from loguru import logger
+
+from changedetectionio.content_fetchers import SCREENSHOT_MAX_HEIGHT_DEFAULT
+from changedetectionio.content_fetchers.base import manage_user_agent
+from changedetectionio.safe_jinja import render as jinja_render
+
+
+
+# Two flags, tell the JS which of the "Selector" or "Value" field should be enabled in the front end
+# 0- off, 1- on
+browser_step_ui_config = {'Choose one': '0 0',
+                          #                 'Check checkbox': '1 0',
+                          #                 'Click button containing text': '0 1',
+                          #                 'Scroll to bottom': '0 0',
+                          #                 'Scroll to element': '1 0',
+                          #                 'Scroll to top': '0 0',
+                          #                 'Switch to iFrame by index number': '0 1'
+                          #                 'Uncheck checkbox': '1 0',
+                          # @todo
+                          'Check checkbox': '1 0',
+                          'Click X,Y': '0 1',
+                          'Click element if exists': '1 0',
+                          'Click element': '1 0',
+                          'Click element containing text': '0 1',
+                          'Click element containing text if exists': '0 1',
+                          'Enter text in field': '1 1',
+                          'Execute JS': '0 1',
+#                          'Extract text and use as filter': '1 0',
+                          'Goto site': '0 0',
+                          'Goto URL': '0 1',
+                          'Make all child elements visible': '1 0',
+                          'Press Enter': '0 0',
+                          'Select by label': '1 1',
+                          '<select> by option text': '1 1',
+                          'Scroll down': '0 0',
+                          'Uncheck checkbox': '1 0',
+                          'Wait for seconds': '0 1',
+                          'Wait for text': '0 1',
+                          'Wait for text in element': '1 1',
+                          'Remove elements': '1 0',
+                          #                          'Press Page Down': '0 0',
+                          #                          'Press Page Up': '0 0',
+                          # weird bug, come back to it later
+                          }
+
+
+# Good reference - https://playwright.dev/python/docs/input
+#                  https://pythonmana.com/2021/12/202112162236307035.html
+#
+# ONLY Works in Playwright because we need the fullscreen screenshot
+class steppable_browser_interface():
+    page = None
+    start_url = None
+    action_timeout = 10 * 1000
+
+    def __init__(self, start_url):
+        self.start_url = start_url
+
+    # Convert and perform "Click Button" for example
+    async def call_action(self, action_name, selector=None, optional_value=None):
+        if self.page is None:
+            logger.warning("Cannot call action on None page object")
+            return
+            
+        now = time.time()
+        call_action_name = re.sub('[^0-9a-zA-Z]+', '_', action_name.lower())
+        if call_action_name == 'choose_one':
+            return
+
+        logger.debug(f"> Action calling '{call_action_name}'")
+        # https://playwright.dev/python/docs/selectors#xpath-selectors
+        if selector and selector.startswith('/') and not selector.startswith('//'):
+            selector = "xpath=" + selector
+
+        # Check if action handler exists
+        if not hasattr(self, "action_" + call_action_name):
+            logger.warning(f"Action handler for '{call_action_name}' not found")
+            return
+            
+        action_handler = getattr(self, "action_" + call_action_name)
+
+        # Support for Jinja2 variables in the value and selector
+        if selector and ('{%' in selector or '{{' in selector):
+            selector = jinja_render(template_str=selector)
+
+        if optional_value and ('{%' in optional_value or '{{' in optional_value):
+            optional_value = jinja_render(template_str=optional_value)
+
+        # Trigger click and cautiously handle potential navigation
+        # This means the page redirects/reloads/changes JS etc etc
+        if call_action_name.startswith('click_'):
+            try:
+                # Set up navigation expectation before the click (like sync version)
+                async with self.page.expect_event("framenavigated", timeout=3000) as navigation_info:
+                    await action_handler(selector, optional_value)
+                
+                # Check if navigation actually occurred
+                try:
+                    await navigation_info.value  # This waits for the navigation promise
+                    logger.debug(f"Navigation occurred on {call_action_name}.")
+                except Exception:
+                    logger.debug(f"No navigation occurred within timeout when calling {call_action_name}, that's OK, continuing.")
+                    
+            except Exception as e:
+                # If expect_event itself times out, that means no navigation occurred - that's OK
+                if "framenavigated" in str(e) and "exceeded" in str(e):
+                    logger.debug(f"No navigation occurred within timeout when calling {call_action_name}, that's OK, continuing.")
+                else:
+                    raise e
+        else:
+            # Some other action that probably a navigation is not expected
+            await action_handler(selector, optional_value)
+
+
+        # Safely wait for timeout
+        await self.page.wait_for_timeout(1.5 * 1000)
+        logger.debug(f"Call action done in {time.time()-now:.2f}s")
+
+    async def action_goto_url(self, selector=None, value=None):
+        if not value:
+            logger.warning("No URL provided for goto_url action")
+            return None
+            
+        now = time.time()
+        response = await self.page.goto(value, timeout=0, wait_until='load')
+        logger.debug(f"Time to goto URL {time.time()-now:.2f}s")
+        return response
+
+    # Incase they request to go back to the start
+    async def action_goto_site(self, selector=None, value=None):
+        return await self.action_goto_url(value=re.sub(r'^source:', '', self.start_url, flags=re.IGNORECASE))
+
+    async def action_click_element_containing_text(self, selector=None, value=''):
+        logger.debug("Clicking element containing text")
+        if not value or not len(value.strip()):
+            return
+            
+        elem = self.page.get_by_text(value)
+        if await elem.count():
+            await elem.first.click(delay=randint(200, 500), timeout=self.action_timeout)
+
+
+    async def action_click_element_containing_text_if_exists(self, selector=None, value=''):
+        logger.debug("Clicking element containing text if exists")
+        if not value or not len(value.strip()):
+            return
+            
+        elem = self.page.get_by_text(value)
+        count = await elem.count()
+        logger.debug(f"Clicking element containing text - {count} elements found")
+        if count:
+            await elem.first.click(delay=randint(200, 500), timeout=self.action_timeout)
+                
+
+    async def action_enter_text_in_field(self, selector, value):
+        if not selector or not len(selector.strip()):
+            return
+
+        await self.page.fill(selector, value, timeout=self.action_timeout)
+
+    async def action_execute_js(self, selector, value):
+        if not value:
+            return None
+            
+        return await self.page.evaluate(value)
+
+    async def action_click_element(self, selector, value):
+        logger.debug("Clicking element")
+        if not selector or not len(selector.strip()):
+            return
+
+        await self.page.click(selector=selector, timeout=self.action_timeout + 20 * 1000, delay=randint(200, 500))
+
+    async def action_click_element_if_exists(self, selector, value):
+        import playwright._impl._errors as _api_types
+        logger.debug("Clicking element if exists")
+        if not selector or not len(selector.strip()):
+            return
+            
+        try:
+            await self.page.click(selector, timeout=self.action_timeout, delay=randint(200, 500))
+        except _api_types.TimeoutError:
+            return
+        except _api_types.Error:
+            # Element was there, but page redrew and now its long long gone
+            return
+                
+
+    async def action_click_x_y(self, selector, value):
+        if not value or not re.match(r'^\s?\d+\s?,\s?\d+\s?$', value):
+            logger.warning("'Click X,Y' step should be in the format of '100 , 90'")
+            return
+
+        try:
+            x, y = value.strip().split(',')
+            x = int(float(x.strip()))
+            y = int(float(y.strip()))
+            
+            await self.page.mouse.click(x=x, y=y, delay=randint(200, 500))
+                
+        except Exception as e:
+            logger.error(f"Error parsing x,y coordinates: {str(e)}")
+
+    async def action__select_by_option_text(self, selector, value):
+        if not selector or not len(selector.strip()):
+            return
+
+        await self.page.select_option(selector, label=value, timeout=self.action_timeout)
+
+    async def action_scroll_down(self, selector, value):
+        # Some sites this doesnt work on for some reason
+        await self.page.mouse.wheel(0, 600)
+        await self.page.wait_for_timeout(1000)
+
+    async def action_wait_for_seconds(self, selector, value):
+        try:
+            seconds = float(value.strip()) if value else 1.0
+            await self.page.wait_for_timeout(seconds * 1000)
+        except (ValueError, TypeError) as e:
+            logger.error(f"Invalid value for wait_for_seconds: {str(e)}")
+
+    async def action_wait_for_text(self, selector, value):
+        if not value:
+            return
+            
+        import json
+        v = json.dumps(value)
+        await self.page.wait_for_function(
+            f'document.querySelector("body").innerText.includes({v});',
+            timeout=30000
+        )
+            
+
+    async def action_wait_for_text_in_element(self, selector, value):
+        if not selector or not value:
+            return
+            
+        import json
+        s = json.dumps(selector)
+        v = json.dumps(value)
+        
+        await self.page.wait_for_function(
+            f'document.querySelector({s}).innerText.includes({v});',
+            timeout=30000
+        )
+
+    # @todo - in the future make some popout interface to capture what needs to be set
+    # https://playwright.dev/python/docs/api/class-keyboard
+    async def action_press_enter(self, selector, value):
+        await self.page.keyboard.press("Enter", delay=randint(200, 500))
+            
+
+    async def action_press_page_up(self, selector, value):
+        await self.page.keyboard.press("PageUp", delay=randint(200, 500))
+
+    async def action_press_page_down(self, selector, value):
+        await self.page.keyboard.press("PageDown", delay=randint(200, 500))
+
+    async def action_check_checkbox(self, selector, value):
+        if not selector:
+            return
+
+        await self.page.locator(selector).check(timeout=self.action_timeout)
+
+    async def action_uncheck_checkbox(self, selector, value):
+        if not selector:
+            return
+            
+        await self.page.locator(selector).uncheck(timeout=self.action_timeout)
+            
+
+    async def action_remove_elements(self, selector, value):
+        """Removes all elements matching the given selector from the DOM."""
+        if not selector:
+            return
+            
+        await self.page.locator(selector).evaluate_all("els => els.forEach(el => el.remove())")
+
+    async def action_make_all_child_elements_visible(self, selector, value):
+        """Recursively makes all child elements inside the given selector fully visible."""
+        if not selector:
+            return
+            
+        await self.page.locator(selector).locator("*").evaluate_all("""
+            els => els.forEach(el => {
+                el.style.display = 'block';   // Forces it to be displayed
+                el.style.visibility = 'visible';   // Ensures it's not hidden
+                el.style.opacity = '1';   // Fully opaque
+                el.style.position = 'relative';   // Avoids 'absolute' hiding
+                el.style.height = 'auto';   // Expands collapsed elements
+                el.style.width = 'auto';   // Ensures full visibility
+                el.removeAttribute('hidden');   // Removes hidden attribute
+                el.classList.remove('hidden', 'd-none');  // Removes common CSS hidden classes
+            })
+        """)
+
+# Responsible for maintaining a live 'context' with the chrome CDP
+# @todo - how long do contexts live for anyway?
+class browsersteps_live_ui(steppable_browser_interface):
+    context = None
+    page = None
+    render_extra_delay = 1
+    stale = False
+    # bump and kill this if idle after X sec
+    age_start = 0
+    headers = {}
+    # Track if resources are properly cleaned up
+    _is_cleaned_up = False
+    
+    # use a special driver, maybe locally etc
+    command_executor = os.getenv(
+        "PLAYWRIGHT_BROWSERSTEPS_DRIVER_URL"
+    )
+    # if not..
+    if not command_executor:
+        command_executor = os.getenv(
+            "PLAYWRIGHT_DRIVER_URL",
+            'ws://playwright-chrome:3000'
+        ).strip('"')
+
+    browser_type = os.getenv("PLAYWRIGHT_BROWSER_TYPE", 'chromium').strip('"')
+
+    def __init__(self, playwright_browser, proxy=None, headers=None, start_url=None):
+        self.headers = headers or {}
+        self.age_start = time.time()
+        self.playwright_browser = playwright_browser
+        self.start_url = start_url
+        self._is_cleaned_up = False
+        self.proxy = proxy
+        # Note: connect() is now async and must be called separately
+
+    def __del__(self):
+        # Ensure cleanup happens if object is garbage collected
+        # Note: cleanup is now async, so we can only mark as cleaned up here
+        self._is_cleaned_up = True
+
+    # Connect and setup a new context
+    async def connect(self, proxy=None):
+        # Should only get called once - test that
+        keep_open = 1000 * 60 * 5
+        now = time.time()
+
+        # @todo handle multiple contexts, bind a unique id from the browser on each req?
+        self.context = await self.playwright_browser.new_context(
+            accept_downloads=False,  # Should never be needed
+            bypass_csp=True,  # This is needed to enable JavaScript execution on GitHub and others
+            extra_http_headers=self.headers,
+            ignore_https_errors=True,
+            proxy=proxy,
+            service_workers=os.getenv('PLAYWRIGHT_SERVICE_WORKERS', 'allow'),
+            # Should be `allow` or `block` - sites like YouTube can transmit large amounts of data via Service Workers
+            user_agent=manage_user_agent(headers=self.headers),
+        )
+
+        self.page = await self.context.new_page()
+
+        # self.page.set_default_navigation_timeout(keep_open)
+        self.page.set_default_timeout(keep_open)
+        # Set event handlers
+        self.page.on("close", self.mark_as_closed)
+        # Listen for all console events and handle errors
+        self.page.on("console", lambda msg: print(f"Browser steps console - {msg.type}: {msg.text} {msg.args}"))
+
+        logger.debug(f"Time to browser setup {time.time()-now:.2f}s")
+        await self.page.wait_for_timeout(1 * 1000)
+
+    def mark_as_closed(self):
+        logger.debug("Page closed, cleaning up..")
+        # Note: This is called from a sync context (event handler)
+        # so we'll just mark as cleaned up and let __del__ handle the rest
+        self._is_cleaned_up = True
+
+    async def cleanup(self):
+        """Properly clean up all resources to prevent memory leaks"""
+        if self._is_cleaned_up:
+            return
+            
+        logger.debug("Cleaning up browser steps resources")
+        
+        # Clean up page
+        if hasattr(self, 'page') and self.page is not None:
+            try:
+                # Force garbage collection before closing
+                await self.page.request_gc()
+            except Exception as e:
+                logger.debug(f"Error during page garbage collection: {str(e)}")
+                
+            try:
+                # Remove event listeners before closing
+                self.page.remove_listener("close", self.mark_as_closed)
+            except Exception as e:
+                logger.debug(f"Error removing event listeners: {str(e)}")
+                
+            try:
+                await self.page.close()
+            except Exception as e:
+                logger.debug(f"Error closing page: {str(e)}")
+            
+            self.page = None
+
+        # Clean up context
+        if hasattr(self, 'context') and self.context is not None:
+            try:
+                await self.context.close()
+            except Exception as e:
+                logger.debug(f"Error closing context: {str(e)}")
+            
+            self.context = None
+            
+        self._is_cleaned_up = True
+        logger.debug("Browser steps resources cleanup complete")
+
+    @property
+    def has_expired(self):
+        if not self.page or self._is_cleaned_up:
+            return True
+        
+        # Check if session has expired based on age
+        max_age_seconds = int(os.getenv("BROWSER_STEPS_MAX_AGE_SECONDS", 60 * 10))  # Default 10 minutes
+        if (time.time() - self.age_start) > max_age_seconds:
+            logger.debug(f"Browser steps session expired after {max_age_seconds} seconds")
+            return True
+            
+        return False
+
+    async def get_current_state(self):
+        """Return the screenshot and interactive elements mapping, generally always called after action_()"""
+        import importlib.resources
+        import json
+        # because we for now only run browser steps in playwright mode (not puppeteer mode)
+        from changedetectionio.content_fetchers.playwright import capture_full_page_async
+
+        # Safety check - don't proceed if resources are cleaned up
+        if self._is_cleaned_up or self.page is None:
+            logger.warning("Attempted to get current state after cleanup")
+            return (None, None)
+
+        xpath_element_js = importlib.resources.files("changedetectionio.content_fetchers.res").joinpath('xpath_element_scraper.js').read_text()
+
+        now = time.time()
+        await self.page.wait_for_timeout(1 * 1000)
+
+        screenshot = None
+        xpath_data = None
+        
+        try:
+            # Get screenshot first
+            screenshot = await capture_full_page_async(page=self.page)
+            if not screenshot:
+                logger.error("No screenshot was retrieved :((")
+
+            logger.debug(f"Time to get screenshot from browser {time.time() - now:.2f}s")
+
+            # Then get interactive elements
+            now = time.time()
+            await self.page.evaluate("var include_filters=''")
+            await self.page.request_gc()
+
+            scan_elements = 'a,button,input,select,textarea,i,th,td,p,li,h1,h2,h3,h4,div,span'
+
+            MAX_TOTAL_HEIGHT = int(os.getenv("SCREENSHOT_MAX_HEIGHT", SCREENSHOT_MAX_HEIGHT_DEFAULT))
+            xpath_data = json.loads(await self.page.evaluate(xpath_element_js, {
+                "visualselector_xpath_selectors": scan_elements,
+                "max_height": MAX_TOTAL_HEIGHT
+            }))
+            await self.page.request_gc()
+
+            # Sort elements by size
+            xpath_data['size_pos'] = sorted(xpath_data['size_pos'], key=lambda k: k['width'] * k['height'], reverse=True)
+            logger.debug(f"Time to scrape xPath element data in browser {time.time()-now:.2f}s")
+            
+        except Exception as e:
+            logger.error(f"Error getting current state: {str(e)}")
+            # If the page has navigated (common with logins) then the context is destroyed on navigation, continue
+            # I'm not sure that this is required anymore because we have the "expect navigation wrapper" at the top
+            if "Execution context was destroyed" in str(e):
+                logger.debug("Execution context was destroyed, most likely because of navigation, continuing...")
+            pass
+
+            # Attempt recovery - force garbage collection
+            try:
+                await self.page.request_gc()
+            except:
+                pass
+        
+        # Request garbage collection one final time
+        try:
+            await self.page.request_gc()
+        except:
+            pass
+            
+        return (screenshot, xpath_data)
+

changedetectionio/blueprint/check_proxies/__init__.py

@@ -0,0 +1,124 @@
+import importlib
+from concurrent.futures import ThreadPoolExecutor
+
+from changedetectionio.processors.text_json_diff.processor import FilterNotFoundInResponse
+from changedetectionio.store import ChangeDetectionStore
+
+from functools import wraps
+
+from flask import Blueprint
+from flask_login import login_required
+
+STATUS_CHECKING = 0
+STATUS_FAILED = 1
+STATUS_OK = 2
+THREADPOOL_MAX_WORKERS = 3
+_DEFAULT_POOL = ThreadPoolExecutor(max_workers=THREADPOOL_MAX_WORKERS)
+
+
+# Maybe use fetch-time if its >5 to show some expected load time?
+def threadpool(f, executor=None):
+    @wraps(f)
+    def wrap(*args, **kwargs):
+        return (executor or _DEFAULT_POOL).submit(f, *args, **kwargs)
+
+    return wrap
+
+
+def construct_blueprint(datastore: ChangeDetectionStore):
+    check_proxies_blueprint = Blueprint('check_proxies', __name__)
+    checks_in_progress = {}
+
+    @threadpool
+    def long_task(uuid, preferred_proxy):
+        import time
+        from changedetectionio.content_fetchers import exceptions as content_fetcher_exceptions
+        from changedetectionio.safe_jinja import render as jinja_render
+
+        status = {'status': '', 'length': 0, 'text': ''}
+
+        contents = ''
+        now = time.time()
+        try:
+            processor_module = importlib.import_module("changedetectionio.processors.text_json_diff.processor")
+            update_handler = processor_module.perform_site_check(datastore=datastore,
+                                                                 watch_uuid=uuid
+                                                                 )
+
+            update_handler.call_browser(preferred_proxy_id=preferred_proxy)
+        # title, size is len contents not len xfer
+        except content_fetcher_exceptions.Non200ErrorCodeReceived as e:
+            if e.status_code == 404:
+                status.update({'status': 'OK', 'length': len(contents), 'text': f"OK but 404 (page not found)"})
+            elif e.status_code == 403 or e.status_code == 401:
+                status.update({'status': 'ERROR', 'length': len(contents), 'text': f"{e.status_code} - Access denied"})
+            else:
+                status.update({'status': 'ERROR', 'length': len(contents), 'text': f"Status code: {e.status_code}"})
+        except FilterNotFoundInResponse:
+            status.update({'status': 'OK', 'length': len(contents), 'text': f"OK but CSS/xPath filter not found (page changed layout?)"})
+        except content_fetcher_exceptions.EmptyReply as e:
+            if e.status_code == 403 or e.status_code == 401:
+                status.update({'status': 'ERROR OTHER', 'length': len(contents), 'text': f"Got empty reply with code {e.status_code} - Access denied"})
+            else:
+                status.update({'status': 'ERROR OTHER', 'length': len(contents) if contents else 0, 'text': f"Empty reply with code {e.status_code}, needs chrome?"})
+        except content_fetcher_exceptions.ReplyWithContentButNoText as e:
+            txt = f"Got reply but with no content - Status code {e.status_code} - It's possible that the filters were found, but contained no usable text (or contained only an image)."
+            status.update({'status': 'ERROR', 'text': txt})
+        except Exception as e:
+            status.update({'status': 'ERROR OTHER', 'length': len(contents) if contents else 0, 'text': 'Error: '+type(e).__name__+str(e)})
+        else:
+            status.update({'status': 'OK', 'length': len(contents), 'text': ''})
+
+        if status.get('text'):
+            # parse 'text' as text for safety
+            v = {'text': status['text']}
+            status['text'] = jinja_render(template_str='{{text|e}}', **v)
+
+        status['time'] = "{:.2f}s".format(time.time() - now)
+
+        return status
+
+    def _recalc_check_status(uuid):
+
+        results = {}
+        for k, v in checks_in_progress.get(uuid, {}).items():
+            try:
+                r_1 = v.result(timeout=0.05)
+            except Exception as e:
+                # If timeout error?
+                results[k] = {'status': 'RUNNING'}
+
+            else:
+                results[k] = r_1
+
+        return results
+
+    @login_required
+    @check_proxies_blueprint.route("/<string:uuid>/status", methods=['GET'])
+    def get_recheck_status(uuid):
+        results = _recalc_check_status(uuid=uuid)
+        return results
+
+    @login_required
+    @check_proxies_blueprint.route("/<string:uuid>/start", methods=['GET'])
+    def start_check(uuid):
+
+        if not datastore.proxy_list:
+            return
+
+        if checks_in_progress.get(uuid):
+            state = _recalc_check_status(uuid=uuid)
+            for proxy_key, v in state.items():
+                if v.get('status') == 'RUNNING':
+                    return state
+        else:
+            checks_in_progress[uuid] = {}
+
+        for k, v in datastore.proxy_list.items():
+            if not checks_in_progress[uuid].get(k):
+                checks_in_progress[uuid][k] = long_task(uuid=uuid, preferred_proxy=k)
+
+        results = _recalc_check_status(uuid=uuid)
+        return results
+
+    return check_proxies_blueprint

changedetectionio/blueprint/imports/__init__.py

@@ -0,0 +1,75 @@
+from flask import Blueprint, request, redirect, url_for, flash, render_template
+from changedetectionio.store import ChangeDetectionStore
+from changedetectionio.auth_decorator import login_optionally_required
+from changedetectionio import worker_handler
+from changedetectionio.blueprint.imports.importer import (
+    import_url_list, 
+    import_distill_io_json, 
+    import_xlsx_wachete, 
+    import_xlsx_custom
+)
+
+def construct_blueprint(datastore: ChangeDetectionStore, update_q, queuedWatchMetaData):
+    import_blueprint = Blueprint('imports', __name__, template_folder="templates")
+    
+    @import_blueprint.route("/import", methods=['GET', 'POST'])
+    @login_optionally_required
+    def import_page():
+        remaining_urls = []
+        from changedetectionio import forms
+
+        if request.method == 'POST':
+            # URL List import
+            if request.values.get('urls') and len(request.values.get('urls').strip()):
+                # Import and push into the queue for immediate update check
+                importer_handler = import_url_list()
+                importer_handler.run(data=request.values.get('urls'), flash=flash, datastore=datastore, processor=request.values.get('processor', 'text_json_diff'))
+                for uuid in importer_handler.new_uuids:
+                    worker_handler.queue_item_async_safe(update_q, queuedWatchMetaData.PrioritizedItem(priority=1, item={'uuid': uuid}))
+
+                if len(importer_handler.remaining_data) == 0:
+                    return redirect(url_for('watchlist.index'))
+                else:
+                    remaining_urls = importer_handler.remaining_data
+
+            # Distill.io import
+            if request.values.get('distill-io') and len(request.values.get('distill-io').strip()):
+                # Import and push into the queue for immediate update check
+                d_importer = import_distill_io_json()
+                d_importer.run(data=request.values.get('distill-io'), flash=flash, datastore=datastore)
+                for uuid in d_importer.new_uuids:
+                    worker_handler.queue_item_async_safe(update_q, queuedWatchMetaData.PrioritizedItem(priority=1, item={'uuid': uuid}))
+
+            # XLSX importer
+            if request.files and request.files.get('xlsx_file'):
+                file = request.files['xlsx_file']
+
+                if request.values.get('file_mapping') == 'wachete':
+                    w_importer = import_xlsx_wachete()
+                    w_importer.run(data=file, flash=flash, datastore=datastore)
+                else:
+                    w_importer = import_xlsx_custom()
+                    # Building mapping of col # to col # type
+                    map = {}
+                    for i in range(10):
+                        c = request.values.get(f"custom_xlsx[col_{i}]")
+                        v = request.values.get(f"custom_xlsx[col_type_{i}]")
+                        if c and v:
+                            map[int(c)] = v
+
+                    w_importer.import_profile = map
+                    w_importer.run(data=file, flash=flash, datastore=datastore)
+
+                for uuid in w_importer.new_uuids:
+                    worker_handler.queue_item_async_safe(update_q, queuedWatchMetaData.PrioritizedItem(priority=1, item={'uuid': uuid}))
+
+        # Could be some remaining, or we could be on GET
+        form = forms.importForm(formdata=request.form if request.method == 'POST' else None)
+        output = render_template("import.html",
+                                form=form,
+                                import_url_list_remaining="\n".join(remaining_urls),
+                                original_distill_json=''
+                                )
+        return output
+
+    return import_blueprint

changedetectionio/blueprint/imports/importer.py

@@ -0,0 +1,302 @@
+from abc import abstractmethod
+import time
+from wtforms import ValidationError
+from loguru import logger
+
+from changedetectionio.forms import validate_url
+
+
+class Importer():
+    remaining_data = []
+    new_uuids = []
+    good = 0
+
+    def __init__(self):
+        self.new_uuids = []
+        self.good = 0
+        self.remaining_data = []
+        self.import_profile = None
+
+    @abstractmethod
+    def run(self,
+            data,
+            flash,
+            datastore):
+        pass
+
+
+class import_url_list(Importer):
+    """
+    Imports a list, can be in <code>https://example.com tag1, tag2, last tag</code> format
+    """
+    def run(self,
+            data,
+            flash,
+            datastore,
+            processor=None
+            ):
+
+        urls = data.split("\n")
+        good = 0
+        now = time.time()
+
+        if (len(urls) > 5000):
+            flash("Importing 5,000 of the first URLs from your list, the rest can be imported again.")
+
+        for url in urls:
+            url = url.strip()
+            if not len(url):
+                continue
+
+            tags = ""
+
+            # 'tags' should be a csv list after the URL
+            if ' ' in url:
+                url, tags = url.split(" ", 1)
+
+            # Flask wtform validators wont work with basic auth, use validators package
+            # Up to 5000 per batch so we dont flood the server
+            # @todo validators.url will fail when you add your own IP etc
+            if len(url) and 'http' in url.lower() and good < 5000:
+                extras = None
+                if processor:
+                    extras = {'processor': processor}
+                new_uuid = datastore.add_watch(url=url.strip(), tag=tags, write_to_disk_now=False, extras=extras)
+
+                if new_uuid:
+                    # Straight into the queue.
+                    self.new_uuids.append(new_uuid)
+                    good += 1
+                    continue
+
+            # Worked past the 'continue' above, append it to the bad list
+            if self.remaining_data is None:
+                self.remaining_data = []
+            self.remaining_data.append(url)
+
+        flash("{} Imported from list in {:.2f}s, {} Skipped.".format(good, time.time() - now, len(self.remaining_data)))
+
+
+class import_distill_io_json(Importer):
+    def run(self,
+            data,
+            flash,
+            datastore,
+            ):
+
+        import json
+        good = 0
+        now = time.time()
+        self.new_uuids=[]
+
+        # @todo Use JSONSchema like in the API to validate here.
+        
+        try:
+            data = json.loads(data.strip())
+        except json.decoder.JSONDecodeError:
+            flash("Unable to read JSON file, was it broken?", 'error')
+            return
+
+        if not data.get('data'):
+            flash("JSON structure looks invalid, was it broken?", 'error')
+            return
+
+        for d in data.get('data'):
+            d_config = json.loads(d['config'])
+            extras = {'title': d.get('name', None)}
+
+            if len(d['uri']) and good < 5000:
+                try:
+                    # @todo we only support CSS ones at the moment
+                    if d_config['selections'][0]['frames'][0]['excludes'][0]['type'] == 'css':
+                        extras['subtractive_selectors'] = d_config['selections'][0]['frames'][0]['excludes'][0]['expr']
+                except KeyError:
+                    pass
+                except IndexError:
+                    pass
+                extras['include_filters'] = []
+                try:
+                    if d_config['selections'][0]['frames'][0]['includes'][0]['type'] == 'xpath':
+                        extras['include_filters'].append('xpath:' + d_config['selections'][0]['frames'][0]['includes'][0]['expr'])
+                    else:
+                        extras['include_filters'].append(d_config['selections'][0]['frames'][0]['includes'][0]['expr'])
+                except KeyError:
+                    pass
+                except IndexError:
+                    pass
+
+                new_uuid = datastore.add_watch(url=d['uri'].strip(),
+                                               tag=",".join(d.get('tags', [])),
+                                               extras=extras,
+                                               write_to_disk_now=False)
+
+                if new_uuid:
+                    # Straight into the queue.
+                    self.new_uuids.append(new_uuid)
+                    good += 1
+
+        flash("{} Imported from Distill.io in {:.2f}s, {} Skipped.".format(len(self.new_uuids), time.time() - now, len(self.remaining_data)))
+
+
+class import_xlsx_wachete(Importer):
+
+    def run(self,
+            data,
+            flash,
+            datastore,
+            ):
+
+        good = 0
+        now = time.time()
+        self.new_uuids = []
+
+        from openpyxl import load_workbook
+
+        try:
+            wb = load_workbook(data)
+        except Exception as e:
+            # @todo correct except
+            flash("Unable to read export XLSX file, something wrong with the file?", 'error')
+            return
+
+        row_id = 2
+        for row in wb.active.iter_rows(min_row=row_id):
+            try:
+                extras = {}
+                data = {}
+                for cell in row:
+                    if not cell.value:
+                        continue
+                    column_title = wb.active.cell(row=1, column=cell.column).value.strip().lower()
+                    data[column_title] = cell.value
+
+                # Forced switch to webdriver/playwright/etc
+                dynamic_wachet = str(data.get('dynamic wachet', '')).strip().lower()  # Convert bool to str to cover all cases
+                # libreoffice and others can have it as =FALSE() =TRUE(), or bool(true)
+                if 'true' in dynamic_wachet or dynamic_wachet == '1':
+                    extras['fetch_backend'] = 'html_webdriver'
+                elif 'false' in dynamic_wachet or dynamic_wachet == '0':
+                    extras['fetch_backend'] = 'html_requests'
+
+                if data.get('xpath'):
+                    # @todo split by || ?
+                    extras['include_filters'] = [data.get('xpath')]
+                if data.get('name'):
+                    extras['title'] = data.get('name').strip()
+                if data.get('interval (min)'):
+                    minutes = int(data.get('interval (min)'))
+                    hours, minutes = divmod(minutes, 60)
+                    days, hours = divmod(hours, 24)
+                    weeks, days = divmod(days, 7)
+                    extras['time_between_check'] = {'weeks': weeks, 'days': days, 'hours': hours, 'minutes': minutes, 'seconds': 0}
+
+                # At minimum a URL is required.
+                if data.get('url'):
+                    try:
+                        validate_url(data.get('url'))
+                    except ValidationError as e:
+                        logger.error(f">> Import URL error {data.get('url')} {str(e)}")
+                        flash(f"Error processing row number {row_id}, URL value was incorrect, row was skipped.", 'error')
+                        # Don't bother processing anything else on this row
+                        continue
+
+                    new_uuid = datastore.add_watch(url=data['url'].strip(),
+                                                   extras=extras,
+                                                   tag=data.get('folder'),
+                                                   write_to_disk_now=False)
+                    if new_uuid:
+                        # Straight into the queue.
+                        self.new_uuids.append(new_uuid)
+                        good += 1
+            except Exception as e:
+                logger.error(e)
+                flash(f"Error processing row number {row_id}, check all cell data types are correct, row was skipped.", 'error')
+            else:
+                row_id += 1
+
+        flash(
+            "{} imported from Wachete .xlsx in {:.2f}s".format(len(self.new_uuids), time.time() - now))
+
+
+class import_xlsx_custom(Importer):
+
+    def run(self,
+            data,
+            flash,
+            datastore,
+            ):
+
+        good = 0
+        now = time.time()
+        self.new_uuids = []
+
+        from openpyxl import load_workbook
+
+        try:
+            wb = load_workbook(data)
+        except Exception as e:
+            # @todo correct except
+            flash("Unable to read export XLSX file, something wrong with the file?", 'error')
+            return
+
+        # @todo cehck atleast 2 rows, same in other method
+        from changedetectionio.forms import validate_url
+        row_i = 1
+
+        try:
+            for row in wb.active.iter_rows():
+                url = None
+                tags = None
+                extras = {}
+
+                for cell in row:
+                    if not self.import_profile.get(cell.col_idx):
+                        continue
+                    if not cell.value:
+                        continue
+
+                    cell_map = self.import_profile.get(cell.col_idx)
+
+                    cell_val = str(cell.value).strip()  # could be bool
+
+                    if cell_map == 'url':
+                        url = cell.value.strip()
+                        try:
+                            validate_url(url)
+                        except ValidationError as e:
+                            logger.error(f">> Import URL error {url} {str(e)}")
+                            flash(f"Error processing row number {row_i}, URL value was incorrect, row was skipped.", 'error')
+                            # Don't bother processing anything else on this row
+                            url = None
+                            break
+                    elif cell_map == 'tag':
+                        tags = cell.value.strip()
+                    elif cell_map == 'include_filters':
+                        # @todo validate?
+                        extras['include_filters'] = [cell.value.strip()]
+                    elif cell_map == 'interval_minutes':
+                        hours, minutes = divmod(int(cell_val), 60)
+                        days, hours = divmod(hours, 24)
+                        weeks, days = divmod(days, 7)
+                        extras['time_between_check'] = {'weeks': weeks, 'days': days, 'hours': hours, 'minutes': minutes, 'seconds': 0}
+                    else:
+                        extras[cell_map] = cell_val
+
+                # At minimum a URL is required.
+                if url:
+                    new_uuid = datastore.add_watch(url=url,
+                                                   extras=extras,
+                                                   tag=tags,
+                                                   write_to_disk_now=False)
+                    if new_uuid:
+                        # Straight into the queue.
+                        self.new_uuids.append(new_uuid)
+                        good += 1
+        except Exception as e:
+            logger.error(e)
+            flash(f"Error processing row number {row_i}, check all cell data types are correct, row was skipped.", 'error')
+        else:
+            row_i += 1
+
+        flash(
+            "{} imported from custom .xlsx in {:.2f}s".format(len(self.new_uuids), time.time() - now))

changedetectionio/blueprint/imports/templates/import.html

@@ -1,54 +1,55 @@
 {% extends 'base.html' %}
 {% block content %}
-<script type="text/javascript" src="{{url_for('static_content', group='js', filename='tabs.js')}}" defer></script>
+{% from '_helpers.html' import render_field %}
+<script src="{{url_for('static_content', group='js', filename='tabs.js')}}" defer></script>
 <div class="edit-form monospaced-textarea">
 
     <div class="tabs collapsable">
         <ul>
-            <li class="tab" id="default-tab"><a href="#url-list">URL List</a></li>
+            <li class="tab" id=""><a href="#url-list">URL List</a></li>
             <li class="tab"><a href="#distill-io">Distill.io</a></li>
+            <li class="tab"><a href="#xlsx">.XLSX &amp; Wachete</a></li>
         </ul>
     </div>
 
     <div class="box-wrap inner">
-        <form class="pure-form pure-form-aligned" action="{{url_for('import_page')}}" method="POST">
-            <input type="hidden" name="csrf_token" value="{{ csrf_token() }}"/>
+        <form class="pure-form" action="{{url_for('imports.import_page')}}" method="POST" enctype="multipart/form-data">
+            <input type="hidden" name="csrf_token" value="{{ csrf_token() }}">
             <div class="tab-pane-inner" id="url-list">
-                <fieldset class="pure-group">
-                    <legend>
+                <div class="pure-control-group">
                         Enter one URL per line, and optionally add tags for each URL after a space, delineated by comma
                         (,):
                         <br>
-                        <code>https://example.com tag1, tag2, last tag</code>
-                        <br>
+                        <p><strong>Example:  </strong><code>https://example.com tag1, tag2, last tag</code></p>
                         URLs which do not pass validation will stay in the textarea.
-                    </legend>
-
-
+                </div>
+                {{ render_field(form.processor, class="processor") }}
+                
+                <div class="pure-control-group">
                     <textarea name="urls" class="pure-input-1-2" placeholder="https://"
                               style="width: 100%;
                                 font-family:monospace;
                                 white-space: pre;
                                 overflow-wrap: normal;
                                 overflow-x: scroll;" rows="25">{{ import_url_list_remaining }}</textarea>
-                </fieldset>
-
+                 </div>
+                 <div id="quick-watch-processor-type"></div>
 
             </div>
 
             <div class="tab-pane-inner" id="distill-io">
 
 
-                <fieldset class="pure-group">
-                    <legend>
-                        Copy and Paste your Distill.io watch 'export' file, this should be a JSON file.</br>
+
+                    <div class="pure-control-group">
+                        Copy and Paste your Distill.io watch 'export' file, this should be a JSON file.<br>
                         This is <i>experimental</i>, supported fields are <code>name</code>, <code>uri</code>, <code>tags</code>, <code>config:selections</code>, the rest (including <code>schedule</code>) are ignored.
-                        <br/>
+                        <br>
                         <p>
-                        How to export? <a href="https://distill.io/docs/web-monitor/how-export-and-import-monitors/">https://distill.io/docs/web-monitor/how-export-and-import-monitors/</a><br/>
-                        Be sure to set your default fetcher to Chrome if required.</br>
+                        How to export? <a href="https://distill.io/docs/web-monitor/how-export-and-import-monitors/">https://distill.io/docs/web-monitor/how-export-and-import-monitors/</a><br>
+                        Be sure to set your default fetcher to Chrome if required.<br>
                         </p>
-                    </legend>
+                    </div>
 
 
                     <textarea name="distill-io" class="pure-input-1-2" style="width: 100%;
@@ -75,7 +76,43 @@
     ]
 }
 " rows="25">{{ original_distill_json }}</textarea>
-                </fieldset>
+
+            </div>
+            <div class="tab-pane-inner" id="xlsx">
+            <fieldset>
+                <div class="pure-control-group">
+                {{ render_field(form.xlsx_file, class="processor") }}
+                </div>
+                <div class="pure-control-group">
+                    {{ render_field(form.file_mapping, class="processor") }}
+                </div>
+            </fieldset>
+                <div class="pure-control-group">
+                <span class="pure-form-message-inline">
+                    Table of custom column and data types mapping for the <strong>Custom mapping</strong> File mapping type.
+                </span>
+                    <table style="border: 1px solid #aaa; padding: 0.5rem; border-radius: 4px;">
+                        <tr>
+                            <td><strong>Column #</strong></td>
+                            {% for n in range(4) %}
+                                <td><input type="number" name="custom_xlsx[col_{{n}}]" style="width: 4rem;" min="1"></td>
+                            {%  endfor %}
+                        </tr>
+                        <tr>
+                            <td><strong>Type</strong></td>
+                            {% for n in range(4) %}
+                                <td><select name="custom_xlsx[col_type_{{n}}]">
+                                    <option value="" style="color: #aaa"> -- none --</option>
+                                    <option value="url">URL</option>
+                                    <option value="title">Title</option>
+                                    <option value="include_filters">CSS/xPath filter</option>
+                                    <option value="tag">Group / Tag name(s)</option>
+                                    <option value="interval_minutes">Recheck time (minutes)</option>
+                                </select></td>
+                            {%  endfor %}
+                        </tr>
+                    </table>
+                </div>
             </div>
             <button type="submit" class="pure-button pure-input-1-2 pure-button-primary">Import</button>
         </form>
@@ -83,4 +120,4 @@
     </div>
 </div>
 
-{% endblock %}
+{% endblock %}

changedetectionio/blueprint/price_data_follower/__init__.py

@@ -0,0 +1,35 @@
+
+from changedetectionio.strtobool import strtobool
+from flask import Blueprint, flash, redirect, url_for
+from flask_login import login_required
+from changedetectionio.store import ChangeDetectionStore
+from changedetectionio import queuedWatchMetaData
+from changedetectionio import worker_handler
+from queue import PriorityQueue
+
+PRICE_DATA_TRACK_ACCEPT = 'accepted'
+PRICE_DATA_TRACK_REJECT = 'rejected'
+
+def construct_blueprint(datastore: ChangeDetectionStore, update_q: PriorityQueue):
+
+    price_data_follower_blueprint = Blueprint('price_data_follower', __name__)
+
+    @login_required
+    @price_data_follower_blueprint.route("/<string:uuid>/accept", methods=['GET'])
+    def accept(uuid):
+        datastore.data['watching'][uuid]['track_ldjson_price_data'] = PRICE_DATA_TRACK_ACCEPT
+        datastore.data['watching'][uuid]['processor'] = 'restock_diff'
+        datastore.data['watching'][uuid].clear_watch()
+        worker_handler.queue_item_async_safe(update_q, queuedWatchMetaData.PrioritizedItem(priority=1, item={'uuid': uuid}))
+        return redirect(url_for("watchlist.index"))
+
+    @login_required
+    @price_data_follower_blueprint.route("/<string:uuid>/reject", methods=['GET'])
+    def reject(uuid):
+        datastore.data['watching'][uuid]['track_ldjson_price_data'] = PRICE_DATA_TRACK_REJECT
+        return redirect(url_for("watchlist.index"))
+
+
+    return price_data_follower_blueprint
+
+

changedetectionio/blueprint/rss/__init__.py

@@ -0,0 +1 @@
+RSS_FORMAT_TYPES = [('plaintext', 'Plain text'), ('html', 'HTML Color')]

changedetectionio/blueprint/rss/blueprint.py

@@ -0,0 +1,150 @@
+
+from changedetectionio.safe_jinja import render as jinja_render
+from changedetectionio.store import ChangeDetectionStore
+from feedgen.feed import FeedGenerator
+from flask import Blueprint, make_response, request, url_for, redirect
+from loguru import logger
+import datetime
+import pytz
+import re
+import time
+
+
+BAD_CHARS_REGEX=r'[\x00-\x08\x0B\x0C\x0E-\x1F]'
+
+# Anything that is not text/UTF-8 should be stripped before it breaks feedgen (such as binary data etc)
+def scan_invalid_chars_in_rss(content):
+    for match in re.finditer(BAD_CHARS_REGEX, content):
+        i = match.start()
+        bad_char = content[i]
+        hex_value = f"0x{ord(bad_char):02x}"
+        # Grab context
+        start = max(0, i - 20)
+        end = min(len(content), i + 21)
+        context = content[start:end].replace('\n', '\\n').replace('\r', '\\r')
+        logger.warning(f"Invalid char {hex_value} at pos {i}: ...{context}...")
+        # First match is enough
+        return True
+
+    return False
+
+
+def clean_entry_content(content):
+    cleaned = re.sub(BAD_CHARS_REGEX, '', content)
+    return cleaned
+
+def construct_blueprint(datastore: ChangeDetectionStore):
+    rss_blueprint = Blueprint('rss', __name__)
+
+    # Some RSS reader situations ended up with rss/ (forward slash after RSS) due
+    # to some earlier blueprint rerouting work, it should goto feed.
+    @rss_blueprint.route("/", methods=['GET'])
+    def extraslash():
+        return redirect(url_for('rss.feed'))
+
+    # Import the login decorator if needed
+    # from changedetectionio.auth_decorator import login_optionally_required
+    @rss_blueprint.route("", methods=['GET'])
+    def feed():
+        now = time.time()
+        # Always requires token set
+        app_rss_token = datastore.data['settings']['application'].get('rss_access_token')
+        rss_url_token = request.args.get('token')
+        if rss_url_token != app_rss_token:
+            return "Access denied, bad token", 403
+
+        from changedetectionio import diff
+        limit_tag = request.args.get('tag', '').lower().strip()
+        # Be sure limit_tag is a uuid
+        for uuid, tag in datastore.data['settings']['application'].get('tags', {}).items():
+            if limit_tag == tag.get('title', '').lower().strip():
+                limit_tag = uuid
+
+        # Sort by last_changed and add the uuid which is usually the key..
+        sorted_watches = []
+
+        # @todo needs a .itemsWithTag() or something - then we can use that in Jinaj2 and throw this away
+        for uuid, watch in datastore.data['watching'].items():
+            # @todo tag notification_muted skip also (improve Watch model)
+            if datastore.data['settings']['application'].get('rss_hide_muted_watches') and watch.get('notification_muted'):
+                continue
+            if limit_tag and not limit_tag in watch['tags']:
+                continue
+            watch['uuid'] = uuid
+            sorted_watches.append(watch)
+
+        sorted_watches.sort(key=lambda x: x.last_changed, reverse=False)
+
+        fg = FeedGenerator()
+        fg.title('changedetection.io')
+        fg.description('Feed description')
+        fg.link(href='https://changedetection.io')
+
+        html_colour_enable = False
+        if datastore.data['settings']['application'].get('rss_content_format') == 'html':
+            html_colour_enable = True
+
+        for watch in sorted_watches:
+
+            dates = list(watch.history.keys())
+            # Re #521 - Don't bother processing this one if theres less than 2 snapshots, means we never had a change detected.
+            if len(dates) < 2:
+                continue
+
+            if not watch.viewed:
+                # Re #239 - GUID needs to be individual for each event
+                # @todo In the future make this a configurable link back (see work on BASE_URL https://github.com/dgtlmoon/changedetection.io/pull/228)
+                guid = "{}/{}".format(watch['uuid'], watch.last_changed)
+                fe = fg.add_entry()
+
+                # Include a link to the diff page, they will have to login here to see if password protection is enabled.
+                # Description is the page you watch, link takes you to the diff JS UI page
+                # Dict val base_url will get overriden with the env var if it is set.
+                ext_base_url = datastore.data['settings']['application'].get('active_base_url')
+                # @todo fix
+
+                # Because we are called via whatever web server, flask should figure out the right path (
+                diff_link = {'href': url_for('ui.ui_views.diff_history_page', uuid=watch['uuid'], _external=True)}
+
+                fe.link(link=diff_link)
+
+                # Same logic as watch-overview.html
+                if datastore.data['settings']['application']['ui'].get('use_page_title_in_list') or watch.get('use_page_title_in_list'):
+                    watch_label = watch.label
+                else:
+                    watch_label = watch.get('url')
+
+                fe.title(title=watch_label)
+                try:
+
+                    html_diff = diff.render_diff(previous_version_file_contents=watch.get_history_snapshot(dates[-2]),
+                                                 newest_version_file_contents=watch.get_history_snapshot(dates[-1]),
+                                                 include_equal=False,
+                                                 line_feed_sep="<br>",
+                                                 html_colour=html_colour_enable
+                                                 )
+                except FileNotFoundError as e:
+                    html_diff = f"History snapshot file for watch {watch.get('uuid')}@{watch.last_changed} - '{watch.get('title')} not found."
+
+                # @todo Make this configurable and also consider html-colored markup
+                # @todo User could decide if <link> goes to the diff page, or to the watch link
+                rss_template = "<html><body>\n<h4><a href=\"{{watch_url}}\">{{watch_title}}</a></h4>\n<p>{{html_diff}}</p>\n</body></html>\n"
+
+                content = jinja_render(template_str=rss_template, watch_title=watch_label, html_diff=html_diff, watch_url=watch.link)
+
+                # Out of range chars could also break feedgen
+                if scan_invalid_chars_in_rss(content):
+                    content = clean_entry_content(content)
+
+                fe.content(content=content, type='CDATA')
+                fe.guid(guid, permalink=False)
+                dt = datetime.datetime.fromtimestamp(int(watch.newest_history_key))
+                dt = dt.replace(tzinfo=pytz.UTC)
+                fe.pubDate(dt)
+
+        response = make_response(fg.rss_str())
+        response.headers.set('Content-Type', 'application/rss+xml;charset=utf-8')
+        logger.trace(f"RSS generated in {time.time() - now:.3f}s")
+        return response
+
+    return rss_blueprint

changedetectionio/blueprint/settings/__init__.py

@@ -0,0 +1,145 @@
+import os
+from copy import deepcopy
+from datetime import datetime
+from zoneinfo import ZoneInfo, available_timezones
+import secrets
+import flask_login
+from flask import Blueprint, render_template, request, redirect, url_for, flash
+
+from changedetectionio.store import ChangeDetectionStore
+from changedetectionio.auth_decorator import login_optionally_required
+
+
+def construct_blueprint(datastore: ChangeDetectionStore):
+    settings_blueprint = Blueprint('settings', __name__, template_folder="templates")
+
+    @settings_blueprint.route("", methods=['GET', "POST"])
+    @login_optionally_required
+    def settings_page():
+        from changedetectionio import forms
+
+        default = deepcopy(datastore.data['settings'])
+        if datastore.proxy_list is not None:
+            available_proxies = list(datastore.proxy_list.keys())
+            # When enabled
+            system_proxy = datastore.data['settings']['requests']['proxy']
+            # In the case it doesnt exist anymore
+            if not system_proxy in available_proxies:
+                system_proxy = None
+
+            default['requests']['proxy'] = system_proxy if system_proxy is not None else available_proxies[0]
+            # Used by the form handler to keep or remove the proxy settings
+            default['proxy_list'] = available_proxies[0]
+
+        # Don't use form.data on POST so that it doesnt overrid the checkbox status from the POST status
+        form = forms.globalSettingsForm(formdata=request.form if request.method == 'POST' else None,
+                                        data=default,
+                                        extra_notification_tokens=datastore.get_unique_notification_tokens_available()
+                                        )
+
+        # Remove the last option 'System default'
+        form.application.form.notification_format.choices.pop()
+
+        if datastore.proxy_list is None:
+            # @todo - Couldn't get setattr() etc dynamic addition working, so remove it instead
+            del form.requests.form.proxy
+        else:
+            form.requests.form.proxy.choices = []
+            for p in datastore.proxy_list:
+                form.requests.form.proxy.choices.append(tuple((p, datastore.proxy_list[p]['label'])))
+
+        if request.method == 'POST':
+            # Password unset is a GET, but we can lock the session to a salted env password to always need the password
+            if form.application.form.data.get('removepassword_button', False):
+                # SALTED_PASS means the password is "locked" to what we set in the Env var
+                if not os.getenv("SALTED_PASS", False):
+                    datastore.remove_password()
+                    flash("Password protection removed.", 'notice')
+                    flask_login.logout_user()
+                    return redirect(url_for('settings.settings_page'))
+
+            if form.validate():
+                # Don't set password to False when a password is set - should be only removed with the `removepassword` button
+                app_update = dict(deepcopy(form.data['application']))
+
+                # Never update password with '' or False (Added by wtforms when not in submission)
+                if 'password' in app_update and not app_update['password']:
+                    del (app_update['password'])
+
+                datastore.data['settings']['application'].update(app_update)
+                
+                # Handle dynamic worker count adjustment
+                old_worker_count = datastore.data['settings']['requests'].get('workers', 1)
+                new_worker_count = form.data['requests'].get('workers', 1)
+                
+                datastore.data['settings']['requests'].update(form.data['requests'])
+                
+                # Adjust worker count if it changed
+                if new_worker_count != old_worker_count:
+                    from changedetectionio import worker_handler
+                    from changedetectionio.flask_app import update_q, notification_q, app, datastore as ds
+                    
+                    result = worker_handler.adjust_async_worker_count(
+                        new_count=new_worker_count,
+                        update_q=update_q,
+                        notification_q=notification_q,
+                        app=app,
+                        datastore=ds
+                    )
+                    
+                    if result['status'] == 'success':
+                        flash(f"Worker count adjusted: {result['message']}", 'notice')
+                    elif result['status'] == 'not_supported':
+                        flash("Dynamic worker adjustment not supported for sync workers", 'warning')
+                    elif result['status'] == 'error':
+                        flash(f"Error adjusting workers: {result['message']}", 'error')
+
+                if not os.getenv("SALTED_PASS", False) and len(form.application.form.password.encrypted_password):
+                    datastore.data['settings']['application']['password'] = form.application.form.password.encrypted_password
+                    datastore.needs_write_urgent = True
+                    flash("Password protection enabled.", 'notice')
+                    flask_login.logout_user()
+                    return redirect(url_for('watchlist.index'))
+
+                datastore.needs_write_urgent = True
+                flash("Settings updated.")
+
+            else:
+                flash("An error occurred, please see below.", "error")
+
+        # Convert to ISO 8601 format, all date/time relative events stored as UTC time
+        utc_time = datetime.now(ZoneInfo("UTC")).isoformat()
+
+        output = render_template("settings.html",
+                                api_key=datastore.data['settings']['application'].get('api_access_token'),
+                                available_timezones=sorted(available_timezones()),
+                                emailprefix=os.getenv('NOTIFICATION_MAIL_BUTTON_PREFIX', False),
+                                extra_notification_token_placeholder_info=datastore.get_unique_notification_token_placeholders_available(),
+                                form=form,
+                                hide_remove_pass=os.getenv("SALTED_PASS", False),
+                                min_system_recheck_seconds=int(os.getenv('MINIMUM_SECONDS_RECHECK_TIME', 3)),
+                                settings_application=datastore.data['settings']['application'],
+                                timezone_default_config=datastore.data['settings']['application'].get('timezone'),
+                                utc_time=utc_time,
+                                )
+
+        return output
+
+    @settings_blueprint.route("/reset-api-key", methods=['GET'])
+    @login_optionally_required
+    def settings_reset_api_key():
+        secret = secrets.token_hex(16)
+        datastore.data['settings']['application']['api_access_token'] = secret
+        datastore.needs_write_urgent = True
+        flash("API Key was regenerated.")
+        return redirect(url_for('settings.settings_page')+'#api')
+        
+    @settings_blueprint.route("/notification-logs", methods=['GET'])
+    @login_optionally_required
+    def notification_logs():
+        from changedetectionio.flask_app import notification_debug_log
+        output = render_template("notification-log.html",
+                               logs=notification_debug_log if len(notification_debug_log) else ["Notification logs are empty - no notifications sent yet."])
+        return output
+
+    return settings_blueprint

changedetectionio/blueprint/settings/templates/settings.html

@@ -0,0 +1,334 @@
+{% extends 'base.html' %}
+
+{% block content %}
+{% from '_helpers.html' import render_field, render_checkbox_field, render_button, render_time_schedule_form, render_ternary_field %}
+{% from '_common_fields.html' import render_common_settings_form %}
+<script>
+    const notification_base_url="{{url_for('ui.ui_notification.ajax_callback_send_notification_test', mode="global-settings")}}";
+{% if emailprefix %}
+    const email_notification_prefix=JSON.parse('{{emailprefix|tojson}}');
+{% endif %}
+</script>
+<script src="{{url_for('static_content', group='js', filename='tabs.js')}}" defer></script>
+<script src="{{url_for('static_content', group='js', filename='plugins.js')}}" defer></script>
+<script src="{{url_for('static_content', group='js', filename='notifications.js')}}" defer></script>
+<script src="{{url_for('static_content', group='js', filename='vis.js')}}" defer></script>
+<script src="{{url_for('static_content', group='js', filename='global-settings.js')}}" defer></script>
+<script src="{{url_for('static_content', group='js', filename='scheduler.js')}}" defer></script>
+<div class="edit-form">
+    <div class="tabs collapsable">
+        <ul>
+            <li class="tab" id=""><a href="#general">General</a></li>
+            <li class="tab"><a href="#notifications">Notifications</a></li>
+            <li class="tab"><a href="#fetching">Fetching</a></li>
+            <li class="tab"><a href="#filters">Global Filters</a></li>
+            <li class="tab"><a href="#ui-options">UI Options</a></li>
+            <li class="tab"><a href="#api">API</a></li>
+            <li class="tab"><a href="#timedate">Time &amp Date</a></li>
+            <li class="tab"><a href="#proxies">CAPTCHA &amp; Proxies</a></li>
+        </ul>
+    </div>
+    <div class="box-wrap inner">
+        <form class="pure-form pure-form-stacked settings" action="{{url_for('settings.settings_page')}}" method="POST">
+            <input type="hidden" name="csrf_token" value="{{ csrf_token() }}" >
+            <div class="tab-pane-inner" id="general">
+                <fieldset>
+                    <div class="pure-control-group">
+                        {{ render_field(form.requests.form.time_between_check, class="time-check-widget") }}
+                        <span class="pure-form-message-inline">Default recheck time for all watches, current system minimum is <i>{{min_system_recheck_seconds}}</i> seconds (<a href="https://github.com/dgtlmoon/changedetection.io/wiki/Misc-system-settings#enviroment-variables">more info</a>).</span>
+                            <div id="time-between-check-schedule">
+                                <!-- Start Time and End Time -->
+                                <div id="limit-between-time">
+                                    {{ render_time_schedule_form(form.requests, available_timezones, timezone_default_config) }}
+                                </div>
+                        </div>
+                    </div>
+                    <div class="pure-control-group">
+                        {{ render_field(form.requests.form.jitter_seconds, class="jitter_seconds") }}
+                        <span class="pure-form-message-inline">Example - 3 seconds random jitter could trigger up to 3 seconds earlier or up to 3 seconds later</span>
+                    </div>
+                    <div class="pure-control-group">
+                        {{ render_field(form.application.form.filter_failure_notification_threshold_attempts, class="filter_failure_notification_threshold_attempts") }}
+                        <span class="pure-form-message-inline">After this many consecutive times that the CSS/xPath filter is missing, send a notification
+                            <br>
+                        Set to <strong>0</strong> to disable
+                        </span>
+                    </div>
+                    <div class="pure-control-group">
+                        {% if not hide_remove_pass %}
+                            {% if current_user.is_authenticated %}
+                                {{ render_button(form.application.form.removepassword_button) }}
+                            {% else %}
+                            {{ render_field(form.application.form.password) }}
+                            <span class="pure-form-message-inline">Password protection for your changedetection.io application.</span>
+                            {% endif %}
+                        {% else %}
+                            <span class="pure-form-message-inline">Password is locked.</span>
+                        {% endif %}
+                    </div>
+
+                    <div class="pure-control-group">
+                        {{ render_checkbox_field(form.application.form.shared_diff_access, class="shared_diff_access") }}
+                        <span class="pure-form-message-inline">Allow access to view watch diff page when password is enabled (Good for sharing the diff page)
+                        </span>
+                    </div>
+                    <div class="pure-control-group">
+                        {{ render_checkbox_field(form.application.form.rss_hide_muted_watches) }}
+                    </div>
+                    <div class="pure-control-group">
+                        {{ render_field(form.application.form.rss_content_format) }}
+                        <span class="pure-form-message-inline">Love RSS? Does your reader support HTML? Set it here</span>
+                    </div>
+                    <div class="pure-control-group">
+                        {{ render_checkbox_field(form.application.form.empty_pages_are_a_change) }}
+                        <span class="pure-form-message-inline">When a request returns no content, or the HTML does not contain any text, is this considered a change?</span>
+                    </div>
+                {% if form.requests.proxy %}
+                    <div class="pure-control-group inline-radio">
+                        {{ render_field(form.requests.form.proxy, class="fetch-backend-proxy") }}
+                        <span class="pure-form-message-inline">
+                        Choose a default proxy for all watches
+                        </span>
+                    </div>
+                {% endif %}
+                </fieldset>
+            </div>
+
+            <div class="tab-pane-inner" id="notifications">
+                <fieldset>
+                    <div class="field-group">
+                        {{ render_common_settings_form(form.application.form, emailprefix, settings_application, extra_notification_token_placeholder_info) }}
+                    </div>
+                </fieldset>
+                <div class="pure-control-group" id="notification-base-url">
+                    {{ render_field(form.application.form.base_url, class="m-d") }}
+                    <span class="pure-form-message-inline">
+                        Base URL used for the <code>{{ '{{ base_url }}' }}</code> token in notification links.<br>
+                        Default value is the system environment variable '<code>BASE_URL</code>' - <a href="https://github.com/dgtlmoon/changedetection.io/wiki/Configurable-BASE_URL-setting">read more here</a>.
+                    </span>
+                </div>
+            </div>
+
+            <div class="tab-pane-inner" id="fetching">
+                <div class="pure-control-group inline-radio">
+                    {{ render_field(form.application.form.fetch_backend, class="fetch-backend") }}
+                    <span class="pure-form-message-inline">
+                        <p>Use the <strong>Basic</strong> method (default) where your watched sites don't need Javascript to render.</p>
+                        <p>The <strong>Chrome/Javascript</strong> method requires a network connection to a running WebDriver+Chrome server, set by the ENV var 'WEBDRIVER_URL'. </p>
+                    </span>
+                </div>
+                <fieldset class="pure-group" id="webdriver-override-options" data-visible-for="application-fetch_backend=html_webdriver">
+                    <div class="pure-form-message-inline">
+                        <strong>If you're having trouble waiting for the page to be fully rendered (text missing etc), try increasing the 'wait' time here.</strong>
+                        <br>
+                        This will wait <i>n</i> seconds before extracting the text.
+                    </div>
+                    <div class="pure-control-group">
+                        {{ render_field(form.application.form.webdriver_delay) }}
+                    </div>
+                </fieldset>
+                <div class="pure-control-group">
+                    {{ render_field(form.requests.form.workers) }}
+                    {% set worker_info = get_worker_status_info() %}
+                    <span class="pure-form-message-inline">Number of concurrent workers to process watches. More workers = faster processing but higher memory usage.<br>
+                    Currently running: <strong>{{ worker_info.count }}</strong> operational {{ worker_info.type }} workers{% if worker_info.active_workers > 0 %} ({{ worker_info.active_workers }} actively processing){% endif %}.</span>
+                </div>
+                <div class="pure-control-group inline-radio">
+                    {{ render_field(form.requests.form.default_ua) }}
+                    <span class="pure-form-message-inline">
+                        Applied to all requests.<br><br>
+                        Note: Simply changing the User-Agent often does not defeat anti-robot technologies, it's important to consider <a href="https://changedetection.io/tutorial/what-are-main-types-anti-robot-mechanisms">all of the ways that the browser is detected</a>.
+                    </span>
+                </div>
+                <div class="pure-control-group">
+                                        <br>
+                    Tip: <a href="https://github.com/dgtlmoon/changedetection.io/wiki/Proxy-configuration#brightdata-proxy-support">Connect using Bright Data and Oxylabs Proxies, find out more here.</a>
+
+                </div>
+            </div>
+
+            <div class="tab-pane-inner" id="filters">
+
+                    <fieldset class="pure-group">
+                    {{ render_checkbox_field(form.application.form.ignore_whitespace) }}
+                    <span class="pure-form-message-inline">Ignore whitespace, tabs and new-lines/line-feeds when considering if a change was detected.<br>
+                    <i>Note:</i> Changing this will change the status of your existing watches, possibly trigger alerts etc.
+                    </span>
+                    </fieldset>
+                <fieldset class="pure-group">
+                    {{ render_checkbox_field(form.application.form.render_anchor_tag_content) }}
+                    <span class="pure-form-message-inline">Render anchor tag content, default disabled, when enabled renders links as <code>(link text)[https://somesite.com]</code>
+                        <br>
+                    <i>Note:</i> Changing this could affect the content of your existing watches, possibly trigger alerts etc.
+                    </span>
+                    </fieldset>
+                    <fieldset class="pure-group">
+                      {{ render_field(form.application.form.global_subtractive_selectors, rows=5, placeholder="header
+footer
+nav
+.stockticker
+//*[contains(text(), 'Advertisement')]") }}
+                      <span class="pure-form-message-inline">
+                        <ul>
+                          <li> Remove HTML element(s) by CSS and XPath selectors before text conversion. </li>
+                          <li> Don't paste HTML here, use only CSS and XPath selectors </li>
+                          <li> Add multiple elements, CSS or XPath selectors per line to ignore multiple parts of the HTML. </li>
+                        </ul>
+                      </span>
+                    </fieldset>
+                    <fieldset class="pure-group">
+                    {{ render_field(form.application.form.global_ignore_text, rows=5, placeholder="Some text to ignore in a line
+/some.regex\d{2}/ for case-INsensitive regex
+                    ") }}
+                    <span class="pure-form-message-inline">Note: This is applied globally in addition to the per-watch rules.</span><br>
+                    <span class="pure-form-message-inline">
+                        <ul>
+                            <li>Matching text will be <strong>ignored</strong> in the text snapshot (you can still see it but it wont trigger a change)</li>
+                            <li>Note: This is applied globally in addition to the per-watch rules.</li>
+                            <li>Each line processed separately, any line matching will be ignored (removed before creating the checksum)</li>
+                            <li>Regular Expression support, wrap the entire line in forward slash <code>/regex/</code></li>
+                            <li>Changing this will affect the comparison checksum which may trigger an alert</li>
+                        </ul>
+                     </span>
+                    </fieldset>
+           </div>
+
+            <div class="tab-pane-inner" id="api">
+                <h4>API Access</h4>
+                <p>Drive your changedetection.io via API, More about <a href="https://changedetection.io/docs/api_v1/index.html">API access and examples here</a>.</p>
+
+                <div class="pure-control-group">
+                    {{ render_checkbox_field(form.application.form.api_access_token_enabled) }}
+                    <div class="pure-form-message-inline">Restrict API access limit by using <code>x-api-key</code> header - required for the Chrome Extension to work</div><br>
+                    <div class="pure-form-message-inline"><br>API Key <span id="api-key">{{api_key}}</span>
+                        <span style="display:none;" id="api-key-copy" >copy</span>
+                    </div>
+                </div>
+                <div class="pure-control-group">
+                    <a href="{{url_for('settings.settings_reset_api_key')}}" class="pure-button button-small button-cancel">Regenerate API key</a>
+                </div>
+                <div class="pure-control-group">
+                    <h4>Chrome Extension</h4>
+                    <p>Easily add any web-page to your changedetection.io installation from within Chrome.</p>
+                    <strong>Step 1</strong> Install the extension, <strong>Step 2</strong> Navigate to this page,
+                    <strong>Step 3</strong> Open the extension from the toolbar and click "<i>Sync API Access</i>"
+                    <p>
+                        <a id="chrome-extension-link"
+                           title="Try our new Chrome Extension!"
+                           href="https://chromewebstore.google.com/detail/changedetectionio-website/kefcfmgmlhmankjmnbijimhofdjekbop">
+                            <img alt="Chrome store icon" src="{{ url_for('static_content', group='images', filename='google-chrome-icon.png') }}" alt="Chrome">
+                            Chrome Webstore
+                        </a>
+                    </p>
+                </div>
+            </div>
+            <div class="tab-pane-inner" id="timedate">
+                <div class="pure-control-group">
+                    Ensure the settings below are correct, they are used to manage the time schedule for checking your web page watches.
+                </div>
+                <div class="pure-control-group">
+                    <p><strong>UTC Time &amp Date from Server:</strong> <span id="utc-time" >{{ utc_time }}</span></p>
+                    <p><strong>Local Time &amp Date in Browser:</strong> <span class="local-time" data-utc="{{ utc_time }}"></span></p>
+                    <p>
+                       {{ render_field(form.application.form.timezone) }}
+                        <datalist id="timezones" style="display: none;">
+                            {% for tz_name in available_timezones %}
+                                <option value="{{ tz_name }}">{{ tz_name }}</option>
+                            {% endfor %}
+                        </datalist>
+                    </p>
+                </div>
+            </div>
+            <div class="tab-pane-inner" id="ui-options">
+                <div class="pure-control-group">
+                    {{ render_checkbox_field(form.application.form.ui.form.open_diff_in_new_tab, class="open_diff_in_new_tab") }}
+                    <span class="pure-form-message-inline">Enable this setting to open the diff page in a new tab. If disabled, the diff page will open in the current tab.</span>
+                </div>
+                <div class="pure-control-group">
+                    {{ render_checkbox_field(form.application.form.ui.form.socket_io_enabled, class="socket_io_enabled") }}
+                    <span class="pure-form-message-inline">Realtime UI Updates Enabled - (Restart required if this is changed)</span>
+                </div>
+                <div class="pure-control-group">
+                    {{ render_checkbox_field(form.application.form.ui.form.favicons_enabled, class="") }}
+                    <span class="pure-form-message-inline">Enable or Disable Favicons next to the watch list</span>
+                </div>
+                <div class="pure-control-group">
+                    {{ render_checkbox_field(form.application.form.ui.use_page_title_in_list) }}
+                </div>
+                <div class="pure-control-group">
+                    {{ render_field(form.application.form.pager_size) }}
+                    <span class="pure-form-message-inline">Number of items per page in the watch overview list, 0 to disable.</span>
+                </div>
+
+            </div>
+            <div class="tab-pane-inner" id="proxies">
+                <div id="recommended-proxy">
+                    <div>
+                        <img style="height: 2em;" src="{{url_for('static_content', group='images', filename='brightdata.svg')}}" alt="BrightData Proxy Provider">
+                        <p>BrightData offer world-class proxy services, "Data Center" proxies are a very affordable way to proxy your requests, whilst <strong><a href="https://brightdata.grsm.io/n0r16zf7eivq">WebUnlocker</a></strong> can help solve most CAPTCHAs.</p>
+                        <p>
+                            BrightData offer many <a href="https://brightdata.com/proxy-types" target="new">many different types of proxies</a>, it is worth reading about what is best for your use-case.
+                        </p>
+
+                        <p>
+                            When you have <a href="https://brightdata.grsm.io/n0r16zf7eivq">registered</a>, enabled the required services, visit the <A href="https://brightdata.com/cp/api_example?">API example page</A>, then select <strong>Python</strong>, set the country you wish to use, then copy+paste the access Proxy URL into the "Extra Proxies" boxes below.<br>
+                        </p>
+                        <p>
+                            The Proxy URL with BrightData should start with <code>http://brd-customer...</code>
+                        </p>
+                        <p>When you sign up using <a href="https://brightdata.grsm.io/n0r16zf7eivq">https://brightdata.grsm.io/n0r16zf7eivq</a> BrightData will match any first deposit up to $150</p>
+                    </div>
+                    <div>
+                        <img style="height: 2em;"
+                             src="{{url_for('static_content', group='images', filename='oxylabs.svg')}}"
+                             alt="Oxylabs Proxy Provider">
+                        <p>
+                            Collect public data at scale with industry-leading web scraping solutions and the world’s
+                            largest ethical proxy network.
+                        </p>
+                        <p>
+                            Oxylabs also provide a <a href="https://oxylabs.io/products/web-unblocker"><strong>WebUnlocker</strong></a>
+                            proxy that bypasses sophisticated anti-bot systems, so you don’t have to.<br>
+                        </p>
+                        <p>
+                            Serve over <a href="https://oxylabs.io/location-proxy">195 countries</a>, providing <a
+                                href="https://oxylabs.io/products/residential-proxy-pool">Residential</a>, <a
+                                href="https://oxylabs.io/products/mobile-proxies">Mobile</a> and <a
+                                href="https://oxylabs.io/products/rotating-isp-proxies">ISP proxies</a> and much more.
+                        </p>
+                        <p>
+                            Use the promo code <strong>boost35</strong> with this link <a href="https://oxylabs.go2cloud.org/SH2d">https://oxylabs.go2cloud.org/SH2d</a> for 35% off Residential, Mobile proxies, Web Unblocker, and Scraper APIs. Built-in proxies enable you to access data from all around the world and help overcome anti-bot solutions.
+
+                        </p>
+
+                        
+                    </div>
+                </div>
+
+               <p><strong>Tip</strong>: "Residential" and "Mobile" proxy type can be more successfull than "Data Center" for blocked websites.
+
+                <div class="pure-control-group" id="extra-proxies-setting">
+                {{ render_field(form.requests.form.extra_proxies) }}
+                <span class="pure-form-message-inline">"Name" will be used for selecting the proxy in the Watch Edit settings</span><br>
+                <span class="pure-form-message-inline">SOCKS5 proxies with authentication are only supported with 'plain requests' fetcher, for other fetchers you should whitelist the IP access instead</span>
+                </div>
+                <div class="pure-control-group" id="extra-browsers-setting">
+                    <p>
+                    <span class="pure-form-message-inline"><i>Extra Browsers</i> can be attached to further defeat CAPTCHA's on websites that are particularly hard to scrape.</span><br>
+                    <span class="pure-form-message-inline">Simply paste the connection address into the box, <a href="https://changedetection.io/tutorial/using-bright-datas-scraping-browser-pass-captchas-and-other-protection-when-monitoring">More instructions and examples here</a> </span>
+                    </p>
+                    {{ render_field(form.requests.form.extra_browsers) }}
+                </div>
+            </div>
+            <div id="actions">
+                <div class="pure-control-group">
+                    {{ render_button(form.save_button) }}
+                    <a href="{{url_for('watchlist.index')}}" class="pure-button button-cancel">Back</a>
+                    <a href="{{url_for('ui.clear_all_history')}}" class="pure-button button-error">Clear Snapshot History</a>
+                </div>
+            </div>
+        </form>
+    </div>
+</div>
+
+{% endblock %}

changedetectionio/blueprint/tags/README.md

@@ -0,0 +1,9 @@
+# Groups tags
+
+## How it works
+
+Watch has a list() of tag UUID's, which relate to a config under application.settings.tags
+
+The 'tag' is actually a watch, because they basically will eventually share 90% of the same config.
+
+So a tag is like an abstract of a watch

changedetectionio/blueprint/tags/__init__.py

@@ -0,0 +1,191 @@
+from flask import Blueprint, request, render_template, flash, url_for, redirect
+
+
+from changedetectionio.store import ChangeDetectionStore
+from changedetectionio.flask_app import login_optionally_required
+
+
+def construct_blueprint(datastore: ChangeDetectionStore):
+    tags_blueprint = Blueprint('tags', __name__, template_folder="templates")
+
+    @tags_blueprint.route("/list", methods=['GET'])
+    @login_optionally_required
+    def tags_overview_page():
+        from .form import SingleTag
+        add_form = SingleTag(request.form)
+
+        sorted_tags = sorted(datastore.data['settings']['application'].get('tags').items(), key=lambda x: x[1]['title'])
+
+        from collections import Counter
+
+        tag_count = Counter(tag for watch in datastore.data['watching'].values() if watch.get('tags') for tag in watch['tags'])
+
+        output = render_template("groups-overview.html",
+                                 available_tags=sorted_tags,
+                                 form=add_form,
+                                 tag_count=tag_count
+                                 )
+
+        return output
+
+    @tags_blueprint.route("/add", methods=['POST'])
+    @login_optionally_required
+    def form_tag_add():
+        from .form import SingleTag
+        add_form = SingleTag(request.form)
+
+        if not add_form.validate():
+            for widget, l in add_form.errors.items():
+                flash(','.join(l), 'error')
+            return redirect(url_for('tags.tags_overview_page'))
+
+        title = request.form.get('name').strip()
+
+        if datastore.tag_exists_by_name(title):
+            flash(f'The tag "{title}" already exists', "error")
+            return redirect(url_for('tags.tags_overview_page'))
+
+        datastore.add_tag(title)
+        flash("Tag added")
+
+
+        return redirect(url_for('tags.tags_overview_page'))
+
+    @tags_blueprint.route("/mute/<string:uuid>", methods=['GET'])
+    @login_optionally_required
+    def mute(uuid):
+        if datastore.data['settings']['application']['tags'].get(uuid):
+            datastore.data['settings']['application']['tags'][uuid]['notification_muted'] = not datastore.data['settings']['application']['tags'][uuid]['notification_muted']
+        return redirect(url_for('tags.tags_overview_page'))
+
+    @tags_blueprint.route("/delete/<string:uuid>", methods=['GET'])
+    @login_optionally_required
+    def delete(uuid):
+        removed = 0
+        # Delete the tag, and any tag reference
+        if datastore.data['settings']['application']['tags'].get(uuid):
+            del datastore.data['settings']['application']['tags'][uuid]
+
+        for watch_uuid, watch in datastore.data['watching'].items():
+            if watch.get('tags') and uuid in watch['tags']:
+                removed += 1
+                watch['tags'].remove(uuid)
+
+        flash(f"Tag deleted and removed from {removed} watches")
+        return redirect(url_for('tags.tags_overview_page'))
+
+    @tags_blueprint.route("/unlink/<string:uuid>", methods=['GET'])
+    @login_optionally_required
+    def unlink(uuid):
+        unlinked = 0
+        for watch_uuid, watch in datastore.data['watching'].items():
+            if watch.get('tags') and uuid in watch['tags']:
+                unlinked += 1
+                watch['tags'].remove(uuid)
+
+        flash(f"Tag unlinked removed from {unlinked} watches")
+        return redirect(url_for('tags.tags_overview_page'))
+
+    @tags_blueprint.route("/delete_all", methods=['GET'])
+    @login_optionally_required
+    def delete_all():
+        for watch_uuid, watch in datastore.data['watching'].items():
+            watch['tags'] = []
+        datastore.data['settings']['application']['tags'] = {}
+
+        flash(f"All tags deleted")
+        return redirect(url_for('tags.tags_overview_page'))
+
+    @tags_blueprint.route("/edit/<string:uuid>", methods=['GET'])
+    @login_optionally_required
+    def form_tag_edit(uuid):
+        from changedetectionio.blueprint.tags.form import group_restock_settings_form
+        if uuid == 'first':
+            uuid = list(datastore.data['settings']['application']['tags'].keys()).pop()
+
+        default = datastore.data['settings']['application']['tags'].get(uuid)
+        if not default:
+            flash("Tag not found", "error")
+            return redirect(url_for('watchlist.index'))
+
+        form = group_restock_settings_form(
+                                       formdata=request.form if request.method == 'POST' else None,
+                                       data=default,
+                                       extra_notification_tokens=datastore.get_unique_notification_tokens_available(),
+                                       default_system_settings = datastore.data['settings'],
+                                       )
+
+        template_args = {
+            'data': default,
+            'form': form,
+            'watch': default,
+            'extra_notification_token_placeholder_info': datastore.get_unique_notification_token_placeholders_available(),
+        }
+
+        included_content = {}
+        if form.extra_form_content():
+            # So that the extra panels can access _helpers.html etc, we set the environment to load from templates/
+            # And then render the code from the module
+            from jinja2 import Environment, FileSystemLoader
+            import importlib.resources
+            templates_dir = str(importlib.resources.files("changedetectionio").joinpath('templates'))
+            env = Environment(loader=FileSystemLoader(templates_dir))
+            template_str = """{% from '_helpers.html' import render_field, render_checkbox_field, render_button %}
+        <script>        
+            $(document).ready(function () {
+                toggleOpacity('#overrides_watch', '#restock-fieldset-price-group', true);
+            });
+        </script>            
+                <fieldset>
+                    <div class="pure-control-group">
+                        <fieldset class="pure-group">
+                        {{ render_checkbox_field(form.overrides_watch) }}
+                        <span class="pure-form-message-inline">Used for watches in "Restock & Price detection" mode</span>
+                        </fieldset>
+                </fieldset>
+                """
+            template_str += form.extra_form_content()
+            template = env.from_string(template_str)
+            included_content = template.render(**template_args)
+
+        output = render_template("edit-tag.html",
+                                 settings_application=datastore.data['settings']['application'],
+                                 extra_tab_content=form.extra_tab_content() if form.extra_tab_content() else None,
+                                 extra_form_content=included_content,
+                                 **template_args
+                                 )
+
+        return output
+
+
+    @tags_blueprint.route("/edit/<string:uuid>", methods=['POST'])
+    @login_optionally_required
+    def form_tag_edit_submit(uuid):
+        from changedetectionio.blueprint.tags.form import group_restock_settings_form
+        if uuid == 'first':
+            uuid = list(datastore.data['settings']['application']['tags'].keys()).pop()
+
+        default = datastore.data['settings']['application']['tags'].get(uuid)
+
+        form = group_restock_settings_form(formdata=request.form if request.method == 'POST' else None,
+                               data=default,
+                               extra_notification_tokens=datastore.get_unique_notification_tokens_available()
+                               )
+        # @todo subclass form so validation works
+        #if not form.validate():
+#            for widget, l in form.errors.items():
+#                flash(','.join(l), 'error')
+#           return redirect(url_for('tags.form_tag_edit_submit', uuid=uuid))
+
+        datastore.data['settings']['application']['tags'][uuid].update(form.data)
+        datastore.data['settings']['application']['tags'][uuid]['processor'] = 'restock_diff'
+        datastore.needs_write_urgent = True
+        flash("Updated")
+
+        return redirect(url_for('tags.tags_overview_page'))
+
+
+    @tags_blueprint.route("/delete/<string:uuid>", methods=['GET'])
+    def form_tag_delete(uuid):
+        return redirect(url_for('tags.tags_overview_page'))
+    return tags_blueprint

changedetectionio/blueprint/tags/form.py

@@ -0,0 +1,21 @@
+from wtforms import (
+    Form,
+    StringField,
+    SubmitField,
+    validators,
+)
+from wtforms.fields.simple import BooleanField
+
+from changedetectionio.processors.restock_diff.forms import processor_settings_form as restock_settings_form
+
+class group_restock_settings_form(restock_settings_form):
+    overrides_watch = BooleanField('Activate for individual watches in this tag/group?', default=False)
+
+class SingleTag(Form):
+
+    name = StringField('Tag name', [validators.InputRequired()], render_kw={"placeholder": "Name"})
+    save_button = SubmitField('Save', render_kw={"class": "pure-button pure-button-primary"})
+
+
+
+

changedetectionio/blueprint/tags/templates/edit-tag.html

@@ -0,0 +1,100 @@
+{% extends 'base.html' %}
+{% block content %}
+{% from '_helpers.html' import render_field, render_checkbox_field, render_button, render_ternary_field %}
+{% from '_common_fields.html' import render_common_settings_form %}
+<script>
+    const notification_base_url="{{url_for('ui.ui_notification.ajax_callback_send_notification_test', mode="group-settings")}}";
+</script>
+
+<script src="{{url_for('static_content', group='js', filename='tabs.js')}}" defer></script>
+<script>
+
+/*{% if emailprefix %}*/
+    /*const email_notification_prefix=JSON.parse('{{ emailprefix|tojson }}');*/
+/*{% endif %}*/
+
+{% set has_tag_filters_extra='' %}
+
+</script>
+
+<script src="{{url_for('static_content', group='js', filename='watch-settings.js')}}" defer></script>
+<script src="{{url_for('static_content', group='js', filename='notifications.js')}}" defer></script>
+
+<div class="edit-form monospaced-textarea">
+
+    <div class="tabs collapsable">
+        <ul>
+            <li class="tab" id=""><a href="#general">General</a></li>
+            <li class="tab"><a href="#filters-and-triggers">Filters &amp; Triggers</a></li>
+            {% if extra_tab_content %}
+            <li class="tab"><a href="#extras_tab">{{ extra_tab_content }}</a></li>
+            {% endif %}
+            <li class="tab"><a href="#notifications">Notifications</a></li>
+        </ul>
+    </div>
+
+    <div class="box-wrap inner">
+        <form class="pure-form pure-form-stacked"
+              action="{{ url_for('tags.form_tag_edit', uuid=data.uuid) }}" method="POST">
+             <input type="hidden" name="csrf_token" value="{{ csrf_token() }}">
+
+            <div class="tab-pane-inner" id="general">
+                <fieldset>
+                    <div class="pure-control-group">
+                        {{ render_field(form.title, placeholder="https://...", required=true, class="m-d") }}
+                    </div>
+                </fieldset>
+            </div>
+
+            <div class="tab-pane-inner" id="filters-and-triggers">
+                <p>These settings are <strong><i>added</i></strong> to any existing watch configurations.</p>
+                {% include "edit/include_subtract.html" %}
+                <div class="text-filtering border-fieldset">
+                    <h3>Text filtering</h3>
+                    {% include "edit/text-options.html" %}
+                </div>
+            </div>
+
+        {# rendered sub Template #}
+        {% if extra_form_content %}
+            <div class="tab-pane-inner" id="extras_tab">
+            {{ extra_form_content|safe }}
+            </div>
+        {% endif %}
+            <div class="tab-pane-inner" id="notifications">
+                <fieldset>
+                    <div  class="pure-control-group inline-radio">
+                      {{ render_ternary_field(form.notification_muted, BooleanField=True) }}
+                    </div>
+                    {% if 1 %}
+                    <div class="pure-control-group inline-radio">
+                      {{ render_checkbox_field(form.notification_screenshot) }}
+                        <span class="pure-form-message-inline">
+                            <strong>Use with caution!</strong> This will easily fill up your email storage quota or flood other storages.
+                        </span>
+                    </div>
+                    {% endif %}
+                    <div class="field-group" id="notification-field-group">
+                        {% if has_default_notification_urls %}
+                        <div class="inline-warning">
+                            <img class="inline-warning-icon" src="{{url_for('static_content', group='images', filename='notice.svg')}}" alt="Look out!" title="Lookout!" >
+                            There are <a href="{{ url_for('settings.settings_page')}}#notifications">system-wide notification URLs enabled</a>, this form will override notification settings for this watch only &dash; an empty Notification URL list here will still send notifications.
+                        </div>
+                        {% endif %}
+                        <a href="#notifications" id="notification-setting-reset-to-default" class="pure-button button-xsmall" style="right: 20px; top: 20px; position: absolute; background-color: #5f42dd; border-radius: 4px; font-size: 70%; color: #fff">Use system defaults</a>
+
+                        {{ render_common_settings_form(form, emailprefix, settings_application, extra_notification_token_placeholder_info) }}
+                    </div>
+                </fieldset>
+            </div>
+
+            <div id="actions">
+                <div class="pure-control-group">
+                    {{ render_button(form.save_button) }}
+                </div>
+            </div>
+        </form>
+    </div>
+</div>
+
+{% endblock %}

changedetectionio/blueprint/tags/templates/groups-overview.html

@@ -0,0 +1,62 @@
+{% extends 'base.html' %}
+{% block content %}
+{% from '_helpers.html' import render_simple_field, render_field %}
+<script src="{{url_for('static_content', group='js', filename='jquery-3.6.0.min.js')}}"></script>
+
+<div class="box">
+    <form class="pure-form" action="{{ url_for('tags.form_tag_add') }}" method="POST" id="new-watch-form">
+        <input type="hidden" name="csrf_token" value="{{ csrf_token() }}" >
+        <fieldset>
+            <legend>Add a new organisational tag</legend>
+            <div id="watch-add-wrapper-zone">
+                <div>
+                    {{ render_simple_field(form.name, placeholder="Watch group / tag") }}
+                </div>
+                <div>
+                    {{ render_simple_field(form.save_button, title="Save" ) }}
+                </div>
+            </div>
+            <br>
+            <div style="color: #fff;">Groups allows you to manage filters and notifications for multiple watches under a single organisational tag.</div>
+        </fieldset>
+    </form>
+    <!-- @todo maybe some overview matrix, 'tick' with which has notification, filter rules etc -->
+    <div id="watch-table-wrapper">
+
+        <table class="pure-table pure-table-striped watch-table group-overview-table">
+            <thead>
+            <tr>
+                <th></th>
+                <th># Watches</th>
+                <th>Tag / Label name</th>
+                <th></th>
+            </tr>
+            </thead>
+            <tbody>
+            <!--
+            @Todo - connect Last checked, Last Changed, Number of Watches etc
+            --->
+            {% if not available_tags|length %}
+            <tr>
+                <td colspan="3">No website organisational tags/groups configured</td>
+            </tr>
+            {% endif %}
+            {% for uuid, tag in available_tags  %}
+            <tr id="{{ uuid }}" class="{{ loop.cycle('pure-table-odd', 'pure-table-even') }}">
+                <td class="watch-controls">
+                    <a class="link-mute state-{{'on' if tag.notification_muted else 'off'}}" href="{{url_for('tags.mute', uuid=tag.uuid)}}"><img src="{{url_for('static_content', group='images', filename='bell-off.svg')}}" alt="Mute notifications" title="Mute notifications" class="icon icon-mute" ></a>
+                </td>
+                <td>{{ "{:,}".format(tag_count[uuid]) if uuid in tag_count else 0 }}</td>
+                <td class="title-col inline"> <a href="{{url_for('watchlist.index', tag=uuid) }}">{{ tag.title }}</a></td>
+                <td>
+                    <a class="pure-button pure-button-primary" href="{{ url_for('tags.form_tag_edit', uuid=uuid) }}">Edit</a>&nbsp;
+                    <a class="pure-button pure-button-primary" href="{{ url_for('tags.delete', uuid=uuid) }}" title="Deletes and removes tag">Delete</a>
+                    <a class="pure-button pure-button-primary" href="{{ url_for('tags.unlink', uuid=uuid) }}" title="Keep the tag but unlink any watches">Unlink</a>
+                </td>
+            </tr>
+            {% endfor %}
+            </tbody>
+        </table>
+    </div>
+</div>
+{% endblock %}

changedetectionio/blueprint/ui/__init__.py

@@ -0,0 +1,325 @@
+import time
+from flask import Blueprint, request, redirect, url_for, flash, render_template, session
+from loguru import logger
+
+from changedetectionio.store import ChangeDetectionStore
+from changedetectionio.blueprint.ui.edit import construct_blueprint as construct_edit_blueprint
+from changedetectionio.blueprint.ui.notification import construct_blueprint as construct_notification_blueprint
+from changedetectionio.blueprint.ui.views import construct_blueprint as construct_views_blueprint
+
+def _handle_operations(op, uuids, datastore, worker_handler, update_q, queuedWatchMetaData, watch_check_update, extra_data=None, emit_flash=True):
+    from flask import request, flash
+
+    if op == 'delete':
+        for uuid in uuids:
+            if datastore.data['watching'].get(uuid):
+                datastore.delete(uuid)
+        if emit_flash:
+            flash(f"{len(uuids)} watches deleted")
+
+    elif op == 'pause':
+        for uuid in uuids:
+            if datastore.data['watching'].get(uuid):
+                datastore.data['watching'][uuid]['paused'] = True
+        if emit_flash:
+            flash(f"{len(uuids)} watches paused")
+
+    elif op == 'unpause':
+        for uuid in uuids:
+            if datastore.data['watching'].get(uuid):
+                datastore.data['watching'][uuid.strip()]['paused'] = False
+        if emit_flash:
+            flash(f"{len(uuids)} watches unpaused")
+
+    elif (op == 'mark-viewed'):
+        for uuid in uuids:
+            if datastore.data['watching'].get(uuid):
+                datastore.set_last_viewed(uuid, int(time.time()))
+        if emit_flash:
+            flash(f"{len(uuids)} watches updated")
+
+    elif (op == 'mute'):
+        for uuid in uuids:
+            if datastore.data['watching'].get(uuid):
+                datastore.data['watching'][uuid]['notification_muted'] = True
+        if emit_flash:
+            flash(f"{len(uuids)} watches muted")
+
+    elif (op == 'unmute'):
+        for uuid in uuids:
+            if datastore.data['watching'].get(uuid):
+                datastore.data['watching'][uuid]['notification_muted'] = False
+        if emit_flash:
+            flash(f"{len(uuids)} watches un-muted")
+
+    elif (op == 'recheck'):
+        for uuid in uuids:
+            if datastore.data['watching'].get(uuid):
+                # Recheck and require a full reprocessing
+                worker_handler.queue_item_async_safe(update_q, queuedWatchMetaData.PrioritizedItem(priority=1, item={'uuid': uuid}))
+        if emit_flash:
+            flash(f"{len(uuids)} watches queued for rechecking")
+
+    elif (op == 'clear-errors'):
+        for uuid in uuids:
+            if datastore.data['watching'].get(uuid):
+                datastore.data['watching'][uuid]["last_error"] = False
+        if emit_flash:
+            flash(f"{len(uuids)} watches errors cleared")
+
+    elif (op == 'clear-history'):
+        for uuid in uuids:
+            if datastore.data['watching'].get(uuid):
+                datastore.clear_watch_history(uuid)
+        if emit_flash:
+            flash(f"{len(uuids)} watches cleared/reset.")
+
+    elif (op == 'notification-default'):
+        from changedetectionio.notification import (
+            default_notification_format_for_watch
+        )
+        for uuid in uuids:
+            if datastore.data['watching'].get(uuid):
+                datastore.data['watching'][uuid]['notification_title'] = None
+                datastore.data['watching'][uuid]['notification_body'] = None
+                datastore.data['watching'][uuid]['notification_urls'] = []
+                datastore.data['watching'][uuid]['notification_format'] = default_notification_format_for_watch
+        if emit_flash:
+            flash(f"{len(uuids)} watches set to use default notification settings")
+
+    elif (op == 'assign-tag'):
+        op_extradata = extra_data
+        if op_extradata:
+            tag_uuid = datastore.add_tag(title=op_extradata)
+            if op_extradata and tag_uuid:
+                for uuid in uuids:
+                    if datastore.data['watching'].get(uuid):
+                        # Bug in old versions caused by bad edit page/tag handler
+                        if isinstance(datastore.data['watching'][uuid]['tags'], str):
+                            datastore.data['watching'][uuid]['tags'] = []
+
+                        datastore.data['watching'][uuid]['tags'].append(tag_uuid)
+        if emit_flash:
+            flash(f"{len(uuids)} watches were tagged")
+
+    if uuids:
+        for uuid in uuids:
+            watch_check_update.send(watch_uuid=uuid)
+
+def construct_blueprint(datastore: ChangeDetectionStore, update_q, worker_handler, queuedWatchMetaData, watch_check_update):
+    ui_blueprint = Blueprint('ui', __name__, template_folder="templates")
+    
+    # Register the edit blueprint
+    edit_blueprint = construct_edit_blueprint(datastore, update_q, queuedWatchMetaData)
+    ui_blueprint.register_blueprint(edit_blueprint)
+    
+    # Register the notification blueprint
+    notification_blueprint = construct_notification_blueprint(datastore)
+    ui_blueprint.register_blueprint(notification_blueprint)
+    
+    # Register the views blueprint
+    views_blueprint = construct_views_blueprint(datastore, update_q, queuedWatchMetaData, watch_check_update)
+    ui_blueprint.register_blueprint(views_blueprint)
+
+    # Import the login decorator
+    from changedetectionio.auth_decorator import login_optionally_required
+
+    @ui_blueprint.route("/clear_history/<string:uuid>", methods=['GET'])
+    @login_optionally_required
+    def clear_watch_history(uuid):
+        try:
+            datastore.clear_watch_history(uuid)
+        except KeyError:
+            flash('Watch not found', 'error')
+        else:
+            flash("Cleared snapshot history for watch {}".format(uuid))
+        return redirect(url_for('watchlist.index'))
+
+    @ui_blueprint.route("/clear_history", methods=['GET', 'POST'])
+    @login_optionally_required
+    def clear_all_history():
+        if request.method == 'POST':
+            confirmtext = request.form.get('confirmtext')
+
+            if confirmtext == 'clear':
+                for uuid in datastore.data['watching'].keys():
+                    datastore.clear_watch_history(uuid)
+                flash("Cleared snapshot history for all watches")
+            else:
+                flash('Incorrect confirmation text.', 'error')
+
+            return redirect(url_for('watchlist.index'))
+
+        output = render_template("clear_all_history.html")
+        return output
+
+    # Clear all statuses, so we do not see the 'unviewed' class
+    @ui_blueprint.route("/form/mark-all-viewed", methods=['GET'])
+    @login_optionally_required
+    def mark_all_viewed():
+        # Save the current newest history as the most recently viewed
+        with_errors = request.args.get('with_errors') == "1"
+        tag_limit = request.args.get('tag')
+        logger.debug(f"Limiting to tag {tag_limit}")
+        now = int(time.time())
+        for watch_uuid, watch in datastore.data['watching'].items():
+            if with_errors and not watch.get('last_error'):
+                continue
+
+            if tag_limit and ( not watch.get('tags') or tag_limit not in watch['tags'] ):
+                logger.debug(f"Skipping watch {watch_uuid}")
+                continue
+
+            datastore.set_last_viewed(watch_uuid, now)
+
+        return redirect(url_for('watchlist.index', tag=tag_limit))
+
+    @ui_blueprint.route("/delete", methods=['GET'])
+    @login_optionally_required
+    def form_delete():
+        uuid = request.args.get('uuid')
+
+        if uuid != 'all' and not uuid in datastore.data['watching'].keys():
+            flash('The watch by UUID {} does not exist.'.format(uuid), 'error')
+            return redirect(url_for('watchlist.index'))
+
+        # More for testing, possible to return the first/only
+        if uuid == 'first':
+            uuid = list(datastore.data['watching'].keys()).pop()
+        datastore.delete(uuid)
+        flash('Deleted.')
+
+        return redirect(url_for('watchlist.index'))
+
+    @ui_blueprint.route("/clone", methods=['GET'])
+    @login_optionally_required
+    def form_clone():
+        uuid = request.args.get('uuid')
+        # More for testing, possible to return the first/only
+        if uuid == 'first':
+            uuid = list(datastore.data['watching'].keys()).pop()
+
+        new_uuid = datastore.clone(uuid)
+
+        if not datastore.data['watching'].get(uuid).get('paused'):
+            worker_handler.queue_item_async_safe(update_q, queuedWatchMetaData.PrioritizedItem(priority=5, item={'uuid': new_uuid}))
+
+        flash('Cloned, you are editing the new watch.')
+
+        return redirect(url_for("ui.ui_edit.edit_page", uuid=new_uuid))
+
+    @ui_blueprint.route("/checknow", methods=['GET'])
+    @login_optionally_required
+    def form_watch_checknow():
+        # Forced recheck will skip the 'skip if content is the same' rule (, 'reprocess_existing_data': True})))
+        tag = request.args.get('tag')
+        uuid = request.args.get('uuid')
+        with_errors = request.args.get('with_errors') == "1"
+
+        i = 0
+
+        running_uuids = worker_handler.get_running_uuids()
+
+        if uuid:
+            if uuid not in running_uuids:
+                worker_handler.queue_item_async_safe(update_q, queuedWatchMetaData.PrioritizedItem(priority=1, item={'uuid': uuid}))
+                i += 1
+
+        else:
+            # Recheck all, including muted
+            # Get most overdue first
+            for k in sorted(datastore.data['watching'].items(), key=lambda item: item[1].get('last_checked', 0)):
+                watch_uuid = k[0]
+                watch = k[1]
+                if not watch['paused']:
+                    if watch_uuid not in running_uuids:
+                        if with_errors and not watch.get('last_error'):
+                            continue
+
+                        if tag != None and tag not in watch['tags']:
+                            continue
+
+                        worker_handler.queue_item_async_safe(update_q, queuedWatchMetaData.PrioritizedItem(priority=1, item={'uuid': watch_uuid}))
+                        i += 1
+
+        if i == 1:
+            flash("Queued 1 watch for rechecking.")
+        if i > 1:
+            flash(f"Queued {i} watches for rechecking.")
+        if i == 0:
+            flash("No watches available to recheck.")
+
+        return redirect(url_for('watchlist.index'))
+
+    @ui_blueprint.route("/form/checkbox-operations", methods=['POST'])
+    @login_optionally_required
+    def form_watch_list_checkbox_operations():
+        op = request.form['op']
+        uuids = [u.strip() for u in request.form.getlist('uuids') if u]
+        extra_data = request.form.get('op_extradata', '').strip()
+        _handle_operations(
+            datastore=datastore,
+            extra_data=extra_data,
+            queuedWatchMetaData=queuedWatchMetaData,
+            uuids=uuids,
+            worker_handler=worker_handler,
+            update_q=update_q,
+            watch_check_update=watch_check_update,
+            op=op,
+        )
+
+        return redirect(url_for('watchlist.index'))
+
+
+    @ui_blueprint.route("/share-url/<string:uuid>", methods=['GET'])
+    @login_optionally_required
+    def form_share_put_watch(uuid):
+        """Given a watch UUID, upload the info and return a share-link
+           the share-link can be imported/added"""
+        import requests
+        import json
+        from copy import deepcopy
+
+        # more for testing
+        if uuid == 'first':
+            uuid = list(datastore.data['watching'].keys()).pop()
+
+        # copy it to memory as trim off what we dont need (history)
+        watch = deepcopy(datastore.data['watching'].get(uuid))
+        # For older versions that are not a @property
+        if (watch.get('history')):
+            del (watch['history'])
+
+        # for safety/privacy
+        for k in list(watch.keys()):
+            if k.startswith('notification_'):
+                del watch[k]
+
+        for r in['uuid', 'last_checked', 'last_changed']:
+            if watch.get(r):
+                del (watch[r])
+
+        # Add the global stuff which may have an impact
+        watch['ignore_text'] += datastore.data['settings']['application']['global_ignore_text']
+        watch['subtractive_selectors'] += datastore.data['settings']['application']['global_subtractive_selectors']
+
+        watch_json = json.dumps(watch)
+
+        try:
+            r = requests.request(method="POST",
+                                 data={'watch': watch_json},
+                                 url="https://changedetection.io/share/share",
+                                 headers={'App-Guid': datastore.data['app_guid']})
+            res = r.json()
+
+            # Add to the flask session
+            session['share-link'] = f"https://changedetection.io/share/{res['share_key']}"
+
+
+        except Exception as e:
+            logger.error(f"Error sharing -{str(e)}")
+            flash(f"Could not share, something went wrong while communicating with the share server - {str(e)}", 'error')
+
+        return redirect(url_for('watchlist.index'))
+
+    return ui_blueprint

changedetectionio/blueprint/ui/edit.py

@@ -0,0 +1,340 @@
+import time
+from copy import deepcopy
+import os
+import importlib.resources
+from flask import Blueprint, request, redirect, url_for, flash, render_template, make_response, send_from_directory, abort
+from loguru import logger
+from jinja2 import Environment, FileSystemLoader
+
+from changedetectionio.store import ChangeDetectionStore
+from changedetectionio.auth_decorator import login_optionally_required
+from changedetectionio.time_handler import is_within_schedule
+from changedetectionio import worker_handler
+
+def construct_blueprint(datastore: ChangeDetectionStore, update_q, queuedWatchMetaData):
+    edit_blueprint = Blueprint('ui_edit', __name__, template_folder="../ui/templates")
+    
+    def _watch_has_tag_options_set(watch):
+        """This should be fixed better so that Tag is some proper Model, a tag is just a Watch also"""
+        for tag_uuid, tag in datastore.data['settings']['application'].get('tags', {}).items():
+            if tag_uuid in watch.get('tags', []) and (tag.get('include_filters') or tag.get('subtractive_selectors')):
+                return True
+
+    @edit_blueprint.route("/edit/<string:uuid>", methods=['GET', 'POST'])
+    @login_optionally_required
+    # https://stackoverflow.com/questions/42984453/wtforms-populate-form-with-data-if-data-exists
+    # https://wtforms.readthedocs.io/en/3.0.x/forms/#wtforms.form.Form.populate_obj ?
+    def edit_page(uuid):
+        from changedetectionio import forms
+        from changedetectionio.blueprint.browser_steps.browser_steps import browser_step_ui_config
+        from changedetectionio import processors
+        import importlib
+
+        # More for testing, possible to return the first/only
+        if not datastore.data['watching'].keys():
+            flash("No watches to edit", "error")
+            return redirect(url_for('watchlist.index'))
+
+        if uuid == 'first':
+            uuid = list(datastore.data['watching'].keys()).pop()
+
+        if not uuid in datastore.data['watching']:
+            flash("No watch with the UUID %s found." % (uuid), "error")
+            return redirect(url_for('watchlist.index'))
+
+        switch_processor = request.args.get('switch_processor')
+        if switch_processor:
+            for p in processors.available_processors():
+                if p[0] == switch_processor:
+                    datastore.data['watching'][uuid]['processor'] = switch_processor
+                    flash(f"Switched to mode - {p[1]}.")
+                    datastore.clear_watch_history(uuid)
+                    redirect(url_for('ui_edit.edit_page', uuid=uuid))
+
+        # be sure we update with a copy instead of accidently editing the live object by reference
+        default = deepcopy(datastore.data['watching'][uuid])
+
+        # Defaults for proxy choice
+        if datastore.proxy_list is not None:  # When enabled
+            # @todo
+            # Radio needs '' not None, or incase that the chosen one no longer exists
+            if default['proxy'] is None or not any(default['proxy'] in tup for tup in datastore.proxy_list):
+                default['proxy'] = ''
+        # proxy_override set to the json/text list of the items
+
+        # Does it use some custom form? does one exist?
+        processor_name = datastore.data['watching'][uuid].get('processor', '')
+        processor_classes = next((tpl for tpl in processors.find_processors() if tpl[1] == processor_name), None)
+        if not processor_classes:
+            flash(f"Cannot load the edit form for processor/plugin '{processor_classes[1]}', plugin missing?", 'error')
+            return redirect(url_for('watchlist.index'))
+
+        parent_module = processors.get_parent_module(processor_classes[0])
+
+        try:
+            # Get the parent of the "processor.py" go up one, get the form (kinda spaghetti but its reusing existing code)
+            forms_module = importlib.import_module(f"{parent_module.__name__}.forms")
+            # Access the 'processor_settings_form' class from the 'forms' module
+            form_class = getattr(forms_module, 'processor_settings_form')
+        except ModuleNotFoundError as e:
+            # .forms didnt exist
+            form_class = forms.processor_text_json_diff_form
+        except AttributeError as e:
+            # .forms exists but no useful form
+            form_class = forms.processor_text_json_diff_form
+
+        form = form_class(formdata=request.form if request.method == 'POST' else None,
+                          data=default,
+                          extra_notification_tokens=default.extra_notification_token_values(),
+                          default_system_settings=datastore.data['settings']
+                          )
+
+        # For the form widget tag UUID back to "string name" for the field
+        form.tags.datastore = datastore
+
+        # Used by some forms that need to dig deeper
+        form.datastore = datastore
+        form.watch = default
+
+        for p in datastore.extra_browsers:
+            form.fetch_backend.choices.append(p)
+
+        form.fetch_backend.choices.append(("system", 'System settings default'))
+
+        # form.browser_steps[0] can be assumed that we 'goto url' first
+
+        if datastore.proxy_list is None:
+            # @todo - Couldn't get setattr() etc dynamic addition working, so remove it instead
+            del form.proxy
+        else:
+            form.proxy.choices = [('', 'Default')]
+            for p in datastore.proxy_list:
+                form.proxy.choices.append(tuple((p, datastore.proxy_list[p]['label'])))
+
+
+        if request.method == 'POST' and form.validate():
+
+            # If they changed processor, it makes sense to reset it.
+            if datastore.data['watching'][uuid].get('processor') != form.data.get('processor'):
+                datastore.data['watching'][uuid].clear_watch()
+                flash("Reset watch history due to change of processor")
+
+            extra_update_obj = {
+                'consecutive_filter_failures': 0,
+                'last_error' : False
+            }
+
+            if request.args.get('unpause_on_save'):
+                extra_update_obj['paused'] = False
+
+            extra_update_obj['time_between_check'] = form.time_between_check.data
+
+             # Ignore text
+            form_ignore_text = form.ignore_text.data
+            datastore.data['watching'][uuid]['ignore_text'] = form_ignore_text
+
+            # Be sure proxy value is None
+            if datastore.proxy_list is not None and form.data['proxy'] == '':
+                extra_update_obj['proxy'] = None
+
+            # Unsetting all filter_text methods should make it go back to default
+            # This particularly affects tests running
+            if 'filter_text_added' in form.data and not form.data.get('filter_text_added') \
+                    and 'filter_text_replaced' in form.data and not form.data.get('filter_text_replaced') \
+                    and 'filter_text_removed' in form.data and not form.data.get('filter_text_removed'):
+                extra_update_obj['filter_text_added'] = True
+                extra_update_obj['filter_text_replaced'] = True
+                extra_update_obj['filter_text_removed'] = True
+
+            # Because wtforms doesn't support accessing other data in process_ , but we convert the CSV list of tags back to a list of UUIDs
+            tag_uuids = []
+            if form.data.get('tags'):
+                # Sometimes in testing this can be list, dont know why
+                if type(form.data.get('tags')) == list:
+                    extra_update_obj['tags'] = form.data.get('tags')
+                else:
+                    for t in form.data.get('tags').split(','):
+                        tag_uuids.append(datastore.add_tag(title=t))
+                    extra_update_obj['tags'] = tag_uuids
+
+            datastore.data['watching'][uuid].update(form.data)
+            datastore.data['watching'][uuid].update(extra_update_obj)
+
+            if not datastore.data['watching'][uuid].get('tags'):
+                # Force it to be a list, because form.data['tags'] will be string if nothing found
+                # And del(form.data['tags'] ) wont work either for some reason
+                datastore.data['watching'][uuid]['tags'] = []
+
+            # Recast it if need be to right data Watch handler
+            watch_class = processors.get_custom_watch_obj_for_processor(form.data.get('processor'))
+            datastore.data['watching'][uuid] = watch_class(datastore_path=datastore.datastore_path, default=datastore.data['watching'][uuid])
+            flash("Updated watch - unpaused!" if request.args.get('unpause_on_save') else "Updated watch.")
+
+            # Re #286 - We wait for syncing new data to disk in another thread every 60 seconds
+            # But in the case something is added we should save straight away
+            datastore.needs_write_urgent = True
+
+            # Do not queue on edit if its not within the time range
+
+            # @todo maybe it should never queue anyway on edit...
+            is_in_schedule = True
+            watch = datastore.data['watching'].get(uuid)
+
+            if watch.get('time_between_check_use_default'):
+                time_schedule_limit = datastore.data['settings']['requests'].get('time_schedule_limit', {})
+            else:
+                time_schedule_limit = watch.get('time_schedule_limit')
+
+            tz_name = time_schedule_limit.get('timezone')
+            if not tz_name:
+                tz_name = datastore.data['settings']['application'].get('timezone', 'UTC')
+
+            if time_schedule_limit and time_schedule_limit.get('enabled'):
+                try:
+                    is_in_schedule = is_within_schedule(time_schedule_limit=time_schedule_limit,
+                                                      default_tz=tz_name
+                                                      )
+                except Exception as e:
+                    logger.error(
+                        f"{uuid} - Recheck scheduler, error handling timezone, check skipped - TZ name '{tz_name}' - {str(e)}")
+                    return False
+
+            #############################
+            if not datastore.data['watching'][uuid].get('paused') and is_in_schedule:
+                # Queue the watch for immediate recheck, with a higher priority
+                worker_handler.queue_item_async_safe(update_q, queuedWatchMetaData.PrioritizedItem(priority=1, item={'uuid': uuid}))
+
+            # Diff page [edit] link should go back to diff page
+            if request.args.get("next") and request.args.get("next") == 'diff':
+                return redirect(url_for('ui.ui_views.diff_history_page', uuid=uuid))
+
+            return redirect(url_for('watchlist.index', tag=request.args.get("tag",'')))
+
+        else:
+            if request.method == 'POST' and not form.validate():
+                flash("An error occurred, please see below.", "error")
+
+            # JQ is difficult to install on windows and must be manually added (outside requirements.txt)
+            jq_support = True
+            try:
+                import jq
+            except ModuleNotFoundError:
+                jq_support = False
+
+            watch = datastore.data['watching'].get(uuid)
+
+            # if system or watch is configured to need a chrome type browser
+            system_uses_webdriver = datastore.data['settings']['application']['fetch_backend'] == 'html_webdriver'
+            watch_needs_selenium_or_playwright = False
+            if (watch.get('fetch_backend') == 'system' and system_uses_webdriver) or watch.get('fetch_backend') == 'html_webdriver' or watch.get('fetch_backend', '').startswith('extra_browser_'):
+                watch_needs_selenium_or_playwright = True
+
+
+            from zoneinfo import available_timezones
+
+            # Only works reliably with Playwright
+
+            # Import the global plugin system
+            from changedetectionio.pluggy_interface import collect_ui_edit_stats_extras
+            
+            template_args = {
+                'available_processors': processors.available_processors(),
+                'available_timezones': sorted(available_timezones()),
+                'browser_steps_config': browser_step_ui_config,
+                'emailprefix': os.getenv('NOTIFICATION_MAIL_BUTTON_PREFIX', False),
+                'extra_classes': 'checking-now' if worker_handler.is_watch_running(uuid) else '',
+                'extra_notification_token_placeholder_info': datastore.get_unique_notification_token_placeholders_available(),
+                'extra_processor_config': form.extra_tab_content(),
+                'extra_title': f" - Edit - {watch.label}",
+                'form': form,
+                'has_default_notification_urls': True if len(datastore.data['settings']['application']['notification_urls']) else False,
+                'has_extra_headers_file': len(datastore.get_all_headers_in_textfile_for_watch(uuid=uuid)) > 0,
+                'has_special_tag_options': _watch_has_tag_options_set(watch=watch),
+                'jq_support': jq_support,
+                'playwright_enabled': os.getenv('PLAYWRIGHT_DRIVER_URL', False),
+                'settings_application': datastore.data['settings']['application'],
+                'system_has_playwright_configured': os.getenv('PLAYWRIGHT_DRIVER_URL'),
+                'system_has_webdriver_configured': os.getenv('WEBDRIVER_URL'),
+                'ui_edit_stats_extras': collect_ui_edit_stats_extras(watch),
+                'visual_selector_data_ready': datastore.visualselector_data_is_ready(watch_uuid=uuid),
+                'timezone_default_config': datastore.data['settings']['application'].get('timezone'),
+                'using_global_webdriver_wait': not default['webdriver_delay'],
+                'uuid': uuid,
+                'watch': watch,
+                'watch_needs_selenium_or_playwright': watch_needs_selenium_or_playwright,
+            }
+
+            included_content = None
+            if form.extra_form_content():
+                # So that the extra panels can access _helpers.html etc, we set the environment to load from templates/
+                # And then render the code from the module
+                templates_dir = str(importlib.resources.files("changedetectionio").joinpath('templates'))
+                env = Environment(loader=FileSystemLoader(templates_dir))
+                template = env.from_string(form.extra_form_content())
+                included_content = template.render(**template_args)
+
+            output = render_template("edit.html",
+                                     extra_tab_content=form.extra_tab_content() if form.extra_tab_content() else None,
+                                     extra_form_content=included_content,
+                                     **template_args
+                                     )
+
+        return output
+
+    @edit_blueprint.route("/edit/<string:uuid>/get-html", methods=['GET'])
+    @login_optionally_required
+    def watch_get_latest_html(uuid):
+        from io import BytesIO
+        from flask import send_file
+        import brotli
+
+        watch = datastore.data['watching'].get(uuid)
+        if watch and watch.history.keys() and os.path.isdir(watch.watch_data_dir):
+            latest_filename = list(watch.history.keys())[-1]
+            html_fname = os.path.join(watch.watch_data_dir, f"{latest_filename}.html.br")
+            with open(html_fname, 'rb') as f:
+                if html_fname.endswith('.br'):
+                    # Read and decompress the Brotli file
+                    decompressed_data = brotli.decompress(f.read())
+                else:
+                    decompressed_data = f.read()
+
+            buffer = BytesIO(decompressed_data)
+
+            return send_file(buffer, as_attachment=True, download_name=f"{latest_filename}.html", mimetype='text/html')
+
+        # Return a 500 error
+        abort(500)
+
+    # Ajax callback
+    @edit_blueprint.route("/edit/<string:uuid>/preview-rendered", methods=['POST'])
+    @login_optionally_required
+    def watch_get_preview_rendered(uuid):
+        '''For when viewing the "preview" of the rendered text from inside of Edit'''
+        from flask import jsonify
+        from changedetectionio.processors.text_json_diff import prepare_filter_prevew
+        result = prepare_filter_prevew(watch_uuid=uuid, form_data=request.form, datastore=datastore)
+        return jsonify(result)
+
+    @edit_blueprint.route("/highlight_submit_ignore_url", methods=['POST'])
+    @login_optionally_required
+    def highlight_submit_ignore_url():
+        import re
+        mode = request.form.get('mode')
+        selection = request.form.get('selection')
+
+        uuid = request.args.get('uuid','')
+        if datastore.data["watching"].get(uuid):
+            if mode == 'exact':
+                for l in selection.splitlines():
+                    datastore.data["watching"][uuid]['ignore_text'].append(l.strip())
+            elif mode == 'digit-regex':
+                for l in selection.splitlines():
+                    # Replace any series of numbers with a regex
+                    s = re.escape(l.strip())
+                    s = re.sub(r'[0-9]+', r'\\d+', s)
+                    datastore.data["watching"][uuid]['ignore_text'].append('/' + s + '/')
+
+        return f"<a href={url_for('ui.ui_views.preview_page', uuid=uuid)}>Click to preview</a>"
+    
+    return edit_blueprint

changedetectionio/blueprint/ui/notification.py

@@ -0,0 +1,108 @@
+from flask import Blueprint, request, make_response
+import random
+from loguru import logger
+
+from changedetectionio.store import ChangeDetectionStore
+from changedetectionio.auth_decorator import login_optionally_required
+
+def construct_blueprint(datastore: ChangeDetectionStore):
+    notification_blueprint = Blueprint('ui_notification', __name__, template_folder="../ui/templates")
+    
+    # AJAX endpoint for sending a test
+    @notification_blueprint.route("/notification/send-test/<string:watch_uuid>", methods=['POST'])
+    @notification_blueprint.route("/notification/send-test", methods=['POST'])
+    @notification_blueprint.route("/notification/send-test/", methods=['POST'])
+    @login_optionally_required
+    def ajax_callback_send_notification_test(watch_uuid=None):
+
+        # Watch_uuid could be unset in the case it`s used in tag editor, global settings
+        import apprise
+        from changedetectionio.notification.handler import process_notification
+        from changedetectionio.notification.apprise_plugin.assets import apprise_asset
+
+        from changedetectionio.notification.apprise_plugin.custom_handlers import apprise_http_custom_handler
+
+        apobj = apprise.Apprise(asset=apprise_asset)
+
+        is_global_settings_form = request.args.get('mode', '') == 'global-settings'
+        is_group_settings_form = request.args.get('mode', '') == 'group-settings'
+
+        # Use an existing random one on the global/main settings form
+        if not watch_uuid and (is_global_settings_form or is_group_settings_form) \
+                and datastore.data.get('watching'):
+            logger.debug(f"Send test notification - Choosing random Watch {watch_uuid}")
+            watch_uuid = random.choice(list(datastore.data['watching'].keys()))
+
+        if not watch_uuid:
+            return make_response("Error: You must have atleast one watch configured for 'test notification' to work", 400)
+
+        watch = datastore.data['watching'].get(watch_uuid)
+
+        notification_urls = None
+
+        if request.form.get('notification_urls'):
+            notification_urls = request.form['notification_urls'].strip().splitlines()
+
+        if not notification_urls:
+            logger.debug("Test notification - Trying by group/tag in the edit form if available")
+            # On an edit page, we should also fire off to the tags if they have notifications
+            if request.form.get('tags') and request.form['tags'].strip():
+                for k in request.form['tags'].split(','):
+                    tag = datastore.tag_exists_by_name(k.strip())
+                    notification_urls = tag.get('notifications_urls') if tag and tag.get('notifications_urls') else None
+
+        if not notification_urls and not is_global_settings_form and not is_group_settings_form:
+            # In the global settings, use only what is typed currently in the text box
+            logger.debug("Test notification - Trying by global system settings notifications")
+            if datastore.data['settings']['application'].get('notification_urls'):
+                notification_urls = datastore.data['settings']['application']['notification_urls']
+
+        if not notification_urls:
+            return 'Error: No Notification URLs set/found'
+
+        for n_url in notification_urls:
+            if len(n_url.strip()):
+                if not apobj.add(n_url):
+                    return f'Error:  {n_url} is not a valid AppRise URL.'
+
+        try:
+            # use the same as when it is triggered, but then override it with the form test values
+            n_object = {
+                'watch_url': request.form.get('window_url', "https://changedetection.io"),
+                'notification_urls': notification_urls
+            }
+
+            # Only use if present, if not set in n_object it should use the default system value
+            if 'notification_format' in request.form and request.form['notification_format'].strip():
+                n_object['notification_format'] = request.form.get('notification_format', '').strip()
+
+            if 'notification_title' in request.form and request.form['notification_title'].strip():
+                n_object['notification_title'] = request.form.get('notification_title', '').strip()
+            elif datastore.data['settings']['application'].get('notification_title'):
+                n_object['notification_title'] = datastore.data['settings']['application'].get('notification_title')
+            else:
+                n_object['notification_title'] = "Test title"
+
+            if 'notification_body' in request.form and request.form['notification_body'].strip():
+                n_object['notification_body'] = request.form.get('notification_body', '').strip()
+            elif datastore.data['settings']['application'].get('notification_body'):
+                n_object['notification_body'] = datastore.data['settings']['application'].get('notification_body')
+            else:
+                n_object['notification_body'] = "Test body"
+
+            n_object['as_async'] = False
+            n_object.update(watch.extra_notification_token_values())
+            sent_obj = process_notification(n_object, datastore)
+
+        except Exception as e:
+            e_str = str(e)
+            # Remove this text which is not important and floods the container
+            e_str = e_str.replace(
+                "DEBUG - <class 'apprise.decorators.base.CustomNotifyPlugin.instantiate_plugin.<locals>.CustomNotifyPluginWrapper'>",
+                '')
+
+            return make_response(e_str, 400)
+
+        return 'OK - Sent test notifications'
+
+    return notification_blueprint

changedetectionio/blueprint/ui/templates/clear_all_history.html

@@ -0,0 +1,49 @@
+{% extends 'base.html' %} {% block content %}
+<div class="edit-form">
+  <div class="box-wrap inner">
+    <form
+      class="pure-form pure-form-stacked"
+      action="{{url_for('ui.clear_all_history')}}"
+      method="POST"
+    >
+      <input type="hidden" name="csrf_token" value="{{ csrf_token() }}" >
+      <fieldset>
+        <div class="pure-control-group">
+          This will remove version history (snapshots) for ALL watches, but keep
+          your list of URLs! <br />
+          You may like to use the <strong>BACKUP</strong> link first.<br />
+        </div>
+        <br />
+        <div class="pure-control-group">
+          <label for="confirmtext">Confirmation text</label>
+          <input
+            type="text"
+            id="confirmtext"
+            required=""
+            name="confirmtext"
+            value=""
+            size="10"
+          />
+          <span class="pure-form-message-inline"
+            >Type in the word <strong>clear</strong> to confirm that you
+            understand.</span
+          >
+        </div>
+        <br />
+        <div class="pure-control-group">
+          <button type="submit" class="pure-button pure-button-primary">
+            Clear History!
+          </button>
+        </div>
+        <br />
+        <div class="pure-control-group">
+          <a href="{{url_for('watchlist.index')}}" class="pure-button button-cancel"
+            >Cancel</a
+          >
+        </div>
+      </fieldset>
+    </form>
+  </div>
+</div>
+
+{% endblock %}

changedetectionio/blueprint/ui/templates/edit.html

@@ -0,0 +1,485 @@
+{% extends 'base.html' %}
+{% block content %}
+{% from '_helpers.html' import render_field, render_checkbox_field, render_button, render_time_schedule_form, playwright_warning, only_playwright_type_watches_warning, render_conditions_fieldlist_of_formfields_as_table, render_ternary_field %}
+{% from '_common_fields.html' import render_common_settings_form %}
+<script src="{{url_for('static_content', group='js', filename='tabs.js')}}" defer></script>
+<script src="{{url_for('static_content', group='js', filename='vis.js')}}" defer></script>
+<script src="{{url_for('static_content', group='js', filename='global-settings.js')}}" defer></script>
+<script src="{{url_for('static_content', group='js', filename='scheduler.js')}}" defer></script>
+<script src="{{url_for('static_content', group='js', filename='conditions.js')}}" defer></script>
+
+
+<script>
+    const browser_steps_available_screenshots=JSON.parse('{{ watch.get_browsersteps_available_screenshots|tojson }}');
+    const browser_steps_config=JSON.parse('{{ browser_steps_config|tojson }}');
+    <!-- Should be _external so that firefox and others load it more reliably -->
+    const browser_steps_fetch_screenshot_image_url="{{url_for('browser_steps.browser_steps_fetch_screenshot_image', uuid=uuid, _external=True)}}";
+    const browser_steps_last_error_step={{ watch.browser_steps_last_error_step|tojson }};
+    const browser_steps_start_url="{{url_for('browser_steps.browsersteps_start_session', uuid=uuid)}}";
+    const browser_steps_sync_url="{{url_for('browser_steps.browsersteps_ui_update', uuid=uuid)}}";
+{% if emailprefix %}
+    const email_notification_prefix=JSON.parse('{{ emailprefix|tojson }}');
+{% endif %}
+    const notification_base_url="{{url_for('ui.ui_notification.ajax_callback_send_notification_test', watch_uuid=uuid)}}";
+    const playwright_enabled={% if playwright_enabled %}true{% else %}false{% endif %};
+    const recheck_proxy_start_url="{{url_for('check_proxies.start_check', uuid=uuid)}}";
+    const proxy_recheck_status_url="{{url_for('check_proxies.get_recheck_status', uuid=uuid)}}";
+    const screenshot_url="{{url_for('static_content', group='screenshot', filename=uuid)}}";
+    const watch_visual_selector_data_url="{{url_for('static_content', group='visual_selector_data', filename=uuid)}}";
+    const default_system_fetch_backend="{{ settings_application['fetch_backend'] }}";
+</script>
+<script src="{{url_for('static_content', group='js', filename='plugins.js')}}" defer></script>
+<script src="{{url_for('static_content', group='js', filename='watch-settings.js')}}" defer></script>
+<script src="{{url_for('static_content', group='js', filename='notifications.js')}}" defer></script>
+<script src="{{url_for('static_content', group='js', filename='visual-selector.js')}}" defer></script>
+{% if playwright_enabled %}
+<script src="{{url_for('static_content', group='js', filename='browser-steps.js')}}" defer></script>
+{% endif %}
+
+{% set has_tag_filters_extra="WARNING: Watch has tag/groups set with special filters\n" if has_special_tag_options else '' %}
+<script src="{{url_for('static_content', group='js', filename='recheck-proxy.js')}}" defer></script>
+
+<div class="edit-form monospaced-textarea">
+
+    <div class="tabs collapsable">
+        <ul>
+            <li class="tab"><a href="#general">General</a></li>
+            <li class="tab"><a href="#request">Request</a></li>
+            {% if extra_tab_content %}
+            <li class="tab"><a href="#extras_tab">{{ extra_tab_content }}</a></li>
+            {% endif %}
+            <li class="tab"><a id="browsersteps-tab" href="#browser-steps">Browser Steps</a></li>
+        <!-- should goto extra forms? -->
+            {% if watch['processor'] == 'text_json_diff' %}
+            <li class="tab"><a id="visualselector-tab" href="#visualselector">Visual Filter Selector</a></li>
+            <li class="tab" id="filters-and-triggers-tab"><a href="#filters-and-triggers">Filters &amp; Triggers</a></li>
+            <li class="tab" id="conditions-tab"><a href="#conditions">Conditions</a></li>
+            {% endif %}
+            <li class="tab"><a href="#notifications">Notifications</a></li>
+            <li class="tab"><a href="#stats">Stats</a></li>
+        </ul>
+    </div>
+
+    <div class="box-wrap inner">
+        <form class="pure-form pure-form-stacked"
+              action="{{ url_for('ui.ui_edit.edit_page', uuid=uuid, next = request.args.get('next'), unpause_on_save = request.args.get('unpause_on_save'), tag = request.args.get('tag')) }}" method="POST">
+             <input type="hidden" name="csrf_token" value="{{ csrf_token() }}">
+
+            <div class="tab-pane-inner" id="general">
+                <fieldset>
+                    <div class="pure-control-group">
+                        {{ render_field(form.url, placeholder="https://...", required=true, class="m-d") }}
+                        <div class="pure-form-message">Some sites use JavaScript to create the content, for this you should <a href="https://github.com/dgtlmoon/changedetection.io/wiki/Fetching-pages-with-WebDriver">use the Chrome/WebDriver Fetcher</a></div>
+                        <div class="pure-form-message">Variables are supported in the URL (<a href="https://github.com/dgtlmoon/changedetection.io/wiki/Handling-variables-in-the-watched-URL">help and examples here</a>).</div>
+                    </div>
+                    <div class="pure-control-group">
+                        {{ render_field(form.tags) }}
+                        <span class="pure-form-message-inline">Organisational tag/group name used in the main listing page</span>
+                    </div>
+                    <div class="pure-control-group inline-radio">
+                        {{ render_field(form.processor) }}
+                    </div>
+                    <div class="pure-control-group">
+                        {{ render_field(form.title, class="m-d", placeholder=watch.label) }}
+                        <span class="pure-form-message-inline">Automatically uses the page title if found, you can also use your own title/description here</span>
+                    </div>
+                    <div class="pure-control-group time-between-check border-fieldset">
+
+                        {{ render_checkbox_field(form.time_between_check_use_default, class="use-default-timecheck") }}
+                        <br>
+                        <div id="time-check-widget-wrapper">
+                            {{ render_field(form.time_between_check, class="time-check-widget") }}
+
+                            <span class="pure-form-message-inline">
+                             The interval/amount of time between each check.
+                            </span>
+                        </div>
+                        <div id="time-between-check-schedule">
+                            <!-- Start Time and End Time -->
+                            <div id="limit-between-time">
+                                {{ render_time_schedule_form(form, available_timezones, timezone_default_config) }}
+                            </div>
+                        </div>
+<br>
+              </div>
+
+                    <div class="pure-control-group">
+                        {{ render_checkbox_field(form.filter_failure_notification_send) }}
+                        <span class="pure-form-message-inline">
+                         Sends a notification when the filter can no longer be seen on the page, good for knowing when the page changed and your filter will not work anymore.
+                        </span>
+                    </div>
+                    <div class="pure-control-group">
+                        {{ render_ternary_field(form.use_page_title_in_list) }}
+                    </div>
+                </fieldset>
+            </div>
+
+            <div class="tab-pane-inner" id="request">
+                    <div class="pure-control-group inline-radio">
+                        {{ render_field(form.fetch_backend, class="fetch-backend") }}
+                        <span class="pure-form-message-inline">
+                            <p>Use the <strong>Basic</strong> method (default) where your watched site doesn't need Javascript to render.</p>
+                            <p>The <strong>Chrome/Javascript</strong> method requires a network connection to a running WebDriver+Chrome server, set by the ENV var 'WEBDRIVER_URL'. </p>
+                            Tip: <a href="https://github.com/dgtlmoon/changedetection.io/wiki/Proxy-configuration#brightdata-proxy-support">Connect using Bright Data and Oxylabs Proxies, find out more here.</a>
+                        </span>
+                    </div>
+                {% if form.proxy %}
+                    <div class="pure-control-group inline-radio">
+                          <div>{{ form.proxy.label }} <a href="" id="check-all-proxies" class="pure-button button-secondary button-xsmall" >Check/Scan all</a></div>
+                          <div>{{ form.proxy(class="fetch-backend-proxy") }}</div>
+                        <span class="pure-form-message-inline">
+                        Choose a proxy for this watch
+                        </span>
+                    </div>
+                {% endif %}
+
+                <!-- webdriver always -->
+                <fieldset data-visible-for="fetch_backend=html_webdriver"  style="display: none;">
+                    <div class="pure-control-group">
+                        {{ render_field(form.webdriver_delay) }}
+                        <div class="pure-form-message-inline">
+                            <strong>If you're having trouble waiting for the page to be fully rendered (text missing etc), try increasing the 'wait' time here.</strong>
+                            <br>
+                            This will wait <i>n</i> seconds before extracting the text.
+                            {% if using_global_webdriver_wait %}
+                            <br><strong>Using the current global default settings</strong>
+                            {% endif %}
+                        </div>
+                    </div>
+                    <div class="pure-control-group">
+                        <a class="pure-button button-secondary button-xsmall show-advanced">Show advanced options</a>
+                    </div>
+                    <div class="advanced-options"  style="display: none;">
+                        {{ render_field(form.webdriver_js_execute_code) }}
+                        <div class="pure-form-message-inline">
+                            Run this code before performing change detection, handy for filling in fields and other
+                            actions <a
+                                href="https://github.com/dgtlmoon/changedetection.io/wiki/Run-JavaScript-before-change-detection">More
+                            help and examples here</a>
+                        </div>
+                    </div>
+                </fieldset>
+                <!-- html requests always -->
+                <fieldset data-visible-for="fetch_backend=html_requests">
+                    <div class="pure-control-group">
+                        <a class="pure-button button-secondary button-xsmall show-advanced">Show advanced options</a>
+                    </div>
+                    <div class="advanced-options"  style="display: none;">
+                        <div class="pure-control-group" id="request-method">
+                            {{ render_field(form.method) }}
+                        </div>
+                        <div id="request-body">
+                                            {{ render_field(form.body, rows=7, placeholder="Example
+{
+   \"name\":\"John\",
+   \"age\":30,
+   \"car\":null,
+   \"year\":{% now 'Europe/Berlin', '%Y' %}
+}") }}
+                        </div>
+                        <div class="pure-form-message">Variables are supported in the request body (<a href="https://github.com/dgtlmoon/changedetection.io/wiki/Handling-variables-in-the-watched-URL">help and examples here</a>).</div>
+                    </div>
+                </fieldset>
+            <!-- hmm -->
+                <div class="pure-control-group advanced-options"  style="display: none;">
+                    {{ render_field(form.headers, rows=7, placeholder="Example
+Cookie: foobar
+User-Agent: wonderbra 1.0
+Math: {{ 1 + 1 }}") }}
+                        <div class="pure-form-message">Variables are supported in the request header values (<a href="https://github.com/dgtlmoon/changedetection.io/wiki/Handling-variables-in-the-watched-URL">help and examples here</a>).</div>
+                        <div class="pure-form-message-inline">
+                            {% if has_extra_headers_file %}
+                                <strong>Alert! Extra headers file found and will be added to this watch!</strong>
+                            {% else %}
+                                Headers can be also read from a file in your data-directory <a href="https://github.com/dgtlmoon/changedetection.io/wiki/Adding-headers-from-an-external-file">Read more here</a>
+                            {% endif %}
+                            <br>
+                            (Not supported by Selenium browser)
+                        </div>
+                    </div>
+            <fieldset data-visible-for="fetch_backend=html_requests fetch_backend=html_webdriver" >
+                    <div class="pure-control-group inline-radio advanced-options"  style="display: none;">
+                    {{ render_checkbox_field(form.ignore_status_codes) }}
+                    </div>
+            </fieldset>
+            </div>
+
+            <div class="tab-pane-inner" id="browser-steps">
+            {% if watch_needs_selenium_or_playwright %}
+                {# Only works with playwright #}
+                {% if system_has_playwright_configured %}
+                <img class="beta-logo" src="{{url_for('static_content', group='images', filename='beta-logo.png')}}" alt="New beta functionality">
+                <fieldset>
+                    <div class="pure-control-group">
+                        <!--
+                        Too hard right now, better to just send the events to the fetcher for now and leave it in the final screenshot
+                        and/or report an error
+                        <a id="play-steps" class="pure-button button-secondary button-xsmall" style="font-size: 70%">Play steps  ▶</a>
+                        -->
+
+                        <!---  Do this later -->
+                        <div class="checkbox" style="display: none;">
+                            <input type=checkbox id="include_text_elements" > <label for="include_text_elements">Turn on text finder</label>
+                        </div>
+
+                        <div id="loading-status-text" style="display: none;">Please wait, first browser step can take a little time to load..<div class="spinner"></div></div>
+                        <div class="flex-wrapper" >
+
+                            <div id="browser-steps-ui" class="noselect">
+                                <div class="noselect"  id="browsersteps-selector-wrapper" style="width: 100%">
+                                    <span class="loader" >
+                                        <span id="browsersteps-click-start">
+                                            <h2 >Click here to Start</h2>
+                                            <svg style="height: 3.5rem;" version="1.1" viewBox="0 0 32 32"  xml:space="preserve" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink"><g id="start"/><g id="play_x5F_alt"><path d="M16,0C7.164,0,0,7.164,0,16s7.164,16,16,16s16-7.164,16-16S24.836,0,16,0z M10,24V8l16.008,8L10,24z" style="fill: var(--color-grey-400);"/></g></svg><br>
+                                            Please allow 10-15 seconds for the browser to connect.<br>
+                                        </span>
+                                        <div class="spinner"  style="display: none;"></div>
+                                    </span>
+                                    <img class="noselect" id="browsersteps-img" src="" style="max-width: 100%; width: 100%;" >
+                                    <canvas  class="noselect" id="browsersteps-selector-canvas" style="max-width: 100%; width: 100%;"></canvas>
+                                </div>
+                            </div>
+                            <div id="browser-steps-fieldlist" >
+                                <span id="browser-seconds-remaining">Press "Play" to start.</span> <span style="font-size: 80%;"> (<a target="newwindow" href="https://github.com/dgtlmoon/changedetection.io/pull/478/files#diff-1a79d924d1840c485238e66772391268a89c95b781d69091384cf1ea1ac146c9R4">?</a>) </span>
+                                {{ render_field(form.browser_steps) }}
+                            </div>
+                        </div>
+                    </div>
+                </fieldset>
+                {% else %}
+                    {# it's configured to use selenium or chrome but system says its not configured #}
+                    {{ playwright_warning() }}
+                    {% if system_has_webdriver_configured %}
+                        <strong>Selenium/Webdriver cant be used here because it wont fetch screenshots reliably.</strong>
+                    {% endif %}
+                {% endif %}
+            {% else %}
+                {# "This functionality needs chrome.." #}
+                {{ only_playwright_type_watches_warning() }}
+            {% endif %}
+            </div>
+
+
+            <div class="tab-pane-inner" id="notifications">
+                <fieldset>
+                    <div  class="pure-control-group inline-radio">
+                      {{ render_ternary_field(form.notification_muted, BooleanField=true) }}
+                    </div>
+                    {% if watch_needs_selenium_or_playwright %}
+                    <div class="pure-control-group inline-radio">
+                      {{ render_checkbox_field(form.notification_screenshot) }}
+                        <span class="pure-form-message-inline">
+                            <strong>Use with caution!</strong> This will easily fill up your email storage quota or flood other storages.
+                        </span>
+                    </div>
+                    {% endif %}
+                    <div class="field-group" id="notification-field-group">
+                        {% if has_default_notification_urls %}
+                        <div class="inline-warning">
+                            <img class="inline-warning-icon" src="{{url_for('static_content', group='images', filename='notice.svg')}}" alt="Look out!" title="Lookout!" >
+                            There are <a href="{{ url_for('settings.settings_page')}}#notifications">system-wide notification URLs enabled</a>, this form will override notification settings for this watch only &dash; an empty Notification URL list here will still send notifications.
+                        </div>
+                        {% endif %}
+                        <a href="#notifications" id="notification-setting-reset-to-default" class="pure-button button-xsmall" style="right: 20px; top: 20px; position: absolute; background-color: #5f42dd; border-radius: 4px; font-size: 70%; color: #fff">Use system defaults</a>
+                        {{ render_common_settings_form(form, emailprefix, settings_application, extra_notification_token_placeholder_info) }}
+                    </div>
+                </fieldset>
+            </div>
+
+            {% if watch['processor'] == 'text_json_diff' %}
+
+            <div class="tab-pane-inner" id="conditions">
+                    <script>
+                        const verify_condition_rule_url="{{url_for('conditions.verify_condition_single_rule', watch_uuid=uuid)}}";
+                    </script>
+                <div class="pure-control-group">
+                    {{ render_field(form.conditions_match_logic) }}
+                    {{ render_conditions_fieldlist_of_formfields_as_table(form.conditions) }}
+                    <div class="pure-form-message-inline">
+
+                        <p id="verify-state-text">Use the verify (✓) button to test if a condition passes against the current snapshot.</p>
+                       Read a quick tutorial about <a href="https://changedetection.io/tutorial/conditional-actions-web-page-changes">using conditional web page changes here</a>.<br>
+                    </div>
+                </div>
+            </div>
+            <div class="tab-pane-inner" id="filters-and-triggers">
+                <span id="activate-text-preview" class="pure-button pure-button-primary button-xsmall">Activate preview</span>
+              <div>
+              <div id="edit-text-filter">
+                    <div class="pure-control-group" id="pro-tips">
+                            <strong>Pro-tips:</strong><br>
+                            <ul>
+                                <li>
+                                    Use the preview page to see your filters and triggers highlighted.
+                                </li>
+                                <li>
+                                    Some sites use JavaScript to create the content, for this you should <a href="https://github.com/dgtlmoon/changedetection.io/wiki/Fetching-pages-with-WebDriver">use the Chrome/WebDriver Fetcher</a>
+                                </li>
+                            </ul>
+                    </div>
+
+{% include "edit/include_subtract.html" %}
+                <div class="text-filtering border-fieldset">
+                <fieldset class="pure-group" id="text-filtering-type-options">
+                    <h3>Text filtering</h3>
+                        Limit trigger/ignore/block/extract to;<br>
+                        {{ render_checkbox_field(form.filter_text_added) }}
+                        {{ render_checkbox_field(form.filter_text_replaced) }}
+                        {{ render_checkbox_field(form.filter_text_removed) }}
+                    <span class="pure-form-message-inline">Note: Depending on the length and similarity of the text on each line, the algorithm may consider an <strong>addition</strong> instead of <strong>replacement</strong> for example.</span><br>
+                    <span class="pure-form-message-inline">&nbsp;So it's always better to select <strong>Added</strong>+<strong>Replaced</strong> when you're interested in new content.</span><br>
+                    <span class="pure-form-message-inline">&nbsp;When content is merely moved in a list, it will also trigger an <strong>addition</strong>, consider enabling <code><strong>Only trigger when unique lines appear</strong></code></span>
+                </fieldset>
+                <fieldset class="pure-control-group">
+                    {{ render_checkbox_field(form.check_unique_lines) }}
+                    <span class="pure-form-message-inline">Good for websites that just move the content around, and you want to know when NEW content is added, compares new lines against all history for this watch.</span>
+                </fieldset>
+                <fieldset class="pure-control-group">
+                    {{ render_checkbox_field(form.remove_duplicate_lines) }}
+                    <span class="pure-form-message-inline">Remove duplicate lines of text</span>
+                </fieldset>
+                <fieldset class="pure-control-group">
+                    {{ render_checkbox_field(form.sort_text_alphabetically) }}
+                    <span class="pure-form-message-inline">Helps reduce changes detected caused by sites shuffling lines around, combine with <i>check unique lines</i> below.</span>
+                </fieldset>
+                <fieldset class="pure-control-group">
+                    {{ render_checkbox_field(form.trim_text_whitespace) }}
+                    <span class="pure-form-message-inline">Remove any whitespace before and after each line of text</span>
+                </fieldset>
+                {% include "edit/text-options.html" %}
+                </div>
+              </div>
+              <div id="text-preview" style="display: none;" >
+                    <script>
+                        const preview_text_edit_filters_url="{{url_for('ui.ui_edit.watch_get_preview_rendered', uuid=uuid)}}";
+                    </script>
+                    <br>
+                    {#<div id="text-preview-controls"><span id="text-preview-refresh" class="pure-button button-xsmall">Refresh</span></div>#}
+                    <div class="minitabs-wrapper">
+                      <div class="minitabs-content">
+                          <div id="text-preview-inner" class="monospace-preview">
+                              <p>Loading...</p>
+                          </div>
+                          <div id="text-preview-before-inner" style="display: none;" class="monospace-preview">
+                              <p>Loading...</p>
+                          </div>
+                      </div>
+                    </div>
+            </div>
+          </div>
+        </div>
+
+        {% endif %}
+        {# rendered sub Template #}
+        {% if extra_form_content %}
+            <div class="tab-pane-inner" id="extras_tab">
+            {{ extra_form_content|safe }}
+            </div>
+        {% endif %}
+            {% if watch['processor'] == 'text_json_diff' %}
+            <div class="tab-pane-inner visual-selector-ui" id="visualselector">
+                <img class="beta-logo" src="{{url_for('static_content', group='images', filename='beta-logo.png')}}" alt="New beta functionality">
+
+                <fieldset>
+                    <div class="pure-control-group">
+                        {% if watch_needs_selenium_or_playwright %}
+                            {% if system_has_playwright_configured %}
+                            <span class="pure-form-message-inline" id="visual-selector-heading">
+                                The Visual Selector tool lets you select the <i>text</i> elements that will be used for the change detection. It automatically fills-in the filters in the "CSS/JSONPath/JQ/XPath Filters" box of the <a href="#filters-and-triggers">Filters & Triggers</a> tab. Use <strong>Shift+Click</strong> to select multiple items.
+                            </span>
+
+                            <div id="selector-header">
+                                <a id="clear-selector" class="pure-button button-secondary button-xsmall" style="font-size: 70%">Clear selection</a>
+                                <!-- visual selector IMG will try to load, it will either replace this or on error replace it with some handy text -->
+                                <i class="fetching-update-notice" style="font-size: 80%;">One moment, fetching screenshot and element information..</i>
+                            </div>
+                            <div id="selector-wrapper" style="display: none">
+                                <!-- request the screenshot and get the element offset info ready -->
+                                <!-- use img src ready load to know everything is ready to map out -->
+                                <!-- @todo: maybe something interesting like a field to select 'elements that contain text... and their parents n' -->
+                                <img id="selector-background" >
+                                <canvas id="selector-canvas"></canvas>
+                            </div>
+                            <div id="selector-current-xpath" style="overflow-x: hidden"><strong>Currently:</strong>&nbsp;<span class="text">Loading...</span></div>
+                        {% else %}
+                            {# The watch needed chrome but system says that playwright is not ready #}
+                            {{ playwright_warning() }}
+                        {% endif %}
+                            {% if system_has_webdriver_configured %}
+                                <strong>Selenium/Webdriver cant be used here because it wont fetch screenshots reliably.</strong>
+                            {% endif %}
+                    {% else %}
+                        {# "This functionality needs chrome.." #}
+                        {{ only_playwright_type_watches_warning() }}
+                    {% endif %}
+                    </div>
+                </fieldset>
+            </div>
+            {% endif %}
+            <div class="tab-pane-inner" id="stats">
+                <div class="pure-control-group">
+                    <style>
+                    #stats-table tr > td:first-child {
+                        font-weight: bold;
+                    }
+                    </style>
+                    <table class="pure-table" id="stats-table">
+                        <tbody>
+                        <tr>
+                            <td>Check count</td>
+                            <td>{{ "{:,}".format( watch.check_count) }}</td>
+                        </tr>
+                        <tr>
+                            <td>Consecutive filter failures</td>
+                            <td>{{ "{:,}".format( watch.consecutive_filter_failures) }}</td>
+                        </tr>
+                        <tr>
+                            <td>History length</td>
+                            <td>{{ "{:,}".format(watch.history|length) }}</td>
+                        </tr>
+                        <tr>
+                            <td>Last fetch duration</td>
+                            <td>{{ watch.fetch_time }}s</td>
+                        </tr>
+                        <tr>
+                            <td>Notification alert count</td>
+                            <td>{{ watch.notification_alert_count }}</td>
+                        </tr>
+                        <tr>
+                            <td>Server type reply</td>
+                            <td>{{ watch.get('remote_server_reply') }}</td>
+                        </tr>
+                        </tbody>
+                    </table>
+
+                    {% if ui_edit_stats_extras %}
+                    <div class="plugin-stats-extras"> <!-- from pluggy plugin -->
+                        {{ ui_edit_stats_extras|safe }}
+                    </div>
+                    {% endif %}
+
+                    {% if watch.history_n %}
+                        <p>
+                             <a href="{{url_for('ui.ui_edit.watch_get_latest_html', uuid=uuid)}}" class="pure-button button-small">Download latest HTML snapshot</a>
+                        </p>
+                    {% endif %}
+
+                </div>
+            </div>
+            <div id="actions">
+                <div class="pure-control-group">
+                    {{ render_button(form.save_button) }}
+                    <a href="{{url_for('ui.form_delete', uuid=uuid)}}"
+                       class="pure-button button-error ">Delete</a>
+                    {% if watch.history_n %}<a href="{{url_for('ui.clear_watch_history', uuid=uuid)}}"
+                       class="pure-button button-error">Clear History</a>{% endif %}
+                    <a href="{{url_for('ui.form_clone', uuid=uuid)}}"
+                       class="pure-button">Clone &amp; Edit</a>
+                </div>
+            </div>
+        </form>
+    </div>
+</div>
+
+{% endblock %}

changedetectionio/blueprint/ui/views.py

@@ -0,0 +1,236 @@
+from flask import Blueprint, request, redirect, url_for, flash, render_template, make_response, send_from_directory, abort
+import os
+import time
+from loguru import logger
+
+from changedetectionio.store import ChangeDetectionStore
+from changedetectionio.auth_decorator import login_optionally_required
+from changedetectionio import html_tools
+from changedetectionio import worker_handler
+
+def construct_blueprint(datastore: ChangeDetectionStore, update_q, queuedWatchMetaData, watch_check_update):
+    views_blueprint = Blueprint('ui_views', __name__, template_folder="../ui/templates")
+    
+    @views_blueprint.route("/preview/<string:uuid>", methods=['GET'])
+    @login_optionally_required
+    def preview_page(uuid):
+        content = []
+        versions = []
+        timestamp = None
+
+        # More for testing, possible to return the first/only
+        if uuid == 'first':
+            uuid = list(datastore.data['watching'].keys()).pop()
+
+        try:
+            watch = datastore.data['watching'][uuid]
+        except KeyError:
+            flash("No history found for the specified link, bad link?", "error")
+            return redirect(url_for('watchlist.index'))
+
+        system_uses_webdriver = datastore.data['settings']['application']['fetch_backend'] == 'html_webdriver'
+        extra_stylesheets = [url_for('static_content', group='styles', filename='diff.css')]
+
+        is_html_webdriver = False
+        if (watch.get('fetch_backend') == 'system' and system_uses_webdriver) or watch.get('fetch_backend') == 'html_webdriver' or watch.get('fetch_backend', '').startswith('extra_browser_'):
+            is_html_webdriver = True
+        triggered_line_numbers = []
+        if datastore.data['watching'][uuid].history_n == 0 and (watch.get_error_text() or watch.get_error_snapshot()):
+            flash("Preview unavailable - No fetch/check completed or triggers not reached", "error")
+        else:
+            # So prepare the latest preview or not
+            preferred_version = request.args.get('version')
+            versions = list(watch.history.keys())
+            timestamp = versions[-1]
+            if preferred_version and preferred_version in versions:
+                timestamp = preferred_version
+
+            try:
+                versions = list(watch.history.keys())
+                content = watch.get_history_snapshot(timestamp)
+
+                triggered_line_numbers = html_tools.strip_ignore_text(content=content,
+                                                                      wordlist=watch['trigger_text'],
+                                                                      mode='line numbers'
+                                                                      )
+
+            except Exception as e:
+                content.append({'line': f"File doesnt exist or unable to read timestamp {timestamp}", 'classes': ''})
+
+        output = render_template("preview.html",
+                                 content=content,
+                                 current_version=timestamp,
+                                 history_n=watch.history_n,
+                                 extra_stylesheets=extra_stylesheets,
+                                 extra_title=f" - Diff - {watch.label} @ {timestamp}",
+                                 triggered_line_numbers=triggered_line_numbers,
+                                 current_diff_url=watch['url'],
+                                 screenshot=watch.get_screenshot(),
+                                 watch=watch,
+                                 uuid=uuid,
+                                 is_html_webdriver=is_html_webdriver,
+                                 last_error=watch['last_error'],
+                                 last_error_text=watch.get_error_text(),
+                                 last_error_screenshot=watch.get_error_snapshot(),
+                                 versions=versions
+                                )
+
+        return output
+
+    @views_blueprint.route("/diff/<string:uuid>", methods=['POST'])
+    @login_optionally_required
+    def diff_history_page_build_report(uuid):
+        from changedetectionio import forms
+
+        # More for testing, possible to return the first/only
+        if uuid == 'first':
+            uuid = list(datastore.data['watching'].keys()).pop()
+
+        try:
+            watch = datastore.data['watching'][uuid]
+        except KeyError:
+            flash("No history found for the specified link, bad link?", "error")
+            return redirect(url_for('watchlist.index'))
+
+        # For submission of requesting an extract
+        extract_form = forms.extractDataForm(formdata=request.form,
+                                             data={'extract_regex': request.form.get('extract_regex', '')}
+                                             )
+        if not extract_form.validate():
+            flash("An error occurred, please see below.", "error")
+            return _render_diff_template(uuid, extract_form)
+
+        else:
+            extract_regex = request.form.get('extract_regex', '').strip()
+            output = watch.extract_regex_from_all_history(extract_regex)
+            if output:
+                watch_dir = os.path.join(datastore.datastore_path, uuid)
+                response = make_response(send_from_directory(directory=watch_dir, path=output, as_attachment=True))
+                response.headers['Content-type'] = 'text/csv'
+                response.headers['Cache-Control'] = 'no-cache, no-store, must-revalidate'
+                response.headers['Pragma'] = 'no-cache'
+                response.headers['Expires'] = "0"
+                return response
+
+            flash('No matches found while scanning all of the watch history for that RegEx.', 'error')
+        return redirect(url_for('ui.ui_views.diff_history_page', uuid=uuid) + '#extract')
+
+    def _render_diff_template(uuid, extract_form=None):
+        """Helper function to render the diff template with all required data"""
+        from changedetectionio import forms
+
+        # More for testing, possible to return the first/only
+        if uuid == 'first':
+            uuid = list(datastore.data['watching'].keys()).pop()
+
+        extra_stylesheets = [url_for('static_content', group='styles', filename='diff.css')]
+        try:
+            watch = datastore.data['watching'][uuid]
+        except KeyError:
+            flash("No history found for the specified link, bad link?", "error")
+            return redirect(url_for('watchlist.index'))
+
+        # Use provided form or create a new one
+        if extract_form is None:
+            extract_form = forms.extractDataForm(formdata=request.form,
+                                                 data={'extract_regex': request.form.get('extract_regex', '')}
+                                                 )
+
+        history = watch.history
+        dates = list(history.keys())
+
+        # If a "from_version" was requested, then find it (or the closest one)
+        # Also set "from version" to be the closest version to the one that was last viewed.
+
+        best_last_viewed_timestamp = watch.get_from_version_based_on_last_viewed
+        from_version_timestamp = best_last_viewed_timestamp if best_last_viewed_timestamp else dates[-2]
+        from_version = request.args.get('from_version', from_version_timestamp )
+
+        # Use the current one if nothing was specified
+        to_version = request.args.get('to_version', str(dates[-1]))
+
+        try:
+            to_version_file_contents = watch.get_history_snapshot(timestamp=to_version)
+        except Exception as e:
+            logger.error(f"Unable to read watch history to-version for version {to_version}: {str(e)}")
+            to_version_file_contents = f"Unable to read to-version at {to_version}.\n"
+
+        try:
+            from_version_file_contents = watch.get_history_snapshot(timestamp=from_version)
+        except Exception as e:
+            logger.error(f"Unable to read watch history from-version for version {from_version}: {str(e)}")
+            from_version_file_contents = f"Unable to read to-version {from_version}.\n"
+
+        screenshot_url = watch.get_screenshot()
+
+        system_uses_webdriver = datastore.data['settings']['application']['fetch_backend'] == 'html_webdriver'
+
+        is_html_webdriver = False
+        if (watch.get('fetch_backend') == 'system' and system_uses_webdriver) or watch.get('fetch_backend') == 'html_webdriver' or watch.get('fetch_backend', '').startswith('extra_browser_'):
+            is_html_webdriver = True
+
+        password_enabled_and_share_is_off = False
+        if datastore.data['settings']['application'].get('password') or os.getenv("SALTED_PASS", False):
+            password_enabled_and_share_is_off = not datastore.data['settings']['application'].get('shared_diff_access')
+
+        datastore.set_last_viewed(uuid, time.time())
+
+        return render_template("diff.html",
+                                 current_diff_url=watch['url'],
+                                 from_version=str(from_version),
+                                 to_version=str(to_version),
+                                 extra_stylesheets=extra_stylesheets,
+                                 extra_title=f" - Diff - {watch.label}",
+                                 extract_form=extract_form,
+                                 is_html_webdriver=is_html_webdriver,
+                                 last_error=watch['last_error'],
+                                 last_error_screenshot=watch.get_error_snapshot(),
+                                 last_error_text=watch.get_error_text(),
+                                 left_sticky=True,
+                                 newest=to_version_file_contents,
+                                 newest_version_timestamp=dates[-1],
+                                 password_enabled_and_share_is_off=password_enabled_and_share_is_off,
+                                 from_version_file_contents=from_version_file_contents,
+                                 to_version_file_contents=to_version_file_contents,
+                                 screenshot=screenshot_url,
+                                 uuid=uuid,
+                                 versions=dates, # All except current/last
+                                 watch_a=watch
+                                 )
+
+    @views_blueprint.route("/diff/<string:uuid>", methods=['GET'])
+    @login_optionally_required
+    def diff_history_page(uuid):
+        return _render_diff_template(uuid)
+
+    @views_blueprint.route("/form/add/quickwatch", methods=['POST'])
+    @login_optionally_required
+    def form_quick_watch_add():
+        from changedetectionio import forms
+        form = forms.quickWatchForm(request.form)
+
+        if not form.validate():
+            for widget, l in form.errors.items():
+                flash(','.join(l), 'error')
+            return redirect(url_for('watchlist.index'))
+
+        url = request.form.get('url').strip()
+        if datastore.url_exists(url):
+            flash(f'Warning, URL {url} already exists', "notice")
+
+        add_paused = request.form.get('edit_and_watch_submit_button') != None
+        processor = request.form.get('processor', 'text_json_diff')
+        new_uuid = datastore.add_watch(url=url, tag=request.form.get('tags').strip(), extras={'paused': add_paused, 'processor': processor})
+
+        if new_uuid:
+            if add_paused:
+                flash('Watch added in Paused state, saving will unpause.')
+                return redirect(url_for('ui.ui_edit.edit_page', uuid=new_uuid, unpause_on_save=1, tag=request.args.get('tag')))
+            else:
+                # Straight into the queue.
+                worker_handler.queue_item_async_safe(update_q, queuedWatchMetaData.PrioritizedItem(priority=1, item={'uuid': new_uuid}))
+                flash("Watch added.")
+
+        return redirect(url_for('watchlist.index', tag=request.args.get('tag','')))
+
+    return views_blueprint

changedetectionio/blueprint/watchlist/__init__.py

@@ -0,0 +1,116 @@
+import os
+import time
+
+from flask import Blueprint, request, make_response, render_template, redirect, url_for, flash, session
+from flask_login import current_user
+from flask_paginate import Pagination, get_page_parameter
+
+from changedetectionio import forms
+from changedetectionio.store import ChangeDetectionStore
+from changedetectionio.auth_decorator import login_optionally_required
+
+def construct_blueprint(datastore: ChangeDetectionStore, update_q, queuedWatchMetaData):
+    watchlist_blueprint = Blueprint('watchlist', __name__, template_folder="templates")
+    
+    @watchlist_blueprint.route("/", methods=['GET'])
+    @login_optionally_required
+    def index():
+        active_tag_req = request.args.get('tag', '').lower().strip()
+        active_tag_uuid = active_tag = None
+
+        # Be sure limit_tag is a uuid
+        if active_tag_req:
+            for uuid, tag in datastore.data['settings']['application'].get('tags', {}).items():
+                if active_tag_req == tag.get('title', '').lower().strip() or active_tag_req == uuid:
+                    active_tag = tag
+                    active_tag_uuid = uuid
+                    break
+
+        # Redirect for the old rss path which used the /?rss=true
+        if request.args.get('rss'):
+            return redirect(url_for('rss.feed', tag=active_tag_uuid))
+
+        op = request.args.get('op')
+        if op:
+            uuid = request.args.get('uuid')
+            if op == 'pause':
+                datastore.data['watching'][uuid].toggle_pause()
+            elif op == 'mute':
+                datastore.data['watching'][uuid].toggle_mute()
+
+            datastore.needs_write = True
+            return redirect(url_for('watchlist.index', tag = active_tag_uuid))
+
+        # Sort by last_changed and add the uuid which is usually the key..
+        sorted_watches = []
+        with_errors = request.args.get('with_errors') == "1"
+        unread_only = request.args.get('unread') == "1"
+        errored_count = 0
+        search_q = request.args.get('q').strip().lower() if request.args.get('q') else False
+        for uuid, watch in datastore.data['watching'].items():
+            if with_errors and not watch.get('last_error'):
+                continue
+
+            if unread_only and (watch.viewed or watch.last_changed == 0) :
+                continue
+
+            if active_tag_uuid and not active_tag_uuid in watch['tags']:
+                    continue
+            if watch.get('last_error'):
+                errored_count += 1
+
+            if search_q:
+                if (watch.get('title') and search_q in watch.get('title').lower()) or search_q in watch.get('url', '').lower():
+                    sorted_watches.append(watch)
+                elif watch.get('last_error') and search_q in watch.get('last_error').lower():
+                    sorted_watches.append(watch)
+            else:
+                sorted_watches.append(watch)
+
+        form = forms.quickWatchForm(request.form)
+        page = request.args.get(get_page_parameter(), type=int, default=1)
+        total_count = len(sorted_watches)
+
+        pagination = Pagination(page=page,
+                                total=total_count,
+                                per_page=datastore.data['settings']['application'].get('pager_size', 50), css_framework="semantic")
+
+        sorted_tags = sorted(datastore.data['settings']['application'].get('tags').items(), key=lambda x: x[1]['title'])
+
+        output = render_template(
+            "watch-overview.html",
+            active_tag=active_tag,
+            active_tag_uuid=active_tag_uuid,
+            app_rss_token=datastore.data['settings']['application'].get('rss_access_token'),
+            datastore=datastore,
+            errored_count=errored_count,
+            form=form,
+            guid=datastore.data['app_guid'],
+            has_proxies=datastore.proxy_list,
+            hosted_sticky=os.getenv("SALTED_PASS", False) == False,
+            now_time_server=round(time.time()),
+            pagination=pagination,
+            queued_uuids=[q_uuid.item['uuid'] for q_uuid in update_q.queue],
+            search_q=request.args.get('q', '').strip(),
+            sort_attribute=request.args.get('sort') if request.args.get('sort') else request.cookies.get('sort'),
+            sort_order=request.args.get('order') if request.args.get('order') else request.cookies.get('order'),
+            system_default_fetcher=datastore.data['settings']['application'].get('fetch_backend'),
+            tags=sorted_tags,
+            unread_changes_count=datastore.unread_changes_count,
+            watches=sorted_watches
+        )
+
+        if session.get('share-link'):
+            del (session['share-link'])
+
+        resp = make_response(output)
+
+        # The template can run on cookie or url query info
+        if request.args.get('sort'):
+            resp.set_cookie('sort', request.args.get('sort'))
+        if request.args.get('order'):
+            resp.set_cookie('order', request.args.get('order'))
+
+        return resp
+        
+    return watchlist_blueprint

changedetectionio/blueprint/watchlist/templates/watch-overview.html

@@ -0,0 +1,273 @@
+{%- extends 'base.html' -%}
+{%- block content -%}
+{%- from '_helpers.html' import render_simple_field, render_field, render_nolabel_field, sort_by_title -%}
+<script src="{{url_for('static_content', group='js', filename='jquery-3.6.0.min.js')}}"></script>
+<script src="{{url_for('static_content', group='js', filename='watch-overview.js')}}" defer></script>
+<script>let nowtimeserver={{ now_time_server }};</script>
+<script>let favicon_baseURL="{{ url_for('static_content', group='favicon', filename="PLACEHOLDER")}}";</script>
+<script>
+// Initialize Feather icons after the page loads
+document.addEventListener('DOMContentLoaded', function() {
+    feather.replace();
+});
+</script>
+<style>
+.checking-now .last-checked {
+    background-image: linear-gradient(to bottom, transparent 0%, rgba(0,0,0,0.05) 40%, rgba(0,0,0,0.1) 100%);
+    background-size: 0 100%;
+    background-repeat: no-repeat;
+    transition: background-size 0.9s ease
+}
+</style>
+<div class="box" id="form-quick-watch-add">
+
+    <form class="pure-form" action="{{ url_for('ui.ui_views.form_quick_watch_add', tag=active_tag_uuid) }}" method="POST" id="new-watch-form">
+        <input type="hidden" name="csrf_token" value="{{ csrf_token() }}" >
+        <fieldset>
+            <legend>Add a new web page change detection watch</legend>
+            <div id="watch-add-wrapper-zone">
+                    {{ render_nolabel_field(form.url, placeholder="https://...", required=true) }}
+                    {{ render_nolabel_field(form.watch_submit_button, title="Watch this URL!" ) }}
+                    {{ render_nolabel_field(form.edit_and_watch_submit_button, title="Edit first then Watch") }}
+            </div>
+            <div id="watch-group-tag">
+               {{ render_field(form.tags, value=active_tag.title if active_tag_uuid else '', placeholder="Watch group / tag", class="transparent-field") }}
+            </div>
+            <div id="quick-watch-processor-type">
+                {{ render_simple_field(form.processor) }}
+            </div>
+
+        </fieldset>
+        <span style="color:#eee; font-size: 80%;"><img alt="Create a shareable link" style="height: 1em;display:inline-block;" src="{{url_for('static_content', group='images', filename='spread-white.svg')}}" > Tip: You can also add 'shared' watches. <a href="https://github.com/dgtlmoon/changedetection.io/wiki/Sharing-a-Watch">More info</a></span>
+    </form>
+</div>
+<div class="box">
+    <form class="pure-form" action="{{ url_for('ui.form_watch_list_checkbox_operations') }}" method="POST" id="watch-list-form">
+    <input type="hidden" name="csrf_token" value="{{ csrf_token() }}" >
+    <input type="hidden" id="op_extradata" name="op_extradata" value="" >
+    <div id="checkbox-operations">
+        <button class="pure-button button-secondary button-xsmall"  name="op" value="pause"><i data-feather="pause" style="width: 14px; height: 14px; stroke: white; margin-right: 4px;"></i>Pause</button>
+        <button class="pure-button button-secondary button-xsmall"  name="op" value="unpause"><i data-feather="play" style="width: 14px; height: 14px; stroke: white; margin-right: 4px;"></i>UnPause</button>
+        <button class="pure-button button-secondary button-xsmall"  name="op" value="mute"><i data-feather="volume-x" style="width: 14px; height: 14px; stroke: white; margin-right: 4px;"></i>Mute</button>
+        <button class="pure-button button-secondary button-xsmall"  name="op" value="unmute"><i data-feather="volume-2" style="width: 14px; height: 14px; stroke: white; margin-right: 4px;"></i>UnMute</button>
+        <button class="pure-button button-secondary button-xsmall" name="op" value="recheck"><i data-feather="refresh-cw" style="width: 14px; height: 14px; stroke: white; margin-right: 4px;"></i>Recheck</button>
+        <button class="pure-button button-secondary button-xsmall" name="op" value="assign-tag" id="checkbox-assign-tag"><i data-feather="tag" style="width: 14px; height: 14px; stroke: white; margin-right: 4px;"></i>Tag</button>
+        <button class="pure-button button-secondary button-xsmall" name="op" value="mark-viewed"><i data-feather="eye" style="width: 14px; height: 14px; stroke: white; margin-right: 4px;"></i>Mark viewed</button>
+        <button class="pure-button button-secondary button-xsmall" name="op" value="notification-default"><i data-feather="bell" style="width: 14px; height: 14px; stroke: white; margin-right: 4px;"></i>Use default notification</button>
+        <button class="pure-button button-secondary button-xsmall" name="op" value="clear-errors"><i data-feather="x-circle" style="width: 14px; height: 14px; stroke: white; margin-right: 4px;"></i>Clear errors</button>
+        <button class="pure-button button-secondary button-xsmall" style="background: #dd4242;" name="op" value="clear-history"><i data-feather="trash-2" style="width: 14px; height: 14px; stroke: white; margin-right: 4px;"></i>Clear/reset history</button>
+        <button class="pure-button button-secondary button-xsmall" style="background: #dd4242;" name="op" value="delete"><i data-feather="trash" style="width: 14px; height: 14px; stroke: white; margin-right: 4px;"></i>Delete</button>
+    </div>
+    {%- if watches|length >= pagination.per_page -%}
+        {{ pagination.info }}
+    {%- endif -%}
+    {%- if search_q -%}<div id="search-result-info">Searching "<strong><i>{{search_q}}</i></strong>"</div>{%- endif -%}
+    <div>
+        <a href="{{url_for('watchlist.index')}}" class="pure-button button-tag {{'active' if not active_tag_uuid }}">All</a>
+
+    <!-- tag list -->
+    {%- for uuid, tag in tags -%}
+        {%- if tag != "" -%}
+            <a href="{{url_for('watchlist.index', tag=uuid) }}" class="pure-button button-tag {{'active' if active_tag_uuid == uuid }}">{{ tag.title }}</a>
+        {%- endif -%}
+    {%- endfor -%}
+    </div>
+
+    {%- set sort_order = sort_order or 'asc' -%}
+    {%- set sort_attribute = sort_attribute or 'last_changed'  -%}
+    {%- set pagination_page = request.args.get('page', 0) -%}
+    {%- set cols_required = 6 -%}
+    {%- set any_has_restock_price_processor = datastore.any_watches_have_processor_by_name("restock_diff") -%}
+    {%- if any_has_restock_price_processor -%}
+        {%- set cols_required = cols_required + 1 -%}
+    {%- endif -%}
+    {%- set ui_settings = datastore.data['settings']['application']['ui'] -%}
+    {%- set wrapper_classes = [
+        'has-unread-changes' if unread_changes_count else '',
+        'has-error' if errored_count else '',
+    ] -%}
+    <div id="watch-table-wrapper" class="{{ wrapper_classes | reject('equalto', '') | join(' ') }}">
+        {%- set table_classes = [
+            'favicon-enabled' if 'favicons_enabled' not in ui_settings or ui_settings['favicons_enabled'] else 'favicon-not-enabled',
+        ] -%}
+        <table class="pure-table pure-table-striped watch-table {{ table_classes | reject('equalto', '') | join(' ') }}">
+            <thead>
+            <tr>
+                {%- set link_order = "desc" if sort_order  == 'asc' else "asc" -%}
+                {%- set arrow_span = "" -%}
+                <th><input style="vertical-align: middle" type="checkbox" id="check-all" > <a class="{{ 'active '+link_order if sort_attribute == 'date_created' else 'inactive' }}"  href="{{url_for('watchlist.index', sort='date_created', order=link_order, tag=active_tag_uuid)}}"># <span class='arrow {{link_order}}'></span></a></th>
+                <th>
+                    <a class="{{ 'active '+link_order if sort_attribute == 'paused' else 'inactive' }}" href="{{url_for('watchlist.index', sort='paused', order=link_order, tag=active_tag_uuid)}}"><i data-feather="pause" style="vertical-align: bottom; width: 14px; height: 14px;  margin-right: 4px;"></i><span class='arrow {{link_order}}'></span></a>
+                    &nbsp;
+                    <a class="{{ 'active '+link_order if sort_attribute == 'notification_muted' else 'inactive' }}" href="{{url_for('watchlist.index', sort='notification_muted', order=link_order, tag=active_tag_uuid)}}"><i data-feather="volume-2" style="vertical-align: bottom; width: 14px; height: 14px;  margin-right: 4px;"></i><span class='arrow {{link_order}}'></span></a>
+                </th>
+                <th><a class="{{ 'active '+link_order if sort_attribute == 'label' else 'inactive' }}" href="{{url_for('watchlist.index', sort='label', order=link_order, tag=active_tag_uuid)}}">Website <span class='arrow {{link_order}}'></span></a></th>
+             {%- if any_has_restock_price_processor -%}
+                <th>Restock &amp; Price</th>
+             {%- endif -%}
+                <th><a class="{{ 'active '+link_order if sort_attribute == 'last_checked' else 'inactive' }}" href="{{url_for('watchlist.index', sort='last_checked', order=link_order, tag=active_tag_uuid)}}"><span class="hide-on-mobile">Last</span> Checked <span class='arrow {{link_order}}'></span></a></th>
+                <th><a class="{{ 'active '+link_order if sort_attribute == 'last_changed' else 'inactive' }}" href="{{url_for('watchlist.index', sort='last_changed', order=link_order, tag=active_tag_uuid)}}"><span class="hide-on-mobile">Last</span> Changed <span class='arrow {{link_order}}'></span></a></th>
+                <th class="empty-cell"></th>
+            </tr>
+            </thead>
+            <tbody>
+            {%- if not watches|length -%}
+            <tr>
+                <td colspan="{{ cols_required }}" style="text-wrap: wrap;">No website watches configured, please add a URL in the box above, or <a href="{{ url_for('imports.import_page')}}" >import a list</a>.</td>
+            </tr>
+            {%- endif -%}
+
+            {%- for watch in (watches|sort(attribute=sort_attribute, reverse=sort_order == 'asc'))|pagination_slice(skip=pagination.skip) -%}
+                {%- set checking_now = is_checking_now(watch) -%}
+                {%- set history_n = watch.history_n -%}
+                {%- set favicon = watch.get_favicon_filename() -%}
+                {%- set system_use_url_watchlist = datastore.data['settings']['application']['ui'].get('use_page_title_in_list')  -%}
+                {#  Class settings mirrored in changedetectionio/static/js/realtime.js for the frontend #}
+                {%- set row_classes = [
+                    loop.cycle('pure-table-odd', 'pure-table-even'),
+                    'processor-' ~ watch['processor'],
+                    'has-error' if watch.compile_error_texts()|length > 2 else '',
+                    'paused' if watch.paused is defined and watch.paused != False else '',
+                    'unviewed' if watch.has_unviewed else '',
+                    'has-restock-info' if watch.has_restock_info else 'no-restock-info',
+                    'has-favicon' if favicon else '',
+                    'in-stock' if watch.has_restock_info and watch['restock']['in_stock'] else '',
+                    'not-in-stock' if watch.has_restock_info and not watch['restock']['in_stock'] else '',
+                    'queued' if watch.uuid in queued_uuids else '',
+                    'checking-now' if checking_now else '',
+                    'notification_muted' if watch.notification_muted else '',
+                    'single-history' if history_n == 1 else '',
+                    'multiple-history' if history_n >= 2 else '',
+                    'use-html-title' if system_use_url_watchlist else 'no-html-title',
+                ] -%}
+            <tr id="{{ watch.uuid }}" data-watch-uuid="{{ watch.uuid }}" class="{{ row_classes | reject('equalto', '') | join(' ') }}">
+                <td class="inline checkbox-uuid" ><div><input name="uuids"  type="checkbox" value="{{ watch.uuid}} " > <span class="counter-i">{{ loop.index+pagination.skip }}</span></div></td>
+                <td class="inline watch-controls">
+                    <div>
+                    <a class="ajax-op state-off pause-toggle" data-op="pause" href="{{url_for('watchlist.index', op='pause', uuid=watch.uuid, tag=active_tag_uuid)}}"><img src="{{url_for('static_content', group='images', filename='pause.svg')}}" alt="Pause checks" title="Pause checks" class="icon icon-pause" ></a>
+                    <a class="ajax-op state-on pause-toggle"  data-op="pause" style="display: none" href="{{url_for('watchlist.index', op='pause', uuid=watch.uuid, tag=active_tag_uuid)}}"><img src="{{url_for('static_content', group='images', filename='play.svg')}}" alt="UnPause checks" title="UnPause checks" class="icon icon-unpause" ></a>
+                    <a class="ajax-op state-off mute-toggle" data-op="mute" href="{{url_for('watchlist.index', op='mute', uuid=watch.uuid, tag=active_tag_uuid)}}"><img src="{{url_for('static_content', group='images', filename='bell-off.svg')}}" alt="Mute notification" title="Mute notification" class="icon icon-mute" ></a>
+                    <a class="ajax-op state-on mute-toggle" data-op="mute"  style="display: none" href="{{url_for('watchlist.index', op='mute', uuid=watch.uuid, tag=active_tag_uuid)}}"><img src="{{url_for('static_content', group='images', filename='bell-off.svg')}}" alt="UnMute notification" title="UnMute notification" class="icon icon-mute" ></a>
+                    </div>
+                </td>
+
+                <td class="title-col inline">
+                    <div class="flex-wrapper">
+                    {% if 'favicons_enabled' not in ui_settings or ui_settings['favicons_enabled'] %}
+                        <div>{# A page might have hundreds of these images, set IMG options for lazy loading, don't set SRC if we dont have it so it doesnt fetch the placeholder'  #}
+                            <img alt="Favicon thumbnail" class="favicon" loading="lazy" decoding="async" fetchpriority="low" {% if favicon %} src="{{url_for('static_content', group='favicon', filename=watch.uuid)}}" {% else %} src='data:image/svg+xml;utf8,%3Csvg xmlns="http://www.w3.org/2000/svg" width="7.087" height="7.087" viewBox="0 0 7.087 7.087"%3E%3Ccircle cx="3.543" cy="3.543" r="3.279" stroke="%23e1e1e1" stroke-width="0.45" fill="none" opacity="0.74"/%3E%3C/svg%3E' {%  endif %} />
+                        </div>
+                    {%  endif %}
+                        <div>
+                        <span class="watch-title">
+                            {% if system_use_url_watchlist or watch.get('use_page_title_in_list') %}
+                                {{ watch.label }}
+                            {% else %}
+                                {{ watch.get('title') or watch.link }}
+                            {% endif %}
+                           <a class="external" target="_blank" rel="noopener" href="{{ watch.link.replace('source:','') }}">&nbsp;</a>
+                        </span>
+                            <div class="error-text" style="display:none;">{{ watch.compile_error_texts(has_proxies=datastore.proxy_list) }}</div>
+                            {%- if watch['processor'] == 'text_json_diff'  -%}
+                                {%- if watch['has_ldjson_price_data'] and not watch['track_ldjson_price_data']  -%}
+                                <div class="ldjson-price-track-offer">Switch to Restock & Price watch mode? <a href="{{url_for('price_data_follower.accept', uuid=watch.uuid)}}" class="pure-button button-xsmall">Yes</a> <a href="{{url_for('price_data_follower.reject', uuid=watch.uuid)}}" class="">No</a></div>
+                                {%- endif -%}
+                            {%- endif -%}
+                            {%- if watch['processor'] == 'restock_diff' -%}
+                                <span class="tracking-ldjson-price-data" title="Automatically following embedded price information"><img src="{{url_for('static_content', group='images', filename='price-tag-icon.svg')}}"  class="status-icon price-follow-tag-icon" > Price</span>
+                            {%- endif -%}
+                            {%- for watch_tag_uuid, watch_tag in datastore.get_all_tags_for_watch(watch['uuid']).items() -%}
+                              <span class="watch-tag-list">{{ watch_tag.title }}</span>
+                            {%- endfor -%}
+                        </div>
+                    <div class="status-icons">
+                            <a class="link-spread" href="{{url_for('ui.form_share_put_watch', uuid=watch.uuid)}}"><img src="{{url_for('static_content', group='images', filename='spread.svg')}}" class="status-icon icon icon-spread" title="Create a link to share watch config with others" ></a>
+                            {%- if watch.get_fetch_backend == "html_webdriver"
+                                 or ( watch.get_fetch_backend == "system" and system_default_fetcher == 'html_webdriver'  )
+                                 or "extra_browser_" in watch.get_fetch_backend
+                            -%}
+                            <img class="status-icon" src="{{url_for('static_content', group='images', filename='google-chrome-icon.png')}}" alt="Using a Chrome browser" title="Using a Chrome browser" >
+                            {%- endif -%}
+                            {%- if watch.is_pdf  -%}<img class="status-icon" src="{{url_for('static_content', group='images', filename='pdf-icon.svg')}}" alt="Converting PDF to text" >{%- endif -%}
+                            {%- if watch.has_browser_steps -%}<img class="status-icon status-browsersteps" src="{{url_for('static_content', group='images', filename='steps.svg')}}" alt="Browser Steps is enabled" >{%- endif -%}
+
+                    </div>
+                    </div>
+                </td>
+{%- if any_has_restock_price_processor -%}
+                <td class="restock-and-price">
+                    {%- if watch['processor'] == 'restock_diff'  -%}
+                        {%- if watch.has_restock_info -%}
+                            <span class="restock-label {{'in-stock' if watch['restock']['in_stock'] else 'not-in-stock' }}" title="Detecting restock and price">
+                                <!-- maybe some object watch['processor'][restock_diff] or.. -->
+                                 {%- if watch['restock']['in_stock']-%}  In stock {%- else-%}  Not in stock {%- endif -%}
+                            </span>
+                        {%- endif -%}
+
+                        {%- if watch.get('restock') and watch['restock']['price'] != None -%}
+                            {%- if watch['restock']['price'] != None -%}
+                                <span class="restock-label price" title="Price">
+                                {{ watch['restock']['price']|format_number_locale }} {{ watch['restock']['currency'] }}
+                                </span>
+                            {%- endif -%}
+                        {%- elif not watch.has_restock_info -%}
+                            <span class="restock-label error">No information</span>
+                        {%- endif -%}
+                    {%- endif -%}
+                </td>
+{%- endif -%}
+            {#last_checked becomes fetch-start-time#}
+                <td class="last-checked" data-timestamp="{{ watch.last_checked }}" data-fetchduration={{ watch.fetch_time }} data-eta_complete="{{ watch.last_checked+watch.fetch_time }}" >
+                    <div class="spinner-wrapper" style="display:none;" >
+                        <span class="spinner"></span><span>&nbsp;Checking now</span>
+                    </div>
+                    <span class="innertext">{{watch|format_last_checked_time|safe}}</span>
+                </td>
+                <td class="last-changed" data-timestamp="{{ watch.last_changed }}">{%- if watch.history_n >=2 and watch.last_changed >0 -%}
+                    {{watch.last_changed|format_timestamp_timeago}}
+                    {%- else -%}
+                    Not yet
+                    {%- endif -%}
+                </td>
+                <td class="buttons">
+                    <div>
+                    {%- set target_attr = ' target="' ~ watch.uuid ~ '"' if datastore.data['settings']['application']['ui'].get('open_diff_in_new_tab') else '' -%}
+                    <a href="" class="already-in-queue-button recheck pure-button pure-button-primary" style="display: none;" disabled="disabled">Queued</a>
+                    <a href="{{ url_for('ui.form_watch_checknow', uuid=watch.uuid, tag=request.args.get('tag')) }}" data-op='recheck' class="ajax-op recheck pure-button pure-button-primary">Recheck</a>
+                    <a href="{{ url_for('ui.ui_edit.edit_page', uuid=watch.uuid, tag=active_tag_uuid)}}#general" class="pure-button pure-button-primary">Edit</a>
+                    <a href="{{ url_for('ui.ui_views.diff_history_page', uuid=watch.uuid)}}" {{target_attr}} class="pure-button pure-button-primary history-link" style="display: none;">History</a>
+                    <a href="{{ url_for('ui.ui_views.preview_page', uuid=watch.uuid)}}" {{target_attr}} class="pure-button pure-button-primary preview-link" style="display: none;">Preview</a>
+                    </div>
+                </td>
+            </tr>
+            {%- endfor -%}
+            </tbody>
+        </table>
+        <ul id="post-list-buttons">
+            <li id="post-list-with-errors" style="display: none;" >
+                <a href="{{url_for('watchlist.index', with_errors=1, tag=request.args.get('tag')) }}" class="pure-button button-tag button-error">With errors ({{ errored_count }})</a>
+            </li>
+            <li id="post-list-mark-views" style="display: none;" >
+                <a href="{{url_for('ui.mark_all_viewed',with_errors=request.args.get('with_errors',0)) }}" class="pure-button button-tag " id="mark-all-viewed">Mark all viewed</a>
+            </li>
+        {%-  if active_tag_uuid -%}
+            <li id="post-list-mark-views-tag">
+                <a href="{{url_for('ui.mark_all_viewed', tag=active_tag_uuid) }}" class="pure-button button-tag " id="mark-all-viewed">Mark all viewed in '{{active_tag.title}}'</a>
+            </li>
+        {%-  endif -%}
+            <li id="post-list-unread" style="display: none;" >
+                <a href="{{url_for('watchlist.index', unread=1, tag=request.args.get('tag')) }}" class="pure-button button-tag">Unread (<span id="unread-tab-counter">{{ unread_changes_count }}</span>)</a>
+            </li>
+            <li>
+               <a href="{{ url_for('ui.form_watch_checknow', tag=active_tag_uuid, with_errors=request.args.get('with_errors',0)) }}" class="pure-button button-tag" id="recheck-all">Recheck
+                all {% if active_tag_uuid %}  in '{{active_tag.title}}'{%endif%}</a>
+            </li>
+            <li>
+                <a href="{{ url_for('rss.feed', tag=active_tag_uuid, token=app_rss_token)}}"><img alt="RSS Feed" id="feed-icon" src="{{url_for('static_content', group='images', filename='generic_feed-icon.svg')}}" height="15"></a>
+            </li>
+        </ul>
+        {{ pagination.links }}
+    </div>
+    </form>
+</div>
+{%- endblock -%}

changedetectionio/changedetection.py

@@ -1,114 +0,0 @@
-#!/usr/bin/python3
-
-# Launch as a eventlet.wsgi server instance.
-
-import getopt
-import os
-import sys
-
-import eventlet
-import eventlet.wsgi
-from . import store, changedetection_app, content_fetcher
-from . import __version__
-
-def main():
-    ssl_mode = False
-    host = ''
-    port = os.environ.get('PORT') or 5000
-    do_cleanup = False
-    datastore_path = None
-
-    # On Windows, create and use a default path.
-    if os.name == 'nt':
-        datastore_path = os.path.expandvars(r'%APPDATA%\changedetection.io')
-        os.makedirs(datastore_path, exist_ok=True)
-    else:
-        # Must be absolute so that send_from_directory doesnt try to make it relative to backend/
-        datastore_path = os.path.join(os.getcwd(), "../datastore")
-
-    try:
-        opts, args = getopt.getopt(sys.argv[1:], "Ccsd:h:p:", "port")
-    except getopt.GetoptError:
-        print('backend.py -s SSL enable -h [host] -p [port] -d [datastore path]')
-        sys.exit(2)
-
-    create_datastore_dir = False
-
-    for opt, arg in opts:
-        #        if opt == '--clear-all-history':
-        # Remove history, the actual files you need to delete manually.
-        #            for uuid, watch in datastore.data['watching'].items():
-        #                watch.update({'history': {}, 'last_checked': 0, 'last_changed': 0, 'previous_md5': None})
-
-        if opt == '-s':
-            ssl_mode = True
-
-        if opt == '-h':
-            host = arg
-
-        if opt == '-p':
-            port = int(arg)
-
-        if opt == '-d':
-            datastore_path = arg
-
-        # Cleanup (remove text files that arent in the index)
-        if opt == '-c':
-            do_cleanup = True
-
-        # Create the datadir if it doesnt exist
-        if opt == '-C':
-            create_datastore_dir = True
-
-    # isnt there some @thingy to attach to each route to tell it, that this route needs a datastore
-    app_config = {'datastore_path': datastore_path}
-
-    if not os.path.isdir(app_config['datastore_path']):
-        if create_datastore_dir:
-            os.mkdir(app_config['datastore_path'])
-        else:
-            print(
-                "ERROR: Directory path for the datastore '{}' does not exist, cannot start, please make sure the directory exists or specify a directory with the -d option.\n"
-                "Or use the -C parameter to create the directory.".format(app_config['datastore_path']), file=sys.stderr)
-            sys.exit(2)
-
-    datastore = store.ChangeDetectionStore(datastore_path=app_config['datastore_path'], version_tag=__version__)
-    app = changedetection_app(app_config, datastore)
-
-    # Go into cleanup mode
-    if do_cleanup:
-        datastore.remove_unused_snapshots()
-
-    app.config['datastore_path'] = datastore_path
-
-
-    @app.context_processor
-    def inject_version():
-        return dict(right_sticky="v{}".format(datastore.data['version_tag']),
-                    new_version_available=app.config['NEW_VERSION_AVAILABLE'],
-                    has_password=datastore.data['settings']['application']['password'] != False
-                    )
-
-    # Proxy sub-directory support
-    # Set environment var USE_X_SETTINGS=1 on this script
-    # And then in your proxy_pass settings
-    #
-    #         proxy_set_header Host "localhost";
-    #         proxy_set_header X-Forwarded-Prefix /app;
-
-    if os.getenv('USE_X_SETTINGS'):
-        print ("USE_X_SETTINGS is ENABLED\n")
-        from werkzeug.middleware.proxy_fix import ProxyFix
-        app.wsgi_app = ProxyFix(app.wsgi_app, x_prefix=1, x_host=1)
-
-    if ssl_mode:
-        # @todo finalise SSL config, but this should get you in the right direction if you need it.
-        eventlet.wsgi.server(eventlet.wrap_ssl(eventlet.listen((host, port)),
-                                               certfile='cert.pem',
-                                               keyfile='privkey.pem',
-                                               server_side=True), app)
-
-    else:
-        eventlet.wsgi.server(eventlet.listen((host, int(port))), app)
-
-

changedetectionio/conditions/__init__.py

@@ -0,0 +1,167 @@
+from json_logic.builtins import BUILTINS
+
+from .exceptions import EmptyConditionRuleRowNotUsable
+from .pluggy_interface import plugin_manager  # Import the pluggy plugin manager
+from . import default_plugin
+from loguru import logger
+# List of all supported JSON Logic operators
+operator_choices = [
+    (None, "Choose one - Operator"),
+    (">", "Greater Than"),
+    ("<", "Less Than"),
+    (">=", "Greater Than or Equal To"),
+    ("<=", "Less Than or Equal To"),
+    ("==", "Equals"),
+    ("!=", "Not Equals"),
+    ("in", "Contains"),
+]
+
+# Fields available in the rules
+field_choices = [
+    (None, "Choose one - Field"),
+]
+
+# The data we will feed the JSON Rules to see if it passes the test/conditions or not
+EXECUTE_DATA = {}
+
+
+# Define the extended operations dictionary
+CUSTOM_OPERATIONS = {
+    **BUILTINS,  # Include all standard operators
+}
+
+def filter_complete_rules(ruleset):
+    rules = [
+        rule for rule in ruleset
+        if all(value not in ("", False, "None", None) for value in [rule["operator"], rule["field"], rule["value"]])
+    ]
+    return rules
+
+def convert_to_jsonlogic(logic_operator: str, rule_dict: list):
+    """
+    Convert a structured rule dict into a JSON Logic rule.
+
+    :param rule_dict: Dictionary containing conditions.
+    :return: JSON Logic rule as a dictionary.
+    """
+
+
+    json_logic_conditions = []
+
+    for condition in rule_dict:
+        operator = condition["operator"]
+        field = condition["field"]
+        value = condition["value"]
+
+        if not operator or operator == 'None' or not value or not field:
+            raise EmptyConditionRuleRowNotUsable()
+
+        # Convert value to int/float if possible
+        try:
+            if isinstance(value, str) and "." in value and str != "None":
+                value = float(value)
+            else:
+                value = int(value)
+        except (ValueError, TypeError):
+            pass  # Keep as a string if conversion fails
+
+        # Handle different JSON Logic operators properly
+        if operator == "in":
+            json_logic_conditions.append({"in": [value, {"var": field}]})  # value first
+        elif operator in ("!", "!!", "-"):
+            json_logic_conditions.append({operator: [{"var": field}]})  # Unary operators
+        elif operator in ("min", "max", "cat"):
+            json_logic_conditions.append({operator: value})  # Multi-argument operators
+        else:
+            json_logic_conditions.append({operator: [{"var": field}, value]})  # Standard binary operators
+
+    return {logic_operator: json_logic_conditions} if len(json_logic_conditions) > 1 else json_logic_conditions[0]
+
+
+def execute_ruleset_against_all_plugins(current_watch_uuid: str, application_datastruct, ephemeral_data={} ):
+    """
+    Build our data and options by calling our plugins then pass it to jsonlogic and see if the conditions pass
+
+    :param ruleset: JSON Logic rule dictionary.
+    :param extracted_data: Dictionary containing the facts.   <-- maybe the app struct+uuid
+    :return: Dictionary of plugin results.
+    """
+    from json_logic import jsonLogic
+
+    EXECUTE_DATA = {}
+    result = True
+    
+    watch = application_datastruct['watching'].get(current_watch_uuid)
+
+    if watch and watch.get("conditions"):
+        logic_operator = "and" if watch.get("conditions_match_logic", "ALL") == "ALL" else "or"
+        complete_rules = filter_complete_rules(watch['conditions'])
+        if complete_rules:
+            # Give all plugins a chance to update the data dict again (that we will test the conditions against)
+            for plugin in plugin_manager.get_plugins():
+                try:
+                    import concurrent.futures
+                    import time
+                    
+                    with concurrent.futures.ThreadPoolExecutor() as executor:
+                        future = executor.submit(
+                            plugin.add_data,
+                            current_watch_uuid=current_watch_uuid,
+                            application_datastruct=application_datastruct,
+                            ephemeral_data=ephemeral_data
+                        )
+                        logger.debug(f"Trying plugin {plugin}....")
+
+                        # Set a timeout of 10 seconds
+                        try:
+                            new_execute_data = future.result(timeout=10)
+                            if new_execute_data and isinstance(new_execute_data, dict):
+                                EXECUTE_DATA.update(new_execute_data)
+
+                        except concurrent.futures.TimeoutError:
+                            # The plugin took too long, abort processing for this watch
+                            raise Exception(f"Plugin {plugin.__class__.__name__} took more than 10 seconds to run.")
+                except Exception as e:
+                    # Log the error but continue with the next plugin
+                    import logging
+                    logging.error(f"Error executing plugin {plugin.__class__.__name__}: {str(e)}")
+                    continue
+
+            # Create the ruleset
+            ruleset = convert_to_jsonlogic(logic_operator=logic_operator, rule_dict=complete_rules)
+            
+            # Pass the custom operations dictionary to jsonLogic
+            if not jsonLogic(logic=ruleset, data=EXECUTE_DATA, operations=CUSTOM_OPERATIONS):
+                result = False
+
+    return {'executed_data': EXECUTE_DATA, 'result': result}
+
+# Load plugins dynamically
+for plugin in plugin_manager.get_plugins():
+    new_ops = plugin.register_operators()
+    if isinstance(new_ops, dict):
+        CUSTOM_OPERATIONS.update(new_ops)
+
+    new_operator_choices = plugin.register_operator_choices()
+    if isinstance(new_operator_choices, list):
+        operator_choices.extend(new_operator_choices)
+
+    new_field_choices = plugin.register_field_choices()
+    if isinstance(new_field_choices, list):
+        field_choices.extend(new_field_choices)
+
+def collect_ui_edit_stats_extras(watch):
+    """Collect and combine HTML content from all plugins that implement ui_edit_stats_extras"""
+    extras_content = []
+    
+    for plugin in plugin_manager.get_plugins():
+        try:
+            content = plugin.ui_edit_stats_extras(watch=watch)
+            if content:
+                extras_content.append(content)
+        except Exception as e:
+            # Skip plugins that don't implement the hook or have errors
+            pass
+            
+    return "\n".join(extras_content) if extras_content else ""
+

changedetectionio/conditions/blueprint.py

@@ -0,0 +1,81 @@
+# Flask Blueprint Definition
+import json
+
+from flask import Blueprint
+
+from changedetectionio.conditions import execute_ruleset_against_all_plugins
+
+
+def construct_blueprint(datastore):
+    from changedetectionio.flask_app import login_optionally_required
+
+    conditions_blueprint = Blueprint('conditions', __name__, template_folder="templates")
+
+    @conditions_blueprint.route("/<string:watch_uuid>/verify-condition-single-rule", methods=['POST'])
+    @login_optionally_required
+    def verify_condition_single_rule(watch_uuid):
+        """Verify a single condition rule against the current snapshot"""
+        from changedetectionio.processors.text_json_diff import prepare_filter_prevew
+        from flask import request, jsonify
+        from copy import deepcopy
+
+        ephemeral_data = {}
+
+        # Get the watch data
+        watch = datastore.data['watching'].get(watch_uuid)
+        if not watch:
+            return jsonify({'status': 'error', 'message': 'Watch not found'}), 404
+
+        # First use prepare_filter_prevew to process the form data
+        # This will return text_after_filter which is after all current form settings are applied
+        # Create ephemeral data with the text from the current snapshot
+
+        try:
+            # Call prepare_filter_prevew to get a processed version of the content with current form settings
+            # We'll ignore the returned response and just use the datastore which is modified by the function
+
+            # this should apply all filters etc so then we can run the CONDITIONS against the final output text
+            result = prepare_filter_prevew(datastore=datastore,
+                                           form_data=request.form,
+                                           watch_uuid=watch_uuid)
+
+            ephemeral_data['text'] = result.get('after_filter', '')
+            # Create a temporary watch data structure with this single rule
+            tmp_watch_data = deepcopy(datastore.data['watching'].get(watch_uuid))
+
+            # Override the conditions in the temporary watch
+            rule_json = request.args.get("rule")
+            rule = json.loads(rule_json) if rule_json else None
+
+            # Should be key/value of field, operator, value
+            tmp_watch_data['conditions'] = [rule]
+            tmp_watch_data['conditions_match_logic'] = "ALL"  # Single rule, so use ALL
+
+            # Create a temporary application data structure for the rule check
+            temp_app_data = {
+                'watching': {
+                    watch_uuid: tmp_watch_data
+                }
+            }
+
+            # Execute the rule against the current snapshot with form data
+            result = execute_ruleset_against_all_plugins(
+                current_watch_uuid=watch_uuid,
+                application_datastruct=temp_app_data,
+                ephemeral_data=ephemeral_data
+            )
+
+            return jsonify({
+                'status': 'success',
+                'result': result.get('result'),
+                'data': result.get('executed_data'),
+                'message': 'Condition passes' if result else 'Condition does not pass'
+            })
+
+        except Exception as e:
+            return jsonify({
+                'status': 'error',
+                'message': f'Error verifying condition: {str(e)}'
+            }), 500
+
+    return conditions_blueprint

changedetectionio/conditions/default_plugin.py

@@ -0,0 +1,83 @@
+import re
+
+import pluggy
+from price_parser import Price
+from loguru import logger
+
+hookimpl = pluggy.HookimplMarker("changedetectionio_conditions")
+
+
+@hookimpl
+def register_operators():
+    def starts_with(_, text, prefix):
+        return text.lower().strip().startswith(str(prefix).strip().lower())
+
+    def ends_with(_, text, suffix):
+        return text.lower().strip().endswith(str(suffix).strip().lower())
+
+    def length_min(_, text, strlen):
+        return len(text) >= int(strlen)
+
+    def length_max(_, text, strlen):
+        return len(text) <= int(strlen)
+
+    # Custom function for case-insensitive regex matching
+    def contains_regex(_, text, pattern):
+        """Returns True if `text` contains `pattern` (case-insensitive regex match)."""
+        return bool(re.search(pattern, str(text), re.IGNORECASE))
+
+    # Custom function for NOT matching case-insensitive regex
+    def not_contains_regex(_, text, pattern):
+        """Returns True if `text` does NOT contain `pattern` (case-insensitive regex match)."""
+        return not bool(re.search(pattern, str(text), re.IGNORECASE))
+
+    def not_contains(_, text, pattern):
+        return not pattern in text
+
+    return {
+        "!in": not_contains,
+        "!contains_regex": not_contains_regex,
+        "contains_regex": contains_regex,
+        "ends_with": ends_with,
+        "length_max": length_max,
+        "length_min": length_min,
+        "starts_with": starts_with,
+    }
+
+@hookimpl
+def register_operator_choices():
+    return [
+        ("!in", "Does NOT Contain"),
+        ("starts_with", "Text Starts With"),
+        ("ends_with", "Text Ends With"),
+        ("length_min", "Length minimum"),
+        ("length_max", "Length maximum"),
+        ("contains_regex", "Text Matches Regex"),
+        ("!contains_regex", "Text Does NOT Match Regex"),
+    ]
+
+@hookimpl
+def register_field_choices():
+    return [
+        ("extracted_number", "Extracted number after 'Filters & Triggers'"),
+#        ("meta_description", "Meta Description"),
+#        ("meta_keywords", "Meta Keywords"),
+        ("page_filtered_text", "Page text after 'Filters & Triggers'"),
+        #("page_title", "Page <title>"), # actual page title <title>
+    ]
+
+@hookimpl
+def add_data(current_watch_uuid, application_datastruct, ephemeral_data):
+
+    res = {}
+    if 'text' in ephemeral_data:
+        res['page_filtered_text'] = ephemeral_data['text']
+
+        # Better to not wrap this in try/except so that the UI can see any errors
+        price = Price.fromstring(ephemeral_data.get('text'))
+        if price and price.amount != None:
+            # This is slightly misleading, it's extracting a PRICE not a Number..
+            res['extracted_number'] = float(price.amount)
+            logger.debug(f"Extracted number result: '{price}' - returning float({res['extracted_number']})")
+
+    return res

changedetectionio/conditions/exceptions.py

@@ -0,0 +1,6 @@
+class EmptyConditionRuleRowNotUsable(Exception):
+    def __init__(self):
+        super().__init__("One of the 'conditions' rulesets is incomplete, cannot run.")
+
+    def __str__(self):
+        return self.args[0]

changedetectionio/conditions/form.py

@@ -0,0 +1,44 @@
+# Condition Rule Form (for each rule row)
+from wtforms import Form, SelectField, StringField, validators
+from wtforms import validators
+
+class ConditionFormRow(Form):
+
+    # ✅ Ensure Plugins Are Loaded BEFORE Importing Choices
+    from changedetectionio.conditions import plugin_manager
+    from changedetectionio.conditions import operator_choices, field_choices
+    field = SelectField(
+        "Field",
+        choices=field_choices,
+        validators=[validators.Optional()]
+    )
+
+    operator = SelectField(
+        "Operator",
+        choices=operator_choices,
+        validators=[validators.Optional()]
+    )
+
+    value = StringField("Value", validators=[validators.Optional()], render_kw={"placeholder": "A value"})
+
+    def validate(self, extra_validators=None):
+        # First, run the default validators
+        if not super().validate(extra_validators):
+            return False
+
+        # Custom validation logic
+        # If any of the operator/field/value is set, then they must be all set
+        if any(value not in ("", False, "None", None) for value in [self.operator.data, self.field.data, self.value.data]):
+            if not self.operator.data or self.operator.data == 'None':
+                self.operator.errors.append("Operator is required.")
+                return False
+
+            if not self.field.data or self.field.data == 'None':
+                self.field.errors.append("Field is required.")
+                return False
+
+            if not self.value.data:
+                self.value.errors.append("Value is required.")
+                return False
+
+        return True  # Only return True if all conditions pass

changedetectionio/conditions/pluggy_interface.py

@@ -0,0 +1,74 @@
+import pluggy
+import os
+import importlib
+import sys
+from . import default_plugin
+
+# ✅ Ensure that the namespace in HookspecMarker matches PluginManager
+PLUGIN_NAMESPACE = "changedetectionio_conditions"
+
+hookspec = pluggy.HookspecMarker(PLUGIN_NAMESPACE)
+hookimpl = pluggy.HookimplMarker(PLUGIN_NAMESPACE)
+
+
+class ConditionsSpec:
+    """Hook specifications for extending JSON Logic conditions."""
+
+    @hookspec
+    def register_operators():
+        """Return a dictionary of new JSON Logic operators."""
+        pass
+
+    @hookspec
+    def register_operator_choices():
+        """Return a list of new operator choices."""
+        pass
+
+    @hookspec
+    def register_field_choices():
+        """Return a list of new field choices."""
+        pass
+
+    @hookspec
+    def add_data(current_watch_uuid, application_datastruct, ephemeral_data):
+        """Add to the datadict"""
+        pass
+        
+    @hookspec
+    def ui_edit_stats_extras(watch):
+        """Return HTML content to add to the stats tab in the edit view"""
+        pass
+
+# ✅ Set up Pluggy Plugin Manager
+plugin_manager = pluggy.PluginManager(PLUGIN_NAMESPACE)
+
+# ✅ Register hookspecs (Ensures they are detected)
+plugin_manager.add_hookspecs(ConditionsSpec)
+
+# ✅ Register built-in plugins manually
+plugin_manager.register(default_plugin, "default_plugin")
+
+# ✅ Load plugins from the plugins directory
+def load_plugins_from_directory():
+    plugins_dir = os.path.join(os.path.dirname(__file__), 'plugins')
+    if not os.path.exists(plugins_dir):
+        return
+        
+    # Get all Python files (excluding __init__.py)
+    for filename in os.listdir(plugins_dir):
+        if filename.endswith(".py") and filename != "__init__.py":
+            module_name = filename[:-3]  # Remove .py extension
+            module_path = f"changedetectionio.conditions.plugins.{module_name}"
+            
+            try:
+                module = importlib.import_module(module_path)
+                # Register the plugin with pluggy
+                plugin_manager.register(module, module_name)
+            except (ImportError, AttributeError) as e:
+                print(f"Error loading plugin {module_name}: {e}")
+
+# Load plugins from the plugins directory
+load_plugins_from_directory()
+
+# ✅ Discover installed plugins from external packages (if any)
+plugin_manager.load_setuptools_entrypoints(PLUGIN_NAMESPACE)

changedetectionio/conditions/plugins/__init__.py

@@ -0,0 +1 @@
+# Import plugins package to make them discoverable

changedetectionio/conditions/plugins/levenshtein_plugin.py

@@ -0,0 +1,119 @@
+import pluggy
+from loguru import logger
+
+LEVENSHTEIN_MAX_LEN_FOR_EDIT_STATS=100000
+
+# Support both plugin systems
+conditions_hookimpl = pluggy.HookimplMarker("changedetectionio_conditions")
+global_hookimpl = pluggy.HookimplMarker("changedetectionio")
+
+def levenshtein_ratio_recent_history(watch, incoming_text=None):
+    try:
+        from Levenshtein import ratio, distance
+        k = list(watch.history.keys())
+        a = None
+        b = None
+
+        # When called from ui_edit_stats_extras, we don't have incoming_text
+        if incoming_text is None:
+            a = watch.get_history_snapshot(timestamp=k[-1])  # Latest snapshot
+            b = watch.get_history_snapshot(timestamp=k[-2])  # Previous snapshot
+
+        # Needs atleast one snapshot
+        elif len(k) >= 1: # Should be atleast one snapshot to compare against
+            a = watch.get_history_snapshot(timestamp=k[-1]) # Latest saved snapshot
+            b = incoming_text if incoming_text else k[-2]
+
+        if a and b:
+            distance_value = distance(a, b)
+            ratio_value = ratio(a, b)
+            return {
+                'distance': distance_value,
+                'ratio': ratio_value,
+                'percent_similar': round(ratio_value * 100, 2)
+            }
+    except Exception as e:
+        logger.warning(f"Unable to calc similarity: {str(e)}")
+
+    return ''
+
+@conditions_hookimpl
+def register_operators():
+    pass
+
+@conditions_hookimpl
+def register_operator_choices():
+    pass
+
+
+@conditions_hookimpl
+def register_field_choices():
+    return [
+        ("levenshtein_ratio", "Levenshtein - Text similarity ratio"),
+        ("levenshtein_distance", "Levenshtein - Text change distance"),
+    ]
+
+@conditions_hookimpl
+def add_data(current_watch_uuid, application_datastruct, ephemeral_data):
+    res = {}
+    watch = application_datastruct['watching'].get(current_watch_uuid)
+    # ephemeral_data['text'] will be the current text after filters, they may have edited filters but not saved them yet etc
+
+    if watch and 'text' in ephemeral_data:
+        lev_data = levenshtein_ratio_recent_history(watch, ephemeral_data.get('text',''))
+        if isinstance(lev_data, dict):
+            res['levenshtein_ratio'] = lev_data.get('ratio', 0)
+            res['levenshtein_similarity'] = lev_data.get('percent_similar', 0)
+            res['levenshtein_distance'] = lev_data.get('distance', 0)
+
+    return res
+
+@global_hookimpl
+def ui_edit_stats_extras(watch):
+    """Add Levenshtein stats to the UI using the global plugin system"""
+    """Generate the HTML for Levenshtein stats - shared by both plugin systems"""
+    if len(watch.history.keys()) < 2:
+        return "<p>Not enough history to calculate Levenshtein metrics</p>"
+
+
+    # Protection against the algorithm getting stuck on huge documents
+    k = list(watch.history.keys())
+    if any(
+            len(watch.get_history_snapshot(timestamp=k[idx])) > LEVENSHTEIN_MAX_LEN_FOR_EDIT_STATS
+            for idx in (-1, -2)
+            if len(k) >= abs(idx)
+    ):
+        return "<p>Snapshot too large for edit statistics, skipping.</p>"
+
+    try:
+        lev_data = levenshtein_ratio_recent_history(watch)
+        if not lev_data or not isinstance(lev_data, dict):
+            return "<p>Unable to calculate Levenshtein metrics</p>"
+            
+        html = f"""
+        <div class="levenshtein-stats">
+            <h4>Levenshtein Text Similarity Details</h4>
+            <table class="pure-table">
+                <tbody>
+                    <tr>
+                        <td>Raw distance (edits needed)</td>
+                        <td>{lev_data['distance']}</td>
+                    </tr>
+                    <tr>
+                        <td>Similarity ratio</td>
+                        <td>{lev_data['ratio']:.4f}</td>
+                    </tr>
+                    <tr>
+                        <td>Percent similar</td>
+                        <td>{lev_data['percent_similar']}%</td>
+                    </tr>
+                </tbody>
+            </table>
+            <p style="font-size: 80%;">Levenshtein metrics compare the last two snapshots, measuring how many character edits are needed to transform one into the other.</p>
+        </div>
+        """
+        return html
+    except Exception as e:
+        logger.error(f"Error generating Levenshtein UI extras: {str(e)}")
+        return "<p>Error calculating Levenshtein metrics</p>"
+        

changedetectionio/conditions/plugins/wordcount_plugin.py

@@ -0,0 +1,82 @@
+import pluggy
+from loguru import logger
+
+# Support both plugin systems
+conditions_hookimpl = pluggy.HookimplMarker("changedetectionio_conditions")
+global_hookimpl = pluggy.HookimplMarker("changedetectionio")
+
+def count_words_in_history(watch, incoming_text=None):
+    """Count words in snapshot text"""
+    try:
+        if incoming_text is not None:
+            # When called from add_data with incoming text
+            return len(incoming_text.split())
+        elif watch.history.keys():
+            # When called from UI extras to count latest snapshot
+            latest_key = list(watch.history.keys())[-1]
+            latest_content = watch.get_history_snapshot(latest_key)
+            return len(latest_content.split())
+        return 0
+    except Exception as e:
+        logger.error(f"Error counting words: {str(e)}")
+        return 0
+
+# Implement condition plugin hooks
+@conditions_hookimpl
+def register_operators():
+    # No custom operators needed
+    return {}
+
+@conditions_hookimpl
+def register_operator_choices():
+    # No custom operator choices needed
+    return []
+
+@conditions_hookimpl
+def register_field_choices():
+    # Add a field that will be available in conditions
+    return [
+        ("word_count", "Word count of content"),
+    ]
+
+@conditions_hookimpl
+def add_data(current_watch_uuid, application_datastruct, ephemeral_data):
+    """Add word count data for conditions"""
+    result = {}
+    watch = application_datastruct['watching'].get(current_watch_uuid)
+    
+    if watch and 'text' in ephemeral_data:
+        word_count = count_words_in_history(watch, ephemeral_data['text'])
+        result['word_count'] = word_count
+    
+    return result
+
+def _generate_stats_html(watch):
+    """Generate the HTML content for the stats tab"""
+    word_count = count_words_in_history(watch)
+    
+    html = f"""
+    <div class="word-count-stats">
+        <h4>Content Analysis</h4>
+        <table class="pure-table">
+            <tbody>
+                <tr>
+                    <td>Word count (latest snapshot)</td>
+                    <td>{word_count}</td>
+                </tr>
+            </tbody>
+        </table>
+        <p style="font-size: 80%;">Word count is a simple measure of content length, calculated by splitting text on whitespace.</p>
+    </div>
+    """
+    return html
+
+@conditions_hookimpl
+def ui_edit_stats_extras(watch):
+    """Add word count stats to the UI through conditions plugin system"""
+    return _generate_stats_html(watch)
+
+@global_hookimpl
+def ui_edit_stats_extras(watch):
+    """Add word count stats to the UI using the global plugin system"""
+    return _generate_stats_html(watch)

changedetectionio/content_fetcher.py

@@ -1,556 +0,0 @@
-from abc import ABC, abstractmethod
-import chardet
-import json
-import os
-import requests
-import time
-import sys
-
-class PageUnloadable(Exception):
-    def __init__(self, status_code, url):
-        # Set this so we can use it in other parts of the app
-        self.status_code = status_code
-        self.url = url
-        return
-    pass
-
-class EmptyReply(Exception):
-    def __init__(self, status_code, url):
-        # Set this so we can use it in other parts of the app
-        self.status_code = status_code
-        self.url = url
-        return
-    pass
-
-class ScreenshotUnavailable(Exception):
-    def __init__(self, status_code, url):
-        # Set this so we can use it in other parts of the app
-        self.status_code = status_code
-        self.url = url
-        return
-    pass
-
-class ReplyWithContentButNoText(Exception):
-    def __init__(self, status_code, url):
-        # Set this so we can use it in other parts of the app
-        self.status_code = status_code
-        self.url = url
-        return
-    pass
-
-
-class Fetcher():
-    error = None
-    status_code = None
-    content = None
-    headers = None
-
-    fetcher_description = "No description"
-    webdriver_js_execute_code = None
-    xpath_element_js = """               
-                // Include the getXpath script directly, easier than fetching
-                !function(e,n){"object"==typeof exports&&"undefined"!=typeof module?module.exports=n():"function"==typeof define&&define.amd?define(n):(e=e||self).getXPath=n()}(this,function(){return function(e){var n=e;if(n&&n.id)return'//*[@id="'+n.id+'"]';for(var o=[];n&&Node.ELEMENT_NODE===n.nodeType;){for(var i=0,r=!1,d=n.previousSibling;d;)d.nodeType!==Node.DOCUMENT_TYPE_NODE&&d.nodeName===n.nodeName&&i++,d=d.previousSibling;for(d=n.nextSibling;d;){if(d.nodeName===n.nodeName){r=!0;break}d=d.nextSibling}o.push((n.prefix?n.prefix+":":"")+n.localName+(i||r?"["+(i+1)+"]":"")),n=n.parentNode}return o.length?"/"+o.reverse().join("/"):""}});
-
-
-                const findUpTag = (el) => {
-                  let r = el
-                  chained_css = [];
-                  depth=0;
-            
-                // Strategy 1: Keep going up until we hit an ID tag, imagine it's like  #list-widget div h4
-                  while (r.parentNode) {
-                    if(depth==5) {
-                      break;
-                    }
-                    if('' !==r.id) {
-                      chained_css.unshift("#"+CSS.escape(r.id));
-                      final_selector= chained_css.join(' > ');
-                      // Be sure theres only one, some sites have multiples of the same ID tag :-(
-                      if (window.document.querySelectorAll(final_selector).length ==1 ) {
-                        return final_selector;
-                        }
-                      return null;
-                    } else {
-                      chained_css.unshift(r.tagName.toLowerCase());
-                    }
-                    r=r.parentNode;
-                    depth+=1;
-                  }
-                  return null;
-                }
-
-
-                // @todo - if it's SVG or IMG, go into image diff mode
-                var elements = window.document.querySelectorAll("div,span,form,table,tbody,tr,td,a,p,ul,li,h1,h2,h3,h4, header, footer, section, article, aside, details, main, nav, section, summary");
-                var size_pos=[];
-                // after page fetch, inject this JS
-                // build a map of all elements and their positions (maybe that only include text?)
-                var bbox;
-                for (var i = 0; i < elements.length; i++) {   
-                 bbox = elements[i].getBoundingClientRect();
-
-                 // forget really small ones
-                 if (bbox['width'] <20 && bbox['height'] < 20 ) {
-                   continue;
-                 }
-
-                 // @todo the getXpath kind of sucks, it doesnt know when there is for example just one ID sometimes
-                 // it should not traverse when we know we can anchor off just an ID one level up etc..
-                 // maybe, get current class or id, keep traversing up looking for only class or id until there is just one match 
-
-                 // 1st primitive - if it has class, try joining it all and select, if theres only one.. well thats us.
-                 xpath_result=false;
-                 
-                 try {
-                   var d= findUpTag(elements[i]);
-                   if (d) {
-                     xpath_result =d;
-                   }                
-                 } catch (e) {
-                   console.log(e);
-                 }
-                 
-                 // You could swap it and default to getXpath and then try the smarter one
-                 // default back to the less intelligent one
-                 if (!xpath_result) {
-                    try {
-                       // I've seen on FB and eBay that this doesnt work
-                       // ReferenceError: getXPath is not defined at eval (eval at evaluate (:152:29), <anonymous>:67:20) at UtilityScript.evaluate (<anonymous>:159:18) at UtilityScript.<anonymous> (<anonymous>:1:44)
-                       xpath_result = getXPath(elements[i]);
-                     } catch (e) {
-                       console.log(e);
-                       continue;
-                     }            
-                 }
-                 
-                 if(window.getComputedStyle(elements[i]).visibility === "hidden") {
-                   continue;
-                 }
-
-                 size_pos.push({
-                   xpath: xpath_result,
-                   width: Math.round(bbox['width']), 
-                   height: Math.round(bbox['height']), 
-                   left: Math.floor(bbox['left']), 
-                   top: Math.floor(bbox['top']), 
-                   childCount: elements[i].childElementCount
-                 });                 
-                }
-
-
-                // inject the current one set in the css_filter, which may be a CSS rule
-                // used for displaying the current one in VisualSelector, where its not one we generated.
-                if (css_filter.length) {
-                   q=false;                   
-                   try {
-                       // is it xpath?
-                       if (css_filter.startsWith('/') || css_filter.startsWith('xpath:')) {
-                         q=document.evaluate(css_filter.replace('xpath:',''), document, null, XPathResult.FIRST_ORDERED_NODE_TYPE, null).singleNodeValue;
-                       } else {
-                         q=document.querySelector(css_filter);
-                       }                       
-                   } catch (e) {
-                    // Maybe catch DOMException and alert? 
-                     console.log(e);                       
-                   }
-                   bbox=false;
-                   if(q) {
-                     bbox = q.getBoundingClientRect();
-                   }
-                                   
-                   if (bbox && bbox['width'] >0 && bbox['height']>0) {                       
-                       size_pos.push({
-                           xpath: css_filter,
-                           width: bbox['width'], 
-                           height: bbox['height'],
-                           left: bbox['left'],
-                           top: bbox['top'],
-                           childCount: q.childElementCount
-                         });
-                     }
-                }
-                // Window.width required for proper scaling in the frontend
-                return {'size_pos':size_pos, 'browser_width': window.innerWidth};
-    """
-    xpath_data = None
-
-    # Will be needed in the future by the VisualSelector, always get this where possible.
-    screenshot = False
-    system_http_proxy = os.getenv('HTTP_PROXY')
-    system_https_proxy = os.getenv('HTTPS_PROXY')
-
-    # Time ONTOP of the system defined env minimum time
-    render_extract_delay=0
-
-    @abstractmethod
-    def get_error(self):
-        return self.error
-
-    @abstractmethod
-    def run(self,
-            url,
-            timeout,
-            request_headers,
-            request_body,
-            request_method,
-            ignore_status_codes=False,
-            current_css_filter=None):
-        # Should set self.error, self.status_code and self.content
-        pass
-
-    @abstractmethod
-    def quit(self):
-        return
-
-    @abstractmethod
-    def get_last_status_code(self):
-        return self.status_code
-
-    @abstractmethod
-    # Return true/false if this checker is ready to run, in the case it needs todo some special config check etc
-    def is_ready(self):
-        return True
-
-
-#   Maybe for the future, each fetcher provides its own diff output, could be used for text, image
-#   the current one would return javascript output (as we use JS to generate the diff)
-#
-def available_fetchers():
-    # See the if statement at the bottom of this file for how we switch between playwright and webdriver
-    import inspect
-    p = []
-    for name, obj in inspect.getmembers(sys.modules[__name__], inspect.isclass):
-        if inspect.isclass(obj):
-            # @todo html_ is maybe better as fetcher_ or something
-            # In this case, make sure to edit the default one in store.py and fetch_site_status.py
-            if name.startswith('html_'):
-                t = tuple([name, obj.fetcher_description])
-                p.append(t)
-
-    return p
-
-
-class base_html_playwright(Fetcher):
-    fetcher_description = "Playwright {}/Javascript".format(
-        os.getenv("PLAYWRIGHT_BROWSER_TYPE", 'chromium').capitalize()
-    )
-    if os.getenv("PLAYWRIGHT_DRIVER_URL"):
-        fetcher_description += " via '{}'".format(os.getenv("PLAYWRIGHT_DRIVER_URL"))
-
-    browser_type = ''
-    command_executor = ''
-
-    # Configs for Proxy setup
-    # In the ENV vars, is prefixed with "playwright_proxy_", so it is for example "playwright_proxy_server"
-    playwright_proxy_settings_mappings = ['bypass', 'server', 'username', 'password']
-
-    proxy = None
-
-    def __init__(self, proxy_override=None):
-
-        # .strip('"') is going to save someone a lot of time when they accidently wrap the env value
-        self.browser_type = os.getenv("PLAYWRIGHT_BROWSER_TYPE", 'chromium').strip('"')
-        self.command_executor = os.getenv(
-            "PLAYWRIGHT_DRIVER_URL",
-            'ws://playwright-chrome:3000'
-        ).strip('"')
-
-        # If any proxy settings are enabled, then we should setup the proxy object
-        proxy_args = {}
-        for k in self.playwright_proxy_settings_mappings:
-            v = os.getenv('playwright_proxy_' + k, False)
-            if v:
-                proxy_args[k] = v.strip('"')
-
-        if proxy_args:
-            self.proxy = proxy_args
-
-        # allow per-watch proxy selection override
-        if proxy_override:
-            self.proxy = {'server': proxy_override}
-
-    def run(self,
-            url,
-            timeout,
-            request_headers,
-            request_body,
-            request_method,
-            ignore_status_codes=False,
-            current_css_filter=None):
-
-        from playwright.sync_api import sync_playwright
-        import playwright._impl._api_types
-        from playwright._impl._api_types import Error, TimeoutError
-        response = None
-        with sync_playwright() as p:
-            browser_type = getattr(p, self.browser_type)
-
-            # Seemed to cause a connection Exception even tho I can see it connect
-            # self.browser = browser_type.connect(self.command_executor, timeout=timeout*1000)
-            # 60,000 connection timeout only
-            browser = browser_type.connect_over_cdp(self.command_executor, timeout=60000)
-
-            # Set user agent to prevent Cloudflare from blocking the browser
-            # Use the default one configured in the App.py model that's passed from fetch_site_status.py
-            context = browser.new_context(
-                user_agent=request_headers['User-Agent'] if request_headers.get('User-Agent') else 'Mozilla/5.0',
-                proxy=self.proxy,
-                # This is needed to enable JavaScript execution on GitHub and others
-                bypass_csp=True,
-                # Should never be needed
-                accept_downloads=False
-            )
-
-            if len(request_headers):
-                context.set_extra_http_headers(request_headers)
-
-            page = context.new_page()
-            try:
-                page.set_default_navigation_timeout(90000)
-                page.set_default_timeout(90000)
-
-                # Listen for all console events and handle errors
-                page.on("console", lambda msg: print(f"Playwright console: Watch URL: {url} {msg.type}: {msg.text} {msg.args}"))
-
-                # Bug - never set viewport size BEFORE page.goto
-
-                # Waits for the next navigation. Using Python context manager
-                # prevents a race condition between clicking and waiting for a navigation.
-                with page.expect_navigation():
-                    response = page.goto(url, wait_until='load')
-
-                if self.webdriver_js_execute_code is not None:
-                    page.evaluate(self.webdriver_js_execute_code)
-
-            except playwright._impl._api_types.TimeoutError as e:
-                context.close()
-                browser.close()
-                # This can be ok, we will try to grab what we could retrieve
-                pass
-            except Exception as e:
-                print ("other exception when page.goto")
-                print (str(e))
-                context.close()
-                browser.close()
-                raise PageUnloadable(url=url, status_code=None)
-
-            if response is None:
-                context.close()
-                browser.close()
-                print ("response object was none")
-                raise EmptyReply(url=url, status_code=None)
-
-            # Bug 2(?) Set the viewport size AFTER loading the page
-            page.set_viewport_size({"width": 1280, "height": 1024})            
-            extra_wait = int(os.getenv("WEBDRIVER_DELAY_BEFORE_CONTENT_READY", 5)) + self.render_extract_delay
-            time.sleep(extra_wait)
-            self.content = page.content()
-            self.status_code = response.status
-
-            if len(self.content.strip()) == 0:
-                context.close()
-                browser.close()
-                print ("Content was empty")
-                raise EmptyReply(url=url, status_code=None)
-            
-            self.headers = response.all_headers()
-
-            if current_css_filter is not None:
-                page.evaluate("var css_filter={}".format(json.dumps(current_css_filter)))
-            else:
-                page.evaluate("var css_filter=''")
-
-            self.xpath_data = page.evaluate("async () => {" + self.xpath_element_js + "}")
-
-            # Bug 3 in Playwright screenshot handling
-            # Some bug where it gives the wrong screenshot size, but making a request with the clip set first seems to solve it
-            # JPEG is better here because the screenshots can be very very large
-
-            # Screenshots also travel via the ws:// (websocket) meaning that the binary data is base64 encoded
-            # which will significantly increase the IO size between the server and client, it's recommended to use the lowest
-            # acceptable screenshot quality here
-            try:
-                # Quality set to 1 because it's not used, just used as a work-around for a bug, no need to change this.
-                page.screenshot(type='jpeg', clip={'x': 1.0, 'y': 1.0, 'width': 1280, 'height': 1024}, quality=1)
-                # The actual screenshot
-                self.screenshot = page.screenshot(type='jpeg', full_page=True, quality=int(os.getenv("PLAYWRIGHT_SCREENSHOT_QUALITY", 72)))
-            except Exception as e:
-                context.close()
-                browser.close()
-                raise ScreenshotUnavailable(url=url, status_code=None)
-
-            context.close()
-            browser.close()
-
-
-class base_html_webdriver(Fetcher):
-    if os.getenv("WEBDRIVER_URL"):
-        fetcher_description = "WebDriver Chrome/Javascript via '{}'".format(os.getenv("WEBDRIVER_URL"))
-    else:
-        fetcher_description = "WebDriver Chrome/Javascript"
-
-    command_executor = ''
-
-    # Configs for Proxy setup
-    # In the ENV vars, is prefixed with "webdriver_", so it is for example "webdriver_sslProxy"
-    selenium_proxy_settings_mappings = ['proxyType', 'ftpProxy', 'httpProxy', 'noProxy',
-                                        'proxyAutoconfigUrl', 'sslProxy', 'autodetect',
-                                        'socksProxy', 'socksVersion', 'socksUsername', 'socksPassword']
-    proxy = None
-
-    def __init__(self, proxy_override=None):
-        from selenium.webdriver.common.proxy import Proxy as SeleniumProxy
-
-        # .strip('"') is going to save someone a lot of time when they accidently wrap the env value
-        self.command_executor = os.getenv("WEBDRIVER_URL", 'http://browser-chrome:4444/wd/hub').strip('"')
-
-        # If any proxy settings are enabled, then we should setup the proxy object
-        proxy_args = {}
-        for k in self.selenium_proxy_settings_mappings:
-            v = os.getenv('webdriver_' + k, False)
-            if v:
-                proxy_args[k] = v.strip('"')
-
-        # Map back standard HTTP_ and HTTPS_PROXY to webDriver httpProxy/sslProxy
-        if not proxy_args.get('webdriver_httpProxy') and self.system_http_proxy:
-            proxy_args['httpProxy'] = self.system_http_proxy
-        if not proxy_args.get('webdriver_sslProxy') and self.system_https_proxy:
-            proxy_args['httpsProxy'] = self.system_https_proxy
-
-        # Allows override the proxy on a per-request basis
-        if proxy_override is not None:
-            proxy_args['httpProxy'] = proxy_override
-
-        if proxy_args:
-            self.proxy = SeleniumProxy(raw=proxy_args)
-
-    def run(self,
-            url,
-            timeout,
-            request_headers,
-            request_body,
-            request_method,
-            ignore_status_codes=False,
-            current_css_filter=None):
-
-        from selenium import webdriver
-        from selenium.webdriver.common.desired_capabilities import DesiredCapabilities
-        from selenium.common.exceptions import WebDriverException
-        # request_body, request_method unused for now, until some magic in the future happens.
-
-        # check env for WEBDRIVER_URL
-        self.driver = webdriver.Remote(
-            command_executor=self.command_executor,
-            desired_capabilities=DesiredCapabilities.CHROME,
-            proxy=self.proxy)
-
-        try:
-            self.driver.get(url)
-        except WebDriverException as e:
-            # Be sure we close the session window
-            self.quit()
-            raise
-
-        self.driver.set_window_size(1280, 1024)
-        self.driver.implicitly_wait(int(os.getenv("WEBDRIVER_DELAY_BEFORE_CONTENT_READY", 5)))
-
-        if self.webdriver_js_execute_code is not None:
-            self.driver.execute_script(self.webdriver_js_execute_code)
-            # Selenium doesn't automatically wait for actions as good as Playwright, so wait again
-            self.driver.implicitly_wait(int(os.getenv("WEBDRIVER_DELAY_BEFORE_CONTENT_READY", 5)))
-
-        self.screenshot = self.driver.get_screenshot_as_png()
-
-        # @todo - how to check this? is it possible?
-        self.status_code = 200
-        # @todo somehow we should try to get this working for WebDriver
-        # raise EmptyReply(url=url, status_code=r.status_code)
-
-        # @todo - dom wait loaded?
-        time.sleep(int(os.getenv("WEBDRIVER_DELAY_BEFORE_CONTENT_READY", 5)) + self.render_extract_delay)
-        self.content = self.driver.page_source
-        self.headers = {}
-
-    # Does the connection to the webdriver work? run a test connection.
-    def is_ready(self):
-        from selenium import webdriver
-        from selenium.webdriver.common.desired_capabilities import DesiredCapabilities
-        from selenium.common.exceptions import WebDriverException
-
-        self.driver = webdriver.Remote(
-            command_executor=self.command_executor,
-            desired_capabilities=DesiredCapabilities.CHROME)
-
-        # driver.quit() seems to cause better exceptions
-        self.quit()
-        return True
-
-    def quit(self):
-        if self.driver:
-            try:
-                self.driver.quit()
-            except Exception as e:
-                print("Exception in chrome shutdown/quit" + str(e))
-
-
-# "html_requests" is listed as the default fetcher in store.py!
-class html_requests(Fetcher):
-    fetcher_description = "Basic fast Plaintext/HTTP Client"
-
-    def __init__(self, proxy_override=None):
-        self.proxy_override = proxy_override
-
-    def run(self,
-            url,
-            timeout,
-            request_headers,
-            request_body,
-            request_method,
-            ignore_status_codes=False,
-            current_css_filter=None):
-
-        proxies={}
-
-        # Allows override the proxy on a per-request basis
-        if self.proxy_override:
-            proxies = {'http': self.proxy_override, 'https': self.proxy_override, 'ftp': self.proxy_override}
-        else:
-            if self.system_http_proxy:
-                proxies['http'] = self.system_http_proxy
-            if self.system_https_proxy:
-                proxies['https'] = self.system_https_proxy
-
-        r = requests.request(method=request_method,
-                             data=request_body,
-                             url=url,
-                             headers=request_headers,
-                             timeout=timeout,
-                             proxies=proxies,
-                             verify=False)
-
-        # If the response did not tell us what encoding format to expect, Then use chardet to override what `requests` thinks.
-        # For example - some sites don't tell us it's utf-8, but return utf-8 content
-        # This seems to not occur when using webdriver/selenium, it seems to detect the text encoding more reliably.
-        # https://github.com/psf/requests/issues/1604 good info about requests encoding detection
-        if not r.headers.get('content-type') or not 'charset=' in r.headers.get('content-type'):
-            encoding = chardet.detect(r.content)['encoding']
-            if encoding:
-                r.encoding = encoding
-
-        # @todo test this
-        # @todo maybe you really want to test zero-byte return pages?
-        if (not ignore_status_codes and not r) or not r.content or not len(r.content):
-            raise EmptyReply(url=url, status_code=r.status_code)
-
-        self.status_code = r.status_code
-        self.content = r.text
-        self.headers = r.headers
-
-
-# Decide which is the 'real' HTML webdriver, this is more a system wide config
-# rather than site-specific.
-use_playwright_as_chrome_fetcher = os.getenv('PLAYWRIGHT_DRIVER_URL', False)
-if use_playwright_as_chrome_fetcher:
-    html_webdriver = base_html_playwright
-else:
-    html_webdriver = base_html_webdriver

changedetectionio/content_fetchers/__init__.py

@@ -0,0 +1,64 @@
+import sys
+from changedetectionio.strtobool import strtobool
+from loguru import logger
+from changedetectionio.content_fetchers.exceptions import BrowserStepsStepException
+import os
+
+# Visual Selector scraper - 'Button' is there because some sites have <button>OUT OF STOCK</button>.
+visualselector_xpath_selectors = 'div,span,form,table,tbody,tr,td,a,p,ul,li,h1,h2,h3,h4,header,footer,section,article,aside,details,main,nav,section,summary,button'
+
+SCREENSHOT_MAX_HEIGHT_DEFAULT = 20000
+SCREENSHOT_DEFAULT_QUALITY = 40
+
+# Maximum total height for the final image (When in stitch mode).
+# We limit this to 16000px due to the huge amount of RAM that was being used
+# Example: 16000 × 1400 × 3 = 67,200,000 bytes ≈ 64.1 MB (not including buffers in PIL etc)
+SCREENSHOT_MAX_TOTAL_HEIGHT = int(os.getenv("SCREENSHOT_MAX_HEIGHT", SCREENSHOT_MAX_HEIGHT_DEFAULT))
+
+# The size at which we will switch to stitching method, when below this (and
+# MAX_TOTAL_HEIGHT which can be set by a user) we will use the default
+# screenshot method.
+SCREENSHOT_SIZE_STITCH_THRESHOLD = 8000
+
+# available_fetchers() will scan this implementation looking for anything starting with html_
+# this information is used in the form selections
+from changedetectionio.content_fetchers.requests import fetcher as html_requests
+
+
+import importlib.resources
+XPATH_ELEMENT_JS = importlib.resources.files("changedetectionio.content_fetchers.res").joinpath('xpath_element_scraper.js').read_text(encoding='utf-8')
+INSTOCK_DATA_JS = importlib.resources.files("changedetectionio.content_fetchers.res").joinpath('stock-not-in-stock.js').read_text(encoding='utf-8')
+FAVICON_FETCHER_JS = importlib.resources.files("changedetectionio.content_fetchers.res").joinpath('favicon-fetcher.js').read_text(encoding='utf-8')
+
+
+def available_fetchers():
+    # See the if statement at the bottom of this file for how we switch between playwright and webdriver
+    import inspect
+    p = []
+    for name, obj in inspect.getmembers(sys.modules[__name__], inspect.isclass):
+        if inspect.isclass(obj):
+            # @todo html_ is maybe better as fetcher_ or something
+            # In this case, make sure to edit the default one in store.py and fetch_site_status.py
+            if name.startswith('html_'):
+                t = tuple([name, obj.fetcher_description])
+                p.append(t)
+
+    return p
+
+
+# Decide which is the 'real' HTML webdriver, this is more a system wide config
+# rather than site-specific.
+use_playwright_as_chrome_fetcher = os.getenv('PLAYWRIGHT_DRIVER_URL', False)
+if use_playwright_as_chrome_fetcher:
+    # @note - For now, browser steps always uses playwright
+    if not strtobool(os.getenv('FAST_PUPPETEER_CHROME_FETCHER', 'False')):
+        logger.debug('Using Playwright library as fetcher')
+        from .playwright import fetcher as html_webdriver
+    else:
+        logger.debug('Using direct Python Puppeteer library as fetcher')
+        from .puppeteer import fetcher as html_webdriver
+
+else:
+    logger.debug("Falling back to selenium as fetcher")
+    from .webdriver_selenium import fetcher as html_webdriver
+

changedetectionio/content_fetchers/base.py

@@ -0,0 +1,178 @@
+import os
+from abc import abstractmethod
+from loguru import logger
+
+from changedetectionio.content_fetchers import BrowserStepsStepException
+
+
+def manage_user_agent(headers, current_ua=''):
+    """
+    Basic setting of user-agent
+
+    NOTE!!!!!! The service that does the actual Chrome fetching should handle any anti-robot techniques
+    THERE ARE MANY WAYS THAT IT CAN BE DETECTED AS A ROBOT!!
+    This does not take care of
+    - Scraping of 'navigator' (platform, productSub, vendor, oscpu etc etc) browser object (navigator.appVersion) etc
+    - TCP/IP fingerprint JA3 etc
+    - Graphic rendering fingerprinting
+    - Your IP being obviously in a pool of bad actors
+    - Too many requests
+    - Scraping of SCH-UA browser replies (thanks google!!)
+    - Scraping of ServiceWorker, new window calls etc
+
+    See https://filipvitas.medium.com/how-to-set-user-agent-header-with-puppeteer-js-and-not-fail-28c7a02165da
+    Puppeteer requests https://github.com/dgtlmoon/pyppeteerstealth
+
+    :param page:
+    :param headers:
+    :return:
+    """
+    # Ask it what the user agent is, if its obviously ChromeHeadless, switch it to the default
+    ua_in_custom_headers = headers.get('User-Agent')
+    if ua_in_custom_headers:
+        return ua_in_custom_headers
+
+    if not ua_in_custom_headers and current_ua:
+        current_ua = current_ua.replace('HeadlessChrome', 'Chrome')
+        return current_ua
+
+    return None
+
+
+class Fetcher():
+    browser_connection_is_custom = None
+    browser_connection_url = None
+    browser_steps = None
+    browser_steps_screenshot_path = None
+    content = None
+    error = None
+    fetcher_description = "No description"
+    headers = {}
+    favicon_blob = None
+    instock_data = None
+    instock_data_js = ""
+    status_code = None
+    webdriver_js_execute_code = None
+    xpath_data = None
+    xpath_element_js = ""
+
+    # Will be needed in the future by the VisualSelector, always get this where possible.
+    screenshot = False
+    system_http_proxy = os.getenv('HTTP_PROXY')
+    system_https_proxy = os.getenv('HTTPS_PROXY')
+
+    # Time ONTOP of the system defined env minimum time
+    render_extract_delay = 0
+
+    @abstractmethod
+    def get_error(self):
+        return self.error
+
+    @abstractmethod
+    async def run(self,
+                  fetch_favicon=True,
+                  current_include_filters=None,
+                  empty_pages_are_a_change=False,
+                  ignore_status_codes=False,
+                  is_binary=False,
+                  request_body=None,
+                  request_headers=None,
+                  request_method=None,
+                  timeout=None,
+                  url=None,
+                  ):
+        # Should set self.error, self.status_code and self.content
+        pass
+
+    @abstractmethod
+    def quit(self, watch=None):
+        return
+
+    @abstractmethod
+    def get_last_status_code(self):
+        return self.status_code
+
+    @abstractmethod
+    def screenshot_step(self, step_n):
+        if self.browser_steps_screenshot_path and not os.path.isdir(self.browser_steps_screenshot_path):
+            logger.debug(f"> Creating data dir {self.browser_steps_screenshot_path}")
+            os.mkdir(self.browser_steps_screenshot_path)
+        return None
+
+    @abstractmethod
+    # Return true/false if this checker is ready to run, in the case it needs todo some special config check etc
+    def is_ready(self):
+        return True
+
+    def get_all_headers(self):
+        """
+        Get all headers but ensure all keys are lowercase
+        :return:
+        """
+        return {k.lower(): v for k, v in self.headers.items()}
+
+    def browser_steps_get_valid_steps(self):
+        if self.browser_steps is not None and len(self.browser_steps):
+            valid_steps = list(filter(
+                lambda s: (s['operation'] and len(s['operation']) and s['operation'] != 'Choose one'),
+                self.browser_steps))
+
+            # Just incase they selected Goto site by accident with older JS
+            if valid_steps and valid_steps[0]['operation'] == 'Goto site':
+                del(valid_steps[0])
+
+            return valid_steps
+
+        return None
+
+    async def iterate_browser_steps(self, start_url=None):
+        from changedetectionio.blueprint.browser_steps.browser_steps import steppable_browser_interface
+        from playwright._impl._errors import TimeoutError, Error
+        from changedetectionio.safe_jinja import render as jinja_render
+        step_n = 0
+
+        if self.browser_steps is not None and len(self.browser_steps):
+            interface = steppable_browser_interface(start_url=start_url)
+            interface.page = self.page
+            valid_steps = self.browser_steps_get_valid_steps()
+
+            for step in valid_steps:
+                step_n += 1
+                logger.debug(f">> Iterating check - browser Step n {step_n} - {step['operation']}...")
+                await self.screenshot_step("before-" + str(step_n))
+                await self.save_step_html("before-" + str(step_n))
+
+                try:
+                    optional_value = step['optional_value']
+                    selector = step['selector']
+                    # Support for jinja2 template in step values, with date module added
+                    if '{%' in step['optional_value'] or '{{' in step['optional_value']:
+                        optional_value = jinja_render(template_str=step['optional_value'])
+                    if '{%' in step['selector'] or '{{' in step['selector']:
+                        selector = jinja_render(template_str=step['selector'])
+
+                    await getattr(interface, "call_action")(action_name=step['operation'],
+                                                      selector=selector,
+                                                      optional_value=optional_value)
+                    await self.screenshot_step(step_n)
+                    await self.save_step_html(step_n)
+                except (Error, TimeoutError) as e:
+                    logger.debug(str(e))
+                    # Stop processing here
+                    raise BrowserStepsStepException(step_n=step_n, original_e=e)
+
+    # It's always good to reset these
+    def delete_browser_steps_screenshots(self):
+        import glob
+        if self.browser_steps_screenshot_path is not None:
+            dest = os.path.join(self.browser_steps_screenshot_path, 'step_*.jpeg')
+            files = glob.glob(dest)
+            for f in files:
+                if os.path.isfile(f):
+                    os.unlink(f)
+
+    def save_step_html(self, step_n):
+        if self.browser_steps_screenshot_path and not os.path.isdir(self.browser_steps_screenshot_path):
+            logger.debug(f"> Creating data dir {self.browser_steps_screenshot_path}")
+            os.mkdir(self.browser_steps_screenshot_path)
+        pass

changedetectionio/content_fetchers/exceptions/__init__.py

@@ -0,0 +1,97 @@
+from loguru import logger
+
+class Non200ErrorCodeReceived(Exception):
+    def __init__(self, status_code, url, screenshot=None, xpath_data=None, page_html=None):
+        # Set this so we can use it in other parts of the app
+        self.status_code = status_code
+        self.url = url
+        self.screenshot = screenshot
+        self.xpath_data = xpath_data
+        self.page_text = None
+
+        if page_html:
+            from changedetectionio import html_tools
+            self.page_text = html_tools.html_to_text(page_html)
+        return
+
+
+class checksumFromPreviousCheckWasTheSame(Exception):
+    def __init__(self):
+        return
+
+
+class JSActionExceptions(Exception):
+    def __init__(self, status_code, url, screenshot, message=''):
+        self.status_code = status_code
+        self.url = url
+        self.screenshot = screenshot
+        self.message = message
+        return
+
+class BrowserConnectError(Exception):
+    msg = ''
+    def __init__(self, msg):
+        self.msg = msg
+        logger.error(f"Browser connection error {msg}")
+        return
+
+class BrowserFetchTimedOut(Exception):
+    msg = ''
+    def __init__(self, msg):
+        self.msg = msg
+        logger.error(f"Browser processing took too long - {msg}")
+        return
+
+class BrowserStepsStepException(Exception):
+    def __init__(self, step_n, original_e):
+        self.step_n = step_n
+        self.original_e = original_e
+        logger.debug(f"Browser Steps exception at step {self.step_n} {str(original_e)}")
+        return
+
+
+# @todo - make base Exception class that announces via logger()
+class PageUnloadable(Exception):
+    def __init__(self, status_code=None, url='', message='', screenshot=False):
+        # Set this so we can use it in other parts of the app
+        self.status_code = status_code
+        self.url = url
+        self.screenshot = screenshot
+        self.message = message
+        return
+
+class BrowserStepsInUnsupportedFetcher(Exception):
+    def __init__(self, url):
+        self.url = url
+        return
+
+class EmptyReply(Exception):
+    def __init__(self, status_code, url, screenshot=None):
+        # Set this so we can use it in other parts of the app
+        self.status_code = status_code
+        self.url = url
+        self.screenshot = screenshot
+        return
+
+
+class ScreenshotUnavailable(Exception):
+    def __init__(self, status_code, url, page_html=None):
+        # Set this so we can use it in other parts of the app
+        self.status_code = status_code
+        self.url = url
+        if page_html:
+            from changedetectionio.html_tools import html_to_text
+            self.page_text = html_to_text(page_html)
+        return
+
+
+class ReplyWithContentButNoText(Exception):
+    def __init__(self, status_code, url, screenshot=None, has_filters=False, html_content='', xpath_data=None):
+        # Set this so we can use it in other parts of the app
+        self.status_code = status_code
+        self.url = url
+        self.screenshot = screenshot
+        self.has_filters = has_filters
+        self.html_content = html_content
+        self.xpath_data = xpath_data
+        return

changedetectionio/content_fetchers/playwright.py

@@ -0,0 +1,333 @@
+import json
+import os
+from urllib.parse import urlparse
+
+from loguru import logger
+
+from changedetectionio.content_fetchers import SCREENSHOT_MAX_HEIGHT_DEFAULT, visualselector_xpath_selectors, \
+    SCREENSHOT_SIZE_STITCH_THRESHOLD, SCREENSHOT_MAX_TOTAL_HEIGHT, XPATH_ELEMENT_JS, INSTOCK_DATA_JS, FAVICON_FETCHER_JS
+from changedetectionio.content_fetchers.base import Fetcher, manage_user_agent
+from changedetectionio.content_fetchers.exceptions import PageUnloadable, Non200ErrorCodeReceived, EmptyReply, ScreenshotUnavailable
+
+async def capture_full_page_async(page):
+    import os
+    import time
+    from multiprocessing import Process, Pipe
+
+    start = time.time()
+
+    page_height = await page.evaluate("document.documentElement.scrollHeight")
+    page_width = await page.evaluate("document.documentElement.scrollWidth")
+    original_viewport = page.viewport_size
+
+    logger.debug(f"Playwright viewport size {page.viewport_size} page height {page_height} page width {page_width}")
+
+    # Use an approach similar to puppeteer: set a larger viewport and take screenshots in chunks
+    step_size = SCREENSHOT_SIZE_STITCH_THRESHOLD # Size that won't cause GPU to overflow
+    screenshot_chunks = []
+    y = 0
+
+    if page_height > page.viewport_size['height']:
+        if page_height < step_size:
+            step_size = page_height # Incase page is bigger than default viewport but smaller than proposed step size
+        logger.debug(f"Setting bigger viewport to step through large page width W{page.viewport_size['width']}xH{step_size} because page_height > viewport_size")
+        # Set viewport to a larger size to capture more content at once
+        await page.set_viewport_size({'width': page.viewport_size['width'], 'height': step_size})
+
+    # Capture screenshots in chunks up to the max total height
+    while y < min(page_height, SCREENSHOT_MAX_TOTAL_HEIGHT):
+        await page.request_gc()
+        await page.evaluate(f"window.scrollTo(0, {y})")
+        await page.request_gc()
+        screenshot_chunks.append(await page.screenshot(
+            type="jpeg",
+            full_page=False,
+            quality=int(os.getenv("SCREENSHOT_QUALITY", 72))
+        ))
+        y += step_size
+        await page.request_gc()
+
+    # Restore original viewport size
+    await page.set_viewport_size({'width': original_viewport['width'], 'height': original_viewport['height']})
+
+    # If we have multiple chunks, stitch them together
+    if len(screenshot_chunks) > 1:
+        from changedetectionio.content_fetchers.screenshot_handler import stitch_images_worker
+        logger.debug(f"Screenshot stitching {len(screenshot_chunks)} chunks together")
+        parent_conn, child_conn = Pipe()
+        p = Process(target=stitch_images_worker, args=(child_conn, screenshot_chunks, page_height, SCREENSHOT_MAX_TOTAL_HEIGHT))
+        p.start()
+        screenshot = parent_conn.recv_bytes()
+        p.join()
+        logger.debug(
+            f"Screenshot (chunked/stitched) - Page height: {page_height} Capture height: {SCREENSHOT_MAX_TOTAL_HEIGHT} - Stitched together in {time.time() - start:.2f}s")
+        # Explicit cleanup
+        del screenshot_chunks
+        del p
+        del parent_conn, child_conn
+        screenshot_chunks = None
+        return screenshot
+
+    logger.debug(
+        f"Screenshot Page height: {page_height} Capture height: {SCREENSHOT_MAX_TOTAL_HEIGHT} - Stitched together in {time.time() - start:.2f}s")
+
+    return screenshot_chunks[0]
+
+class fetcher(Fetcher):
+    fetcher_description = "Playwright {}/Javascript".format(
+        os.getenv("PLAYWRIGHT_BROWSER_TYPE", 'chromium').capitalize()
+    )
+    if os.getenv("PLAYWRIGHT_DRIVER_URL"):
+        fetcher_description += " via '{}'".format(os.getenv("PLAYWRIGHT_DRIVER_URL"))
+
+    browser_type = ''
+    command_executor = ''
+
+    # Configs for Proxy setup
+    # In the ENV vars, is prefixed with "playwright_proxy_", so it is for example "playwright_proxy_server"
+    playwright_proxy_settings_mappings = ['bypass', 'server', 'username', 'password']
+
+    proxy = None
+
+    def __init__(self, proxy_override=None, custom_browser_connection_url=None):
+        super().__init__()
+
+        self.browser_type = os.getenv("PLAYWRIGHT_BROWSER_TYPE", 'chromium').strip('"')
+
+        if custom_browser_connection_url:
+            self.browser_connection_is_custom = True
+            self.browser_connection_url = custom_browser_connection_url
+        else:
+            # Fallback to fetching from system
+            # .strip('"') is going to save someone a lot of time when they accidently wrap the env value
+            self.browser_connection_url = os.getenv("PLAYWRIGHT_DRIVER_URL", 'ws://playwright-chrome:3000').strip('"')
+
+        # If any proxy settings are enabled, then we should setup the proxy object
+        proxy_args = {}
+        for k in self.playwright_proxy_settings_mappings:
+            v = os.getenv('playwright_proxy_' + k, False)
+            if v:
+                proxy_args[k] = v.strip('"')
+
+        if proxy_args:
+            self.proxy = proxy_args
+
+        # allow per-watch proxy selection override
+        if proxy_override:
+            self.proxy = {'server': proxy_override}
+
+        if self.proxy:
+            # Playwright needs separate username and password values
+            parsed = urlparse(self.proxy.get('server'))
+            if parsed.username:
+                self.proxy['username'] = parsed.username
+                self.proxy['password'] = parsed.password
+
+    async def screenshot_step(self, step_n=''):
+        super().screenshot_step(step_n=step_n)
+        screenshot = await capture_full_page_async(page=self.page)
+
+
+        if self.browser_steps_screenshot_path is not None:
+            destination = os.path.join(self.browser_steps_screenshot_path, 'step_{}.jpeg'.format(step_n))
+            logger.debug(f"Saving step screenshot to {destination}")
+            with open(destination, 'wb') as f:
+                f.write(screenshot)
+
+    async def save_step_html(self, step_n):
+        super().save_step_html(step_n=step_n)
+        content = await self.page.content()
+        destination = os.path.join(self.browser_steps_screenshot_path, 'step_{}.html'.format(step_n))
+        logger.debug(f"Saving step HTML to {destination}")
+        with open(destination, 'w') as f:
+            f.write(content)
+
+    async def run(self,
+                  fetch_favicon=True,
+                  current_include_filters=None,
+                  empty_pages_are_a_change=False,
+                  ignore_status_codes=False,
+                  is_binary=False,
+                  request_body=None,
+                  request_headers=None,
+                  request_method=None,
+                  timeout=None,
+                  url=None,
+                  ):
+
+        from playwright.async_api import async_playwright
+        import playwright._impl._errors
+        import time
+        self.delete_browser_steps_screenshots()
+        response = None
+
+        async with async_playwright() as p:
+            browser_type = getattr(p, self.browser_type)
+
+            # Seemed to cause a connection Exception even tho I can see it connect
+            # self.browser = browser_type.connect(self.command_executor, timeout=timeout*1000)
+            # 60,000 connection timeout only
+            browser = await browser_type.connect_over_cdp(self.browser_connection_url, timeout=60000)
+
+            # SOCKS5 with authentication is not supported (yet)
+            # https://github.com/microsoft/playwright/issues/10567
+
+            # Set user agent to prevent Cloudflare from blocking the browser
+            # Use the default one configured in the App.py model that's passed from fetch_site_status.py
+            context = await browser.new_context(
+                accept_downloads=False,  # Should never be needed
+                bypass_csp=True,  # This is needed to enable JavaScript execution on GitHub and others
+                extra_http_headers=request_headers,
+                ignore_https_errors=True,
+                proxy=self.proxy,
+                service_workers=os.getenv('PLAYWRIGHT_SERVICE_WORKERS', 'allow'), # Should be `allow` or `block` - sites like YouTube can transmit large amounts of data via Service Workers
+                user_agent=manage_user_agent(headers=request_headers),
+            )
+
+            self.page = await context.new_page()
+
+            # Listen for all console events and handle errors
+            self.page.on("console", lambda msg: logger.debug(f"Playwright console: Watch URL: {url} {msg.type}: {msg.text} {msg.args}"))
+
+            # Re-use as much code from browser steps as possible so its the same
+            from changedetectionio.blueprint.browser_steps.browser_steps import steppable_browser_interface
+            browsersteps_interface = steppable_browser_interface(start_url=url)
+            browsersteps_interface.page = self.page
+
+            response = await browsersteps_interface.action_goto_url(value=url)
+
+            if response is None:
+                await context.close()
+                await browser.close()
+                logger.debug("Content Fetcher > Response object from the browser communication was none")
+                raise EmptyReply(url=url, status_code=None)
+
+            # In async_playwright, all_headers() returns a coroutine
+            try:
+                self.headers = await response.all_headers()
+            except TypeError:
+                # Fallback for sync version
+                self.headers = response.all_headers()
+
+            try:
+                if self.webdriver_js_execute_code is not None and len(self.webdriver_js_execute_code):
+                    await browsersteps_interface.action_execute_js(value=self.webdriver_js_execute_code, selector=None)
+            except playwright._impl._errors.TimeoutError as e:
+                await context.close()
+                await browser.close()
+                # This can be ok, we will try to grab what we could retrieve
+                pass
+            except Exception as e:
+                logger.debug(f"Content Fetcher > Other exception when executing custom JS code {str(e)}")
+                await context.close()
+                await browser.close()
+                raise PageUnloadable(url=url, status_code=None, message=str(e))
+
+            extra_wait = int(os.getenv("WEBDRIVER_DELAY_BEFORE_CONTENT_READY", 5)) + self.render_extract_delay
+            await self.page.wait_for_timeout(extra_wait * 1000)
+
+            try:
+                self.status_code = response.status
+            except Exception as e:
+                # https://github.com/dgtlmoon/changedetection.io/discussions/2122#discussioncomment-8241962
+                logger.critical(f"Response from the browser/Playwright did not have a status_code! Response follows.")
+                logger.critical(response)
+                await context.close()
+                await browser.close()
+                raise PageUnloadable(url=url, status_code=None, message=str(e))
+
+            if fetch_favicon:
+                try:
+                    self.favicon_blob = await self.page.evaluate(FAVICON_FETCHER_JS)
+                    await self.page.request_gc()
+                except Exception as e:
+                    logger.error(f"Error fetching FavIcon info {str(e)}, continuing.")
+
+            if self.status_code != 200 and not ignore_status_codes:
+                screenshot = await capture_full_page_async(self.page)
+                raise Non200ErrorCodeReceived(url=url, status_code=self.status_code, screenshot=screenshot)
+
+            if not empty_pages_are_a_change and len((await self.page.content()).strip()) == 0:
+                logger.debug("Content Fetcher > Content was empty, empty_pages_are_a_change = False")
+                await context.close()
+                await browser.close()
+                raise EmptyReply(url=url, status_code=response.status)
+
+            # Run Browser Steps here
+            if self.browser_steps_get_valid_steps():
+                await self.iterate_browser_steps(start_url=url)
+
+            await self.page.wait_for_timeout(extra_wait * 1000)
+
+            now = time.time()
+            # So we can find an element on the page where its selector was entered manually (maybe not xPath etc)
+            if current_include_filters is not None:
+                await self.page.evaluate("var include_filters={}".format(json.dumps(current_include_filters)))
+            else:
+                await self.page.evaluate("var include_filters=''")
+            await self.page.request_gc()
+
+            # request_gc before and after evaluate to free up memory
+            # @todo browsersteps etc
+            MAX_TOTAL_HEIGHT = int(os.getenv("SCREENSHOT_MAX_HEIGHT", SCREENSHOT_MAX_HEIGHT_DEFAULT))
+            self.xpath_data = await self.page.evaluate(XPATH_ELEMENT_JS, {
+                "visualselector_xpath_selectors": visualselector_xpath_selectors,
+                "max_height": MAX_TOTAL_HEIGHT
+            })
+            await self.page.request_gc()
+
+            self.instock_data = await self.page.evaluate(INSTOCK_DATA_JS)
+            await self.page.request_gc()
+
+            self.content = await self.page.content()
+            await self.page.request_gc()
+            logger.debug(f"Scrape xPath element data in browser done in {time.time() - now:.2f}s")
+
+
+            # Bug 3 in Playwright screenshot handling
+            # Some bug where it gives the wrong screenshot size, but making a request with the clip set first seems to solve it
+            # JPEG is better here because the screenshots can be very very large
+
+            # Screenshots also travel via the ws:// (websocket) meaning that the binary data is base64 encoded
+            # which will significantly increase the IO size between the server and client, it's recommended to use the lowest
+            # acceptable screenshot quality here
+            try:
+                # The actual screenshot - this always base64 and needs decoding! horrible! huge CPU usage
+                self.screenshot = await capture_full_page_async(page=self.page)
+
+            except Exception as e:
+                # It's likely the screenshot was too long/big and something crashed
+                raise ScreenshotUnavailable(url=url, status_code=self.status_code)
+            finally:
+                # Request garbage collection one more time before closing
+                try:
+                    await self.page.request_gc()
+                except:
+                    pass
+                
+                # Clean up resources properly
+                try:
+                    await self.page.request_gc()
+                except:
+                    pass
+
+                try:
+                    await self.page.close()
+                except:
+                    pass
+                self.page = None
+
+                try:
+                    await context.close()
+                except:
+                    pass
+                context = None
+
+                try:
+                    await browser.close()
+                except:
+                    pass
+                browser = None
+
+
+

changedetectionio/content_fetchers/puppeteer.py

@@ -0,0 +1,386 @@
+import asyncio
+import json
+import os
+import websockets.exceptions
+from urllib.parse import urlparse
+
+from loguru import logger
+
+from changedetectionio.content_fetchers import SCREENSHOT_MAX_HEIGHT_DEFAULT, visualselector_xpath_selectors, \
+    SCREENSHOT_SIZE_STITCH_THRESHOLD, SCREENSHOT_DEFAULT_QUALITY, XPATH_ELEMENT_JS, INSTOCK_DATA_JS, \
+    SCREENSHOT_MAX_TOTAL_HEIGHT, FAVICON_FETCHER_JS
+from changedetectionio.content_fetchers.base import Fetcher, manage_user_agent
+from changedetectionio.content_fetchers.exceptions import PageUnloadable, Non200ErrorCodeReceived, EmptyReply, BrowserFetchTimedOut, \
+    BrowserConnectError
+
+
+# Bug 3 in Playwright screenshot handling
+# Some bug where it gives the wrong screenshot size, but making a request with the clip set first seems to solve it
+
+# Screenshots also travel via the ws:// (websocket) meaning that the binary data is base64 encoded
+# which will significantly increase the IO size between the server and client, it's recommended to use the lowest
+# acceptable screenshot quality here
+async def capture_full_page(page):
+    import os
+    import time
+    from multiprocessing import Process, Pipe
+
+    start = time.time()
+
+    page_height = await page.evaluate("document.documentElement.scrollHeight")
+    page_width = await page.evaluate("document.documentElement.scrollWidth")
+    original_viewport = page.viewport
+
+    logger.debug(f"Puppeteer viewport size {page.viewport} page height {page_height} page width {page_width}")
+
+    # Bug 3 in Playwright screenshot handling
+    # Some bug where it gives the wrong screenshot size, but making a request with the clip set first seems to solve it
+    # JPEG is better here because the screenshots can be very very large
+
+    # Screenshots also travel via the ws:// (websocket) meaning that the binary data is base64 encoded
+    # which will significantly increase the IO size between the server and client, it's recommended to use the lowest
+    # acceptable screenshot quality here
+
+
+    step_size = SCREENSHOT_SIZE_STITCH_THRESHOLD # Something that will not cause the GPU to overflow when taking the screenshot
+    screenshot_chunks = []
+    y = 0
+    if page_height > page.viewport['height']:
+        if page_height < step_size:
+            step_size = page_height # Incase page is bigger than default viewport but smaller than proposed step size
+        await page.setViewport({'width': page.viewport['width'], 'height': step_size})
+
+    while y < min(page_height, SCREENSHOT_MAX_TOTAL_HEIGHT):
+        # better than scrollTo incase they override it in the page
+        await page.evaluate(
+            """(y) => {
+                document.documentElement.scrollTop = y;
+                document.body.scrollTop = y;
+            }""",
+            y
+        )
+
+        screenshot_chunks.append(await page.screenshot(type_='jpeg',
+                                                       fullPage=False,
+                                                       quality=int(os.getenv("SCREENSHOT_QUALITY", 72))))
+        y += step_size
+
+    await page.setViewport({'width': original_viewport['width'], 'height': original_viewport['height']})
+
+    if len(screenshot_chunks) > 1:
+        from changedetectionio.content_fetchers.screenshot_handler import stitch_images_worker
+        logger.debug(f"Screenshot stitching {len(screenshot_chunks)} chunks together")
+        parent_conn, child_conn = Pipe()
+        p = Process(target=stitch_images_worker, args=(child_conn, screenshot_chunks, page_height, SCREENSHOT_MAX_TOTAL_HEIGHT))
+        p.start()
+        screenshot = parent_conn.recv_bytes()
+        p.join()
+        logger.debug(
+            f"Screenshot (chunked/stitched) - Page height: {page_height} Capture height: {SCREENSHOT_MAX_TOTAL_HEIGHT} - Stitched together in {time.time() - start:.2f}s")
+
+        screenshot_chunks = None
+        return screenshot
+
+    logger.debug(
+        f"Screenshot Page height: {page_height} Capture height: {SCREENSHOT_MAX_TOTAL_HEIGHT} - Stitched together in {time.time() - start:.2f}s")
+    return screenshot_chunks[0]
+
+
+class fetcher(Fetcher):
+    fetcher_description = "Puppeteer/direct {}/Javascript".format(
+        os.getenv("PLAYWRIGHT_BROWSER_TYPE", 'chromium').capitalize()
+    )
+    if os.getenv("PLAYWRIGHT_DRIVER_URL"):
+        fetcher_description += " via '{}'".format(os.getenv("PLAYWRIGHT_DRIVER_URL"))
+
+    browser_type = ''
+    command_executor = ''
+
+    proxy = None
+
+    def __init__(self, proxy_override=None, custom_browser_connection_url=None):
+        super().__init__()
+
+        if custom_browser_connection_url:
+            self.browser_connection_is_custom = True
+            self.browser_connection_url = custom_browser_connection_url
+        else:
+            # Fallback to fetching from system
+            # .strip('"') is going to save someone a lot of time when they accidently wrap the env value
+            self.browser_connection_url = os.getenv("PLAYWRIGHT_DRIVER_URL", 'ws://playwright-chrome:3000').strip('"')
+
+        # allow per-watch proxy selection override
+        # @todo check global too?
+        if proxy_override:
+            # Playwright needs separate username and password values
+            parsed = urlparse(proxy_override)
+            if parsed:
+                self.proxy = {'username': parsed.username, 'password': parsed.password}
+                # Add the proxy server chrome start option, the username and password never gets added here
+                # (It always goes in via await self.page.authenticate(self.proxy))
+
+                # @todo filter some injection attack?
+                # check scheme when no scheme
+                proxy_url = parsed.scheme + "://" if parsed.scheme else 'http://'
+                r = "?" if not '?' in self.browser_connection_url else '&'
+                port = ":"+str(parsed.port) if parsed.port else ''
+                q = "?"+parsed.query if parsed.query else ''
+                proxy_url += f"{parsed.hostname}{port}{parsed.path}{q}"
+                self.browser_connection_url += f"{r}--proxy-server={proxy_url}"
+
+    # def screenshot_step(self, step_n=''):
+    #     screenshot = self.page.screenshot(type='jpeg', full_page=True, quality=85)
+    #
+    #     if self.browser_steps_screenshot_path is not None:
+    #         destination = os.path.join(self.browser_steps_screenshot_path, 'step_{}.jpeg'.format(step_n))
+    #         logger.debug(f"Saving step screenshot to {destination}")
+    #         with open(destination, 'wb') as f:
+    #             f.write(screenshot)
+    #
+    # def save_step_html(self, step_n):
+    #     content = self.page.content()
+    #     destination = os.path.join(self.browser_steps_screenshot_path, 'step_{}.html'.format(step_n))
+    #     logger.debug(f"Saving step HTML to {destination}")
+    #     with open(destination, 'w') as f:
+    #         f.write(content)
+
+    async def fetch_page(self,
+                         current_include_filters,
+                         empty_pages_are_a_change,
+                         fetch_favicon,
+                         ignore_status_codes,
+                         is_binary,
+                         request_body,
+                         request_headers,
+                         request_method,
+                         timeout,
+                         url,
+                         ):
+        import re
+        self.delete_browser_steps_screenshots()
+
+        n = int(os.getenv("WEBDRIVER_DELAY_BEFORE_CONTENT_READY", 5)) + self.render_extract_delay
+        extra_wait = min(n, 15)
+
+        logger.debug(f"Extra wait set to {extra_wait}s, requested was {n}s.")
+
+        from pyppeteer import Pyppeteer
+        pyppeteer_instance = Pyppeteer()
+
+        # Connect directly using the specified browser_ws_endpoint
+        # @todo timeout
+        try:
+            browser = await pyppeteer_instance.connect(browserWSEndpoint=self.browser_connection_url,
+                                                       ignoreHTTPSErrors=True
+                                                       )
+        except websockets.exceptions.InvalidStatusCode as e:
+            raise BrowserConnectError(msg=f"Error while trying to connect the browser, Code {e.status_code} (check your access, whitelist IP, password etc)")
+        except websockets.exceptions.InvalidURI:
+            raise BrowserConnectError(msg=f"Error connecting to the browser, check your browser connection address (should be ws:// or wss://")
+        except Exception as e:
+            raise BrowserConnectError(msg=f"Error connecting to the browser - Exception '{str(e)}'")
+
+        # more reliable is to just request a new page
+        self.page = await browser.newPage()
+        
+        # Add console handler to capture console.log from favicon fetcher
+        #self.page.on('console', lambda msg: logger.debug(f"Browser console [{msg.type}]: {msg.text}"))
+
+        if '--window-size' in self.browser_connection_url:
+            # Be sure the viewport is always the window-size, this is often not the same thing
+            match = re.search(r'--window-size=(\d+),(\d+)', self.browser_connection_url)
+            if match:
+                logger.debug(f"Setting viewport to same as --window-size in browser connection URL {int(match.group(1))},{int(match.group(2))}")
+                await self.page.setViewport({
+                    "width": int(match.group(1)),
+                    "height": int(match.group(2))
+                })
+                logger.debug(f"Puppeteer viewport size {self.page.viewport}")
+
+        try:
+            from pyppeteerstealth import inject_evasions_into_page
+        except ImportError:
+            logger.debug("pyppeteerstealth module not available, skipping")
+            pass
+        else:
+            # I tried hooking events via self.page.on(Events.Page.DOMContentLoaded, inject_evasions_requiring_obj_to_page)
+            # But I could never get it to fire reliably, so we just inject it straight after
+            await inject_evasions_into_page(self.page)
+
+        # This user agent is similar to what was used when tweaking the evasions in inject_evasions_into_page(..)
+        user_agent = None
+        if request_headers and request_headers.get('User-Agent'):
+            # Request_headers should now be CaaseInsensitiveDict
+            # Remove it so it's not sent again with headers after
+            user_agent = request_headers.pop('User-Agent').strip()
+            await self.page.setUserAgent(user_agent)
+
+        if not user_agent:
+            # Attempt to strip 'HeadlessChrome' etc
+            await self.page.setUserAgent(manage_user_agent(headers=request_headers, current_ua=await self.page.evaluate('navigator.userAgent')))
+
+        await self.page.setBypassCSP(True)
+        if request_headers:
+            await self.page.setExtraHTTPHeaders(request_headers)
+
+        # SOCKS5 with authentication is not supported (yet)
+        # https://github.com/microsoft/playwright/issues/10567
+        self.page.setDefaultNavigationTimeout(0)
+        await self.page.setCacheEnabled(True)
+        if self.proxy and self.proxy.get('username'):
+            # Setting Proxy-Authentication header is deprecated, and doing so can trigger header change errors from Puppeteer
+            # https://github.com/puppeteer/puppeteer/issues/676 ?
+            # https://help.brightdata.com/hc/en-us/articles/12632549957649-Proxy-Manager-How-to-Guides#h_01HAKWR4Q0AFS8RZTNYWRDFJC2
+            # https://cri.dev/posts/2020-03-30-How-to-solve-Puppeteer-Chrome-Error-ERR_INVALID_ARGUMENT/
+            await self.page.authenticate(self.proxy)
+
+        # Re-use as much code from browser steps as possible so its the same
+        # from changedetectionio.blueprint.browser_steps.browser_steps import steppable_browser_interface
+
+        # not yet used here, we fallback to playwright when browsersteps is required
+        #            browsersteps_interface = steppable_browser_interface()
+        #            browsersteps_interface.page = self.page
+
+        async def handle_frame_navigation(event):
+            logger.debug(f"Frame navigated: {event}")
+            w = extra_wait - 2 if extra_wait > 4 else 2
+            logger.debug(f"Waiting {w} seconds before calling Page.stopLoading...")
+            await asyncio.sleep(w)
+            logger.debug("Issuing stopLoading command...")
+            await self.page._client.send('Page.stopLoading')
+            logger.debug("stopLoading command sent!")
+
+        self.page._client.on('Page.frameStartedNavigating', lambda event: asyncio.create_task(handle_frame_navigation(event)))
+        self.page._client.on('Page.frameStartedLoading', lambda event: asyncio.create_task(handle_frame_navigation(event)))
+        self.page._client.on('Page.frameStoppedLoading', lambda event: logger.debug(f"Frame stopped loading: {event}"))
+
+        response = None
+        attempt=0
+        while not response:
+            logger.debug(f"Attempting page fetch {url} attempt {attempt}")
+            response = await self.page.goto(url)
+            await asyncio.sleep(1 + extra_wait)
+            if response:
+                break
+            if not response:
+                logger.warning("Page did not fetch! trying again!")
+            if response is None and attempt>=2:
+                await self.page.close()
+                await browser.close()
+                logger.warning(f"Content Fetcher > Response object was none (as in, the response from the browser was empty, not just the content) exiting attmpt {attempt}")
+                raise EmptyReply(url=url, status_code=None)
+            attempt+=1
+
+        self.headers = response.headers
+
+        try:
+            if self.webdriver_js_execute_code is not None and len(self.webdriver_js_execute_code):
+                await self.page.evaluate(self.webdriver_js_execute_code)
+        except Exception as e:
+            logger.warning("Got exception when running evaluate on custom JS code")
+            logger.error(str(e))
+            await self.page.close()
+            await browser.close()
+            # This can be ok, we will try to grab what we could retrieve
+            raise PageUnloadable(url=url, status_code=None, message=str(e))
+
+        try:
+            self.status_code = response.status
+        except Exception as e:
+            # https://github.com/dgtlmoon/changedetection.io/discussions/2122#discussioncomment-8241962
+            logger.critical(f"Response from the browser/Playwright did not have a status_code! Response follows.")
+            logger.critical(response)
+            await self.page.close()
+            await browser.close()
+            raise PageUnloadable(url=url, status_code=None, message=str(e))
+
+        if fetch_favicon:
+            try:
+                self.favicon_blob = await self.page.evaluate(FAVICON_FETCHER_JS)
+            except Exception as e:
+                logger.error(f"Error fetching FavIcon info {str(e)}, continuing.")
+
+        if self.status_code != 200 and not ignore_status_codes:
+            screenshot = await capture_full_page(page=self.page)
+
+            raise Non200ErrorCodeReceived(url=url, status_code=self.status_code, screenshot=screenshot)
+
+        content = await self.page.content
+
+        if not empty_pages_are_a_change and len(content.strip()) == 0:
+            logger.error("Content Fetcher > Content was empty (empty_pages_are_a_change is False), closing browsers")
+            await self.page.close()
+            await browser.close()
+            raise EmptyReply(url=url, status_code=response.status)
+
+        # Run Browser Steps here
+        # @todo not yet supported, we switch to playwright in this case
+        #            if self.browser_steps_get_valid_steps():
+        #                self.iterate_browser_steps()
+
+
+        # So we can find an element on the page where its selector was entered manually (maybe not xPath etc)
+        # Setup the xPath/VisualSelector scraper
+        if current_include_filters:
+            js = json.dumps(current_include_filters)
+            await self.page.evaluate(f"var include_filters={js}")
+        else:
+            await self.page.evaluate(f"var include_filters=''")
+
+        MAX_TOTAL_HEIGHT = int(os.getenv("SCREENSHOT_MAX_HEIGHT", SCREENSHOT_MAX_HEIGHT_DEFAULT))
+        self.xpath_data = await self.page.evaluate(XPATH_ELEMENT_JS, {
+            "visualselector_xpath_selectors": visualselector_xpath_selectors,
+            "max_height": MAX_TOTAL_HEIGHT
+        })
+        if not self.xpath_data:
+            raise Exception(f"Content Fetcher > xPath scraper failed. Please report this URL so we can fix it :)")
+
+        self.instock_data = await self.page.evaluate(INSTOCK_DATA_JS)
+
+        self.content = await self.page.content
+
+        self.screenshot = await capture_full_page(page=self.page)
+
+        # It's good to log here in the case that the browser crashes on shutting down but we still get the data we need
+        logger.success(f"Fetching '{url}' complete, closing page")
+        await self.page.close()
+        logger.success(f"Fetching '{url}' complete, closing browser")
+        await browser.close()
+        logger.success(f"Fetching '{url}' complete, exiting puppeteer fetch.")
+
+    async def main(self, **kwargs):
+        await self.fetch_page(**kwargs)
+
+    async def run(self,
+                  fetch_favicon=True,
+                  current_include_filters=None,
+                  empty_pages_are_a_change=False,
+                  ignore_status_codes=False,
+                  is_binary=False,
+                  request_body=None,
+                  request_headers=None,
+                  request_method=None,
+                  timeout=None,
+                  url=None,
+                  ):
+
+        #@todo make update_worker async which could run any of these content_fetchers within memory and time constraints
+        max_time = int(os.getenv('PUPPETEER_MAX_PROCESSING_TIMEOUT_SECONDS', 180))
+
+        # Now we run this properly in async context since we're called from async worker
+        try:
+            await asyncio.wait_for(self.main(
+                current_include_filters=current_include_filters,
+                empty_pages_are_a_change=empty_pages_are_a_change,
+                fetch_favicon=fetch_favicon,
+                ignore_status_codes=ignore_status_codes,
+                is_binary=is_binary,
+                request_body=request_body,
+                request_headers=request_headers,
+                request_method=request_method,
+                timeout=timeout,
+                url=url,
+            ), timeout=max_time
+            )
+        except asyncio.TimeoutError:
+            raise (BrowserFetchTimedOut(msg=f"Browser connected but was unable to process the page in {max_time} seconds."))

changedetectionio/content_fetchers/requests.py

@@ -0,0 +1,150 @@
+from loguru import logger
+import hashlib
+import os
+import asyncio
+from changedetectionio import strtobool
+from changedetectionio.content_fetchers.exceptions import BrowserStepsInUnsupportedFetcher, EmptyReply, Non200ErrorCodeReceived
+from changedetectionio.content_fetchers.base import Fetcher
+
+
+# "html_requests" is listed as the default fetcher in store.py!
+class fetcher(Fetcher):
+    fetcher_description = "Basic fast Plaintext/HTTP Client"
+
+    def __init__(self, proxy_override=None, custom_browser_connection_url=None):
+        super().__init__()
+        self.proxy_override = proxy_override
+        # browser_connection_url is none because its always 'launched locally'
+
+    def _run_sync(self,
+            url,
+            timeout,
+            request_headers,
+            request_body,
+            request_method,
+            ignore_status_codes=False,
+            current_include_filters=None,
+            is_binary=False,
+            empty_pages_are_a_change=False):
+        """Synchronous version of run - the original requests implementation"""
+
+        import chardet
+        import requests
+        from requests.exceptions import ProxyError, ConnectionError, RequestException
+
+        if self.browser_steps_get_valid_steps():
+            raise BrowserStepsInUnsupportedFetcher(url=url)
+
+        proxies = {}
+
+        # Allows override the proxy on a per-request basis
+        # https://requests.readthedocs.io/en/latest/user/advanced/#socks
+        # Should also work with `socks5://user:pass@host:port` type syntax.
+
+        if self.proxy_override:
+            proxies = {'http': self.proxy_override, 'https': self.proxy_override, 'ftp': self.proxy_override}
+        else:
+            if self.system_http_proxy:
+                proxies['http'] = self.system_http_proxy
+            if self.system_https_proxy:
+                proxies['https'] = self.system_https_proxy
+
+        session = requests.Session()
+
+        if strtobool(os.getenv('ALLOW_FILE_URI', 'false')) and url.startswith('file://'):
+            from requests_file import FileAdapter
+            session.mount('file://', FileAdapter())
+        try:
+            r = session.request(method=request_method,
+                                data=request_body.encode('utf-8') if type(request_body) is str else request_body,
+                                url=url,
+                                headers=request_headers,
+                                timeout=timeout,
+                                proxies=proxies,
+                                verify=False)
+        except Exception as e:
+            msg = str(e)
+            if proxies and 'SOCKSHTTPSConnectionPool' in msg:
+                msg = f"Proxy connection failed? {msg}"
+            raise Exception(msg) from e
+
+        # If the response did not tell us what encoding format to expect, Then use chardet to override what `requests` thinks.
+        # For example - some sites don't tell us it's utf-8, but return utf-8 content
+        # This seems to not occur when using webdriver/selenium, it seems to detect the text encoding more reliably.
+        # https://github.com/psf/requests/issues/1604 good info about requests encoding detection
+        if not is_binary:
+            # Don't run this for PDF (and requests identified as binary) takes a _long_ time
+            if not r.headers.get('content-type') or not 'charset=' in r.headers.get('content-type'):
+                encoding = chardet.detect(r.content)['encoding']
+                if encoding:
+                    r.encoding = encoding
+
+        self.headers = r.headers
+
+        if not r.content or not len(r.content):
+            logger.debug(f"Requests returned empty content for '{url}'")
+            if not empty_pages_are_a_change:
+                raise EmptyReply(url=url, status_code=r.status_code)
+            else:
+                logger.debug(f"URL {url} gave zero byte content reply with Status Code {r.status_code}, but empty_pages_are_a_change = True")
+
+        # @todo test this
+        # @todo maybe you really want to test zero-byte return pages?
+        if r.status_code != 200 and not ignore_status_codes:
+            # maybe check with content works?
+            raise Non200ErrorCodeReceived(url=url, status_code=r.status_code, page_html=r.text)
+
+        self.status_code = r.status_code
+        if is_binary:
+            # Binary files just return their checksum until we add something smarter
+            self.content = hashlib.md5(r.content).hexdigest()
+        else:
+            self.content = r.text
+
+        self.raw_content = r.content
+
+    async def run(self,
+                  fetch_favicon=True,
+                  current_include_filters=None,
+                  empty_pages_are_a_change=False,
+                  ignore_status_codes=False,
+                  is_binary=False,
+                  request_body=None,
+                  request_headers=None,
+                  request_method=None,
+                  timeout=None,
+                  url=None,
+                  ):
+        """Async wrapper that runs the synchronous requests code in a thread pool"""
+        
+        loop = asyncio.get_event_loop()
+        
+        # Run the synchronous _run_sync in a thread pool to avoid blocking the event loop
+        await loop.run_in_executor(
+            None,  # Use default ThreadPoolExecutor
+            lambda: self._run_sync(
+                url=url,
+                timeout=timeout,
+                request_headers=request_headers,
+                request_body=request_body,
+                request_method=request_method,
+                ignore_status_codes=ignore_status_codes,
+                current_include_filters=current_include_filters,
+                is_binary=is_binary,
+                empty_pages_are_a_change=empty_pages_are_a_change
+            )
+        )
+
+    def quit(self, watch=None):
+
+        # In case they switched to `requests` fetcher from something else
+        # Then the screenshot could be old, in any case, it's not used here.
+        # REMOVE_REQUESTS_OLD_SCREENSHOTS - Mainly used for testing
+        if strtobool(os.getenv("REMOVE_REQUESTS_OLD_SCREENSHOTS", 'true')):
+            screenshot = watch.get_screenshot()
+            if screenshot:
+                try:
+                    os.unlink(screenshot)
+                except Exception as e:
+                    logger.warning(f"Failed to unlink screenshot: {screenshot} - {e}")
+

changedetectionio/content_fetchers/res/__init__.py

@@ -0,0 +1 @@
+# resources for browser injection/scraping

changedetectionio/content_fetchers/res/favicon-fetcher.js

@@ -0,0 +1,101 @@
+(async () => {
+  // Define the function inside the IIFE for console testing
+  window.getFaviconAsBlob = async function() {
+    const links = Array.from(document.querySelectorAll(
+      'link[rel~="apple-touch-icon"], link[rel~="icon"]'
+    ));
+
+    const icons = links.map(link => {
+      const sizesStr = link.getAttribute('sizes');
+      let size = 0;
+      if (sizesStr) {
+        const [w] = sizesStr.split('x').map(Number);
+        if (!isNaN(w)) size = w;
+      } else {
+        size = 16;
+      }
+      return {
+        size,
+        rel: link.getAttribute('rel'),
+        href: link.href,
+        hasSizes: !!sizesStr
+      };
+    });
+
+    // If no icons found, add fallback favicon.ico
+    if (icons.length === 0) {
+      icons.push({
+        size: 16,
+        rel: 'icon',
+        href: '/favicon.ico',
+        hasSizes: false
+      });
+    }
+
+    // sort preference: highest resolution first, then apple-touch-icon, then regular icons
+    icons.sort((a, b) => {
+      // First priority: actual size (highest first)
+      if (a.size !== b.size) {
+        return b.size - a.size;
+      }
+      
+      // Second priority: apple-touch-icon over regular icon
+      const isAppleA = /apple-touch-icon/.test(a.rel);
+      const isAppleB = /apple-touch-icon/.test(b.rel);
+      if (isAppleA && !isAppleB) return -1;
+      if (!isAppleA && isAppleB) return 1;
+      
+      // Third priority: icons with no size attribute (fallback icons) last
+      const hasNoSizeA = !a.hasSizes;
+      const hasNoSizeB = !b.hasSizes;
+      if (hasNoSizeA && !hasNoSizeB) return 1;
+      if (!hasNoSizeA && hasNoSizeB) return -1;
+      
+      return 0;
+    });
+
+    const timeoutMs = 2000;
+
+    for (const icon of icons) {
+      try {
+        const controller = new AbortController();
+        const timeout = setTimeout(() => controller.abort(), timeoutMs);
+
+        const resp = await fetch(icon.href, {
+          signal: controller.signal,
+          redirect: 'follow'
+        });
+
+        clearTimeout(timeout);
+
+        if (!resp.ok) {
+          continue;
+        }
+
+        const blob = await resp.blob();
+
+        // Convert blob to base64
+        const reader = new FileReader();
+        return await new Promise(resolve => {
+          reader.onloadend = () => {
+            resolve({
+              url: icon.href,
+              base64: reader.result.split(",")[1]
+            });
+          };
+          reader.readAsDataURL(blob);
+        });
+
+      } catch (e) {
+        continue;
+      }
+    }
+
+    // nothing found
+    return null;
+  };
+
+  // Auto-execute and return result for page.evaluate()
+  return await window.getFaviconAsBlob();
+})();
+

changedetectionio/content_fetchers/res/stock-not-in-stock.js

@@ -0,0 +1,239 @@
+async () => {
+
+    function isItemInStock() {
+        // @todo Pass these in so the same list can be used in non-JS fetchers
+        const outOfStockTexts = [
+            ' أخبرني عندما يتوفر',
+            '0 in stock',
+            'actuellement indisponible',
+            'agotado',
+            'article épuisé',
+            'artikel zurzeit vergriffen',
+            'as soon as stock is available',
+            'aucune offre n\'est disponible',
+            'ausverkauft', // sold out
+            'available for back order',
+            'awaiting stock',
+            'back in stock soon',
+            'back-order or out of stock',
+            'backordered',
+            'backorder',
+            'benachrichtigt mich', // notify me
+            'binnenkort leverbaar', // coming soon
+            'brak na stanie',
+            'brak w magazynie',
+            'coming soon',
+            'currently have any tickets for this',
+            'currently unavailable',
+            'dieser artikel ist bald wieder verfügbar',
+            'dostępne wkrótce',
+            'en rupture',
+            'esgotado',
+            'in kürze lieferbar',
+            'indisponible',
+            'indisponível',
+            'isn\'t in stock right now',
+            'isnt in stock right now',
+            'isn’t in stock right now',
+            'item is no longer available',
+            'let me know when it\'s available',
+            'mail me when available',
+            'message if back in stock',
+            'mevcut değil',
+            'more on order',
+            'nachricht bei',
+            'nicht auf lager',
+            'nicht lagernd',
+            'nicht lieferbar',
+            'nicht verfügbar',
+            'nicht vorrätig',
+            'nicht mehr lieferbar',
+            'nicht zur verfügung',
+            'nie znaleziono produktów',
+            'niet beschikbaar',
+            'niet leverbaar',
+            'niet op voorraad',
+            'no disponible',
+            'no featured offers available',
+            'no longer available',
+            'no longer in stock',
+            'no tickets available',
+            'non disponibile',
+            'non disponible',
+            'not available',
+            'not currently available',
+            'not in stock',
+            'notify me when available',
+            'notify me',
+            'notify when available',
+            'não disponível',
+            'não estamos a aceitar encomendas',
+            'out of stock',
+            'out-of-stock',
+            'plus disponible',
+            'prodotto esaurito',
+            'produkt niedostępny',
+            'rupture',
+            'sold out',
+            'sold-out',
+            'stok habis',
+            'stok kosong',
+            'stok varian ini habis',
+            'stokta yok',
+            'temporarily out of stock',
+            'temporarily unavailable',
+            'there were no search results for',
+            'this item is currently unavailable',
+            'tickets unavailable',
+            'tidak dijual',
+            'tidak tersedia',
+            'tijdelijk uitverkocht',
+            'tiket tidak tersedia',
+            'to subscribe to back in stock',
+            'tükendi',
+            'unavailable nearby',
+            'unavailable tickets',
+            'vergriffen',
+            'vorbestellen',
+            'vorbestellung ist bald möglich',
+            'we couldn\'t find any products that match',
+            'we do not currently have an estimate of when this product will be back in stock.',
+            'we don\'t currently have any',
+            'we don\'t know when or if this item will be back in stock.',
+            'we were not able to find a match',
+            'when this arrives in stock',
+            'when this item is available to order',
+            'zur zeit nicht an lager',
+            'épuisé',
+            '品切れ',
+            '已售',
+            '已售完',
+            '품절'
+        ];
+
+
+        const vh = Math.max(document.documentElement.clientHeight || 0, window.innerHeight || 0);
+
+        function getElementBaseText(element) {
+            // .textContent can include text from children which may give the wrong results
+            // scan only immediate TEXT_NODEs, which will be a child of the element
+            var text = "";
+            for (var i = 0; i < element.childNodes.length; ++i)
+                if (element.childNodes[i].nodeType === Node.TEXT_NODE)
+                    text += element.childNodes[i].textContent;
+            return text.toLowerCase().trim();
+        }
+
+        const negateOutOfStockRegex = new RegExp('^([0-9] in stock|add to cart|in stock|arrives approximately)', 'ig');
+        // The out-of-stock or in-stock-text is generally always above-the-fold
+        // and often below-the-fold is a list of related products that may or may not contain trigger text
+        // so it's good to filter to just the 'above the fold' elements
+        // and it should be atleast 100px from the top to ignore items in the toolbar, sometimes menu items like "Coming soon" exist
+
+        function elementIsInEyeBallRange(element) {
+            // outside the 'fold' or some weird text in the heading area
+            // .getBoundingClientRect() was causing a crash in chrome 119, can only be run on contentVisibility != hidden
+            // Note: theres also an automated test that places the 'out of stock' text fairly low down
+            // Skip text that could be in the header area
+            if (element.getBoundingClientRect().bottom + window.scrollY <= 300 ) {
+                return false;
+            }
+            // Skip text that could be much further down (like a list of "you may like" products that have 'sold out' in there
+            if (element.getBoundingClientRect().bottom + window.scrollY >= 1300 ) {
+                return false;
+            }
+            return true;
+        }
+
+// @todo - if it's SVG or IMG, go into image diff mode
+
+        function collectVisibleElements(parent, visibleElements) {
+            if (!parent) return; // Base case: if parent is null or undefined, return
+
+            // Add the parent itself to the visible elements array if it's of the specified types
+            visibleElements.push(parent);
+
+            // Iterate over the parent's children
+            const children = parent.children;
+            for (let i = 0; i < children.length; i++) {
+                const child = children[i];
+                if (
+                    child.nodeType === Node.ELEMENT_NODE &&
+                    window.getComputedStyle(child).display !== 'none' &&
+                    window.getComputedStyle(child).visibility !== 'hidden' &&
+                    child.offsetWidth >= 0 &&
+                    child.offsetHeight >= 0 &&
+                    window.getComputedStyle(child).contentVisibility !== 'hidden'
+                ) {
+                    // If the child is an element and is visible, recursively collect visible elements
+                    collectVisibleElements(child, visibleElements);
+                }
+            }
+        }
+
+        const elementsToScan = [];
+        collectVisibleElements(document.body, elementsToScan);
+
+        var elementText = "";
+
+        // REGEXS THAT REALLY MEAN IT'S IN STOCK
+        for (let i = elementsToScan.length - 1; i >= 0; i--) {
+            const element = elementsToScan[i];
+
+            if (!elementIsInEyeBallRange(element)) {
+                continue
+            }
+
+            elementText = "";
+            try {
+                if (element.tagName.toLowerCase() === "input") {
+                    elementText = element.value.toLowerCase().trim();
+                } else {
+                    elementText = getElementBaseText(element);
+                }
+            } catch (e) {
+                console.warn('stock-not-in-stock.js scraper - handling element for gettext failed', e);
+            }
+            if (elementText.length) {
+                // try which ones could mean its in stock
+                if (negateOutOfStockRegex.test(elementText) && !elementText.includes('(0 products)')) {
+                    console.log(`Negating/overriding 'Out of Stock' back to "Possibly in stock" found "${elementText}"`)
+                    element.style.border = "2px solid green"; // highlight the element that was detected as in stock
+                    return 'Possibly in stock';
+                }
+            }
+        }
+
+        // OTHER STUFF THAT COULD BE THAT IT'S OUT OF STOCK
+        for (let i = elementsToScan.length - 1; i >= 0; i--) {
+            const element = elementsToScan[i];
+
+            if (!elementIsInEyeBallRange(element)) {
+                continue
+            }
+            elementText = "";
+            if (element.tagName.toLowerCase() === "input") {
+                elementText = element.value.toLowerCase().trim();
+            } else {
+                elementText = getElementBaseText(element);
+            }
+
+            if (elementText.length) {
+                // and these mean its out of stock
+                for (const outOfStockText of outOfStockTexts) {
+                    if (elementText.includes(outOfStockText)) {
+                        console.log(`Selected 'Out of Stock' - found text "${outOfStockText}" - "${elementText}" - offset top ${element.getBoundingClientRect().top}, page height is ${vh}`)
+                        element.style.border = "2px solid red"; // highlight the element that was detected as out of stock
+                        return outOfStockText; // item is out of stock
+                    }
+                }
+            }
+        }
+
+        console.log(`Returning 'Possibly in stock' - cant' find any useful matching text`)
+        return 'Possibly in stock'; // possibly in stock, cant decide otherwise.
+    }
+
+// returns the element text that makes it think it's out of stock
+    return isItemInStock().trim()
+}

changedetectionio/content_fetchers/res/xpath_element_scraper.js

@@ -0,0 +1,284 @@
+async (options) => {
+
+    let visualselector_xpath_selectors = options.visualselector_xpath_selectors
+    let max_height = options.max_height
+
+    var scroll_y = 0;
+    try {
+        scroll_y = +document.documentElement.scrollTop || document.body.scrollTop
+    } catch (e) {
+        console.log(e);
+    }
+
+// Include the getXpath script directly, easier than fetching
+    function getxpath(e) {
+        var n = e;
+        if (n && n.id) return '//*[@id="' + n.id + '"]';
+        for (var o = []; n && Node.ELEMENT_NODE === n.nodeType;) {
+            for (var i = 0, r = !1, d = n.previousSibling; d;) d.nodeType !== Node.DOCUMENT_TYPE_NODE && d.nodeName === n.nodeName && i++, d = d.previousSibling;
+            for (d = n.nextSibling; d;) {
+                if (d.nodeName === n.nodeName) {
+                    r = !0;
+                    break
+                }
+                d = d.nextSibling
+            }
+            o.push((n.prefix ? n.prefix + ":" : "") + n.localName + (i || r ? "[" + (i + 1) + "]" : "")), n = n.parentNode
+        }
+        return o.length ? "/" + o.reverse().join("/") : ""
+    }
+
+    const findUpTag = (el) => {
+        let r = el
+        chained_css = [];
+        depth = 0;
+
+        //  Strategy 1: If it's an input, with name, and there's only one, prefer that
+        if (el.name !== undefined && el.name.length) {
+            var proposed = el.tagName + "[name=\"" + CSS.escape(el.name) + "\"]";
+            var proposed_element = window.document.querySelectorAll(proposed);
+            if (proposed_element.length) {
+                if (proposed_element.length === 1) {
+                    return proposed;
+                } else {
+                    // Some sites change ID but name= stays the same, we can hit it if we know the index
+                    // Find all the elements that match and work out the input[n]
+                    var n = Array.from(proposed_element).indexOf(el);
+                    // Return a Playwright selector for nthinput[name=zipcode]
+                    return proposed + " >> nth=" + n;
+                }
+            }
+        }
+
+        // Strategy 2: Keep going up until we hit an ID tag, imagine it's like  #list-widget div h4
+        while (r.parentNode) {
+            if (depth === 5) {
+                break;
+            }
+            if ('' !== r.id) {
+                chained_css.unshift("#" + CSS.escape(r.id));
+                final_selector = chained_css.join(' > ');
+                // Be sure theres only one, some sites have multiples of the same ID tag :-(
+                if (window.document.querySelectorAll(final_selector).length === 1) {
+                    return final_selector;
+                }
+                return null;
+            } else {
+                chained_css.unshift(r.tagName.toLowerCase());
+            }
+            r = r.parentNode;
+            depth += 1;
+        }
+        return null;
+    }
+
+
+// @todo - if it's SVG or IMG, go into image diff mode
+
+    var size_pos = [];
+// after page fetch, inject this JS
+// build a map of all elements and their positions (maybe that only include text?)
+    var bbox;
+    console.log(`Scanning for "${visualselector_xpath_selectors}"`);
+
+    function collectVisibleElements(parent, visibleElements) {
+        if (!parent) return; // Base case: if parent is null or undefined, return
+
+
+        // Add the parent itself to the visible elements array if it's of the specified types
+        const tagName = parent.tagName.toLowerCase();
+        if (visualselector_xpath_selectors.split(',').includes(tagName)) {
+            visibleElements.push(parent);
+        }
+
+        // Iterate over the parent's children
+        const children = parent.children;
+        for (let i = 0; i < children.length; i++) {
+            const child = children[i];
+            const computedStyle = window.getComputedStyle(child);
+
+            if (
+                child.nodeType === Node.ELEMENT_NODE &&
+                computedStyle.display !== 'none' &&
+                computedStyle.visibility !== 'hidden' &&
+                child.offsetWidth >= 0 &&
+                child.offsetHeight >= 0 &&
+                computedStyle.contentVisibility !== 'hidden'
+            ) {
+                // If the child is an element and is visible, recursively collect visible elements
+                collectVisibleElements(child, visibleElements);
+            }
+        }
+    }
+
+// Create an array to hold the visible elements
+    const visibleElementsArray = [];
+
+// Call collectVisibleElements with the starting parent element
+    collectVisibleElements(document.body, visibleElementsArray);
+
+
+    visibleElementsArray.forEach(function (element) {
+
+        bbox = element.getBoundingClientRect();
+
+        // Skip really small ones, and where width or height ==0
+        if (bbox['width'] * bbox['height'] < 10) {
+            return
+        }
+
+        // Don't include elements that are offset from canvas
+        if (bbox['top'] + scroll_y < 0 || bbox['left'] < 0) {
+            return
+        }
+
+        // @todo the getXpath kind of sucks, it doesnt know when there is for example just one ID sometimes
+        // it should not traverse when we know we can anchor off just an ID one level up etc..
+        // maybe, get current class or id, keep traversing up looking for only class or id until there is just one match
+
+        // 1st primitive - if it has class, try joining it all and select, if theres only one.. well thats us.
+        xpath_result = false;
+        try {
+            var d = findUpTag(element);
+            if (d) {
+                xpath_result = d;
+            }
+        } catch (e) {
+            console.log(e);
+        }
+        // You could swap it and default to getXpath and then try the smarter one
+        // default back to the less intelligent one
+        if (!xpath_result) {
+            try {
+                // I've seen on FB and eBay that this doesnt work
+                // ReferenceError: getXPath is not defined at eval (eval at evaluate (:152:29), <anonymous>:67:20) at UtilityScript.evaluate (<anonymous>:159:18) at UtilityScript.<anonymous> (<anonymous>:1:44)
+                xpath_result = getxpath(element);
+            } catch (e) {
+                console.log(e);
+                return
+            }
+        }
+
+        let label = "not-interesting" // A placeholder, the actual labels for training are done by hand for now
+
+        let text = element.textContent.trim().slice(0, 30).trim();
+        while (/\n{2,}|\t{2,}/.test(text)) {
+            text = text.replace(/\n{2,}/g, '\n').replace(/\t{2,}/g, '\t')
+        }
+
+        // Try to identify any possible currency amounts "Sale: 4000" or "Sale now 3000 Kc", can help with the training.
+        const hasDigitCurrency = (/\d/.test(text.slice(0, 6)) || /\d/.test(text.slice(-6))) && /([€£$¥₩₹]|USD|AUD|EUR|Kč|kr|SEK|,–)/.test(text);
+        const computedStyle = window.getComputedStyle(element);
+
+        if (Math.floor(bbox['top']) + scroll_y > max_height) {
+            return
+        }
+
+        size_pos.push({
+            xpath: xpath_result,
+            width: Math.round(bbox['width']),
+            height: Math.round(bbox['height']),
+            left: Math.floor(bbox['left']),
+            top: Math.floor(bbox['top']) + scroll_y,
+            // tagName used by Browser Steps
+            tagName: (element.tagName) ? element.tagName.toLowerCase() : '',
+            // tagtype used by Browser Steps
+            tagtype: (element.tagName.toLowerCase() === 'input' && element.type) ? element.type.toLowerCase() : '',
+            isClickable: computedStyle.cursor === "pointer",
+            // Used by the keras trainer
+            fontSize: computedStyle.getPropertyValue('font-size'),
+            fontWeight: computedStyle.getPropertyValue('font-weight'),
+            hasDigitCurrency: hasDigitCurrency,
+            label: label,
+        });
+
+    });
+
+
+// Inject the current one set in the include_filters, which may be a CSS rule
+// used for displaying the current one in VisualSelector, where its not one we generated.
+    if (include_filters.length) {
+        let results;
+        // Foreach filter, go and find it on the page and add it to the results so we can visualise it again
+        for (const f of include_filters) {
+            bbox = false;
+
+            if (!f.length) {
+                console.log("xpath_element_scraper: Empty filter, skipping");
+                continue;
+            }
+
+            try {
+                // is it xpath?
+                if (f.startsWith('/') || f.startsWith('xpath')) {
+                    var qry_f = f.replace(/xpath(:|\d:)/, '')
+                    console.log("[xpath] Scanning for included filter " + qry_f)
+                    let xpathResult = document.evaluate(qry_f, document, null, XPathResult.ORDERED_NODE_SNAPSHOT_TYPE, null);
+                    results = [];
+                    for (let i = 0; i < xpathResult.snapshotLength; i++) {
+                        results.push(xpathResult.snapshotItem(i));
+                    }
+                } else {
+                    console.log("[css] Scanning for included filter " + f)
+                    console.log("[css] Scanning for included filter " + f);
+                    results = document.querySelectorAll(f);
+                }
+            } catch (e) {
+                // Maybe catch DOMException and alert?
+                console.log("xpath_element_scraper: Exception selecting element from filter " + f);
+                console.log(e);
+            }
+
+            if (results != null && results.length) {
+
+                // Iterate over the results
+                results.forEach(node => {
+                    // Try to resolve //something/text() back to its /something so we can atleast get the bounding box
+                    try {
+                        if (typeof node.nodeName == 'string' && node.nodeName === '#text') {
+                            node = node.parentElement
+                        }
+                    } catch (e) {
+                        console.log(e)
+                        console.log("xpath_element_scraper: #text resolver")
+                    }
+
+                    // #1231 - IN the case XPath attribute filter is applied, we will have to traverse up and find the element.
+                    if (typeof node.getBoundingClientRect == 'function') {
+                        bbox = node.getBoundingClientRect();
+                        console.log("xpath_element_scraper: Got filter element, scroll from top was " + scroll_y)
+                    } else {
+                        try {
+                            // Try and see we can find its ownerElement
+                            bbox = node.ownerElement.getBoundingClientRect();
+                            console.log("xpath_element_scraper: Got filter by ownerElement element, scroll from top was " + scroll_y)
+                        } catch (e) {
+                            console.log(e)
+                            console.log("xpath_element_scraper: error looking up node.ownerElement")
+                        }
+                    }
+
+                    if (bbox && bbox['width'] > 0 && bbox['height'] > 0) {
+                        size_pos.push({
+                            xpath: f,
+                            width: parseInt(bbox['width']),
+                            height: parseInt(bbox['height']),
+                            left: parseInt(bbox['left']),
+                            top: parseInt(bbox['top']) + scroll_y,
+                            highlight_as_custom_filter: true
+                        });
+                    }
+                });
+            }
+        }
+    }
+
+// Sort the elements so we find the smallest one first, in other words, we find the smallest one matching in that area
+// so that we dont select the wrapping element by mistake and be unable to select what we want
+    size_pos.sort((a, b) => (a.width * a.height > b.width * b.height) ? 1 : -1)
+
+// browser_width required for proper scaling in the frontend
+    // Return as a string to save playwright for juggling thousands of objects
+    return JSON.stringify({'size_pos': size_pos, 'browser_width': window.innerWidth});
+}
+

changedetectionio/content_fetchers/screenshot_handler.py

@@ -0,0 +1,73 @@
+# Pages with a vertical height longer than this will use the 'stitch together' method.
+
+# - Many GPUs have a max texture size of 16384x16384px (or lower on older devices).
+# - If a page is taller than ~8000–10000px, it risks exceeding GPU memory limits.
+# - This is especially important on headless Chromium, where Playwright may fail to allocate a massive full-page buffer.
+
+from loguru import logger
+
+from changedetectionio.content_fetchers import SCREENSHOT_MAX_HEIGHT_DEFAULT, SCREENSHOT_DEFAULT_QUALITY
+
+
+def stitch_images_worker(pipe_conn, chunks_bytes, original_page_height, capture_height):
+    import os
+    import io
+    from PIL import Image, ImageDraw, ImageFont
+
+    try:
+
+        # Load images from byte chunks
+        images = [Image.open(io.BytesIO(b)) for b in chunks_bytes]
+        total_height = sum(im.height for im in images)
+        max_width = max(im.width for im in images)
+
+        # Create stitched image
+        stitched = Image.new('RGB', (max_width, total_height))
+        y_offset = 0
+        for im in images:
+            stitched.paste(im, (0, y_offset))
+            y_offset += im.height
+
+        # Draw caption on top (overlaid, not extending canvas)
+        draw = ImageDraw.Draw(stitched)
+
+        if original_page_height > capture_height:
+            caption_text = f"WARNING: Screenshot was {original_page_height}px but trimmed to {capture_height}px because it was too long"
+            padding = 10
+            font_size = 35
+            font_color = (255, 0, 0)
+            background_color = (255, 255, 255)
+
+
+            # Try to load a proper font
+            try:
+                font = ImageFont.truetype("arial.ttf", font_size)
+            except IOError:
+                font = ImageFont.load_default()
+
+            bbox = draw.textbbox((0, 0), caption_text, font=font)
+            text_width = bbox[2] - bbox[0]
+            text_height = bbox[3] - bbox[1]
+
+            # Draw white rectangle background behind text
+            rect_top = 0
+            rect_bottom = text_height + 2 * padding
+            draw.rectangle([(0, rect_top), (max_width, rect_bottom)], fill=background_color)
+
+            # Draw text centered horizontally, 10px padding from top of the rectangle
+            text_x = (max_width - text_width) // 2
+            text_y = padding
+            draw.text((text_x, text_y), caption_text, font=font, fill=font_color)
+
+        # Encode and send image
+        output = io.BytesIO()
+        stitched.save(output, format="JPEG", quality=int(os.getenv("SCREENSHOT_QUALITY", SCREENSHOT_DEFAULT_QUALITY)))
+        pipe_conn.send_bytes(output.getvalue())
+
+        stitched.close()
+    except Exception as e:
+        pipe_conn.send(f"error:{e}")
+    finally:
+        pipe_conn.close()
+
+

changedetectionio/content_fetchers/webdriver_selenium.py

@@ -0,0 +1,143 @@
+import os
+import time
+
+from loguru import logger
+from changedetectionio.content_fetchers.base import Fetcher
+
+
+class fetcher(Fetcher):
+    if os.getenv("WEBDRIVER_URL"):
+        fetcher_description = f"WebDriver Chrome/Javascript via \"{os.getenv('WEBDRIVER_URL', '')}\""
+    else:
+        fetcher_description = "WebDriver Chrome/Javascript"
+
+    proxy = None
+    proxy_url = None
+
+    def __init__(self, proxy_override=None, custom_browser_connection_url=None):
+        super().__init__()
+        from urllib.parse import urlparse
+        from selenium.webdriver.common.proxy import Proxy
+
+        # .strip('"') is going to save someone a lot of time when they accidently wrap the env value
+        if not custom_browser_connection_url:
+            self.browser_connection_url = os.getenv("WEBDRIVER_URL", 'http://browser-chrome:4444/wd/hub').strip('"')
+        else:
+            self.browser_connection_is_custom = True
+            self.browser_connection_url = custom_browser_connection_url
+
+        ##### PROXY SETUP #####
+
+        proxy_sources = [
+            self.system_http_proxy,
+            self.system_https_proxy,
+            os.getenv('webdriver_proxySocks'),
+            os.getenv('webdriver_socksProxy'),
+            os.getenv('webdriver_proxyHttp'),
+            os.getenv('webdriver_httpProxy'),
+            os.getenv('webdriver_proxyHttps'),
+            os.getenv('webdriver_httpsProxy'),
+            os.getenv('webdriver_sslProxy'),
+            proxy_override,  # last one should override
+        ]
+        # The built in selenium proxy handling is super unreliable!!! so we just grab which ever proxy setting we can find and throw it in --proxy-server=
+        for k in filter(None, proxy_sources):
+            if not k:
+                continue
+            self.proxy_url = k.strip()
+
+    async def run(self,
+                  fetch_favicon=True,
+                  current_include_filters=None,
+                  empty_pages_are_a_change=False,
+                  ignore_status_codes=False,
+                  is_binary=False,
+                  request_body=None,
+                  request_headers=None,
+                  request_method=None,
+                  timeout=None,
+                  url=None,
+                  ):
+
+        import asyncio
+
+        # Wrap the entire selenium operation in a thread executor
+        def _run_sync():
+            from selenium.webdriver.chrome.options import Options as ChromeOptions
+            # request_body, request_method unused for now, until some magic in the future happens.
+
+            options = ChromeOptions()
+
+            # Load Chrome options from env
+            CHROME_OPTIONS = [
+                line.strip()
+                for line in os.getenv("CHROME_OPTIONS", "").strip().splitlines()
+                if line.strip()
+            ]
+
+            for opt in CHROME_OPTIONS:
+                options.add_argument(opt)
+
+            # 1. proxy_config /Proxy(proxy_config) selenium object is REALLY unreliable
+            # 2. selenium-wire cant be used because the websocket version conflicts with pypeteer-ng
+            # 3. selenium only allows ONE runner at a time by default!
+            # 4. driver must use quit() or it will continue to block/hold the selenium process!!
+
+            if self.proxy_url:
+                options.add_argument(f'--proxy-server={self.proxy_url}')
+
+            from selenium.webdriver.remote.remote_connection import RemoteConnection
+            from selenium.webdriver.remote.webdriver import WebDriver as RemoteWebDriver
+            driver = None
+            try:
+                # Create the RemoteConnection and set timeout (e.g., 30 seconds)
+                remote_connection = RemoteConnection(
+                    self.browser_connection_url,
+                )
+                remote_connection.set_timeout(30)  # seconds
+
+                # Now create the driver with the RemoteConnection
+                driver = RemoteWebDriver(
+                    command_executor=remote_connection,
+                    options=options
+                )
+
+                driver.set_page_load_timeout(int(os.getenv("WEBDRIVER_PAGELOAD_TIMEOUT", 45)))
+            except Exception as e:
+                if driver:
+                    driver.quit()
+                raise e
+
+            try:
+                driver.get(url)
+
+                if not "--window-size" in os.getenv("CHROME_OPTIONS", ""):
+                    driver.set_window_size(1280, 1024)
+
+                driver.implicitly_wait(int(os.getenv("WEBDRIVER_DELAY_BEFORE_CONTENT_READY", 5)))
+
+                if self.webdriver_js_execute_code is not None:
+                    driver.execute_script(self.webdriver_js_execute_code)
+                    # Selenium doesn't automatically wait for actions as good as Playwright, so wait again
+                    driver.implicitly_wait(int(os.getenv("WEBDRIVER_DELAY_BEFORE_CONTENT_READY", 5)))
+
+                # @todo - how to check this? is it possible?
+                self.status_code = 200
+                # @todo somehow we should try to get this working for WebDriver
+                # raise EmptyReply(url=url, status_code=r.status_code)
+
+                # @todo - dom wait loaded?
+                import time
+                time.sleep(int(os.getenv("WEBDRIVER_DELAY_BEFORE_CONTENT_READY", 5)) + self.render_extract_delay)
+                self.content = driver.page_source
+                self.headers = {}
+                self.screenshot = driver.get_screenshot_as_png()
+            except Exception as e:
+                driver.quit()
+                raise e
+
+            driver.quit()
+
+        # Run the selenium operations in a thread pool to avoid blocking the event loop
+        loop = asyncio.get_event_loop()
+        await loop.run_in_executor(None, _run_sync)

changedetectionio/custom_queue.py

@@ -0,0 +1,535 @@
+import queue
+import asyncio
+from blinker import signal
+from loguru import logger
+
+
+class NotificationQueue(queue.Queue):
+    """
+    Extended Queue that sends a 'notification_event' signal when notifications are added.
+    
+    This class extends the standard Queue and adds a signal emission after a notification
+    is put into the queue. The signal includes the watch UUID if available.
+    """
+    
+    def __init__(self, maxsize=0):
+        super().__init__(maxsize)
+        try:
+            self.notification_event_signal = signal('notification_event')
+        except Exception as e:
+            logger.critical(f"Exception creating notification_event signal: {e}")
+
+    def put(self, item, block=True, timeout=None):
+        # Call the parent's put method first
+        super().put(item, block, timeout)
+        
+        # After putting the notification in the queue, emit signal with watch UUID
+        try:
+            if self.notification_event_signal and isinstance(item, dict):
+                watch_uuid = item.get('uuid')
+                if watch_uuid:
+                    # Send the notification_event signal with the watch UUID
+                    self.notification_event_signal.send(watch_uuid=watch_uuid)
+                    logger.trace(f"NotificationQueue: Emitted notification_event signal for watch UUID {watch_uuid}")
+                else:
+                    # Send signal without UUID for system notifications
+                    self.notification_event_signal.send()
+                    logger.trace("NotificationQueue: Emitted notification_event signal for system notification")
+        except Exception as e:
+            logger.error(f"Exception emitting notification_event signal: {e}")
+
+class SignalPriorityQueue(queue.PriorityQueue):
+    """
+    Extended PriorityQueue that sends a signal when items with a UUID are added.
+    
+    This class extends the standard PriorityQueue and adds a signal emission
+    after an item is put into the queue. If the item contains a UUID, the signal
+    is sent with that UUID as a parameter.
+    """
+    
+    def __init__(self, maxsize=0):
+        super().__init__(maxsize)
+        try:
+            self.queue_length_signal = signal('queue_length')
+        except Exception as e:
+            logger.critical(f"Exception: {e}")
+
+    def put(self, item, block=True, timeout=None):
+        # Call the parent's put method first
+        super().put(item, block, timeout)
+        
+        # After putting the item in the queue, check if it has a UUID and emit signal
+        if hasattr(item, 'item') and isinstance(item.item, dict) and 'uuid' in item.item:
+            uuid = item.item['uuid']
+            # Get the signal and send it if it exists
+            watch_check_update = signal('watch_check_update')
+            if watch_check_update:
+                # Send the watch_uuid parameter
+                watch_check_update.send(watch_uuid=uuid)
+        
+        # Send queue_length signal with current queue size
+        try:
+
+            if self.queue_length_signal:
+                self.queue_length_signal.send(length=self.qsize())
+        except Exception as e:
+            logger.critical(f"Exception: {e}")
+
+    def get(self, block=True, timeout=None):
+        # Call the parent's get method first
+        item = super().get(block, timeout)
+        
+        # Send queue_length signal with current queue size
+        try:
+            if self.queue_length_signal:
+                self.queue_length_signal.send(length=self.qsize())
+        except Exception as e:
+            logger.critical(f"Exception: {e}")
+        return item
+    
+    def get_uuid_position(self, target_uuid):
+        """
+        Find the position of a watch UUID in the priority queue.
+        Optimized for large queues - O(n) complexity instead of O(n log n).
+        
+        Args:
+            target_uuid: The UUID to search for
+            
+        Returns:
+            dict: Contains position info or None if not found
+                - position: 0-based position in queue (0 = next to be processed)
+                - total_items: total number of items in queue
+                - priority: the priority value of the found item
+        """
+        with self.mutex:
+            queue_list = list(self.queue)
+            total_items = len(queue_list)
+            
+            if total_items == 0:
+                return {
+                    'position': None,
+                    'total_items': 0,
+                    'priority': None,
+                    'found': False
+                }
+            
+            # Find the target item and its priority first - O(n)
+            target_item = None
+            target_priority = None
+            
+            for item in queue_list:
+                if (hasattr(item, 'item') and 
+                    isinstance(item.item, dict) and 
+                    item.item.get('uuid') == target_uuid):
+                    target_item = item
+                    target_priority = item.priority
+                    break
+            
+            if target_item is None:
+                return {
+                    'position': None,
+                    'total_items': total_items,
+                    'priority': None,
+                    'found': False
+                }
+            
+            # Count how many items have higher priority (lower numbers) - O(n)
+            position = 0
+            for item in queue_list:
+                # Items with lower priority numbers are processed first
+                if item.priority < target_priority:
+                    position += 1
+                elif item.priority == target_priority and item != target_item:
+                    # For same priority, count items that come before this one
+                    # (Note: this is approximate since heap order isn't guaranteed for equal priorities)
+                    position += 1
+            
+            return {
+                'position': position,
+                'total_items': total_items,
+                'priority': target_priority,
+                'found': True
+            }
+    
+    def get_all_queued_uuids(self, limit=None, offset=0):
+        """
+        Get UUIDs currently in the queue with their positions.
+        For large queues, use limit/offset for pagination.
+        
+        Args:
+            limit: Maximum number of items to return (None = all)
+            offset: Number of items to skip (for pagination)
+        
+        Returns:
+            dict: Contains items and metadata
+                - items: List of dicts with uuid, position, and priority
+                - total_items: Total number of items in queue
+                - returned_items: Number of items returned
+                - has_more: Whether there are more items after this page
+        """
+        with self.mutex:
+            queue_list = list(self.queue)
+            total_items = len(queue_list)
+            
+            if total_items == 0:
+                return {
+                    'items': [],
+                    'total_items': 0,
+                    'returned_items': 0,
+                    'has_more': False
+                }
+            
+            # For very large queues, warn about performance
+            if total_items > 1000 and limit is None:
+                logger.warning(f"Getting all {total_items} queued items without limit - this may be slow")
+            
+            # Sort only if we need exact positions (expensive for large queues)
+            if limit is not None and limit <= 100:
+                # For small requests, we can afford to sort
+                queue_items = sorted(queue_list)
+                end_idx = min(offset + limit, len(queue_items)) if limit else len(queue_items)
+                items_to_process = queue_items[offset:end_idx]
+                
+                result = []
+                for position, item in enumerate(items_to_process, start=offset):
+                    if (hasattr(item, 'item') and 
+                        isinstance(item.item, dict) and 
+                        'uuid' in item.item):
+                        
+                        result.append({
+                            'uuid': item.item['uuid'],
+                            'position': position,
+                            'priority': item.priority
+                        })
+                
+                return {
+                    'items': result,
+                    'total_items': total_items,
+                    'returned_items': len(result),
+                    'has_more': (offset + len(result)) < total_items
+                }
+            else:
+                # For large requests, return items with approximate positions
+                # This is much faster O(n) instead of O(n log n)
+                result = []
+                processed = 0
+                skipped = 0
+                
+                for item in queue_list:
+                    if (hasattr(item, 'item') and 
+                        isinstance(item.item, dict) and 
+                        'uuid' in item.item):
+                        
+                        if skipped < offset:
+                            skipped += 1
+                            continue
+                        
+                        if limit and processed >= limit:
+                            break
+                        
+                        # Approximate position based on priority comparison
+                        approx_position = sum(1 for other in queue_list if other.priority < item.priority)
+                        
+                        result.append({
+                            'uuid': item.item['uuid'],
+                            'position': approx_position,  # Approximate
+                            'priority': item.priority
+                        })
+                        processed += 1
+                
+                return {
+                    'items': result,
+                    'total_items': total_items,
+                    'returned_items': len(result),
+                    'has_more': (offset + len(result)) < total_items,
+                    'note': 'Positions are approximate for performance with large queues'
+                }
+    
+    def get_queue_summary(self):
+        """
+        Get a quick summary of queue state without expensive operations.
+        O(n) complexity - fast even for large queues.
+        
+        Returns:
+            dict: Queue summary statistics
+        """
+        with self.mutex:
+            queue_list = list(self.queue)
+            total_items = len(queue_list)
+            
+            if total_items == 0:
+                return {
+                    'total_items': 0,
+                    'priority_breakdown': {},
+                    'immediate_items': 0,
+                    'clone_items': 0,
+                    'scheduled_items': 0
+                }
+            
+            # Count items by priority type - O(n)
+            immediate_items = 0  # priority 1
+            clone_items = 0      # priority 5  
+            scheduled_items = 0  # priority > 100 (timestamps)
+            priority_counts = {}
+            
+            for item in queue_list:
+                priority = item.priority
+                priority_counts[priority] = priority_counts.get(priority, 0) + 1
+                
+                if priority == 1:
+                    immediate_items += 1
+                elif priority == 5:
+                    clone_items += 1
+                elif priority > 100:
+                    scheduled_items += 1
+            
+            return {
+                'total_items': total_items,
+                'priority_breakdown': priority_counts,
+                'immediate_items': immediate_items,
+                'clone_items': clone_items,
+                'scheduled_items': scheduled_items,
+                'min_priority': min(priority_counts.keys()) if priority_counts else None,
+                'max_priority': max(priority_counts.keys()) if priority_counts else None
+            }
+
+
+class AsyncSignalPriorityQueue(asyncio.PriorityQueue):
+    """
+    Async version of SignalPriorityQueue that sends signals when items are added/removed.
+    
+    This class extends asyncio.PriorityQueue and maintains the same signal behavior
+    as the synchronous version for real-time UI updates.
+    """
+    
+    def __init__(self, maxsize=0):
+        super().__init__(maxsize)
+        try:
+            self.queue_length_signal = signal('queue_length')
+        except Exception as e:
+            logger.critical(f"Exception: {e}")
+
+    async def put(self, item):
+        # Call the parent's put method first
+        await super().put(item)
+        
+        # After putting the item in the queue, check if it has a UUID and emit signal
+        if hasattr(item, 'item') and isinstance(item.item, dict) and 'uuid' in item.item:
+            uuid = item.item['uuid']
+            # Get the signal and send it if it exists
+            watch_check_update = signal('watch_check_update')
+            if watch_check_update:
+                # Send the watch_uuid parameter
+                watch_check_update.send(watch_uuid=uuid)
+        
+        # Send queue_length signal with current queue size
+        try:
+            if self.queue_length_signal:
+                self.queue_length_signal.send(length=self.qsize())
+        except Exception as e:
+            logger.critical(f"Exception: {e}")
+
+    async def get(self):
+        # Call the parent's get method first
+        item = await super().get()
+        
+        # Send queue_length signal with current queue size
+        try:
+            if self.queue_length_signal:
+                self.queue_length_signal.send(length=self.qsize())
+        except Exception as e:
+            logger.critical(f"Exception: {e}")
+        return item
+    
+    @property
+    def queue(self):
+        """
+        Provide compatibility with sync PriorityQueue.queue access
+        Returns the internal queue for template access
+        """
+        return self._queue if hasattr(self, '_queue') else []
+    
+    def get_uuid_position(self, target_uuid):
+        """
+        Find the position of a watch UUID in the async priority queue.
+        Optimized for large queues - O(n) complexity instead of O(n log n).
+        
+        Args:
+            target_uuid: The UUID to search for
+            
+        Returns:
+            dict: Contains position info or None if not found
+                - position: 0-based position in queue (0 = next to be processed)
+                - total_items: total number of items in queue
+                - priority: the priority value of the found item
+        """
+        queue_list = list(self._queue)
+        total_items = len(queue_list)
+        
+        if total_items == 0:
+            return {
+                'position': None,
+                'total_items': 0,
+                'priority': None,
+                'found': False
+            }
+        
+        # Find the target item and its priority first - O(n)
+        target_item = None
+        target_priority = None
+        
+        for item in queue_list:
+            if (hasattr(item, 'item') and 
+                isinstance(item.item, dict) and 
+                item.item.get('uuid') == target_uuid):
+                target_item = item
+                target_priority = item.priority
+                break
+        
+        if target_item is None:
+            return {
+                'position': None,
+                'total_items': total_items,
+                'priority': None,
+                'found': False
+            }
+        
+        # Count how many items have higher priority (lower numbers) - O(n)
+        position = 0
+        for item in queue_list:
+            if item.priority < target_priority:
+                position += 1
+            elif item.priority == target_priority and item != target_item:
+                position += 1
+        
+        return {
+            'position': position,
+            'total_items': total_items,
+            'priority': target_priority,
+            'found': True
+        }
+    
+    def get_all_queued_uuids(self, limit=None, offset=0):
+        """
+        Get UUIDs currently in the async queue with their positions.
+        For large queues, use limit/offset for pagination.
+        
+        Args:
+            limit: Maximum number of items to return (None = all)
+            offset: Number of items to skip (for pagination)
+        
+        Returns:
+            dict: Contains items and metadata (same structure as sync version)
+        """
+        queue_list = list(self._queue)
+        total_items = len(queue_list)
+        
+        if total_items == 0:
+            return {
+                'items': [],
+                'total_items': 0,
+                'returned_items': 0,
+                'has_more': False
+            }
+        
+        # Same logic as sync version but without mutex
+        if limit is not None and limit <= 100:
+            queue_items = sorted(queue_list)
+            end_idx = min(offset + limit, len(queue_items)) if limit else len(queue_items)
+            items_to_process = queue_items[offset:end_idx]
+            
+            result = []
+            for position, item in enumerate(items_to_process, start=offset):
+                if (hasattr(item, 'item') and 
+                    isinstance(item.item, dict) and 
+                    'uuid' in item.item):
+                    
+                    result.append({
+                        'uuid': item.item['uuid'],
+                        'position': position,
+                        'priority': item.priority
+                    })
+            
+            return {
+                'items': result,
+                'total_items': total_items,
+                'returned_items': len(result),
+                'has_more': (offset + len(result)) < total_items
+            }
+        else:
+            # Fast approximate positions for large queues
+            result = []
+            processed = 0
+            skipped = 0
+            
+            for item in queue_list:
+                if (hasattr(item, 'item') and 
+                    isinstance(item.item, dict) and 
+                    'uuid' in item.item):
+                    
+                    if skipped < offset:
+                        skipped += 1
+                        continue
+                    
+                    if limit and processed >= limit:
+                        break
+                    
+                    approx_position = sum(1 for other in queue_list if other.priority < item.priority)
+                    
+                    result.append({
+                        'uuid': item.item['uuid'],
+                        'position': approx_position,
+                        'priority': item.priority
+                    })
+                    processed += 1
+            
+            return {
+                'items': result,
+                'total_items': total_items,
+                'returned_items': len(result),
+                'has_more': (offset + len(result)) < total_items,
+                'note': 'Positions are approximate for performance with large queues'
+            }
+    
+    def get_queue_summary(self):
+        """
+        Get a quick summary of async queue state.
+        O(n) complexity - fast even for large queues.
+        """
+        queue_list = list(self._queue)
+        total_items = len(queue_list)
+        
+        if total_items == 0:
+            return {
+                'total_items': 0,
+                'priority_breakdown': {},
+                'immediate_items': 0,
+                'clone_items': 0,
+                'scheduled_items': 0
+            }
+        
+        immediate_items = 0
+        clone_items = 0
+        scheduled_items = 0
+        priority_counts = {}
+        
+        for item in queue_list:
+            priority = item.priority
+            priority_counts[priority] = priority_counts.get(priority, 0) + 1
+            
+            if priority == 1:
+                immediate_items += 1
+            elif priority == 5:
+                clone_items += 1
+            elif priority > 100:
+                scheduled_items += 1
+        
+        return {
+            'total_items': total_items,
+            'priority_breakdown': priority_counts,
+            'immediate_items': immediate_items,
+            'clone_items': clone_items,
+            'scheduled_items': scheduled_items,
+            'min_priority': min(priority_counts.keys()) if priority_counts else None,
+            'max_priority': max(priority_counts.keys()) if priority_counts else None
+        }

changedetectionio/dev-docker/Dockerfile

@@ -1,14 +0,0 @@
-FROM python:3.8-slim
-
-# https://stackoverflow.com/questions/58701233/docker-logs-erroneously-appears-empty-until-container-stops
-ENV PYTHONUNBUFFERED=1
-
-WORKDIR /app
-
-RUN [ ! -d "/datastore" ] && mkdir /datastore
-
-COPY sleep.py /
-CMD [ "python", "/sleep.py" ]
-
-
-

changedetectionio/dev-docker/sleep.py

@@ -1,7 +0,0 @@
-import time
-
-print ("Sleep loop, you should run your script from the console")
-
-while True: 
-    # Wait for 5 seconds
-    time.sleep(2)

changedetectionio/diff.py

@@ -1,52 +1,113 @@
-# used for the notifications, the front-end is using a JS library
-
 import difflib
+from typing import List, Iterator, Union
+
+REMOVED_STYLE = "background-color: #fadad7; color: #b30000;"
+ADDED_STYLE = "background-color: #eaf2c2; color: #406619;"
+
+def same_slicer(lst: List[str], start: int, end: int) -> List[str]:
+    """Return a slice of the list, or a single element if start == end."""
+    return lst[start:end] if start != end else [lst[start]]
+
+def customSequenceMatcher(
+    before: List[str],
+    after: List[str],
+    include_equal: bool = False,
+    include_removed: bool = True,
+    include_added: bool = True,
+    include_replaced: bool = True,
+    include_change_type_prefix: bool = True,
+    html_colour: bool = False
+) -> Iterator[List[str]]:
+    """
+    Compare two sequences and yield differences based on specified parameters.
+
+    Args:
+        before (List[str]): Original sequence
+        after (List[str]): Modified sequence
+        include_equal (bool): Include unchanged parts
+        include_removed (bool): Include removed parts
+        include_added (bool): Include added parts
+        include_replaced (bool): Include replaced parts
+        include_change_type_prefix (bool): Add prefixes to indicate change types
+        html_colour (bool): Use HTML background colors for differences
+
+    Yields:
+        List[str]: Differences between sequences
+    """
+    cruncher = difflib.SequenceMatcher(isjunk=lambda x: x in " \t", a=before, b=after)
 
 
-def same_slicer(l, a, b):
-    if a == b:
-        return [l[a]]
-    else:
-        return l[a:b]
 
-# like .compare but a little different output
-def customSequenceMatcher(before, after, include_equal=False):
-    cruncher = difflib.SequenceMatcher(isjunk=lambda x: x in " \\t", a=before, b=after)
-
-    # @todo Line-by-line mode instead of buncghed, including `after` that is not in `before` (maybe unset?)
     for tag, alo, ahi, blo, bhi in cruncher.get_opcodes():
         if include_equal and tag == 'equal':
-            g = before[alo:ahi]
-            yield g
-        elif tag == 'delete':
-            g = ["(removed) " + i for i in same_slicer(before, alo, ahi)]
-            yield g
-        elif tag == 'replace':
-            g = ["(changed) " + i for i in same_slicer(before, alo, ahi)]
-            g += ["(into   ) " + i for i in same_slicer(after, blo, bhi)]
-            yield g
-        elif tag == 'insert':
-            g = ["(added  ) " + i for i in same_slicer(after, blo, bhi)]
-            yield g
+            yield before[alo:ahi]
+        elif include_removed and tag == 'delete':
+            if html_colour:
+                yield [f'<span style="{REMOVED_STYLE}">{line}</span>' for line in same_slicer(before, alo, ahi)]
+            else:
+                yield [f"(removed) {line}" for line in same_slicer(before, alo, ahi)] if include_change_type_prefix else same_slicer(before, alo, ahi)
+        elif include_replaced and tag == 'replace':
+            if html_colour:
+                yield [f'<span style="{REMOVED_STYLE}">{line}</span>' for line in same_slicer(before, alo, ahi)] + \
+                      [f'<span style="{ADDED_STYLE}">{line}</span>' for line in same_slicer(after, blo, bhi)]
+            else:
+                yield [f"(changed) {line}" for line in same_slicer(before, alo, ahi)] + \
+                      [f"(into) {line}" for line in same_slicer(after, blo, bhi)] if include_change_type_prefix else same_slicer(before, alo, ahi) + same_slicer(after, blo, bhi)
+        elif include_added and tag == 'insert':
+            if html_colour:
+                yield [f'<span style="{ADDED_STYLE}">{line}</span>' for line in same_slicer(after, blo, bhi)]
+            else:
+                yield [f"(added) {line}" for line in same_slicer(after, blo, bhi)] if include_change_type_prefix else same_slicer(after, blo, bhi)
 
-# only_differences - only return info about the differences, no context
-# line_feed_sep could be "<br/>" or "<li>" or "\n" etc
-def render_diff(previous_file, newest_file, include_equal=False, line_feed_sep="\n"):
-    with open(newest_file, 'r') as f:
-        newest_version_file_contents = f.read()
-        newest_version_file_contents = [line.rstrip() for line in newest_version_file_contents.splitlines()]
+def render_diff(
+    previous_version_file_contents: str,
+    newest_version_file_contents: str,
+    include_equal: bool = False,
+    include_removed: bool = True,
+    include_added: bool = True,
+    include_replaced: bool = True,
+    line_feed_sep: str = "\n",
+    include_change_type_prefix: bool = True,
+    patch_format: bool = False,
+    html_colour: bool = False
+) -> str:
+    """
+    Render the difference between two file contents.
 
-    if previous_file:
-        with open(previous_file, 'r') as f:
-            previous_version_file_contents = f.read()
-            previous_version_file_contents = [line.rstrip() for line in previous_version_file_contents.splitlines()]
-    else:
-        previous_version_file_contents = ""
+    Args:
+        previous_version_file_contents (str): Original file contents
+        newest_version_file_contents (str): Modified file contents
+        include_equal (bool): Include unchanged parts
+        include_removed (bool): Include removed parts
+        include_added (bool): Include added parts
+        include_replaced (bool): Include replaced parts
+        line_feed_sep (str): Separator for lines in output
+        include_change_type_prefix (bool): Add prefixes to indicate change types
+        patch_format (bool): Use patch format for output
+        html_colour (bool): Use HTML background colors for differences
 
-    rendered_diff = customSequenceMatcher(previous_version_file_contents,
-                                          newest_version_file_contents,
-                                          include_equal)
+    Returns:
+        str: Rendered difference
+    """
+    newest_lines = [line.rstrip() for line in newest_version_file_contents.splitlines()]
+    previous_lines = [line.rstrip() for line in previous_version_file_contents.splitlines()] if previous_version_file_contents else []
 
-    # Recursively join lists
-    f = lambda L: line_feed_sep.join([f(x) if type(x) is list else x for x in L])
-    return f(rendered_diff)
+    if patch_format:
+        patch = difflib.unified_diff(previous_lines, newest_lines)
+        return line_feed_sep.join(patch)
+
+    rendered_diff = customSequenceMatcher(
+        before=previous_lines,
+        after=newest_lines,
+        include_equal=include_equal,
+        include_removed=include_removed,
+        include_added=include_added,
+        include_replaced=include_replaced,
+        include_change_type_prefix=include_change_type_prefix,
+        html_colour=html_colour
+    )
+
+    def flatten(lst: List[Union[str, List[str]]]) -> str:
+        return line_feed_sep.join(flatten(x) if isinstance(x, list) else x for x in lst)
+
+    return flatten(rendered_diff)

changedetectionio/fetch_site_status.py

@@ -1,315 +0,0 @@
-import hashlib
-import logging
-import os
-import re
-import time
-import urllib3
-
-from changedetectionio import content_fetcher, html_tools
-
-urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)
-
-
-# Some common stuff here that can be moved to a base class
-# (set_proxy_from_list)
-class perform_site_check():
-
-    def __init__(self, *args, datastore, **kwargs):
-        super().__init__(*args, **kwargs)
-        self.datastore = datastore
-
-    # If there was a proxy list enabled, figure out what proxy_args/which proxy to use
-    # if watch.proxy use that
-    # fetcher.proxy_override = watch.proxy or main config proxy
-    # Allows override the proxy on a per-request basis
-    # ALWAYS use the first one is nothing selected
-
-    def set_proxy_from_list(self, watch):
-        proxy_args = None
-        if self.datastore.proxy_list is None:
-            return None
-
-        # If its a valid one
-        if any([watch['proxy'] in p for p in self.datastore.proxy_list]):
-            proxy_args = watch['proxy']
-
-        # not valid (including None), try the system one
-        else:
-            system_proxy = self.datastore.data['settings']['requests']['proxy']
-            # Is not None and exists
-            if any([system_proxy in p for p in self.datastore.proxy_list]):
-                proxy_args = system_proxy
-
-        # Fallback - Did not resolve anything, use the first available
-        if proxy_args is None:
-            proxy_args = self.datastore.proxy_list[0][0]
-
-        return proxy_args
-
-    # Doesn't look like python supports forward slash auto enclosure in re.findall
-    # So convert it to inline flag "foobar(?i)" type configuration
-    def forward_slash_enclosed_regex_to_options(self, regex):
-        res = re.search(r'^/(.*?)/(\w+)$', regex, re.IGNORECASE)
-
-        if res:
-            regex = res.group(1)
-            regex += '(?{})'.format(res.group(2))
-        else:
-            regex += '(?{})'.format('i')
-
-        return regex
-
-
-    def run(self, uuid):
-        timestamp = int(time.time())  # used for storage etc too
-
-        changed_detected = False
-        screenshot = False  # as bytes
-        stripped_text_from_html = ""
-
-        watch = self.datastore.data['watching'][uuid]
-
-        # Protect against file:// access
-        if re.search(r'^file', watch['url'], re.IGNORECASE) and not os.getenv('ALLOW_FILE_URI', False):
-            raise Exception(
-                "file:// type access is denied for security reasons."
-            )
-
-        # Unset any existing notification error
-        update_obj = {'last_notification_error': False, 'last_error': False}
-
-        extra_headers = self.datastore.get_val(uuid, 'headers')
-
-        # Tweak the base config with the per-watch ones
-        request_headers = self.datastore.data['settings']['headers'].copy()
-        request_headers.update(extra_headers)
-
-        # https://github.com/psf/requests/issues/4525
-        # Requests doesnt yet support brotli encoding, so don't put 'br' here, be totally sure that the user cannot
-        # do this by accident.
-        if 'Accept-Encoding' in request_headers and "br" in request_headers['Accept-Encoding']:
-            request_headers['Accept-Encoding'] = request_headers['Accept-Encoding'].replace(', br', '')
-
-        timeout = self.datastore.data['settings']['requests']['timeout']
-        url = self.datastore.get_val(uuid, 'url')
-        request_body = self.datastore.get_val(uuid, 'body')
-        request_method = self.datastore.get_val(uuid, 'method')
-        ignore_status_code = self.datastore.get_val(uuid, 'ignore_status_codes')
-
-        # source: support
-        is_source = False
-        if url.startswith('source:'):
-            url = url.replace('source:', '')
-            is_source = True
-
-        # Pluggable content fetcher
-        prefer_backend = watch['fetch_backend']
-        if hasattr(content_fetcher, prefer_backend):
-            klass = getattr(content_fetcher, prefer_backend)
-        else:
-            # If the klass doesnt exist, just use a default
-            klass = getattr(content_fetcher, "html_requests")
-
-
-        proxy_args = self.set_proxy_from_list(watch)
-        fetcher = klass(proxy_override=proxy_args)
-
-        # Configurable per-watch or global extra delay before extracting text (for webDriver types)
-        system_webdriver_delay = self.datastore.data['settings']['application'].get('webdriver_delay', None)
-        if watch['webdriver_delay'] is not None:
-            fetcher.render_extract_delay = watch['webdriver_delay']
-        elif system_webdriver_delay is not None:
-            fetcher.render_extract_delay = system_webdriver_delay
-
-        if watch['webdriver_js_execute_code'] is not None and watch['webdriver_js_execute_code'].strip():
-            fetcher.webdriver_js_execute_code = watch['webdriver_js_execute_code']
-
-        fetcher.run(url, timeout, request_headers, request_body, request_method, ignore_status_code, watch['css_filter'])
-        fetcher.quit()
-
-        # Fetching complete, now filters
-        # @todo move to class / maybe inside of fetcher abstract base?
-
-        # @note: I feel like the following should be in a more obvious chain system
-        #  - Check filter text
-        #  - Is the checksum different?
-        #  - Do we convert to JSON?
-        # https://stackoverflow.com/questions/41817578/basic-method-chaining ?
-        # return content().textfilter().jsonextract().checksumcompare() ?
-
-        is_json = 'application/json' in fetcher.headers.get('Content-Type', '')
-        is_html = not is_json
-
-        # source: support, basically treat it as plaintext
-        if is_source:
-            is_html = False
-            is_json = False
-
-        css_filter_rule = watch['css_filter']
-        subtractive_selectors = watch.get(
-            "subtractive_selectors", []
-        ) + self.datastore.data["settings"]["application"].get(
-            "global_subtractive_selectors", []
-        )
-
-        has_filter_rule = css_filter_rule and len(css_filter_rule.strip())
-        has_subtractive_selectors = subtractive_selectors and len(subtractive_selectors[0].strip())
-
-        if is_json and not has_filter_rule:
-            css_filter_rule = "json:$"
-            has_filter_rule = True
-
-        if has_filter_rule:
-            if 'json:' in css_filter_rule:
-                stripped_text_from_html = html_tools.extract_json_as_string(content=fetcher.content, jsonpath_filter=css_filter_rule)
-                is_html = False
-
-        if is_html or is_source:
-            
-            # CSS Filter, extract the HTML that matches and feed that into the existing inscriptis::get_text
-            fetcher.content = html_tools.workarounds_for_obfuscations(fetcher.content)
-            html_content = fetcher.content
-
-            # If not JSON,  and if it's not text/plain..
-            if 'text/plain' in fetcher.headers.get('Content-Type', '').lower():
-                # Don't run get_text or xpath/css filters on plaintext
-                stripped_text_from_html = html_content
-            else:
-                # Then we assume HTML
-                if has_filter_rule:
-                    # For HTML/XML we offer xpath as an option, just start a regular xPath "/.."
-                    if css_filter_rule[0] == '/' or css_filter_rule.startswith('xpath:'):
-                        html_content = html_tools.xpath_filter(xpath_filter=css_filter_rule.replace('xpath:', ''),
-                                                               html_content=fetcher.content)
-                    else:
-                        # CSS Filter, extract the HTML that matches and feed that into the existing inscriptis::get_text
-                        html_content = html_tools.css_filter(css_filter=css_filter_rule, html_content=fetcher.content)
-
-                if has_subtractive_selectors:
-                    html_content = html_tools.element_removal(subtractive_selectors, html_content)
-
-                if not is_source:
-                    # extract text
-                    stripped_text_from_html = \
-                        html_tools.html_to_text(
-                            html_content,
-                            render_anchor_tag_content=self.datastore.data["settings"][
-                                "application"].get(
-                                "render_anchor_tag_content", False)
-                        )
-
-                elif is_source:
-                    stripped_text_from_html = html_content
-
-            # Re #340 - return the content before the 'ignore text' was applied
-            text_content_before_ignored_filter = stripped_text_from_html.encode('utf-8')
-
-        # Re #340 - return the content before the 'ignore text' was applied
-        text_content_before_ignored_filter = stripped_text_from_html.encode('utf-8')
-
-        # Treat pages with no renderable text content as a change? No by default
-        empty_pages_are_a_change = self.datastore.data['settings']['application'].get('empty_pages_are_a_change', False)
-        if not is_json and not empty_pages_are_a_change and len(stripped_text_from_html.strip()) == 0:
-            raise content_fetcher.ReplyWithContentButNoText(url=url, status_code=200)
-
-        # We rely on the actual text in the html output.. many sites have random script vars etc,
-        # in the future we'll implement other mechanisms.
-
-        update_obj["last_check_status"] = fetcher.get_last_status_code()
-
-        # If there's text to skip
-        # @todo we could abstract out the get_text() to handle this cleaner
-        text_to_ignore = watch.get('ignore_text', []) + self.datastore.data['settings']['application'].get('global_ignore_text', [])
-        if len(text_to_ignore):
-            stripped_text_from_html = html_tools.strip_ignore_text(stripped_text_from_html, text_to_ignore)
-        else:
-            stripped_text_from_html = stripped_text_from_html.encode('utf8')
-
-        # 615 Extract text by regex
-        extract_text = watch.get('extract_text', [])
-        if len(extract_text) > 0:
-            regex_matched_output = []
-            for s_re in extract_text:
-                # incase they specified something in '/.../x'
-                regex = self.forward_slash_enclosed_regex_to_options(s_re)
-                result = re.findall(regex.encode('utf-8'), stripped_text_from_html)
-
-                for l in result:
-                    if type(l) is tuple:
-                        #@todo - some formatter option default (between groups)
-                        regex_matched_output += list(l) + [b'\n']
-                    else:
-                        # @todo - some formatter option default (between each ungrouped result)
-                        regex_matched_output += [l] + [b'\n']
-
-            # Now we will only show what the regex matched
-            stripped_text_from_html = b''
-            text_content_before_ignored_filter = b''
-            if regex_matched_output:
-                # @todo some formatter for presentation?
-                stripped_text_from_html = b''.join(regex_matched_output)
-                text_content_before_ignored_filter = stripped_text_from_html
-
-
-        # Re #133 - if we should strip whitespaces from triggering the change detected comparison
-        if self.datastore.data['settings']['application'].get('ignore_whitespace', False):
-            fetched_md5 = hashlib.md5(stripped_text_from_html.translate(None, b'\r\n\t ')).hexdigest()
-        else:
-            fetched_md5 = hashlib.md5(stripped_text_from_html).hexdigest()
-
-        ############ Blocking rules, after checksum #################
-        blocked = False
-
-        if len(watch['trigger_text']):
-            # Assume blocked
-            blocked = True
-            # Filter and trigger works the same, so reuse it
-            # It should return the line numbers that match
-            result = html_tools.strip_ignore_text(content=str(stripped_text_from_html),
-                                                  wordlist=watch['trigger_text'],
-                                                  mode="line numbers")
-            # Unblock if the trigger was found
-            if result:
-                blocked = False
-
-
-        if len(watch['text_should_not_be_present']):
-            # If anything matched, then we should block a change from happening
-            result = html_tools.strip_ignore_text(content=str(stripped_text_from_html),
-                                                  wordlist=watch['text_should_not_be_present'],
-                                                  mode="line numbers")
-            if result:
-                blocked = True
-
-        # The main thing that all this at the moment comes down to :)
-        if watch['previous_md5'] != fetched_md5:
-            changed_detected = True
-
-        # Looks like something changed, but did it match all the rules?
-        if blocked:
-            changed_detected = False
-
-        # Extract title as title
-        if is_html:
-            if self.datastore.data['settings']['application']['extract_title_as_title'] or watch['extract_title_as_title']:
-                if not watch['title'] or not len(watch['title']):
-                    update_obj['title'] = html_tools.extract_element(find='title', html_content=fetcher.content)
-
-        if changed_detected:
-            if watch.get('check_unique_lines', False):
-                has_unique_lines = watch.lines_contain_something_unique_compared_to_history(lines=stripped_text_from_html.splitlines())
-                # One or more lines? unsure?
-                if not has_unique_lines:
-                    logging.debug("check_unique_lines: UUID {} didnt have anything new setting change_detected=False".format(uuid))
-                    changed_detected = False
-                else:
-                    logging.debug("check_unique_lines: UUID {} had unique content".format(uuid))
-
-        # Always record the new checksum
-        update_obj["previous_md5"] = fetched_md5
-
-        # On the first run of a site, watch['previous_md5'] will be None, set it the current one.
-        if not watch.get('previous_md5'):
-            watch['previous_md5'] = fetched_md5
-
-        return changed_detected, update_obj, text_content_before_ignored_filter, fetcher.screenshot, fetcher.xpath_data

changedetectionio/flask_app.py

@@ -0,0 +1,870 @@
+#!/usr/bin/env python3
+
+import flask_login
+import locale
+import os
+import queue
+import sys
+import threading
+import time
+import timeago
+from blinker import signal
+
+from changedetectionio.strtobool import strtobool
+from threading import Event
+from changedetectionio.queue_handlers import RecheckPriorityQueue, NotificationQueue
+from changedetectionio import worker_handler
+
+from flask import (
+    Flask,
+    abort,
+    flash,
+    redirect,
+    render_template,
+    request,
+    send_from_directory,
+    url_for,
+)
+from flask_compress import Compress as FlaskCompress
+from flask_login import current_user
+from flask_restful import abort, Api
+from flask_cors import CORS
+
+# Create specific signals for application events
+# Make this a global singleton to avoid multiple signal objects
+watch_check_update = signal('watch_check_update', doc='Signal sent when a watch check is completed')
+from flask_wtf import CSRFProtect
+from loguru import logger
+
+from changedetectionio import __version__
+from changedetectionio import queuedWatchMetaData
+from changedetectionio.api import Watch, WatchHistory, WatchSingleHistory, CreateWatch, Import, SystemInfo, Tag, Tags, Notifications, WatchFavicon
+from changedetectionio.api.Search import Search
+from .time_handler import is_within_schedule
+
+datastore = None
+
+# Local
+ticker_thread = None
+extra_stylesheets = []
+
+# Use bulletproof janus-based queues for sync/async reliability  
+update_q = RecheckPriorityQueue()
+notification_q = NotificationQueue()
+MAX_QUEUE_SIZE = 2000
+
+app = Flask(__name__,
+            static_url_path="",
+            static_folder="static",
+            template_folder="templates")
+
+# Will be initialized in changedetection_app
+socketio_server = None
+
+# Enable CORS, especially useful for the Chrome extension to operate from anywhere
+CORS(app)
+
+# Super handy for compressing large BrowserSteps responses and others
+FlaskCompress(app)
+
+# Stop browser caching of assets
+app.config['SEND_FILE_MAX_AGE_DEFAULT'] = 0
+app.config.exit = Event()
+
+app.config['NEW_VERSION_AVAILABLE'] = False
+
+if os.getenv('FLASK_SERVER_NAME'):
+    app.config['SERVER_NAME'] = os.getenv('FLASK_SERVER_NAME')
+
+#app.config["EXPLAIN_TEMPLATE_LOADING"] = True
+
+# Disables caching of the templates
+app.config['TEMPLATES_AUTO_RELOAD'] = True
+app.jinja_env.add_extension('jinja2.ext.loopcontrols')
+csrf = CSRFProtect()
+csrf.init_app(app)
+notification_debug_log=[]
+
+# Locale for correct presentation of prices etc
+default_locale = locale.getdefaultlocale()
+logger.info(f"System locale default is {default_locale}")
+try:
+    locale.setlocale(locale.LC_ALL, default_locale)
+except locale.Error:
+    logger.warning(f"Unable to set locale {default_locale}, locale is not installed maybe?")
+
+watch_api = Api(app, decorators=[csrf.exempt])
+
+def init_app_secret(datastore_path):
+    secret = ""
+
+    path = os.path.join(datastore_path, "secret.txt")
+
+    try:
+        with open(path, "r") as f:
+            secret = f.read()
+
+    except FileNotFoundError:
+        import secrets
+        with open(path, "w") as f:
+            secret = secrets.token_hex(32)
+            f.write(secret)
+
+    return secret
+
+
+@app.template_global()
+def get_darkmode_state():
+    css_dark_mode = request.cookies.get('css_dark_mode', 'false')
+    return 'true' if css_dark_mode and strtobool(css_dark_mode) else 'false'
+
+@app.template_global()
+def get_css_version():
+    return __version__
+
+@app.template_global()
+def get_socketio_path():
+    """Generate the correct Socket.IO path prefix for the client"""
+    # If behind a proxy with a sub-path, we need to respect that path
+    prefix = ""
+    if os.getenv('USE_X_SETTINGS') and 'X-Forwarded-Prefix' in request.headers:
+        prefix = request.headers['X-Forwarded-Prefix']
+
+    # Socket.IO will be available at {prefix}/socket.io/
+    return prefix
+
+
+@app.template_filter('format_number_locale')
+def _jinja2_filter_format_number_locale(value: float) -> str:
+    "Formats for example 4000.10 to the local locale default of 4,000.10"
+    # Format the number with two decimal places (locale format string will return 6 decimal)
+    formatted_value = locale.format_string("%.2f", value, grouping=True)
+
+    return formatted_value
+
+@app.template_global('is_checking_now')
+def _watch_is_checking_now(watch_obj, format="%Y-%m-%d %H:%M:%S"):
+    return worker_handler.is_watch_running(watch_obj['uuid'])
+
+@app.template_global('get_watch_queue_position')
+def _get_watch_queue_position(watch_obj):
+    """Get the position of a watch in the queue"""
+    uuid = watch_obj['uuid']
+    return update_q.get_uuid_position(uuid)
+
+@app.template_global('get_current_worker_count')
+def _get_current_worker_count():
+    """Get the current number of operational workers"""
+    return worker_handler.get_worker_count()
+
+@app.template_global('get_worker_status_info')
+def _get_worker_status_info():
+    """Get detailed worker status information for display"""
+    status = worker_handler.get_worker_status()
+    running_uuids = worker_handler.get_running_uuids()
+    
+    return {
+        'count': status['worker_count'],
+        'type': status['worker_type'],
+        'active_workers': len(running_uuids),
+        'processing_watches': running_uuids,
+        'loop_running': status.get('async_loop_running', None)
+    }
+
+
+# We use the whole watch object from the store/JSON so we can see if there's some related status in terms of a thread
+# running or something similar.
+@app.template_filter('format_last_checked_time')
+def _jinja2_filter_datetime(watch_obj, format="%Y-%m-%d %H:%M:%S"):
+
+    if watch_obj['last_checked'] == 0:
+        return 'Not yet'
+
+    return timeago.format(int(watch_obj['last_checked']), time.time())
+
+@app.template_filter('format_timestamp_timeago')
+def _jinja2_filter_datetimestamp(timestamp, format="%Y-%m-%d %H:%M:%S"):
+    if not timestamp:
+        return 'Not yet'
+
+    return timeago.format(int(timestamp), time.time())
+
+
+@app.template_filter('pagination_slice')
+def _jinja2_filter_pagination_slice(arr, skip):
+    per_page = datastore.data['settings']['application'].get('pager_size', 50)
+    if per_page:
+        return arr[skip:skip + per_page]
+
+    return arr
+
+@app.template_filter('format_seconds_ago')
+def _jinja2_filter_seconds_precise(timestamp):
+    if timestamp == False:
+        return 'Not yet'
+
+    return format(int(time.time()-timestamp), ',d')
+
+# Import login_optionally_required from auth_decorator
+from changedetectionio.auth_decorator import login_optionally_required
+
+# When nobody is logged in Flask-Login's current_user is set to an AnonymousUser object.
+class User(flask_login.UserMixin):
+    id=None
+
+    def set_password(self, password):
+        return True
+    def get_user(self, email="defaultuser@changedetection.io"):
+        return self
+    def is_authenticated(self):
+        return True
+    def is_active(self):
+        return True
+    def is_anonymous(self):
+        return False
+    def get_id(self):
+        return str(self.id)
+
+    # Compare given password against JSON store or Env var
+    def check_password(self, password):
+        import base64
+        import hashlib
+
+        # Can be stored in env (for deployments) or in the general configs
+        raw_salt_pass = os.getenv("SALTED_PASS", False)
+
+        if not raw_salt_pass:
+            raw_salt_pass = datastore.data['settings']['application'].get('password')
+
+        raw_salt_pass = base64.b64decode(raw_salt_pass)
+        salt_from_storage = raw_salt_pass[:32]  # 32 is the length of the salt
+
+        # Use the exact same setup you used to generate the key, but this time put in the password to check
+        new_key = hashlib.pbkdf2_hmac(
+            'sha256',
+            password.encode('utf-8'),  # Convert the password to bytes
+            salt_from_storage,
+            100000
+        )
+        new_key = salt_from_storage + new_key
+
+        return new_key == raw_salt_pass
+
+    pass
+
+
+def changedetection_app(config=None, datastore_o=None):
+    logger.trace("TRACE log is enabled")
+
+    global datastore, socketio_server
+    datastore = datastore_o
+
+    # so far just for read-only via tests, but this will be moved eventually to be the main source
+    # (instead of the global var)
+    app.config['DATASTORE'] = datastore_o
+    
+    # Store the signal in the app config to ensure it's accessible everywhere
+    app.config['watch_check_update_SIGNAL'] = watch_check_update
+
+    login_manager = flask_login.LoginManager(app)
+    login_manager.login_view = 'login'
+    app.secret_key = init_app_secret(config['datastore_path'])
+    
+    # Set up a request hook to check authentication for all routes
+    @app.before_request
+    def check_authentication():
+        has_password_enabled = datastore.data['settings']['application'].get('password') or os.getenv("SALTED_PASS", False)
+
+        if has_password_enabled and not flask_login.current_user.is_authenticated:
+            # Permitted
+            if request.endpoint and request.endpoint == 'static_content' and request.view_args:
+                # Handled by static_content handler
+                return None
+            # Permitted
+            elif request.endpoint and 'login' in request.endpoint:
+                return None
+            elif request.endpoint and 'diff_history_page' in request.endpoint and datastore.data['settings']['application'].get('shared_diff_access'):
+                return None
+            elif request.method in flask_login.config.EXEMPT_METHODS:
+                return None
+            elif app.config.get('LOGIN_DISABLED'):
+                return None
+            # RSS access with token is allowed
+            elif request.endpoint and 'rss.feed' in request.endpoint:
+                return None
+            # Socket.IO routes - need separate handling
+            elif request.path.startswith('/socket.io/'):
+                return None
+            # API routes - use their own auth mechanism (@auth.check_token)
+            elif request.path.startswith('/api/'):
+                return None
+            else:
+                return login_manager.unauthorized()
+
+
+    watch_api.add_resource(WatchSingleHistory,
+                           '/api/v1/watch/<string:uuid>/history/<string:timestamp>',
+                           resource_class_kwargs={'datastore': datastore, 'update_q': update_q})
+    watch_api.add_resource(WatchFavicon,
+                           '/api/v1/watch/<string:uuid>/favicon',
+                           resource_class_kwargs={'datastore': datastore})
+    watch_api.add_resource(WatchHistory,
+                           '/api/v1/watch/<string:uuid>/history',
+                           resource_class_kwargs={'datastore': datastore})
+
+    watch_api.add_resource(CreateWatch, '/api/v1/watch',
+                           resource_class_kwargs={'datastore': datastore, 'update_q': update_q})
+
+    watch_api.add_resource(Watch, '/api/v1/watch/<string:uuid>',
+                           resource_class_kwargs={'datastore': datastore, 'update_q': update_q})
+
+    watch_api.add_resource(SystemInfo, '/api/v1/systeminfo',
+                           resource_class_kwargs={'datastore': datastore, 'update_q': update_q})
+
+    watch_api.add_resource(Import,
+                           '/api/v1/import',
+                           resource_class_kwargs={'datastore': datastore})
+
+    watch_api.add_resource(Tags, '/api/v1/tags',
+                           resource_class_kwargs={'datastore': datastore})
+
+    watch_api.add_resource(Tag, '/api/v1/tag', '/api/v1/tag/<string:uuid>',
+                           resource_class_kwargs={'datastore': datastore, 'update_q': update_q})
+                           
+    watch_api.add_resource(Search, '/api/v1/search',
+                           resource_class_kwargs={'datastore': datastore})
+
+    watch_api.add_resource(Notifications, '/api/v1/notifications',
+                           resource_class_kwargs={'datastore': datastore})
+
+    @login_manager.user_loader
+    def user_loader(email):
+        user = User()
+        user.get_user(email)
+        return user
+
+    @login_manager.unauthorized_handler
+    def unauthorized_handler():
+        flash("You must be logged in, please log in.", 'error')
+        return redirect(url_for('login', next=url_for('watchlist.index')))
+
+    @app.route('/logout')
+    def logout():
+        flask_login.logout_user()
+        return redirect(url_for('watchlist.index'))
+
+    # https://github.com/pallets/flask/blob/93dd1709d05a1cf0e886df6223377bdab3b077fb/examples/tutorial/flaskr/__init__.py#L39
+    # You can divide up the stuff like this
+    @app.route('/login', methods=['GET', 'POST'])
+    def login():
+
+        if request.method == 'GET':
+            if flask_login.current_user.is_authenticated:
+                flash("Already logged in")
+                return redirect(url_for("watchlist.index"))
+
+            output = render_template("login.html")
+            return output
+
+        user = User()
+        user.id = "defaultuser@changedetection.io"
+
+        password = request.form.get('password')
+
+        if (user.check_password(password)):
+            flask_login.login_user(user, remember=True)
+
+            # For now there's nothing else interesting here other than the index/list page
+            # It's more reliable and safe to ignore the 'next' redirect
+            # When we used...
+            # next = request.args.get('next')
+            # return redirect(next or url_for('watchlist.index'))
+            # We would sometimes get login loop errors on sites hosted in sub-paths
+
+            # note for the future:
+            #            if not is_safe_url(next):
+            #                return flask.abort(400)
+            return redirect(url_for('watchlist.index'))
+
+        else:
+            flash('Incorrect password', 'error')
+
+        return redirect(url_for('login'))
+
+    @app.before_request
+    def before_request_handle_cookie_x_settings():
+        # Set the auth cookie path if we're running as X-settings/X-Forwarded-Prefix
+        if os.getenv('USE_X_SETTINGS') and 'X-Forwarded-Prefix' in request.headers:
+            app.config['REMEMBER_COOKIE_PATH'] = request.headers['X-Forwarded-Prefix']
+            app.config['SESSION_COOKIE_PATH'] = request.headers['X-Forwarded-Prefix']
+        return None
+
+    @app.route("/static/<string:group>/<string:filename>", methods=['GET'])
+    def static_content(group, filename):
+        from flask import make_response
+        import re
+        group = re.sub(r'[^\w.-]+', '', group.lower())
+        filename = re.sub(r'[^\w.-]+', '', filename.lower())
+
+        if group == 'screenshot':
+            # Could be sensitive, follow password requirements
+            if datastore.data['settings']['application']['password'] and not flask_login.current_user.is_authenticated:
+                if not datastore.data['settings']['application'].get('shared_diff_access'):
+                    abort(403)
+
+            screenshot_filename = "last-screenshot.png" if not request.args.get('error_screenshot') else "last-error-screenshot.png"
+
+            # These files should be in our subdirectory
+            try:
+                # set nocache, set content-type
+                response = make_response(send_from_directory(os.path.join(datastore_o.datastore_path, filename), screenshot_filename))
+                response.headers['Content-type'] = 'image/png'
+                response.headers['Cache-Control'] = 'no-cache, no-store, must-revalidate'
+                response.headers['Pragma'] = 'no-cache'
+                response.headers['Expires'] = 0
+                return response
+
+            except FileNotFoundError:
+                abort(404)
+
+        if group == 'favicon':
+            # Could be sensitive, follow password requirements
+            if datastore.data['settings']['application']['password'] and not flask_login.current_user.is_authenticated:
+                abort(403)
+            # Get the watch object
+            watch = datastore.data['watching'].get(filename)
+            if not watch:
+                abort(404)
+
+            favicon_filename = watch.get_favicon_filename()
+            if favicon_filename:
+                try:
+                    import magic
+                    mime = magic.from_file(
+                        os.path.join(watch.watch_data_dir, favicon_filename),
+                        mime=True
+                    )
+                except ImportError:
+                    # Fallback, no python-magic
+                    import mimetypes
+                    mime, encoding = mimetypes.guess_type(favicon_filename)
+
+                response = make_response(send_from_directory(watch.watch_data_dir, favicon_filename))
+                response.headers['Content-type'] = mime
+                response.headers['Cache-Control'] = 'max-age=300, must-revalidate'  # Cache for 5 minutes, then revalidate
+                return response
+
+        if group == 'visual_selector_data':
+            # Could be sensitive, follow password requirements
+            if datastore.data['settings']['application']['password'] and not flask_login.current_user.is_authenticated:
+                abort(403)
+
+            # These files should be in our subdirectory
+            try:
+                # set nocache, set content-type,
+                # `filename` is actually directory UUID of the watch
+                watch_directory = str(os.path.join(datastore_o.datastore_path, filename))
+                response = None
+                if os.path.isfile(os.path.join(watch_directory, "elements.deflate")):
+                    response = make_response(send_from_directory(watch_directory, "elements.deflate"))
+                    response.headers['Content-Type'] = 'application/json'
+                    response.headers['Content-Encoding'] = 'deflate'
+                else:
+                    logger.error(f'Request elements.deflate at "{watch_directory}" but was not found.')
+                    abort(404)
+
+                if response:
+                    response.headers['Cache-Control'] = 'no-cache, no-store, must-revalidate'
+                    response.headers['Pragma'] = 'no-cache'
+                    response.headers['Expires'] = "0"
+
+                return response
+
+            except FileNotFoundError:
+                abort(404)
+
+        # These files should be in our subdirectory
+        try:
+            return send_from_directory(f"static/{group}", path=filename)
+        except FileNotFoundError:
+            abort(404)
+
+
+    import changedetectionio.blueprint.browser_steps as browser_steps
+    app.register_blueprint(browser_steps.construct_blueprint(datastore), url_prefix='/browser-steps')
+
+    from changedetectionio.blueprint.imports import construct_blueprint as construct_import_blueprint
+    app.register_blueprint(construct_import_blueprint(datastore, update_q, queuedWatchMetaData), url_prefix='/imports')
+
+    import changedetectionio.blueprint.price_data_follower as price_data_follower
+    app.register_blueprint(price_data_follower.construct_blueprint(datastore, update_q), url_prefix='/price_data_follower')
+
+    import changedetectionio.blueprint.tags as tags
+    app.register_blueprint(tags.construct_blueprint(datastore), url_prefix='/tags')
+
+    import changedetectionio.blueprint.check_proxies as check_proxies
+    app.register_blueprint(check_proxies.construct_blueprint(datastore=datastore), url_prefix='/check_proxy')
+
+    import changedetectionio.blueprint.backups as backups
+    app.register_blueprint(backups.construct_blueprint(datastore), url_prefix='/backups')
+
+    import changedetectionio.blueprint.settings as settings
+    app.register_blueprint(settings.construct_blueprint(datastore), url_prefix='/settings')
+
+    import changedetectionio.conditions.blueprint as conditions
+    app.register_blueprint(conditions.construct_blueprint(datastore), url_prefix='/conditions')
+
+    import changedetectionio.blueprint.rss.blueprint as rss
+    app.register_blueprint(rss.construct_blueprint(datastore), url_prefix='/rss')
+
+    # watchlist UI buttons etc
+    import changedetectionio.blueprint.ui as ui
+    app.register_blueprint(ui.construct_blueprint(datastore, update_q, worker_handler, queuedWatchMetaData, watch_check_update))
+
+    import changedetectionio.blueprint.watchlist as watchlist
+    app.register_blueprint(watchlist.construct_blueprint(datastore=datastore, update_q=update_q, queuedWatchMetaData=queuedWatchMetaData), url_prefix='')
+
+    # Initialize Socket.IO server conditionally based on settings
+    socket_io_enabled = datastore.data['settings']['application']['ui'].get('socket_io_enabled', True)
+    if socket_io_enabled:
+        from changedetectionio.realtime.socket_server import init_socketio
+        global socketio_server
+        socketio_server = init_socketio(app, datastore)
+        logger.info("Socket.IO server initialized")
+    else:
+        logger.info("Socket.IO server disabled via settings")
+        socketio_server = None
+
+    # Memory cleanup endpoint
+    @app.route('/gc-cleanup', methods=['GET'])
+    @login_optionally_required
+    def gc_cleanup():
+        from changedetectionio.gc_cleanup import memory_cleanup
+        from flask import jsonify
+
+        result = memory_cleanup(app)
+        return jsonify({"status": "success", "message": "Memory cleanup completed", "result": result})
+
+    # Worker health check endpoint
+    @app.route('/worker-health', methods=['GET'])
+    @login_optionally_required
+    def worker_health():
+        from flask import jsonify
+        
+        expected_workers = int(os.getenv("FETCH_WORKERS", datastore.data['settings']['requests']['workers']))
+        
+        # Get basic status
+        status = worker_handler.get_worker_status()
+        
+        # Perform health check
+        health_result = worker_handler.check_worker_health(
+            expected_count=expected_workers,
+            update_q=update_q,
+            notification_q=notification_q,
+            app=app,
+            datastore=datastore
+        )
+        
+        return jsonify({
+            "status": "success",
+            "worker_status": status,
+            "health_check": health_result,
+            "expected_workers": expected_workers
+        })
+
+    # Queue status endpoint
+    @app.route('/queue-status', methods=['GET'])
+    @login_optionally_required
+    def queue_status():
+        from flask import jsonify, request
+        
+        # Get specific UUID position if requested
+        target_uuid = request.args.get('uuid')
+        
+        if target_uuid:
+            position_info = update_q.get_uuid_position(target_uuid)
+            return jsonify({
+                "status": "success",
+                "uuid": target_uuid,
+                "queue_position": position_info
+            })
+        else:
+            # Get pagination parameters
+            limit = request.args.get('limit', type=int)
+            offset = request.args.get('offset', type=int, default=0)
+            summary_only = request.args.get('summary', type=bool, default=False)
+            
+            if summary_only:
+                # Fast summary for large queues
+                summary = update_q.get_queue_summary()
+                return jsonify({
+                    "status": "success",
+                    "queue_summary": summary
+                })
+            else:
+                # Get queued items with pagination support
+                if limit is None:
+                    # Default limit for large queues to prevent performance issues
+                    queue_size = update_q.qsize()
+                    if queue_size > 100:
+                        limit = 50
+                        logger.warning(f"Large queue ({queue_size} items) detected, limiting to {limit} items. Use ?limit=N for more.")
+                
+                all_queued = update_q.get_all_queued_uuids(limit=limit, offset=offset)
+                return jsonify({
+                    "status": "success",
+                    "queue_size": update_q.qsize(),
+                    "queued_data": all_queued
+                })
+
+    # Start the async workers during app initialization
+    # Can be overridden by ENV or use the default settings
+    n_workers = int(os.getenv("FETCH_WORKERS", datastore.data['settings']['requests']['workers']))
+    logger.info(f"Starting {n_workers} workers during app initialization")
+    worker_handler.start_workers(n_workers, update_q, notification_q, app, datastore)
+
+    # @todo handle ctrl break
+    ticker_thread = threading.Thread(target=ticker_thread_check_time_launch_checks).start()
+    threading.Thread(target=notification_runner).start()
+
+    in_pytest = "pytest" in sys.modules or "PYTEST_CURRENT_TEST" in os.environ
+    # Check for new release version, but not when running in test/build or pytest
+    if not os.getenv("GITHUB_REF", False) and not strtobool(os.getenv('DISABLE_VERSION_CHECK', 'no')) and not in_pytest:
+        threading.Thread(target=check_for_new_version).start()
+
+    # Return the Flask app - the Socket.IO will be attached to it but initialized separately
+    # This avoids circular dependencies
+    return app
+
+
+# Check for new version and anonymous stats
+def check_for_new_version():
+    import requests
+    import urllib3
+    urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)
+
+    while not app.config.exit.is_set():
+        try:
+            r = requests.post("https://changedetection.io/check-ver.php",
+                              data={'version': __version__,
+                                    'app_guid': datastore.data['app_guid'],
+                                    'watch_count': len(datastore.data['watching'])
+                                    },
+
+                              verify=False)
+        except:
+            pass
+
+        try:
+            if "new_version" in r.text:
+                app.config['NEW_VERSION_AVAILABLE'] = True
+        except:
+            pass
+
+        # Check daily
+        app.config.exit.wait(86400)
+
+
+def notification_runner():
+    global notification_debug_log
+    from datetime import datetime
+    import json
+    with app.app_context():
+        while not app.config.exit.is_set():
+            try:
+                # At the moment only one thread runs (single runner)
+                n_object = notification_q.get(block=False)
+            except queue.Empty:
+                time.sleep(1)
+
+            else:
+
+                now = datetime.now()
+                sent_obj = None
+
+                try:
+                    from changedetectionio.notification.handler import process_notification
+
+                    # Fallback to system config if not set
+                    if not n_object.get('notification_body') and datastore.data['settings']['application'].get('notification_body'):
+                        n_object['notification_body'] = datastore.data['settings']['application'].get('notification_body')
+
+                    if not n_object.get('notification_title') and datastore.data['settings']['application'].get('notification_title'):
+                        n_object['notification_title'] = datastore.data['settings']['application'].get('notification_title')
+
+                    if not n_object.get('notification_format') and datastore.data['settings']['application'].get('notification_format'):
+                        n_object['notification_format'] = datastore.data['settings']['application'].get('notification_format')
+                    if n_object.get('notification_urls', {}):
+                        sent_obj = process_notification(n_object, datastore)
+
+                except Exception as e:
+                    logger.error(f"Watch URL: {n_object['watch_url']}  Error {str(e)}")
+
+                    # UUID wont be present when we submit a 'test' from the global settings
+                    if 'uuid' in n_object:
+                        datastore.update_watch(uuid=n_object['uuid'],
+                                               update_obj={'last_notification_error': "Notification error detected, goto notification log."})
+
+                    log_lines = str(e).splitlines()
+                    notification_debug_log += log_lines
+
+                    with app.app_context():
+                        app.config['watch_check_update_SIGNAL'].send(app_context=app, watch_uuid=n_object.get('uuid'))
+
+                # Process notifications
+                notification_debug_log+= ["{} - SENDING - {}".format(now.strftime("%Y/%m/%d %H:%M:%S,000"), json.dumps(sent_obj))]
+                # Trim the log length
+                notification_debug_log = notification_debug_log[-100:]
+
+
+
+# Threaded runner, look for new watches to feed into the Queue.
+def ticker_thread_check_time_launch_checks():
+    import random
+    proxy_last_called_time = {}
+    last_health_check = 0
+
+    recheck_time_minimum_seconds = int(os.getenv('MINIMUM_SECONDS_RECHECK_TIME', 3))
+    logger.debug(f"System env MINIMUM_SECONDS_RECHECK_TIME {recheck_time_minimum_seconds}")
+
+    # Workers are now started during app initialization, not here
+
+    while not app.config.exit.is_set():
+
+        # Periodic worker health check (every 60 seconds)
+        now = time.time()
+        if now - last_health_check > 60:
+            expected_workers = int(os.getenv("FETCH_WORKERS", datastore.data['settings']['requests']['workers']))
+            health_result = worker_handler.check_worker_health(
+                expected_count=expected_workers,
+                update_q=update_q,
+                notification_q=notification_q,
+                app=app,
+                datastore=datastore
+            )
+            
+            if health_result['status'] != 'healthy':
+                logger.warning(f"Worker health check: {health_result['message']}")
+                
+            last_health_check = now
+
+        # Get a list of watches by UUID that are currently fetching data
+        running_uuids = worker_handler.get_running_uuids()
+
+        # Re #232 - Deepcopy the data incase it changes while we're iterating through it all
+        watch_uuid_list = []
+        while True:
+            try:
+                # Get a list of watches sorted by last_checked, [1] because it gets passed a tuple
+                # This is so we examine the most over-due first
+                for k in sorted(datastore.data['watching'].items(), key=lambda item: item[1].get('last_checked',0)):
+                    watch_uuid_list.append(k[0])
+
+            except RuntimeError as e:
+                # RuntimeError: dictionary changed size during iteration
+                time.sleep(0.1)
+                watch_uuid_list = []
+            else:
+                break
+
+        # Re #438 - Don't place more watches in the queue to be checked if the queue is already large
+        while update_q.qsize() >= 2000:
+            logger.warning(f"Recheck watches queue size limit reached ({MAX_QUEUE_SIZE}), skipping adding more items")
+            time.sleep(3)
+
+
+        recheck_time_system_seconds = int(datastore.threshold_seconds)
+
+        # Check for watches outside of the time threshold to put in the thread queue.
+        for uuid in watch_uuid_list:
+            now = time.time()
+            watch = datastore.data['watching'].get(uuid)
+            if not watch:
+                logger.error(f"Watch: {uuid} no longer present.")
+                continue
+
+            # No need todo further processing if it's paused
+            if watch['paused']:
+                continue
+
+            # @todo - Maybe make this a hook?
+            # Time schedule limit - Decide between watch or global settings
+            if watch.get('time_between_check_use_default'):
+                time_schedule_limit = datastore.data['settings']['requests'].get('time_schedule_limit', {})
+                logger.trace(f"{uuid} Time scheduler - Using system/global settings")
+            else:
+                time_schedule_limit = watch.get('time_schedule_limit')
+                logger.trace(f"{uuid} Time scheduler - Using watch settings (not global settings)")
+            tz_name = datastore.data['settings']['application'].get('timezone', 'UTC')
+
+            if time_schedule_limit and time_schedule_limit.get('enabled'):
+                try:
+                    result = is_within_schedule(time_schedule_limit=time_schedule_limit,
+                                                default_tz=tz_name
+                                                )
+                    if not result:
+                        logger.trace(f"{uuid} Time scheduler - not within schedule skipping.")
+                        continue
+                except Exception as e:
+                    logger.error(
+                        f"{uuid} - Recheck scheduler, error handling timezone, check skipped - TZ name '{tz_name}' - {str(e)}")
+                    return False
+            # If they supplied an individual entry minutes to threshold.
+            threshold = recheck_time_system_seconds if watch.get('time_between_check_use_default') else watch.threshold_seconds()
+
+            # #580 - Jitter plus/minus amount of time to make the check seem more random to the server
+            jitter = datastore.data['settings']['requests'].get('jitter_seconds', 0)
+            if jitter > 0:
+                if watch.jitter_seconds == 0:
+                    watch.jitter_seconds = random.uniform(-abs(jitter), jitter)
+
+            seconds_since_last_recheck = now - watch['last_checked']
+
+            if seconds_since_last_recheck >= (threshold + watch.jitter_seconds) and seconds_since_last_recheck >= recheck_time_minimum_seconds:
+                if not uuid in running_uuids and uuid not in [q_uuid.item['uuid'] for q_uuid in update_q.queue]:
+
+                    # Proxies can be set to have a limit on seconds between which they can be called
+                    watch_proxy = datastore.get_preferred_proxy_for_watch(uuid=uuid)
+                    if watch_proxy and watch_proxy in list(datastore.proxy_list.keys()):
+                        # Proxy may also have some threshold minimum
+                        proxy_list_reuse_time_minimum = int(datastore.proxy_list.get(watch_proxy, {}).get('reuse_time_minimum', 0))
+                        if proxy_list_reuse_time_minimum:
+                            proxy_last_used_time = proxy_last_called_time.get(watch_proxy, 0)
+                            time_since_proxy_used = int(time.time() - proxy_last_used_time)
+                            if time_since_proxy_used < proxy_list_reuse_time_minimum:
+                                # Not enough time difference reached, skip this watch
+                                logger.debug(f"> Skipped UUID {uuid} "
+                                        f"using proxy '{watch_proxy}', not "
+                                        f"enough time between proxy requests "
+                                        f"{time_since_proxy_used}s/{proxy_list_reuse_time_minimum}s")
+                                continue
+                            else:
+                                # Record the last used time
+                                proxy_last_called_time[watch_proxy] = int(time.time())
+
+                    # Use Epoch time as priority, so we get a "sorted" PriorityQueue, but we can still push a priority 1 into it.
+                    priority = int(time.time())
+
+                    # Into the queue with you
+                    queued_successfully = worker_handler.queue_item_async_safe(update_q,
+                                                                               queuedWatchMetaData.PrioritizedItem(priority=priority,
+                                                                                                                   item={'uuid': uuid})
+                                                                               )
+                    if queued_successfully:
+                        logger.debug(
+                            f"> Queued watch UUID {uuid} "
+                            f"last checked at {watch['last_checked']} "
+                            f"queued at {now:0.2f} priority {priority} "
+                            f"jitter {watch.jitter_seconds:0.2f}s, "
+                            f"{now - watch['last_checked']:0.2f}s since last checked")
+                    else:
+                        logger.critical(f"CRITICAL: Failed to queue watch UUID {uuid} in ticker thread!")
+                        
+                    # Reset for next time
+                    watch.jitter_seconds = 0
+
+        # Wait before checking the list again - saves CPU
+        time.sleep(1)
+
+        # Should be low so we can break this out in testing
+        app.config.exit.wait(1)

changedetectionio/forms.py

@@ -1,11 +1,17 @@
+import os
 import re
+from loguru import logger
+from wtforms.widgets.core import TimeInput
+
+from changedetectionio.blueprint.rss import RSS_FORMAT_TYPES
+from changedetectionio.conditions.form import ConditionFormRow
+from changedetectionio.strtobool import strtobool
 
 from wtforms import (
     BooleanField,
-    Field,
     Form,
+    Field,
     IntegerField,
-    PasswordField,
     RadioField,
     SelectField,
     StringField,
@@ -13,30 +19,46 @@ from wtforms import (
     TextAreaField,
     fields,
     validators,
-    widgets,
+    widgets
 )
+from flask_wtf.file import FileField, FileAllowed
+from wtforms.fields import FieldList
+from wtforms.utils import unset_value
+
 from wtforms.validators import ValidationError
 
-from changedetectionio import content_fetcher
+from validators.url import url as url_validator
+
+from changedetectionio.widgets import TernaryNoneBooleanField
+
+
+# default
+# each select <option data-enabled="enabled-0-0"
+from changedetectionio.blueprint.browser_steps.browser_steps import browser_step_ui_config
+
+from changedetectionio import html_tools, content_fetchers
+
 from changedetectionio.notification import (
-    default_notification_body,
-    default_notification_format,
-    default_notification_title,
     valid_notification_formats,
 )
 
 from wtforms.fields import FormField
 
+dictfilt = lambda x, y: dict([ (i,x[i]) for i in x if i in set(y) ])
+
 valid_method = {
     'GET',
     'POST',
     'PUT',
     'PATCH',
     'DELETE',
+    'OPTIONS',
 }
 
 default_method = 'GET'
-
+allow_simplehost = not strtobool(os.getenv('BLOCK_SIMPLEHOSTS', 'False'))
+REQUIRE_ATLEAST_ONE_TIME_PART_MESSAGE_DEFAULT='At least one time interval (weeks, days, hours, minutes, or seconds) must be specified.'
+REQUIRE_ATLEAST_ONE_TIME_PART_WHEN_NOT_GLOBAL_DEFAULT='At least one time interval (weeks, days, hours, minutes, or seconds) must be specified when not using global settings.'
 
 class StringListField(StringField):
     widget = widgets.TextArea()
@@ -89,6 +111,139 @@ class SaltyPasswordField(StringField):
         else:
             self.data = False
 
+class StringTagUUID(StringField):
+
+   # process_formdata(self, valuelist) handled manually in POST handler
+
+    # Is what is shown when field <input> is rendered
+    def _value(self):
+        # Tag UUID to name, on submit it will convert it back (in the submit handler of init.py)
+        if self.data and type(self.data) is list:
+            tag_titles = []
+            for i in self.data:
+                tag = self.datastore.data['settings']['application']['tags'].get(i)
+                if tag:
+                    tag_title = tag.get('title')
+                    if tag_title:
+                        tag_titles.append(tag_title)
+
+            return ', '.join(tag_titles)
+
+        if not self.data:
+            return ''
+
+        return 'error'
+
+class TimeDurationForm(Form):
+    hours = SelectField(choices=[(f"{i}", f"{i}") for i in range(0, 25)], default="24",  validators=[validators.Optional()])
+    minutes = SelectField(choices=[(f"{i}", f"{i}") for i in range(0, 60)], default="00", validators=[validators.Optional()])
+
+class TimeStringField(Field):
+    """
+    A WTForms field for time inputs (HH:MM) that stores the value as a string.
+    """
+    widget = TimeInput()  # Use the built-in time input widget
+
+    def _value(self):
+        """
+        Returns the value for rendering in the form.
+        """
+        return self.data if self.data is not None else ""
+
+    def process_formdata(self, valuelist):
+        """
+        Processes the raw input from the form and stores it as a string.
+        """
+        if valuelist:
+            time_str = valuelist[0]
+            # Simple validation for HH:MM format
+            if not time_str or len(time_str.split(":")) != 2:
+                raise ValidationError("Invalid time format. Use HH:MM.")
+            self.data = time_str
+
+
+class validateTimeZoneName(object):
+    """
+       Flask wtform validators wont work with basic auth
+    """
+
+    def __init__(self, message=None):
+        self.message = message
+
+    def __call__(self, form, field):
+        from zoneinfo import available_timezones
+        python_timezones = available_timezones()
+        if field.data and field.data not in python_timezones:
+            raise ValidationError("Not a valid timezone name")
+
+class ScheduleLimitDaySubForm(Form):
+    enabled = BooleanField("not set", default=True)
+    start_time = TimeStringField("Start At", default="00:00", validators=[validators.Optional()])
+    duration = FormField(TimeDurationForm, label="Run duration")
+
+class ScheduleLimitForm(Form):
+    enabled = BooleanField("Use time scheduler", default=False)
+    # Because the label for=""" doesnt line up/work with the actual checkbox
+    monday = FormField(ScheduleLimitDaySubForm, label="")
+    tuesday = FormField(ScheduleLimitDaySubForm, label="")
+    wednesday = FormField(ScheduleLimitDaySubForm, label="")
+    thursday = FormField(ScheduleLimitDaySubForm, label="")
+    friday = FormField(ScheduleLimitDaySubForm, label="")
+    saturday = FormField(ScheduleLimitDaySubForm, label="")
+    sunday = FormField(ScheduleLimitDaySubForm, label="")
+
+    timezone = StringField("Optional timezone to run in",
+                                  render_kw={"list": "timezones"},
+                                  validators=[validateTimeZoneName()]
+                                  )
+    def __init__(
+        self,
+        formdata=None,
+        obj=None,
+        prefix="",
+        data=None,
+        meta=None,
+        **kwargs,
+    ):
+        super().__init__(formdata, obj, prefix, data, meta, **kwargs)
+        self.monday.form.enabled.label.text="Monday"
+        self.tuesday.form.enabled.label.text = "Tuesday"
+        self.wednesday.form.enabled.label.text = "Wednesday"
+        self.thursday.form.enabled.label.text = "Thursday"
+        self.friday.form.enabled.label.text = "Friday"
+        self.saturday.form.enabled.label.text = "Saturday"
+        self.sunday.form.enabled.label.text = "Sunday"
+
+
+def validate_time_between_check_has_values(form):
+    """
+    Custom validation function for TimeBetweenCheckForm.
+    Returns True if at least one time interval field has a value > 0.
+    """
+    res = any([
+        form.weeks.data and int(form.weeks.data) > 0,
+        form.days.data and int(form.days.data) > 0,
+        form.hours.data and int(form.hours.data) > 0,
+        form.minutes.data and int(form.minutes.data) > 0,
+        form.seconds.data and int(form.seconds.data) > 0
+    ])
+
+    return res
+
+
+class RequiredTimeInterval(object):
+    """
+    WTForms validator that ensures at least one time interval field has a value > 0.
+    Use this with FormField(TimeBetweenCheckForm, validators=[RequiredTimeInterval()]).
+    """
+    def __init__(self, message=None):
+        self.message = message or 'At least one time interval (weeks, days, hours, minutes, or seconds) must be specified.'
+
+    def __call__(self, form, field):
+        if not validate_time_between_check_has_values(field.form):
+            raise ValidationError(self.message)
+
+
 class TimeBetweenCheckForm(Form):
     weeks = IntegerField('Weeks', validators=[validators.Optional(), validators.NumberRange(min=0, message="Should contain zero or more seconds")])
     days = IntegerField('Days', validators=[validators.Optional(), validators.NumberRange(min=0, message="Should contain zero or more seconds")])
@@ -97,33 +252,160 @@ class TimeBetweenCheckForm(Form):
     seconds = IntegerField('Seconds', validators=[validators.Optional(), validators.NumberRange(min=0, message="Should contain zero or more seconds")])
     # @todo add total seconds minimum validatior = minimum_seconds_recheck_time
 
+    def __init__(self, formdata=None, obj=None, prefix="", data=None, meta=None, **kwargs):
+        super().__init__(formdata, obj, prefix, data, meta, **kwargs)
+        self.require_at_least_one = kwargs.get('require_at_least_one', False)
+        self.require_at_least_one_message = kwargs.get('require_at_least_one_message', REQUIRE_ATLEAST_ONE_TIME_PART_MESSAGE_DEFAULT)
+
+    def validate(self, **kwargs):
+        """Custom validation that can optionally require at least one time interval."""
+        # Run normal field validation first
+        if not super().validate(**kwargs):
+            return False
+
+        # Apply optional "at least one" validation
+        if self.require_at_least_one:
+            if not validate_time_between_check_has_values(self):
+                # Add error to the form's general errors (not field-specific)
+                if not hasattr(self, '_formdata_errors'):
+                    self._formdata_errors = []
+                self._formdata_errors.append(self.require_at_least_one_message)
+                return False
+
+        return True
+
+
+class EnhancedFormField(FormField):
+    """
+    An enhanced FormField that supports conditional validation with top-level error messages.
+    Adds a 'top_errors' property for validation errors at the FormField level.
+    """
+
+    def __init__(self, form_class, label=None, validators=None, separator="-",
+                 conditional_field=None, conditional_message=None, conditional_test_function=None, **kwargs):
+        """
+        Initialize EnhancedFormField with optional conditional validation.
+
+        :param conditional_field: Name of the field this FormField depends on (e.g. 'time_between_check_use_default')
+        :param conditional_message: Error message to show when validation fails
+        :param conditional_test_function: Custom function to test if FormField has valid values.
+                                        Should take self.form as parameter and return True if valid.
+        """
+        super().__init__(form_class, label, validators, separator, **kwargs)
+        self.top_errors = []
+        self.conditional_field = conditional_field
+        self.conditional_message = conditional_message or "At least one field must have a value when not using defaults."
+        self.conditional_test_function = conditional_test_function
+
+    def validate(self, form, extra_validators=()):
+        """
+        Custom validation that supports conditional logic and stores top-level errors.
+        """
+        self.top_errors = []
+
+        # First run the normal FormField validation
+        base_valid = super().validate(form, extra_validators)
+
+        # Apply conditional validation if configured
+        if self.conditional_field and hasattr(form, self.conditional_field):
+            conditional_field_obj = getattr(form, self.conditional_field)
+
+            # If the conditional field is False/unchecked, check if this FormField has any values
+            if not conditional_field_obj.data:
+                # Use custom test function if provided, otherwise use generic fallback
+                if self.conditional_test_function:
+                    has_any_value = self.conditional_test_function(self.form)
+                else:
+                    # Generic fallback - check if any field has truthy data
+                    has_any_value = any(field.data for field in self.form if hasattr(field, 'data') and field.data)
+
+                if not has_any_value:
+                    self.top_errors.append(self.conditional_message)
+                    base_valid = False
+
+        return base_valid
+
+
+class RequiredFormField(FormField):
+    """
+    A FormField that passes require_at_least_one=True to TimeBetweenCheckForm.
+    Use this when you want the sub-form to always require at least one value.
+    """
+
+    def __init__(self, form_class, label=None, validators=None, separator="-", **kwargs):
+        super().__init__(form_class, label, validators, separator, **kwargs)
+
+    def process(self, formdata, data=unset_value, extra_filters=None):
+        if extra_filters:
+            raise TypeError(
+                "FormField cannot take filters, as the encapsulated"
+                "data is not mutable."
+            )
+
+        if data is unset_value:
+            try:
+                data = self.default()
+            except TypeError:
+                data = self.default
+            self._obj = data
+
+        self.object_data = data
+
+        prefix = self.name + self.separator
+        # Pass require_at_least_one=True to the sub-form
+        if isinstance(data, dict):
+            self.form = self.form_class(formdata=formdata, prefix=prefix, require_at_least_one=True, **data)
+        else:
+            self.form = self.form_class(formdata=formdata, obj=data, prefix=prefix, require_at_least_one=True)
+
+    @property
+    def errors(self):
+        """Include sub-form validation errors"""
+        form_errors = self.form.errors
+        # Add any general form errors to a special 'form' key
+        if hasattr(self.form, '_formdata_errors') and self.form._formdata_errors:
+            form_errors = dict(form_errors)  # Make a copy
+            form_errors['form'] = self.form._formdata_errors
+        return form_errors
+
+
 # Separated by  key:value
 class StringDictKeyValue(StringField):
     widget = widgets.TextArea()
 
     def _value(self):
         if self.data:
-            output = u''
-            for k in self.data.keys():
-                output += "{}: {}\r\n".format(k, self.data[k])
-
+            output = ''
+            for k, v in self.data.items():
+                output += f"{k}: {v}\r\n"
             return output
         else:
-            return u''
+            return ''
 
-    # incoming
+    # incoming data processing + validation
     def process_formdata(self, valuelist):
+        self.data = {}
+        errors = []
         if valuelist:
-            self.data = {}
-            # Remove empty strings
-            cleaned = list(filter(None, valuelist[0].split("\n")))
-            for s in cleaned:
-                parts = s.strip().split(':', 1)
-                if len(parts) == 2:
-                    self.data.update({parts[0].strip(): parts[1].strip()})
+            # Remove empty strings (blank lines)
+            cleaned = [line.strip() for line in valuelist[0].split("\n") if line.strip()]
+            for idx, s in enumerate(cleaned, start=1):
+                if ':' not in s:
+                    errors.append(f"Line {idx} is missing a ':' separator.")
+                    continue
+                parts = s.split(':', 1)
+                key = parts[0].strip()
+                value = parts[1].strip()
 
-        else:
-            self.data = {}
+                if not key:
+                    errors.append(f"Line {idx} has an empty key.")
+                if not value:
+                    errors.append(f"Line {idx} has an empty value.")
+
+                self.data[key] = value
+
+        if errors:
+            raise ValidationError("Invalid input:\n" + "\n".join(errors))
 
 class ValidateContentFetcherIsReady(object):
     """
@@ -133,31 +415,31 @@ class ValidateContentFetcherIsReady(object):
         self.message = message
 
     def __call__(self, form, field):
-        import urllib3.exceptions
-        from changedetectionio import content_fetcher
+        return
 
+# AttributeError: module 'changedetectionio.content_fetcher' has no attribute 'extra_browser_unlocked<>ASDF213r123r'
         # Better would be a radiohandler that keeps a reference to each class
-        if field.data is not None:
-            klass = getattr(content_fetcher, field.data)
-            some_object = klass()
-            try:
-                ready = some_object.is_ready()
-
-            except urllib3.exceptions.MaxRetryError as e:
-                driver_url = some_object.command_executor
-                message = field.gettext('Content fetcher \'%s\' did not respond.' % (field.data))
-                message += '<br/>' + field.gettext(
-                    'Be sure that the selenium/webdriver runner is running and accessible via network from this container/host.')
-                message += '<br/>' + field.gettext('Did you follow the instructions in the wiki?')
-                message += '<br/><br/>' + field.gettext('WebDriver Host: %s' % (driver_url))
-                message += '<br/><a href="https://github.com/dgtlmoon/changedetection.io/wiki/Fetching-pages-with-WebDriver">Go here for more information</a>'
-                message += '<br/>'+field.gettext('Content fetcher did not respond properly, unable to use it.\n %s' % (str(e)))
-
-                raise ValidationError(message)
-
-            except Exception as e:
-                message = field.gettext('Content fetcher \'%s\' did not respond properly, unable to use it.\n %s')
-                raise ValidationError(message % (field.data, e))
+        # if field.data is not None and field.data != 'system':
+        #     klass = getattr(content_fetcher, field.data)
+        #     some_object = klass()
+        #     try:
+        #         ready = some_object.is_ready()
+        #
+        #     except urllib3.exceptions.MaxRetryError as e:
+        #         driver_url = some_object.command_executor
+        #         message = field.gettext('Content fetcher \'%s\' did not respond.' % (field.data))
+        #         message += '<br>' + field.gettext(
+        #             'Be sure that the selenium/webdriver runner is running and accessible via network from this container/host.')
+        #         message += '<br>' + field.gettext('Did you follow the instructions in the wiki?')
+        #         message += '<br><br>' + field.gettext('WebDriver Host: %s' % (driver_url))
+        #         message += '<br><a href="https://github.com/dgtlmoon/changedetection.io/wiki/Fetching-pages-with-WebDriver">Go here for more information</a>'
+        #         message += '<br>'+field.gettext('Content fetcher did not respond properly, unable to use it.\n %s' % (str(e)))
+        #
+        #         raise ValidationError(message)
+        #
+        #     except Exception as e:
+        #         message = field.gettext('Content fetcher \'%s\' did not respond properly, unable to use it.\n %s')
+        #         raise ValidationError(message % (field.data, e))
 
 
 class ValidateNotificationBodyAndTitleWhenURLisSet(object):
@@ -185,30 +467,59 @@ class ValidateAppRiseServers(object):
 
     def __call__(self, form, field):
         import apprise
-        apobj = apprise.Apprise()
+        from .notification.apprise_plugin.assets import apprise_asset
+        from .notification.apprise_plugin.custom_handlers import apprise_http_custom_handler  # noqa: F401
+
+        apobj = apprise.Apprise(asset=apprise_asset)
 
         for server_url in field.data:
-            if not apobj.add(server_url):
-                message = field.gettext('\'%s\' is not a valid AppRise URL.' % (server_url))
+            url = server_url.strip()
+            if url.startswith("#"):
+                continue
+
+            if not apobj.add(url):
+                message = field.gettext('\'%s\' is not a valid AppRise URL.' % (url))
                 raise ValidationError(message)
 
-class ValidateTokensList(object):
+class ValidateJinja2Template(object):
     """
     Validates that a {token} is from a valid set
     """
-    def __init__(self, message=None):
-        self.message = message
-
     def __call__(self, form, field):
         from changedetectionio import notification
-        regex = re.compile('{.*?}')
-        for p in re.findall(regex, field.data):
-            if not p.strip('{}') in notification.valid_tokens:
-                message = field.gettext('Token \'%s\' is not a valid token.')
-                raise ValidationError(message % (p))
-            
+
+        from jinja2 import BaseLoader, TemplateSyntaxError, UndefinedError
+        from jinja2.sandbox import ImmutableSandboxedEnvironment
+        from jinja2.meta import find_undeclared_variables
+        import jinja2.exceptions
+
+        # Might be a list of text, or might be just text (like from the apprise url list)
+        joined_data = ' '.join(map(str, field.data)) if isinstance(field.data, list) else f"{field.data}"
+
+        try:
+            jinja2_env = ImmutableSandboxedEnvironment(loader=BaseLoader, extensions=['jinja2_time.TimeExtension'])
+            jinja2_env.globals.update(notification.valid_tokens)
+            # Extra validation tokens provided on the form_class(... extra_tokens={}) setup
+            if hasattr(field, 'extra_notification_tokens'):
+                jinja2_env.globals.update(field.extra_notification_tokens)
+
+            jinja2_env.from_string(joined_data).render()
+        except TemplateSyntaxError as e:
+            raise ValidationError(f"This is not a valid Jinja2 template: {e}") from e
+        except UndefinedError as e:
+            raise ValidationError(f"A variable or function is not defined: {e}") from e
+        except jinja2.exceptions.SecurityError as e:
+            raise ValidationError(f"This is not a valid Jinja2 template: {e}") from e
+
+        ast = jinja2_env.parse(joined_data)
+        undefined = ", ".join(find_undeclared_variables(ast))
+        if undefined:
+            raise ValidationError(
+                f"The following tokens used in the notification are not valid: {undefined}"
+            )
+
 class validateURL(object):
-    
+
     """
        Flask wtform validators wont work with basic auth
     """
@@ -217,12 +528,37 @@ class validateURL(object):
         self.message = message
 
     def __call__(self, form, field):
-        import validators
+        # This should raise a ValidationError() or not
+        validate_url(field.data)
+
+
+def validate_url(test_url):
+    # If hosts that only contain alphanumerics are allowed ("localhost" for example)
+    try:
+        url_validator(test_url, simple_host=allow_simplehost)
+    except validators.ValidationError:
+        #@todo check for xss
+        message = f"'{test_url}' is not a valid URL."
+        # This should be wtforms.validators.
+        raise ValidationError(message)
+
+    from .model.Watch import is_safe_url
+    if not is_safe_url(test_url):
+        # This should be wtforms.validators.
+        raise ValidationError('Watch protocol is not permitted by SAFE_PROTOCOL_REGEX or incorrect URL format')
+
+
+class ValidateSinglePythonRegexString(object):
+    def __init__(self, message=None):
+        self.message = message
+
+    def __call__(self, form, field):
         try:
-            validators.url(field.data.strip())
-        except validators.ValidationFailure:
-            message = field.gettext('\'%s\' is not a valid URL.' % (field.data.strip()))
-            raise ValidationError(message)
+            re.compile(field.data)
+        except re.error:
+            message = field.gettext('RegEx \'%s\' is not a valid regular expression.')
+            raise ValidationError(message % (field.data))
+
 
 class ValidateListRegex(object):
     """
@@ -234,15 +570,15 @@ class ValidateListRegex(object):
     def __call__(self, form, field):
 
         for line in field.data:
-            if line[0] == '/' and line[-1] == '/':
-                # Because internally we dont wrap in /
-                line = line.strip('/')
+            if re.search(html_tools.PERL_STYLE_REGEX, line, re.IGNORECASE):
                 try:
-                    re.compile(line)
+                    regex = html_tools.perl_style_slash_enclosed_regex_to_options(line)
+                    re.compile(regex)
                 except re.error:
                     message = field.gettext('RegEx \'%s\' is not a valid regular expression.')
                     raise ValidationError(message % (line))
 
+
 class ValidateCSSJSONXPATHInput(object):
     """
     Filter validation
@@ -267,11 +603,30 @@ class ValidateCSSJSONXPATHInput(object):
                 return
 
             # Does it look like XPath?
-            if line.strip()[0] == '/':
+            if line.strip()[0] == '/' or line.strip().startswith('xpath:'):
+                if not self.allow_xpath:
+                    raise ValidationError("XPath not permitted in this field!")
+                from lxml import etree, html
+                import elementpath
+                # xpath 2.0-3.1
+                from elementpath.xpath3 import XPath3Parser
+                tree = html.fromstring("<html></html>")
+                line = line.replace('xpath:', '')
+
+                try:
+                    elementpath.select(tree, line.strip(), parser=XPath3Parser)
+                except elementpath.ElementPathError as e:
+                    message = field.gettext('\'%s\' is not a valid XPath expression. (%s)')
+                    raise ValidationError(message % (line, str(e)))
+                except:
+                    raise ValidationError("A system-error occurred when validating your XPath expression")
+
+            if line.strip().startswith('xpath1:'):
                 if not self.allow_xpath:
                     raise ValidationError("XPath not permitted in this field!")
                 from lxml import etree, html
                 tree = html.fromstring("<html></html>")
+                line = re.sub(r'^xpath1:', '', line)
 
                 try:
                     tree.xpath(line.strip())
@@ -303,59 +658,143 @@ class ValidateCSSJSONXPATHInput(object):
 
                 # Re #265 - maybe in the future fetch the page and offer a
                 # warning/notice that its possible the rule doesnt yet match anything?
+                if not self.allow_json:
+                    raise ValidationError("jq not permitted in this field!")
 
+            if 'jq:' in line:
+                try:
+                    import jq
+                except ModuleNotFoundError:
+                    # `jq` requires full compilation in windows and so isn't generally available
+                    raise ValidationError("jq not support not found")
+
+                input = line.replace('jq:', '')
+
+                try:
+                    jq.compile(input)
+                except (ValueError) as e:
+                    message = field.gettext('\'%s\' is not a valid jq expression. (%s)')
+                    raise ValidationError(message % (input, str(e)))
+                except:
+                    raise ValidationError("A system-error occurred when validating your jq expression")
 
 class quickWatchForm(Form):
+    from . import processors
+
     url = fields.URLField('URL', validators=[validateURL()])
-    tag = StringField('Group tag', [validators.Optional()])
+    tags = StringTagUUID('Group tag', [validators.Optional()])
+    watch_submit_button = SubmitField('Watch', render_kw={"class": "pure-button pure-button-primary"})
+    processor = RadioField(u'Processor', choices=processors.available_processors(), default="text_json_diff")
+    edit_and_watch_submit_button = SubmitField('Edit > Watch', render_kw={"class": "pure-button pure-button-primary"})
+
+
 
 # Common to a single watch and the global settings
 class commonSettingsForm(Form):
+    from . import processors
 
-    notification_urls = StringListField('Notification URL list', validators=[validators.Optional(), ValidateNotificationBodyAndTitleWhenURLisSet(), ValidateAppRiseServers()])
-    notification_title = StringField('Notification title', default=default_notification_title, validators=[validators.Optional(), ValidateTokensList()])
-    notification_body = TextAreaField('Notification body', default=default_notification_body, validators=[validators.Optional(), ValidateTokensList()])
-    notification_format = SelectField('Notification format', choices=valid_notification_formats.keys(), default=default_notification_format)
-    fetch_backend = RadioField(u'Fetch method', choices=content_fetcher.available_fetchers(), validators=[ValidateContentFetcherIsReady()])
-    extract_title_as_title = BooleanField('Extract <title> from document and use as watch title', default=False)
-    webdriver_delay = IntegerField('Wait seconds before extracting text', validators=[validators.Optional(), validators.NumberRange(min=1, message="Should contain one or more seconds")] )
+    def __init__(self, formdata=None, obj=None, prefix="", data=None, meta=None, **kwargs):
+        super().__init__(formdata, obj, prefix, data, meta, **kwargs)
+        self.notification_body.extra_notification_tokens = kwargs.get('extra_notification_tokens', {})
+        self.notification_title.extra_notification_tokens = kwargs.get('extra_notification_tokens', {})
+        self.notification_urls.extra_notification_tokens = kwargs.get('extra_notification_tokens', {})
 
-class watchForm(commonSettingsForm):
+    fetch_backend = RadioField(u'Fetch Method', choices=content_fetchers.available_fetchers(), validators=[ValidateContentFetcherIsReady()])
+    notification_body = TextAreaField('Notification Body', default='{{ watch_url }} had a change.', validators=[validators.Optional(), ValidateJinja2Template()])
+    notification_format = SelectField('Notification format', choices=valid_notification_formats.keys())
+    notification_title = StringField('Notification Title', default='ChangeDetection.io Notification - {{ watch_url }}', validators=[validators.Optional(), ValidateJinja2Template()])
+    notification_urls = StringListField('Notification URL List', validators=[validators.Optional(), ValidateAppRiseServers(), ValidateJinja2Template()])
+    processor = RadioField( label=u"Processor - What do you want to achieve?", choices=processors.available_processors(), default="text_json_diff")
+    timezone = StringField("Timezone for watch schedule", render_kw={"list": "timezones"}, validators=[validateTimeZoneName()])
+    webdriver_delay = IntegerField('Wait seconds before extracting text', validators=[validators.Optional(), validators.NumberRange(min=1, message="Should contain one or more seconds")])
+
+
+class importForm(Form):
+    from . import processors
+    processor = RadioField(u'Processor', choices=processors.available_processors(), default="text_json_diff")
+    urls = TextAreaField('URLs')
+    xlsx_file = FileField('Upload .xlsx file', validators=[FileAllowed(['xlsx'], 'Must be .xlsx file!')])
+    file_mapping = SelectField('File mapping', [validators.DataRequired()], choices={('wachete', 'Wachete mapping'), ('custom','Custom mapping')})
+
+class SingleBrowserStep(Form):
+
+    operation = SelectField('Operation', [validators.Optional()], choices=browser_step_ui_config.keys())
+
+    # maybe better to set some <script>var..
+    selector = StringField('Selector', [validators.Optional()], render_kw={"placeholder": "CSS or xPath selector"})
+    optional_value = StringField('value', [validators.Optional()], render_kw={"placeholder": "Value"})
+#   @todo move to JS? ajax fetch new field?
+#    remove_button = SubmitField('-', render_kw={"type": "button", "class": "pure-button pure-button-primary", 'title': 'Remove'})
+#    add_button = SubmitField('+', render_kw={"type": "button", "class": "pure-button pure-button-primary", 'title': 'Add new step after'})
+
+class processor_text_json_diff_form(commonSettingsForm):
 
     url = fields.URLField('URL', validators=[validateURL()])
-    tag = StringField('Group tag', [validators.Optional()], default='')
+    tags = StringTagUUID('Group tag', [validators.Optional()], default='')
 
-    time_between_check = FormField(TimeBetweenCheckForm)
+    time_between_check = EnhancedFormField(
+        TimeBetweenCheckForm,
+        conditional_field='time_between_check_use_default',
+        conditional_message=REQUIRE_ATLEAST_ONE_TIME_PART_WHEN_NOT_GLOBAL_DEFAULT,
+        conditional_test_function=validate_time_between_check_has_values
+    )
 
-    css_filter = StringField('CSS/JSON/XPATH Filter', [ValidateCSSJSONXPATHInput()], default='')
+    time_schedule_limit = FormField(ScheduleLimitForm)
 
-    subtractive_selectors = StringListField('Remove elements', [ValidateCSSJSONXPATHInput(allow_xpath=False, allow_json=False)])
+    time_between_check_use_default = BooleanField('Use global settings for time between check and scheduler.', default=False)
+
+    include_filters = StringListField('CSS/JSONPath/JQ/XPath Filters', [ValidateCSSJSONXPATHInput()], default='')
+
+    subtractive_selectors = StringListField('Remove elements', [ValidateCSSJSONXPATHInput(allow_json=False)])
 
     extract_text = StringListField('Extract text', [ValidateListRegex()])
 
     title = StringField('Title', default='')
 
-    ignore_text = StringListField('Ignore text', [ValidateListRegex()])
+    ignore_text = StringListField('Ignore lines containing', [ValidateListRegex()])
     headers = StringDictKeyValue('Request headers')
     body = TextAreaField('Request body', [validators.Optional()])
     method = SelectField('Request method', choices=valid_method, default=default_method)
     ignore_status_codes = BooleanField('Ignore status codes (process non-2xx status codes as normal)', default=False)
-    check_unique_lines = BooleanField('Only trigger when new lines appear', default=False)
-    trigger_text = StringListField('Trigger/wait for text', [validators.Optional(), ValidateListRegex()])
-    text_should_not_be_present = StringListField('Block change-detection if text matches', [validators.Optional(), ValidateListRegex()])
+    check_unique_lines = BooleanField('Only trigger when unique lines appear in all history', default=False)
+    remove_duplicate_lines = BooleanField('Remove duplicate lines of text', default=False)
+    sort_text_alphabetically =  BooleanField('Sort text alphabetically', default=False)
+    trim_text_whitespace = BooleanField('Trim whitespace before and after text', default=False)
 
+    filter_text_added = BooleanField('Added lines', default=True)
+    filter_text_replaced = BooleanField('Replaced/changed lines', default=True)
+    filter_text_removed = BooleanField('Removed lines', default=True)
+
+    trigger_text = StringListField('Keyword triggers - Trigger/wait for text', [validators.Optional(), ValidateListRegex()])
+    if os.getenv("PLAYWRIGHT_DRIVER_URL"):
+        browser_steps = FieldList(FormField(SingleBrowserStep), min_entries=10)
+    text_should_not_be_present = StringListField('Block change-detection while text matches', [validators.Optional(), ValidateListRegex()])
     webdriver_js_execute_code = TextAreaField('Execute JavaScript before change detection', render_kw={"rows": "5"}, validators=[validators.Optional()])
 
     save_button = SubmitField('Save', render_kw={"class": "pure-button pure-button-primary"})
-    save_and_preview_button = SubmitField('Save & Preview', render_kw={"class": "pure-button pure-button-primary"})
+
     proxy = RadioField('Proxy')
+    # filter_failure_notification_send @todo make ternary
     filter_failure_notification_send = BooleanField(
         'Send a notification when the filter can no longer be found on the page', default=False)
+    notification_muted = TernaryNoneBooleanField('Notifications', default=None, yes_text="Muted", no_text="On")
+    notification_screenshot = BooleanField('Attach screenshot to notification (where possible)', default=False)
+
+    conditions_match_logic = RadioField(u'Match', choices=[('ALL', 'Match all of the following'),('ANY', 'Match any of the following')], default='ALL')
+    conditions = FieldList(FormField(ConditionFormRow), min_entries=1)  # Add rule logic here
+    use_page_title_in_list = TernaryNoneBooleanField('Use page <title> in list', default=None)
+
+    def extra_tab_content(self):
+        return None
+
+    def extra_form_content(self):
+        return None
 
     def validate(self, **kwargs):
         if not super().validate():
             return False
 
+        from changedetectionio.safe_jinja import render as jinja_render
         result = True
 
         # Fail form validation when a body is set for a GET
@@ -363,43 +802,164 @@ class watchForm(commonSettingsForm):
             self.body.errors.append('Body must be empty when Request Method is set to GET')
             result = False
 
+        # Attempt to validate jinja2 templates in the URL
+        try:
+            jinja_render(template_str=self.url.data)
+        except ModuleNotFoundError as e:
+            # incase jinja2_time or others is missing
+            logger.error(e)
+            self.url.errors.append(f'Invalid template syntax configuration: {e}')
+            result = False
+        except Exception as e:
+            logger.error(e)
+            self.url.errors.append(f'Invalid template syntax: {e}')
+            result = False
+
+        # Attempt to validate jinja2 templates in the body
+        if self.body.data and self.body.data.strip():
+            try:
+                jinja_render(template_str=self.body.data)
+            except ModuleNotFoundError as e:
+                # incase jinja2_time or others is missing
+                logger.error(e)
+                self.body.errors.append(f'Invalid template syntax configuration: {e}')
+                result = False
+            except Exception as e:
+                logger.error(e)
+                self.body.errors.append(f'Invalid template syntax: {e}')
+                result = False
+
+        # Attempt to validate jinja2 templates in the headers
+        if len(self.headers.data) > 0:
+            try:
+                for header, value in self.headers.data.items():
+                    jinja_render(template_str=value)
+            except ModuleNotFoundError as e:
+                # incase jinja2_time or others is missing
+                logger.error(e)
+                self.headers.errors.append(f'Invalid template syntax configuration: {e}')
+                result = False
+            except Exception as e:
+                logger.error(e)
+                self.headers.errors.append(f'Invalid template syntax in "{header}" header: {e}')
+                result = False
+
         return result
 
+    def __init__(
+            self,
+            formdata=None,
+            obj=None,
+            prefix="",
+            data=None,
+            meta=None,
+            **kwargs,
+    ):
+        super().__init__(formdata, obj, prefix, data, meta, **kwargs)
+        if kwargs and kwargs.get('default_system_settings'):
+            default_tz = kwargs.get('default_system_settings').get('application', {}).get('timezone')
+            if default_tz:
+                self.time_schedule_limit.form.timezone.render_kw['placeholder'] = default_tz
+
+
+
+class SingleExtraProxy(Form):
+
+    # maybe better to set some <script>var..
+    proxy_name = StringField('Name', [validators.Optional()], render_kw={"placeholder": "Name"})
+    proxy_url = StringField('Proxy URL', [validators.Optional()], render_kw={"placeholder": "socks5:// or regular proxy http://user:pass@...:3128", "size":50})
+    # @todo do the validation here instead
+
+class SingleExtraBrowser(Form):
+    browser_name = StringField('Name', [validators.Optional()], render_kw={"placeholder": "Name"})
+    browser_connection_url = StringField('Browser connection URL', [validators.Optional()], render_kw={"placeholder": "wss://brightdata... wss://oxylabs etc", "size":50})
+    # @todo do the validation here instead
+
+class DefaultUAInputForm(Form):
+    html_requests = StringField('Plaintext requests', validators=[validators.Optional()], render_kw={"placeholder": "<default>"})
+    if os.getenv("PLAYWRIGHT_DRIVER_URL") or os.getenv("WEBDRIVER_URL"):
+        html_webdriver = StringField('Chrome requests', validators=[validators.Optional()], render_kw={"placeholder": "<default>"})
 
 # datastore.data['settings']['requests']..
 class globalSettingsRequestForm(Form):
-    time_between_check = FormField(TimeBetweenCheckForm)
+    time_between_check = RequiredFormField(TimeBetweenCheckForm)
+    time_schedule_limit = FormField(ScheduleLimitForm)
     proxy = RadioField('Proxy')
     jitter_seconds = IntegerField('Random jitter seconds ± check',
                                   render_kw={"style": "width: 5em;"},
                                   validators=[validators.NumberRange(min=0, message="Should contain zero or more seconds")])
+    
+    workers = IntegerField('Number of fetch workers',
+                          render_kw={"style": "width: 5em;"},
+                          validators=[validators.NumberRange(min=1, max=50,
+                                                             message="Should be between 1 and 50")])
+    
+    extra_proxies = FieldList(FormField(SingleExtraProxy), min_entries=5)
+    extra_browsers = FieldList(FormField(SingleExtraBrowser), min_entries=5)
+
+    default_ua = FormField(DefaultUAInputForm, label="Default User-Agent overrides")
+
+    def validate_extra_proxies(self, extra_validators=None):
+        for e in self.data['extra_proxies']:
+            if e.get('proxy_name') or e.get('proxy_url'):
+                if not e.get('proxy_name','').strip() or not e.get('proxy_url','').strip():
+                    self.extra_proxies.errors.append('Both a name, and a Proxy URL is required.')
+                    return False
+
+class globalSettingsApplicationUIForm(Form):
+    open_diff_in_new_tab = BooleanField("Open 'History' page in a new tab", default=True, validators=[validators.Optional()])
+    socket_io_enabled = BooleanField('Realtime UI Updates Enabled', default=True, validators=[validators.Optional()])
+    favicons_enabled = BooleanField('Favicons Enabled', default=True, validators=[validators.Optional()])
+    use_page_title_in_list = BooleanField('Use page <title> in watch overview list') #BooleanField=True
 
 # datastore.data['settings']['application']..
 class globalSettingsApplicationForm(commonSettingsForm):
 
-    base_url = StringField('Base URL', validators=[validators.Optional()])
-    global_subtractive_selectors = StringListField('Remove elements', [ValidateCSSJSONXPATHInput(allow_xpath=False, allow_json=False)])
-    global_ignore_text = StringListField('Ignore Text', [ValidateListRegex()])
-    ignore_whitespace = BooleanField('Ignore whitespace')
-    real_browser_save_screenshot = BooleanField('Save last screenshot when using Chrome?')
-    removepassword_button = SubmitField('Remove password', render_kw={"class": "pure-button pure-button-primary"})
-    empty_pages_are_a_change =  BooleanField('Treat empty pages as a change?', default=False)
-    render_anchor_tag_content = BooleanField('Render anchor tag content', default=False)
-    fetch_backend = RadioField('Fetch Method', default="html_requests", choices=content_fetcher.available_fetchers(), validators=[ValidateContentFetcherIsReady()])
     api_access_token_enabled = BooleanField('API access token security check enabled', default=True, validators=[validators.Optional()])
+    base_url = StringField('Notification base URL override',
+                           validators=[validators.Optional()],
+                           render_kw={"placeholder": os.getenv('BASE_URL', 'Not set')}
+                           )
+    empty_pages_are_a_change =  BooleanField('Treat empty pages as a change?', default=False)
+    fetch_backend = RadioField('Fetch Method', default="html_requests", choices=content_fetchers.available_fetchers(), validators=[ValidateContentFetcherIsReady()])
+    global_ignore_text = StringListField('Ignore Text', [ValidateListRegex()])
+    global_subtractive_selectors = StringListField('Remove elements', [ValidateCSSJSONXPATHInput(allow_json=False)])
+    ignore_whitespace = BooleanField('Ignore whitespace')
     password = SaltyPasswordField()
+    pager_size = IntegerField('Pager size',
+                              render_kw={"style": "width: 5em;"},
+                              validators=[validators.NumberRange(min=0,
+                                                                 message="Should be atleast zero (disabled)")])
 
+    rss_content_format = SelectField('RSS Content format', choices=RSS_FORMAT_TYPES)
+
+    removepassword_button = SubmitField('Remove password', render_kw={"class": "pure-button pure-button-primary"})
+    render_anchor_tag_content = BooleanField('Render anchor tag content', default=False)
+    shared_diff_access = BooleanField('Allow anonymous access to watch history page when password is enabled', default=False, validators=[validators.Optional()])
+    rss_hide_muted_watches = BooleanField('Hide muted watches from RSS feed', default=True,
+                                      validators=[validators.Optional()])
     filter_failure_notification_threshold_attempts = IntegerField('Number of times the filter can be missing before sending a notification',
                                                                   render_kw={"style": "width: 5em;"},
                                                                   validators=[validators.NumberRange(min=0,
                                                                                                      message="Should contain zero or more attempts")])
+    ui = FormField(globalSettingsApplicationUIForm)
 
 
 class globalSettingsForm(Form):
     # Define these as FormFields/"sub forms", this way it matches the JSON storage
     # datastore.data['settings']['application']..
     # datastore.data['settings']['requests']..
+    def __init__(self, formdata=None, obj=None, prefix="", data=None, meta=None, **kwargs):
+        super().__init__(formdata, obj, prefix, data, meta, **kwargs)
+        self.application.notification_body.extra_notification_tokens = kwargs.get('extra_notification_tokens', {})
+        self.application.notification_title.extra_notification_tokens = kwargs.get('extra_notification_tokens', {})
+        self.application.notification_urls.extra_notification_tokens = kwargs.get('extra_notification_tokens', {})
 
     requests = FormField(globalSettingsRequestForm)
     application = FormField(globalSettingsApplicationForm)
     save_button = SubmitField('Save', render_kw={"class": "pure-button pure-button-primary"})
+
+
+class extractDataForm(Form):
+    extract_regex = StringField('RegEx to extract', validators=[validators.DataRequired(), ValidateSinglePythonRegexString()])
+    extract_submit_button = SubmitField('Extract as CSV', render_kw={"class": "pure-button pure-button-primary"})

changedetectionio/gc_cleanup.py

@@ -0,0 +1,162 @@
+#!/usr/bin/env python3
+
+import ctypes
+import gc
+import re
+import psutil
+import sys
+import threading
+import importlib
+from loguru import logger
+
+def memory_cleanup(app=None):
+    """
+    Perform comprehensive memory cleanup operations and log memory usage
+    at each step with nicely formatted numbers.
+    
+    Args:
+        app: Optional Flask app instance for clearing Flask-specific caches
+        
+    Returns:
+        str: Status message
+    """
+    # Get current process
+    process = psutil.Process()
+    
+    # Log initial memory usage with nicely formatted numbers
+    current_memory = process.memory_info().rss / 1024 / 1024
+    logger.debug(f"Memory cleanup started - Current memory usage: {current_memory:,.2f} MB")
+
+    # 1. Standard garbage collection - force full collection on all generations
+    gc.collect(0)  # Collect youngest generation
+    gc.collect(1)  # Collect middle generation
+    gc.collect(2)  # Collect oldest generation
+
+    # Run full collection again to ensure maximum cleanup
+    gc.collect()
+    current_memory = process.memory_info().rss / 1024 / 1024
+    logger.debug(f"After full gc.collect() - Memory usage: {current_memory:,.2f} MB")
+    
+
+    # 3. Call libc's malloc_trim to release memory back to the OS
+    libc = ctypes.CDLL("libc.so.6")
+    libc.malloc_trim(0)
+    current_memory = process.memory_info().rss / 1024 / 1024
+    logger.debug(f"After malloc_trim(0) - Memory usage: {current_memory:,.2f} MB")
+    
+    # 4. Clear Python's regex cache
+    re.purge()
+    current_memory = process.memory_info().rss / 1024 / 1024
+    logger.debug(f"After re.purge() - Memory usage: {current_memory:,.2f} MB")
+
+    # 5. Reset thread-local storage
+    # Create a new thread local object to encourage cleanup of old ones
+    threading.local()
+    current_memory = process.memory_info().rss / 1024 / 1024
+    logger.debug(f"After threading.local() - Memory usage: {current_memory:,.2f} MB")
+
+    # 6. Clear sys.intern cache if Python version supports it
+    try:
+        sys.intern.clear()
+        current_memory = process.memory_info().rss / 1024 / 1024
+        logger.debug(f"After sys.intern.clear() - Memory usage: {current_memory:,.2f} MB")
+    except (AttributeError, TypeError):
+        logger.debug("sys.intern.clear() not supported in this Python version")
+    
+    # 7. Clear XML/lxml caches if available
+    try:
+        # Check if lxml.etree is in use
+        lxml_etree = sys.modules.get('lxml.etree')
+        if lxml_etree:
+            # Clear module-level caches
+            if hasattr(lxml_etree, 'clear_error_log'):
+                lxml_etree.clear_error_log()
+            
+            # Check for _ErrorLog and _RotatingErrorLog objects and clear them
+            for obj in gc.get_objects():
+                if hasattr(obj, '__class__') and hasattr(obj.__class__, '__name__'):
+                    class_name = obj.__class__.__name__
+                    if class_name in ('_ErrorLog', '_RotatingErrorLog', '_DomainErrorLog') and hasattr(obj, 'clear'):
+                        try:
+                            obj.clear()
+                        except (AttributeError, TypeError):
+                            pass
+                    
+                    # Clear Element objects which can hold references to documents
+                    elif class_name in ('_Element', 'ElementBase') and hasattr(obj, 'clear'):
+                        try:
+                            obj.clear()
+                        except (AttributeError, TypeError):
+                            pass
+            
+            current_memory = process.memory_info().rss / 1024 / 1024
+            logger.debug(f"After lxml.etree cleanup - Memory usage: {current_memory:,.2f} MB")
+
+        # Check if lxml.html is in use
+        lxml_html = sys.modules.get('lxml.html')
+        if lxml_html:
+            # Clear HTML-specific element types
+            for obj in gc.get_objects():
+                if hasattr(obj, '__class__') and hasattr(obj.__class__, '__name__'):
+                    class_name = obj.__class__.__name__
+                    if class_name in ('HtmlElement', 'FormElement', 'InputElement',
+                                    'SelectElement', 'TextareaElement', 'CheckboxGroup',
+                                    'RadioGroup', 'MultipleSelectOptions', 'FieldsDict') and hasattr(obj, 'clear'):
+                        try:
+                            obj.clear()
+                        except (AttributeError, TypeError):
+                            pass
+
+            current_memory = process.memory_info().rss / 1024 / 1024
+            logger.debug(f"After lxml.html cleanup - Memory usage: {current_memory:,.2f} MB")
+    except (ImportError, AttributeError):
+        logger.debug("lxml cleanup not applicable")
+    
+    # 8. Clear JSON parser caches if applicable
+    try:
+        # Check if json module is being used and try to clear its cache
+        json_module = sys.modules.get('json')
+        if json_module and hasattr(json_module, '_default_encoder'):
+            json_module._default_encoder.markers.clear()
+            current_memory = process.memory_info().rss / 1024 / 1024
+            logger.debug(f"After JSON parser cleanup - Memory usage: {current_memory:,.2f} MB")
+    except (AttributeError, KeyError):
+        logger.debug("JSON cleanup not applicable")
+    
+    # 9. Force Python's memory allocator to release unused memory
+    try:
+        if hasattr(sys, 'pypy_version_info'):
+            # PyPy has different memory management
+            gc.collect()
+        else:
+            # CPython - try to release unused memory
+            ctypes.pythonapi.PyGC_Collect()
+            current_memory = process.memory_info().rss / 1024 / 1024
+            logger.debug(f"After PyGC_Collect - Memory usage: {current_memory:,.2f} MB")
+    except (AttributeError, TypeError):
+        logger.debug("PyGC_Collect not supported")
+    
+    # 10. Clear Flask-specific caches if applicable
+    if app:
+        try:
+            # Clear Flask caches if they exist
+            for key in list(app.config.get('_cache', {}).keys()):
+                app.config['_cache'].pop(key, None)
+            
+            # Clear Jinja2 template cache if available
+            if hasattr(app, 'jinja_env') and hasattr(app.jinja_env, 'cache'):
+                app.jinja_env.cache.clear()
+            
+            current_memory = process.memory_info().rss / 1024 / 1024
+            logger.debug(f"After Flask cache clear - Memory usage: {current_memory:,.2f} MB")
+        except (AttributeError, KeyError):
+            logger.debug("No Flask cache to clear")
+    
+    # Final garbage collection pass
+    gc.collect()
+    libc.malloc_trim(0)
+    
+    # Log final memory usage
+    final_memory = process.memory_info().rss / 1024 / 1024
+    logger.info(f"Memory cleanup completed - Final memory usage: {final_memory:,.2f} MB")
+    return "cleaned"