Merge branch 'master' into history-preview-ignore-text-highlighting

Co-authored-by: silversub <silversub@gmail.com>

.dockerignore

@@ -1,18 +1,62 @@
-.git
-.github
-changedetectionio/processors/__pycache__
-changedetectionio/api/__pycache__
-changedetectionio/model/__pycache__
-changedetectionio/blueprint/price_data_follower/__pycache__
-changedetectionio/blueprint/tags/__pycache__
-changedetectionio/blueprint/__pycache__
-changedetectionio/blueprint/browser_steps/__pycache__
-changedetectionio/fetchers/__pycache__
-changedetectionio/tests/visualselector/__pycache__
-changedetectionio/tests/restock/__pycache__
-changedetectionio/tests/__pycache__
-changedetectionio/tests/fetchers/__pycache__
-changedetectionio/tests/unit/__pycache__
-changedetectionio/tests/proxy_list/__pycache__
-changedetectionio/__pycache__
+# Git
+.git/
+.gitignore
 
+# GitHub
+.github/
+
+# Byte-compiled / optimized / DLL files
+**/__pycache__
+**/*.py[cod]
+
+# Caches
+.mypy_cache/
+.pytest_cache/
+.ruff_cache/
+
+# Distribution / packaging
+build/
+dist/
+*.egg-info*
+
+# Virtual environment
+.env
+.venv/
+venv/
+
+# IntelliJ IDEA
+.idea/
+
+# Visual Studio
+.vscode/
+
+# Test and development files
+test-datastore/
+tests/
+*.md
+!README.md
+
+# Temporary and log files
+*.log
+*.tmp
+tmp/
+temp/
+
+# Training data and large files
+train-data/
+works-data/
+
+# Container files
+Dockerfile*
+docker-compose*.yml
+.dockerignore
+
+# Development certificates and keys
+*.pem
+*.key
+*.crt
+profile_output.prof
+
+# Large binary files that shouldn't be in container
+*.pdf
+chrome.json

.github/ISSUE_TEMPLATE/bug_report.md

@@ -27,6 +27,10 @@ A clear and concise description of what the bug is.
 **Version**
 *Exact version* in the top right area: 0....
 
+**How did you install?**
+
+Docker, Pip, from source directly etc
+
 **To Reproduce**
 
 Steps to reproduce the behavior:

.github/actions/extract-memory-report/action.yml

@@ -0,0 +1,51 @@
+name: 'Extract Memory Test Report'
+description: 'Extracts and displays memory test report from a container'
+inputs:
+  container-name:
+    description: 'Name of the container to extract logs from'
+    required: true
+  python-version:
+    description: 'Python version for artifact naming'
+    required: true
+  output-dir:
+    description: 'Directory to store output logs'
+    required: false
+    default: 'output-logs'
+
+runs:
+  using: "composite"
+  steps:
+    - name: Create output directory
+      shell: bash
+      run: |
+        mkdir -p ${{ inputs.output-dir }}
+
+    - name: Dump container log
+      shell: bash
+      run: |
+        echo "Disabled for now"
+#        return
+#        docker logs ${{ inputs.container-name }} > ${{ inputs.output-dir }}/${{ inputs.container-name }}-stdout-${{ inputs.python-version }}.txt 2>&1 || echo "Could not get stdout"
+#        docker logs ${{ inputs.container-name }} 2> ${{ inputs.output-dir }}/${{ inputs.container-name }}-stderr-${{ inputs.python-version }}.txt || echo "Could not get stderr"
+
+    - name: Extract and display memory test report
+      shell: bash
+      run: |
+        echo "Disabled for now"
+#        echo "Extracting test-memory.log from container..."
+#        docker cp ${{ inputs.container-name }}:/app/changedetectionio/test-memory.log ${{ inputs.output-dir }}/test-memory-${{ inputs.python-version }}.log || echo "test-memory.log not found in container"
+#
+#        echo "=== Top 10 Highest Peak Memory Tests ==="
+#        if [ -f ${{ inputs.output-dir }}/test-memory-${{ inputs.python-version }}.log ]; then
+#          grep "Peak memory:" ${{ inputs.output-dir }}/test-memory-${{ inputs.python-version }}.log | \
+#            sed 's/.*Peak memory: //' | \
+#            paste -d'|' - <(grep "Peak memory:" ${{ inputs.output-dir }}/test-memory-${{ inputs.python-version }}.log) | \
+#            sort -t'|' -k1 -nr | \
+#            cut -d'|' -f2 | \
+#            head -10
+#          echo ""
+#          echo "=== Full Memory Test Report ==="
+#          cat ${{ inputs.output-dir }}/test-memory-${{ inputs.python-version }}.log
+#        else
+#          echo "No memory log available"
+#        fi

.github/dependabot.yml

@@ -0,0 +1,14 @@
+version: 2
+updates:
+  - package-ecosystem: github-actions
+    directory: /
+    schedule:
+      interval: "weekly"
+    groups:
+      all:
+        patterns:
+        - "*"
+  - package-ecosystem: pip
+    directory: /
+    schedule:
+      interval: "weekly"

.github/test/Dockerfile-alpine

@@ -2,30 +2,35 @@
 # Test that we can still build on Alpine (musl modified libc https://musl.libc.org/)
 # Some packages wont install via pypi because they dont have a wheel available under this architecture.
 
-FROM ghcr.io/linuxserver/baseimage-alpine:3.18
+FROM ghcr.io/linuxserver/baseimage-alpine:3.22
 ENV PYTHONUNBUFFERED=1
 
 COPY requirements.txt /requirements.txt
 
 RUN \
-  apk add --update --no-cache --virtual=build-dependencies \
+ apk add --update --no-cache --virtual=build-dependencies \
+    build-base \
     cargo \
-    g++ \
-    gcc \
+    git \
+    jpeg-dev \
     libc-dev \
     libffi-dev \
     libxslt-dev \
-    make \
     openssl-dev \
-    py3-wheel \
     python3-dev \
+    file \
+    zip \
     zlib-dev && \
   apk add --update --no-cache \
+    libjpeg \
     libxslt \
-    python3 \
-    py3-pip && \
+    file \
+    nodejs \
+    poppler-utils \
+    python3 && \
   echo "**** pip3 install test of changedetection.io ****" && \
-  pip3 install -U pip wheel setuptools && \
-  pip3 install -U --no-cache-dir --find-links https://wheel-index.linuxserver.io/alpine-3.18/ -r /requirements.txt && \
+  python3 -m venv /lsiopy  && \
+  pip install -U pip wheel setuptools && \
+  pip install -U --no-cache-dir --find-links https://wheel-index.linuxserver.io/alpine-3.22/ -r /requirements.txt && \
   apk del --purge \
     build-dependencies

.github/workflows/codeql-analysis.yml

@@ -30,11 +30,11 @@ jobs:
 
     steps:
     - name: Checkout repository
-      uses: actions/checkout@v4
+      uses: actions/checkout@v5
 
     # Initializes the CodeQL tools for scanning.
     - name: Initialize CodeQL
-      uses: github/codeql-action/init@v2
+      uses: github/codeql-action/init@v4
       with:
         languages: ${{ matrix.language }}
         # If you wish to specify custom queries, you can do so here or in a config file.
@@ -45,7 +45,7 @@ jobs:
     # Autobuild attempts to build any compiled languages  (C/C++, C#, or Java).
     # If this step fails, then you should remove it and run the build manually (see below)
     - name: Autobuild
-      uses: github/codeql-action/autobuild@v2
+      uses: github/codeql-action/autobuild@v4
 
     # ℹ️ Command-line programs to run using the OS shell.
     # 📚 https://git.io/JvXDl
@@ -59,4 +59,4 @@ jobs:
     #   make release
 
     - name: Perform CodeQL Analysis
-      uses: github/codeql-action/analyze@v2
+      uses: github/codeql-action/analyze@v4

.github/workflows/containers.yml

@@ -39,12 +39,20 @@ jobs:
     # Or if we are in a tagged release scenario.
     if: ${{ github.event.workflow_run.conclusion == 'success' }} || ${{ github.event.release.tag_name }} != ''
     steps:
-      - uses: actions/checkout@v4
+      - uses: actions/checkout@v5
       - name: Set up Python 3.11
-        uses: actions/setup-python@v4
+        uses: actions/setup-python@v6
         with:
           python-version: 3.11
 
+      - name: Cache pip packages
+        uses: actions/cache@v4
+        with:
+          path: ~/.cache/pip
+          key: ${{ runner.os }}-pip-${{ hashFiles('requirements.txt') }}
+          restore-keys: |
+            ${{ runner.os }}-pip-
+
       - name: Install dependencies
         run: |
           python -m pip install --upgrade pip
@@ -88,46 +96,57 @@ jobs:
       - name: Build and push :dev
         id: docker_build
         if: ${{ github.ref }} == "refs/heads/master"
-        uses: docker/build-push-action@v5
+        uses: docker/build-push-action@v6
         with:
           context: ./
           file: ./Dockerfile
           push: true
           tags: |
             ${{ secrets.DOCKER_HUB_USERNAME }}/changedetection.io:dev,ghcr.io/${{ github.repository }}:dev
-          platforms: linux/amd64,linux/arm64,linux/arm/v6,linux/arm/v7,linux/arm/v8
-          cache-from: type=local,src=/tmp/.buildx-cache
-          cache-to: type=local,dest=/tmp/.buildx-cache
+          platforms: linux/amd64,linux/arm64,linux/arm/v7,linux/arm/v8
+          cache-from: type=gha
+          cache-to: type=gha,mode=max
+
 # Looks like this was disabled
 #          provenance: false
 
       # A new tagged release is required, which builds :tag and :latest
+      - name: Debug release info
+        if: github.event_name == 'release' && startsWith(github.event.release.tag_name, '0.')
+        run: |
+          echo "Release tag: ${{ github.event.release.tag_name }}"
+          echo "Github ref: ${{ github.ref }}"
+          echo "Github ref name: ${{ github.ref_name }}"
+          
+      - name: Docker meta :tag
+        if: github.event_name == 'release' && startsWith(github.event.release.tag_name, '0.')
+        uses: docker/metadata-action@v5
+        id: meta
+        with:
+            images: |
+                ${{ secrets.DOCKER_HUB_USERNAME }}/changedetection.io
+                ghcr.io/dgtlmoon/changedetection.io
+            tags: |
+                type=semver,pattern={{version}},value=${{ github.event.release.tag_name }}
+                type=semver,pattern={{major}}.{{minor}},value=${{ github.event.release.tag_name }}
+                type=semver,pattern={{major}},value=${{ github.event.release.tag_name }}
+                type=raw,value=latest
+
       - name: Build and push :tag
         id: docker_build_tag_release
         if: github.event_name == 'release' && startsWith(github.event.release.tag_name, '0.')
-        uses: docker/build-push-action@v5
+        uses: docker/build-push-action@v6
         with:
           context: ./
           file: ./Dockerfile
           push: true
-          tags: |
-            ${{ secrets.DOCKER_HUB_USERNAME }}/changedetection.io:${{ github.event.release.tag_name }}
-            ghcr.io/dgtlmoon/changedetection.io:${{ github.event.release.tag_name }}
-            ${{ secrets.DOCKER_HUB_USERNAME }}/changedetection.io:latest
-            ghcr.io/dgtlmoon/changedetection.io:latest
-          platforms: linux/amd64,linux/arm64,linux/arm/v6,linux/arm/v7,linux/arm/v8
-          cache-from: type=local,src=/tmp/.buildx-cache
-          cache-to: type=local,dest=/tmp/.buildx-cache
+          tags: ${{ steps.meta.outputs.tags }}
+          platforms: linux/amd64,linux/arm64,linux/arm/v7,linux/arm/v8
+          cache-from: type=gha
+          cache-to: type=gha,mode=max
 # Looks like this was disabled
 #          provenance: false
 
       - name: Image digest
         run: echo step SHA ${{ steps.vars.outputs.sha_short }} tag ${{steps.vars.outputs.tag}} branch ${{steps.vars.outputs.branch}} digest ${{ steps.docker_build.outputs.digest }}
 
-      - name: Cache Docker layers
-        uses: actions/cache@v3
-        with:
-          path: /tmp/.buildx-cache
-          key: ${{ runner.os }}-buildx-${{ github.sha }}
-          restore-keys: |
-            ${{ runner.os }}-buildx-

.github/workflows/pypi-release.yml

@@ -0,0 +1,101 @@
+name: Publish Python 🐍distribution 📦 to PyPI and TestPyPI
+
+on: push
+jobs:
+  build:
+    name: Build distribution 📦
+    runs-on: ubuntu-latest
+
+    steps:
+    - uses: actions/checkout@v5
+    - name: Set up Python
+      uses: actions/setup-python@v6
+      with:
+        python-version: "3.11"
+    - name: Install pypa/build
+      run: >-
+        python3 -m
+        pip install
+        build
+        --user
+    - name: Build a binary wheel and a source tarball
+      run: python3 -m build
+    - name: Store the distribution packages
+      uses: actions/upload-artifact@v5
+      with:
+        name: python-package-distributions
+        path: dist/
+
+
+  test-pypi-package:
+    name: Test the built package works basically.
+    runs-on: ubuntu-latest
+    needs:
+    - build
+    steps:
+    - name: Download all the dists
+      uses: actions/download-artifact@v6
+      with:
+        name: python-package-distributions
+        path: dist/
+    - name: Set up Python 3.11
+      uses: actions/setup-python@v6
+      with:
+        python-version: '3.11'
+
+    - name: Test that the basic pip built package runs without error
+      run: |
+        set -ex
+        ls -alR 
+        
+        # Install the first wheel found in dist/
+        WHEEL=$(find dist -type f -name "*.whl" -print -quit)
+        echo Installing $WHEEL
+        python3 -m pip install --upgrade pip
+        python3 -m pip install "$WHEEL"
+        changedetection.io -d /tmp -p 10000 &
+        
+        sleep 3
+        curl --retry-connrefused --retry 6 http://127.0.0.1:10000/static/styles/pure-min.css >/dev/null
+        curl --retry-connrefused --retry 6 http://127.0.0.1:10000/ >/dev/null
+        
+        # --- API test ---
+        # This also means that the docs/api-spec.yml was shipped and could be read
+        test -f /tmp/url-watches.json
+        API_KEY=$(jq -r '.. | .api_access_token? // empty' /tmp/url-watches.json)
+        echo Test API KEY is $API_KEY
+        curl -X POST "http://127.0.0.1:10000/api/v1/watch" \
+          -H "x-api-key: ${API_KEY}" \
+          -H "Content-Type: application/json" \
+          --show-error --fail \
+          --retry 6 --retry-delay 1 --retry-connrefused \
+          -d '{
+            "url": "https://example.com",
+            "title": "Example Site Monitor",
+            "time_between_check": { "hours": 1 }
+          }'
+          
+        killall changedetection.io
+
+
+  publish-to-pypi:
+    name: >-
+      Publish Python 🐍 distribution 📦 to PyPI
+    if: startsWith(github.ref, 'refs/tags/')  # only publish to PyPI on tag pushes
+    needs:
+    - test-pypi-package
+    runs-on: ubuntu-latest
+    environment:
+      name: release
+      url: https://pypi.org/p/changedetection.io
+    permissions:
+      id-token: write  # IMPORTANT: mandatory for trusted publishing
+
+    steps:
+    - name: Download all the dists
+      uses: actions/download-artifact@v6
+      with:
+        name: python-package-distributions
+        path: dist/
+    - name: Publish distribution 📦 to PyPI
+      uses: pypa/gh-action-pypi-publish@release/v1

.github/workflows/test-container-build.yml

@@ -11,25 +11,53 @@ on:
       - requirements.txt
       - Dockerfile
       - .github/workflows/*
+      - .github/test/Dockerfile*
 
   pull_request:
     paths:
       - requirements.txt
       - Dockerfile
       - .github/workflows/*
+      - .github/test/Dockerfile*
 
   # Changes to requirements.txt packages and Dockerfile may or may not always be compatible with arm etc, so worth testing
   # @todo: some kind of path filter for requirements.txt and Dockerfile
 jobs:
-  test-container-build:
+  builder:
+    name: Build ${{ matrix.platform }} (${{ matrix.dockerfile == './Dockerfile' && 'main' || 'alpine' }})
     runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        include:
+          # Main Dockerfile platforms
+          - platform: linux/amd64
+            dockerfile: ./Dockerfile
+          - platform: linux/arm64
+            dockerfile: ./Dockerfile
+          - platform: linux/arm/v7
+            dockerfile: ./Dockerfile
+          - platform: linux/arm/v8
+            dockerfile: ./Dockerfile
+          # Alpine Dockerfile platforms (musl via alpine check)
+          - platform: linux/amd64
+            dockerfile: ./.github/test/Dockerfile-alpine
+          - platform: linux/arm64
+            dockerfile: ./.github/test/Dockerfile-alpine
     steps:
-        - uses: actions/checkout@v4
+        - uses: actions/checkout@v5
         - name: Set up Python 3.11
-          uses: actions/setup-python@v4
+          uses: actions/setup-python@v6
           with:
             python-version: 3.11
 
+        - name: Cache pip packages
+          uses: actions/cache@v4
+          with:
+            path: ~/.cache/pip
+            key: ${{ runner.os }}-pip-${{ hashFiles('requirements.txt') }}
+            restore-keys: |
+              ${{ runner.os }}-pip-
+
         # Just test that the build works, some libraries won't compile on ARM/rPi etc
         - name: Set up QEMU
           uses: docker/setup-qemu-action@v3
@@ -45,24 +73,14 @@ jobs:
             version: latest
             driver-opts: image=moby/buildkit:master
 
-        # https://github.com/dgtlmoon/changedetection.io/pull/1067
-        # Check we can still build under alpine/musl
-        - name: Test that the docker containers can build (musl via alpine check)
-          id: docker_build_musl
-          uses: docker/build-push-action@v5
-          with:
-            context: ./
-            file: ./.github/test/Dockerfile-alpine
-            platforms: linux/amd64,linux/arm64
-
-        - name: Test that the docker containers can build
+        - name: Test that the docker containers can build (${{ matrix.platform }} - ${{ matrix.dockerfile }})
           id: docker_build
-          uses: docker/build-push-action@v5
+          uses: docker/build-push-action@v6
           # https://github.com/docker/build-push-action#customizing
           with:
             context: ./
-            file: ./Dockerfile
-            platforms: linux/amd64,linux/arm64,linux/arm/v6,linux/arm/v7,linux/arm/v8
-            cache-from: type=local,src=/tmp/.buildx-cache
-            cache-to: type=local,dest=/tmp/.buildx-cache
+            file: ${{ matrix.dockerfile }}
+            platforms: ${{ matrix.platform }}
+            cache-from: type=gha
+            cache-to: type=gha,mode=min
 

.github/workflows/test-only.yml

@@ -4,99 +4,52 @@ name: ChangeDetection.io App Test
 on: [push, pull_request]
 
 jobs:
-  test-application:
+  lint-code:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v4
-
-      # Mainly just for link/flake8
-      - name: Set up Python 3.11
-        uses: actions/setup-python@v4
-        with:
-          python-version: '3.11'
-
-      - name: Lint with flake8
+      - uses: actions/checkout@v5
+      - name: Lint with Ruff
         run: |
-          pip3 install flake8
-          # stop the build if there are Python syntax errors or undefined names
-          flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics
-          # exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide
-          flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics
-
-      - name: Spin up ancillary testable services
+          pip install ruff
+          # Check for syntax errors and undefined names
+          ruff check . --select E9,F63,F7,F82
+          # Complete check with errors treated as warnings
+          ruff check . --exit-zero
+      - name: Validate OpenAPI spec
         run: |
-          
-          docker network create changedet-network
+          pip install openapi-spec-validator
+          python3 -c "from openapi_spec_validator import validate_spec; import yaml; validate_spec(yaml.safe_load(open('docs/api-spec.yaml')))"
 
-          # Selenium+browserless
-          docker run --network changedet-network -d --hostname selenium  -p 4444:4444 --rm --shm-size="2g"  selenium/standalone-chrome:4
-          docker run --network changedet-network -d --hostname browserless -e "FUNCTION_BUILT_INS=[\"fs\",\"crypto\"]" -e "DEFAULT_LAUNCH_ARGS=[\"--window-size=1920,1080\"]" --rm  -p 3000:3000  --shm-size="2g"  browserless/chrome:1.60-chrome-stable
+  test-application-3-10:
+    # Only run on push to master (including PR merges)
+    if: github.event_name == 'push' && github.ref == 'refs/heads/master'
+    needs: lint-code
+    uses: ./.github/workflows/test-stack-reusable-workflow.yml
+    with:
+      python-version: '3.10'
 
-      - name: Build changedetection.io container for testing
-        run: |         
-          # Build a changedetection.io container and start testing inside
-          docker build . -t test-changedetectionio
-          # Debug info
-          docker run test-changedetectionio  bash -c 'pip list'
 
-      - name: Spin up ancillary SMTP+Echo message test server
-        run: |
-          # Debug SMTP server/echo message back server
-          docker run --network changedet-network -d -p 11025:11025 -p 11080:11080  --hostname mailserver test-changedetectionio  bash -c 'python changedetectionio/tests/smtp/smtp-test-server.py' 
+  test-application-3-11:
+    # Always run
+    needs: lint-code
+    uses: ./.github/workflows/test-stack-reusable-workflow.yml
+    with:
+      python-version: '3.11'
 
-      - name: Test built container with pytest
-        run: |
-          # Unit tests
-          docker run test-changedetectionio  bash -c 'python3 -m unittest changedetectionio.tests.unit.test_notification_diff'
-          
-          # All tests
-          docker run --network changedet-network  test-changedetectionio  bash -c 'cd changedetectionio && ./run_basic_tests.sh'
+  test-application-3-12:
+    # Only run on push to master (including PR merges)
+    if: github.event_name == 'push' && github.ref == 'refs/heads/master'
+    needs: lint-code
+    uses: ./.github/workflows/test-stack-reusable-workflow.yml
+    with:
+      python-version: '3.12'
+      skip-pypuppeteer: true
 
-      - name: Test built container selenium+browserless/playwright
-        run: |
-          
-          # Selenium fetch
-          docker run --rm -e "WEBDRIVER_URL=http://selenium:4444/wd/hub" --network changedet-network test-changedetectionio  bash -c 'cd changedetectionio;pytest tests/fetchers/test_content.py && pytest tests/test_errorhandling.py'
-          
-          # Playwright/Browserless fetch
-          docker run --rm -e "PLAYWRIGHT_DRIVER_URL=ws://browserless:3000" --network changedet-network test-changedetectionio  bash -c 'cd changedetectionio;pytest tests/fetchers/test_content.py && pytest tests/test_errorhandling.py && pytest tests/visualselector/test_fetch_data.py'
-          
-          # Settings headers playwright tests - Call back in from Browserless, check headers
-          docker run --name "changedet" --hostname changedet --rm -e "FLASK_SERVER_NAME=changedet" -e "PLAYWRIGHT_DRIVER_URL=ws://browserless:3000?dumpio=true" --network changedet-network test-changedetectionio  bash -c 'cd changedetectionio; pytest --live-server-host=0.0.0.0  --live-server-port=5004 tests/test_request.py'
-          docker run --name "changedet" --hostname changedet --rm -e "FLASK_SERVER_NAME=changedet" -e "WEBDRIVER_URL=http://selenium:4444/wd/hub" --network changedet-network test-changedetectionio  bash -c 'cd changedetectionio; pytest --live-server-host=0.0.0.0  --live-server-port=5004 tests/test_request.py'
-          docker run --name "changedet" --hostname changedet --rm -e "FLASK_SERVER_NAME=changedet" -e "USE_EXPERIMENTAL_PUPPETEER_FETCH=yes" -e "PLAYWRIGHT_DRIVER_URL=ws://browserless:3000?dumpio=true" --network changedet-network test-changedetectionio  bash -c 'cd changedetectionio; pytest --live-server-host=0.0.0.0  --live-server-port=5004 tests/test_request.py'          
-          
-          # restock detection via playwright - added name=changedet here so that playwright/browserless can connect to it
-          docker run --rm --name "changedet" -e "FLASK_SERVER_NAME=changedet" -e "PLAYWRIGHT_DRIVER_URL=ws://browserless:3000" --network changedet-network test-changedetectionio  bash -c 'cd changedetectionio;pytest --live-server-port=5004 --live-server-host=0.0.0.0 tests/restock/test_restock.py'
-
-      - name: Test SMTP notification mime types
-        run: |
-          # SMTP content types - needs the 'Debug SMTP server/echo message back server' container from above
-          docker run --rm  --network changedet-network test-changedetectionio bash -c 'cd changedetectionio;pytest tests/smtp/test_notification_smtp.py'
-
-      - name: Test with puppeteer fetcher and disk cache
-        run: |
-          docker run --rm -e "PUPPETEER_DISK_CACHE=/tmp/data/" -e "USE_EXPERIMENTAL_PUPPETEER_FETCH=yes" -e "PLAYWRIGHT_DRIVER_URL=ws://browserless:3000" --network changedet-network test-changedetectionio  bash -c 'cd changedetectionio;pytest tests/fetchers/test_content.py && pytest tests/test_errorhandling.py && pytest tests/visualselector/test_fetch_data.py'
-          # Browserless would have had -e "FUNCTION_BUILT_INS=[\"fs\",\"crypto\"]" added above
-
-      - name: Test proxy interaction
-        run: |
-          cd changedetectionio
-          ./run_proxy_tests.sh
-          # And again with PLAYWRIGHT_DRIVER_URL=..
-          cd ..
-
-      - name: Test changedetection.io container starts+runs basically without error
-        run: |
-          docker run -p 5556:5000 -d test-changedetectionio
-          sleep 3
-          # Should return 0 (no error) when grep finds it
-          curl -s http://localhost:5556 |grep -q checkbox-uuid
-          
-          # and IPv6
-          curl -s -g -6 "http://[::1]:5556"|grep -q checkbox-uuid
-          
-
-#export WEBDRIVER_URL=http://localhost:4444/wd/hub
-#pytest tests/fetchers/test_content.py
-#pytest tests/test_errorhandling.py
+  test-application-3-13:
+    # Only run on push to master (including PR merges)
+    if: github.event_name == 'push' && github.ref == 'refs/heads/master'
+    needs: lint-code
+    uses: ./.github/workflows/test-stack-reusable-workflow.yml
+    with:
+      python-version: '3.13'
+      skip-pypuppeteer: true

.github/workflows/test-pip-build.yml

@@ -1,36 +0,0 @@
-name: ChangeDetection.io PIP package test
-
-# Triggers the workflow on push or pull request events
-
-# This line doesnt work, even tho it is the documented one
-on: [push, pull_request]
-
-  # Changes to requirements.txt packages and Dockerfile may or may not always be compatible with arm etc, so worth testing
-  # @todo: some kind of path filter for requirements.txt and Dockerfile
-jobs:
-  test-pip-build-basics:
-    runs-on: ubuntu-latest
-    steps:
-        - uses: actions/checkout@v4
-
-        - name: Set up Python 3.11
-          uses: actions/setup-python@v4
-          with:
-            python-version: 3.11
-
-
-        - name: Test that the basic pip built package runs without error
-          run: |
-            set -e
-            mkdir dist
-            pip3 install wheel
-            python3 setup.py bdist_wheel            
-            pip3 install -r requirements.txt
-            rm ./changedetection.py
-            rm -rf changedetectio
-            
-            pip3 install dist/changedetection.io*.whl
-            changedetection.io -d /tmp -p 10000 &
-            sleep 3
-            curl http://127.0.0.1:10000/static/styles/pure-min.css >/dev/null
-            killall -9 changedetection.io

.github/workflows/test-stack-reusable-workflow.yml

@@ -0,0 +1,442 @@
+name: ChangeDetection.io App Test
+
+on:
+  workflow_call:
+    inputs:
+      python-version:
+        description: 'Python version to use'
+        required: true
+        type: string
+        default: '3.11'
+      skip-pypuppeteer:
+        description: 'Skip PyPuppeteer (not supported in 3.11/3.12)'
+        required: false
+        type: boolean
+        default: false
+
+jobs:
+  # Build the Docker image once and share it with all test jobs
+  build:
+    runs-on: ubuntu-latest
+    env:
+      PYTHON_VERSION: ${{ inputs.python-version }}
+    steps:
+      - uses: actions/checkout@v5
+
+      - name: Set up Python ${{ env.PYTHON_VERSION }}
+        uses: actions/setup-python@v6
+        with:
+          python-version: ${{ env.PYTHON_VERSION }}
+
+      - name: Cache pip packages
+        uses: actions/cache@v4
+        with:
+          path: ~/.cache/pip
+          key: ${{ runner.os }}-pip-py${{ env.PYTHON_VERSION }}-${{ hashFiles('requirements.txt') }}
+          restore-keys: |
+            ${{ runner.os }}-pip-py${{ env.PYTHON_VERSION }}-
+            ${{ runner.os }}-pip-
+
+      - name: Build changedetection.io container for testing under Python ${{ env.PYTHON_VERSION }}
+        run: |
+          echo "---- Building for Python ${{ env.PYTHON_VERSION }} -----"
+          docker build --build-arg PYTHON_VERSION=${{ env.PYTHON_VERSION }} --build-arg LOGGER_LEVEL=TRACE -t test-changedetectionio .
+          docker run test-changedetectionio bash -c 'pip list'
+
+      - name: We should be Python ${{ env.PYTHON_VERSION }} ...
+        run: |
+          docker run test-changedetectionio bash -c 'python3 --version'
+
+      - name: Save Docker image
+        run: |
+          docker save test-changedetectionio -o /tmp/test-changedetectionio.tar
+
+      - name: Upload Docker image artifact
+        uses: actions/upload-artifact@v5
+        with:
+          name: test-changedetectionio-${{ env.PYTHON_VERSION }}
+          path: /tmp/test-changedetectionio.tar
+          retention-days: 1
+
+  # Unit tests (lightweight, no ancillary services needed)
+  unit-tests:
+    runs-on: ubuntu-latest
+    needs: build
+    timeout-minutes: 10
+    env:
+      PYTHON_VERSION: ${{ inputs.python-version }}
+    steps:
+      - uses: actions/checkout@v5
+
+      - name: Download Docker image artifact
+        uses: actions/download-artifact@v5
+        with:
+          name: test-changedetectionio-${{ env.PYTHON_VERSION }}
+          path: /tmp
+
+      - name: Load Docker image
+        run: |
+          docker load -i /tmp/test-changedetectionio.tar
+
+      - name: Run Unit Tests
+        run: |
+          docker run test-changedetectionio bash -c 'python3 -m unittest changedetectionio.tests.unit.test_notification_diff'
+          docker run test-changedetectionio bash -c 'python3 -m unittest changedetectionio.tests.unit.test_watch_model'
+          docker run test-changedetectionio bash -c 'python3 -m unittest changedetectionio.tests.unit.test_jinja2_security'
+          docker run test-changedetectionio bash -c 'python3 -m unittest changedetectionio.tests.unit.test_semver'
+
+  # Basic pytest tests with ancillary services
+  basic-tests:
+    runs-on: ubuntu-latest
+    needs: build
+    timeout-minutes: 25
+    env:
+      PYTHON_VERSION: ${{ inputs.python-version }}
+    steps:
+      - uses: actions/checkout@v5
+
+      - name: Download Docker image artifact
+        uses: actions/download-artifact@v5
+        with:
+          name: test-changedetectionio-${{ env.PYTHON_VERSION }}
+          path: /tmp
+
+      - name: Load Docker image
+        run: |
+          docker load -i /tmp/test-changedetectionio.tar
+
+      - name: Test built container with Pytest
+        run: |
+          docker network inspect changedet-network >/dev/null 2>&1 || docker network create changedet-network
+          docker run --name test-cdio-basic-tests --network changedet-network test-changedetectionio bash -c 'cd changedetectionio && ./run_basic_tests.sh'
+
+      - name: Extract memory report and logs
+        if: always()
+        uses: ./.github/actions/extract-memory-report
+        with:
+          container-name: test-cdio-basic-tests
+          python-version: ${{ env.PYTHON_VERSION }}
+
+      - name: Store test artifacts
+        if: always()
+        uses: actions/upload-artifact@v5
+        with:
+          name: test-cdio-basic-tests-output-py${{ env.PYTHON_VERSION }}
+          path: output-logs
+
+  # Playwright tests
+  playwright-tests:
+    runs-on: ubuntu-latest
+    needs: build
+    timeout-minutes: 10
+    env:
+      PYTHON_VERSION: ${{ inputs.python-version }}
+    steps:
+      - uses: actions/checkout@v5
+
+      - name: Download Docker image artifact
+        uses: actions/download-artifact@v5
+        with:
+          name: test-changedetectionio-${{ env.PYTHON_VERSION }}
+          path: /tmp
+
+      - name: Load Docker image
+        run: |
+          docker load -i /tmp/test-changedetectionio.tar
+
+      - name: Spin up ancillary services
+        run: |
+          docker network create changedet-network
+          docker run --network changedet-network -d -e "LOG_LEVEL=TRACE" --cap-add=SYS_ADMIN --name sockpuppetbrowser --hostname sockpuppetbrowser --rm -p 3000:3000 dgtlmoon/sockpuppetbrowser:latest
+          docker run --network changedet-network -d -e "LOG_LEVEL=TRACE" --cap-add=SYS_ADMIN --name sockpuppetbrowser-custom-url --hostname sockpuppetbrowser-custom-url -p 3001:3000 --rm dgtlmoon/sockpuppetbrowser:latest
+
+      - name: Playwright - Specific tests in built container
+        run: |
+          docker run --rm -e "FLASK_SERVER_NAME=cdio" -e "PLAYWRIGHT_DRIVER_URL=ws://sockpuppetbrowser:3000" --network changedet-network --hostname=cdio test-changedetectionio bash -c 'cd changedetectionio;pytest -vv --capture=tee-sys --showlocals --tb=long --live-server-host=0.0.0.0 --live-server-port=5004 tests/fetchers/test_content.py'
+          docker run --rm -e "FLASK_SERVER_NAME=cdio" -e "PLAYWRIGHT_DRIVER_URL=ws://sockpuppetbrowser:3000" --network changedet-network --hostname=cdio test-changedetectionio bash -c 'cd changedetectionio;pytest -vv --capture=tee-sys --showlocals --tb=long --live-server-host=0.0.0.0 --live-server-port=5004 tests/test_errorhandling.py'
+          docker run --rm -e "FLASK_SERVER_NAME=cdio" -e "PLAYWRIGHT_DRIVER_URL=ws://sockpuppetbrowser:3000" --network changedet-network --hostname=cdio test-changedetectionio bash -c 'cd changedetectionio;pytest -vv --capture=tee-sys --showlocals --tb=long --live-server-host=0.0.0.0 --live-server-port=5004 tests/visualselector/test_fetch_data.py'
+          docker run --rm -e "FLASK_SERVER_NAME=cdio" -e "PLAYWRIGHT_DRIVER_URL=ws://sockpuppetbrowser:3000" --network changedet-network --hostname=cdio test-changedetectionio bash -c 'cd changedetectionio;pytest -vv --capture=tee-sys --showlocals --tb=long --live-server-host=0.0.0.0 --live-server-port=5004 tests/fetchers/test_custom_js_before_content.py'
+
+      - name: Playwright - Headers and requests
+        run: |
+          docker run --name "changedet" --hostname changedet --rm -e "FLASK_SERVER_NAME=changedet" -e "PLAYWRIGHT_DRIVER_URL=ws://sockpuppetbrowser:3000?dumpio=true" --network changedet-network test-changedetectionio bash -c 'find .; cd changedetectionio; pytest --live-server-host=0.0.0.0 --live-server-port=5004 tests/test_request.py; pwd;find .'
+
+      - name: Playwright - Restock detection
+        run: |
+          docker run --rm --name "changedet" -e "FLASK_SERVER_NAME=changedet" -e "PLAYWRIGHT_DRIVER_URL=ws://sockpuppetbrowser:3000" --network changedet-network test-changedetectionio bash -c 'cd changedetectionio;pytest --live-server-port=5004 --live-server-host=0.0.0.0 tests/restock/test_restock.py'
+
+  # Pyppeteer tests
+  pyppeteer-tests:
+    runs-on: ubuntu-latest
+    needs: build
+    if: ${{ inputs.skip-pypuppeteer == false }}
+    timeout-minutes: 10
+    env:
+      PYTHON_VERSION: ${{ inputs.python-version }}
+    steps:
+      - uses: actions/checkout@v5
+
+      - name: Download Docker image artifact
+        uses: actions/download-artifact@v5
+        with:
+          name: test-changedetectionio-${{ env.PYTHON_VERSION }}
+          path: /tmp
+
+      - name: Load Docker image
+        run: |
+          docker load -i /tmp/test-changedetectionio.tar
+
+      - name: Spin up ancillary services
+        run: |
+          docker network create changedet-network
+          docker run --network changedet-network -d -e "LOG_LEVEL=TRACE" --cap-add=SYS_ADMIN --name sockpuppetbrowser --hostname sockpuppetbrowser --rm -p 3000:3000 dgtlmoon/sockpuppetbrowser:latest
+
+      - name: Pyppeteer - Specific tests in built container
+        run: |
+          docker run --rm -e "FLASK_SERVER_NAME=cdio" -e "FAST_PUPPETEER_CHROME_FETCHER=True" -e "PLAYWRIGHT_DRIVER_URL=ws://sockpuppetbrowser:3000" --network changedet-network --hostname=cdio test-changedetectionio bash -c 'cd changedetectionio;pytest --live-server-host=0.0.0.0 --live-server-port=5004 tests/fetchers/test_content.py'
+          docker run --rm -e "FLASK_SERVER_NAME=cdio" -e "FAST_PUPPETEER_CHROME_FETCHER=True" -e "PLAYWRIGHT_DRIVER_URL=ws://sockpuppetbrowser:3000" --network changedet-network --hostname=cdio test-changedetectionio bash -c 'cd changedetectionio;pytest --live-server-host=0.0.0.0 --live-server-port=5004 tests/test_errorhandling.py'
+          docker run --rm -e "FLASK_SERVER_NAME=cdio" -e "FAST_PUPPETEER_CHROME_FETCHER=True" -e "PLAYWRIGHT_DRIVER_URL=ws://sockpuppetbrowser:3000" --network changedet-network --hostname=cdio test-changedetectionio bash -c 'cd changedetectionio;pytest --live-server-host=0.0.0.0 --live-server-port=5004 tests/visualselector/test_fetch_data.py'
+          docker run --rm -e "FLASK_SERVER_NAME=cdio" -e "FAST_PUPPETEER_CHROME_FETCHER=True" -e "PLAYWRIGHT_DRIVER_URL=ws://sockpuppetbrowser:3000" --network changedet-network --hostname=cdio test-changedetectionio bash -c 'cd changedetectionio;pytest --live-server-host=0.0.0.0 --live-server-port=5004 tests/fetchers/test_custom_js_before_content.py'
+
+      - name: Pyppeteer - Headers and requests checks
+        run: |
+          docker run --name "changedet" --hostname changedet --rm -e "FAST_PUPPETEER_CHROME_FETCHER=True" -e "FLASK_SERVER_NAME=changedet" -e "PLAYWRIGHT_DRIVER_URL=ws://sockpuppetbrowser:3000?dumpio=true" --network changedet-network test-changedetectionio bash -c 'cd changedetectionio; pytest --live-server-host=0.0.0.0 --live-server-port=5004 tests/test_request.py'
+
+      - name: Pyppeteer - Restock detection
+        run: |
+          docker run --rm --name "changedet" -e "FLASK_SERVER_NAME=changedet" -e "FAST_PUPPETEER_CHROME_FETCHER=True" -e "PLAYWRIGHT_DRIVER_URL=ws://sockpuppetbrowser:3000" --network changedet-network test-changedetectionio bash -c 'cd changedetectionio;pytest --live-server-port=5004 --live-server-host=0.0.0.0 tests/restock/test_restock.py'
+
+  # Selenium tests
+  selenium-tests:
+    runs-on: ubuntu-latest
+    needs: build
+    timeout-minutes: 10
+    env:
+      PYTHON_VERSION: ${{ inputs.python-version }}
+    steps:
+      - uses: actions/checkout@v5
+
+      - name: Download Docker image artifact
+        uses: actions/download-artifact@v5
+        with:
+          name: test-changedetectionio-${{ env.PYTHON_VERSION }}
+          path: /tmp
+
+      - name: Load Docker image
+        run: |
+          docker load -i /tmp/test-changedetectionio.tar
+
+      - name: Spin up ancillary services
+        run: |
+          docker network create changedet-network
+          docker run --network changedet-network -d --hostname selenium -p 4444:4444 --rm --shm-size="2g" selenium/standalone-chrome:4
+          sleep 3
+
+      - name: Specific tests for headers and requests checks with Selenium
+        run: |
+
+          docker run --name "changedet" --hostname changedet --rm -e "FLASK_SERVER_NAME=changedet" -e "WEBDRIVER_URL=http://selenium:4444/wd/hub" --network changedet-network test-changedetectionio bash -c 'cd changedetectionio; pytest --live-server-host=0.0.0.0 --live-server-port=5004 tests/test_request.py'
+
+      - name: Specific tests in built container for Selenium
+        run: |
+          docker run --rm -e "WEBDRIVER_URL=http://selenium:4444/wd/hub" --network changedet-network test-changedetectionio bash -c 'cd changedetectionio;pytest tests/fetchers/test_content.py && pytest tests/test_errorhandling.py'
+
+
+  # SMTP tests
+  smtp-tests:
+    runs-on: ubuntu-latest
+    needs: build
+    timeout-minutes: 10
+    env:
+      PYTHON_VERSION: ${{ inputs.python-version }}
+    steps:
+      - uses: actions/checkout@v5
+
+      - name: Download Docker image artifact
+        uses: actions/download-artifact@v5
+        with:
+          name: test-changedetectionio-${{ env.PYTHON_VERSION }}
+          path: /tmp
+
+      - name: Load Docker image
+        run: |
+          docker load -i /tmp/test-changedetectionio.tar
+
+      - name: Spin up SMTP test server
+        run: |
+          docker network create changedet-network
+          docker run --network changedet-network -d -p 11025:11025 -p 11080:11080 --hostname mailserver test-changedetectionio bash -c 'pip3 install aiosmtpd && python changedetectionio/tests/smtp/smtp-test-server.py'
+
+      - name: Test SMTP notification mime types
+        run: |
+          docker run --rm --network changedet-network test-changedetectionio bash -c 'cd changedetectionio;pytest tests/smtp/test_notification_smtp.py'
+
+  # Proxy tests
+  proxy-tests:
+    runs-on: ubuntu-latest
+    needs: build
+    timeout-minutes: 10
+    env:
+      PYTHON_VERSION: ${{ inputs.python-version }}
+    steps:
+      - uses: actions/checkout@v5
+
+      - name: Download Docker image artifact
+        uses: actions/download-artifact@v5
+        with:
+          name: test-changedetectionio-${{ env.PYTHON_VERSION }}
+          path: /tmp
+
+      - name: Load Docker image
+        run: |
+          docker load -i /tmp/test-changedetectionio.tar
+
+      - name: Spin up services
+        run: |
+          docker network create changedet-network
+          docker run --network changedet-network -d --hostname selenium -p 4444:4444 --rm --shm-size="2g" selenium/standalone-chrome:4
+          docker run --network changedet-network -d -e "LOG_LEVEL=TRACE" --cap-add=SYS_ADMIN --name sockpuppetbrowser --hostname sockpuppetbrowser --rm -p 3000:3000 dgtlmoon/sockpuppetbrowser:latest
+          docker run --network changedet-network -d -e "LOG_LEVEL=TRACE" --cap-add=SYS_ADMIN --name sockpuppetbrowser-custom-url --hostname sockpuppetbrowser-custom-url -p 3001:3000 --rm dgtlmoon/sockpuppetbrowser:latest
+
+      - name: Test proxy Squid style interaction
+        run: |
+          cd changedetectionio
+          ./run_proxy_tests.sh
+          docker ps
+          cd ..
+
+      - name: Test proxy SOCKS5 style interaction
+        run: |
+          cd changedetectionio
+          ./run_socks_proxy_tests.sh
+          cd ..
+
+  # Custom browser URL tests
+  custom-browser-tests:
+    runs-on: ubuntu-latest
+    needs: build
+    timeout-minutes: 10
+    env:
+      PYTHON_VERSION: ${{ inputs.python-version }}
+    steps:
+      - uses: actions/checkout@v5
+
+      - name: Download Docker image artifact
+        uses: actions/download-artifact@v5
+        with:
+          name: test-changedetectionio-${{ env.PYTHON_VERSION }}
+          path: /tmp
+
+      - name: Load Docker image
+        run: |
+          docker load -i /tmp/test-changedetectionio.tar
+
+      - name: Spin up ancillary services
+        run: |
+          docker network create changedet-network
+          docker run --network changedet-network -d -e "LOG_LEVEL=TRACE" --cap-add=SYS_ADMIN --name sockpuppetbrowser --hostname sockpuppetbrowser --rm -p 3000:3000 dgtlmoon/sockpuppetbrowser:latest
+          docker run --network changedet-network -d -e "LOG_LEVEL=TRACE" --cap-add=SYS_ADMIN --name sockpuppetbrowser-custom-url --hostname sockpuppetbrowser-custom-url -p 3001:3000 --rm dgtlmoon/sockpuppetbrowser:latest
+
+      - name: Test custom browser URL
+        run: |
+          cd changedetectionio
+          ./run_custom_browser_url_tests.sh
+
+  # Container startup tests
+  container-tests:
+    runs-on: ubuntu-latest
+    needs: build
+    timeout-minutes: 10
+    env:
+      PYTHON_VERSION: ${{ inputs.python-version }}
+    steps:
+      - uses: actions/checkout@v5
+
+      - name: Download Docker image artifact
+        uses: actions/download-artifact@v5
+        with:
+          name: test-changedetectionio-${{ env.PYTHON_VERSION }}
+          path: /tmp
+
+      - name: Load Docker image
+        run: |
+          docker load -i /tmp/test-changedetectionio.tar
+
+      - name: Test container starts+runs basically without error
+        run: |
+          docker run --name test-changedetectionio -p 5556:5000 -d test-changedetectionio
+          sleep 3
+          curl --retry-connrefused --retry 6 -s http://localhost:5556 |grep -q checkbox-uuid
+          curl --retry-connrefused --retry 6 -s -g -6 "http://[::1]:5556"|grep -q checkbox-uuid
+          docker logs test-changedetectionio 2>/dev/null | grep 'TRACE log is enabled' || exit 1
+          docker logs test-changedetectionio 2>/dev/null | grep 'DEBUG' || exit 1
+          docker kill test-changedetectionio
+
+      - name: Test HTTPS SSL mode
+        run: |
+          openssl req -x509 -newkey rsa:4096 -keyout privkey.pem -out cert.pem -days 365 -nodes -subj "/CN=localhost"
+          docker run --name test-changedetectionio-ssl --rm -e SSL_CERT_FILE=cert.pem -e SSL_PRIVKEY_FILE=privkey.pem -p 5000:5000 -v ./cert.pem:/app/cert.pem -v ./privkey.pem:/app/privkey.pem -d test-changedetectionio
+          sleep 3
+          curl --retry-connrefused --retry 6 -k https://localhost:5000 -v|grep -q checkbox-uuid
+          docker kill test-changedetectionio-ssl
+
+      - name: Test IPv6 Mode
+        run: |
+          docker run --name test-changedetectionio-ipv6 --rm -p 5000:5000 -e LISTEN_HOST=:: -d test-changedetectionio
+          sleep 3
+          curl --retry-connrefused --retry 6 http://[::1]:5000 -v|grep -q checkbox-uuid
+          docker kill test-changedetectionio-ipv6
+
+  # Signal tests
+  signal-tests:
+    runs-on: ubuntu-latest
+    needs: build
+    timeout-minutes: 10
+    env:
+      PYTHON_VERSION: ${{ inputs.python-version }}
+    steps:
+      - uses: actions/checkout@v5
+
+      - name: Download Docker image artifact
+        uses: actions/download-artifact@v5
+        with:
+          name: test-changedetectionio-${{ env.PYTHON_VERSION }}
+          path: /tmp
+
+      - name: Load Docker image
+        run: |
+          docker load -i /tmp/test-changedetectionio.tar
+
+      - name: Test SIGTERM and SIGINT signal shutdown
+        run: |
+          echo SIGINT Shutdown request test
+          docker run --name sig-test -d test-changedetectionio
+          sleep 3
+          echo ">>> Sending SIGINT to sig-test container"
+          docker kill --signal=SIGINT sig-test
+          sleep 3
+          docker ps
+          docker logs sig-test 2>&1 | grep 'Shutdown: Got Signal - SIGINT' || exit 1
+          test -z "`docker ps|grep sig-test`"
+          if [ $? -ne 0 ]; then
+            echo "Looks like container was running when it shouldnt be"
+            docker ps
+            exit 1
+          fi
+          docker rm sig-test
+
+          echo SIGTERM Shutdown request test
+          docker run --name sig-test -d test-changedetectionio
+          sleep 3
+          echo ">>> Sending SIGTERM to sig-test container"
+          docker kill --signal=SIGTERM sig-test
+          sleep 3
+          docker ps
+          docker logs sig-test 2>&1 | grep 'Shutdown: Got Signal - SIGTERM' || exit 1
+          test -z "`docker ps|grep sig-test`"
+          if [ $? -ne 0 ]; then
+            echo "Looks like container was running when it shouldnt be"
+            docker ps
+            exit 1
+          fi
+          docker rm sig-test

.gitignore

@@ -1,14 +1,30 @@
-__pycache__
-.idea
-*.pyc
-datastore/url-watches.json
-datastore/*
-__pycache__
-.pytest_cache
-build
-dist
-venv
-test-datastore/*
-test-datastore
+# Byte-compiled / optimized / DLL files
+**/__pycache__
+**/*.py[cod]
+
+# Caches
+.mypy_cache/
+.pytest_cache/
+.ruff_cache/
+
+# Distribution / packaging
+build/
+dist/
 *.egg-info*
+
+# Virtual environment
+.env
+.venv/
+venv/
+.python-version
+
+# IDEs
+.idea
 .vscode/settings.json
+
+# Datastore files
+datastore/
+test-datastore/
+
+# Memory consumption log
+test-memory.log

.pre-commit-config.yaml

@@ -0,0 +1,9 @@
+repos:
+  - repo: https://github.com/astral-sh/ruff-pre-commit
+    rev: v0.11.2
+    hooks:
+      # Lint (and apply safe fixes)
+      - id: ruff
+        args: [--fix]
+      # Fomrat
+      - id: ruff-format

.ruff.toml

@@ -0,0 +1,48 @@
+# Minimum supported version
+target-version = "py310"
+
+# Formatting options
+line-length = 100
+indent-width = 4
+
+exclude = [
+    "__pycache__",
+    ".eggs",
+    ".git",
+    ".tox",
+    ".venv",
+    "*.egg-info",
+    "*.pyc",
+]
+
+[lint]
+# https://docs.astral.sh/ruff/rules/
+select = [
+    "B", # flake8-bugbear
+    "B9",
+    "C", 
+    "E", # pycodestyle
+    "F", # Pyflakes
+    "I", # isort
+    "N", # pep8-naming
+    "UP", # pyupgrade
+    "W", # pycodestyle
+]
+ignore = [
+    "B007", # unused-loop-control-variable
+    "B909", # loop-iterator-mutation
+    "E203", # whitespace-before-punctuation
+    "E266", # multiple-leading-hashes-for-block-comment
+    "E501", # redundant-backslash
+    "F403", # undefined-local-with-import-star
+    "N802", # invalid-function-name
+    "N806", # non-lowercase-variable-in-function
+    "N815", # mixed-case-variable-in-class-scope
+]
+
+[lint.mccabe]
+max-complexity = 12
+
+[format]
+indent-style = "space"
+quote-style = "preserve"

COMMERCIAL_LICENCE.md

@@ -0,0 +1,54 @@
+# Generally
+
+In any commercial activity involving 'Hosting' (as defined herein), whether in part or in full, this license must be executed and adhered to.
+
+# Commercial License Agreement
+
+This Commercial License Agreement ("Agreement") is entered into by and between Web Technologies s.r.o. here-in ("Licensor") and (your company or personal name) _____________ ("Licensee"). This Agreement sets forth the terms and conditions under which Licensor provides its software ("Software") and services to Licensee for the purpose of reselling the software either in part or full, as part of any commercial activity where the activity involves a third party.
+
+### Definition of Hosting
+
+For the purposes of this Agreement, "hosting" means making the functionality of the Program or modified version available to third parties as a service. This includes, without limitation:
+- Enabling third parties to interact with the functionality of the Program or modified version remotely through a computer network.
+- Offering a service the value of which entirely or primarily derives from the value of the Program or modified version.
+- Offering a service that accomplishes for users the primary purpose of the Program or modified version.
+
+## 1. Grant of License
+Subject to the terms and conditions of this Agreement, Licensor grants Licensee a non-exclusive, non-transferable license to install, use, and resell the Software. Licensee may:
+- Resell the Software as part of a service offering or as a standalone product.
+- Host the Software on a server and provide it as a hosted service (e.g., Software as a Service - SaaS).
+- Integrate the Software into a larger product or service that is then sold or provided for commercial purposes, where the software is used either in part or full.
+
+## 2. License Fees
+Licensee agrees to pay Licensor the license fees specified in the ordering document. License fees are due and payable as specified in the ordering document. The fees may include initial licensing costs and recurring fees based on the number of end users, instances of the Software resold, or revenue generated from the resale activities.
+
+## 3. Resale Conditions
+Licensee must comply with the following conditions when reselling the Software, whether the software is resold in part or full:
+- Provide end users with access to the source code under the same open-source license conditions as provided by Licensor.
+- Clearly state in all marketing and sales materials that the Software is provided under a commercial license from Licensor, and provide a link back to https://changedetection.io.
+- Ensure end users are aware of and agree to the terms of the commercial license prior to resale.
+- Do not sublicense or transfer the Software to third parties except as part of an authorized resale activity.
+
+## 4. Hosting and Provision of Services
+Licensee may host the Software (either in part or full) on its servers and provide it as a hosted service to end users. The following conditions apply:
+- Licensee must ensure that all hosted versions of the Software comply with the terms of this Agreement.
+- Licensee must provide Licensor with regular reports detailing the number of end users and instances of the hosted service.
+- Any modifications to the Software made by Licensee for hosting purposes must be made available to end users under the same open-source license conditions, unless agreed otherwise.
+
+## 5. Services
+Licensor will provide support and maintenance services as described in the support policy referenced in the ordering document should such an agreement be signed by all parties. Additional fees may apply for support services provided to end users resold by Licensee.
+
+## 6. Reporting and Audits
+Licensee agrees to provide Licensor with regular reports detailing the number of instances, end users, and revenue generated from the resale of the Software. Licensor reserves the right to audit Licensee’s records to ensure compliance with this Agreement.
+
+## 7. Term and Termination
+This Agreement shall commence on the effective date and continue for the period set forth in the ordering document unless terminated earlier in accordance with this Agreement. Either party may terminate this Agreement if the other party breaches any material term and fails to cure such breach within thirty (30) days after receipt of written notice.
+
+## 8. Limitation of Liability and Disclaimer of Warranty
+Executing this commercial license does not waive the Limitation of Liability or Disclaimer of Warranty as stated in the open-source LICENSE provided with the Software. The Software is provided "as is," without warranty of any kind, express or implied, including but not limited to the warranties of merchantability, fitness for a particular purpose, and noninfringement. In no event shall the authors or copyright holders be liable for any claim, damages, or other liability, whether in an action of contract, tort, or otherwise, arising from, out of, or in connection with the Software or the use or other dealings in the Software.
+
+## 9. Governing Law
+This Agreement shall be governed by and construed in accordance with the laws of the Czech Republic.
+
+## Contact Information
+For commercial licensing inquiries, please contact contact@changedetection.io and dgtlmoon@gmail.com.

CONTRIBUTING.md

@@ -2,7 +2,7 @@ Contributing is always welcome!
 
 I am no professional flask developer, if you know a better way that something can be done, please let me know!
 
-Otherwise, it's always best to PR into the `dev` branch.
+Otherwise, it's always best to PR into the `master` branch.
 
 Please be sure that all new functionality has a matching test!
 

Dockerfile

@@ -1,8 +1,10 @@
 # pip dependencies install stage
-FROM python:3.11-slim-bookworm as builder
+
+ARG PYTHON_VERSION=3.11
+
+FROM python:${PYTHON_VERSION}-slim-bookworm AS builder
 
 # See `cryptography` pin comment in requirements.txt
-ARG CRYPTOGRAPHY_DONT_BUILD_RUST=1
 
 RUN apt-get update && apt-get install -y --no-install-recommends \
     g++ \
@@ -13,6 +15,8 @@ RUN apt-get update && apt-get install -y --no-install-recommends \
     libssl-dev \
     libxslt-dev \
     make \
+    patch \
+    pkg-config \
     zlib1g-dev
 
 RUN mkdir /install
@@ -20,26 +24,48 @@ WORKDIR /install
 
 COPY requirements.txt /requirements.txt
 
-# Instructing pip to fetch wheels from piwheels.org" on ARMv6 and ARMv7 machines
-RUN if [ "$(dpkg --print-architecture)" = "armhf" ] || [ "$(dpkg --print-architecture)" = "armel" ]; then \
-      printf "[global]\nextra-index-url=https://www.piwheels.org/simple\n" > /etc/pip.conf; \
-    fi;
-
-RUN pip install --target=/dependencies -r /requirements.txt
+# Use cache mounts and multiple wheel sources for faster ARM builds
+ENV PIP_CACHE_DIR=/tmp/pip-cache
+# Help Rust find OpenSSL for cryptography package compilation on ARM
+ENV PKG_CONFIG_PATH="/usr/lib/pkgconfig:/usr/lib/arm-linux-gnueabihf/pkgconfig:/usr/lib/aarch64-linux-gnu/pkgconfig"
+ENV PKG_CONFIG_ALLOW_SYSTEM_CFLAGS=1
+ENV OPENSSL_DIR="/usr"
+ENV OPENSSL_LIB_DIR="/usr/lib/arm-linux-gnueabihf"
+ENV OPENSSL_INCLUDE_DIR="/usr/include/openssl"
+# Additional environment variables for cryptography Rust build
+ENV CRYPTOGRAPHY_DONT_BUILD_RUST=1
+RUN --mount=type=cache,target=/tmp/pip-cache \
+    pip install \
+    --prefer-binary \
+    --extra-index-url https://www.piwheels.org/simple \
+    --extra-index-url https://pypi.anaconda.org/ARM-software/simple \
+    --cache-dir=/tmp/pip-cache \
+    --target=/dependencies \
+    -r /requirements.txt
 
 # Playwright is an alternative to Selenium
 # Excluded this package from requirements.txt to prevent arm/v6 and arm/v7 builds from failing
 # https://github.com/dgtlmoon/changedetection.io/pull/1067 also musl/alpine (not supported)
-RUN pip install --target=/dependencies playwright~=1.39 \
+RUN --mount=type=cache,target=/tmp/pip-cache \
+    pip install \
+    --prefer-binary \
+    --cache-dir=/tmp/pip-cache \
+    --target=/dependencies \
+    playwright~=1.48.0 \
     || echo "WARN: Failed to install Playwright. The application can still run, but the Playwright option will be disabled."
 
 # Final image stage
-FROM python:3.11-slim-bookworm
+FROM python:${PYTHON_VERSION}-slim-bookworm
+LABEL org.opencontainers.image.source="https://github.com/dgtlmoon/changedetection.io"
 
 RUN apt-get update && apt-get install -y --no-install-recommends \
     libxslt1.1 \
+    # For presenting price amounts correctly in the restock/price detection overview
+    locales \
     # For pdftohtml
     poppler-utils \
+    # favicon type detection and other uses
+    file \
     zlib1g \
     && apt-get clean && rm -rf /var/lib/apt/lists/*
 
@@ -58,12 +84,25 @@ ENV PYTHONPATH=/usr/local
 
 EXPOSE 5000
 
-# The actual flask app
+# The actual flask app module
 COPY changedetectionio /app/changedetectionio
 
-# The eventlet server wrapper
+# Also for OpenAPI validation wrapper - needs the YML
+RUN [ ! -d "/app/docs" ] && mkdir /app/docs
+COPY docs/api-spec.yaml /app/docs/api-spec.yaml
+
+# Starting wrapper
 COPY changedetection.py /app/changedetection.py
 
-WORKDIR /app
+# Github Action test purpose(test-only.yml).
+# On production, it is effectively LOGGER_LEVEL=''.
+ARG LOGGER_LEVEL=''
+ENV LOGGER_LEVEL="$LOGGER_LEVEL"
+
+# Default
+ENV LC_ALL=en_US.UTF-8
+
+WORKDIR /app
+CMD ["python", "./changedetection.py", "-d", "/datastore"]
+
 
-CMD [ "python", "./changedetection.py" , "-d", "/datastore"]

LICENSE

@@ -186,7 +186,7 @@
       same "printed page" as the copyright notice for easier
       identification within third-party archives.
 
-   Copyright [yyyy] [name of copyright owner]
+   Copyright 2025 Web Technologies s.r.o.
 
    Licensed under the Apache License, Version 2.0 (the "License");
    you may not use this file except in compliance with the License.

MANIFEST.in

@@ -1,15 +1,23 @@
 recursive-include changedetectionio/api *
+include docs/api-spec.yaml
 recursive-include changedetectionio/blueprint *
+recursive-include changedetectionio/conditions *
+recursive-include changedetectionio/content_fetchers *
+recursive-include changedetectionio/jinja2_custom *
 recursive-include changedetectionio/model *
+recursive-include changedetectionio/notification *
 recursive-include changedetectionio/processors *
-recursive-include changedetectionio/res *
+recursive-include changedetectionio/realtime *
 recursive-include changedetectionio/static *
 recursive-include changedetectionio/templates *
 recursive-include changedetectionio/tests *
+recursive-include changedetectionio/widgets *
 prune changedetectionio/static/package-lock.json
 prune changedetectionio/static/styles/node_modules
 prune changedetectionio/static/styles/package-lock.json
 include changedetection.py
+include requirements.txt
+include README-pip.md
 global-exclude *.pyc
 global-exclude node_modules
 global-exclude venv

Procfile

@@ -1 +0,0 @@
-web: python3 ./changedetection.py -C -d ./datastore -p $PORT

README-pip.md

@@ -1,11 +1,21 @@
-## Web Site Change Detection, Monitoring and Notification.
+# Monitor website changes
 
-Live your data-life pro-actively, track website content changes and receive notifications via Discord, Email, Slack, Telegram and 70+ more
+Detect WebPage Changes Automatically — Monitor Web Page Changes in Real Time
+
+Monitor websites for updates — get notified via Discord, Email, Slack, Telegram, Webhook and many more.
+
+Detect web page content changes and get instant alerts.
+
+
+[Changedetection.io is the best tool to monitor web-pages for changes](https://changedetection.io) Track website content changes and receive notifications via Discord, Email, Slack, Telegram and 90+ more
+
+Ideal for monitoring price changes, content edits, conditional changes and more.
 
 [<img src="https://raw.githubusercontent.com/dgtlmoon/changedetection.io/master/docs/screenshot.png" style="max-width:100%;" alt="Self-hosted web page change monitoring, list of websites with changes"  title="Self-hosted web page change monitoring, list of websites with changes"  />](https://changedetection.io)
 
 
-[**Don't have time? Let us host it for you! try our extremely affordable subscription use our proxies and support!**](https://changedetection.io) 
+[**Don't have time? Try our extremely affordable subscription use our proxies and support!**](https://changedetection.io) 
+
 
 
 ### Target specific parts of the webpage using the Visual Selector tool.

README.md

@@ -1,22 +1,26 @@
-## Web Site Change Detection, Restock monitoring and notifications.
+# Detect Website Changes Automatically — Monitor Web Page Changes in Real Time
 
-**_Detect website content changes and perform meaningful actions - trigger notifications via Discord, Email, Slack, Telegram, API calls and many more._**
+Monitor websites for updates — get notified via Discord, Email, Slack, Telegram, Webhook and many more.
 
-_Live your data-life pro-actively._ 
+**Detect web page content changes and get instant alerts.**  
+
+Ideal for monitoring price changes, content edits, conditional changes and more.
 
 
-[<img src="https://raw.githubusercontent.com/dgtlmoon/changedetection.io/master/docs/screenshot.png" style="max-width:100%;" alt="Self-hosted web site page change monitoring"  title="Self-hosted web site page change monitoring"  />](https://changedetection.io?src=github)
+[<img src="https://raw.githubusercontent.com/dgtlmoon/changedetection.io/master/docs/screenshot.png" style="max-width:100%;" alt="Web site page change monitoring"  title="Web site page change monitoring"  />](https://changedetection.io?src=github)
 
 [![Release Version][release-shield]][release-link] [![Docker Pulls][docker-pulls]][docker-link] [![License][license-shield]](LICENSE.md)
 
 ![changedetection.io](https://github.com/dgtlmoon/changedetection.io/actions/workflows/test-only.yml/badge.svg?branch=master)
 
-[**Don't have time? Let us host it for you! try our $8.99/month subscription - use our proxies and support!**](https://changedetection.io) , _half the price of other website change monitoring services!_
+[**Get started with website page change monitoring straight away. Don't have time? Try our $8.99/month subscription, use our proxies and support!**](https://changedetection.io) , _half the price of other website change monitoring services!_
+
 
 - Chrome browser included.
+- Nothing to install, access via browser login after signup.
 - Super fast, no registration needed setup.
 - Get started watching and receiving website change notifications straight away.
-
+- See our [tutorials and how-to page for more inspiration](https://changedetection.io/tutorials) 
 
 ### Target specific parts of the webpage using the Visual Selector tool.
 
@@ -40,6 +44,20 @@ Using the **Browser Steps** configuration, add basic steps before performing cha
 After **Browser Steps** have been run, then visit the **Visual Selector** tab to refine the content you're interested in.
 Requires Playwright to be enabled.
 
+### Awesome restock and price change notifications
+
+Enable the _"Re-stock & Price detection for single product pages"_ option to activate the best way to monitor product pricing, this will extract any meta-data in the HTML page and give you many options to follow the pricing of the product.
+
+Easily organise and monitor prices for products from the dashboard, get alerts and notifications when the price of a product changes or comes back in stock again!
+
+[<img src="docs/restock-overview.png" style="max-width:100%;" alt="Easily keep an eye on product price changes directly from the UI"  title="Easily keep an eye on product price changes directly from the UI" />](https://changedetection.io?src=github)
+
+Set price change notification parameters, upper and lower price, price change percentage and more.
+Always know when a product for sale drops in price.
+
+[<img src="docs/restock-settings.png" style="max-width:100%;" alt="Set upper lower and percentage price change notification values"  title="Set upper lower and percentage price change notification values" />](https://changedetection.io?src=github)
+
+
 
 ### Example use cases
 
@@ -74,7 +92,7 @@ _Need an actual Chrome runner with Javascript support? We support fetching via W
 #### Key Features
 
 - Lots of trigger filters, such as "Trigger on text", "Remove text by selector", "Ignore text", "Extract text", also using regular-expressions!
-- Target elements with xPath(1.0) and CSS Selectors, Easily monitor complex JSON with JSONPath or jq
+- Target elements with xPath 1 and xPath 2, CSS Selectors, Easily monitor complex JSON with JSONPath or jq
 - Switch between fast non-JS and Chrome JS based "fetchers"
 - Track changes in PDF files (Monitor text changed in the PDF, Also monitor PDF filesize and checksums)
 - Easily specify how often a site should be checked
@@ -84,12 +102,33 @@ _Need an actual Chrome runner with Javascript support? We support fetching via W
 - Configurable [proxy per watch](https://github.com/dgtlmoon/changedetection.io/wiki/Proxy-configuration)
 - Send a screenshot with the notification when a change is detected in the web page
 
-We [recommend and use Bright Data](https://brightdata.grsm.io/n0r16zf7eivq) global proxy services, Bright Data will match any first deposit up to $100 using our signup link.
-
-[Oxylabs](https://oxylabs.go2cloud.org/SH2d) is also an excellent proxy provider and well worth using, they offer Residental, ISP, Rotating and many other proxy types to suit your project. 
+We [recommend and use Bright Data](https://brightdata.grsm.io/n0r16zf7eivq) global proxy services, Bright Data will match any first deposit up to $150 using our signup link.
 
 Please :star: star :star: this project and help it grow! https://github.com/dgtlmoon/changedetection.io/
 
+### Conditional web page changes
+
+Easily [configure conditional actions](https://changedetection.io/tutorial/conditional-actions-web-page-changes), for example, only trigger when a price is above or below a preset amount, or [when a web page includes (or does not include) a keyword](https://changedetection.io/tutorial/how-monitor-keywords-any-website)
+
+<img src="./docs/web-page-change-conditions.png" style="max-width:80%;" alt="Conditional web page changes"  title="Conditional web page changes"  />
+
+### Schedule web page watches in any timezone, limit by day of week and time.
+
+Easily set a re-check schedule, for example you could limit the web page change detection to only operate during business hours.
+Or perhaps based on a foreign timezone (for example, you want to check for the latest news-headlines in a foreign country at 0900 AM),
+
+<img src="./docs/scheduler.png" style="max-width:80%;" alt="How to monitor web page changes according to a schedule"  title="How to monitor web page changes according to a schedule"  />
+
+Includes quick short-cut buttons to setup a schedule for **business hours only**, or **weekends**.
+
+### We have a Chrome extension!
+
+Easily add the current web page to your changedetection.io tool, simply install the extension and click "Sync" to connect it to your existing changedetection.io install.
+
+[<img src="./docs/chrome-extension-screenshot.png" style="max-width:80%;" alt="Chrome Extension to easily add the current web-page to detect a change."  title="Chrome Extension to easily add the current web-page to detect a change."  />](https://chromewebstore.google.com/detail/changedetectionio-website/kefcfmgmlhmankjmnbijimhofdjekbop)
+
+[Goto the Chrome Webstore to download the extension.](https://chromewebstore.google.com/detail/changedetectionio-website/kefcfmgmlhmankjmnbijimhofdjekbop) ( Or check out the [GitHub repo](https://github.com/dgtlmoon/changedetection.io-browser-extension) ) 
+
 ## Installation
 
 ### Docker
@@ -97,7 +136,7 @@ Please :star: star :star: this project and help it grow! https://github.com/dgtl
 With Docker composer, just clone this repository and..
 
 ```bash
-$ docker-compose up -d
+$ docker compose up -d
 ```
 
 Docker standalone
@@ -136,10 +175,10 @@ docker rm $(docker ps -a -f name=changedetection.io -q)
 docker run -d --restart always -p "127.0.0.1:5000:5000" -v datastore-volume:/datastore --name changedetection.io dgtlmoon/changedetection.io
 ```
 
-### docker-compose
+### docker compose
 
 ```bash
-docker-compose pull && docker-compose up -d
+docker compose pull && docker compose up -d
 ```
 
 See the wiki for more information https://github.com/dgtlmoon/changedetection.io/wiki
@@ -241,20 +280,17 @@ Excel import is recommended - that way you can better organise tags/groups of we
 
 ## API Support
 
-Supports managing the website watch list [via our API](https://changedetection.io/docs/api_v1/index.html)
+Full REST API for programmatic management of watches, tags, notifications and more. 
+
+- **[Interactive API Documentation](https://changedetection.io/docs/api_v1/index.html)** - Complete API reference with live testing
+- **[OpenAPI Specification](docs/api-spec.yaml)** - Generate SDKs for any programming language
 
 ## Support us
 
 Do you use changedetection.io to make money? does it save you time or money? Does it make your life easier? less stressful? Remember, we write this software when we should be doing actual paid work, we have to buy food and pay rent just like you.
 
 
-Firstly, consider taking out a [change detection monthly subscription - unlimited checks and watches](https://changedetection.io?src=github) , even if you don't use it, you still get the warm fuzzy feeling of helping out the project. (And who knows, you might just use it!)
-
-Or directly donate an amount PayPal [![Donate](https://img.shields.io/badge/Donate-PayPal-green.svg)](https://www.paypal.com/donate/?hosted_button_id=7CP6HR9ZCNDYJ)
-
-Or BTC `1PLFN327GyUarpJd7nVe7Reqg9qHx5frNn`
-
-<img src="https://raw.githubusercontent.com/dgtlmoon/changedetection.io/master/docs/btc-support.png" style="max-width:50%;" alt="Support us!"  />
+Consider taking out an officially supported [website change detection subscription](https://changedetection.io?src=github) , even if you don't use it, you still get the warm fuzzy feeling of helping out the project. (And who knows, you might just use it!)
 
 ## Commercial Support
 
@@ -268,3 +304,17 @@ I offer commercial support, this software is depended on by network security, ae
 [license-shield]: https://img.shields.io/github/license/dgtlmoon/changedetection.io.svg?style=for-the-badge
 [release-link]: https://github.com/dgtlmoon/changedetection.io/releases
 [docker-link]: https://hub.docker.com/r/dgtlmoon/changedetection.io
+
+## Commercial Licencing
+
+If you are reselling this software either in part or full as part of any commercial arrangement, you must abide by our COMMERCIAL_LICENCE.md found in our code repository, please contact dgtlmoon@gmail.com and contact@changedetection.io .
+
+## Third-party licenses
+
+changedetectionio.html_tools.elementpath_tostring: Copyright (c), 2018-2021, SISSA (Scuola Internazionale Superiore di Studi Avanzati), Licensed under [MIT license](https://github.com/sissaschool/elementpath/blob/master/LICENSE)
+
+## Contributors
+
+Recognition of fantastic contributors to the project
+
+- Constantin Hong https://github.com/Constantin1489

app.json

@@ -1,21 +0,0 @@
-{
-  "name": "ChangeDetection.io",
-  "description": "The best and simplest self-hosted open source website change detection monitoring and notification service.",
-  "keywords": [
-    "changedetection",
-    "website monitoring"
-  ],
-  "repository": "https://github.com/dgtlmoon/changedetection.io",
-  "success_url": "/",
-  "scripts": {
-  },
-  "env": {
-  },
-  "formation": {
-    "web": {
-      "quantity": 1,
-      "size": "free"
-    }
-  },
-  "image": "heroku/python"
-}

changedetection.py

@@ -1,44 +1,8 @@
-#!/usr/bin/python3
+#!/usr/bin/env python3
 
-# Entry-point for running from the CLI when not installed via Pip, Pip will handle the console_scripts entry_points's from setup.py
-# It's recommended to use `pip3 install changedetection.io` and start with `changedetection.py` instead, it will be linkd to your global path.
-# or Docker.
-# Read more https://github.com/dgtlmoon/changedetection.io/wiki
+# Only exists for direct CLI usage
 
-from changedetectionio import changedetection
-import multiprocessing
-import sys
-import os
-
-def sigchld_handler(_signo, _stack_frame):
-    import sys
-    print('Shutdown: Got SIGCHLD')
-    # https://stackoverflow.com/questions/40453496/python-multiprocessing-capturing-signals-to-restart-child-processes-or-shut-do
-    pid, status = os.waitpid(-1, os.WNOHANG | os.WUNTRACED | os.WCONTINUED)
-
-    print('Sub-process: pid %d status %d' % (pid, status))
-    if status != 0:
-        sys.exit(1)
-
-    raise SystemExit
+import changedetectionio
 
 if __name__ == '__main__':
-
-    #signal.signal(signal.SIGCHLD, sigchld_handler)
-
-    # The only way I could find to get Flask to shutdown, is to wrap it and then rely on the subsystem issuing SIGTERM/SIGKILL
-    parse_process = multiprocessing.Process(target=changedetection.main)
-    parse_process.daemon = True
-    parse_process.start()
-    import time
-
-    try:
-        while True:
-            time.sleep(1)
-            if not parse_process.is_alive():
-                # Process died/crashed for some reason, exit with error set
-                sys.exit(1)
-
-    except KeyboardInterrupt:
-        #parse_process.terminate() not needed, because this process will issue it to the sub-process anyway
-        print ("Exited - CTRL+C")
+    changedetectionio.main()

changedetectionio/PLUGIN_README.md

@@ -0,0 +1,98 @@
+# Creating Plugins for changedetection.io
+
+This document describes how to create plugins for changedetection.io. Plugins can be used to extend the functionality of the application in various ways.
+
+## Plugin Types
+
+### UI Stats Tab Plugins
+
+These plugins can add content to the Stats tab in the Edit page. This is useful for adding custom statistics or visualizations about a watch.
+
+#### Creating a UI Stats Tab Plugin
+
+1. Create a Python file in a directory that will be loaded by the plugin system.
+
+2. Use the `global_hookimpl` decorator to implement the `ui_edit_stats_extras` hook:
+
+```python
+import pluggy
+from loguru import logger
+
+global_hookimpl = pluggy.HookimplMarker("changedetectionio")
+
+@global_hookimpl
+def ui_edit_stats_extras(watch):
+    """Add custom content to the stats tab"""
+    # Calculate or retrieve your stats
+    my_stat = calculate_something(watch)
+    
+    # Return HTML content as a string
+    html = f"""
+    <div class="my-plugin-stats">
+        <h4>My Plugin Statistics</h4>
+        <p>My statistic: {my_stat}</p>
+    </div>
+    """
+    return html
+```
+
+3. The HTML you return will be included in the Stats tab.
+
+## Plugin Loading
+
+Plugins can be loaded from:
+
+1. Built-in plugin directories in the codebase
+2. External packages using setuptools entry points
+
+To add a new plugin directory, modify the `plugin_dirs` dictionary in `pluggy_interface.py`.
+
+## Example Plugin
+
+Here's a simple example of a plugin that adds a word count statistic to the Stats tab:
+
+```python
+import pluggy
+from loguru import logger
+
+global_hookimpl = pluggy.HookimplMarker("changedetectionio")
+
+def count_words_in_history(watch):
+    """Count words in the latest snapshot"""
+    try:
+        if not watch.history.keys():
+            return 0
+            
+        latest_key = list(watch.history.keys())[-1]
+        latest_content = watch.get_history_snapshot(latest_key)
+        return len(latest_content.split())
+    except Exception as e:
+        logger.error(f"Error counting words: {str(e)}")
+        return 0
+
+@global_hookimpl
+def ui_edit_stats_extras(watch):
+    """Add word count to the Stats tab"""
+    word_count = count_words_in_history(watch)
+    
+    html = f"""
+    <div class="word-count-stats">
+        <h4>Content Analysis</h4>
+        <table class="pure-table">
+            <tbody>
+                <tr>
+                    <td>Word count (latest snapshot)</td>
+                    <td>{word_count}</td>
+                </tr>
+            </tbody>
+        </table>
+    </div>
+    """
+    return html
+```
+
+## Testing Your Plugin
+
+1. Place your plugin in one of the directories scanned by the plugin system
+2. Restart changedetection.io
+3. Go to the Edit page of a watch and check the Stats tab to see your content

changedetectionio/api/Import.py

@@ -0,0 +1,67 @@
+from changedetectionio.strtobool import strtobool
+from flask_restful import abort, Resource
+from flask import request
+from functools import wraps
+from . import auth, validate_openapi_request
+from ..validate_url import is_safe_valid_url
+
+
+def default_content_type(content_type='text/plain'):
+    """Decorator to set a default Content-Type header if none is provided."""
+    def decorator(f):
+        @wraps(f)
+        def wrapper(*args, **kwargs):
+            if not request.content_type:
+                # Set default content type in the request environment
+                request.environ['CONTENT_TYPE'] = content_type
+            return f(*args, **kwargs)
+        return wrapper
+    return decorator
+
+
+class Import(Resource):
+    def __init__(self, **kwargs):
+        # datastore is a black box dependency
+        self.datastore = kwargs['datastore']
+
+    @auth.check_token
+    @default_content_type('text/plain') #3547 #3542
+    @validate_openapi_request('importWatches')
+    def post(self):
+        """Import a list of watched URLs."""
+
+        extras = {}
+
+        if request.args.get('proxy'):
+            plist = self.datastore.proxy_list
+            if not request.args.get('proxy') in plist:
+                return "Invalid proxy choice, currently supported proxies are '{}'".format(', '.join(plist)), 400
+            else:
+                extras['proxy'] = request.args.get('proxy')
+
+        dedupe = strtobool(request.args.get('dedupe', 'true'))
+
+        tags = request.args.get('tag')
+        tag_uuids = request.args.get('tag_uuids')
+
+        if tag_uuids:
+            tag_uuids = tag_uuids.split(',')
+
+        urls = request.get_data().decode('utf8').splitlines()
+        added = []
+        for url in urls:
+            url = url.strip()
+            if not len(url):
+                continue
+
+            # If hosts that only contain alphanumerics are allowed ("localhost" for example)
+            if not is_safe_valid_url(url):
+                return f"Invalid or unsupported URL - {url}", 400
+
+            if dedupe and self.datastore.url_exists(url):
+                continue
+
+            new_uuid = self.datastore.add_watch(url=url, extras=extras, tag=tags, tag_uuids=tag_uuids)
+            added.append(new_uuid)
+
+        return added

changedetectionio/api/Notifications.py

@@ -0,0 +1,108 @@
+from flask_expects_json import expects_json
+from flask_restful import Resource, abort
+from flask import request
+from . import auth, validate_openapi_request
+from . import schema_create_notification_urls, schema_delete_notification_urls
+
+class Notifications(Resource):
+    def __init__(self, **kwargs):
+        # datastore is a black box dependency
+        self.datastore = kwargs['datastore']
+
+    @auth.check_token
+    @validate_openapi_request('getNotifications')
+    def get(self):
+        """Return Notification URL List."""
+
+        notification_urls = self.datastore.data.get('settings', {}).get('application', {}).get('notification_urls', [])        
+
+        return {
+                'notification_urls': notification_urls,
+               }, 200
+    
+    @auth.check_token
+    @validate_openapi_request('addNotifications')
+    @expects_json(schema_create_notification_urls)
+    def post(self):
+        """Create Notification URLs."""
+
+        json_data = request.get_json()
+        notification_urls = json_data.get("notification_urls", [])
+
+        from wtforms import ValidationError
+        try:
+            validate_notification_urls(notification_urls)
+        except ValidationError as e:
+            return str(e), 400
+
+        added_urls = []
+
+        for url in notification_urls:
+            clean_url = url.strip()
+            added_url = self.datastore.add_notification_url(clean_url)
+            if added_url:
+                added_urls.append(added_url)
+
+        if not added_urls:
+            return "No valid notification URLs were added", 400
+
+        return {'notification_urls': added_urls}, 201
+    
+    @auth.check_token
+    @validate_openapi_request('replaceNotifications')
+    @expects_json(schema_create_notification_urls)
+    def put(self):
+        """Replace Notification URLs."""
+        json_data = request.get_json()
+        notification_urls = json_data.get("notification_urls", [])
+
+        from wtforms import ValidationError
+        try:
+            validate_notification_urls(notification_urls)
+        except ValidationError as e:
+            return str(e), 400
+        
+        if not isinstance(notification_urls, list):
+            return "Invalid input format", 400
+
+        clean_urls = [url.strip() for url in notification_urls if isinstance(url, str)]
+        self.datastore.data['settings']['application']['notification_urls'] = clean_urls
+        self.datastore.needs_write = True
+
+        return {'notification_urls': clean_urls}, 200
+        
+    @auth.check_token
+    @validate_openapi_request('deleteNotifications')
+    @expects_json(schema_delete_notification_urls)
+    def delete(self):
+        """Delete Notification URLs."""
+
+        json_data = request.get_json()
+        urls_to_delete = json_data.get("notification_urls", [])
+        if not isinstance(urls_to_delete, list):
+            abort(400, message="Expected a list of notification URLs.")
+
+        notification_urls = self.datastore.data['settings']['application'].get('notification_urls', [])
+        deleted = []
+
+        for url in urls_to_delete:
+            clean_url = url.strip()
+            if clean_url in notification_urls:
+                notification_urls.remove(clean_url)
+                deleted.append(clean_url)
+
+        if not deleted:
+            abort(400, message="No matching notification URLs found.")
+
+        self.datastore.data['settings']['application']['notification_urls'] = notification_urls
+        self.datastore.needs_write = True
+
+        return 'OK', 204
+    
+def validate_notification_urls(notification_urls):
+    from changedetectionio.forms import ValidateAppRiseServers
+    validator = ValidateAppRiseServers()
+    class DummyForm: pass
+    dummy_form = DummyForm()
+    field = type("Field", (object,), {"data": notification_urls, "gettext": lambda self, x: x})()
+    validator(dummy_form, field)

changedetectionio/api/Search.py

@@ -0,0 +1,39 @@
+from flask_restful import Resource, abort
+from flask import request
+from . import auth, validate_openapi_request
+
+class Search(Resource):
+    def __init__(self, **kwargs):
+        # datastore is a black box dependency
+        self.datastore = kwargs['datastore']
+
+    @auth.check_token
+    @validate_openapi_request('searchWatches')
+    def get(self):
+        """Search for watches by URL or title text."""
+        query = request.args.get('q', '').strip()
+        tag_limit = request.args.get('tag', '').strip()
+        from changedetectionio.strtobool import strtobool
+        partial = bool(strtobool(request.args.get('partial', '0'))) if 'partial' in request.args else False
+
+        # Require a search query
+        if not query:
+            abort(400, message="Search query 'q' parameter is required")
+
+        # Use the search function from the datastore
+        matching_uuids = self.datastore.search_watches_for_url(query=query, tag_limit=tag_limit, partial=partial)
+
+        # Build the response with watch details
+        results = {}
+        for uuid in matching_uuids:
+            watch = self.datastore.data['watching'].get(uuid)
+            results[uuid] = {
+                'last_changed': watch.last_changed,
+                'last_checked': watch['last_checked'],
+                'last_error': watch['last_error'],
+                'title': watch['title'],
+                'url': watch['url'],
+                'viewed': watch.viewed
+            }
+
+        return results, 200

changedetectionio/api/SystemInfo.py

@@ -0,0 +1,40 @@
+from flask_restful import Resource
+from . import auth, validate_openapi_request
+
+
+class SystemInfo(Resource):
+    def __init__(self, **kwargs):
+        # datastore is a black box dependency
+        self.datastore = kwargs['datastore']
+        self.update_q = kwargs['update_q']
+
+    @auth.check_token
+    @validate_openapi_request('getSystemInfo')
+    def get(self):
+        """Return system info."""
+        import time
+        overdue_watches = []
+
+        # Check all watches and report which have not been checked but should have been
+
+        for uuid, watch in self.datastore.data.get('watching', {}).items():
+            # see if now - last_checked is greater than the time that should have been
+            # this is not super accurate (maybe they just edited it) but better than nothing
+            t = watch.threshold_seconds()
+            if not t:
+                # Use the system wide default
+                t = self.datastore.threshold_seconds
+
+            time_since_check = time.time() - watch.get('last_checked')
+
+            # Allow 5 minutes of grace time before we decide it's overdue
+            if time_since_check - (5 * 60) > t:
+                overdue_watches.append(uuid)
+        from changedetectionio import __version__ as main_version
+        return {
+                   'queue_size': self.update_q.qsize(),
+                   'overdue_watches': overdue_watches,
+                   'uptime': round(time.time() - self.datastore.start_time, 2),
+                   'watch_count': len(self.datastore.data.get('watching', {})),
+                   'version': main_version
+               }, 200

changedetectionio/api/Tags.py

@@ -0,0 +1,119 @@
+from changedetectionio import queuedWatchMetaData
+from changedetectionio import worker_handler
+from flask_expects_json import expects_json
+from flask_restful import abort, Resource
+
+from flask import request
+from . import auth
+
+# Import schemas from __init__.py
+from . import schema_tag, schema_create_tag, schema_update_tag, validate_openapi_request
+
+
+class Tag(Resource):
+    def __init__(self, **kwargs):
+        # datastore is a black box dependency
+        self.datastore = kwargs['datastore']
+        self.update_q = kwargs['update_q']
+
+    # Get information about a single tag
+    # curl http://localhost:5000/api/v1/tag/<string:uuid>
+    @auth.check_token
+    @validate_openapi_request('getTag')
+    def get(self, uuid):
+        """Get data for a single tag/group, toggle notification muting, or recheck all."""
+        from copy import deepcopy
+        tag = deepcopy(self.datastore.data['settings']['application']['tags'].get(uuid))
+        if not tag:
+            abort(404, message=f'No tag exists with the UUID of {uuid}')
+
+        if request.args.get('recheck'):
+            # Recheck all, including muted
+            # Get most overdue first
+            i=0
+            for k in sorted(self.datastore.data['watching'].items(), key=lambda item: item[1].get('last_checked', 0)):
+                watch_uuid = k[0]
+                watch = k[1]
+                if not watch['paused'] and tag['uuid'] not in watch['tags']:
+                    continue
+                worker_handler.queue_item_async_safe(self.update_q, queuedWatchMetaData.PrioritizedItem(priority=1, item={'uuid': watch_uuid}))
+                i+=1
+
+            return f"OK, {i} watches queued", 200
+
+        if request.args.get('muted', '') == 'muted':
+            self.datastore.data['settings']['application']['tags'][uuid]['notification_muted'] = True
+            return "OK", 200
+        elif request.args.get('muted', '') == 'unmuted':
+            self.datastore.data['settings']['application']['tags'][uuid]['notification_muted'] = False
+            return "OK", 200
+
+        return tag
+
+    @auth.check_token
+    @validate_openapi_request('deleteTag')
+    def delete(self, uuid):
+        """Delete a tag/group and remove it from all watches."""
+        if not self.datastore.data['settings']['application']['tags'].get(uuid):
+            abort(400, message='No tag exists with the UUID of {}'.format(uuid))
+
+        # Delete the tag, and any tag reference
+        del self.datastore.data['settings']['application']['tags'][uuid]
+        
+        # Remove tag from all watches
+        for watch_uuid, watch in self.datastore.data['watching'].items():
+            if watch.get('tags') and uuid in watch['tags']:
+                watch['tags'].remove(uuid)
+
+        return 'OK', 204
+
+    @auth.check_token
+    @validate_openapi_request('updateTag')
+    @expects_json(schema_update_tag)
+    def put(self, uuid):
+        """Update tag information."""
+        tag = self.datastore.data['settings']['application']['tags'].get(uuid)
+        if not tag:
+            abort(404, message='No tag exists with the UUID of {}'.format(uuid))
+
+        tag.update(request.json)
+        self.datastore.needs_write_urgent = True
+
+        return "OK", 200
+
+
+    @auth.check_token
+    @validate_openapi_request('createTag')
+    # Only cares for {'title': 'xxxx'}
+    def post(self):
+        """Create a single tag/group."""
+
+        json_data = request.get_json()
+        title = json_data.get("title",'').strip()
+
+
+        new_uuid = self.datastore.add_tag(title=title)
+        if new_uuid:
+            return {'uuid': new_uuid}, 201
+        else:
+            return "Invalid or unsupported tag", 400
+
+class Tags(Resource):
+    def __init__(self, **kwargs):
+        # datastore is a black box dependency
+        self.datastore = kwargs['datastore']
+
+    @auth.check_token
+    @validate_openapi_request('listTags')
+    def get(self):
+        """List tags/groups."""
+        result = {}
+        for uuid, tag in self.datastore.data['settings']['application']['tags'].items():
+            result[uuid] = {
+                'date_created': tag.get('date_created', 0),
+                'notification_muted': tag.get('notification_muted', False),
+                'title': tag.get('title', ''),
+                'uuid': tag.get('uuid')
+            }
+
+        return result, 200

changedetectionio/api/Watch.py

@@ -0,0 +1,403 @@
+import os
+
+from changedetectionio.validate_url import is_safe_valid_url
+
+from flask_expects_json import expects_json
+from changedetectionio import queuedWatchMetaData, strtobool
+from changedetectionio import worker_handler
+from flask_restful import abort, Resource
+from flask import request, make_response, send_from_directory
+from . import auth
+import copy
+
+# Import schemas from __init__.py
+from . import schema, schema_create_watch, schema_update_watch, validate_openapi_request
+from ..notification_service import CUSTOM_LINEBREAK_PLACEHOLDER
+
+
+def validate_time_between_check_required(json_data):
+    """
+    Validate that at least one time interval is specified when not using default settings.
+    Returns None if valid, or error message string if invalid.
+    Defaults to using global settings if time_between_check_use_default is not provided.
+    """
+    # Default to using global settings if not specified
+    use_default = json_data.get('time_between_check_use_default', True)
+
+    # If using default settings, no validation needed
+    if use_default:
+        return None
+
+    # If not using defaults, check if time_between_check exists and has at least one non-zero value
+    time_check = json_data.get('time_between_check')
+    if not time_check:
+        # No time_between_check provided and not using defaults - this is an error
+        return "At least one time interval (weeks, days, hours, minutes, or seconds) must be specified when not using global settings."
+
+    # time_between_check exists, check if it has at least one non-zero value
+    if any([
+        (time_check.get('weeks') or 0) > 0,
+        (time_check.get('days') or 0) > 0,
+        (time_check.get('hours') or 0) > 0,
+        (time_check.get('minutes') or 0) > 0,
+        (time_check.get('seconds') or 0) > 0
+    ]):
+        return None
+
+    # time_between_check exists but all values are 0 or empty - this is an error
+    return "At least one time interval (weeks, days, hours, minutes, or seconds) must be specified when not using global settings."
+
+
+class Watch(Resource):
+    def __init__(self, **kwargs):
+        # datastore is a black box dependency
+        self.datastore = kwargs['datastore']
+        self.update_q = kwargs['update_q']
+
+    # Get information about a single watch, excluding the history list (can be large)
+    # curl http://localhost:5000/api/v1/watch/<string:uuid>
+    # @todo - version2 - ?muted and ?paused should be able to be called together, return the watch struct not "OK"
+    # ?recheck=true
+    @auth.check_token
+    @validate_openapi_request('getWatch')
+    def get(self, uuid):
+        """Get information about a single watch, recheck, pause, or mute."""
+        from copy import deepcopy
+        watch = deepcopy(self.datastore.data['watching'].get(uuid))
+        if not watch:
+            abort(404, message='No watch exists with the UUID of {}'.format(uuid))
+
+        if request.args.get('recheck'):
+            worker_handler.queue_item_async_safe(self.update_q, queuedWatchMetaData.PrioritizedItem(priority=1, item={'uuid': uuid}))
+            return "OK", 200
+        if request.args.get('paused', '') == 'paused':
+            self.datastore.data['watching'].get(uuid).pause()
+            return "OK", 200
+        elif request.args.get('paused', '') == 'unpaused':
+            self.datastore.data['watching'].get(uuid).unpause()
+            return "OK", 200
+        if request.args.get('muted', '') == 'muted':
+            self.datastore.data['watching'].get(uuid).mute()
+            return "OK", 200
+        elif request.args.get('muted', '') == 'unmuted':
+            self.datastore.data['watching'].get(uuid).unmute()
+            return "OK", 200
+
+        # Return without history, get that via another API call
+        # Properties are not returned as a JSON, so add the required props manually
+        watch['history_n'] = watch.history_n
+        # attr .last_changed will check for the last written text snapshot on change
+        watch['last_changed'] = watch.last_changed
+        watch['viewed'] = watch.viewed
+        watch['link'] = watch.link,
+
+        return watch
+
+    @auth.check_token
+    @validate_openapi_request('deleteWatch')
+    def delete(self, uuid):
+        """Delete a watch and related history."""
+        if not self.datastore.data['watching'].get(uuid):
+            abort(400, message='No watch exists with the UUID of {}'.format(uuid))
+
+        self.datastore.delete(uuid)
+        return 'OK', 204
+
+    @auth.check_token
+    @validate_openapi_request('updateWatch')
+    @expects_json(schema_update_watch)
+    def put(self, uuid):
+        """Update watch information."""
+        watch = self.datastore.data['watching'].get(uuid)
+        if not watch:
+            abort(404, message='No watch exists with the UUID of {}'.format(uuid))
+
+        if request.json.get('proxy'):
+            plist = self.datastore.proxy_list
+            if not request.json.get('proxy') in plist:
+                return "Invalid proxy choice, currently supported proxies are '{}'".format(', '.join(plist)), 400
+
+        # Validate time_between_check when not using defaults
+        validation_error = validate_time_between_check_required(request.json)
+        if validation_error:
+            return validation_error, 400
+
+        # XSS etc protection
+        if request.json.get('url') and not is_safe_valid_url(request.json.get('url')):
+            return "Invalid URL", 400
+
+        watch.update(request.json)
+
+        return "OK", 200
+
+
+class WatchHistory(Resource):
+    def __init__(self, **kwargs):
+        # datastore is a black box dependency
+        self.datastore = kwargs['datastore']
+
+    # Get a list of available history for a watch by UUID
+    # curl http://localhost:5000/api/v1/watch/<string:uuid>/history
+    @auth.check_token
+    @validate_openapi_request('getWatchHistory')
+    def get(self, uuid):
+        """Get a list of all historical snapshots available for a watch."""
+        watch = self.datastore.data['watching'].get(uuid)
+        if not watch:
+            abort(404, message='No watch exists with the UUID of {}'.format(uuid))
+        return watch.history, 200
+
+
+class WatchSingleHistory(Resource):
+    def __init__(self, **kwargs):
+        # datastore is a black box dependency
+        self.datastore = kwargs['datastore']
+
+    @auth.check_token
+    @validate_openapi_request('getWatchSnapshot')
+    def get(self, uuid, timestamp):
+        """Get single snapshot from watch."""
+        watch = self.datastore.data['watching'].get(uuid)
+        if not watch:
+            abort(404, message=f"No watch exists with the UUID of {uuid}")
+
+        if not len(watch.history):
+            abort(404, message=f"Watch found but no history exists for the UUID {uuid}")
+
+        if timestamp == 'latest':
+            timestamp = list(watch.history.keys())[-1]
+
+        if request.args.get('html'):
+            content = watch.get_fetched_html(timestamp)
+            if content:
+                response = make_response(content, 200)
+                response.mimetype = "text/html"
+            else:
+                response = make_response("No content found", 404)
+                response.mimetype = "text/plain"
+        else:
+            content = watch.get_history_snapshot(timestamp)
+            response = make_response(content, 200)
+            response.mimetype = "text/plain"
+
+        return response
+
+class WatchHistoryDiff(Resource):
+    def __init__(self, **kwargs):
+        # datastore is a black box dependency
+        self.datastore = kwargs['datastore']
+
+    @auth.check_token
+    @validate_openapi_request('getWatchHistoryDiff')
+    def get(self, uuid, from_timestamp, to_timestamp):
+        """Generate diff between two historical snapshots."""
+        from changedetectionio import diff
+        from changedetectionio.notification.handler import apply_service_tweaks
+
+        watch = self.datastore.data['watching'].get(uuid)
+        if not watch:
+            abort(404, message=f"No watch exists with the UUID of {uuid}")
+
+        if not len(watch.history):
+            abort(404, message=f"Watch found but no history exists for the UUID {uuid}")
+
+        history_keys = list(watch.history.keys())
+
+        # Handle 'latest' keyword for to_timestamp
+        if to_timestamp == 'latest':
+            to_timestamp = history_keys[-1]
+
+        # Handle 'previous' keyword for from_timestamp (second-most-recent)
+        if from_timestamp == 'previous':
+            if len(history_keys) < 2:
+                abort(404, message=f"Not enough history entries. Need at least 2 snapshots for 'previous'")
+            from_timestamp = history_keys[-2]
+
+        # Validate timestamps exist
+        if from_timestamp not in watch.history:
+            abort(404, message=f"From timestamp {from_timestamp} not found in watch history")
+        if to_timestamp not in watch.history:
+            abort(404, message=f"To timestamp {to_timestamp} not found in watch history")
+
+        # Get the format parameter (default to 'text')
+        output_format = request.args.get('format', 'text').lower()
+
+        # Validate format
+        valid_formats = ['text', 'html', 'htmlcolor']
+        if output_format not in valid_formats:
+            abort(400, message=f"Invalid format. Must be one of: {', '.join(valid_formats)}")
+
+        # Get the word_diff parameter (default to False - line-level mode)
+        word_diff = strtobool(request.args.get('word_diff', 'false'))
+
+        # Get the no_markup parameter (default to False)
+        no_markup = strtobool(request.args.get('no_markup', 'false'))
+
+        # Retrieve snapshot contents
+        from_version_file_contents = watch.get_history_snapshot(from_timestamp)
+        to_version_file_contents = watch.get_history_snapshot(to_timestamp)
+
+        # Get diff preferences (using defaults similar to the existing code)
+        diff_prefs = {
+            'diff_ignoreWhitespace': False,
+            'diff_changesOnly': True
+        }
+
+        # Generate the diff
+        content = diff.render_diff(
+            previous_version_file_contents=from_version_file_contents,
+            newest_version_file_contents=to_version_file_contents,
+            ignore_junk=diff_prefs.get('diff_ignoreWhitespace'),
+            include_equal=not diff_prefs.get('diff_changesOnly'),
+            word_diff=word_diff,
+        )
+
+        # Skip formatting if no_markup is set
+        if no_markup:
+            mimetype = "text/plain"
+        else:
+            # Apply formatting based on the requested format
+            if output_format == 'htmlcolor':
+                from changedetectionio.notification.handler import apply_html_color_to_body
+                content = apply_html_color_to_body(n_body=content)
+                mimetype = "text/html"
+            else:
+                # Apply service tweaks for text/html formats
+                # Pass empty URL and title as they're not used for the placeholder replacement we need
+                _, content, _ = apply_service_tweaks(
+                    url='',
+                    n_body=content,
+                    n_title='',
+                    requested_output_format=output_format
+                )
+                mimetype = "text/html" if output_format == 'html' else "text/plain"
+
+            import re
+            if 'html' in output_format:
+                content = re.sub(
+                    re.escape(CUSTOM_LINEBREAK_PLACEHOLDER) + r'\r?\n?',
+                    '<br>\\r\\n',
+                    content
+                )
+            else:
+                # texty types
+                content = re.sub(
+                    re.escape(CUSTOM_LINEBREAK_PLACEHOLDER) + r'\r?\n?',
+                    '\\r\\n',
+                    content
+                )
+
+        response = make_response(content, 200)
+        response.mimetype = mimetype
+        return response
+
+
+class WatchFavicon(Resource):
+    def __init__(self, **kwargs):
+        # datastore is a black box dependency
+        self.datastore = kwargs['datastore']
+
+    @auth.check_token
+    @validate_openapi_request('getWatchFavicon')
+    def get(self, uuid):
+        """Get favicon for a watch."""
+        watch = self.datastore.data['watching'].get(uuid)
+        if not watch:
+            abort(404, message=f"No watch exists with the UUID of {uuid}")
+
+        favicon_filename = watch.get_favicon_filename()
+        if favicon_filename:
+            try:
+                import magic
+                mime = magic.from_file(
+                    os.path.join(watch.watch_data_dir, favicon_filename),
+                    mime=True
+                )
+            except ImportError:
+                # Fallback, no python-magic
+                import mimetypes
+                mime, encoding = mimetypes.guess_type(favicon_filename)
+
+            response = make_response(send_from_directory(watch.watch_data_dir, favicon_filename))
+            response.headers['Content-type'] = mime
+            response.headers['Cache-Control'] = 'max-age=300, must-revalidate'  # Cache for 5 minutes, then revalidate
+            return response
+
+        abort(404, message=f'No Favicon available for {uuid}')
+
+
+class CreateWatch(Resource):
+    def __init__(self, **kwargs):
+        # datastore is a black box dependency
+        self.datastore = kwargs['datastore']
+        self.update_q = kwargs['update_q']
+
+    @auth.check_token
+    @validate_openapi_request('createWatch')
+    @expects_json(schema_create_watch)
+    def post(self):
+        """Create a single watch."""
+
+        json_data = request.get_json()
+        url = json_data['url'].strip()
+
+        if not is_safe_valid_url(url):
+            return "Invalid or unsupported URL", 400
+
+        if json_data.get('proxy'):
+            plist = self.datastore.proxy_list
+            if not json_data.get('proxy') in plist:
+                return "Invalid proxy choice, currently supported proxies are '{}'".format(', '.join(plist)), 400
+
+        # Validate time_between_check when not using defaults
+        validation_error = validate_time_between_check_required(json_data)
+        if validation_error:
+            return validation_error, 400
+
+        extras = copy.deepcopy(json_data)
+
+        # Because we renamed 'tag' to 'tags' but don't want to change the API (can do this in v2 of the API)
+        tags = None
+        if extras.get('tag'):
+            tags = extras.get('tag')
+            del extras['tag']
+
+        del extras['url']
+
+        new_uuid = self.datastore.add_watch(url=url, extras=extras, tag=tags)
+        if new_uuid:
+            worker_handler.queue_item_async_safe(self.update_q, queuedWatchMetaData.PrioritizedItem(priority=1, item={'uuid': new_uuid}))
+            return {'uuid': new_uuid}, 201
+        else:
+            return "Invalid or unsupported URL", 400
+
+    @auth.check_token
+    @validate_openapi_request('listWatches')
+    def get(self):
+        """List watches."""
+        list = {}
+
+        tag_limit = request.args.get('tag', '').lower()
+        for uuid, watch in self.datastore.data['watching'].items():
+            # Watch tags by name (replace the other calls?)
+            tags = self.datastore.get_all_tags_for_watch(uuid=uuid)
+            if tag_limit and not any(v.get('title').lower() == tag_limit for k, v in tags.items()):
+                continue
+
+            list[uuid] = {
+                'last_changed': watch.last_changed,
+                'last_checked': watch['last_checked'],
+                'last_error': watch['last_error'],
+                'link': watch.link,
+                'page_title': watch['page_title'],
+                'title': watch['title'],
+                'url': watch['url'],
+                'viewed': watch.viewed
+            }
+
+        if request.args.get('recheck_all'):
+            for uuid in self.datastore.data['watching'].keys():
+                worker_handler.queue_item_async_safe(self.update_q, queuedWatchMetaData.PrioritizedItem(priority=1, item={'uuid': uuid}))
+            return {'status': "OK"}, 200
+
+        return list, 200

changedetectionio/api/__init__.py

@@ -0,0 +1,86 @@
+import copy
+import functools
+from flask import request, abort
+from loguru import logger
+from . import api_schema
+from ..model import watch_base
+
+# Build a JSON Schema atleast partially based on our Watch model
+watch_base_config = watch_base()
+schema = api_schema.build_watch_json_schema(watch_base_config)
+
+schema_create_watch = copy.deepcopy(schema)
+schema_create_watch['required'] = ['url']
+del schema_create_watch['properties']['last_viewed']
+
+schema_update_watch = copy.deepcopy(schema)
+schema_update_watch['additionalProperties'] = False
+
+# Tag schema is also based on watch_base since Tag inherits from it
+schema_tag = copy.deepcopy(schema)
+schema_create_tag = copy.deepcopy(schema_tag)
+schema_create_tag['required'] = ['title']
+schema_update_tag = copy.deepcopy(schema_tag)
+schema_update_tag['additionalProperties'] = False
+
+schema_notification_urls = copy.deepcopy(schema)
+schema_create_notification_urls = copy.deepcopy(schema_notification_urls)
+schema_create_notification_urls['required'] = ['notification_urls']
+schema_delete_notification_urls = copy.deepcopy(schema_notification_urls)
+schema_delete_notification_urls['required'] = ['notification_urls']
+
+@functools.cache
+def get_openapi_spec():
+    """Lazy load OpenAPI spec and dependencies only when validation is needed."""
+    import os
+    import yaml  # Lazy import - only loaded when API validation is actually used
+    from openapi_core import OpenAPI  # Lazy import - saves ~10.7 MB on startup
+
+    spec_path = os.path.join(os.path.dirname(__file__), '../../docs/api-spec.yaml')
+    if not os.path.exists(spec_path):
+        # Possibly for pip3 packages
+        spec_path = os.path.join(os.path.dirname(__file__), '../docs/api-spec.yaml')
+
+    with open(spec_path, 'r') as f:
+        spec_dict = yaml.safe_load(f)
+    _openapi_spec = OpenAPI.from_dict(spec_dict)
+    return _openapi_spec
+
+def validate_openapi_request(operation_id):
+    """Decorator to validate incoming requests against OpenAPI spec."""
+    def decorator(f):
+        @functools.wraps(f)
+        def wrapper(*args, **kwargs):
+            from werkzeug.exceptions import BadRequest
+            try:
+                # Skip OpenAPI validation for GET requests since they don't have request bodies
+                if request.method.upper() != 'GET':
+                    # Lazy import - only loaded when actually validating a request
+                    from openapi_core.contrib.flask import FlaskOpenAPIRequest
+
+                    spec = get_openapi_spec()
+                    openapi_request = FlaskOpenAPIRequest(request)
+                    result = spec.unmarshal_request(openapi_request)
+                    if result.errors:
+                        error_details = []
+                        for error in result.errors:
+                            error_details.append(str(error))
+                        raise BadRequest(f"OpenAPI validation failed: {error_details}")
+            except BadRequest:
+                # Re-raise BadRequest exceptions (validation failures)
+                raise
+            except Exception as e:
+                # If OpenAPI spec loading fails, log but don't break existing functionality
+                logger.critical(f"OpenAPI validation warning for {operation_id}: {e}")
+                abort(500)
+            return f(*args, **kwargs)
+        return wrapper
+    return decorator
+
+# Import all API resources
+from .Watch import Watch, WatchHistory, WatchSingleHistory, WatchHistoryDiff, CreateWatch, WatchFavicon
+from .Tags import Tags, Tag
+from .Import import Import
+from .SystemInfo import SystemInfo
+from .Notifications import Notifications
+

changedetectionio/api/api_schema.py

@@ -1,5 +1,7 @@
 # Responsible for building the storage dict into a set of rules ("JSON Schema") acceptable via the API
 # Probably other ways to solve this when the backend switches to some ORM
+from changedetectionio.notification import valid_notification_formats
+
 
 def build_time_between_check_json_schema():
     # Setup time between check schema
@@ -76,6 +78,13 @@ def build_watch_json_schema(d):
               ]:
         schema['properties'][v]['anyOf'].append({'type': 'string', "maxLength": 5000})
 
+    for v in ['last_viewed']:
+        schema['properties'][v] = {
+            "type": "integer",
+            "description": "Unix timestamp in seconds of the last time the watch was viewed.",
+            "minimum": 0
+        }
+
     # None or Boolean
     schema['properties']['track_ldjson_price_data']['anyOf'].append({'type': 'boolean'})
 
@@ -98,8 +107,6 @@ def build_watch_json_schema(d):
         }
     }
 
-    from changedetectionio.notification import valid_notification_formats
-
     schema['properties']['notification_format'] = {'type': 'string',
                                                    'enum': list(valid_notification_formats.keys())
                                                    }
@@ -112,6 +119,41 @@ def build_watch_json_schema(d):
 
     schema['properties']['time_between_check'] = build_time_between_check_json_schema()
 
+    schema['properties']['time_between_check_use_default'] = {
+        "type": "boolean",
+        "default": True,
+        "description": "Whether to use global settings for time between checks - defaults to true if not set"
+    }
+
+    schema['properties']['browser_steps'] = {
+        "anyOf": [
+            {
+                "type": "array",
+                "items": {
+                    "type": "object",
+                    "properties": {
+                        "operation": {
+                            "type": ["string", "null"],
+                            "maxLength": 5000  # Allows null and any string up to 5000 chars (including "")
+                        },
+                        "selector": {
+                            "type": ["string", "null"],
+                            "maxLength": 5000
+                        },
+                        "optional_value": {
+                            "type": ["string", "null"],
+                            "maxLength": 5000
+                        }
+                    },
+                    "required": ["operation", "selector", "optional_value"],
+                    "additionalProperties": False  # No extra keys allowed
+                }
+            },
+            {"type": "null"},  # Allows null for `browser_steps`
+            {"type": "array", "maxItems": 0}  # Allows empty array []
+        ]
+    }
+
     # headers ?
     return schema
 

changedetectionio/api/api_v1.py

@@ -1,345 +0,0 @@
-import os
-from distutils.util import strtobool
-
-from flask_expects_json import expects_json
-from changedetectionio import queuedWatchMetaData
-from flask_restful import abort, Resource
-from flask import request, make_response
-import validators
-from . import auth
-import copy
-
-# See docs/README.md for rebuilding the docs/apidoc information
-
-from . import api_schema
-
-# Build a JSON Schema atleast partially based on our Watch model
-from changedetectionio.model.Watch import base_config as watch_base_config
-schema = api_schema.build_watch_json_schema(watch_base_config)
-
-schema_create_watch = copy.deepcopy(schema)
-schema_create_watch['required'] = ['url']
-
-schema_update_watch = copy.deepcopy(schema)
-schema_update_watch['additionalProperties'] = False
-
-class Watch(Resource):
-    def __init__(self, **kwargs):
-        # datastore is a black box dependency
-        self.datastore = kwargs['datastore']
-        self.update_q = kwargs['update_q']
-
-    # Get information about a single watch, excluding the history list (can be large)
-    # curl http://localhost:4000/api/v1/watch/<string:uuid>
-    # @todo - version2 - ?muted and ?paused should be able to be called together, return the watch struct not "OK"
-    # ?recheck=true
-    @auth.check_token
-    def get(self, uuid):
-        """
-        @api {get} /api/v1/watch/:uuid Single watch - get data, recheck, pause, mute.
-        @apiDescription Retrieve watch information and set muted/paused status
-        @apiExample {curl} Example usage:
-            curl http://localhost:4000/api/v1/watch/cc0cfffa-f449-477b-83ea-0caafd1dc091  -H"x-api-key:813031b16330fe25e3780cf0325daa45"
-            curl "http://localhost:4000/api/v1/watch/cc0cfffa-f449-477b-83ea-0caafd1dc091?muted=unmuted"  -H"x-api-key:813031b16330fe25e3780cf0325daa45"
-            curl "http://localhost:4000/api/v1/watch/cc0cfffa-f449-477b-83ea-0caafd1dc091?paused=unpaused"  -H"x-api-key:813031b16330fe25e3780cf0325daa45"
-        @apiName Watch
-        @apiGroup Watch
-        @apiParam {uuid} uuid Watch unique ID.
-        @apiQuery {Boolean} [recheck] Recheck this watch `recheck=1`
-        @apiQuery {String} [paused] =`paused` or =`unpaused` , Sets the PAUSED state
-        @apiQuery {String} [muted] =`muted` or =`unmuted` , Sets the MUTE NOTIFICATIONS state
-        @apiSuccess (200) {String} OK When paused/muted/recheck operation OR full JSON object of the watch
-        @apiSuccess (200) {JSON} WatchJSON JSON Full JSON object of the watch
-        """
-        from copy import deepcopy
-        watch = deepcopy(self.datastore.data['watching'].get(uuid))
-        if not watch:
-            abort(404, message='No watch exists with the UUID of {}'.format(uuid))
-
-        if request.args.get('recheck'):
-            self.update_q.put(queuedWatchMetaData.PrioritizedItem(priority=1, item={'uuid': uuid, 'skip_when_checksum_same': True}))
-            return "OK", 200
-        if request.args.get('paused', '') == 'paused':
-            self.datastore.data['watching'].get(uuid).pause()
-            return "OK", 200
-        elif request.args.get('paused', '') == 'unpaused':
-            self.datastore.data['watching'].get(uuid).unpause()
-            return "OK", 200
-        if request.args.get('muted', '') == 'muted':
-            self.datastore.data['watching'].get(uuid).mute()
-            return "OK", 200
-        elif request.args.get('muted', '') == 'unmuted':
-            self.datastore.data['watching'].get(uuid).unmute()
-            return "OK", 200
-
-        # Return without history, get that via another API call
-        # Properties are not returned as a JSON, so add the required props manually
-        watch['history_n'] = watch.history_n
-        watch['last_changed'] = watch.last_changed
-
-        return watch
-
-    @auth.check_token
-    def delete(self, uuid):
-        """
-        @api {delete} /api/v1/watch/:uuid Delete a watch and related history
-        @apiExample {curl} Example usage:
-            curl http://localhost:4000/api/v1/watch/cc0cfffa-f449-477b-83ea-0caafd1dc091 -X DELETE -H"x-api-key:813031b16330fe25e3780cf0325daa45"
-        @apiParam {uuid} uuid Watch unique ID.
-        @apiName Delete
-        @apiGroup Watch
-        @apiSuccess (200) {String} OK Was deleted
-        """
-        if not self.datastore.data['watching'].get(uuid):
-            abort(400, message='No watch exists with the UUID of {}'.format(uuid))
-
-        self.datastore.delete(uuid)
-        return 'OK', 204
-
-    @auth.check_token
-    @expects_json(schema_update_watch)
-    def put(self, uuid):
-        """
-        @api {put} /api/v1/watch/:uuid Update watch information
-        @apiExample {curl} Example usage:
-            Update (PUT)
-            curl http://localhost:4000/api/v1/watch/cc0cfffa-f449-477b-83ea-0caafd1dc091 -X PUT -H"x-api-key:813031b16330fe25e3780cf0325daa45" -H "Content-Type: application/json" -d '{"url": "https://my-nice.com" , "tag": "new list"}'
-
-        @apiDescription Updates an existing watch using JSON, accepts the same structure as returned in <a href="#api-Watch-Watch">get single watch information</a>
-        @apiParam {uuid} uuid Watch unique ID.
-        @apiName Update a watch
-        @apiGroup Watch
-        @apiSuccess (200) {String} OK Was updated
-        @apiSuccess (500) {String} ERR Some other error
-        """
-        watch = self.datastore.data['watching'].get(uuid)
-        if not watch:
-            abort(404, message='No watch exists with the UUID of {}'.format(uuid))
-
-        if request.json.get('proxy'):
-            plist = self.datastore.proxy_list
-            if not request.json.get('proxy') in plist:
-                return "Invalid proxy choice, currently supported proxies are '{}'".format(', '.join(plist)), 400
-
-        watch.update(request.json)
-
-        return "OK", 200
-
-
-class WatchHistory(Resource):
-    def __init__(self, **kwargs):
-        # datastore is a black box dependency
-        self.datastore = kwargs['datastore']
-
-    # Get a list of available history for a watch by UUID
-    # curl http://localhost:4000/api/v1/watch/<string:uuid>/history
-    def get(self, uuid):
-        """
-        @api {get} /api/v1/watch/<string:uuid>/history Get a list of all historical snapshots available for a watch
-        @apiDescription Requires `uuid`, returns list
-        @apiExample {curl} Example usage:
-            curl http://localhost:4000/api/v1/watch/cc0cfffa-f449-477b-83ea-0caafd1dc091/history -H"x-api-key:813031b16330fe25e3780cf0325daa45" -H "Content-Type: application/json"
-            {
-                "1676649279": "/tmp/data/6a4b7d5c-fee4-4616-9f43-4ac97046b595/cb7e9be8258368262246910e6a2a4c30.txt",
-                "1677092785": "/tmp/data/6a4b7d5c-fee4-4616-9f43-4ac97046b595/e20db368d6fc633e34f559ff67bb4044.txt",
-                "1677103794": "/tmp/data/6a4b7d5c-fee4-4616-9f43-4ac97046b595/02efdd37dacdae96554a8cc85dc9c945.txt"
-            }
-        @apiName Get list of available stored snapshots for watch
-        @apiGroup Watch History
-        @apiSuccess (200) {String} OK
-        @apiSuccess (404) {String} ERR Not found
-        """
-        watch = self.datastore.data['watching'].get(uuid)
-        if not watch:
-            abort(404, message='No watch exists with the UUID of {}'.format(uuid))
-        return watch.history, 200
-
-
-class WatchSingleHistory(Resource):
-    def __init__(self, **kwargs):
-        # datastore is a black box dependency
-        self.datastore = kwargs['datastore']
-
-    @auth.check_token
-    def get(self, uuid, timestamp):
-        """
-        @api {get} /api/v1/watch/<string:uuid>/history/<int:timestamp> Get single snapshot from watch
-        @apiDescription Requires watch `uuid` and `timestamp`. `timestamp` of "`latest`" for latest available snapshot, or <a href="#api-Watch_History-Get_list_of_available_stored_snapshots_for_watch">use the list returned here</a>
-        @apiExample {curl} Example usage:
-            curl http://localhost:4000/api/v1/watch/cc0cfffa-f449-477b-83ea-0caafd1dc091/history/1677092977 -H"x-api-key:813031b16330fe25e3780cf0325daa45" -H "Content-Type: application/json"
-        @apiName Get single snapshot content
-        @apiGroup Watch History
-        @apiSuccess (200) {String} OK
-        @apiSuccess (404) {String} ERR Not found
-        """
-        watch = self.datastore.data['watching'].get(uuid)
-        if not watch:
-            abort(404, message='No watch exists with the UUID of {}'.format(uuid))
-
-        if not len(watch.history):
-            abort(404, message='Watch found but no history exists for the UUID {}'.format(uuid))
-
-        if timestamp == 'latest':
-            timestamp = list(watch.history.keys())[-1]
-
-        content = watch.get_history_snapshot(timestamp)
-
-        response = make_response(content, 200)
-        response.mimetype = "text/plain"
-        return response
-
-
-class CreateWatch(Resource):
-    def __init__(self, **kwargs):
-        # datastore is a black box dependency
-        self.datastore = kwargs['datastore']
-        self.update_q = kwargs['update_q']
-
-    @auth.check_token
-    @expects_json(schema_create_watch)
-    def post(self):
-        """
-        @api {post} /api/v1/watch Create a single watch
-        @apiDescription Requires atleast `url` set, can accept the same structure as <a href="#api-Watch-Watch">get single watch information</a> to create.
-        @apiExample {curl} Example usage:
-            curl http://localhost:4000/api/v1/watch -H"x-api-key:813031b16330fe25e3780cf0325daa45" -H "Content-Type: application/json" -d '{"url": "https://my-nice.com" , "tag": "nice list"}'
-        @apiName Create
-        @apiGroup Watch
-        @apiSuccess (200) {String} OK Was created
-        @apiSuccess (500) {String} ERR Some other error
-        """
-
-        json_data = request.get_json()
-        url = json_data['url'].strip()
-
-        # If hosts that only contain alphanumerics are allowed ("localhost" for example)
-        allow_simplehost = not strtobool(os.getenv('BLOCK_SIMPLEHOSTS', 'False'))
-        if not validators.url(url, simple_host=allow_simplehost):
-            return "Invalid or unsupported URL", 400
-
-        if json_data.get('proxy'):
-            plist = self.datastore.proxy_list
-            if not json_data.get('proxy') in plist:
-                return "Invalid proxy choice, currently supported proxies are '{}'".format(', '.join(plist)), 400
-
-        extras = copy.deepcopy(json_data)
-
-        # Because we renamed 'tag' to 'tags' but don't want to change the API (can do this in v2 of the API)
-        tags = None
-        if extras.get('tag'):
-            tags = extras.get('tag')
-            del extras['tag']
-
-        del extras['url']
-
-        new_uuid = self.datastore.add_watch(url=url, extras=extras, tag=tags)
-        if new_uuid:
-            self.update_q.put(queuedWatchMetaData.PrioritizedItem(priority=1, item={'uuid': new_uuid, 'skip_when_checksum_same': True}))
-            return {'uuid': new_uuid}, 201
-        else:
-            return "Invalid or unsupported URL", 400
-
-    @auth.check_token
-    def get(self):
-        """
-        @api {get} /api/v1/watch List watches
-        @apiDescription Return concise list of available watches and some very basic info
-        @apiExample {curl} Example usage:
-            curl http://localhost:4000/api/v1/watch -H"x-api-key:813031b16330fe25e3780cf0325daa45"
-            {
-                "6a4b7d5c-fee4-4616-9f43-4ac97046b595": {
-                    "last_changed": 1677103794,
-                    "last_checked": 1677103794,
-                    "last_error": false,
-                    "title": "",
-                    "url": "http://www.quotationspage.com/random.php"
-                },
-                "e6f5fd5c-dbfe-468b-b8f3-f9d6ff5ad69b": {
-                    "last_changed": 0,
-                    "last_checked": 1676662819,
-                    "last_error": false,
-                    "title": "QuickLook",
-                    "url": "https://github.com/QL-Win/QuickLook/tags"
-                }
-            }
-
-        @apiParam {String} [recheck_all]       Optional Set to =1 to force recheck of all watches
-        @apiParam {String} [tag]               Optional name of tag to limit results
-        @apiName ListWatches
-        @apiGroup Watch Management
-        @apiSuccess (200) {String} OK JSON dict
-        """
-        list = {}
-
-        tag_limit = request.args.get('tag', '').lower()
-
-
-        for uuid, watch in self.datastore.data['watching'].items():
-            # Watch tags by name (replace the other calls?)
-            tags = self.datastore.get_all_tags_for_watch(uuid=uuid)
-            if tag_limit and not any(v.get('title').lower() == tag_limit for k, v in tags.items()):
-                continue
-
-            list[uuid] = {'url': watch['url'],
-                       'title': watch['title'],
-                       'last_checked': watch['last_checked'],
-                       'last_changed': watch.last_changed,
-                       'last_error': watch['last_error']}
-
-        if request.args.get('recheck_all'):
-            for uuid in self.datastore.data['watching'].keys():
-                self.update_q.put(queuedWatchMetaData.PrioritizedItem(priority=1, item={'uuid': uuid, 'skip_when_checksum_same': True}))
-            return {'status': "OK"}, 200
-
-        return list, 200
-
-class SystemInfo(Resource):
-    def __init__(self, **kwargs):
-        # datastore is a black box dependency
-        self.datastore = kwargs['datastore']
-        self.update_q = kwargs['update_q']
-
-    @auth.check_token
-    def get(self):
-        """
-        @api {get} /api/v1/systeminfo Return system info
-        @apiDescription Return some info about the current system state
-        @apiExample {curl} Example usage:
-            curl http://localhost:4000/api/v1/systeminfo -H"x-api-key:813031b16330fe25e3780cf0325daa45"
-            HTTP/1.0 200
-            {
-                'queue_size': 10 ,
-                'overdue_watches': ["watch-uuid-list"],
-                'uptime': 38344.55,
-                'watch_count': 800,
-                'version': "0.40.1"
-            }
-        @apiName Get Info
-        @apiGroup System Information
-        """
-        import time
-        overdue_watches = []
-
-        # Check all watches and report which have not been checked but should have been
-
-        for uuid, watch in self.datastore.data.get('watching', {}).items():
-            # see if now - last_checked is greater than the time that should have been
-            # this is not super accurate (maybe they just edited it) but better than nothing
-            t = watch.threshold_seconds()
-            if not t:
-                # Use the system wide default
-                t = self.datastore.threshold_seconds
-
-            time_since_check = time.time() - watch.get('last_checked')
-
-            # Allow 5 minutes of grace time before we decide it's overdue
-            if time_since_check - (5 * 60) > t:
-                overdue_watches.append(uuid)
-        from changedetectionio import __version__ as main_version
-        return {
-                   'queue_size': self.update_q.qsize(),
-                   'overdue_watches': overdue_watches,
-                   'uptime': round(time.time() - self.datastore.start_time, 2),
-                   'watch_count': len(self.datastore.data.get('watching', {})),
-                   'version': main_version
-               }, 200

changedetectionio/api/auth.py

@@ -11,22 +11,14 @@ def check_token(f):
         datastore = args[0].datastore
 
         config_api_token_enabled = datastore.data['settings']['application'].get('api_access_token_enabled')
-        if not config_api_token_enabled:
-            return
-
-        try:
-            api_key_header = request.headers['x-api-key']
-        except KeyError:
-            return make_response(
-                jsonify("No authorization x-api-key header."), 403
-            )
-
         config_api_token = datastore.data['settings']['application'].get('api_access_token')
 
-        if api_key_header != config_api_token:
-            return make_response(
-                jsonify("Invalid access - API key invalid."), 403
-            )
+        # config_api_token_enabled - a UI option in settings if access should obey the key or not
+        if config_api_token_enabled:
+            if request.headers.get('x-api-key') != config_api_token:
+                return make_response(
+                    jsonify("Invalid access - API key invalid."), 403
+                )
 
         return f(*args, **kwargs)
 

changedetectionio/apprise_asset.py

@@ -1,11 +0,0 @@
-import apprise
-
-# Create our AppriseAsset and populate it with some of our new values:
-# https://github.com/caronc/apprise/wiki/Development_API#the-apprise-asset-object
-asset = apprise.AppriseAsset(
-   image_url_logo='https://raw.githubusercontent.com/dgtlmoon/changedetection.io/master/changedetectionio/static/images/avatar-256x256.png'
-)
-
-asset.app_id = "changedetection.io"
-asset.app_desc = "ChangeDetection.io best and simplest website monitoring and change detection"
-asset.app_url = "https://changedetection.io"

changedetectionio/async_update_worker.py

@@ -0,0 +1,496 @@
+from .processors.exceptions import ProcessorException
+import changedetectionio.content_fetchers.exceptions as content_fetchers_exceptions
+from changedetectionio.processors.text_json_diff.processor import FilterNotFoundInResponse
+from changedetectionio import html_tools
+from changedetectionio.flask_app import watch_check_update
+
+import asyncio
+import importlib
+import os
+import queue
+import time
+
+from loguru import logger
+
+# Async version of update_worker
+# Processes jobs from AsyncSignalPriorityQueue instead of threaded queue
+
+async def async_update_worker(worker_id, q, notification_q, app, datastore):
+    """
+    Async worker function that processes watch check jobs from the queue.
+    
+    Args:
+        worker_id: Unique identifier for this worker
+        q: AsyncSignalPriorityQueue containing jobs to process
+        notification_q: Standard queue for notifications
+        app: Flask application instance
+        datastore: Application datastore
+    """
+    # Set a descriptive name for this task
+    task = asyncio.current_task()
+    if task:
+        task.set_name(f"async-worker-{worker_id}")
+    
+    logger.info(f"Starting async worker {worker_id}")
+    
+    while not app.config.exit.is_set():
+        update_handler = None
+        watch = None
+
+        try:
+            # Use native janus async interface - no threads needed!
+            queued_item_data = await asyncio.wait_for(q.async_get(), timeout=1.0)
+            
+        except asyncio.TimeoutError:
+            # No jobs available, continue loop
+            continue
+        except Exception as e:
+            logger.critical(f"CRITICAL: Worker {worker_id} failed to get queue item: {type(e).__name__}: {e}")
+            
+            # Log queue health for debugging
+            try:
+                queue_size = q.qsize()
+                is_empty = q.empty()
+                logger.critical(f"CRITICAL: Worker {worker_id} queue health - size: {queue_size}, empty: {is_empty}")
+            except Exception as health_e:
+                logger.critical(f"CRITICAL: Worker {worker_id} queue health check failed: {health_e}")
+            
+            await asyncio.sleep(0.1)
+            continue
+        
+        uuid = queued_item_data.item.get('uuid')
+        fetch_start_time = round(time.time())
+        
+        # Mark this UUID as being processed
+        from changedetectionio import worker_handler
+        worker_handler.set_uuid_processing(uuid, processing=True)
+        
+        try:
+            if uuid in list(datastore.data['watching'].keys()) and datastore.data['watching'][uuid].get('url'):
+                changed_detected = False
+                contents = b''
+                process_changedetection_results = True
+                update_obj = {}
+
+                # Clear last errors
+                datastore.data['watching'][uuid]['browser_steps_last_error_step'] = None
+                datastore.data['watching'][uuid]['last_checked'] = fetch_start_time
+
+                watch = datastore.data['watching'].get(uuid)
+
+                logger.info(f"Worker {worker_id} processing watch UUID {uuid} Priority {queued_item_data.priority} URL {watch['url']}")
+
+                try:
+                    watch_check_update.send(watch_uuid=uuid)
+
+                    # Processor is what we are using for detecting the "Change"
+                    processor = watch.get('processor', 'text_json_diff')
+
+                    # Init a new 'difference_detection_processor'
+                    processor_module_name = f"changedetectionio.processors.{processor}.processor"
+                    try:
+                        processor_module = importlib.import_module(processor_module_name)
+                    except ModuleNotFoundError as e:
+                        print(f"Processor module '{processor}' not found.")
+                        raise e
+
+                    update_handler = processor_module.perform_site_check(datastore=datastore,
+                                                                         watch_uuid=uuid)
+
+                    # All fetchers are now async, so call directly
+                    await update_handler.call_browser()
+
+                    # Run change detection (this is synchronous)
+                    changed_detected, update_obj, contents = update_handler.run_changedetection(watch=watch)
+
+                except PermissionError as e:
+                    logger.critical(f"File permission error updating file, watch: {uuid}")
+                    logger.critical(str(e))
+                    process_changedetection_results = False
+
+                except ProcessorException as e:
+                    if e.screenshot:
+                        watch.save_screenshot(screenshot=e.screenshot)
+                    if e.xpath_data:
+                        watch.save_xpath_data(data=e.xpath_data)
+                    datastore.update_watch(uuid=uuid, update_obj={'last_error': e.message})
+                    process_changedetection_results = False
+
+                except content_fetchers_exceptions.ReplyWithContentButNoText as e:
+                    extra_help = ""
+                    if e.has_filters:
+                        has_img = html_tools.include_filters(include_filters='img',
+                                                             html_content=e.html_content)
+                        if has_img:
+                            extra_help = ", it's possible that the filters you have give an empty result or contain only an image."
+                        else:
+                            extra_help = ", it's possible that the filters were found, but contained no usable text."
+
+                    datastore.update_watch(uuid=uuid, update_obj={
+                        'last_error': f"Got HTML content but no text found (With {e.status_code} reply code){extra_help}"
+                    })
+
+                    if e.screenshot:
+                        watch.save_screenshot(screenshot=e.screenshot, as_error=True)
+
+                    if e.xpath_data:
+                        watch.save_xpath_data(data=e.xpath_data)
+                        
+                    process_changedetection_results = False
+
+                except content_fetchers_exceptions.Non200ErrorCodeReceived as e:
+                    if e.status_code == 403:
+                        err_text = "Error - 403 (Access denied) received"
+                    elif e.status_code == 404:
+                        err_text = "Error - 404 (Page not found) received"
+                    elif e.status_code == 407:
+                        err_text = "Error - 407 (Proxy authentication required) received, did you need a username and password for the proxy?"
+                    elif e.status_code == 500:
+                        err_text = "Error - 500 (Internal server error) received from the web site"
+                    else:
+                        extra = ' (Access denied or blocked)' if str(e.status_code).startswith('4') else ''
+                        err_text = f"Error - Request returned a HTTP error code {e.status_code}{extra}"
+
+                    if e.screenshot:
+                        watch.save_screenshot(screenshot=e.screenshot, as_error=True)
+                    if e.xpath_data:
+                        watch.save_xpath_data(data=e.xpath_data, as_error=True)
+                    if e.page_text:
+                        watch.save_error_text(contents=e.page_text)
+
+                    datastore.update_watch(uuid=uuid, update_obj={'last_error': err_text})
+                    process_changedetection_results = False
+
+                except FilterNotFoundInResponse as e:
+                    if not datastore.data['watching'].get(uuid):
+                        continue
+
+                    err_text = "Warning, no filters were found, no change detection ran - Did the page change layout? update your Visual Filter if necessary."
+                    datastore.update_watch(uuid=uuid, update_obj={'last_error': err_text})
+
+                    # Filter wasnt found, but we should still update the visual selector so that they can have a chance to set it up again
+                    if e.screenshot:
+                        watch.save_screenshot(screenshot=e.screenshot)
+
+                    if e.xpath_data:
+                        watch.save_xpath_data(data=e.xpath_data)
+
+                    # Only when enabled, send the notification
+                    if watch.get('filter_failure_notification_send', False):
+                        c = watch.get('consecutive_filter_failures', 0)
+                        c += 1
+                        # Send notification if we reached the threshold?
+                        threshold = datastore.data['settings']['application'].get('filter_failure_notification_threshold_attempts', 0)
+                        logger.debug(f"Filter for {uuid} not found, consecutive_filter_failures: {c} of threshold {threshold}")
+                        if c >= threshold:
+                            if not watch.get('notification_muted'):
+                                logger.debug(f"Sending filter failed notification for {uuid}")
+                                await send_filter_failure_notification(uuid, notification_q, datastore)
+                            c = 0
+                            logger.debug(f"Reset filter failure count back to zero")
+
+                        datastore.update_watch(uuid=uuid, update_obj={'consecutive_filter_failures': c})
+                    else:
+                        logger.trace(f"{uuid} - filter_failure_notification_send not enabled, skipping")
+
+                    process_changedetection_results = False
+
+                except content_fetchers_exceptions.checksumFromPreviousCheckWasTheSame as e:
+                    # Yes fine, so nothing todo, don't continue to process.
+                    process_changedetection_results = False
+                    changed_detected = False
+                    
+                except content_fetchers_exceptions.BrowserConnectError as e:
+                    datastore.update_watch(uuid=uuid,
+                                         update_obj={'last_error': e.msg})
+                    process_changedetection_results = False
+                    
+                except content_fetchers_exceptions.BrowserFetchTimedOut as e:
+                    datastore.update_watch(uuid=uuid,
+                                         update_obj={'last_error': e.msg})
+                    process_changedetection_results = False
+                    
+                except content_fetchers_exceptions.BrowserStepsStepException as e:
+                    if not datastore.data['watching'].get(uuid):
+                        continue
+
+                    error_step = e.step_n + 1
+                    from playwright._impl._errors import TimeoutError, Error
+
+                    # Generally enough info for TimeoutError (couldnt locate the element after default seconds)
+                    err_text = f"Browser step at position {error_step} could not run, check the watch, add a delay if necessary, view Browser Steps to see screenshot at that step."
+
+                    if e.original_e.name == "TimeoutError":
+                        # Just the first line is enough, the rest is the stack trace
+                        err_text += " Could not find the target."
+                    else:
+                        # Other Error, more info is good.
+                        err_text += " " + str(e.original_e).splitlines()[0]
+
+                    logger.debug(f"BrowserSteps exception at step {error_step} {str(e.original_e)}")
+
+                    datastore.update_watch(uuid=uuid,
+                                         update_obj={'last_error': err_text,
+                                                   'browser_steps_last_error_step': error_step})
+
+                    if watch.get('filter_failure_notification_send', False):
+                        c = watch.get('consecutive_filter_failures', 0)
+                        c += 1
+                        # Send notification if we reached the threshold?
+                        threshold = datastore.data['settings']['application'].get('filter_failure_notification_threshold_attempts', 0)
+                        logger.error(f"Step for {uuid} not found, consecutive_filter_failures: {c}")
+                        if threshold > 0 and c >= threshold:
+                            if not watch.get('notification_muted'):
+                                await send_step_failure_notification(watch_uuid=uuid, step_n=e.step_n, notification_q=notification_q, datastore=datastore)
+                            c = 0
+
+                        datastore.update_watch(uuid=uuid, update_obj={'consecutive_filter_failures': c})
+
+                    process_changedetection_results = False
+
+                except content_fetchers_exceptions.EmptyReply as e:
+                    # Some kind of custom to-str handler in the exception handler that does this?
+                    err_text = "EmptyReply - try increasing 'Wait seconds before extracting text', Status Code {}".format(e.status_code)
+                    datastore.update_watch(uuid=uuid, update_obj={'last_error': err_text,
+                                                                'last_check_status': e.status_code})
+                    process_changedetection_results = False
+                    
+                except content_fetchers_exceptions.ScreenshotUnavailable as e:
+                    err_text = "Screenshot unavailable, page did not render fully in the expected time or page was too long - try increasing 'Wait seconds before extracting text'"
+                    datastore.update_watch(uuid=uuid, update_obj={'last_error': err_text,
+                                                                'last_check_status': e.status_code})
+                    process_changedetection_results = False
+                    
+                except content_fetchers_exceptions.JSActionExceptions as e:
+                    err_text = "Error running JS Actions - Page request - "+e.message
+                    if e.screenshot:
+                        watch.save_screenshot(screenshot=e.screenshot, as_error=True)
+                    datastore.update_watch(uuid=uuid, update_obj={'last_error': err_text,
+                                                                'last_check_status': e.status_code})
+                    process_changedetection_results = False
+                    
+                except content_fetchers_exceptions.PageUnloadable as e:
+                    err_text = "Page request from server didnt respond correctly"
+                    if e.message:
+                        err_text = "{} - {}".format(err_text, e.message)
+
+                    if e.screenshot:
+                        watch.save_screenshot(screenshot=e.screenshot, as_error=True)
+
+                    datastore.update_watch(uuid=uuid, update_obj={'last_error': err_text,
+                                                                'last_check_status': e.status_code,
+                                                                'has_ldjson_price_data': None})
+                    process_changedetection_results = False
+                    
+                except content_fetchers_exceptions.BrowserStepsInUnsupportedFetcher as e:
+                    err_text = "This watch has Browser Steps configured and so it cannot run with the 'Basic fast Plaintext/HTTP Client', either remove the Browser Steps or select a Chrome fetcher."
+                    datastore.update_watch(uuid=uuid, update_obj={'last_error': err_text})
+                    process_changedetection_results = False
+                    logger.error(f"Exception (BrowserStepsInUnsupportedFetcher) reached processing watch UUID: {uuid}")
+
+                except Exception as e:
+                    logger.error(f"Worker {worker_id} exception processing watch UUID: {uuid}")
+                    logger.error(str(e))
+                    datastore.update_watch(uuid=uuid, update_obj={'last_error': "Exception: " + str(e)})
+                    process_changedetection_results = False
+
+                else:
+                    if not datastore.data['watching'].get(uuid):
+                        continue
+
+                    update_obj['content-type'] = update_handler.fetcher.get_all_headers().get('content-type', '').lower()
+
+                    if not watch.get('ignore_status_codes'):
+                        update_obj['consecutive_filter_failures'] = 0
+
+                    update_obj['last_error'] = False
+                    cleanup_error_artifacts(uuid, datastore)
+
+                if not datastore.data['watching'].get(uuid):
+                    continue
+
+                if process_changedetection_results:
+                    try:
+                        datastore.update_watch(uuid=uuid, update_obj=update_obj)
+
+                        if changed_detected or not watch.history_n:
+                            if update_handler.screenshot:
+                                watch.save_screenshot(screenshot=update_handler.screenshot)
+
+                            if update_handler.xpath_data:
+                                watch.save_xpath_data(data=update_handler.xpath_data)
+
+                            # Ensure unique timestamp for history
+                            if watch.newest_history_key and int(fetch_start_time) == int(watch.newest_history_key):
+                                logger.warning(f"Timestamp {fetch_start_time} already exists, waiting 1 seconds")
+                                fetch_start_time += 1
+                                await asyncio.sleep(1)
+
+                            watch.save_history_text(contents=contents,
+                                                    timestamp=int(fetch_start_time),
+                                                    snapshot_id=update_obj.get('previous_md5', 'none'))
+
+                            empty_pages_are_a_change = datastore.data['settings']['application'].get('empty_pages_are_a_change', False)
+                            if update_handler.fetcher.content or (not update_handler.fetcher.content and empty_pages_are_a_change):
+                                watch.save_last_fetched_html(contents=update_handler.fetcher.content, timestamp=int(fetch_start_time))
+
+                            # Explicitly delete large content variables to free memory IMMEDIATELY after saving
+                            # These are no longer needed after being saved to history
+                            del contents
+
+                            # Send notifications on second+ check
+                            if watch.history_n >= 2:
+                                logger.info(f"Change detected in UUID {uuid} - {watch['url']}")
+                                if not watch.get('notification_muted'):
+                                    await send_content_changed_notification(uuid, notification_q, datastore)
+
+                    except Exception as e:
+                        logger.critical(f"Worker {worker_id} exception in process_changedetection_results")
+                        logger.critical(str(e))
+                        datastore.update_watch(uuid=uuid, update_obj={'last_error': str(e)})
+
+                # Always record attempt count
+                count = watch.get('check_count', 0) + 1
+
+                # Always record page title (used in notifications, and can change even when the content is the same)
+                try:
+                    page_title = html_tools.extract_title(data=update_handler.fetcher.content)
+                    logger.debug(f"UUID: {uuid} Page <title> is '{page_title}'")
+                    datastore.update_watch(uuid=uuid, update_obj={'page_title': page_title})
+                except Exception as e:
+                    logger.warning(f"UUID: {uuid} Exception when extracting <title> - {str(e)}")
+
+                # Record server header
+                try:
+                    server_header = update_handler.fetcher.headers.get('server', '').strip().lower()[:255]
+                    datastore.update_watch(uuid=uuid, update_obj={'remote_server_reply': server_header})
+                except Exception as e:
+                    pass
+
+                # Store favicon if necessary
+                if update_handler.fetcher.favicon_blob and update_handler.fetcher.favicon_blob.get('base64'):
+                    watch.bump_favicon(url=update_handler.fetcher.favicon_blob.get('url'),
+                                       favicon_base_64=update_handler.fetcher.favicon_blob.get('base64')
+                                       )
+
+                datastore.update_watch(uuid=uuid, update_obj={'fetch_time': round(time.time() - fetch_start_time, 3),
+                                                               'check_count': count})
+
+                # NOW clear fetcher content - after all processing is complete
+                # This is the last point where we need the fetcher data
+                if update_handler and hasattr(update_handler, 'fetcher') and update_handler.fetcher:
+                    update_handler.fetcher.clear_content()
+                    logger.debug(f"Cleared fetcher content for UUID {uuid}")
+
+        except Exception as e:
+            logger.error(f"Worker {worker_id} unexpected error processing {uuid}: {e}")
+            logger.error(f"Worker {worker_id} traceback:", exc_info=True)
+            
+            # Also update the watch with error information
+            if datastore and uuid in datastore.data['watching']:
+                datastore.update_watch(uuid=uuid, update_obj={'last_error': f"Worker error: {str(e)}"})
+        
+        finally:
+            # Always cleanup - this runs whether there was an exception or not
+            if uuid:
+                try:
+                    # Mark UUID as no longer being processed
+                    worker_handler.set_uuid_processing(uuid, processing=False)
+                    
+                    # Send completion signal
+                    if watch:
+                        #logger.info(f"Worker {worker_id} sending completion signal for UUID {watch['uuid']}")
+                        watch_check_update.send(watch_uuid=watch['uuid'])
+
+                    # Explicitly clean up update_handler and all its references
+                    if update_handler:
+                        # Clear fetcher content using the proper method
+                        if hasattr(update_handler, 'fetcher') and update_handler.fetcher:
+                            update_handler.fetcher.clear_content()
+
+                        # Clear processor references
+                        if hasattr(update_handler, 'content_processor'):
+                            update_handler.content_processor = None
+
+                        update_handler = None
+
+                    # Clear local contents variable if it still exists
+                    if 'contents' in locals():
+                        del contents
+
+                    # Note: We don't set watch = None here because:
+                    # 1. watch is just a local reference to datastore.data['watching'][uuid]
+                    # 2. Setting it to None doesn't affect the datastore
+                    # 3. GC can't collect the object anyway (still referenced by datastore)
+                    # 4. It would just cause confusion
+
+                    logger.debug(f"Worker {worker_id} completed watch {uuid} in {time.time()-fetch_start_time:.2f}s")
+                except Exception as cleanup_error:
+                    logger.error(f"Worker {worker_id} error during cleanup: {cleanup_error}")
+            
+            # Brief pause before continuing to avoid tight error loops (only on error)
+            if 'e' in locals():
+                await asyncio.sleep(1.0)
+            else:
+                # Small yield for normal completion
+                await asyncio.sleep(0.01)
+
+        # Check if we should exit
+        if app.config.exit.is_set():
+            break
+
+    # Check if we're in pytest environment - if so, be more gentle with logging
+    import sys
+    in_pytest = "pytest" in sys.modules or "PYTEST_CURRENT_TEST" in os.environ
+    
+    if not in_pytest:
+        logger.info(f"Worker {worker_id} shutting down")
+
+
+def cleanup_error_artifacts(uuid, datastore):
+    """Helper function to clean up error artifacts"""
+    cleanup_files = ["last-error-screenshot.png", "last-error.txt"]
+    for f in cleanup_files:
+        full_path = os.path.join(datastore.datastore_path, uuid, f)
+        if os.path.isfile(full_path):
+            os.unlink(full_path)
+
+
+
+async def send_content_changed_notification(watch_uuid, notification_q, datastore):
+    """Helper function to queue notifications using the new notification service"""
+    try:
+        from changedetectionio.notification_service import create_notification_service
+        
+        # Create notification service instance
+        notification_service = create_notification_service(datastore, notification_q)
+        
+        notification_service.send_content_changed_notification(watch_uuid)
+    except Exception as e:
+        logger.error(f"Error sending notification for {watch_uuid}: {e}")
+
+
+async def send_filter_failure_notification(watch_uuid, notification_q, datastore):
+    """Helper function to send filter failure notifications using the new notification service"""
+    try:
+        from changedetectionio.notification_service import create_notification_service
+        
+        # Create notification service instance
+        notification_service = create_notification_service(datastore, notification_q)
+        
+        notification_service.send_filter_failure_notification(watch_uuid)
+    except Exception as e:
+        logger.error(f"Error sending filter failure notification for {watch_uuid}: {e}")
+
+
+async def send_step_failure_notification(watch_uuid, step_n, notification_q, datastore):
+    """Helper function to send step failure notifications using the new notification service"""
+    try:
+        from changedetectionio.notification_service import create_notification_service
+        
+        # Create notification service instance
+        notification_service = create_notification_service(datastore, notification_q)
+        
+        notification_service.send_step_failure_notification(watch_uuid, step_n)
+    except Exception as e:
+        logger.error(f"Error sending step failure notification for {watch_uuid}: {e}")

changedetectionio/auth_decorator.py

@@ -0,0 +1,33 @@
+import os
+from functools import wraps
+from flask import current_app, redirect, request
+from loguru import logger
+
+def login_optionally_required(func):
+    """
+    If password authentication is enabled, verify the user is logged in.
+    To be used as a decorator for routes that should optionally require login.
+    This version is blueprint-friendly as it uses current_app instead of directly accessing app.
+    """
+    @wraps(func)
+    def decorated_view(*args, **kwargs):
+        from flask import current_app
+        import flask_login
+        from flask_login import current_user
+
+        # Access datastore through the app config
+        datastore = current_app.config['DATASTORE']
+        has_password_enabled = datastore.data['settings']['application'].get('password') or os.getenv("SALTED_PASS", False)
+
+        # Permitted
+        if request.endpoint and 'diff_history_page' in request.endpoint and datastore.data['settings']['application'].get('shared_diff_access'):
+            return func(*args, **kwargs)
+        elif request.method in flask_login.config.EXEMPT_METHODS:
+            return func(*args, **kwargs)
+        elif current_app.config.get('LOGIN_DISABLED'):
+            return func(*args, **kwargs)
+        elif has_password_enabled and not current_user.is_authenticated:
+            return current_app.login_manager.unauthorized()
+
+        return func(*args, **kwargs)
+    return decorated_view

changedetectionio/blueprint/backups/__init__.py

@@ -0,0 +1,164 @@
+import datetime
+import glob
+import threading
+
+from flask import Blueprint, render_template, send_from_directory, flash, url_for, redirect, abort
+import os
+
+from changedetectionio.store import ChangeDetectionStore
+from changedetectionio.flask_app import login_optionally_required
+from loguru import logger
+
+BACKUP_FILENAME_FORMAT = "changedetection-backup-{}.zip"
+
+
+def create_backup(datastore_path, watches: dict):
+    logger.debug("Creating backup...")
+    import zipfile
+    from pathlib import Path
+
+    # create a ZipFile object
+    timestamp = datetime.datetime.now().strftime("%Y%m%d%H%M%S")
+    backupname = BACKUP_FILENAME_FORMAT.format(timestamp)
+    backup_filepath = os.path.join(datastore_path, backupname)
+
+    with zipfile.ZipFile(backup_filepath.replace('.zip', '.tmp'), "w",
+                         compression=zipfile.ZIP_DEFLATED,
+                         compresslevel=8) as zipObj:
+
+        # Add the index
+        zipObj.write(os.path.join(datastore_path, "url-watches.json"), arcname="url-watches.json")
+
+        # Add the flask app secret
+        zipObj.write(os.path.join(datastore_path, "secret.txt"), arcname="secret.txt")
+
+        # Add any data in the watch data directory.
+        for uuid, w in watches.items():
+            for f in Path(w.watch_data_dir).glob('*'):
+                zipObj.write(f,
+                             # Use the full path to access the file, but make the file 'relative' in the Zip.
+                             arcname=os.path.join(f.parts[-2], f.parts[-1]),
+                             compress_type=zipfile.ZIP_DEFLATED,
+                             compresslevel=8)
+
+        # Create a list file with just the URLs, so it's easier to port somewhere else in the future
+        list_file = "url-list.txt"
+        with open(os.path.join(datastore_path, list_file), "w") as f:
+            for uuid in watches:
+                url = watches[uuid]["url"]
+                f.write("{}\r\n".format(url))
+        list_with_tags_file = "url-list-with-tags.txt"
+        with open(
+                os.path.join(datastore_path, list_with_tags_file), "w"
+        ) as f:
+            for uuid in watches:
+                url = watches[uuid].get('url')
+                tag = watches[uuid].get('tags', {})
+                f.write("{} {}\r\n".format(url, tag))
+
+        # Add it to the Zip
+        zipObj.write(
+            os.path.join(datastore_path, list_file),
+            arcname=list_file,
+            compress_type=zipfile.ZIP_DEFLATED,
+            compresslevel=8,
+        )
+        zipObj.write(
+            os.path.join(datastore_path, list_with_tags_file),
+            arcname=list_with_tags_file,
+            compress_type=zipfile.ZIP_DEFLATED,
+            compresslevel=8,
+        )
+
+    # Now it's done, rename it so it shows up finally and its completed being written.
+    os.rename(backup_filepath.replace('.zip', '.tmp'), backup_filepath.replace('.tmp', '.zip'))
+
+
+def construct_blueprint(datastore: ChangeDetectionStore):
+    backups_blueprint = Blueprint('backups', __name__, template_folder="templates")
+    backup_threads = []
+
+    @login_optionally_required
+    @backups_blueprint.route("/request-backup", methods=['GET'])
+    def request_backup():
+        if any(thread.is_alive() for thread in backup_threads):
+            flash("A backup is already running, check back in a few minutes", "error")
+            return redirect(url_for('backups.index'))
+
+        if len(find_backups()) > int(os.getenv("MAX_NUMBER_BACKUPS", 100)):
+            flash("Maximum number of backups reached, please remove some", "error")
+            return redirect(url_for('backups.index'))
+
+        # Be sure we're written fresh
+        datastore.sync_to_json()
+        zip_thread = threading.Thread(target=create_backup, args=(datastore.datastore_path, datastore.data.get("watching")))
+        zip_thread.start()
+        backup_threads.append(zip_thread)
+        flash("Backup building in background, check back in a few minutes.")
+
+        return redirect(url_for('backups.index'))
+
+    def find_backups():
+        backup_filepath = os.path.join(datastore.datastore_path, BACKUP_FILENAME_FORMAT.format("*"))
+        backups = glob.glob(backup_filepath)
+        backup_info = []
+
+        for backup in backups:
+            size = os.path.getsize(backup) / (1024 * 1024)
+            creation_time = os.path.getctime(backup)
+            backup_info.append({
+                'filename': os.path.basename(backup),
+                'filesize': f"{size:.2f}",
+                'creation_time': creation_time
+            })
+
+        backup_info.sort(key=lambda x: x['creation_time'], reverse=True)
+
+        return backup_info
+
+    @login_optionally_required
+    @backups_blueprint.route("/download/<string:filename>", methods=['GET'])
+    def download_backup(filename):
+        import re
+        filename = filename.strip()
+        backup_filename_regex = BACKUP_FILENAME_FORMAT.format("\d+")
+
+        full_path = os.path.join(os.path.abspath(datastore.datastore_path), filename)
+        if not full_path.startswith(os.path.abspath(datastore.datastore_path)):
+            abort(404)
+
+        if filename == 'latest':
+            backups = find_backups()
+            filename = backups[0]['filename']
+
+        if not re.match(r"^" + backup_filename_regex + "$", filename):
+            abort(400)  # Bad Request if the filename doesn't match the pattern
+
+        logger.debug(f"Backup download request for '{full_path}'")
+        return send_from_directory(os.path.abspath(datastore.datastore_path), filename, as_attachment=True)
+
+    @login_optionally_required
+    @backups_blueprint.route("", methods=['GET'])
+    def index():
+        backups = find_backups()
+        output = render_template("overview.html",
+                                 available_backups=backups,
+                                 backup_running=any(thread.is_alive() for thread in backup_threads)
+                                 )
+
+        return output
+
+    @login_optionally_required
+    @backups_blueprint.route("/remove-backups", methods=['GET'])
+    def remove_backups():
+
+        backup_filepath = os.path.join(datastore.datastore_path, BACKUP_FILENAME_FORMAT.format("*"))
+        backups = glob.glob(backup_filepath)
+        for backup in backups:
+            os.unlink(backup)
+
+        flash("Backups were deleted.")
+
+        return redirect(url_for('backups.index'))
+
+    return backups_blueprint

changedetectionio/blueprint/backups/templates/overview.html

@@ -0,0 +1,36 @@
+{% extends 'base.html' %}
+{% block content %}
+    {% from '_helpers.html' import render_simple_field, render_field %}
+    <div class="edit-form">
+        <div class="box-wrap inner">
+            <h4>Backups</h4>
+            {% if backup_running %}
+                <p>
+                    <strong>A backup is running!</strong>
+                </p>
+            {% endif %}
+            <p>
+                Here you can download and request a new backup, when a backup is completed you will see it listed below.
+            </p>
+            <br>
+                {% if available_backups %}
+                    <ul>
+                    {% for backup in available_backups %}
+                        <li><a href="{{ url_for('backups.download_backup', filename=backup["filename"]) }}">{{ backup["filename"] }}</a> {{  backup["filesize"] }} Mb</li>
+                    {% endfor %}
+                    </ul>
+                {% else %}
+                    <p>
+                    <strong>No backups found.</strong>
+                    </p>
+                {% endif %}
+
+            <a class="pure-button pure-button-primary" href="{{ url_for('backups.request_backup') }}">Create backup</a>
+            {% if available_backups %}
+                <a class="pure-button button-small button-error " href="{{ url_for('backups.remove_backups') }}">Remove backups</a>
+            {% endif %}
+        </div>
+    </div>
+
+
+{% endblock %}

changedetectionio/blueprint/browser_steps/TODO.txt

@@ -0,0 +1,7 @@
+- This needs an abstraction to directly handle the puppeteer connection methods
+- Then remove the playwright stuff
+- Remove hack redirect at line 65 changedetectionio/processors/__init__.py
+
+The screenshots are base64 encoded/decoded which is very CPU intensive for large screenshots (in playwright) but not
+in the direct puppeteer connection (they are binary end to end)
+

changedetectionio/blueprint/browser_steps/__init__.py

@@ -4,73 +4,74 @@
 # Why?
 # `browsersteps_playwright_browser_interface.chromium.connect_over_cdp()` will only run once without async()
 # - this flask app is not async()
-# - browserless has a single timeout/keepalive which applies to the session made at .connect_over_cdp()
+# - A single timeout/keepalive which applies to the session made at .connect_over_cdp()
 #
 # So it means that we must unfortunately for now just keep a single timer since .connect_over_cdp() was run
 # and know when that reaches timeout/keepalive :( when that time is up, restart the connection and tell the user
 # that their time is up, insert another coin. (reload)
 #
-# Bigger picture
-# - It's horrible that we have this click+wait deal, some nice socket.io solution using something similar
-# to what the browserless debug UI already gives us would be smarter..
 #
-# OR
-# - Some API call that should be hacked into browserless or playwright that we can "/api/bump-keepalive/{session_id}/60"
-# So we can tell it that we need more time (run this on each action)
-#
-# OR
-# - use multiprocessing to bump this over to its own process and add some transport layer (queue/pipes)
 
-from distutils.util import strtobool
+from changedetectionio.strtobool import strtobool
 from flask import Blueprint, request, make_response
-import logging
 import os
-import re
 
 from changedetectionio.store import ChangeDetectionStore
-from changedetectionio import login_optionally_required
+from changedetectionio.flask_app import login_optionally_required
+from loguru import logger
 
 browsersteps_sessions = {}
 io_interface_context = None
+import json
+import hashlib
+from flask import Response
+import asyncio
+import threading
 
+def run_async_in_browser_loop(coro):
+    """Run async coroutine using the existing async worker event loop"""
+    from changedetectionio import worker_handler
+    
+    # Use the existing async worker event loop instead of creating a new one
+    if worker_handler.USE_ASYNC_WORKERS and worker_handler.async_loop and not worker_handler.async_loop.is_closed():
+        logger.debug("Browser steps using existing async worker event loop")
+        future = asyncio.run_coroutine_threadsafe(coro, worker_handler.async_loop)
+        return future.result()
+    else:
+        # Fallback: create a new event loop (for sync workers or if async loop not available)
+        logger.debug("Browser steps creating temporary event loop")
+        loop = asyncio.new_event_loop()
+        asyncio.set_event_loop(loop)
+        try:
+            return loop.run_until_complete(coro)
+        finally:
+            loop.close()
 
 def construct_blueprint(datastore: ChangeDetectionStore):
     browser_steps_blueprint = Blueprint('browser_steps', __name__, template_folder="templates")
 
-    def start_browsersteps_session(watch_uuid):
-        from . import nonContext
+    async def start_browsersteps_session(watch_uuid):
         from . import browser_steps
         import time
-        global browsersteps_sessions
-        global io_interface_context
-
+        from playwright.async_api import async_playwright
 
         # We keep the playwright session open for many minutes
         keepalive_seconds = int(os.getenv('BROWSERSTEPS_MINUTES_KEEPALIVE', 10)) * 60
 
         browsersteps_start_session = {'start_time': time.time()}
 
-        # You can only have one of these running
-        # This should be very fine to leave running for the life of the application
-        # @idea - Make it global so the pool of watch fetchers can use it also
-        if not io_interface_context:
-            io_interface_context = nonContext.c_sync_playwright()
-            # Start the Playwright context, which is actually a nodejs sub-process and communicates over STDIN/STDOUT pipes
-            io_interface_context = io_interface_context.start()
+        # Create a new async playwright instance for browser steps
+        playwright_instance = async_playwright()
+        playwright_context = await playwright_instance.start()
 
         keepalive_ms = ((keepalive_seconds + 3) * 1000)
-        base_url = os.getenv('PLAYWRIGHT_DRIVER_URL', '')
+        base_url = os.getenv('PLAYWRIGHT_DRIVER_URL', '').strip('"')
         a = "?" if not '?' in base_url else '&'
         base_url += a + f"timeout={keepalive_ms}"
 
-        try:
-            browsersteps_start_session['browser'] = io_interface_context.chromium.connect_over_cdp(base_url)
-        except Exception as e:
-            if 'ECONNREFUSED' in str(e):
-                return make_response('Unable to start the Playwright Browser session, is it running?', 401)
-            else:
-                # Other errors, bad URL syntax, bad reply etc
-                return make_response(str(e), 401)
+        browser = await playwright_context.chromium.connect_over_cdp(base_url, timeout=keepalive_ms)
+        browsersteps_start_session['browser'] = browser
+        browsersteps_start_session['playwright_context'] = playwright_context
 
         proxy_id = datastore.get_preferred_proxy_for_watch(uuid=watch_uuid)
         proxy = None
@@ -89,15 +90,23 @@ def construct_blueprint(datastore: ChangeDetectionStore):
                 if parsed.password:
                     proxy['password'] = parsed.password
 
-                print("Browser Steps: UUID {} selected proxy {}".format(watch_uuid, proxy_url))
+                logger.debug(f"Browser Steps: UUID {watch_uuid} selected proxy {proxy_url}")
 
         # Tell Playwright to connect to Chrome and setup a new session via our stepper interface
-        browsersteps_start_session['browserstepper'] = browser_steps.browsersteps_live_ui(
-            playwright_browser=browsersteps_start_session['browser'],
-            proxy=proxy)
+        browserstepper = browser_steps.browsersteps_live_ui(
+            playwright_browser=browser,
+            proxy=proxy,
+            start_url=datastore.data['watching'][watch_uuid].link,
+            headers=datastore.data['watching'][watch_uuid].get('headers')
+        )
+        
+        # Initialize the async connection
+        await browserstepper.connect(proxy=proxy)
+        
+        browsersteps_start_session['browserstepper'] = browserstepper
 
         # For test
-        #browsersteps_start_session['browserstepper'].action_goto_url(value="http://example.com?time="+str(time.time()))
+        #await browsersteps_start_session['browserstepper'].action_goto_url(value="http://example.com?time="+str(time.time()))
 
         return browsersteps_start_session
 
@@ -106,20 +115,30 @@ def construct_blueprint(datastore: ChangeDetectionStore):
     @browser_steps_blueprint.route("/browsersteps_start_session", methods=['GET'])
     def browsersteps_start_session():
         # A new session was requested, return sessionID
-
+        import asyncio
         import uuid
-        global browsersteps_sessions
-
         browsersteps_session_id = str(uuid.uuid4())
         watch_uuid = request.args.get('uuid')
 
         if not watch_uuid:
             return make_response('No Watch UUID specified', 500)
 
-        print("Starting connection with playwright")
-        logging.debug("browser_steps.py connecting")
-        browsersteps_sessions[browsersteps_session_id] = start_browsersteps_session(watch_uuid)
-        print("Starting connection with playwright - done")
+        logger.debug("Starting connection with playwright")
+        logger.debug("browser_steps.py connecting")
+
+        try:
+            # Run the async function in the dedicated browser steps event loop
+            browsersteps_sessions[browsersteps_session_id] = run_async_in_browser_loop(
+                start_browsersteps_session(watch_uuid)
+            )
+        except Exception as e:
+            if 'ECONNREFUSED' in str(e):
+                return make_response('Unable to start the Playwright Browser session, is sockpuppetbrowser running? Network configuration is OK?', 401)
+            else:
+                # Other errors, bad URL syntax, bad reply etc
+                return make_response(str(e), 401)
+
+        logger.debug("Starting connection with playwright - done")
         return {'browsersteps_session_id': browsersteps_session_id}
 
     @login_optionally_required
@@ -152,8 +171,7 @@ def construct_blueprint(datastore: ChangeDetectionStore):
     @browser_steps_blueprint.route("/browsersteps_update", methods=['POST'])
     def browsersteps_ui_update():
         import base64
-        import playwright._impl._api_types
-        global browsersteps_sessions
+        import playwright._impl._errors
         from changedetectionio.blueprint.browser_steps import browser_steps
 
         remaining =0
@@ -167,41 +185,30 @@ def construct_blueprint(datastore: ChangeDetectionStore):
         if not browsersteps_sessions.get(browsersteps_session_id):
             return make_response('No session exists under that ID', 500)
 
-
+        is_last_step = False
         # Actions - step/apply/etc, do the thing and return state
         if request.method == 'POST':
             # @todo - should always be an existing session
             step_operation = request.form.get('operation')
             step_selector = request.form.get('selector')
             step_optional_value = request.form.get('optional_value')
-            step_n = int(request.form.get('step_n'))
             is_last_step = strtobool(request.form.get('is_last_step'))
 
-            if step_operation == 'Goto site':
-                step_operation = 'goto_url'
-                step_optional_value = datastore.data['watching'][uuid].get('url')
-                step_selector = None
-
-            # @todo try.. accept.. nice errors not popups..
             try:
-
-                browsersteps_sessions[browsersteps_session_id]['browserstepper'].call_action(action_name=step_operation,
-                                         selector=step_selector,
-                                         optional_value=step_optional_value)
+                # Run the async call_action method in the dedicated browser steps event loop
+                run_async_in_browser_loop(
+                    browsersteps_sessions[browsersteps_session_id]['browserstepper'].call_action(
+                        action_name=step_operation,
+                        selector=step_selector,
+                        optional_value=step_optional_value
+                    )
+                )
 
             except Exception as e:
-                print("Exception when calling step operation", step_operation, str(e))
+                logger.error(f"Exception when calling step operation {step_operation} {str(e)}")
                 # Try to find something of value to give back to the user
                 return make_response(str(e).splitlines()[0], 401)
 
-            # Get visual selector ready/update its data (also use the current filter info from the page?)
-            # When the last 'apply' button was pressed
-            # @todo this adds overhead because the xpath selection is happening twice
-            u = browsersteps_sessions[browsersteps_session_id]['browserstepper'].page.url
-            if is_last_step and u:
-                (screenshot, xpath_data) = browsersteps_sessions[browsersteps_session_id]['browserstepper'].request_visualselector_data()
-                datastore.save_screenshot(watch_uuid=uuid, screenshot=screenshot)
-                datastore.save_xpath_data(watch_uuid=uuid, data=xpath_data)
 
 #        if not this_session.page:
 #            cleanup_playwright_session()
@@ -209,31 +216,36 @@ def construct_blueprint(datastore: ChangeDetectionStore):
 
         # Screenshots and other info only needed on requesting a step (POST)
         try:
-            state = browsersteps_sessions[browsersteps_session_id]['browserstepper'].get_current_state()
-        except playwright._impl._api_types.Error as e:
-            return make_response("Browser session ran out of time :( Please reload this page."+str(e), 401)
+            # Run the async get_current_state method in the dedicated browser steps event loop
+            (screenshot, xpath_data) = run_async_in_browser_loop(
+                browsersteps_sessions[browsersteps_session_id]['browserstepper'].get_current_state()
+            )
+                
+            if is_last_step:
+                watch = datastore.data['watching'].get(uuid)
+                u = browsersteps_sessions[browsersteps_session_id]['browserstepper'].page.url
+                if watch and u:
+                    watch.save_screenshot(screenshot=screenshot)
+                    watch.save_xpath_data(data=xpath_data)
 
-        # Use send_file() which is way faster than read/write loop on bytes
-        import json
-        from tempfile import mkstemp
-        from flask import send_file
-        tmp_fd, tmp_file = mkstemp(text=True, suffix=".json", prefix="changedetectionio-")
+        except Exception as e:
+            return make_response(f"Error fetching screenshot and element data - {str(e)}", 401)
 
-        output = json.dumps({'screenshot': "data:image/jpeg;base64,{}".format(
-            base64.b64encode(state[0]).decode('ascii')),
-            'xpath_data': state[1],
-            'session_age_start': browsersteps_sessions[browsersteps_session_id]['browserstepper'].age_start,
-            'browser_time_remaining': round(remaining)
-        })
+        # SEND THIS BACK TO THE BROWSER
+        output = {
+            "screenshot": f"data:image/jpeg;base64,{base64.b64encode(screenshot).decode('ascii')}",
+            "xpath_data": xpath_data,
+            "session_age_start": browsersteps_sessions[browsersteps_session_id]['browserstepper'].age_start,
+            "browser_time_remaining": round(remaining)
+        }
+        json_data = json.dumps(output)
 
-        with os.fdopen(tmp_fd, 'w') as f:
-            f.write(output)
+        # Generate an ETag (hash of the response body)
+        etag_hash = hashlib.md5(json_data.encode('utf-8')).hexdigest()
 
-        response = make_response(send_file(path_or_file=tmp_file,
-                                           mimetype='application/json; charset=UTF-8',
-                                           etag=True))
-        # No longer needed
-        os.unlink(tmp_file)
+        # Create the response with ETag
+        response = Response(json_data, mimetype="application/json; charset=UTF-8")
+        response.set_etag(etag_hash)
 
         return response
 

changedetectionio/blueprint/browser_steps/browser_steps.py

@@ -1,9 +1,14 @@
-#!/usr/bin/python3
-
 import os
 import time
 import re
 from random import randint
+from loguru import logger
+
+from changedetectionio.content_fetchers import SCREENSHOT_MAX_HEIGHT_DEFAULT
+from changedetectionio.content_fetchers.base import manage_user_agent
+from changedetectionio.jinja2_custom import render as jinja_render
+
+
 
 # Two flags, tell the JS which of the "Selector" or "Value" field should be enabled in the front end
 # 0- off, 1- on
@@ -21,18 +26,22 @@ browser_step_ui_config = {'Choose one': '0 0',
                           'Click element if exists': '1 0',
                           'Click element': '1 0',
                           'Click element containing text': '0 1',
+                          'Click element containing text if exists': '0 1',
                           'Enter text in field': '1 1',
                           'Execute JS': '0 1',
 #                          'Extract text and use as filter': '1 0',
                           'Goto site': '0 0',
                           'Goto URL': '0 1',
+                          'Make all child elements visible': '1 0',
                           'Press Enter': '0 0',
                           'Select by label': '1 1',
+                          '<select> by option text': '1 1',
                           'Scroll down': '0 0',
                           'Uncheck checkbox': '1 0',
                           'Wait for seconds': '0 1',
                           'Wait for text': '0 1',
                           'Wait for text in element': '1 1',
+                          'Remove elements': '1 0',
                           #                          'Press Page Down': '0 0',
                           #                          'Press Page Up': '0 0',
                           # weird bug, come back to it later
@@ -45,127 +54,251 @@ browser_step_ui_config = {'Choose one': '0 0',
 # ONLY Works in Playwright because we need the fullscreen screenshot
 class steppable_browser_interface():
     page = None
+    start_url = None
+    action_timeout = 10 * 1000
+
+    def __init__(self, start_url):
+        self.start_url = start_url
 
     # Convert and perform "Click Button" for example
-    def call_action(self, action_name, selector=None, optional_value=None):
+    async def call_action(self, action_name, selector=None, optional_value=None):
+        if self.page is None:
+            logger.warning("Cannot call action on None page object")
+            return
+            
         now = time.time()
         call_action_name = re.sub('[^0-9a-zA-Z]+', '_', action_name.lower())
         if call_action_name == 'choose_one':
             return
 
-        print("> action calling", call_action_name)
+        logger.debug(f"> Action calling '{call_action_name}'")
         # https://playwright.dev/python/docs/selectors#xpath-selectors
         if selector and selector.startswith('/') and not selector.startswith('//'):
             selector = "xpath=" + selector
 
+        # Check if action handler exists
+        if not hasattr(self, "action_" + call_action_name):
+            logger.warning(f"Action handler for '{call_action_name}' not found")
+            return
+            
         action_handler = getattr(self, "action_" + call_action_name)
 
         # Support for Jinja2 variables in the value and selector
-        from jinja2 import Environment
-        jinja2_env = Environment(extensions=['jinja2_time.TimeExtension'])
-
         if selector and ('{%' in selector or '{{' in selector):
-            selector = str(jinja2_env.from_string(selector).render())
+            selector = jinja_render(template_str=selector)
 
         if optional_value and ('{%' in optional_value or '{{' in optional_value):
-            optional_value = str(jinja2_env.from_string(optional_value).render())
+            optional_value = jinja_render(template_str=optional_value)
 
-        action_handler(selector, optional_value)
-        self.page.wait_for_timeout(1.5 * 1000)
-        print("Call action done in", time.time() - now)
+        # Trigger click and cautiously handle potential navigation
+        # This means the page redirects/reloads/changes JS etc etc
+        if call_action_name.startswith('click_'):
+            try:
+                # Set up navigation expectation before the click (like sync version)
+                async with self.page.expect_event("framenavigated", timeout=3000) as navigation_info:
+                    await action_handler(selector, optional_value)
+                
+                # Check if navigation actually occurred
+                try:
+                    await navigation_info.value  # This waits for the navigation promise
+                    logger.debug(f"Navigation occurred on {call_action_name}.")
+                except Exception:
+                    logger.debug(f"No navigation occurred within timeout when calling {call_action_name}, that's OK, continuing.")
+                    
+            except Exception as e:
+                # If expect_event itself times out, that means no navigation occurred - that's OK
+                if "framenavigated" in str(e) and "exceeded" in str(e):
+                    logger.debug(f"No navigation occurred within timeout when calling {call_action_name}, that's OK, continuing.")
+                else:
+                    raise e
+        else:
+            # Some other action that probably a navigation is not expected
+            await action_handler(selector, optional_value)
 
-    def action_goto_url(self, selector=None, value=None):
-        # self.page.set_viewport_size({"width": 1280, "height": 5000})
+
+        # Safely wait for timeout
+        await self.page.wait_for_timeout(1.5 * 1000)
+        logger.debug(f"Call action done in {time.time()-now:.2f}s")
+
+    async def action_goto_url(self, selector=None, value=None):
+        if not value:
+            logger.warning("No URL provided for goto_url action")
+            return None
+            
         now = time.time()
-        response = self.page.goto(value, timeout=0, wait_until='load')
-        # Should be the same as the puppeteer_fetch.js methods, means, load with no timeout set (skip timeout)
-        #and also wait for seconds ?
-        #await page.waitForTimeout(1000);
-        #await page.waitForTimeout(extra_wait_ms);
-        print("Time to goto URL ", time.time() - now)
+        response = await self.page.goto(value, timeout=0, wait_until='load')
+        logger.debug(f"Time to goto URL {time.time()-now:.2f}s")
         return response
 
-    def action_click_element_containing_text(self, selector=None, value=''):
-        if not len(value.strip()):
+    # Incase they request to go back to the start
+    async def action_goto_site(self, selector=None, value=None):
+        return await self.action_goto_url(value=re.sub(r'^source:', '', self.start_url, flags=re.IGNORECASE))
+
+    async def action_click_element_containing_text(self, selector=None, value=''):
+        logger.debug("Clicking element containing text")
+        if not value or not len(value.strip()):
             return
+            
         elem = self.page.get_by_text(value)
-        if elem.count():
-            elem.first.click(delay=randint(200, 500), timeout=3000)
+        if await elem.count():
+            await elem.first.click(delay=randint(200, 500), timeout=self.action_timeout)
 
-    def action_enter_text_in_field(self, selector, value):
-        if not len(selector.strip()):
+
+    async def action_click_element_containing_text_if_exists(self, selector=None, value=''):
+        logger.debug("Clicking element containing text if exists")
+        if not value or not len(value.strip()):
+            return
+            
+        elem = self.page.get_by_text(value)
+        count = await elem.count()
+        logger.debug(f"Clicking element containing text - {count} elements found")
+        if count:
+            await elem.first.click(delay=randint(200, 500), timeout=self.action_timeout)
+                
+
+    async def action_enter_text_in_field(self, selector, value):
+        if not selector or not len(selector.strip()):
             return
 
-        self.page.fill(selector, value, timeout=10 * 1000)
+        await self.page.fill(selector, value, timeout=self.action_timeout)
 
-    def action_execute_js(self, selector, value):
-        response = self.page.evaluate(value)
-        return response
+    async def action_execute_js(self, selector, value):
+        if not value:
+            return None
+            
+        return await self.page.evaluate(value)
 
-    def action_click_element(self, selector, value):
-        print("Clicking element")
-        if not len(selector.strip()):
+    async def action_click_element(self, selector, value):
+        logger.debug("Clicking element")
+        if not selector or not len(selector.strip()):
             return
 
-        self.page.click(selector=selector, timeout=30 * 1000, delay=randint(200, 500))
+        await self.page.click(selector=selector, timeout=self.action_timeout + 20 * 1000, delay=randint(200, 500))
 
-    def action_click_element_if_exists(self, selector, value):
-        import playwright._impl._api_types as _api_types
-        print("Clicking element if exists")
-        if not len(selector.strip()):
+    async def action_click_element_if_exists(self, selector, value):
+        import playwright._impl._errors as _api_types
+        logger.debug("Clicking element if exists")
+        if not selector or not len(selector.strip()):
             return
+            
         try:
-            self.page.click(selector, timeout=10 * 1000, delay=randint(200, 500))
-        except _api_types.TimeoutError as e:
+            await self.page.click(selector, timeout=self.action_timeout, delay=randint(200, 500))
+        except _api_types.TimeoutError:
             return
-        except _api_types.Error as e:
+        except _api_types.Error:
             # Element was there, but page redrew and now its long long gone
             return
+                
 
-    def action_click_x_y(self, selector, value):
-        x, y = value.strip().split(',')
-        x = int(float(x.strip()))
-        y = int(float(y.strip()))
-        self.page.mouse.click(x=x, y=y, delay=randint(200, 500))
+    async def action_click_x_y(self, selector, value):
+        if not value or not re.match(r'^\s?\d+\s?,\s?\d+\s?$', value):
+            logger.warning("'Click X,Y' step should be in the format of '100 , 90'")
+            return
 
-    def action_scroll_down(self, selector, value):
+        try:
+            x, y = value.strip().split(',')
+            x = int(float(x.strip()))
+            y = int(float(y.strip()))
+            
+            await self.page.mouse.click(x=x, y=y, delay=randint(200, 500))
+                
+        except Exception as e:
+            logger.error(f"Error parsing x,y coordinates: {str(e)}")
+
+    async def action__select_by_option_text(self, selector, value):
+        if not selector or not len(selector.strip()):
+            return
+
+        await self.page.select_option(selector, label=value, timeout=self.action_timeout)
+
+    async def action_scroll_down(self, selector, value):
         # Some sites this doesnt work on for some reason
-        self.page.mouse.wheel(0, 600)
-        self.page.wait_for_timeout(1000)
+        await self.page.mouse.wheel(0, 600)
+        await self.page.wait_for_timeout(1000)
 
-    def action_wait_for_seconds(self, selector, value):
-        self.page.wait_for_timeout(float(value.strip()) * 1000)
+    async def action_wait_for_seconds(self, selector, value):
+        try:
+            seconds = float(value.strip()) if value else 1.0
+            await self.page.wait_for_timeout(seconds * 1000)
+        except (ValueError, TypeError) as e:
+            logger.error(f"Invalid value for wait_for_seconds: {str(e)}")
 
-    def action_wait_for_text(self, selector, value):
+    async def action_wait_for_text(self, selector, value):
+        if not value:
+            return
+            
         import json
         v = json.dumps(value)
-        self.page.wait_for_function(f'document.querySelector("body").innerText.includes({v});', timeout=30000)
+        await self.page.wait_for_function(
+            f'document.querySelector("body").innerText.includes({v});',
+            timeout=30000
+        )
+            
 
-    def action_wait_for_text_in_element(self, selector, value):
+    async def action_wait_for_text_in_element(self, selector, value):
+        if not selector or not value:
+            return
+            
         import json
         s = json.dumps(selector)
         v = json.dumps(value)
-        self.page.wait_for_function(f'document.querySelector({s}).innerText.includes({v});', timeout=30000)
+        
+        await self.page.wait_for_function(
+            f'document.querySelector({s}).innerText.includes({v});',
+            timeout=30000
+        )
 
     # @todo - in the future make some popout interface to capture what needs to be set
     # https://playwright.dev/python/docs/api/class-keyboard
-    def action_press_enter(self, selector, value):
-        self.page.keyboard.press("Enter", delay=randint(200, 500))
+    async def action_press_enter(self, selector, value):
+        await self.page.keyboard.press("Enter", delay=randint(200, 500))
+            
 
-    def action_press_page_up(self, selector, value):
-        self.page.keyboard.press("PageUp", delay=randint(200, 500))
+    async def action_press_page_up(self, selector, value):
+        await self.page.keyboard.press("PageUp", delay=randint(200, 500))
 
-    def action_press_page_down(self, selector, value):
-        self.page.keyboard.press("PageDown", delay=randint(200, 500))
+    async def action_press_page_down(self, selector, value):
+        await self.page.keyboard.press("PageDown", delay=randint(200, 500))
 
-    def action_check_checkbox(self, selector, value):
-        self.page.locator(selector).check(timeout=1000)
+    async def action_check_checkbox(self, selector, value):
+        if not selector:
+            return
 
-    def action_uncheck_checkbox(self, selector, value):
-        self.page.locator(selector, timeout=1000).uncheck(timeout=1000)
+        await self.page.locator(selector).check(timeout=self.action_timeout)
 
+    async def action_uncheck_checkbox(self, selector, value):
+        if not selector:
+            return
+            
+        await self.page.locator(selector).uncheck(timeout=self.action_timeout)
+            
 
-# Responsible for maintaining a live 'context' with browserless
+    async def action_remove_elements(self, selector, value):
+        """Removes all elements matching the given selector from the DOM."""
+        if not selector:
+            return
+            
+        await self.page.locator(selector).evaluate_all("els => els.forEach(el => el.remove())")
+
+    async def action_make_all_child_elements_visible(self, selector, value):
+        """Recursively makes all child elements inside the given selector fully visible."""
+        if not selector:
+            return
+            
+        await self.page.locator(selector).locator("*").evaluate_all("""
+            els => els.forEach(el => {
+                el.style.display = 'block';   // Forces it to be displayed
+                el.style.visibility = 'visible';   // Ensures it's not hidden
+                el.style.opacity = '1';   // Fully opaque
+                el.style.position = 'relative';   // Avoids 'absolute' hiding
+                el.style.height = 'auto';   // Expands collapsed elements
+                el.style.width = 'auto';   // Ensures full visibility
+                el.removeAttribute('hidden');   // Removes hidden attribute
+                el.classList.remove('hidden', 'd-none');  // Removes common CSS hidden classes
+            })
+        """)
+
+# Responsible for maintaining a live 'context' with the chrome CDP
 # @todo - how long do contexts live for anyway?
 class browsersteps_live_ui(steppable_browser_interface):
     context = None
@@ -174,7 +307,10 @@ class browsersteps_live_ui(steppable_browser_interface):
     stale = False
     # bump and kill this if idle after X sec
     age_start = 0
-
+    headers = {}
+    # Track if resources are properly cleaned up
+    _is_cleaned_up = False
+    
     # use a special driver, maybe locally etc
     command_executor = os.getenv(
         "PLAYWRIGHT_BROWSERSTEPS_DRIVER_URL"
@@ -188,96 +324,174 @@ class browsersteps_live_ui(steppable_browser_interface):
 
     browser_type = os.getenv("PLAYWRIGHT_BROWSER_TYPE", 'chromium').strip('"')
 
-    def __init__(self, playwright_browser, proxy=None):
+    def __init__(self, playwright_browser, proxy=None, headers=None, start_url=None):
+        self.headers = headers or {}
         self.age_start = time.time()
         self.playwright_browser = playwright_browser
-        if self.context is None:
-            self.connect(proxy=proxy)
+        self.start_url = start_url
+        self._is_cleaned_up = False
+        self.proxy = proxy
+        # Note: connect() is now async and must be called separately
+
+    def __del__(self):
+        # Ensure cleanup happens if object is garbage collected
+        # Note: cleanup is now async, so we can only mark as cleaned up here
+        self._is_cleaned_up = True
 
     # Connect and setup a new context
-    def connect(self, proxy=None):
+    async def connect(self, proxy=None):
         # Should only get called once - test that
         keep_open = 1000 * 60 * 5
         now = time.time()
 
         # @todo handle multiple contexts, bind a unique id from the browser on each req?
-        self.context = self.playwright_browser.new_context(
-            # @todo
-            #                user_agent=request_headers['User-Agent'] if request_headers.get('User-Agent') else 'Mozilla/5.0',
-            #               proxy=self.proxy,
-            # This is needed to enable JavaScript execution on GitHub and others
-            bypass_csp=True,
-            # Should never be needed
-            accept_downloads=False,
-            proxy=proxy
+        self.context = await self.playwright_browser.new_context(
+            accept_downloads=False,  # Should never be needed
+            bypass_csp=True,  # This is needed to enable JavaScript execution on GitHub and others
+            extra_http_headers=self.headers,
+            ignore_https_errors=True,
+            proxy=proxy,
+            service_workers=os.getenv('PLAYWRIGHT_SERVICE_WORKERS', 'allow'),
+            # Should be `allow` or `block` - sites like YouTube can transmit large amounts of data via Service Workers
+            user_agent=manage_user_agent(headers=self.headers),
         )
 
-        self.page = self.context.new_page()
+        self.page = await self.context.new_page()
 
         # self.page.set_default_navigation_timeout(keep_open)
         self.page.set_default_timeout(keep_open)
-        # @todo probably this doesnt work
-        self.page.on(
-            "close",
-            self.mark_as_closed,
-        )
+        # Set event handlers
+        self.page.on("close", self.mark_as_closed)
         # Listen for all console events and handle errors
         self.page.on("console", lambda msg: print(f"Browser steps console - {msg.type}: {msg.text} {msg.args}"))
 
-        print("Time to browser setup", time.time() - now)
-        self.page.wait_for_timeout(1 * 1000)
+        logger.debug(f"Time to browser setup {time.time()-now:.2f}s")
+        await self.page.wait_for_timeout(1 * 1000)
 
     def mark_as_closed(self):
-        print("Page closed, cleaning up..")
+        logger.debug("Page closed, cleaning up..")
+        # Note: This is called from a sync context (event handler)
+        # so we'll just mark as cleaned up and let __del__ handle the rest
+        self._is_cleaned_up = True
+
+    async def cleanup(self):
+        """Properly clean up all resources to prevent memory leaks"""
+        if self._is_cleaned_up:
+            return
+            
+        logger.debug("Cleaning up browser steps resources")
+        
+        # Clean up page
+        if hasattr(self, 'page') and self.page is not None:
+            try:
+                # Force garbage collection before closing
+                await self.page.request_gc()
+            except Exception as e:
+                logger.debug(f"Error during page garbage collection: {str(e)}")
+                
+            try:
+                # Remove event listeners before closing
+                self.page.remove_listener("close", self.mark_as_closed)
+            except Exception as e:
+                logger.debug(f"Error removing event listeners: {str(e)}")
+                
+            try:
+                await self.page.close()
+            except Exception as e:
+                logger.debug(f"Error closing page: {str(e)}")
+            
+            self.page = None
+
+        # Clean up context
+        if hasattr(self, 'context') and self.context is not None:
+            try:
+                await self.context.close()
+            except Exception as e:
+                logger.debug(f"Error closing context: {str(e)}")
+            
+            self.context = None
+            
+        self._is_cleaned_up = True
+        logger.debug("Browser steps resources cleanup complete")
 
     @property
     def has_expired(self):
-        if not self.page:
+        if not self.page or self._is_cleaned_up:
             return True
+        
+        # Check if session has expired based on age
+        max_age_seconds = int(os.getenv("BROWSER_STEPS_MAX_AGE_SECONDS", 60 * 10))  # Default 10 minutes
+        if (time.time() - self.age_start) > max_age_seconds:
+            logger.debug(f"Browser steps session expired after {max_age_seconds} seconds")
+            return True
+            
+        return False
 
-
-    def get_current_state(self):
+    async def get_current_state(self):
         """Return the screenshot and interactive elements mapping, generally always called after action_()"""
-        from pkg_resources import resource_string
-        xpath_element_js = resource_string(__name__, "../../res/xpath_element_scraper.js").decode('utf-8')
+        import importlib.resources
+        import json
+        # because we for now only run browser steps in playwright mode (not puppeteer mode)
+        from changedetectionio.content_fetchers.playwright import capture_full_page_async
+
+        # Safety check - don't proceed if resources are cleaned up
+        if self._is_cleaned_up or self.page is None:
+            logger.warning("Attempted to get current state after cleanup")
+            return (None, None)
+
+        xpath_element_js = importlib.resources.files("changedetectionio.content_fetchers.res").joinpath('xpath_element_scraper.js').read_text()
+
         now = time.time()
-        self.page.wait_for_timeout(1 * 1000)
+        await self.page.wait_for_timeout(1 * 1000)
 
-        # The actual screenshot
-        screenshot = self.page.screenshot(type='jpeg', full_page=True, quality=40)
+        screenshot = None
+        xpath_data = None
+        
+        try:
+            # Get screenshot first
+            screenshot = await capture_full_page_async(page=self.page)
+            if not screenshot:
+                logger.error("No screenshot was retrieved :((")
 
-        self.page.evaluate("var include_filters=''")
-        # Go find the interactive elements
-        # @todo in the future, something smarter that can scan for elements with .click/focus etc event handlers?
-        elements = 'a,button,input,select,textarea,i,th,td,p,li,h1,h2,h3,h4,div,span'
-        xpath_element_js = xpath_element_js.replace('%ELEMENTS%', elements)
-        xpath_data = self.page.evaluate("async () => {" + xpath_element_js + "}")
-        # So the JS will find the smallest one first
-        xpath_data['size_pos'] = sorted(xpath_data['size_pos'], key=lambda k: k['width'] * k['height'], reverse=True)
-        print("Time to complete get_current_state of browser", time.time() - now)
-        # except
-        # playwright._impl._api_types.Error: Browser closed.
-        # @todo show some countdown timer?
+            logger.debug(f"Time to get screenshot from browser {time.time() - now:.2f}s")
+
+            # Then get interactive elements
+            now = time.time()
+            await self.page.evaluate("var include_filters=''")
+            await self.page.request_gc()
+
+            scan_elements = 'a,button,input,select,textarea,i,th,td,p,li,h1,h2,h3,h4,div,span'
+
+            MAX_TOTAL_HEIGHT = int(os.getenv("SCREENSHOT_MAX_HEIGHT", SCREENSHOT_MAX_HEIGHT_DEFAULT))
+            xpath_data = json.loads(await self.page.evaluate(xpath_element_js, {
+                "visualselector_xpath_selectors": scan_elements,
+                "max_height": MAX_TOTAL_HEIGHT
+            }))
+            await self.page.request_gc()
+
+            # Sort elements by size
+            xpath_data['size_pos'] = sorted(xpath_data['size_pos'], key=lambda k: k['width'] * k['height'], reverse=True)
+            logger.debug(f"Time to scrape xPath element data in browser {time.time()-now:.2f}s")
+            
+        except Exception as e:
+            logger.error(f"Error getting current state: {str(e)}")
+            # If the page has navigated (common with logins) then the context is destroyed on navigation, continue
+            # I'm not sure that this is required anymore because we have the "expect navigation wrapper" at the top
+            if "Execution context was destroyed" in str(e):
+                logger.debug("Execution context was destroyed, most likely because of navigation, continuing...")
+            pass
+
+            # Attempt recovery - force garbage collection
+            try:
+                await self.page.request_gc()
+            except:
+                pass
+        
+        # Request garbage collection one final time
+        try:
+            await self.page.request_gc()
+        except:
+            pass
+            
         return (screenshot, xpath_data)
 
-    def request_visualselector_data(self):
-        """
-        Does the same that the playwright operation in content_fetcher does
-        This is used to just bump the VisualSelector data so it' ready to go if they click on the tab
-        @todo refactor and remove duplicate code, add include_filters
-        :param xpath_data:
-        :param screenshot:
-        :param current_include_filters:
-        :return:
-        """
-
-        self.page.evaluate("var include_filters=''")
-        from pkg_resources import resource_string
-        # The code that scrapes elements and makes a list of elements/size/position to click on in the VisualSelector
-        xpath_element_js = resource_string(__name__, "../../res/xpath_element_scraper.js").decode('utf-8')
-        from changedetectionio.content_fetcher import visualselector_xpath_selectors
-        xpath_element_js = xpath_element_js.replace('%ELEMENTS%', visualselector_xpath_selectors)
-        xpath_data = self.page.evaluate("async () => {" + xpath_element_js + "}")
-        screenshot = self.page.screenshot(type='jpeg', full_page=True, quality=int(os.getenv("PLAYWRIGHT_SCREENSHOT_QUALITY", 72)))
-
-        return (screenshot, xpath_data)

changedetectionio/blueprint/browser_steps/nonContext.py

@@ -1,18 +0,0 @@
-from playwright.sync_api import PlaywrightContextManager
-import asyncio
-
-# So playwright wants to run as a context manager, but we do something horrible and hacky
-# we are holding the session open for as long as possible, then shutting it down, and opening a new one
-# So it means we don't get to use PlaywrightContextManager' __enter__ __exit__
-# To work around this, make goodbye() act the same as the __exit__()
-#
-# But actually I think this is because the context is opened correctly with __enter__() but we timeout the connection
-# then theres some lock condition where we cant destroy it without it hanging
-
-class c_PlaywrightContextManager(PlaywrightContextManager):
-
-    def goodbye(self) -> None:
-        self.__exit__()
-
-def c_sync_playwright() -> PlaywrightContextManager:
-    return c_PlaywrightContextManager()

changedetectionio/blueprint/check_proxies/__init__.py

@@ -1,14 +1,14 @@
+import importlib
 from concurrent.futures import ThreadPoolExecutor
 
+from changedetectionio.processors.text_json_diff.processor import FilterNotFoundInResponse
+from changedetectionio.store import ChangeDetectionStore
+
 from functools import wraps
 
 from flask import Blueprint
 from flask_login import login_required
 
-from changedetectionio.processors import text_json_diff
-from changedetectionio.store import ChangeDetectionStore
-
-
 STATUS_CHECKING = 0
 STATUS_FAILED = 1
 STATUS_OK = 2
@@ -32,32 +32,36 @@ def construct_blueprint(datastore: ChangeDetectionStore):
     @threadpool
     def long_task(uuid, preferred_proxy):
         import time
-        from changedetectionio import content_fetcher
+        from changedetectionio.content_fetchers import exceptions as content_fetcher_exceptions
+        from changedetectionio.jinja2_custom import render as jinja_render
 
         status = {'status': '', 'length': 0, 'text': ''}
-        from jinja2 import Environment, BaseLoader
 
         contents = ''
         now = time.time()
         try:
-            update_handler = text_json_diff.perform_site_check(datastore=datastore)
-            changed_detected, update_obj, contents = update_handler.run(uuid, preferred_proxy=preferred_proxy, skip_when_checksum_same=False)
+            processor_module = importlib.import_module("changedetectionio.processors.text_json_diff.processor")
+            update_handler = processor_module.perform_site_check(datastore=datastore,
+                                                                 watch_uuid=uuid
+                                                                 )
+
+            update_handler.call_browser(preferred_proxy_id=preferred_proxy)
         # title, size is len contents not len xfer
-        except content_fetcher.Non200ErrorCodeReceived as e:
+        except content_fetcher_exceptions.Non200ErrorCodeReceived as e:
             if e.status_code == 404:
                 status.update({'status': 'OK', 'length': len(contents), 'text': f"OK but 404 (page not found)"})
             elif e.status_code == 403 or e.status_code == 401:
                 status.update({'status': 'ERROR', 'length': len(contents), 'text': f"{e.status_code} - Access denied"})
             else:
                 status.update({'status': 'ERROR', 'length': len(contents), 'text': f"Status code: {e.status_code}"})
-        except text_json_diff.FilterNotFoundInResponse:
+        except FilterNotFoundInResponse:
             status.update({'status': 'OK', 'length': len(contents), 'text': f"OK but CSS/xPath filter not found (page changed layout?)"})
-        except content_fetcher.EmptyReply as e:
+        except content_fetcher_exceptions.EmptyReply as e:
             if e.status_code == 403 or e.status_code == 401:
                 status.update({'status': 'ERROR OTHER', 'length': len(contents), 'text': f"Got empty reply with code {e.status_code} - Access denied"})
             else:
                 status.update({'status': 'ERROR OTHER', 'length': len(contents) if contents else 0, 'text': f"Empty reply with code {e.status_code}, needs chrome?"})
-        except content_fetcher.ReplyWithContentButNoText as e:
+        except content_fetcher_exceptions.ReplyWithContentButNoText as e:
             txt = f"Got reply but with no content - Status code {e.status_code} - It's possible that the filters were found, but contained no usable text (or contained only an image)."
             status.update({'status': 'ERROR', 'text': txt})
         except Exception as e:
@@ -66,7 +70,9 @@ def construct_blueprint(datastore: ChangeDetectionStore):
             status.update({'status': 'OK', 'length': len(contents), 'text': ''})
 
         if status.get('text'):
-            status['text'] = Environment(loader=BaseLoader()).from_string('{{text|e}}').render({'text': status['text']})
+            # parse 'text' as text for safety
+            v = {'text': status['text']}
+            status['text'] = jinja_render(template_str='{{text|e}}', **v)
 
         status['time'] = "{:.2f}s".format(time.time() - now)
 

changedetectionio/blueprint/cookie_preferences.py

@@ -0,0 +1,98 @@
+from typing import Dict, Any
+from flask import request
+
+
+class PreferenceManager:
+    """
+    Manages user preferences with cookie persistence.
+
+    Handles reading from cookies, overriding with URL query parameters,
+    and setting cookies when preferences are updated.
+    """
+
+    def __init__(self, preferences_config: Dict[str, Dict[str, Any]], cookie_scope: str = 'path'):
+        """
+        Initialize the preference manager.
+
+        Args:
+            preferences_config: Dict defining preferences with their defaults and types
+                               e.g., {'diff_type': {'default': 'diffLines', 'type': 'value'}}
+            cookie_scope: 'path' for current path only, 'global' for entire application
+        """
+        self.config = preferences_config
+        self.cookie_scope = cookie_scope
+        self.preferences = {}
+        self.cookies_updated = False
+
+    def load_preferences(self) -> Dict[str, Any]:
+        """
+        Load preferences from cookies and override with URL query parameters.
+
+        URL query parameters act as temporary overrides but don't update cookies.
+
+        Returns:
+            Dict containing current preference values
+        """
+        for key, config in self.config.items():
+            # Read from cookie first (or use default)
+            if config['type'] == 'bool':
+                if key in request.cookies:
+                    # Cookie exists, use its value
+                    self.preferences[key] = request.cookies.get(key) == 'on'
+                else:
+                    # No cookie, use configured default
+                    self.preferences[key] = config['default']
+            else:
+                self.preferences[key] = request.cookies.get(key, config['default'])
+
+            # URL query parameters override (but don't update cookies)
+            if key in request.args:
+                if config['type'] == 'bool':
+                    self.preferences[key] = request.args.get(key) == 'on'
+                else:
+                    self.preferences[key] = request.args.get(key, config['default'])
+
+        return self.preferences
+
+    def load_from_form(self) -> Dict[str, Any]:
+        """
+        Load preferences from POST form data and mark for cookie updates.
+
+        For checkboxes: absence in form.data means unchecked = False.
+
+        Returns:
+            Dict containing preference values from form
+        """
+        self.cookies_updated = True
+
+        for key, config in self.config.items():
+            if config['type'] == 'bool':
+                # Checkbox: present = on, absent = off
+                self.preferences[key] = key in request.form and request.form.get(key) == 'on'
+            else:
+                # Value field: get from form or use default
+                self.preferences[key] = request.form.get(key, config['default'])
+
+        return self.preferences
+
+    def apply_cookies_to_response(self, response, max_age: int = 365 * 24 * 60 * 60):
+        """
+        Apply cookies to the response if preferences were updated.
+
+        Args:
+            response: Flask response object
+            max_age: Cookie expiration time in seconds (default: 1 year)
+
+        Returns:
+            Modified response object
+        """
+        if not self.cookies_updated:
+            return response
+
+        cookie_path = request.path if self.cookie_scope == 'path' else '/'
+
+        for key, value in self.preferences.items():
+            cookie_value = 'on' if value is True else ('off' if value is False else value)
+            response.set_cookie(key, cookie_value, max_age=max_age, path=cookie_path)
+
+        return response

changedetectionio/blueprint/imports/__init__.py

@@ -0,0 +1,75 @@
+from flask import Blueprint, request, redirect, url_for, flash, render_template
+from changedetectionio.store import ChangeDetectionStore
+from changedetectionio.auth_decorator import login_optionally_required
+from changedetectionio import worker_handler
+from changedetectionio.blueprint.imports.importer import (
+    import_url_list, 
+    import_distill_io_json, 
+    import_xlsx_wachete, 
+    import_xlsx_custom
+)
+
+def construct_blueprint(datastore: ChangeDetectionStore, update_q, queuedWatchMetaData):
+    import_blueprint = Blueprint('imports', __name__, template_folder="templates")
+    
+    @import_blueprint.route("/import", methods=['GET', 'POST'])
+    @login_optionally_required
+    def import_page():
+        remaining_urls = []
+        from changedetectionio import forms
+
+        if request.method == 'POST':
+            # URL List import
+            if request.values.get('urls') and len(request.values.get('urls').strip()):
+                # Import and push into the queue for immediate update check
+                importer_handler = import_url_list()
+                importer_handler.run(data=request.values.get('urls'), flash=flash, datastore=datastore, processor=request.values.get('processor', 'text_json_diff'))
+                for uuid in importer_handler.new_uuids:
+                    worker_handler.queue_item_async_safe(update_q, queuedWatchMetaData.PrioritizedItem(priority=1, item={'uuid': uuid}))
+
+                if len(importer_handler.remaining_data) == 0:
+                    return redirect(url_for('watchlist.index'))
+                else:
+                    remaining_urls = importer_handler.remaining_data
+
+            # Distill.io import
+            if request.values.get('distill-io') and len(request.values.get('distill-io').strip()):
+                # Import and push into the queue for immediate update check
+                d_importer = import_distill_io_json()
+                d_importer.run(data=request.values.get('distill-io'), flash=flash, datastore=datastore)
+                for uuid in d_importer.new_uuids:
+                    worker_handler.queue_item_async_safe(update_q, queuedWatchMetaData.PrioritizedItem(priority=1, item={'uuid': uuid}))
+
+            # XLSX importer
+            if request.files and request.files.get('xlsx_file'):
+                file = request.files['xlsx_file']
+
+                if request.values.get('file_mapping') == 'wachete':
+                    w_importer = import_xlsx_wachete()
+                    w_importer.run(data=file, flash=flash, datastore=datastore)
+                else:
+                    w_importer = import_xlsx_custom()
+                    # Building mapping of col # to col # type
+                    map = {}
+                    for i in range(10):
+                        c = request.values.get(f"custom_xlsx[col_{i}]")
+                        v = request.values.get(f"custom_xlsx[col_type_{i}]")
+                        if c and v:
+                            map[int(c)] = v
+
+                    w_importer.import_profile = map
+                    w_importer.run(data=file, flash=flash, datastore=datastore)
+
+                for uuid in w_importer.new_uuids:
+                    worker_handler.queue_item_async_safe(update_q, queuedWatchMetaData.PrioritizedItem(priority=1, item={'uuid': uuid}))
+
+        # Could be some remaining, or we could be on GET
+        form = forms.importForm(formdata=request.form if request.method == 'POST' else None)
+        output = render_template("import.html",
+                                form=form,
+                                import_url_list_remaining="\n".join(remaining_urls),
+                                original_distill_json=''
+                                )
+        return output
+
+    return import_blueprint

changedetectionio/blueprint/imports/importer.py

@@ -1,7 +1,7 @@
-from abc import ABC, abstractmethod
+from abc import abstractmethod
 import time
-import validators
 from wtforms import ValidationError
+from loguru import logger
 
 from changedetectionio.forms import validate_url
 
@@ -56,7 +56,7 @@ class import_url_list(Importer):
 
             # Flask wtform validators wont work with basic auth, use validators package
             # Up to 5000 per batch so we dont flood the server
-            # @todo validators.url failed on local hostnames (such as referring to ourself when using browserless)
+            # @todo validators.url will fail when you add your own IP etc
             if len(url) and 'http' in url.lower() and good < 5000:
                 extras = None
                 if processor:
@@ -195,7 +195,7 @@ class import_xlsx_wachete(Importer):
                     try:
                         validate_url(data.get('url'))
                     except ValidationError as e:
-                        print(">> import URL error", data.get('url'), str(e))
+                        logger.error(f">> Import URL error {data.get('url')} {str(e)}")
                         flash(f"Error processing row number {row_id}, URL value was incorrect, row was skipped.", 'error')
                         # Don't bother processing anything else on this row
                         continue
@@ -209,7 +209,7 @@ class import_xlsx_wachete(Importer):
                         self.new_uuids.append(new_uuid)
                         good += 1
             except Exception as e:
-                print(e)
+                logger.error(e)
                 flash(f"Error processing row number {row_id}, check all cell data types are correct, row was skipped.", 'error')
             else:
                 row_id += 1
@@ -240,7 +240,7 @@ class import_xlsx_custom(Importer):
             return
 
         # @todo cehck atleast 2 rows, same in other method
-        from .forms import validate_url
+        from changedetectionio.forms import validate_url
         row_i = 1
 
         try:
@@ -264,7 +264,7 @@ class import_xlsx_custom(Importer):
                         try:
                             validate_url(url)
                         except ValidationError as e:
-                            print(">> Import URL error", url, str(e))
+                            logger.error(f">> Import URL error {url} {str(e)}")
                             flash(f"Error processing row number {row_i}, URL value was incorrect, row was skipped.", 'error')
                             # Don't bother processing anything else on this row
                             url = None
@@ -293,10 +293,10 @@ class import_xlsx_custom(Importer):
                         self.new_uuids.append(new_uuid)
                         good += 1
         except Exception as e:
-            print(e)
+            logger.error(e)
             flash(f"Error processing row number {row_i}, check all cell data types are correct, row was skipped.", 'error')
         else:
             row_i += 1
 
         flash(
-            "{} imported from custom .xlsx in {:.2f}s".format(len(self.new_uuids), time.time() - now))
+            "{} imported from custom .xlsx in {:.2f}s".format(len(self.new_uuids), time.time() - now))

changedetectionio/blueprint/imports/templates/import.html

@@ -1,6 +1,6 @@
 {% extends 'base.html' %}
 {% block content %}
-{% from '_helpers.jinja' import render_field %}
+{% from '_helpers.html' import render_field %}
 <script src="{{url_for('static_content', group='js', filename='tabs.js')}}" defer></script>
 <div class="edit-form monospaced-textarea">
 
@@ -13,29 +13,27 @@
     </div>
 
     <div class="box-wrap inner">
-        <form class="pure-form" action="{{url_for('import_page')}}" method="POST" enctype="multipart/form-data">
+        <form class="pure-form" action="{{url_for('imports.import_page')}}" method="POST" enctype="multipart/form-data">
             <input type="hidden" name="csrf_token" value="{{ csrf_token() }}">
             <div class="tab-pane-inner" id="url-list">
-                    <legend>
+                <div class="pure-control-group">
                         Enter one URL per line, and optionally add tags for each URL after a space, delineated by comma
                         (,):
                         <br>
-                        <code>https://example.com tag1, tag2, last tag</code>
-                        <br>
+                        <p><strong>Example:  </strong><code>https://example.com tag1, tag2, last tag</code></p>
                         URLs which do not pass validation will stay in the textarea.
-                    </legend>
+                </div>
                 {{ render_field(form.processor, class="processor") }}
-
+                
+                <div class="pure-control-group">
                     <textarea name="urls" class="pure-input-1-2" placeholder="https://"
                               style="width: 100%;
                                 font-family:monospace;
                                 white-space: pre;
                                 overflow-wrap: normal;
                                 overflow-x: scroll;" rows="25">{{ import_url_list_remaining }}</textarea>
-
-<div id="quick-watch-processor-type">
-
-                    </div>
+                 </div>
+                 <div id="quick-watch-processor-type"></div>
 
             </div>
 
@@ -43,7 +41,7 @@
 
 
 
-                    <legend>
+                    <div class="pure-control-group">
                         Copy and Paste your Distill.io watch 'export' file, this should be a JSON file.<br>
                         This is <i>experimental</i>, supported fields are <code>name</code>, <code>uri</code>, <code>tags</code>, <code>config:selections</code>, the rest (including <code>schedule</code>) are ignored.
                         <br>
@@ -51,7 +49,7 @@
                         How to export? <a href="https://distill.io/docs/web-monitor/how-export-and-import-monitors/">https://distill.io/docs/web-monitor/how-export-and-import-monitors/</a><br>
                         Be sure to set your default fetcher to Chrome if required.<br>
                         </p>
-                    </legend>
+                    </div>
 
 
                     <textarea name="distill-io" class="pure-input-1-2" style="width: 100%;
@@ -107,7 +105,7 @@
                                     <option value="" style="color: #aaa"> -- none --</option>
                                     <option value="url">URL</option>
                                     <option value="title">Title</option>
-                                    <option value="include_filter">CSS/xPath filter</option>
+                                    <option value="include_filters">CSS/xPath filter</option>
                                     <option value="tag">Group / Tag name(s)</option>
                                     <option value="interval_minutes">Recheck time (minutes)</option>
                                 </select></td>
@@ -122,4 +120,4 @@
     </div>
 </div>
 
-{% endblock %}
+{% endblock %}

changedetectionio/blueprint/price_data_follower/__init__.py

@@ -1,9 +1,10 @@
 
-from distutils.util import strtobool
+from changedetectionio.strtobool import strtobool
 from flask import Blueprint, flash, redirect, url_for
 from flask_login import login_required
 from changedetectionio.store import ChangeDetectionStore
 from changedetectionio import queuedWatchMetaData
+from changedetectionio import worker_handler
 from queue import PriorityQueue
 
 PRICE_DATA_TRACK_ACCEPT = 'accepted'
@@ -17,15 +18,16 @@ def construct_blueprint(datastore: ChangeDetectionStore, update_q: PriorityQueue
     @price_data_follower_blueprint.route("/<string:uuid>/accept", methods=['GET'])
     def accept(uuid):
         datastore.data['watching'][uuid]['track_ldjson_price_data'] = PRICE_DATA_TRACK_ACCEPT
-        update_q.put(queuedWatchMetaData.PrioritizedItem(priority=1, item={'uuid': uuid, 'skip_when_checksum_same': False}))
-        return redirect(url_for("form_watch_checknow", uuid=uuid))
-
+        datastore.data['watching'][uuid]['processor'] = 'restock_diff'
+        datastore.data['watching'][uuid].clear_watch()
+        worker_handler.queue_item_async_safe(update_q, queuedWatchMetaData.PrioritizedItem(priority=1, item={'uuid': uuid}))
+        return redirect(url_for("watchlist.index"))
 
     @login_required
     @price_data_follower_blueprint.route("/<string:uuid>/reject", methods=['GET'])
     def reject(uuid):
         datastore.data['watching'][uuid]['track_ldjson_price_data'] = PRICE_DATA_TRACK_REJECT
-        return redirect(url_for("index"))
+        return redirect(url_for("watchlist.index"))
 
 
     return price_data_follower_blueprint

changedetectionio/blueprint/rss/__init__.py

@@ -0,0 +1 @@
+RSS_FORMAT_TYPES = [('plaintext', 'Plain text'), ('html', 'HTML Color')]

changedetectionio/blueprint/rss/blueprint.py

@@ -0,0 +1,154 @@
+
+from changedetectionio.jinja2_custom import render as jinja_render
+from changedetectionio.notification.handler import apply_service_tweaks
+from changedetectionio.store import ChangeDetectionStore
+from feedgen.feed import FeedGenerator
+from flask import Blueprint, make_response, request, url_for, redirect
+from loguru import logger
+import datetime
+import pytz
+import re
+import time
+
+
+BAD_CHARS_REGEX=r'[\x00-\x08\x0B\x0C\x0E-\x1F]'
+
+# Anything that is not text/UTF-8 should be stripped before it breaks feedgen (such as binary data etc)
+def scan_invalid_chars_in_rss(content):
+    for match in re.finditer(BAD_CHARS_REGEX, content):
+        i = match.start()
+        bad_char = content[i]
+        hex_value = f"0x{ord(bad_char):02x}"
+        # Grab context
+        start = max(0, i - 20)
+        end = min(len(content), i + 21)
+        context = content[start:end].replace('\n', '\\n').replace('\r', '\\r')
+        logger.warning(f"Invalid char {hex_value} at pos {i}: ...{context}...")
+        # First match is enough
+        return True
+
+    return False
+
+
+def clean_entry_content(content):
+    cleaned = re.sub(BAD_CHARS_REGEX, '', content)
+    return cleaned
+
+def construct_blueprint(datastore: ChangeDetectionStore):
+    rss_blueprint = Blueprint('rss', __name__)
+
+    # Some RSS reader situations ended up with rss/ (forward slash after RSS) due
+    # to some earlier blueprint rerouting work, it should goto feed.
+    @rss_blueprint.route("/", methods=['GET'])
+    def extraslash():
+        return redirect(url_for('rss.feed'))
+
+    # Import the login decorator if needed
+    # from changedetectionio.auth_decorator import login_optionally_required
+    @rss_blueprint.route("", methods=['GET'])
+    def feed():
+        now = time.time()
+        # Always requires token set
+        app_rss_token = datastore.data['settings']['application'].get('rss_access_token')
+        rss_url_token = request.args.get('token')
+        if rss_url_token != app_rss_token:
+            return "Access denied, bad token", 403
+
+        from changedetectionio import diff
+        limit_tag = request.args.get('tag', '').lower().strip()
+        # Be sure limit_tag is a uuid
+        for uuid, tag in datastore.data['settings']['application'].get('tags', {}).items():
+            if limit_tag == tag.get('title', '').lower().strip():
+                limit_tag = uuid
+
+        # Sort by last_changed and add the uuid which is usually the key..
+        sorted_watches = []
+
+        # @todo needs a .itemsWithTag() or something - then we can use that in Jinaj2 and throw this away
+        for uuid, watch in datastore.data['watching'].items():
+            # @todo tag notification_muted skip also (improve Watch model)
+            if datastore.data['settings']['application'].get('rss_hide_muted_watches') and watch.get('notification_muted'):
+                continue
+            if limit_tag and not limit_tag in watch['tags']:
+                continue
+            watch['uuid'] = uuid
+            sorted_watches.append(watch)
+
+        sorted_watches.sort(key=lambda x: x.last_changed, reverse=False)
+
+        fg = FeedGenerator()
+        fg.title('changedetection.io')
+        fg.description('Feed description')
+        fg.link(href='https://changedetection.io')
+
+        html_colour_enable = False
+        if datastore.data['settings']['application'].get('rss_content_format') == 'html':
+            html_colour_enable = True
+
+        for watch in sorted_watches:
+
+            dates = list(watch.history.keys())
+            # Re #521 - Don't bother processing this one if theres less than 2 snapshots, means we never had a change detected.
+            if len(dates) < 2:
+                continue
+
+            if not watch.viewed:
+                # Re #239 - GUID needs to be individual for each event
+                # @todo In the future make this a configurable link back (see work on BASE_URL https://github.com/dgtlmoon/changedetection.io/pull/228)
+                guid = "{}/{}".format(watch['uuid'], watch.last_changed)
+                fe = fg.add_entry()
+
+                # Include a link to the diff page, they will have to login here to see if password protection is enabled.
+                # Description is the page you watch, link takes you to the diff JS UI page
+                # Dict val base_url will get overriden with the env var if it is set.
+                ext_base_url = datastore.data['settings']['application'].get('active_base_url')
+                # @todo fix
+
+                # Because we are called via whatever web server, flask should figure out the right path (
+                diff_link = {'href': url_for('ui.ui_views.diff_history_page', uuid=watch['uuid'], _external=True)}
+
+                fe.link(link=diff_link)
+
+                # Same logic as watch-overview.html
+                if datastore.data['settings']['application']['ui'].get('use_page_title_in_list') or watch.get('use_page_title_in_list'):
+                    watch_label = watch.label
+                else:
+                    watch_label = watch.get('url')
+
+                fe.title(title=watch_label)
+                try:
+
+                    html_diff = diff.render_diff(previous_version_file_contents=watch.get_history_snapshot(dates[-2]),
+                                                 newest_version_file_contents=watch.get_history_snapshot(dates[-1]),
+                                                 include_equal=False
+                                                 )
+
+
+                    requested_output_format = 'htmlcolor' if html_colour_enable else 'html'
+                    html_diff = apply_service_tweaks(url='', n_body=html_diff, n_title=None, requested_output_format=requested_output_format)
+
+                except FileNotFoundError as e:
+                    html_diff = f"History snapshot file for watch {watch.get('uuid')}@{watch.last_changed} - '{watch.get('title')} not found."
+
+                # @todo Make this configurable and also consider html-colored markup
+                # @todo User could decide if <link> goes to the diff page, or to the watch link
+                rss_template = "<html><body>\n<h4><a href=\"{{watch_url}}\">{{watch_title}}</a></h4>\n<p>{{html_diff}}</p>\n</body></html>\n"
+
+                content = jinja_render(template_str=rss_template, watch_title=watch_label, html_diff=html_diff, watch_url=watch.link)
+
+                # Out of range chars could also break feedgen
+                if scan_invalid_chars_in_rss(content):
+                    content = clean_entry_content(content)
+
+                fe.content(content=content, type='CDATA')
+                fe.guid(guid, permalink=False)
+                dt = datetime.datetime.fromtimestamp(int(watch.newest_history_key))
+                dt = dt.replace(tzinfo=pytz.UTC)
+                fe.pubDate(dt)
+
+        response = make_response(fg.rss_str())
+        response.headers.set('Content-Type', 'application/rss+xml;charset=utf-8')
+        logger.trace(f"RSS generated in {time.time() - now:.3f}s")
+        return response
+
+    return rss_blueprint

changedetectionio/blueprint/settings/__init__.py

@@ -0,0 +1,145 @@
+import os
+from copy import deepcopy
+from datetime import datetime
+from zoneinfo import ZoneInfo, available_timezones
+import secrets
+import flask_login
+from flask import Blueprint, render_template, request, redirect, url_for, flash
+
+from changedetectionio.store import ChangeDetectionStore
+from changedetectionio.auth_decorator import login_optionally_required
+
+
+def construct_blueprint(datastore: ChangeDetectionStore):
+    settings_blueprint = Blueprint('settings', __name__, template_folder="templates")
+
+    @settings_blueprint.route("", methods=['GET', "POST"])
+    @login_optionally_required
+    def settings_page():
+        from changedetectionio import forms
+
+        default = deepcopy(datastore.data['settings'])
+        if datastore.proxy_list is not None:
+            available_proxies = list(datastore.proxy_list.keys())
+            # When enabled
+            system_proxy = datastore.data['settings']['requests']['proxy']
+            # In the case it doesnt exist anymore
+            if not system_proxy in available_proxies:
+                system_proxy = None
+
+            default['requests']['proxy'] = system_proxy if system_proxy is not None else available_proxies[0]
+            # Used by the form handler to keep or remove the proxy settings
+            default['proxy_list'] = available_proxies[0]
+
+        # Don't use form.data on POST so that it doesnt overrid the checkbox status from the POST status
+        form = forms.globalSettingsForm(formdata=request.form if request.method == 'POST' else None,
+                                        data=default,
+                                        extra_notification_tokens=datastore.get_unique_notification_tokens_available()
+                                        )
+
+        # Remove the last option 'System default'
+        form.application.form.notification_format.choices.pop()
+
+        if datastore.proxy_list is None:
+            # @todo - Couldn't get setattr() etc dynamic addition working, so remove it instead
+            del form.requests.form.proxy
+        else:
+            form.requests.form.proxy.choices = []
+            for p in datastore.proxy_list:
+                form.requests.form.proxy.choices.append(tuple((p, datastore.proxy_list[p]['label'])))
+
+        if request.method == 'POST':
+            # Password unset is a GET, but we can lock the session to a salted env password to always need the password
+            if form.application.form.data.get('removepassword_button', False):
+                # SALTED_PASS means the password is "locked" to what we set in the Env var
+                if not os.getenv("SALTED_PASS", False):
+                    datastore.remove_password()
+                    flash("Password protection removed.", 'notice')
+                    flask_login.logout_user()
+                    return redirect(url_for('settings.settings_page'))
+
+            if form.validate():
+                # Don't set password to False when a password is set - should be only removed with the `removepassword` button
+                app_update = dict(deepcopy(form.data['application']))
+
+                # Never update password with '' or False (Added by wtforms when not in submission)
+                if 'password' in app_update and not app_update['password']:
+                    del (app_update['password'])
+
+                datastore.data['settings']['application'].update(app_update)
+                
+                # Handle dynamic worker count adjustment
+                old_worker_count = datastore.data['settings']['requests'].get('workers', 1)
+                new_worker_count = form.data['requests'].get('workers', 1)
+                
+                datastore.data['settings']['requests'].update(form.data['requests'])
+                
+                # Adjust worker count if it changed
+                if new_worker_count != old_worker_count:
+                    from changedetectionio import worker_handler
+                    from changedetectionio.flask_app import update_q, notification_q, app, datastore as ds
+                    
+                    result = worker_handler.adjust_async_worker_count(
+                        new_count=new_worker_count,
+                        update_q=update_q,
+                        notification_q=notification_q,
+                        app=app,
+                        datastore=ds
+                    )
+                    
+                    if result['status'] == 'success':
+                        flash(f"Worker count adjusted: {result['message']}", 'notice')
+                    elif result['status'] == 'not_supported':
+                        flash("Dynamic worker adjustment not supported for sync workers", 'warning')
+                    elif result['status'] == 'error':
+                        flash(f"Error adjusting workers: {result['message']}", 'error')
+
+                if not os.getenv("SALTED_PASS", False) and len(form.application.form.password.encrypted_password):
+                    datastore.data['settings']['application']['password'] = form.application.form.password.encrypted_password
+                    datastore.needs_write_urgent = True
+                    flash("Password protection enabled.", 'notice')
+                    flask_login.logout_user()
+                    return redirect(url_for('watchlist.index'))
+
+                datastore.needs_write_urgent = True
+                flash("Settings updated.")
+
+            else:
+                flash("An error occurred, please see below.", "error")
+
+        # Convert to ISO 8601 format, all date/time relative events stored as UTC time
+        utc_time = datetime.now(ZoneInfo("UTC")).isoformat()
+
+        output = render_template("settings.html",
+                                api_key=datastore.data['settings']['application'].get('api_access_token'),
+                                available_timezones=sorted(available_timezones()),
+                                emailprefix=os.getenv('NOTIFICATION_MAIL_BUTTON_PREFIX', False),
+                                extra_notification_token_placeholder_info=datastore.get_unique_notification_token_placeholders_available(),
+                                form=form,
+                                hide_remove_pass=os.getenv("SALTED_PASS", False),
+                                min_system_recheck_seconds=int(os.getenv('MINIMUM_SECONDS_RECHECK_TIME', 3)),
+                                settings_application=datastore.data['settings']['application'],
+                                timezone_default_config=datastore.data['settings']['application'].get('scheduler_timezone_default'),
+                                utc_time=utc_time,
+                                )
+
+        return output
+
+    @settings_blueprint.route("/reset-api-key", methods=['GET'])
+    @login_optionally_required
+    def settings_reset_api_key():
+        secret = secrets.token_hex(16)
+        datastore.data['settings']['application']['api_access_token'] = secret
+        datastore.needs_write_urgent = True
+        flash("API Key was regenerated.")
+        return redirect(url_for('settings.settings_page')+'#api')
+        
+    @settings_blueprint.route("/notification-logs", methods=['GET'])
+    @login_optionally_required
+    def notification_logs():
+        from changedetectionio.flask_app import notification_debug_log
+        output = render_template("notification-log.html",
+                               logs=notification_debug_log if len(notification_debug_log) else ["Notification logs are empty - no notifications sent yet."])
+        return output
+
+    return settings_blueprint

changedetectionio/blueprint/settings/templates/settings.html

@@ -1,18 +1,20 @@
 {% extends 'base.html' %}
 
 {% block content %}
-{% from '_helpers.jinja' import render_field, render_checkbox_field, render_button %}
-{% from '_common_fields.jinja' import render_common_settings_form %}
+{% from '_helpers.html' import render_field, render_checkbox_field, render_button, render_time_schedule_form, render_ternary_field, render_fieldlist_with_inline_errors %}
+{% from '_common_fields.html' import render_common_settings_form %}
 <script>
-    const notification_base_url="{{url_for('ajax_callback_send_notification_test')}}";
+    const notification_base_url="{{url_for('ui.ui_notification.ajax_callback_send_notification_test', mode="global-settings")}}";
 {% if emailprefix %}
     const email_notification_prefix=JSON.parse('{{emailprefix|tojson}}');
 {% endif %}
 </script>
 <script src="{{url_for('static_content', group='js', filename='tabs.js')}}" defer></script>
+<script src="{{url_for('static_content', group='js', filename='plugins.js')}}" defer></script>
 <script src="{{url_for('static_content', group='js', filename='notifications.js')}}" defer></script>
-
+<script src="{{url_for('static_content', group='js', filename='vis.js')}}" defer></script>
 <script src="{{url_for('static_content', group='js', filename='global-settings.js')}}" defer></script>
+<script src="{{url_for('static_content', group='js', filename='scheduler.js')}}" defer></script>
 <div class="edit-form">
     <div class="tabs collapsable">
         <ul>
@@ -20,18 +22,26 @@
             <li class="tab"><a href="#notifications">Notifications</a></li>
             <li class="tab"><a href="#fetching">Fetching</a></li>
             <li class="tab"><a href="#filters">Global Filters</a></li>
+            <li class="tab"><a href="#ui-options">UI Options</a></li>
             <li class="tab"><a href="#api">API</a></li>
+            <li class="tab"><a href="#timedate">Time &amp Date</a></li>
             <li class="tab"><a href="#proxies">CAPTCHA &amp; Proxies</a></li>
         </ul>
     </div>
     <div class="box-wrap inner">
-        <form class="pure-form pure-form-stacked settings" action="{{url_for('settings_page')}}" method="POST">
+        <form class="pure-form pure-form-stacked settings" action="{{url_for('settings.settings_page')}}" method="POST">
             <input type="hidden" name="csrf_token" value="{{ csrf_token() }}" >
             <div class="tab-pane-inner" id="general">
                 <fieldset>
                     <div class="pure-control-group">
                         {{ render_field(form.requests.form.time_between_check, class="time-check-widget") }}
-                        <span class="pure-form-message-inline">Default time for all watches, when the watch does not have a specific time setting.</span>
+                        <span class="pure-form-message-inline">Default recheck time for all watches, current system minimum is <i>{{min_system_recheck_seconds}}</i> seconds (<a href="https://github.com/dgtlmoon/changedetection.io/wiki/Misc-system-settings#enviroment-variables">more info</a>).</span>
+                            <div id="time-between-check-schedule">
+                                <!-- Start Time and End Time -->
+                                <div id="limit-between-time">
+                                    {{ render_time_schedule_form(form.requests, available_timezones, timezone_default_config) }}
+                                </div>
+                        </div>
                     </div>
                     <div class="pure-control-group">
                         {{ render_field(form.requests.form.jitter_seconds, class="jitter_seconds") }}
@@ -62,34 +72,30 @@
                         <span class="pure-form-message-inline">Allow access to view watch diff page when password is enabled (Good for sharing the diff page)
                         </span>
                     </div>
-                    <div class="pure-control-group">
-                        {{ render_field(form.application.form.pager_size) }}
-                        <span class="pure-form-message-inline">Number of items per page in the watch overview list, 0 to disable.</span>
-                    </div>
-
-                    <div class="pure-control-group">
-                        {{ render_checkbox_field(form.application.form.extract_title_as_title) }}
-                        <span class="pure-form-message-inline">Note: This will automatically apply to all existing watches.</span>
-                    </div>
                     <div class="pure-control-group">
                         {{ render_checkbox_field(form.application.form.empty_pages_are_a_change) }}
-                        <span class="pure-form-message-inline">When a page contains HTML, but no renderable text appears (empty page), is this considered a change?</span>
+                        <span class="pure-form-message-inline">When a request returns no content, or the HTML does not contain any text, is this considered a change?</span>
                     </div>
-                {% if form.requests.proxy %}
-                    <div class="pure-control-group inline-radio">
-                        {{ render_field(form.requests.form.proxy, class="fetch-backend-proxy") }}
-                        <span class="pure-form-message-inline">
-                        Choose a default proxy for all watches
-                        </span>
+                    <div class="grey-form-border">
+                        <div class="pure-control-group">
+                            {{ render_checkbox_field(form.application.form.rss_hide_muted_watches) }}
+                        </div>
+                        <div class="pure-control-group">
+                            {{ render_field(form.application.form.rss_content_format) }}
+                            <span class="pure-form-message-inline">Love RSS? Does your reader support HTML? Set it here</span>
+                        </div>
+                        <div class="pure-control-group">
+                            {{ render_checkbox_field(form.application.form.rss_reader_mode) }}
+                            <span class="pure-form-message-inline">Transforms RSS/RDF feed watches into beautiful text only</span>
+                        </div>
                     </div>
-                {% endif %}
                 </fieldset>
             </div>
 
             <div class="tab-pane-inner" id="notifications">
                 <fieldset>
                     <div class="field-group">
-                        {{ render_common_settings_form(form.application.form, emailprefix, settings_application) }}
+                        {{ render_common_settings_form(form.application.form, emailprefix, settings_application, extra_notification_token_placeholder_info) }}
                     </div>
                 </fieldset>
                 <div class="pure-control-group" id="notification-base-url">
@@ -108,10 +114,8 @@
                         <p>Use the <strong>Basic</strong> method (default) where your watched sites don't need Javascript to render.</p>
                         <p>The <strong>Chrome/Javascript</strong> method requires a network connection to a running WebDriver+Chrome server, set by the ENV var 'WEBDRIVER_URL'. </p>
                     </span>
-                    <br>
-                    Tip: <a href="https://github.com/dgtlmoon/changedetection.io/wiki/Proxy-configuration#brightdata-proxy-support">Connect using Bright Data and Oxylabs Proxies, find out more here.</a>
                 </div>
-                <fieldset class="pure-group" id="webdriver-override-options">
+                <fieldset class="pure-group" id="webdriver-override-options" data-visible-for="application-fetch_backend=html_webdriver">
                     <div class="pure-form-message-inline">
                         <strong>If you're having trouble waiting for the page to be fully rendered (text missing etc), try increasing the 'wait' time here.</strong>
                         <br>
@@ -121,6 +125,28 @@
                         {{ render_field(form.application.form.webdriver_delay) }}
                     </div>
                 </fieldset>
+                <div class="pure-control-group">
+                    {{ render_field(form.requests.form.workers) }}
+                    {% set worker_info = get_worker_status_info() %}
+                    <span class="pure-form-message-inline">Number of concurrent workers to process watches. More workers = faster processing but higher memory usage.<br>
+                    Currently running: <strong>{{ worker_info.count }}</strong> operational {{ worker_info.type }} workers{% if worker_info.active_workers > 0 %} ({{ worker_info.active_workers }} actively processing){% endif %}.</span>
+                </div>
+                <div class="pure-control-group">
+                    {{ render_field(form.requests.form.timeout) }}
+                    <span class="pure-form-message-inline">For regular plain requests (not chrome based), maximum number of seconds until timeout, 1-999.<br>
+                </div>
+                <div class="pure-control-group inline-radio">
+                    {{ render_field(form.requests.form.default_ua) }}
+                    <span class="pure-form-message-inline">
+                        Applied to all requests.<br><br>
+                        Note: Simply changing the User-Agent often does not defeat anti-robot technologies, it's important to consider <a href="https://changedetection.io/tutorial/what-are-main-types-anti-robot-mechanisms">all of the ways that the browser is detected</a>.
+                    </span>
+                </div>
+                <div class="pure-control-group">
+                                        <br>
+                    Tip: <a href="https://github.com/dgtlmoon/changedetection.io/wiki/Proxy-configuration#brightdata-proxy-support">Connect using Bright Data and Oxylabs Proxies, find out more here.</a>
+
+                </div>
             </div>
 
             <div class="tab-pane-inner" id="filters">
@@ -142,11 +168,13 @@
                       {{ render_field(form.application.form.global_subtractive_selectors, rows=5, placeholder="header
 footer
 nav
-.stockticker") }}
+.stockticker
+//*[contains(text(), 'Advertisement')]") }}
                       <span class="pure-form-message-inline">
                         <ul>
-                          <li> Remove HTML element(s) by CSS selector before text conversion. </li>
-                          <li> Add multiple elements or CSS selectors per line to ignore multiple parts of the HTML. </li>
+                          <li> Remove HTML element(s) by CSS and XPath selectors before text conversion. </li>
+                          <li> Don't paste HTML here, use only CSS and XPath selectors </li>
+                          <li> Add multiple elements, CSS or XPath selectors per line to ignore multiple parts of the HTML. </li>
                         </ul>
                       </span>
                     </fieldset>
@@ -157,27 +185,87 @@ nav
                     <span class="pure-form-message-inline">Note: This is applied globally in addition to the per-watch rules.</span><br>
                     <span class="pure-form-message-inline">
                         <ul>
+                            <li>Matching text will be <strong>ignored</strong> in the text snapshot (you can still see it but it wont trigger a change)</li>
                             <li>Note: This is applied globally in addition to the per-watch rules.</li>
                             <li>Each line processed separately, any line matching will be ignored (removed before creating the checksum)</li>
                             <li>Regular Expression support, wrap the entire line in forward slash <code>/regex/</code></li>
                             <li>Changing this will affect the comparison checksum which may trigger an alert</li>
-                            <li>Use the preview/show current tab to see ignores</li>
                         </ul>
                      </span>
                     </fieldset>
+                    <fieldset class="pure-group">
+                        {{ render_checkbox_field(form.application.form.strip_ignored_lines) }}
+                        <span class="pure-form-message-inline">Remove any text that appears in the "Ignore text" from the output (otherwise its just ignored for change-detection)<br>
+                        <i>Note:</i> Changing this will change the status of your existing watches, possibly trigger alerts etc.
+                        </span>
+                    </fieldset>
            </div>
 
             <div class="tab-pane-inner" id="api">
-
-                <p>Drive your changedetection.io via API, More about <a href="https://github.com/dgtlmoon/changedetection.io/wiki/API-Reference">API access here</a></p>
+                <h4>API Access</h4>
+                <p>Drive your changedetection.io via API, More about <a href="https://changedetection.io/docs/api_v1/index.html">API access and examples here</a>.</p>
 
                 <div class="pure-control-group">
                     {{ render_checkbox_field(form.application.form.api_access_token_enabled) }}
-                    <div class="pure-form-message-inline">Restrict API access limit by using <code>x-api-key</code> header</div><br>
+                    <div class="pure-form-message-inline">Restrict API access limit by using <code>x-api-key</code> header - required for the Chrome Extension to work</div><br>
                     <div class="pure-form-message-inline"><br>API Key <span id="api-key">{{api_key}}</span>
                         <span style="display:none;" id="api-key-copy" >copy</span>
                     </div>
                 </div>
+                <div class="pure-control-group">
+                    <a href="{{url_for('settings.settings_reset_api_key')}}" class="pure-button button-small button-cancel">Regenerate API key</a>
+                </div>
+                <div class="pure-control-group">
+                    <h4>Chrome Extension</h4>
+                    <p>Easily add any web-page to your changedetection.io installation from within Chrome.</p>
+                    <strong>Step 1</strong> Install the extension, <strong>Step 2</strong> Navigate to this page,
+                    <strong>Step 3</strong> Open the extension from the toolbar and click "<i>Sync API Access</i>"
+                    <p>
+                        <a id="chrome-extension-link"
+                           title="Try our new Chrome Extension!"
+                           href="https://chromewebstore.google.com/detail/changedetectionio-website/kefcfmgmlhmankjmnbijimhofdjekbop">
+                            <img alt="Chrome store icon" src="{{ url_for('static_content', group='images', filename='google-chrome-icon.png') }}" alt="Chrome">
+                            Chrome Webstore
+                        </a>
+                    </p>
+                </div>
+            </div>
+            <div class="tab-pane-inner" id="timedate">
+                <div class="pure-control-group">
+                    Ensure the settings below are correct, they are used to manage the time schedule for checking your web page watches.
+                </div>
+                <div class="pure-control-group">
+                    <p><strong>UTC Time &amp Date from Server:</strong> <span id="utc-time" >{{ utc_time }}</span></p>
+                    <p><strong>Local Time &amp Date in Browser:</strong> <span class="local-time" data-utc="{{ utc_time }}"></span></p>
+                    <p>
+                       {{ render_field(form.application.form.scheduler_timezone_default) }}
+                        <datalist id="timezones" style="display: none;">
+                            {%- for timezone in available_timezones -%}<option value="{{ timezone }}">{{ timezone }}</option>{%- endfor -%}
+                        </datalist>
+                    </p>
+                </div>
+            </div>
+            <div class="tab-pane-inner" id="ui-options">
+                <div class="pure-control-group">
+                    {{ render_checkbox_field(form.application.form.ui.form.open_diff_in_new_tab, class="open_diff_in_new_tab") }}
+                    <span class="pure-form-message-inline">Enable this setting to open the diff page in a new tab. If disabled, the diff page will open in the current tab.</span>
+                </div>
+                <div class="pure-control-group">
+                    {{ render_checkbox_field(form.application.form.ui.form.socket_io_enabled, class="socket_io_enabled") }}
+                    <span class="pure-form-message-inline">Realtime UI Updates Enabled - (Restart required if this is changed)</span>
+                </div>
+                <div class="pure-control-group">
+                    {{ render_checkbox_field(form.application.form.ui.form.favicons_enabled, class="") }}
+                    <span class="pure-form-message-inline">Enable or Disable Favicons next to the watch list</span>
+                </div>
+                <div class="pure-control-group">
+                    {{ render_checkbox_field(form.application.form.ui.use_page_title_in_list) }}
+                </div>
+                <div class="pure-control-group">
+                    {{ render_field(form.application.form.pager_size) }}
+                    <span class="pure-form-message-inline">Number of items per page in the watch overview list, 0 to disable.</span>
+                </div>
+
             </div>
             <div class="tab-pane-inner" id="proxies">
                 <div id="recommended-proxy">
@@ -222,22 +310,37 @@ nav
                         
                     </div>
                 </div>
-                <p>
-                    Your proxy provider may need to whitelist our IP of <code>204.15.192.195</code>
-                </p>
+
                <p><strong>Tip</strong>: "Residential" and "Mobile" proxy type can be more successfull than "Data Center" for blocked websites.
 
-                <div class="pure-control-group">
-                {{ render_field(form.requests.form.extra_proxies) }}
+                <div class="pure-control-group" id="extra-proxies-setting">
+                {{ render_fieldlist_with_inline_errors(form.requests.form.extra_proxies) }}
                 <span class="pure-form-message-inline">"Name" will be used for selecting the proxy in the Watch Edit settings</span><br>
                 <span class="pure-form-message-inline">SOCKS5 proxies with authentication are only supported with 'plain requests' fetcher, for other fetchers you should whitelist the IP access instead</span>
+                {% if form.requests.proxy %}
+                <div>
+                <br>
+                    <div class="inline-radio">
+                        {{ render_field(form.requests.form.proxy, class="fetch-backend-proxy") }}
+                        <span class="pure-form-message-inline">Choose a default proxy for all watches</span>
+                    </div>
                 </div>
+                {% endif %}
+                </div>
+                <div class="pure-control-group" id="extra-browsers-setting">
+                    <p>
+                    <span class="pure-form-message-inline"><i>Extra Browsers</i> can be attached to further defeat CAPTCHA's on websites that are particularly hard to scrape.</span><br>
+                    <span class="pure-form-message-inline">Simply paste the connection address into the box, <a href="https://changedetection.io/tutorial/using-bright-datas-scraping-browser-pass-captchas-and-other-protection-when-monitoring">More instructions and examples here</a> </span>
+                    </p>
+                    {{ render_fieldlist_with_inline_errors(form.requests.form.extra_browsers) }}
+                </div>
+            
             </div>
             <div id="actions">
                 <div class="pure-control-group">
                     {{ render_button(form.save_button) }}
-                    <a href="{{url_for('index')}}" class="pure-button button-small button-cancel">Back</a>
-                    <a href="{{url_for('clear_all_history')}}" class="pure-button button-small button-cancel">Clear Snapshot History</a>
+                    <a href="{{url_for('watchlist.index')}}" class="pure-button button-cancel">Back</a>
+                    <a href="{{url_for('ui.clear_all_history')}}" class="pure-button button-error">Clear Snapshot History</a>
                 </div>
             </div>
         </form>

changedetectionio/blueprint/tags/__init__.py

@@ -1,6 +1,8 @@
-from flask import Blueprint, request, make_response, render_template, flash, url_for, redirect
+from flask import Blueprint, request, render_template, flash, url_for, redirect
+
+
 from changedetectionio.store import ChangeDetectionStore
-from changedetectionio import login_optionally_required
+from changedetectionio.flask_app import login_optionally_required
 
 
 def construct_blueprint(datastore: ChangeDetectionStore):
@@ -11,9 +13,17 @@ def construct_blueprint(datastore: ChangeDetectionStore):
     def tags_overview_page():
         from .form import SingleTag
         add_form = SingleTag(request.form)
+
+        sorted_tags = sorted(datastore.data['settings']['application'].get('tags').items(), key=lambda x: x[1]['title'])
+
+        from collections import Counter
+
+        tag_count = Counter(tag for watch in datastore.data['watching'].values() if watch.get('tags') for tag in watch['tags'])
+
         output = render_template("groups-overview.html",
+                                 available_tags=sorted_tags,
                                  form=add_form,
-                                 available_tags=datastore.data['settings']['application'].get('tags', {}),
+                                 tag_count=tag_count
                                  )
 
         return output
@@ -89,22 +99,60 @@ def construct_blueprint(datastore: ChangeDetectionStore):
     @tags_blueprint.route("/edit/<string:uuid>", methods=['GET'])
     @login_optionally_required
     def form_tag_edit(uuid):
-        from changedetectionio import forms
-
+        from changedetectionio.blueprint.tags.form import group_restock_settings_form
         if uuid == 'first':
             uuid = list(datastore.data['settings']['application']['tags'].keys()).pop()
 
         default = datastore.data['settings']['application']['tags'].get(uuid)
+        if not default:
+            flash("Tag not found", "error")
+            return redirect(url_for('watchlist.index'))
 
-        form = forms.watchForm(formdata=request.form if request.method == 'POST' else None,
-                               data=default,
-                               )
-        form.datastore=datastore # needed?
+        form = group_restock_settings_form(
+                                       formdata=request.form if request.method == 'POST' else None,
+                                       data=default,
+                                       extra_notification_tokens=datastore.get_unique_notification_tokens_available(),
+                                       default_system_settings = datastore.data['settings'],
+                                       )
+
+        template_args = {
+            'data': default,
+            'form': form,
+            'watch': default,
+            'extra_notification_token_placeholder_info': datastore.get_unique_notification_token_placeholders_available(),
+        }
+
+        included_content = {}
+        if form.extra_form_content():
+            # So that the extra panels can access _helpers.html etc, we set the environment to load from templates/
+            # And then render the code from the module
+            from jinja2 import Environment, FileSystemLoader
+            import importlib.resources
+            templates_dir = str(importlib.resources.files("changedetectionio").joinpath('templates'))
+            env = Environment(loader=FileSystemLoader(templates_dir))
+            template_str = """{% from '_helpers.html' import render_field, render_checkbox_field, render_button %}
+        <script>        
+            $(document).ready(function () {
+                toggleOpacity('#overrides_watch', '#restock-fieldset-price-group', true);
+            });
+        </script>            
+                <fieldset>
+                    <div class="pure-control-group">
+                        <fieldset class="pure-group">
+                        {{ render_checkbox_field(form.overrides_watch) }}
+                        <span class="pure-form-message-inline">Used for watches in "Restock & Price detection" mode</span>
+                        </fieldset>
+                </fieldset>
+                """
+            template_str += form.extra_form_content()
+            template = env.from_string(template_str)
+            included_content = template.render(**template_args)
 
         output = render_template("edit-tag.html",
-                                 data=default,
-                                 form=form,
                                  settings_application=datastore.data['settings']['application'],
+                                 extra_tab_content=form.extra_tab_content() if form.extra_tab_content() else None,
+                                 extra_form_content=included_content,
+                                 **template_args
                                  )
 
         return output
@@ -113,14 +161,15 @@ def construct_blueprint(datastore: ChangeDetectionStore):
     @tags_blueprint.route("/edit/<string:uuid>", methods=['POST'])
     @login_optionally_required
     def form_tag_edit_submit(uuid):
-        from changedetectionio import forms
+        from changedetectionio.blueprint.tags.form import group_restock_settings_form
         if uuid == 'first':
             uuid = list(datastore.data['settings']['application']['tags'].keys()).pop()
 
         default = datastore.data['settings']['application']['tags'].get(uuid)
 
-        form = forms.watchForm(formdata=request.form if request.method == 'POST' else None,
+        form = group_restock_settings_form(formdata=request.form if request.method == 'POST' else None,
                                data=default,
+                               extra_notification_tokens=datastore.get_unique_notification_tokens_available()
                                )
         # @todo subclass form so validation works
         #if not form.validate():
@@ -129,6 +178,7 @@ def construct_blueprint(datastore: ChangeDetectionStore):
 #           return redirect(url_for('tags.form_tag_edit_submit', uuid=uuid))
 
         datastore.data['settings']['application']['tags'][uuid].update(form.data)
+        datastore.data['settings']['application']['tags'][uuid]['processor'] = 'restock_diff'
         datastore.needs_write_urgent = True
         flash("Updated")
 

changedetectionio/blueprint/tags/form.py

@@ -1,16 +1,15 @@
 from wtforms import (
-    BooleanField,
     Form,
-    IntegerField,
-    RadioField,
-    SelectField,
     StringField,
     SubmitField,
-    TextAreaField,
     validators,
 )
+from wtforms.fields.simple import BooleanField
 
+from changedetectionio.processors.restock_diff.forms import processor_settings_form as restock_settings_form
 
+class group_restock_settings_form(restock_settings_form):
+    overrides_watch = BooleanField('Activate for individual watches in this tag/group?', default=False)
 
 class SingleTag(Form):
 

changedetectionio/blueprint/tags/templates/edit-tag.html

@@ -1,9 +1,9 @@
 {% extends 'base.html' %}
 {% block content %}
-{% from '_helpers.jinja' import render_field, render_checkbox_field, render_button %}
-{% from '_common_fields.jinja' import render_common_settings_form %}
+{% from '_helpers.html' import render_field, render_checkbox_field, render_button, render_ternary_field %}
+{% from '_common_fields.html' import render_common_settings_form %}
 <script>
-    const notification_base_url="{{url_for('ajax_callback_send_notification_test')}}";
+    const notification_base_url="{{url_for('ui.ui_notification.ajax_callback_send_notification_test', mode="group-settings")}}";
 </script>
 
 <script src="{{url_for('static_content', group='js', filename='tabs.js')}}" defer></script>
@@ -13,11 +13,11 @@
     /*const email_notification_prefix=JSON.parse('{{ emailprefix|tojson }}');*/
 /*{% endif %}*/
 
+{% set has_tag_filters_extra='' %}
 
 </script>
 
 <script src="{{url_for('static_content', group='js', filename='watch-settings.js')}}" defer></script>
-<!--<script src="{{url_for('static_content', group='js', filename='limit.js')}}" defer></script>-->
 <script src="{{url_for('static_content', group='js', filename='notifications.js')}}" defer></script>
 
 <div class="edit-form monospaced-textarea">
@@ -26,6 +26,9 @@
         <ul>
             <li class="tab" id=""><a href="#general">General</a></li>
             <li class="tab"><a href="#filters-and-triggers">Filters &amp; Triggers</a></li>
+            {% if extra_tab_content %}
+            <li class="tab"><a href="#extras_tab">{{ extra_tab_content }}</a></li>
+            {% endif %}
             <li class="tab"><a href="#notifications">Notifications</a></li>
         </ul>
     </div>
@@ -44,64 +47,26 @@
             </div>
 
             <div class="tab-pane-inner" id="filters-and-triggers">
-                    <div class="pure-control-group">
-                        {% set field = render_field(form.include_filters,
-                            rows=5,
-                            placeholder="#example
-xpath://body/div/span[contains(@class, 'example-class')]",
-                            class="m-d")
-                        %}
-                        {{ field }}
-                        {% if '/text()' in  field %}
-                          <span class="pure-form-message-inline"><strong>Note!: //text() function does not work where the &lt;element&gt; contains &lt;![CDATA[]]&gt;</strong></span><br>
-                        {% endif %}
-                        <span class="pure-form-message-inline">One rule per line, <i>any</i> rules that matches will be used.<br>
-
-                    <ul>
-                        <li>CSS - Limit text to this CSS rule, only text matching this CSS rule is included.</li>
-                        <li>JSON - Limit text to this JSON rule, using either <a href="https://pypi.org/project/jsonpath-ng/" target="new">JSONPath</a> or <a href="https://stedolan.github.io/jq/" target="new">jq</a> (if installed).
-                            <ul>
-                                <li>JSONPath: Prefix with <code>json:</code>, use <code>json:$</code> to force re-formatting if required,  <a href="https://jsonpath.com/" target="new">test your JSONPath here</a>.</li>
-                                {% if jq_support %}
-                                <li>jq: Prefix with <code>jq:</code> and <a href="https://jqplay.org/" target="new">test your jq here</a>. Using <a href="https://stedolan.github.io/jq/" target="new">jq</a> allows for complex filtering and processing of JSON data with built-in functions, regex, filtering, and more. See examples and documentation <a href="https://stedolan.github.io/jq/manual/" target="new">here</a>.</li>
-                                {% else %}
-                                <li>jq support not installed</li>
-                                {% endif %}
-                            </ul>
-                        </li>
-                        <li>XPath - Limit text to this XPath rule, simply start with a forward-slash,
-                            <ul>
-                                <li>Example:  <code>//*[contains(@class, 'sametext')]</code> or <code>xpath://*[contains(@class, 'sametext')]</code>, <a
-                                href="http://xpather.com/" target="new">test your XPath here</a></li>
-                                <li>Example: Get all titles from an RSS feed <code>//title/text()</code></li>
-                            </ul>
-                            </li>
-                    </ul>
-                    Please be sure that you thoroughly understand how to write CSS, JSONPath, XPath{% if jq_support %}, or jq selector{%endif%} rules before filing an issue on GitHub! <a
-                                href="https://github.com/dgtlmoon/changedetection.io/wiki/CSS-Selector-help">here for more CSS selector help</a>.<br>
-                </span>
-                    </div>
-                <fieldset class="pure-control-group">
-                    {{ render_field(form.subtractive_selectors, rows=5, placeholder="header
-footer
-nav
-.stockticker") }}
-                    <span class="pure-form-message-inline">
-                        <ul>
-                          <li> Remove HTML element(s) by CSS selector before text conversion. </li>
-                          <li> Add multiple elements or CSS selectors per line to ignore multiple parts of the HTML. </li>
-                        </ul>
-                      </span>
-                </fieldset>
-
+                <p>These settings are <strong><i>added</i></strong> to any existing watch configurations.</p>
+                {% include "edit/include_subtract.html" %}
+                <div class="text-filtering border-fieldset">
+                    <h3>Text filtering</h3>
+                    {% include "edit/text-options.html" %}
+                </div>
             </div>
 
+        {# rendered sub Template #}
+        {% if extra_form_content %}
+            <div class="tab-pane-inner" id="extras_tab">
+            {{ extra_form_content|safe }}
+            </div>
+        {% endif %}
             <div class="tab-pane-inner" id="notifications">
                 <fieldset>
                     <div  class="pure-control-group inline-radio">
-                      {{ render_checkbox_field(form.notification_muted) }}
+                      {{ render_ternary_field(form.notification_muted, BooleanField=True) }}
                     </div>
-                    {% if is_html_webdriver %}
+                    {% if 1 %}
                     <div class="pure-control-group inline-radio">
                       {{ render_checkbox_field(form.notification_screenshot) }}
                         <span class="pure-form-message-inline">
@@ -113,12 +78,12 @@ nav
                         {% if has_default_notification_urls %}
                         <div class="inline-warning">
                             <img class="inline-warning-icon" src="{{url_for('static_content', group='images', filename='notice.svg')}}" alt="Look out!" title="Lookout!" >
-                            There are <a href="{{ url_for('settings_page')}}#notifications">system-wide notification URLs enabled</a>, this form will override notification settings for this watch only &dash; an empty Notification URL list here will still send notifications.
+                            There are <a href="{{ url_for('settings.settings_page')}}#notifications">system-wide notification URLs enabled</a>, this form will override notification settings for this watch only &dash; an empty Notification URL list here will still send notifications.
                         </div>
                         {% endif %}
                         <a href="#notifications" id="notification-setting-reset-to-default" class="pure-button button-xsmall" style="right: 20px; top: 20px; position: absolute; background-color: #5f42dd; border-radius: 4px; font-size: 70%; color: #fff">Use system defaults</a>
 
-                        {{ render_common_settings_form(form, emailprefix, settings_application) }}
+                        {{ render_common_settings_form(form, emailprefix, settings_application, extra_notification_token_placeholder_info) }}
                     </div>
                 </fieldset>
             </div>

changedetectionio/blueprint/tags/templates/groups-overview.html

@@ -1,6 +1,6 @@
 {% extends 'base.html' %}
 {% block content %}
-{% from '_helpers.jinja' import render_simple_field, render_field %}
+{% from '_helpers.html' import render_simple_field, render_field %}
 <script src="{{url_for('static_content', group='js', filename='jquery-3.6.0.min.js')}}"></script>
 
 <div class="box">
@@ -10,7 +10,7 @@
             <legend>Add a new organisational tag</legend>
             <div id="watch-add-wrapper-zone">
                 <div>
-                    {{ render_simple_field(form.name, placeholder="watch label / tag") }}
+                    {{ render_simple_field(form.name, placeholder="Watch group / tag") }}
                 </div>
                 <div>
                     {{ render_simple_field(form.save_button, title="Save" ) }}
@@ -27,6 +27,7 @@
             <thead>
             <tr>
                 <th></th>
+                <th># Watches</th>
                 <th>Tag / Label name</th>
                 <th></th>
             </tr>
@@ -40,12 +41,13 @@
                 <td colspan="3">No website organisational tags/groups configured</td>
             </tr>
             {% endif %}
-            {% for uuid, tag in available_tags.items()  %}
+            {% for uuid, tag in available_tags  %}
             <tr id="{{ uuid }}" class="{{ loop.cycle('pure-table-odd', 'pure-table-even') }}">
                 <td class="watch-controls">
                     <a class="link-mute state-{{'on' if tag.notification_muted else 'off'}}" href="{{url_for('tags.mute', uuid=tag.uuid)}}"><img src="{{url_for('static_content', group='images', filename='bell-off.svg')}}" alt="Mute notifications" title="Mute notifications" class="icon icon-mute" ></a>
                 </td>
-                <td class="title-col inline">{{tag.title}}</td>
+                <td>{{ "{:,}".format(tag_count[uuid]) if uuid in tag_count else 0 }}</td>
+                <td class="title-col inline"> <a href="{{url_for('watchlist.index', tag=uuid) }}">{{ tag.title }}</a></td>
                 <td>
                     <a class="pure-button pure-button-primary" href="{{ url_for('tags.form_tag_edit', uuid=uuid) }}">Edit</a>&nbsp;
                     <a class="pure-button pure-button-primary" href="{{ url_for('tags.delete', uuid=uuid) }}" title="Deletes and removes tag">Delete</a>

changedetectionio/blueprint/ui/__init__.py

@@ -0,0 +1,325 @@
+import time
+from flask import Blueprint, request, redirect, url_for, flash, render_template, session
+from loguru import logger
+
+from changedetectionio.store import ChangeDetectionStore
+from changedetectionio.blueprint.ui.edit import construct_blueprint as construct_edit_blueprint
+from changedetectionio.blueprint.ui.notification import construct_blueprint as construct_notification_blueprint
+from changedetectionio.blueprint.ui.views import construct_blueprint as construct_views_blueprint
+
+def _handle_operations(op, uuids, datastore, worker_handler, update_q, queuedWatchMetaData, watch_check_update, extra_data=None, emit_flash=True):
+    from flask import request, flash
+
+    if op == 'delete':
+        for uuid in uuids:
+            if datastore.data['watching'].get(uuid):
+                datastore.delete(uuid)
+        if emit_flash:
+            flash(f"{len(uuids)} watches deleted")
+
+    elif op == 'pause':
+        for uuid in uuids:
+            if datastore.data['watching'].get(uuid):
+                datastore.data['watching'][uuid]['paused'] = True
+        if emit_flash:
+            flash(f"{len(uuids)} watches paused")
+
+    elif op == 'unpause':
+        for uuid in uuids:
+            if datastore.data['watching'].get(uuid):
+                datastore.data['watching'][uuid.strip()]['paused'] = False
+        if emit_flash:
+            flash(f"{len(uuids)} watches unpaused")
+
+    elif (op == 'mark-viewed'):
+        for uuid in uuids:
+            if datastore.data['watching'].get(uuid):
+                datastore.set_last_viewed(uuid, int(time.time()))
+        if emit_flash:
+            flash(f"{len(uuids)} watches updated")
+
+    elif (op == 'mute'):
+        for uuid in uuids:
+            if datastore.data['watching'].get(uuid):
+                datastore.data['watching'][uuid]['notification_muted'] = True
+        if emit_flash:
+            flash(f"{len(uuids)} watches muted")
+
+    elif (op == 'unmute'):
+        for uuid in uuids:
+            if datastore.data['watching'].get(uuid):
+                datastore.data['watching'][uuid]['notification_muted'] = False
+        if emit_flash:
+            flash(f"{len(uuids)} watches un-muted")
+
+    elif (op == 'recheck'):
+        for uuid in uuids:
+            if datastore.data['watching'].get(uuid):
+                # Recheck and require a full reprocessing
+                worker_handler.queue_item_async_safe(update_q, queuedWatchMetaData.PrioritizedItem(priority=1, item={'uuid': uuid}))
+        if emit_flash:
+            flash(f"{len(uuids)} watches queued for rechecking")
+
+    elif (op == 'clear-errors'):
+        for uuid in uuids:
+            if datastore.data['watching'].get(uuid):
+                datastore.data['watching'][uuid]["last_error"] = False
+        if emit_flash:
+            flash(f"{len(uuids)} watches errors cleared")
+
+    elif (op == 'clear-history'):
+        for uuid in uuids:
+            if datastore.data['watching'].get(uuid):
+                datastore.clear_watch_history(uuid)
+        if emit_flash:
+            flash(f"{len(uuids)} watches cleared/reset.")
+
+    elif (op == 'notification-default'):
+        from changedetectionio.notification import (
+            USE_SYSTEM_DEFAULT_NOTIFICATION_FORMAT_FOR_WATCH
+        )
+        for uuid in uuids:
+            if datastore.data['watching'].get(uuid):
+                datastore.data['watching'][uuid]['notification_title'] = None
+                datastore.data['watching'][uuid]['notification_body'] = None
+                datastore.data['watching'][uuid]['notification_urls'] = []
+                datastore.data['watching'][uuid]['notification_format'] = USE_SYSTEM_DEFAULT_NOTIFICATION_FORMAT_FOR_WATCH
+        if emit_flash:
+            flash(f"{len(uuids)} watches set to use default notification settings")
+
+    elif (op == 'assign-tag'):
+        op_extradata = extra_data
+        if op_extradata:
+            tag_uuid = datastore.add_tag(title=op_extradata)
+            if op_extradata and tag_uuid:
+                for uuid in uuids:
+                    if datastore.data['watching'].get(uuid):
+                        # Bug in old versions caused by bad edit page/tag handler
+                        if isinstance(datastore.data['watching'][uuid]['tags'], str):
+                            datastore.data['watching'][uuid]['tags'] = []
+
+                        datastore.data['watching'][uuid]['tags'].append(tag_uuid)
+        if emit_flash:
+            flash(f"{len(uuids)} watches were tagged")
+
+    if uuids:
+        for uuid in uuids:
+            watch_check_update.send(watch_uuid=uuid)
+
+def construct_blueprint(datastore: ChangeDetectionStore, update_q, worker_handler, queuedWatchMetaData, watch_check_update):
+    ui_blueprint = Blueprint('ui', __name__, template_folder="templates")
+    
+    # Register the edit blueprint
+    edit_blueprint = construct_edit_blueprint(datastore, update_q, queuedWatchMetaData)
+    ui_blueprint.register_blueprint(edit_blueprint)
+    
+    # Register the notification blueprint
+    notification_blueprint = construct_notification_blueprint(datastore)
+    ui_blueprint.register_blueprint(notification_blueprint)
+    
+    # Register the views blueprint
+    views_blueprint = construct_views_blueprint(datastore, update_q, queuedWatchMetaData, watch_check_update)
+    ui_blueprint.register_blueprint(views_blueprint)
+
+    # Import the login decorator
+    from changedetectionio.auth_decorator import login_optionally_required
+
+    @ui_blueprint.route("/clear_history/<string:uuid>", methods=['GET'])
+    @login_optionally_required
+    def clear_watch_history(uuid):
+        try:
+            datastore.clear_watch_history(uuid)
+        except KeyError:
+            flash('Watch not found', 'error')
+        else:
+            flash("Cleared snapshot history for watch {}".format(uuid))
+        return redirect(url_for('watchlist.index'))
+
+    @ui_blueprint.route("/clear_history", methods=['GET', 'POST'])
+    @login_optionally_required
+    def clear_all_history():
+        if request.method == 'POST':
+            confirmtext = request.form.get('confirmtext')
+
+            if confirmtext == 'clear':
+                for uuid in datastore.data['watching'].keys():
+                    datastore.clear_watch_history(uuid)
+                flash("Cleared snapshot history for all watches")
+            else:
+                flash('Incorrect confirmation text.', 'error')
+
+            return redirect(url_for('watchlist.index'))
+
+        output = render_template("clear_all_history.html")
+        return output
+
+    # Clear all statuses, so we do not see the 'unviewed' class
+    @ui_blueprint.route("/form/mark-all-viewed", methods=['GET'])
+    @login_optionally_required
+    def mark_all_viewed():
+        # Save the current newest history as the most recently viewed
+        with_errors = request.args.get('with_errors') == "1"
+        tag_limit = request.args.get('tag')
+        logger.debug(f"Limiting to tag {tag_limit}")
+        now = int(time.time())
+        for watch_uuid, watch in datastore.data['watching'].items():
+            if with_errors and not watch.get('last_error'):
+                continue
+
+            if tag_limit and ( not watch.get('tags') or tag_limit not in watch['tags'] ):
+                logger.debug(f"Skipping watch {watch_uuid}")
+                continue
+
+            datastore.set_last_viewed(watch_uuid, now)
+
+        return redirect(url_for('watchlist.index', tag=tag_limit))
+
+    @ui_blueprint.route("/delete", methods=['GET'])
+    @login_optionally_required
+    def form_delete():
+        uuid = request.args.get('uuid')
+
+        if uuid != 'all' and not uuid in datastore.data['watching'].keys():
+            flash('The watch by UUID {} does not exist.'.format(uuid), 'error')
+            return redirect(url_for('watchlist.index'))
+
+        # More for testing, possible to return the first/only
+        if uuid == 'first':
+            uuid = list(datastore.data['watching'].keys()).pop()
+        datastore.delete(uuid)
+        flash('Deleted.')
+
+        return redirect(url_for('watchlist.index'))
+
+    @ui_blueprint.route("/clone", methods=['GET'])
+    @login_optionally_required
+    def form_clone():
+        uuid = request.args.get('uuid')
+        # More for testing, possible to return the first/only
+        if uuid == 'first':
+            uuid = list(datastore.data['watching'].keys()).pop()
+
+        new_uuid = datastore.clone(uuid)
+
+        if not datastore.data['watching'].get(uuid).get('paused'):
+            worker_handler.queue_item_async_safe(update_q, queuedWatchMetaData.PrioritizedItem(priority=5, item={'uuid': new_uuid}))
+
+        flash('Cloned, you are editing the new watch.')
+
+        return redirect(url_for("ui.ui_edit.edit_page", uuid=new_uuid))
+
+    @ui_blueprint.route("/checknow", methods=['GET'])
+    @login_optionally_required
+    def form_watch_checknow():
+        # Forced recheck will skip the 'skip if content is the same' rule (, 'reprocess_existing_data': True})))
+        tag = request.args.get('tag')
+        uuid = request.args.get('uuid')
+        with_errors = request.args.get('with_errors') == "1"
+
+        i = 0
+
+        running_uuids = worker_handler.get_running_uuids()
+
+        if uuid:
+            if uuid not in running_uuids:
+                worker_handler.queue_item_async_safe(update_q, queuedWatchMetaData.PrioritizedItem(priority=1, item={'uuid': uuid}))
+                i += 1
+
+        else:
+            # Recheck all, including muted
+            # Get most overdue first
+            for k in sorted(datastore.data['watching'].items(), key=lambda item: item[1].get('last_checked', 0)):
+                watch_uuid = k[0]
+                watch = k[1]
+                if not watch['paused']:
+                    if watch_uuid not in running_uuids:
+                        if with_errors and not watch.get('last_error'):
+                            continue
+
+                        if tag != None and tag not in watch['tags']:
+                            continue
+
+                        worker_handler.queue_item_async_safe(update_q, queuedWatchMetaData.PrioritizedItem(priority=1, item={'uuid': watch_uuid}))
+                        i += 1
+
+        if i == 1:
+            flash("Queued 1 watch for rechecking.")
+        if i > 1:
+            flash(f"Queued {i} watches for rechecking.")
+        if i == 0:
+            flash("No watches available to recheck.")
+
+        return redirect(url_for('watchlist.index'))
+
+    @ui_blueprint.route("/form/checkbox-operations", methods=['POST'])
+    @login_optionally_required
+    def form_watch_list_checkbox_operations():
+        op = request.form['op']
+        uuids = [u.strip() for u in request.form.getlist('uuids') if u]
+        extra_data = request.form.get('op_extradata', '').strip()
+        _handle_operations(
+            datastore=datastore,
+            extra_data=extra_data,
+            queuedWatchMetaData=queuedWatchMetaData,
+            uuids=uuids,
+            worker_handler=worker_handler,
+            update_q=update_q,
+            watch_check_update=watch_check_update,
+            op=op,
+        )
+
+        return redirect(url_for('watchlist.index'))
+
+
+    @ui_blueprint.route("/share-url/<string:uuid>", methods=['GET'])
+    @login_optionally_required
+    def form_share_put_watch(uuid):
+        """Given a watch UUID, upload the info and return a share-link
+           the share-link can be imported/added"""
+        import requests
+        import json
+        from copy import deepcopy
+
+        # more for testing
+        if uuid == 'first':
+            uuid = list(datastore.data['watching'].keys()).pop()
+
+        # copy it to memory as trim off what we dont need (history)
+        watch = deepcopy(datastore.data['watching'].get(uuid))
+        # For older versions that are not a @property
+        if (watch.get('history')):
+            del (watch['history'])
+
+        # for safety/privacy
+        for k in list(watch.keys()):
+            if k.startswith('notification_'):
+                del watch[k]
+
+        for r in['uuid', 'last_checked', 'last_changed']:
+            if watch.get(r):
+                del (watch[r])
+
+        # Add the global stuff which may have an impact
+        watch['ignore_text'] += datastore.data['settings']['application']['global_ignore_text']
+        watch['subtractive_selectors'] += datastore.data['settings']['application']['global_subtractive_selectors']
+
+        watch_json = json.dumps(watch)
+
+        try:
+            r = requests.request(method="POST",
+                                 data={'watch': watch_json},
+                                 url="https://changedetection.io/share/share",
+                                 headers={'App-Guid': datastore.data['app_guid']})
+            res = r.json()
+
+            # Add to the flask session
+            session['share-link'] = f"https://changedetection.io/share/{res['share_key']}"
+
+
+        except Exception as e:
+            logger.error(f"Error sharing -{str(e)}")
+            flash(f"Could not share, something went wrong while communicating with the share server - {str(e)}", 'error')
+
+        return redirect(url_for('watchlist.index'))
+
+    return ui_blueprint

changedetectionio/blueprint/ui/edit.py

@@ -0,0 +1,340 @@
+import time
+from copy import deepcopy
+import os
+import importlib.resources
+from flask import Blueprint, request, redirect, url_for, flash, render_template, make_response, send_from_directory, abort
+from loguru import logger
+from jinja2 import Environment, FileSystemLoader
+
+from changedetectionio.store import ChangeDetectionStore
+from changedetectionio.auth_decorator import login_optionally_required
+from changedetectionio.time_handler import is_within_schedule
+from changedetectionio import worker_handler
+
+def construct_blueprint(datastore: ChangeDetectionStore, update_q, queuedWatchMetaData):
+    edit_blueprint = Blueprint('ui_edit', __name__, template_folder="../ui/templates")
+    
+    def _watch_has_tag_options_set(watch):
+        """This should be fixed better so that Tag is some proper Model, a tag is just a Watch also"""
+        for tag_uuid, tag in datastore.data['settings']['application'].get('tags', {}).items():
+            if tag_uuid in watch.get('tags', []) and (tag.get('include_filters') or tag.get('subtractive_selectors')):
+                return True
+
+    @edit_blueprint.route("/edit/<string:uuid>", methods=['GET', 'POST'])
+    @login_optionally_required
+    # https://stackoverflow.com/questions/42984453/wtforms-populate-form-with-data-if-data-exists
+    # https://wtforms.readthedocs.io/en/3.0.x/forms/#wtforms.form.Form.populate_obj ?
+    def edit_page(uuid):
+        from changedetectionio import forms
+        from changedetectionio.blueprint.browser_steps.browser_steps import browser_step_ui_config
+        from changedetectionio import processors
+        import importlib
+
+        # More for testing, possible to return the first/only
+        if not datastore.data['watching'].keys():
+            flash("No watches to edit", "error")
+            return redirect(url_for('watchlist.index'))
+
+        if uuid == 'first':
+            uuid = list(datastore.data['watching'].keys()).pop()
+
+        if not uuid in datastore.data['watching']:
+            flash("No watch with the UUID %s found." % (uuid), "error")
+            return redirect(url_for('watchlist.index'))
+
+        switch_processor = request.args.get('switch_processor')
+        if switch_processor:
+            for p in processors.available_processors():
+                if p[0] == switch_processor:
+                    datastore.data['watching'][uuid]['processor'] = switch_processor
+                    flash(f"Switched to mode - {p[1]}.")
+                    datastore.clear_watch_history(uuid)
+                    redirect(url_for('ui_edit.edit_page', uuid=uuid))
+
+        # be sure we update with a copy instead of accidently editing the live object by reference
+        default = deepcopy(datastore.data['watching'][uuid])
+
+        # Defaults for proxy choice
+        if datastore.proxy_list is not None:  # When enabled
+            # @todo
+            # Radio needs '' not None, or incase that the chosen one no longer exists
+            if default['proxy'] is None or not any(default['proxy'] in tup for tup in datastore.proxy_list):
+                default['proxy'] = ''
+        # proxy_override set to the json/text list of the items
+
+        # Does it use some custom form? does one exist?
+        processor_name = datastore.data['watching'][uuid].get('processor', '')
+        processor_classes = next((tpl for tpl in processors.find_processors() if tpl[1] == processor_name), None)
+        if not processor_classes:
+            flash(f"Cannot load the edit form for processor/plugin '{processor_classes[1]}', plugin missing?", 'error')
+            return redirect(url_for('watchlist.index'))
+
+        parent_module = processors.get_parent_module(processor_classes[0])
+
+        try:
+            # Get the parent of the "processor.py" go up one, get the form (kinda spaghetti but its reusing existing code)
+            forms_module = importlib.import_module(f"{parent_module.__name__}.forms")
+            # Access the 'processor_settings_form' class from the 'forms' module
+            form_class = getattr(forms_module, 'processor_settings_form')
+        except ModuleNotFoundError as e:
+            # .forms didnt exist
+            form_class = forms.processor_text_json_diff_form
+        except AttributeError as e:
+            # .forms exists but no useful form
+            form_class = forms.processor_text_json_diff_form
+
+        form = form_class(formdata=request.form if request.method == 'POST' else None,
+                          data=default,
+                          extra_notification_tokens=default.extra_notification_token_values(),
+                          default_system_settings=datastore.data['settings']
+                          )
+
+        # For the form widget tag UUID back to "string name" for the field
+        form.tags.datastore = datastore
+
+        # Used by some forms that need to dig deeper
+        form.datastore = datastore
+        form.watch = default
+
+        for p in datastore.extra_browsers:
+            form.fetch_backend.choices.append(p)
+
+        form.fetch_backend.choices.append(("system", 'System settings default'))
+
+        # form.browser_steps[0] can be assumed that we 'goto url' first
+
+        if datastore.proxy_list is None:
+            # @todo - Couldn't get setattr() etc dynamic addition working, so remove it instead
+            del form.proxy
+        else:
+            form.proxy.choices = [('', 'Default')]
+            for p in datastore.proxy_list:
+                form.proxy.choices.append(tuple((p, datastore.proxy_list[p]['label'])))
+
+
+        if request.method == 'POST' and form.validate():
+
+            # If they changed processor, it makes sense to reset it.
+            if datastore.data['watching'][uuid].get('processor') != form.data.get('processor'):
+                datastore.data['watching'][uuid].clear_watch()
+                flash("Reset watch history due to change of processor")
+
+            extra_update_obj = {
+                'consecutive_filter_failures': 0,
+                'last_error' : False
+            }
+
+            if request.args.get('unpause_on_save'):
+                extra_update_obj['paused'] = False
+
+            extra_update_obj['time_between_check'] = form.time_between_check.data
+
+             # Ignore text
+            form_ignore_text = form.ignore_text.data
+            datastore.data['watching'][uuid]['ignore_text'] = form_ignore_text
+
+            # Be sure proxy value is None
+            if datastore.proxy_list is not None and form.data['proxy'] == '':
+                extra_update_obj['proxy'] = None
+
+            # Unsetting all filter_text methods should make it go back to default
+            # This particularly affects tests running
+            if 'filter_text_added' in form.data and not form.data.get('filter_text_added') \
+                    and 'filter_text_replaced' in form.data and not form.data.get('filter_text_replaced') \
+                    and 'filter_text_removed' in form.data and not form.data.get('filter_text_removed'):
+                extra_update_obj['filter_text_added'] = True
+                extra_update_obj['filter_text_replaced'] = True
+                extra_update_obj['filter_text_removed'] = True
+
+            # Because wtforms doesn't support accessing other data in process_ , but we convert the CSV list of tags back to a list of UUIDs
+            tag_uuids = []
+            if form.data.get('tags'):
+                # Sometimes in testing this can be list, dont know why
+                if type(form.data.get('tags')) == list:
+                    extra_update_obj['tags'] = form.data.get('tags')
+                else:
+                    for t in form.data.get('tags').split(','):
+                        tag_uuids.append(datastore.add_tag(title=t))
+                    extra_update_obj['tags'] = tag_uuids
+
+            datastore.data['watching'][uuid].update(form.data)
+            datastore.data['watching'][uuid].update(extra_update_obj)
+
+            if not datastore.data['watching'][uuid].get('tags'):
+                # Force it to be a list, because form.data['tags'] will be string if nothing found
+                # And del(form.data['tags'] ) wont work either for some reason
+                datastore.data['watching'][uuid]['tags'] = []
+
+            # Recast it if need be to right data Watch handler
+            watch_class = processors.get_custom_watch_obj_for_processor(form.data.get('processor'))
+            datastore.data['watching'][uuid] = watch_class(datastore_path=datastore.datastore_path, default=datastore.data['watching'][uuid])
+            flash("Updated watch - unpaused!" if request.args.get('unpause_on_save') else "Updated watch.")
+
+            # Re #286 - We wait for syncing new data to disk in another thread every 60 seconds
+            # But in the case something is added we should save straight away
+            datastore.needs_write_urgent = True
+
+            # Do not queue on edit if its not within the time range
+
+            # @todo maybe it should never queue anyway on edit...
+            is_in_schedule = True
+            watch = datastore.data['watching'].get(uuid)
+
+            if watch.get('time_between_check_use_default'):
+                time_schedule_limit = datastore.data['settings']['requests'].get('time_schedule_limit', {})
+            else:
+                time_schedule_limit = watch.get('time_schedule_limit')
+
+            tz_name = time_schedule_limit.get('timezone')
+            if not tz_name:
+                tz_name = datastore.data['settings']['application'].get('scheduler_timezone_default', os.getenv('TZ', 'UTC').strip())
+
+            if time_schedule_limit and time_schedule_limit.get('enabled'):
+                try:
+                    is_in_schedule = is_within_schedule(time_schedule_limit=time_schedule_limit,
+                                                      default_tz=tz_name
+                                                      )
+                except Exception as e:
+                    logger.error(
+                        f"{uuid} - Recheck scheduler, error handling timezone, check skipped - TZ name '{tz_name}' - {str(e)}")
+                    return False
+
+            #############################
+            if not datastore.data['watching'][uuid].get('paused') and is_in_schedule:
+                # Queue the watch for immediate recheck, with a higher priority
+                worker_handler.queue_item_async_safe(update_q, queuedWatchMetaData.PrioritizedItem(priority=1, item={'uuid': uuid}))
+
+            # Diff page [edit] link should go back to diff page
+            if request.args.get("next") and request.args.get("next") == 'diff':
+                return redirect(url_for('ui.ui_views.diff_history_page', uuid=uuid))
+
+            return redirect(url_for('watchlist.index', tag=request.args.get("tag",'')))
+
+        else:
+            if request.method == 'POST' and not form.validate():
+                flash("An error occurred, please see below.", "error")
+
+            # JQ is difficult to install on windows and must be manually added (outside requirements.txt)
+            jq_support = True
+            try:
+                import jq
+            except ModuleNotFoundError:
+                jq_support = False
+
+            watch = datastore.data['watching'].get(uuid)
+
+            # if system or watch is configured to need a chrome type browser
+            system_uses_webdriver = datastore.data['settings']['application']['fetch_backend'] == 'html_webdriver'
+            watch_needs_selenium_or_playwright = False
+            if (watch.get('fetch_backend') == 'system' and system_uses_webdriver) or watch.get('fetch_backend') == 'html_webdriver' or watch.get('fetch_backend', '').startswith('extra_browser_'):
+                watch_needs_selenium_or_playwright = True
+
+
+            from zoneinfo import available_timezones
+
+            # Only works reliably with Playwright
+
+            # Import the global plugin system
+            from changedetectionio.pluggy_interface import collect_ui_edit_stats_extras
+            
+            template_args = {
+                'available_processors': processors.available_processors(),
+                'available_timezones': sorted(available_timezones()),
+                'browser_steps_config': browser_step_ui_config,
+                'emailprefix': os.getenv('NOTIFICATION_MAIL_BUTTON_PREFIX', False),
+                'extra_classes': 'checking-now' if worker_handler.is_watch_running(uuid) else '',
+                'extra_notification_token_placeholder_info': datastore.get_unique_notification_token_placeholders_available(),
+                'extra_processor_config': form.extra_tab_content(),
+                'extra_title': f" - Edit - {watch.label}",
+                'form': form,
+                'has_default_notification_urls': True if len(datastore.data['settings']['application']['notification_urls']) else False,
+                'has_extra_headers_file': len(datastore.get_all_headers_in_textfile_for_watch(uuid=uuid)) > 0,
+                'has_special_tag_options': _watch_has_tag_options_set(watch=watch),
+                'jq_support': jq_support,
+                'playwright_enabled': os.getenv('PLAYWRIGHT_DRIVER_URL', False),
+                'settings_application': datastore.data['settings']['application'],
+                'system_has_playwright_configured': os.getenv('PLAYWRIGHT_DRIVER_URL'),
+                'system_has_webdriver_configured': os.getenv('WEBDRIVER_URL'),
+                'ui_edit_stats_extras': collect_ui_edit_stats_extras(watch),
+                'visual_selector_data_ready': datastore.visualselector_data_is_ready(watch_uuid=uuid),
+                'timezone_default_config': datastore.data['settings']['application'].get('scheduler_timezone_default'),
+                'using_global_webdriver_wait': not default['webdriver_delay'],
+                'uuid': uuid,
+                'watch': watch,
+                'watch_needs_selenium_or_playwright': watch_needs_selenium_or_playwright,
+            }
+
+            included_content = None
+            if form.extra_form_content():
+                # So that the extra panels can access _helpers.html etc, we set the environment to load from templates/
+                # And then render the code from the module
+                templates_dir = str(importlib.resources.files("changedetectionio").joinpath('templates'))
+                env = Environment(loader=FileSystemLoader(templates_dir))
+                template = env.from_string(form.extra_form_content())
+                included_content = template.render(**template_args)
+
+            output = render_template("edit.html",
+                                     extra_tab_content=form.extra_tab_content() if form.extra_tab_content() else None,
+                                     extra_form_content=included_content,
+                                     **template_args
+                                     )
+
+        return output
+
+    @edit_blueprint.route("/edit/<string:uuid>/get-html", methods=['GET'])
+    @login_optionally_required
+    def watch_get_latest_html(uuid):
+        from io import BytesIO
+        from flask import send_file
+        import brotli
+
+        watch = datastore.data['watching'].get(uuid)
+        if watch and watch.history.keys() and os.path.isdir(watch.watch_data_dir):
+            latest_filename = list(watch.history.keys())[-1]
+            html_fname = os.path.join(watch.watch_data_dir, f"{latest_filename}.html.br")
+            with open(html_fname, 'rb') as f:
+                if html_fname.endswith('.br'):
+                    # Read and decompress the Brotli file
+                    decompressed_data = brotli.decompress(f.read())
+                else:
+                    decompressed_data = f.read()
+
+            buffer = BytesIO(decompressed_data)
+
+            return send_file(buffer, as_attachment=True, download_name=f"{latest_filename}.html", mimetype='text/html')
+
+        # Return a 500 error
+        abort(500)
+
+    # Ajax callback
+    @edit_blueprint.route("/edit/<string:uuid>/preview-rendered", methods=['POST'])
+    @login_optionally_required
+    def watch_get_preview_rendered(uuid):
+        '''For when viewing the "preview" of the rendered text from inside of Edit'''
+        from flask import jsonify
+        from changedetectionio.processors.text_json_diff import prepare_filter_prevew
+        result = prepare_filter_prevew(watch_uuid=uuid, form_data=request.form, datastore=datastore)
+        return jsonify(result)
+
+    @edit_blueprint.route("/highlight_submit_ignore_url", methods=['POST'])
+    @login_optionally_required
+    def highlight_submit_ignore_url():
+        import re
+        mode = request.form.get('mode')
+        selection = request.form.get('selection')
+
+        uuid = request.args.get('uuid','')
+        if datastore.data["watching"].get(uuid):
+            if mode == 'exact':
+                for l in selection.splitlines():
+                    datastore.data["watching"][uuid]['ignore_text'].append(l.strip())
+            elif mode == 'digit-regex':
+                for l in selection.splitlines():
+                    # Replace any series of numbers with a regex
+                    s = re.escape(l.strip())
+                    s = re.sub(r'[0-9]+', r'\\d+', s)
+                    datastore.data["watching"][uuid]['ignore_text'].append('/' + s + '/')
+
+        return f"<a href={url_for('ui.ui_views.preview_page', uuid=uuid)}>Click to preview</a>"
+    
+    return edit_blueprint

changedetectionio/blueprint/ui/notification.py

@@ -0,0 +1,112 @@
+from flask import Blueprint, request, make_response
+import random
+from loguru import logger
+
+from changedetectionio.notification_service import NotificationContextData
+from changedetectionio.store import ChangeDetectionStore
+from changedetectionio.auth_decorator import login_optionally_required
+
+def construct_blueprint(datastore: ChangeDetectionStore):
+    notification_blueprint = Blueprint('ui_notification', __name__, template_folder="../ui/templates")
+    
+    # AJAX endpoint for sending a test
+    @notification_blueprint.route("/notification/send-test/<string:watch_uuid>", methods=['POST'])
+    @notification_blueprint.route("/notification/send-test", methods=['POST'])
+    @notification_blueprint.route("/notification/send-test/", methods=['POST'])
+    @login_optionally_required
+    def ajax_callback_send_notification_test(watch_uuid=None):
+
+        # Watch_uuid could be unset in the case it`s used in tag editor, global settings
+        import apprise
+        from changedetectionio.notification.handler import process_notification
+        from changedetectionio.notification.apprise_plugin.assets import apprise_asset
+        from changedetectionio.jinja2_custom import render as jinja_render
+
+        from changedetectionio.notification.apprise_plugin.custom_handlers import apprise_http_custom_handler
+
+        apobj = apprise.Apprise(asset=apprise_asset)
+
+        is_global_settings_form = request.args.get('mode', '') == 'global-settings'
+        is_group_settings_form = request.args.get('mode', '') == 'group-settings'
+
+        # Use an existing random one on the global/main settings form
+        if not watch_uuid and (is_global_settings_form or is_group_settings_form) \
+                and datastore.data.get('watching'):
+            logger.debug(f"Send test notification - Choosing random Watch {watch_uuid}")
+            watch_uuid = random.choice(list(datastore.data['watching'].keys()))
+
+        if not watch_uuid:
+            return make_response("Error: You must have atleast one watch configured for 'test notification' to work", 400)
+
+        watch = datastore.data['watching'].get(watch_uuid)
+        notification_urls = request.form.get('notification_urls','').strip().splitlines()
+
+        if not notification_urls:
+            logger.debug("Test notification - Trying by group/tag in the edit form if available")
+            # On an edit page, we should also fire off to the tags if they have notifications
+            if request.form.get('tags') and request.form['tags'].strip():
+                for k in request.form['tags'].split(','):
+                    tag = datastore.tag_exists_by_name(k.strip())
+                    notification_urls = tag.get('notifications_urls') if tag and tag.get('notifications_urls') else None
+
+        if not notification_urls and not is_global_settings_form and not is_group_settings_form:
+            # In the global settings, use only what is typed currently in the text box
+            logger.debug("Test notification - Trying by global system settings notifications")
+            if datastore.data['settings']['application'].get('notification_urls'):
+                notification_urls = datastore.data['settings']['application']['notification_urls']
+
+        if not notification_urls:
+            return 'Error: No Notification URLs set/found'
+
+        for n_url in notification_urls:
+            # We are ONLY validating the apprise:// part here, convert all tags to something so as not to break apprise URLs
+            generic_notification_context_data = NotificationContextData()
+            generic_notification_context_data.set_random_for_validation()
+            n_url = jinja_render(template_str=n_url, **generic_notification_context_data).strip()
+            if len(n_url.strip()):
+                if not apobj.add(n_url):
+                    return f'Error:  {n_url} is not a valid AppRise URL.'
+
+        try:
+            # use the same as when it is triggered, but then override it with the form test values
+            n_object = NotificationContextData({
+                'watch_url': request.form.get('window_url', "https://changedetection.io"),
+                'notification_urls': notification_urls
+            })
+
+            # Only use if present, if not set in n_object it should use the default system value
+            if 'notification_format' in request.form and request.form['notification_format'].strip():
+                n_object['notification_format'] = request.form.get('notification_format', '').strip()
+            else:
+                n_object['notification_format'] = datastore.data['settings']['application'].get('notification_format')
+
+            if 'notification_title' in request.form and request.form['notification_title'].strip():
+                n_object['notification_title'] = request.form.get('notification_title', '').strip()
+            elif datastore.data['settings']['application'].get('notification_title'):
+                n_object['notification_title'] = datastore.data['settings']['application'].get('notification_title')
+            else:
+                n_object['notification_title'] = "Test title"
+
+            if 'notification_body' in request.form and request.form['notification_body'].strip():
+                n_object['notification_body'] = request.form.get('notification_body', '').strip()
+            elif datastore.data['settings']['application'].get('notification_body'):
+                n_object['notification_body'] = datastore.data['settings']['application'].get('notification_body')
+            else:
+                n_object['notification_body'] = "Test body"
+
+            n_object['as_async'] = False
+            n_object.update(watch.extra_notification_token_values())
+            sent_obj = process_notification(n_object, datastore)
+
+        except Exception as e:
+            e_str = str(e)
+            # Remove this text which is not important and floods the container
+            e_str = e_str.replace(
+                "DEBUG - <class 'apprise.decorators.base.CustomNotifyPlugin.instantiate_plugin.<locals>.CustomNotifyPluginWrapper'>",
+                '')
+
+            return make_response(e_str, 400)
+
+        return 'OK - Sent test notifications'
+
+    return notification_blueprint

changedetectionio/blueprint/ui/templates/clear_all_history.html

@@ -3,7 +3,7 @@
   <div class="box-wrap inner">
     <form
       class="pure-form pure-form-stacked"
-      action="{{url_for('clear_all_history')}}"
+      action="{{url_for('ui.clear_all_history')}}"
       method="POST"
     >
       <input type="hidden" name="csrf_token" value="{{ csrf_token() }}" >
@@ -37,7 +37,7 @@
         </div>
         <br />
         <div class="pure-control-group">
-          <a href="{{url_for('index')}}" class="pure-button button-cancel"
+          <a href="{{url_for('watchlist.index')}}" class="pure-button button-cancel"
             >Cancel</a
           >
         </div>

changedetectionio/blueprint/ui/templates/diff.html

@@ -1,5 +1,5 @@
 {% extends 'base.html' %}
-{% from '_helpers.jinja' import render_field, render_checkbox_field, render_button %}
+{% from '_helpers.html' import render_field, render_checkbox_field, render_button %}
 {% block content %}
 <script>
     const screenshot_url="{{url_for('static_content', group='screenshot', filename=uuid)}}";
@@ -7,16 +7,20 @@
     const error_screenshot_url="{{url_for('static_content', group='screenshot', filename=uuid, error_screenshot=1) }}";
     {% endif %}
 
-    const highlight_submit_ignore_url="{{url_for('highlight_submit_ignore_url', uuid=uuid)}}";
-
+    const highlight_submit_ignore_url="{{url_for('ui.ui_edit.highlight_submit_ignore_url', uuid=uuid)}}";
+    const watch_url= {{watch_a.link|tojson}};
 </script>
+<script src="https://cdn.jsdelivr.net/npm/html2canvas@1.4.1/dist/html2canvas.min.js"></script>
+<script src="https://cdn.jsdelivr.net/npm/piexifjs@1.0.6/piexif.min.js"></script>
+<script src="{{url_for('static_content', group='js', filename='snippet-to-image.js')}}"></script>
 <script src="{{url_for('static_content', group='js', filename='diff-overview.js')}}" defer></script>
 
 <div id="settings">
-    <form class="pure-form " action="" method="GET">
-        <fieldset>
+    <form class="pure-form " action="{{ url_for("ui.ui_views.diff_history_page", uuid=uuid) }}" method="POST" id="diff-form">
+        <fieldset class="diff-fieldset">
             {% if versions|length >= 1 %}
                 <strong>Compare</strong>
+                from {{from_version}} to {{to_version}}<br>
                 <del class="change"><span>from</span></del>
                 <select id="diff-version" name="from_version" class="needs-localtime">
                     {% for version in versions|reverse %}
@@ -33,30 +37,33 @@
                         </option>
                     {% endfor %}
                 </select>
-                <button type="submit" class="pure-button pure-button-primary">Go</button>
+                <button type="submit" class="pure-button pure-button-primary reset-margin">Go</button>
             {% endif %}
         </fieldset>
         <fieldset>
-            <strong>Style</strong>
-            <label for="diffWords" class="pure-checkbox">
-                <input type="radio" name="diff_type" id="diffWords" value="diffWords"> Words</label>
-            <label for="diffLines" class="pure-checkbox">
-                <input type="radio" name="diff_type" id="diffLines" value="diffLines" checked=""> Lines</label>
+                <input type="hidden" name="csrf_token" value="{{ csrf_token() }}">
+                <strong>Style</strong>
 
-            <label for="diffChars" class="pure-checkbox">
-                <input type="radio" name="diff_type" id="diffChars" value="diffChars"> Chars</label>
-            <!-- @todo - when mimetype is JSON, select this by default? -->
-            <label for="diffJson" class="pure-checkbox">
-                <input type="radio" name="diff_type" id="diffJson" value="diffJson"> JSON</label>
+                <label for="diffWords" class="pure-checkbox">
+                    <input type="radio" name="diff_type" id="diffWords" value="diffWords" {% if diff_prefs.diff_type == 'diffWords' %}checked=""{% endif %}> Words</label>
+                <label for="diffLines" class="pure-checkbox">
+                    <input type="radio" name="diff_type" id="diffLines" value="diffLines" {% if diff_prefs.diff_type == 'diffLines' %}checked=""{% endif %}> Lines</label>
 
-            <span>
-        <!-- https://github.com/kpdecker/jsdiff/issues/389 ? -->
-        <label for="ignoreWhitespace" class="pure-checkbox" id="label-diff-ignorewhitespace">
-            <input type="checkbox" id="ignoreWhitespace" name="ignoreWhitespace"> Ignore Whitespace</label>
-    </span>
+                <label for="ignoreWhitespace" class="pure-checkbox" id="label-diff-ignorewhitespace">
+                    <input type="checkbox" name="ignoreWhitespace" {% if diff_prefs.ignoreWhitespace %}checked=""{% endif %}> Ignore Whitespace</label>
+
+                <label for="changesOnly" class="pure-checkbox" id="label-diff-changes">
+                    <input type="checkbox" name="diff_changesOnly" {% if diff_prefs.diff_changesOnly %}checked=""{% endif %}> Changes only</label>
+
+                <label for="changesOnly" class="pure-checkbox" id="label-diff-removed">
+                    <input type="checkbox"  name="diff_removed" {% if diff_prefs.diff_removed %}checked=""{% endif %}> Removed</label>
+                <label for="changesOnly" class="pure-checkbox" id="label-diff-added">
+                    <input type="checkbox"  name="diff_added" {% if diff_prefs.diff_added %}checked=""{% endif %}> Added</label>
+                <label for="changesOnly" class="pure-checkbox" id="label-diff-replaced">
+                    <input type="checkbox"  name="diff_replaced" {% if diff_prefs.diff_replaced %}checked=""{% endif %}> Replaced</label>
+                <input type="submit">
         </fieldset>
     </form>
-
 </div>
 
 <div id="diff-jump">
@@ -88,26 +95,30 @@
     </div>
 
      <div class="tab-pane-inner" id="text">
+         <button id="share-as-image-btn" onclick="diffToJpeg()" title="Share diff as image" style="float: right; margin: 10px; padding: 8px 12px; background: #4CAF50; color: white; border: none; border-radius: 4px; cursor: pointer; font-size: 14px;">
+             <svg xmlns="http://www.w3.org/2000/svg" width="16" height="16" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" style="vertical-align: middle; margin-right: 4px;">
+                 <rect x="3" y="3" width="18" height="18" rx="2" ry="2"></rect>
+                 <circle cx="8.5" cy="8.5" r="1.5"></circle>
+                 <polyline points="21 15 16 10 5 21"></polyline>
+             </svg>
+             Share as Image
+         </button>
          {% if password_enabled_and_share_is_off %}
            <div class="tip">Pro-tip: You can enable <strong>"share access when password is enabled"</strong> from settings</div>
          {% endif %}
 
          <div class="snapshot-age">{{watch_a.snapshot_text_ctime|format_timestamp_timeago}}</div>
-
-         <table>
-             <tbody>
-             <tr>
-                 <!-- just proof of concept copied straight from github.com/kpdecker/jsdiff -->
-                 <td id="a" style="display: none;">{{from_version_file_contents}}</td>
-                 <td id="b" style="display: none;">{{to_version_file_contents}}</td>
-                 <td id="diff-col">
-                     <span id="result" class="highlightable-filter"></span>
-                 </td>
-             </tr>
-             </tbody>
-         </table>
-         Diff algorithm from the amazing <a href="https://github.com/kpdecker/jsdiff">github.com/kpdecker/jsdiff</a>
+            <table>
+                <tbody>
+                <tr>
+                    <td id="diff-col" class="highlightable-filter">
+                        <pre id="difference" style="border-left: 2px solid #ddd;">{{ content| diff_unescape_difference_spans  }}</pre>
+                    </td>
+                </tr>
+                </tbody>
+            </table>
      </div>
+
      <div class="tab-pane-inner" id="screenshot">
          <div class="tip">
              For now, Differences are performed on text, not graphically, only the latest screenshot is available.
@@ -125,7 +136,7 @@
      </div>
     <div class="tab-pane-inner" id="extract">
         <form id="extract-data-form" class="pure-form pure-form-stacked edit-form"
-              action="{{ url_for('diff_history_page', uuid=uuid) }}#extract"
+              action="{{ url_for('ui.ui_views.diff_history_page', uuid=uuid) }}#extract"
               method="POST">
             <input type="hidden" name="csrf_token" value="{{ csrf_token() }}">
 
@@ -159,8 +170,6 @@
 <script>
     const newest_version_timestamp = {{newest_version_timestamp}};
 </script>
-<script src="{{url_for('static_content', group='js', filename='diff.min.js')}}"></script>
-
 <script src="{{url_for('static_content', group='js', filename='diff-render.js')}}"></script>
 
 

changedetectionio/blueprint/ui/templates/edit.html

@@ -1,54 +1,59 @@
 {% extends 'base.html' %}
 {% block content %}
-{% from '_helpers.jinja' import render_field, render_checkbox_field, render_button %}
-{% from '_common_fields.jinja' import render_common_settings_form %}
+{% from '_helpers.html' import render_field, render_checkbox_field, render_button, render_time_schedule_form, playwright_warning, only_playwright_type_watches_warning, highlight_trigger_ignored_explainer, render_conditions_fieldlist_of_formfields_as_table, render_ternary_field %}
+{% from '_common_fields.html' import render_common_settings_form %}
 <script src="{{url_for('static_content', group='js', filename='tabs.js')}}" defer></script>
+<script src="{{url_for('static_content', group='js', filename='vis.js')}}" defer></script>
+<script src="{{url_for('static_content', group='js', filename='global-settings.js')}}" defer></script>
+<script src="{{url_for('static_content', group='js', filename='scheduler.js')}}" defer></script>
+<script src="{{url_for('static_content', group='js', filename='conditions.js')}}" defer></script>
+
+
 <script>
     const browser_steps_available_screenshots=JSON.parse('{{ watch.get_browsersteps_available_screenshots|tojson }}');
     const browser_steps_config=JSON.parse('{{ browser_steps_config|tojson }}');
-    const browser_steps_fetch_screenshot_image_url="{{url_for('browser_steps.browser_steps_fetch_screenshot_image', uuid=uuid)}}";
+    <!-- Should be _external so that firefox and others load it more reliably -->
+    const browser_steps_fetch_screenshot_image_url="{{url_for('browser_steps.browser_steps_fetch_screenshot_image', uuid=uuid, _external=True)}}";
     const browser_steps_last_error_step={{ watch.browser_steps_last_error_step|tojson }};
     const browser_steps_start_url="{{url_for('browser_steps.browsersteps_start_session', uuid=uuid)}}";
     const browser_steps_sync_url="{{url_for('browser_steps.browsersteps_ui_update', uuid=uuid)}}";
 {% if emailprefix %}
     const email_notification_prefix=JSON.parse('{{ emailprefix|tojson }}');
 {% endif %}
-    const notification_base_url="{{url_for('ajax_callback_send_notification_test')}}";
-    const playwright_enabled={% if playwright_enabled %} true {% else %} false {% endif %};
+    const notification_base_url="{{url_for('ui.ui_notification.ajax_callback_send_notification_test', watch_uuid=uuid)}}";
+    const playwright_enabled={% if playwright_enabled %}true{% else %}false{% endif %};
     const recheck_proxy_start_url="{{url_for('check_proxies.start_check', uuid=uuid)}}";
     const proxy_recheck_status_url="{{url_for('check_proxies.get_recheck_status', uuid=uuid)}}";
     const screenshot_url="{{url_for('static_content', group='screenshot', filename=uuid)}}";
     const watch_visual_selector_data_url="{{url_for('static_content', group='visual_selector_data', filename=uuid)}}";
-
+    const default_system_fetch_backend="{{ settings_application['fetch_backend'] }}";
 </script>
-
+<script src="{{url_for('static_content', group='js', filename='plugins.js')}}" defer></script>
 <script src="{{url_for('static_content', group='js', filename='watch-settings.js')}}" defer></script>
-<script src="{{url_for('static_content', group='js', filename='limit.js')}}" defer></script>
 <script src="{{url_for('static_content', group='js', filename='notifications.js')}}" defer></script>
 <script src="{{url_for('static_content', group='js', filename='visual-selector.js')}}" defer></script>
 {% if playwright_enabled %}
 <script src="{{url_for('static_content', group='js', filename='browser-steps.js')}}" defer></script>
 {% endif %}
 
+{% set has_tag_filters_extra="WARNING: Watch has tag/groups set with special filters\n" if has_special_tag_options else '' %}
 <script src="{{url_for('static_content', group='js', filename='recheck-proxy.js')}}" defer></script>
 
 <div class="edit-form monospaced-textarea">
 
     <div class="tabs collapsable">
         <ul>
-            <li class="tab" id=""><a href="#general">General</a></li>
+            <li class="tab"><a href="#general">General</a></li>
             <li class="tab"><a href="#request">Request</a></li>
-            {% if playwright_enabled %}
-            <li class="tab"><a id="browsersteps-tab" href="#browser-steps">Browser Steps</a></li>
+            {% if extra_tab_content %}
+            <li class="tab"><a href="#extras_tab">{{ extra_tab_content }}</a></li>
             {% endif %}
-
+            <li class="tab"><a id="browsersteps-tab" href="#browser-steps">Browser Steps</a></li>
+        <!-- should goto extra forms? -->
             {% if watch['processor'] == 'text_json_diff' %}
             <li class="tab"><a id="visualselector-tab" href="#visualselector">Visual Filter Selector</a></li>
-            <li class="tab"><a href="#filters-and-triggers">Filters &amp; Triggers</a></li>
-            {% endif %}
-
-            {% if watch['processor'] == 'restock_diff' %}
-            <li class="tab"><a href="#restock">Restock Detection</a></li>
+            <li class="tab" id="filters-and-triggers-tab"><a href="#filters-and-triggers">Filters &amp; Triggers</a></li>
+            <li class="tab" id="conditions-tab"><a href="#conditions">Conditions</a></li>
             {% endif %}
             <li class="tab"><a href="#notifications">Notifications</a></li>
             <li class="tab"><a href="#stats">Stats</a></li>
@@ -57,52 +62,56 @@
 
     <div class="box-wrap inner">
         <form class="pure-form pure-form-stacked"
-              action="{{ url_for('edit_page', uuid=uuid, next = request.args.get('next'), unpause_on_save = request.args.get('unpause_on_save')) }}" method="POST">
+              action="{{ url_for('ui.ui_edit.edit_page', uuid=uuid, next = request.args.get('next'), unpause_on_save = request.args.get('unpause_on_save'), tag = request.args.get('tag')) }}" method="POST">
              <input type="hidden" name="csrf_token" value="{{ csrf_token() }}">
 
             <div class="tab-pane-inner" id="general">
                 <fieldset>
                     <div class="pure-control-group">
                         {{ render_field(form.url, placeholder="https://...", required=true, class="m-d") }}
-                        <span class="pure-form-message-inline">Some sites use JavaScript to create the content, for this you should <a href="https://github.com/dgtlmoon/changedetection.io/wiki/Fetching-pages-with-WebDriver">use the Chrome/WebDriver Fetcher</a></span><br>
-                        <span class="pure-form-message-inline">You can use variables in the URL, perfect for inserting the current date and other logic, <a href="https://github.com/dgtlmoon/changedetection.io/wiki/Handling-variables-in-the-watched-URL">help and examples here</a></span><br>
-                        <span class="pure-form-message-inline">
-                        {% if watch['processor'] == 'text_json_diff' %}
-                            Current mode: <strong>Webpage Text/HTML, JSON and PDF changes.</strong><br>
-                          <a href="{{url_for('edit_page', uuid=uuid)}}?switch_processor=restock_diff" class="pure-button button-xsmall">Switch to re-stock detection mode.</a>
-                        {% else %}
-                        Current mode: <strong>Re-stock detection.</strong><br>
-                          <a href="{{url_for('edit_page', uuid=uuid)}}?switch_processor=text_json_diff" class="pure-button button-xsmall">Switch to Webpage Text/HTML, JSON and PDF changes mode.</a>
-                        {% endif %}
-                        </span>
-
-                    </div>
-                    <div class="pure-control-group">
-                        {{ render_field(form.title, class="m-d") }}
+                        <div class="pure-form-message">Some sites use JavaScript to create the content, for this you should <a href="https://github.com/dgtlmoon/changedetection.io/wiki/Fetching-pages-with-WebDriver">use the Chrome/WebDriver Fetcher</a></div>
+                        <div class="pure-form-message">Variables are supported in the URL (<a href="https://github.com/dgtlmoon/changedetection.io/wiki/Handling-variables-in-the-watched-URL">help and examples here</a>).</div>
                     </div>
                     <div class="pure-control-group">
                         {{ render_field(form.tags) }}
                         <span class="pure-form-message-inline">Organisational tag/group name used in the main listing page</span>
                     </div>
-                    <div class="pure-control-group">
-                        {{ render_field(form.time_between_check, class="time-check-widget") }}
-                        {% if has_empty_checktime %}
-                        <span class="pure-form-message-inline">Currently using the <a
-                                href="{{ url_for('settings_page', uuid=uuid) }}">default global settings</a>, change to another value if you want to be specific.</span>
-                        {% else %}
-                        <span class="pure-form-message-inline">Set to blank to use the <a
-                                href="{{ url_for('settings_page', uuid=uuid) }}">default global settings</a>.</span>
-                        {% endif %}
+                    <div class="pure-control-group inline-radio">
+                        {{ render_field(form.processor) }}
                     </div>
                     <div class="pure-control-group">
-                        {{ render_checkbox_field(form.extract_title_as_title) }}
+                        {{ render_field(form.title, class="m-d", placeholder=watch.label) }}
+                        <span class="pure-form-message-inline">Automatically uses the page title if found, you can also use your own title/description here</span>
                     </div>
+                    <div class="pure-control-group time-between-check border-fieldset">
+
+                        {{ render_checkbox_field(form.time_between_check_use_default, class="use-default-timecheck") }}
+                        <br>
+                        <div id="time-check-widget-wrapper">
+                            {{ render_field(form.time_between_check, class="time-check-widget") }}
+
+                            <span class="pure-form-message-inline">
+                             The interval/amount of time between each check.
+                            </span>
+                        </div>
+                        <div id="time-between-check-schedule">
+                            <!-- Start Time and End Time -->
+                            <div id="limit-between-time">
+                                {{ render_time_schedule_form(form, available_timezones, timezone_default_config) }}
+                            </div>
+                        </div>
+<br>
+              </div>
+
                     <div class="pure-control-group">
                         {{ render_checkbox_field(form.filter_failure_notification_send) }}
                         <span class="pure-form-message-inline">
                          Sends a notification when the filter can no longer be seen on the page, good for knowing when the page changed and your filter will not work anymore.
                         </span>
                     </div>
+                    <div class="pure-control-group">
+                        {{ render_ternary_field(form.use_page_title_in_list) }}
+                    </div>
                 </fieldset>
             </div>
 
@@ -124,10 +133,9 @@
                         </span>
                     </div>
                 {% endif %}
-                    <div  class="pure-control-group inline-radio">
-                        {{ render_checkbox_field(form.ignore_status_codes) }}
-                    </div>
-                <fieldset id="webdriver-override-options">
+
+                <!-- webdriver always -->
+                <fieldset data-visible-for="fetch_backend=html_webdriver"  style="display: none;">
                     <div class="pure-control-group">
                         {{ render_field(form.webdriver_delay) }}
                         <div class="pure-form-message-inline">
@@ -140,26 +148,46 @@
                         </div>
                     </div>
                     <div class="pure-control-group">
+                        <a class="pure-button button-secondary button-xsmall show-advanced">Show advanced options</a>
+                    </div>
+                    <div class="advanced-options"  style="display: none;">
                         {{ render_field(form.webdriver_js_execute_code) }}
                         <div class="pure-form-message-inline">
-                            Run this code before performing change detection, handy for filling in fields and other actions <a href="https://github.com/dgtlmoon/changedetection.io/wiki/Run-JavaScript-before-change-detection">More help and examples here</a>
+                            Run this code before performing change detection, handy for filling in fields and other
+                            actions <a
+                                href="https://github.com/dgtlmoon/changedetection.io/wiki/Run-JavaScript-before-change-detection">More
+                            help and examples here</a>
                         </div>
                     </div>
                 </fieldset>
-                <fieldset class="pure-group" id="requests-override-options">
-                    {% if not playwright_enabled %}
-                        <div class="pure-form-message-inline">
-                            <strong>Request override is currently only used by the <i>Basic fast Plaintext/HTTP Client</i> method.</strong>
-                        </div>
-                    {% endif %}
-                    <div class="pure-control-group" id="request-method">
-                        {{ render_field(form.method) }}
+                <!-- html requests always -->
+                <fieldset data-visible-for="fetch_backend=html_requests">
+                    <div class="pure-control-group">
+                        <a class="pure-button button-secondary button-xsmall show-advanced">Show advanced options</a>
                     </div>
-                    <div class="pure-control-group" id="request-headers">
-{{ render_field(form.headers, rows=5, placeholder="Example
+                    <div class="advanced-options"  style="display: none;">
+                        <div class="pure-control-group" id="request-method">
+                            {{ render_field(form.method) }}
+                        </div>
+                        <div id="request-body">
+                                            {{ render_field(form.body, rows=7, placeholder="Example
+{
+   \"name\":\"John\",
+   \"age\":30,
+   \"car\":null,
+   \"year\":{% now 'Europe/Berlin', '%Y' %}
+}") }}
+                        </div>
+                        <div class="pure-form-message">Variables are supported in the request body (<a href="https://github.com/dgtlmoon/changedetection.io/wiki/Handling-variables-in-the-watched-URL">help and examples here</a>).</div>
+                    </div>
+                </fieldset>
+            <!-- hmm -->
+                <div class="pure-control-group advanced-options"  style="display: none;">
+                    {{ render_field(form.headers, rows=7, placeholder="Example
 Cookie: foobar
-User-Agent: wonderbra 1.0") }}
-
+User-Agent: wonderbra 1.0
+Math: {{ 1 + 1 }}") }}
+                        <div class="pure-form-message">Variables are supported in the request header values (<a href="https://github.com/dgtlmoon/changedetection.io/wiki/Handling-variables-in-the-watched-URL">help and examples here</a>).</div>
                         <div class="pure-form-message-inline">
                             {% if has_extra_headers_file %}
                                 <strong>Alert! Extra headers file found and will be added to this watch!</strong>
@@ -169,20 +197,18 @@ User-Agent: wonderbra 1.0") }}
                             <br>
                             (Not supported by Selenium browser)
                         </div>
-
                     </div>
-                    <div class="pure-control-group" id="request-body">
-                                        {{ render_field(form.body, rows=5, placeholder="Example
-{
-   \"name\":\"John\",
-   \"age\":30,
-   \"car\":null
-}") }}
+            <fieldset data-visible-for="fetch_backend=html_requests fetch_backend=html_webdriver" >
+                    <div class="pure-control-group inline-radio advanced-options"  style="display: none;">
+                    {{ render_checkbox_field(form.ignore_status_codes) }}
                     </div>
-                </fieldset>
+            </fieldset>
             </div>
-            {% if playwright_enabled %}
+
             <div class="tab-pane-inner" id="browser-steps">
+            {% if watch_needs_selenium_or_playwright %}
+                {# Only works with playwright #}
+                {% if system_has_playwright_configured %}
                 <img class="beta-logo" src="{{url_for('static_content', group='images', filename='beta-logo.png')}}" alt="New beta functionality">
                 <fieldset>
                     <div class="pure-control-group">
@@ -200,13 +226,12 @@ User-Agent: wonderbra 1.0") }}
                         <div id="loading-status-text" style="display: none;">Please wait, first browser step can take a little time to load..<div class="spinner"></div></div>
                         <div class="flex-wrapper" >
 
-                            <div id="browser-steps-ui" class="noselect"  style="width: 100%; background-color: #eee; border-radius: 5px;">
-
+                            <div id="browser-steps-ui" class="noselect">
                                 <div class="noselect"  id="browsersteps-selector-wrapper" style="width: 100%">
                                     <span class="loader" >
                                         <span id="browsersteps-click-start">
                                             <h2 >Click here to Start</h2>
-                                            <svg style="height: 3.5rem;" version="1.1" viewBox="0 0 32 32"  xml:space="preserve" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink"><g id="Layer_1"/><g id="play_x5F_alt"><path d="M16,0C7.164,0,0,7.164,0,16s7.164,16,16,16s16-7.164,16-16S24.836,0,16,0z M10,24V8l16.008,8L10,24z" style="fill: var(--color-grey-400);"/></g></svg><br>
+                                            <svg style="height: 3.5rem;" version="1.1" viewBox="0 0 32 32"  xml:space="preserve" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink"><g id="start"/><g id="play_x5F_alt"><path d="M16,0C7.164,0,0,7.164,0,16s7.164,16,16,16s16-7.164,16-16S24.836,0,16,0z M10,24V8l16.008,8L10,24z" style="fill: var(--color-grey-400);"/></g></svg><br>
                                             Please allow 10-15 seconds for the browser to connect.<br>
                                         </span>
                                         <div class="spinner"  style="display: none;"></div>
@@ -215,22 +240,33 @@ User-Agent: wonderbra 1.0") }}
                                     <canvas  class="noselect" id="browsersteps-selector-canvas" style="max-width: 100%; width: 100%;"></canvas>
                                 </div>
                             </div>
-                            <div id="browser-steps-fieldlist" style="padding-left: 1em;  width: 350px; font-size: 80%;" >
-                                <span id="browserless-seconds-remaining">Loading</span> <span style="font-size: 80%;"> (<a target=_new href="https://github.com/dgtlmoon/changedetection.io/pull/478/files#diff-1a79d924d1840c485238e66772391268a89c95b781d69091384cf1ea1ac146c9R4">?</a>) </span>
+                            <div id="browser-steps-fieldlist" >
+                                <span id="browser-seconds-remaining">Press "Play" to start.</span> <span style="font-size: 80%;"> (<a target="newwindow" href="https://github.com/dgtlmoon/changedetection.io/pull/478/files#diff-1a79d924d1840c485238e66772391268a89c95b781d69091384cf1ea1ac146c9R4">?</a>) </span>
                                 {{ render_field(form.browser_steps) }}
                             </div>
                         </div>
                     </div>
                 </fieldset>
-            </div>
+                {% else %}
+                    {# it's configured to use selenium or chrome but system says its not configured #}
+                    {{ playwright_warning() }}
+                    {% if system_has_webdriver_configured %}
+                        <strong>Selenium/Webdriver cant be used here because it wont fetch screenshots reliably.</strong>
+                    {% endif %}
+                {% endif %}
+            {% else %}
+                {# "This functionality needs chrome.." #}
+                {{ only_playwright_type_watches_warning() }}
             {% endif %}
+            </div>
+
 
             <div class="tab-pane-inner" id="notifications">
                 <fieldset>
                     <div  class="pure-control-group inline-radio">
-                      {{ render_checkbox_field(form.notification_muted) }}
+                      {{ render_ternary_field(form.notification_muted, BooleanField=true) }}
                     </div>
-                    {% if is_html_webdriver %}
+                    {% if watch_needs_selenium_or_playwright %}
                     <div class="pure-control-group inline-radio">
                       {{ render_checkbox_field(form.notification_screenshot) }}
                         <span class="pure-form-message-inline">
@@ -242,19 +278,36 @@ User-Agent: wonderbra 1.0") }}
                         {% if has_default_notification_urls %}
                         <div class="inline-warning">
                             <img class="inline-warning-icon" src="{{url_for('static_content', group='images', filename='notice.svg')}}" alt="Look out!" title="Lookout!" >
-                            There are <a href="{{ url_for('settings_page')}}#notifications">system-wide notification URLs enabled</a>, this form will override notification settings for this watch only &dash; an empty Notification URL list here will still send notifications.
+                            There are <a href="{{ url_for('settings.settings_page')}}#notifications">system-wide notification URLs enabled</a>, this form will override notification settings for this watch only &dash; an empty Notification URL list here will still send notifications.
                         </div>
                         {% endif %}
                         <a href="#notifications" id="notification-setting-reset-to-default" class="pure-button button-xsmall" style="right: 20px; top: 20px; position: absolute; background-color: #5f42dd; border-radius: 4px; font-size: 70%; color: #fff">Use system defaults</a>
-
-                        {{ render_common_settings_form(form, emailprefix, settings_application) }}
+                        {{ render_common_settings_form(form, emailprefix, settings_application, extra_notification_token_placeholder_info) }}
                     </div>
                 </fieldset>
             </div>
 
             {% if watch['processor'] == 'text_json_diff' %}
+
+            <div class="tab-pane-inner" id="conditions">
+                    <script>
+                        const verify_condition_rule_url="{{url_for('conditions.verify_condition_single_rule', watch_uuid=uuid)}}";
+                    </script>
+                <div class="pure-control-group">
+                    {{ render_field(form.conditions_match_logic) }}
+                    {{ render_conditions_fieldlist_of_formfields_as_table(form.conditions) }}
+                    <div class="pure-form-message-inline">
+
+                        <p id="verify-state-text">Use the verify (✓) button to test if a condition passes against the current snapshot.</p>
+                       Read a quick tutorial about <a href="https://changedetection.io/tutorial/conditional-actions-web-page-changes">using conditional web page changes here</a>.<br>
+                    </div>
+                </div>
+            </div>
             <div class="tab-pane-inner" id="filters-and-triggers">
-                    <div class="pure-control-group">
+                <span id="activate-text-preview" class="pure-button pure-button-primary button-xsmall">Activate preview</span>
+              <div>
+              <div id="edit-text-filter">
+                    <div class="pure-control-group" id="pro-tips">
                             <strong>Pro-tips:</strong><br>
                             <ul>
                                 <li>
@@ -265,164 +318,81 @@ User-Agent: wonderbra 1.0") }}
                                 </li>
                             </ul>
                     </div>
-                    <div class="pure-control-group">
-                        {% set field = render_field(form.include_filters,
-                            rows=5,
-                            placeholder="#example
-xpath://body/div/span[contains(@class, 'example-class')]",
-                            class="m-d")
-                        %}
-                        {{ field }}
-                        {% if '/text()' in  field %}
-                          <span class="pure-form-message-inline"><strong>Note!: //text() function does not work where the &lt;element&gt; contains &lt;![CDATA[]]&gt;</strong></span><br>
-                        {% endif %}
-                        <span class="pure-form-message-inline">One rule per line, <i>any</i> rules that matches will be used.<br>
 
-                    <ul>
-                        <li>CSS - Limit text to this CSS rule, only text matching this CSS rule is included.</li>
-                        <li>JSON - Limit text to this JSON rule, using either <a href="https://pypi.org/project/jsonpath-ng/" target="new">JSONPath</a> or <a href="https://stedolan.github.io/jq/" target="new">jq</a> (if installed).
-                            <ul>
-                                <li>JSONPath: Prefix with <code>json:</code>, use <code>json:$</code> to force re-formatting if required,  <a href="https://jsonpath.com/" target="new">test your JSONPath here</a>.</li>
-                                {% if jq_support %}
-                                <li>jq: Prefix with <code>jq:</code> and <a href="https://jqplay.org/" target="new">test your jq here</a>. Using <a href="https://stedolan.github.io/jq/" target="new">jq</a> allows for complex filtering and processing of JSON data with built-in functions, regex, filtering, and more. See examples and documentation <a href="https://stedolan.github.io/jq/manual/" target="new">here</a>.</li>
-                                {% else %}
-                                <li>jq support not installed</li>
-                                {% endif %}
-                            </ul>
-                        </li>
-                        <li>XPath - Limit text to this XPath rule, simply start with a forward-slash,
-                            <ul>
-                                <li>Example:  <code>//*[contains(@class, 'sametext')]</code> or <code>xpath://*[contains(@class, 'sametext')]</code>, <a
-                                href="http://xpather.com/" target="new">test your XPath here</a></li>
-                                <li>Example: Get all titles from an RSS feed <code>//title/text()</code></li>
-                            </ul>
-                            </li>
-                    </ul>
-                    Please be sure that you thoroughly understand how to write CSS, JSONPath, XPath{% if jq_support %}, or jq selector{%endif%} rules before filing an issue on GitHub! <a
-                                href="https://github.com/dgtlmoon/changedetection.io/wiki/CSS-Selector-help">here for more CSS selector help</a>.<br>
-                </span>
-                    </div>
-                <fieldset class="pure-control-group">
-                    {{ render_field(form.subtractive_selectors, rows=5, placeholder="header
-footer
-nav
-.stockticker") }}
-                    <span class="pure-form-message-inline">
-                        <ul>
-                          <li> Remove HTML element(s) by CSS selector before text conversion. </li>
-                          <li> Add multiple elements or CSS selectors per line to ignore multiple parts of the HTML. </li>
-                        </ul>
-                      </span>
-                </fieldset>
-                <div class="text-filtering">
+{% include "edit/include_subtract.html" %}
+                <div class="text-filtering border-fieldset">
                 <fieldset class="pure-group" id="text-filtering-type-options">
                     <h3>Text filtering</h3>
                         Limit trigger/ignore/block/extract to;<br>
                         {{ render_checkbox_field(form.filter_text_added) }}
                         {{ render_checkbox_field(form.filter_text_replaced) }}
                         {{ render_checkbox_field(form.filter_text_removed) }}
-                    <span class="pure-form-message-inline">Note: Depending on the length and similarity of the text on each line, the algorithm may consider an <strong>addition</strong> instead of <strong>replacement</strong> for example.</span>
-                    <span class="pure-form-message-inline">So it's always better to select <strong>Added</strong>+<strong>Replaced</strong> when you're interested in new content.</span><br>
-                    <span class="pure-form-message-inline">When content is merely moved in a list, it will also trigger an <strong>addition</strong>, consider enabling <code><strong>Only trigger when unique lines appear</strong></code></span>
+                    <span class="pure-form-message-inline">Note: Depending on the length and similarity of the text on each line, the algorithm may consider an <strong>addition</strong> instead of <strong>replacement</strong> for example.</span><br>
+                    <span class="pure-form-message-inline">&nbsp;So it's always better to select <strong>Added</strong>+<strong>Replaced</strong> when you're interested in new content.</span><br>
+                    <span class="pure-form-message-inline">&nbsp;When content is merely moved in a list, it will also trigger an <strong>addition</strong>, consider enabling <code><strong>Only trigger when unique lines appear</strong></code></span>
                 </fieldset>
-
                 <fieldset class="pure-control-group">
                     {{ render_checkbox_field(form.check_unique_lines) }}
                     <span class="pure-form-message-inline">Good for websites that just move the content around, and you want to know when NEW content is added, compares new lines against all history for this watch.</span>
                 </fieldset>
-                <fieldset>
-                    <div class="pure-control-group">
-                        {{ render_field(form.trigger_text, rows=5, placeholder="Some text to wait for in a line
-/some.regex\d{2}/ for case-INsensitive regex
-") }}
-                        <span class="pure-form-message-inline">
-                    <ul>
-                        <li>Text to wait for before triggering a change/notification, all text and regex are tested <i>case-insensitive</i>.</li>
-                        <li>Trigger text is processed from the result-text that comes out of any CSS/JSON Filters for this watch</li>
-                        <li>Each line is processed separately (think of each line as "OR")</li>
-                        <li>Note: Wrap in forward slash / to use regex  example: <code>/foo\d/</code></li>
-                    </ul>
-                        </span>
-                    </div>
+                <fieldset class="pure-control-group">
+                    {{ render_checkbox_field(form.remove_duplicate_lines) }}
+                    <span class="pure-form-message-inline">Remove duplicate lines of text</span>
                 </fieldset>
-                <fieldset class="pure-group">
-                    {{ render_field(form.ignore_text, rows=5, placeholder="Some text to ignore in a line
-/some.regex\d{2}/ for case-INsensitive regex
-") }}
-                    <span class="pure-form-message-inline">
-                        <ul>
-                            <li>Each line processed separately, any line matching will be ignored (removed before creating the checksum)</li>
-                            <li>Regular Expression support, wrap the entire line in forward slash <code>/regex/</code></li>
-                            <li>Changing this will affect the comparison checksum which may trigger an alert</li>
-                            <li>Use the preview/show current tab to see ignores</li>
-                        </ul>
-                </span>
-
+                <fieldset class="pure-control-group">
+                    {{ render_checkbox_field(form.sort_text_alphabetically) }}
+                    <span class="pure-form-message-inline">Helps reduce changes detected caused by sites shuffling lines around, combine with <i>check unique lines</i> below.</span>
                 </fieldset>
-
-                <fieldset>
-                    <div class="pure-control-group">
-                        {{ render_field(form.text_should_not_be_present, rows=5, placeholder="For example: Out of stock
-Sold out
-Not in stock
-Unavailable") }}
-                        <span class="pure-form-message-inline">
-                            <ul>
-                                <li>Block change-detection while this text is on the page, all text and regex are tested <i>case-insensitive</i>, good for waiting for when a product is available again</li>
-                                <li>Block text is processed from the result-text that comes out of any CSS/JSON Filters for this watch</li>
-                                <li>All lines here must not exist (think of each line as "OR")</li>
-                                <li>Note: Wrap in forward slash / to use regex  example: <code>/foo\d/</code></li>
-                            </ul>
-                        </span>
-                    </div>
-                </fieldset>
-                <fieldset>
-                    <div class="pure-control-group">
-                        {{ render_field(form.extract_text, rows=5, placeholder="\d+ online") }}
-                        <span class="pure-form-message-inline">
-                    <ul>
-                        <li>Extracts text in the final output (line by line) after other filters using regular expressions or string match;
-                            <ul>
-                                <li>Regular expression &dash; example <code>/reports.+?2022/i</code></li>
-                                <li>Don't forget to consider the white-space at the start of a line <code>/.+?reports.+?2022/i</code></li>
-                                <li>Use <code>//(?aiLmsux))</code> type flags (more <a href="https://docs.python.org/3/library/re.html#index-15">information here</a>)<br></li>
-                                <li>Keyword example &dash; example <code>Out of stock</code></li>
-                                <li>Use groups to extract just that text &dash; example <code>/reports.+?(\d+)/i</code> returns a list of years only</li>
-                            </ul>
-                        </li>
-                        <li>One line per regular-expression/string match</li>
-                    </ul>
-                        </span>
-                    </div>
+                <fieldset class="pure-control-group">
+                    {{ render_checkbox_field(form.trim_text_whitespace) }}
+                    <span class="pure-form-message-inline">Remove any whitespace before and after each line of text</span>
                 </fieldset>
+                {% include "edit/text-options.html" %}
                 </div>
+              </div>
+              <div id="text-preview" style="display: none;" >
+                <script>
+                    const preview_text_edit_filters_url="{{url_for('ui.ui_edit.watch_get_preview_rendered', uuid=uuid)}}";
+                </script>
+                <br>
+                {#<div id="text-preview-controls"><span id="text-preview-refresh" class="pure-button button-xsmall">Refresh</span></div>#}
+                <div class="minitabs-wrapper">
+                  <div class="minitabs-content">
+                      <div id="text-preview-inner" class="monospace-preview">
+                          <p>Loading...</p>
+                      </div>
+                      <div id="text-preview-before-inner" style="display: none;" class="monospace-preview">
+                          <p>Loading...</p>
+                      </div>
+                  </div>
+                </div>
+                {{ highlight_trigger_ignored_explainer() }}
             </div>
-            {% endif %}
+          </div>
+        </div>
 
-            {% if watch['processor'] == 'restock_diff' %}
-            <div class="tab-pane-inner" id="restock">
-                    <fieldset>
-                        <div class="pure-control-group">
-                            {{ render_checkbox_field(form.in_stock_only) }}
-                            <span class="pure-form-message-inline">Only trigger notifications when page changes from <strong>out of stock</strong> to <strong>back in stock</strong></span>
-                        </div>
-                    </fieldset>
+        {% endif %}
+        {# rendered sub Template #}
+        {% if extra_form_content %}
+            <div class="tab-pane-inner" id="extras_tab">
+            {{ extra_form_content|safe }}
             </div>
-            {% endif %}
-
+        {% endif %}
             {% if watch['processor'] == 'text_json_diff' %}
             <div class="tab-pane-inner visual-selector-ui" id="visualselector">
                 <img class="beta-logo" src="{{url_for('static_content', group='images', filename='beta-logo.png')}}" alt="New beta functionality">
 
                 <fieldset>
                     <div class="pure-control-group">
-                        {% if visualselector_enabled %}
-                            <span class="pure-form-message-inline">
-                                The Visual Selector tool lets you select the <i>text</i> elements that will be used for the change detection &dash; after the <i>Browser Steps</i> has completed.<br><br>
+                        {% if watch_needs_selenium_or_playwright %}
+                            {% if system_has_playwright_configured %}
+                            <span class="pure-form-message-inline" id="visual-selector-heading">
+                                The Visual Selector tool lets you select the <i>text</i> elements that will be used for the change detection. It automatically fills-in the filters in the "CSS/JSONPath/JQ/XPath Filters" box of the <a href="#filters-and-triggers">Filters & Triggers</a> tab. Use <strong>Shift+Click</strong> to select multiple items.
                             </span>
 
                             <div id="selector-header">
                                 <a id="clear-selector" class="pure-button button-secondary button-xsmall" style="font-size: 70%">Clear selection</a>
+                                <!-- visual selector IMG will try to load, it will either replace this or on error replace it with some handy text -->
                                 <i class="fetching-update-notice" style="font-size: 80%;">One moment, fetching screenshot and element information..</i>
                             </div>
                             <div id="selector-wrapper" style="display: none">
@@ -434,12 +404,16 @@ Unavailable") }}
                             </div>
                             <div id="selector-current-xpath" style="overflow-x: hidden"><strong>Currently:</strong>&nbsp;<span class="text">Loading...</span></div>
                         {% else %}
-                            <span class="pure-form-message-inline">
-                                <p>Sorry, this functionality only works with Playwright/Chrome enabled watches.</p>
-                                <p>Enable the Playwright Chrome fetcher, or alternatively try our <a href="https://lemonade.changedetection.io/start">very affordable subscription based service</a>.</p>
-                                <p>This is because Selenium/WebDriver can not extract full page screenshots reliably.</p>
-                            </span>
+                            {# The watch needed chrome but system says that playwright is not ready #}
+                            {{ playwright_warning() }}
                         {% endif %}
+                            {% if system_has_webdriver_configured %}
+                                <strong>Selenium/Webdriver cant be used here because it wont fetch screenshots reliably.</strong>
+                            {% endif %}
+                    {% else %}
+                        {# "This functionality needs chrome.." #}
+                        {{ only_playwright_type_watches_warning() }}
+                    {% endif %}
                     </div>
                 </fieldset>
             </div>
@@ -466,22 +440,43 @@ Unavailable") }}
                             <td>{{ "{:,}".format(watch.history|length) }}</td>
                         </tr>
                         <tr>
-                            <td>Last fetch time</td>
+                            <td>Last fetch duration</td>
                             <td>{{ watch.fetch_time }}s</td>
                         </tr>
+                        <tr>
+                            <td>Notification alert count</td>
+                            <td>{{ watch.notification_alert_count }}</td>
+                        </tr>
+                        <tr>
+                            <td>Server type reply</td>
+                            <td>{{ watch.get('remote_server_reply') }}</td>
+                        </tr>
                         </tbody>
                     </table>
+
+                    {% if ui_edit_stats_extras %}
+                    <div class="plugin-stats-extras"> <!-- from pluggy plugin -->
+                        {{ ui_edit_stats_extras|safe }}
+                    </div>
+                    {% endif %}
+
+                    {% if watch.history_n %}
+                        <p>
+                             <a href="{{url_for('ui.ui_edit.watch_get_latest_html', uuid=uuid)}}" class="pure-button button-small">Download latest HTML snapshot</a>
+                        </p>
+                    {% endif %}
+
                 </div>
             </div>
             <div id="actions">
                 <div class="pure-control-group">
                     {{ render_button(form.save_button) }}
-                    <a href="{{url_for('form_delete', uuid=uuid)}}"
-                       class="pure-button button-small button-error ">Delete</a>
-                    <a href="{{url_for('clear_watch_history', uuid=uuid)}}"
-                       class="pure-button button-small button-error ">Clear History</a>
-                    <a href="{{url_for('form_clone', uuid=uuid)}}"
-                       class="pure-button button-small ">Create Copy</a>
+                    <a href="{{url_for('ui.form_delete', uuid=uuid)}}"
+                       class="pure-button button-error ">Delete</a>
+                    {% if watch.history_n %}<a href="{{url_for('ui.clear_watch_history', uuid=uuid)}}"
+                       class="pure-button button-error">Clear History</a>{% endif %}
+                    <a href="{{url_for('ui.form_clone', uuid=uuid)}}"
+                       class="pure-button">Clone &amp; Edit</a>
                 </div>
             </div>
         </form>

changedetectionio/blueprint/ui/templates/preview.html

@@ -0,0 +1,107 @@
+{% extends 'base.html' %}
+{% from '_helpers.html' import highlight_trigger_ignored_explainer %}
+{% block content %}
+    <script>
+        const screenshot_url = "{{url_for('static_content', group='screenshot', filename=uuid)}}";
+        const triggered_line_numbers = {{ highlight_triggered_line_numbers|tojson }};
+        const ignored_line_numbers = {{ highlight_ignored_line_numbers|tojson }};
+        const blocked_line_numbers = {{ highlight_blocked_line_numbers|tojson }};
+        {% if last_error_screenshot %}
+            const error_screenshot_url = "{{url_for('static_content', group='screenshot', filename=uuid, error_screenshot=1) }}";
+        {% endif %}
+        const highlight_submit_ignore_url = "{{url_for('ui.ui_edit.highlight_submit_ignore_url', uuid=uuid)}}";
+    </script>
+    <script src="{{url_for('static_content', group='js', filename='plugins.js')}}"></script>
+    <script src="{{ url_for('static_content', group='js', filename='diff-overview.js') }}" defer></script>
+    <script src="{{ url_for('static_content', group='js', filename='preview.js') }}" defer></script>
+    <script src="{{ url_for('static_content', group='js', filename='tabs.js') }}" defer></script>
+    {% if versions|length >= 2 %}
+        <div id="settings" style="text-align: center;">
+            <form class="pure-form " action="" method="POST">
+                <fieldset>
+                    <label for="preview-version">Select timestamp</label> <select id="preview-version"
+                                                                                 name="from_version"
+                                                                                 class="needs-localtime">
+                    {% for version in versions|reverse %}
+                        <option value="{{ version }}" {% if version == current_version %} selected="" {% endif %}>
+                            {{ version }}
+                        </option>
+                    {% endfor %}
+                </select>
+                    <button type="submit" class="pure-button pure-button-primary">Go</button>
+
+                </fieldset>
+            </form>
+                <br>
+                <strong>Keyboard: </strong><a href="" class="pure-button pure-button-primary" id="btn-previous">
+                &larr; Previous</a> &nbsp; <a class="pure-button pure-button-primary" id="btn-next" href="">
+                &rarr; Next</a>
+        </div>
+    {% endif %}
+
+    <div class="tabs">
+        <ul>
+            {% if last_error_text %}
+                <li class="tab" id="error-text-tab"><a href="#error-text">Error Text</a></li> {% endif %}
+            {% if last_error_screenshot %}
+                <li class="tab" id="error-screenshot-tab"><a href="#error-screenshot">Error Screenshot</a>
+                </li> {% endif %}
+            {% if history_n > 0 %}
+                <li class="tab" id="text-tab"><a href="#text">Text</a></li>
+                <li class="tab" id="screenshot-tab"><a href="#screenshot">Screenshot</a></li>
+            {% endif %}
+        </ul>
+    </div>
+
+
+    <div id="diff-ui">
+        <div class="tab-pane-inner" id="error-text">
+            <div class="snapshot-age error">{{ watch.error_text_ctime|format_seconds_ago }} seconds ago</div>
+            <pre>
+            {{ last_error_text }}
+        </pre>
+        </div>
+
+        <div class="tab-pane-inner" id="error-screenshot">
+            <div class="snapshot-age error">{{ watch.snapshot_error_screenshot_ctime|format_seconds_ago }} seconds ago
+            </div>
+            <img id="error-screenshot-img" style="max-width: 80%"
+                 alt="Current erroring screenshot from most recent request">
+        </div>
+
+        <div class="tab-pane-inner" id="text">
+            <div class="snapshot-age">{{ current_version|format_timestamp_timeago }}</div>
+            <span class="tip"><strong>Pro-tip</strong>: Highlight text to add to ignore filters</span>
+
+            <table>
+                <tbody>
+                <tr>
+                    <td id="diff-col" class="highlightable-filter">
+                        <pre style="border-left: 2px solid #ddd;">
+{{ content }}
+                        </pre>
+                    </td>
+                </tr>
+                </tbody>
+            </table>
+            {{ highlight_trigger_ignored_explainer() }}
+        </div>
+
+        <div class="tab-pane-inner" id="screenshot">
+            <div class="tip">
+                For now, Differences are performed on text, not graphically, only the latest screenshot is available.
+            </div>
+            <br>
+            {% if is_html_webdriver %}
+                {% if screenshot %}
+                    <div class="snapshot-age">{{ watch.snapshot_screenshot_ctime|format_timestamp_timeago }}</div>
+                    <img style="max-width: 80%" id="screenshot-img" alt="Current screenshot from most recent request">
+                {% else %}
+                    No screenshot available just yet! Try rechecking the page.
+                {% endif %}
+            {% else %}
+                <strong>Screenshot requires Playwright/WebDriver enabled</strong>
+            {% endif %}
+        </div>
+    </div>
+{% endblock %}

changedetectionio/blueprint/ui/views.py

@@ -0,0 +1,362 @@
+from flask import Blueprint, request, redirect, url_for, flash, render_template, make_response, send_from_directory, abort
+import os
+import time
+import re
+from loguru import logger
+from markupsafe import Markup
+
+from changedetectionio.diff import REMOVED_STYLE, ADDED_STYLE, REMOVED_INNER_STYLE, ADDED_INNER_STYLE
+from changedetectionio.notification.handler import apply_html_color_to_body
+from changedetectionio.notification_service import CUSTOM_LINEBREAK_PLACEHOLDER
+from changedetectionio.store import ChangeDetectionStore
+from changedetectionio.auth_decorator import login_optionally_required
+from changedetectionio import html_tools, diff
+from changedetectionio import worker_handler
+from changedetectionio.blueprint.cookie_preferences import PreferenceManager
+
+def construct_blueprint(datastore: ChangeDetectionStore, update_q, queuedWatchMetaData, watch_check_update):
+    views_blueprint = Blueprint('ui_views', __name__, template_folder="../ui/templates")
+
+    # Diff display preferences configuration
+    DIFF_PREFERENCES_CONFIG = {
+        'diff_changesOnly': {'default': False, 'type': 'bool'},
+        'diff_ignoreWhitespace': {'default': False, 'type': 'bool'},
+        'diff_removed': {'default': True, 'type': 'bool'},
+        'diff_added': {'default': True, 'type': 'bool'},
+        'diff_replaced': {'default': True, 'type': 'bool'},
+        'diff_type': {'default': 'diffLines', 'type': 'value'},
+    }
+
+    @views_blueprint.app_template_filter('diff_unescape_difference_spans')
+    def diff_unescape_difference_spans(content):
+        """Emulate Jinja2's auto-escape, then selectively unescape our diff spans."""
+        from markupsafe import escape
+
+        if not content:
+            return Markup('')
+
+        # Step 1: Escape everything like Jinja2 would (this makes it XSS-safe)
+        escaped_content = escape(str(content))
+
+        # Step 2: Unescape only our exact diff spans generated by apply_html_color_to_body()
+        # Pattern matches the exact structure:
+        # <span style="{STYLE}" role="{ROLE}" aria-label="{LABEL}" title="{TITLE}">
+
+        # Unescape outer span opening tags with full attributes (role, aria-label, title)
+        # Matches removed/added/changed/changed_into spans
+        result = re.sub(
+            rf'&lt;span style=&#34;({re.escape(REMOVED_STYLE)}|{re.escape(ADDED_STYLE)})&#34; '
+            rf'role=&#34;(deletion|insertion|note)&#34; '
+            rf'aria-label=&#34;([^&]+?)&#34; '
+            rf'title=&#34;([^&]+?)&#34;&gt;',
+            r'<span style="\1" role="\2" aria-label="\3" title="\4">',
+            str(escaped_content),
+            flags=re.IGNORECASE
+        )
+
+        # Unescape inner span opening tags (without additional attributes)
+        # This matches the darker background styles for changed parts within lines
+        result = re.sub(
+            rf'&lt;span style=&#34;({re.escape(REMOVED_INNER_STYLE)}|{re.escape(ADDED_INNER_STYLE)})&#34;&gt;',
+            r'<span style="\1">',
+            result,
+            flags=re.IGNORECASE
+        )
+
+        # Unescape closing tags (but only as many as we opened)
+        open_count = result.count('<span style=')
+        close_count = str(escaped_content).count('&lt;/span&gt;')
+
+        # Replace up to the number of spans we opened
+        for _ in range(min(open_count, close_count)):
+            result = result.replace('&lt;/span&gt;', '</span>', 1)
+
+        # Not necessary because the CSS/HTML will lay it out by linefeed
+        result = result.replace(CUSTOM_LINEBREAK_PLACEHOLDER, '')
+        return Markup(result)
+
+    @views_blueprint.route("/preview/<string:uuid>", methods=['GET'])
+    @login_optionally_required
+    def preview_page(uuid):
+        content = []
+        versions = []
+        timestamp = None
+
+        # More for testing, possible to return the first/only
+        if uuid == 'first':
+            uuid = list(datastore.data['watching'].keys()).pop()
+
+        try:
+            watch = datastore.data['watching'][uuid]
+        except KeyError:
+            flash("No history found for the specified link, bad link?", "error")
+            return redirect(url_for('watchlist.index'))
+
+        system_uses_webdriver = datastore.data['settings']['application']['fetch_backend'] == 'html_webdriver'
+        extra_stylesheets = [url_for('static_content', group='styles', filename='diff.css')]
+
+        is_html_webdriver = False
+        if (watch.get('fetch_backend') == 'system' and system_uses_webdriver) or watch.get('fetch_backend') == 'html_webdriver' or watch.get('fetch_backend', '').startswith('extra_browser_'):
+            is_html_webdriver = True
+
+        triggered_line_numbers = []
+        ignored_line_numbers = []
+        blocked_line_numbers = []
+
+        if datastore.data['watching'][uuid].history_n == 0 and (watch.get_error_text() or watch.get_error_snapshot()):
+            flash("Preview unavailable - No fetch/check completed or triggers not reached", "error")
+        else:
+            # So prepare the latest preview or not
+            preferred_version = request.args.get('version')
+            versions = list(watch.history.keys())
+            timestamp = versions[-1]
+            if preferred_version and preferred_version in versions:
+                timestamp = preferred_version
+
+            try:
+                versions = list(watch.history.keys())
+                content = watch.get_history_snapshot(timestamp)
+
+                triggered_line_numbers = html_tools.strip_ignore_text(content=content,
+                                                                      wordlist=watch.get('trigger_text'),
+                                                                      mode='line numbers'
+                                                                      )
+                ignored_line_numbers = html_tools.strip_ignore_text(content=content,
+                                                                      wordlist=watch.get('ignore_text'),
+                                                                      mode='line numbers'
+                                                                      )
+                blocked_line_numbers = html_tools.strip_ignore_text(content=content,
+                                                                      wordlist=watch.get("text_should_not_be_present"),
+                                                                      mode='line numbers'
+                                                                      )
+            except Exception as e:
+                content.append({'line': f"File doesnt exist or unable to read timestamp {timestamp}", 'classes': ''})
+
+        output = render_template("preview.html",
+                                 content=content,
+                                 current_diff_url=watch['url'],
+                                 current_version=timestamp,
+                                 extra_stylesheets=extra_stylesheets,
+                                 extra_title=f" - Diff - {watch.label} @ {timestamp}",
+                                 highlight_ignored_line_numbers=ignored_line_numbers,
+                                 highlight_triggered_line_numbers=triggered_line_numbers,
+                                 highlight_blocked_line_numbers=blocked_line_numbers,
+                                 history_n=watch.history_n,
+                                 is_html_webdriver=is_html_webdriver,
+                                 last_error=watch['last_error'],
+                                 last_error_screenshot=watch.get_error_snapshot(),
+                                 last_error_text=watch.get_error_text(),
+                                 screenshot=watch.get_screenshot(),
+                                 uuid=uuid,
+                                 versions=versions,
+                                 watch=watch,
+                                 )
+
+        return output
+
+    @views_blueprint.route("/diff/<string:uuid>", methods=['POST'])
+    @login_optionally_required
+    def diff_history_page_build_report(uuid):
+        from changedetectionio import forms
+
+        # More for testing, possible to return the first/only
+        if uuid == 'first':
+            uuid = list(datastore.data['watching'].keys()).pop()
+
+        try:
+            watch = datastore.data['watching'][uuid]
+        except KeyError:
+            flash("No history found for the specified link, bad link?", "error")
+            return redirect(url_for('watchlist.index'))
+
+        # For submission of requesting an extract
+        extract_form = forms.extractDataForm(formdata=request.form,
+                                             data={'extract_regex': request.form.get('extract_regex', '')}
+                                             )
+        if not extract_form.validate():
+            flash("An error occurred, please see below.", "error")
+            return _render_diff_template(uuid, extract_form)
+
+        else:
+            extract_regex = request.form.get('extract_regex', '').strip()
+            output = watch.extract_regex_from_all_history(extract_regex)
+            if output:
+                watch_dir = os.path.join(datastore.datastore_path, uuid)
+                response = make_response(send_from_directory(directory=watch_dir, path=output, as_attachment=True))
+                response.headers['Content-type'] = 'text/csv'
+                response.headers['Cache-Control'] = 'no-cache, no-store, must-revalidate'
+                response.headers['Pragma'] = 'no-cache'
+                response.headers['Expires'] = "0"
+                return response
+
+            flash('No matches found while scanning all of the watch history for that RegEx.', 'error')
+        return redirect(url_for('ui.ui_views.diff_history_page', uuid=uuid) + '#extract')
+
+    def _render_diff_template(uuid, extract_form=None):
+        """Helper function to render the diff template with all required data"""
+        from changedetectionio import forms
+
+        # More for testing, possible to return the first/only
+        if uuid == 'first':
+            uuid = list(datastore.data['watching'].keys()).pop()
+
+        extra_stylesheets = [url_for('static_content', group='styles', filename='diff.css')]
+        try:
+            watch = datastore.data['watching'][uuid]
+        except KeyError:
+            flash("No history found for the specified link, bad link?", "error")
+            return redirect(url_for('watchlist.index'))
+
+        # Use provided form or create a new one
+        if extract_form is None:
+            extract_form = forms.extractDataForm(formdata=request.form,
+                                                 data={'extract_regex': request.form.get('extract_regex', '')}
+                                                 )
+
+        # Handle diff display preferences using PreferenceManager
+        # Load preferences from cookies, with URL query params as temporary overrides
+        pref_manager = PreferenceManager(DIFF_PREFERENCES_CONFIG, cookie_scope='global')
+        diff_prefs = pref_manager.load_preferences()
+
+        history = watch.history
+        dates = list(history.keys())
+
+        # If a "from_version" was requested, then find it (or the closest one)
+        # Also set "from version" to be the closest version to the one that was last viewed.
+
+        best_last_viewed_timestamp = watch.get_from_version_based_on_last_viewed
+        from_version_timestamp = best_last_viewed_timestamp if best_last_viewed_timestamp else dates[-2]
+        from_version = request.args.get('from_version', from_version_timestamp )
+
+        # Use the current one if nothing was specified
+        to_version = request.args.get('to_version', str(dates[-1]))
+
+        try:
+            to_version_file_contents = watch.get_history_snapshot(timestamp=to_version)
+        except Exception as e:
+            logger.error(f"Unable to read watch history to-version for version {to_version}: {str(e)}")
+            to_version_file_contents = f"Unable to read to-version at {to_version}.\n"
+
+        try:
+            from_version_file_contents = watch.get_history_snapshot(timestamp=from_version)
+        except Exception as e:
+            logger.error(f"Unable to read watch history from-version for version {from_version}: {str(e)}")
+            from_version_file_contents = f"Unable to read to-version {from_version}.\n"
+
+        screenshot_url = watch.get_screenshot()
+
+        system_uses_webdriver = datastore.data['settings']['application']['fetch_backend'] == 'html_webdriver'
+
+        is_html_webdriver = False
+        if (watch.get('fetch_backend') == 'system' and system_uses_webdriver) or watch.get('fetch_backend') == 'html_webdriver' or watch.get('fetch_backend', '').startswith('extra_browser_'):
+            is_html_webdriver = True
+
+        password_enabled_and_share_is_off = False
+        if datastore.data['settings']['application'].get('password') or os.getenv("SALTED_PASS", False):
+            password_enabled_and_share_is_off = not datastore.data['settings']['application'].get('shared_diff_access')
+
+        datastore.set_last_viewed(uuid, time.time())
+
+        content = diff.render_diff(previous_version_file_contents=from_version_file_contents,
+                                   newest_version_file_contents=to_version_file_contents,
+                                   #                                   include_removed=diff_prefs.get('removed'),
+                                   #                                   include_added=diff_prefs.get('added'),
+                                   #                                   include_replaced=diff_prefs.get('replaced'),
+                                   ignore_junk=diff_prefs.get('diff_ignoreWhitespace'),
+                                   include_equal=not diff_prefs.get('diff_changesOnly'),
+                                   word_diff=diff_prefs.get('diff_type') == 'diffWords',
+                                   )
+        content = apply_html_color_to_body(n_body=content)
+        content = content.replace(CUSTOM_LINEBREAK_PLACEHOLDER, "\n")
+
+        output = render_template("diff.html",
+                                 content=content,
+                                 current_diff_url=watch['url'],
+                                 diff_prefs=diff_prefs,
+                                 extra_stylesheets=extra_stylesheets,
+                                 extra_title=f" - {watch.label} - History",
+                                 extract_form=extract_form,
+                                 from_version=str(from_version),
+                                 is_html_webdriver=is_html_webdriver,
+                                 last_error=watch['last_error'],
+                                 last_error_screenshot=watch.get_error_snapshot(),
+                                 last_error_text=watch.get_error_text(),
+                                 left_sticky=True,
+                                 newest=to_version_file_contents,
+                                 newest_version_timestamp=dates[-1],
+                                 password_enabled_and_share_is_off=password_enabled_and_share_is_off,
+                                 screenshot=screenshot_url,
+                                 to_version=str(to_version),
+                                 uuid=uuid,
+                                 versions=dates, # All except current/last
+                                 watch_a=watch
+                                 )
+        return output
+
+    @views_blueprint.route("/diff/<string:uuid>", methods=['GET'])
+    @login_optionally_required
+    def diff_history_page(uuid):
+        return _render_diff_template(uuid)
+
+    @views_blueprint.route("/diff/<string:uuid>/style", methods=['POST'])
+    @login_optionally_required
+    def diff_history_page_set_preferences(uuid):
+        """Handle POST request to set diff display preferences via cookies"""
+        # Load preferences from POST form and set cookies
+        pref_manager = PreferenceManager(DIFF_PREFERENCES_CONFIG, cookie_scope='global')
+        diff_prefs = pref_manager.load_from_form()
+
+        # Build redirect params including preferences (for shareable URLs) and preserved params
+        redirect_params = {}
+
+        # Add diff preferences to URL so it's shareable
+        for key, value in diff_prefs.items():
+            if isinstance(value, bool):
+                redirect_params[key] = 'on' if value else 'off'
+            else:
+                redirect_params[key] = value
+
+        # Preserve query parameters (from_version, to_version) but exclude csrf_token
+        for param in ['from_version', 'to_version']:
+            if param in request.args:
+                redirect_params[param] = request.args.get(param)
+
+        # Ensure csrf_token is never included in redirect URL
+        redirect_params.pop('csrf_token', None)
+
+        # Redirect back to GET with all params and apply cookies
+        redirect_url = url_for('ui.ui_views.diff_history_page', uuid=uuid, **redirect_params) + '#text'
+        response = make_response(redirect(redirect_url))
+        return pref_manager.apply_cookies_to_response(response)
+
+    @views_blueprint.route("/form/add/quickwatch", methods=['POST'])
+    @login_optionally_required
+    def form_quick_watch_add():
+        from changedetectionio import forms
+        form = forms.quickWatchForm(request.form)
+
+        if not form.validate():
+            for widget, l in form.errors.items():
+                flash(','.join(l), 'error')
+            return redirect(url_for('watchlist.index'))
+
+        url = request.form.get('url').strip()
+        if datastore.url_exists(url):
+            flash(f'Warning, URL {url} already exists', "notice")
+
+        add_paused = request.form.get('edit_and_watch_submit_button') != None
+        processor = request.form.get('processor', 'text_json_diff')
+        new_uuid = datastore.add_watch(url=url, tag=request.form.get('tags').strip(), extras={'paused': add_paused, 'processor': processor})
+
+        if new_uuid:
+            if add_paused:
+                flash('Watch added in Paused state, saving will unpause.')
+                return redirect(url_for('ui.ui_edit.edit_page', uuid=new_uuid, unpause_on_save=1, tag=request.args.get('tag')))
+            else:
+                # Straight into the queue.
+                worker_handler.queue_item_async_safe(update_q, queuedWatchMetaData.PrioritizedItem(priority=1, item={'uuid': new_uuid}))
+                flash("Watch added.")
+
+        return redirect(url_for('watchlist.index', tag=request.args.get('tag','')))
+
+    return views_blueprint

changedetectionio/blueprint/watchlist/__init__.py

@@ -0,0 +1,116 @@
+import os
+import time
+
+from flask import Blueprint, request, make_response, render_template, redirect, url_for, flash, session
+from flask_login import current_user
+from flask_paginate import Pagination, get_page_parameter
+
+from changedetectionio import forms
+from changedetectionio.store import ChangeDetectionStore
+from changedetectionio.auth_decorator import login_optionally_required
+
+def construct_blueprint(datastore: ChangeDetectionStore, update_q, queuedWatchMetaData):
+    watchlist_blueprint = Blueprint('watchlist', __name__, template_folder="templates")
+    
+    @watchlist_blueprint.route("/", methods=['GET'])
+    @login_optionally_required
+    def index():
+        active_tag_req = request.args.get('tag', '').lower().strip()
+        active_tag_uuid = active_tag = None
+
+        # Be sure limit_tag is a uuid
+        if active_tag_req:
+            for uuid, tag in datastore.data['settings']['application'].get('tags', {}).items():
+                if active_tag_req == tag.get('title', '').lower().strip() or active_tag_req == uuid:
+                    active_tag = tag
+                    active_tag_uuid = uuid
+                    break
+
+        # Redirect for the old rss path which used the /?rss=true
+        if request.args.get('rss'):
+            return redirect(url_for('rss.feed', tag=active_tag_uuid))
+
+        op = request.args.get('op')
+        if op:
+            uuid = request.args.get('uuid')
+            if op == 'pause':
+                datastore.data['watching'][uuid].toggle_pause()
+            elif op == 'mute':
+                datastore.data['watching'][uuid].toggle_mute()
+
+            datastore.needs_write = True
+            return redirect(url_for('watchlist.index', tag = active_tag_uuid))
+
+        # Sort by last_changed and add the uuid which is usually the key..
+        sorted_watches = []
+        with_errors = request.args.get('with_errors') == "1"
+        unread_only = request.args.get('unread') == "1"
+        errored_count = 0
+        search_q = request.args.get('q').strip().lower() if request.args.get('q') else False
+        for uuid, watch in datastore.data['watching'].items():
+            if with_errors and not watch.get('last_error'):
+                continue
+
+            if unread_only and (watch.viewed or watch.last_changed == 0) :
+                continue
+
+            if active_tag_uuid and not active_tag_uuid in watch['tags']:
+                    continue
+            if watch.get('last_error'):
+                errored_count += 1
+
+            if search_q:
+                if (watch.get('title') and search_q in watch.get('title').lower()) or search_q in watch.get('url', '').lower():
+                    sorted_watches.append(watch)
+                elif watch.get('last_error') and search_q in watch.get('last_error').lower():
+                    sorted_watches.append(watch)
+            else:
+                sorted_watches.append(watch)
+
+        form = forms.quickWatchForm(request.form)
+        page = request.args.get(get_page_parameter(), type=int, default=1)
+        total_count = len(sorted_watches)
+
+        pagination = Pagination(page=page,
+                                total=total_count,
+                                per_page=datastore.data['settings']['application'].get('pager_size', 50), css_framework="semantic")
+
+        sorted_tags = sorted(datastore.data['settings']['application'].get('tags').items(), key=lambda x: x[1]['title'])
+
+        output = render_template(
+            "watch-overview.html",
+            active_tag=active_tag,
+            active_tag_uuid=active_tag_uuid,
+            app_rss_token=datastore.data['settings']['application'].get('rss_access_token'),
+            datastore=datastore,
+            errored_count=errored_count,
+            form=form,
+            guid=datastore.data['app_guid'],
+            has_proxies=datastore.proxy_list,
+            hosted_sticky=os.getenv("SALTED_PASS", False) == False,
+            now_time_server=round(time.time()),
+            pagination=pagination,
+            queued_uuids=[q_uuid.item['uuid'] for q_uuid in update_q.queue],
+            search_q=request.args.get('q', '').strip(),
+            sort_attribute=request.args.get('sort') if request.args.get('sort') else request.cookies.get('sort'),
+            sort_order=request.args.get('order') if request.args.get('order') else request.cookies.get('order'),
+            system_default_fetcher=datastore.data['settings']['application'].get('fetch_backend'),
+            tags=sorted_tags,
+            unread_changes_count=datastore.unread_changes_count,
+            watches=sorted_watches
+        )
+
+        if session.get('share-link'):
+            del (session['share-link'])
+
+        resp = make_response(output)
+
+        # The template can run on cookie or url query info
+        if request.args.get('sort'):
+            resp.set_cookie('sort', request.args.get('sort'))
+        if request.args.get('order'):
+            resp.set_cookie('order', request.args.get('order'))
+
+        return resp
+        
+    return watchlist_blueprint

changedetectionio/blueprint/watchlist/templates/watch-overview.html

@@ -0,0 +1,273 @@
+{%- extends 'base.html' -%}
+{%- block content -%}
+{%- from '_helpers.html' import render_simple_field, render_field, render_nolabel_field, sort_by_title -%}
+<script src="{{url_for('static_content', group='js', filename='jquery-3.6.0.min.js')}}"></script>
+<script src="{{url_for('static_content', group='js', filename='watch-overview.js')}}" defer></script>
+<script>let nowtimeserver={{ now_time_server }};</script>
+<script>let favicon_baseURL="{{ url_for('static_content', group='favicon', filename="PLACEHOLDER")}}";</script>
+<script>
+// Initialize Feather icons after the page loads
+document.addEventListener('DOMContentLoaded', function() {
+    feather.replace();
+});
+</script>
+<style>
+.checking-now .last-checked {
+    background-image: linear-gradient(to bottom, transparent 0%, rgba(0,0,0,0.05) 40%, rgba(0,0,0,0.1) 100%);
+    background-size: 0 100%;
+    background-repeat: no-repeat;
+    transition: background-size 0.9s ease
+}
+</style>
+<div class="box" id="form-quick-watch-add">
+
+    <form class="pure-form" action="{{ url_for('ui.ui_views.form_quick_watch_add', tag=active_tag_uuid) }}" method="POST" id="new-watch-form">
+        <input type="hidden" name="csrf_token" value="{{ csrf_token() }}" >
+        <fieldset>
+            <legend>Add a new web page change detection watch</legend>
+            <div id="watch-add-wrapper-zone">
+                    {{ render_nolabel_field(form.url, placeholder="https://...", required=true) }}
+                    {{ render_nolabel_field(form.watch_submit_button, title="Watch this URL!" ) }}
+                    {{ render_nolabel_field(form.edit_and_watch_submit_button, title="Edit first then Watch") }}
+            </div>
+            <div id="watch-group-tag">
+               {{ render_field(form.tags, value=active_tag.title if active_tag_uuid else '', placeholder="Watch group / tag", class="transparent-field") }}
+            </div>
+            <div id="quick-watch-processor-type">
+                {{ render_simple_field(form.processor) }}
+            </div>
+
+        </fieldset>
+        <span style="color:#eee; font-size: 80%;"><img alt="Create a shareable link" style="height: 1em;display:inline-block;" src="{{url_for('static_content', group='images', filename='spread-white.svg')}}" > Tip: You can also add 'shared' watches. <a href="https://github.com/dgtlmoon/changedetection.io/wiki/Sharing-a-Watch">More info</a></span>
+    </form>
+</div>
+<div class="box">
+    <form class="pure-form" action="{{ url_for('ui.form_watch_list_checkbox_operations') }}" method="POST" id="watch-list-form">
+    <input type="hidden" name="csrf_token" value="{{ csrf_token() }}" >
+    <input type="hidden" id="op_extradata" name="op_extradata" value="" >
+    <div id="checkbox-operations">
+        <button class="pure-button button-secondary button-xsmall"  name="op" value="pause"><i data-feather="pause" style="width: 14px; height: 14px; stroke: white; margin-right: 4px;"></i>Pause</button>
+        <button class="pure-button button-secondary button-xsmall"  name="op" value="unpause"><i data-feather="play" style="width: 14px; height: 14px; stroke: white; margin-right: 4px;"></i>UnPause</button>
+        <button class="pure-button button-secondary button-xsmall"  name="op" value="mute"><i data-feather="volume-x" style="width: 14px; height: 14px; stroke: white; margin-right: 4px;"></i>Mute</button>
+        <button class="pure-button button-secondary button-xsmall"  name="op" value="unmute"><i data-feather="volume-2" style="width: 14px; height: 14px; stroke: white; margin-right: 4px;"></i>UnMute</button>
+        <button class="pure-button button-secondary button-xsmall" name="op" value="recheck"><i data-feather="refresh-cw" style="width: 14px; height: 14px; stroke: white; margin-right: 4px;"></i>Recheck</button>
+        <button class="pure-button button-secondary button-xsmall" name="op" value="assign-tag" id="checkbox-assign-tag"><i data-feather="tag" style="width: 14px; height: 14px; stroke: white; margin-right: 4px;"></i>Tag</button>
+        <button class="pure-button button-secondary button-xsmall" name="op" value="mark-viewed"><i data-feather="eye" style="width: 14px; height: 14px; stroke: white; margin-right: 4px;"></i>Mark viewed</button>
+        <button class="pure-button button-secondary button-xsmall" name="op" value="notification-default"><i data-feather="bell" style="width: 14px; height: 14px; stroke: white; margin-right: 4px;"></i>Use default notification</button>
+        <button class="pure-button button-secondary button-xsmall" name="op" value="clear-errors"><i data-feather="x-circle" style="width: 14px; height: 14px; stroke: white; margin-right: 4px;"></i>Clear errors</button>
+        <button class="pure-button button-secondary button-xsmall" style="background: #dd4242;" name="op" value="clear-history"><i data-feather="trash-2" style="width: 14px; height: 14px; stroke: white; margin-right: 4px;"></i>Clear/reset history</button>
+        <button class="pure-button button-secondary button-xsmall" style="background: #dd4242;" name="op" value="delete"><i data-feather="trash" style="width: 14px; height: 14px; stroke: white; margin-right: 4px;"></i>Delete</button>
+    </div>
+    {%- if watches|length >= pagination.per_page -%}
+        {{ pagination.info }}
+    {%- endif -%}
+    {%- if search_q -%}<div id="search-result-info">Searching "<strong><i>{{search_q}}</i></strong>"</div>{%- endif -%}
+    <div>
+        <a href="{{url_for('watchlist.index')}}" class="pure-button button-tag {{'active' if not active_tag_uuid }}">All</a>
+
+    <!-- tag list -->
+    {%- for uuid, tag in tags -%}
+        {%- if tag != "" -%}
+            <a href="{{url_for('watchlist.index', tag=uuid) }}" class="pure-button button-tag {{'active' if active_tag_uuid == uuid }}">{{ tag.title }}</a>
+        {%- endif -%}
+    {%- endfor -%}
+    </div>
+
+    {%- set sort_order = sort_order or 'asc' -%}
+    {%- set sort_attribute = sort_attribute or 'last_changed'  -%}
+    {%- set pagination_page = request.args.get('page', 0) -%}
+    {%- set cols_required = 6 -%}
+    {%- set any_has_restock_price_processor = datastore.any_watches_have_processor_by_name("restock_diff") -%}
+    {%- if any_has_restock_price_processor -%}
+        {%- set cols_required = cols_required + 1 -%}
+    {%- endif -%}
+    {%- set ui_settings = datastore.data['settings']['application']['ui'] -%}
+    {%- set wrapper_classes = [
+        'has-unread-changes' if unread_changes_count else '',
+        'has-error' if errored_count else '',
+    ] -%}
+    <div id="watch-table-wrapper" class="{{ wrapper_classes | reject('equalto', '') | join(' ') }}">
+        {%- set table_classes = [
+            'favicon-enabled' if 'favicons_enabled' not in ui_settings or ui_settings['favicons_enabled'] else 'favicon-not-enabled',
+        ] -%}
+        <table class="pure-table pure-table-striped watch-table {{ table_classes | reject('equalto', '') | join(' ') }}">
+            <thead>
+            <tr>
+                {%- set link_order = "desc" if sort_order  == 'asc' else "asc" -%}
+                {%- set arrow_span = "" -%}
+                <th><input style="vertical-align: middle" type="checkbox" id="check-all" > <a class="{{ 'active '+link_order if sort_attribute == 'date_created' else 'inactive' }}"  href="{{url_for('watchlist.index', sort='date_created', order=link_order, tag=active_tag_uuid)}}"># <span class='arrow {{link_order}}'></span></a></th>
+                <th>
+                    <a class="{{ 'active '+link_order if sort_attribute == 'paused' else 'inactive' }}" href="{{url_for('watchlist.index', sort='paused', order=link_order, tag=active_tag_uuid)}}"><i data-feather="pause" style="vertical-align: bottom; width: 14px; height: 14px;  margin-right: 4px;"></i><span class='arrow {{link_order}}'></span></a>
+                    &nbsp;
+                    <a class="{{ 'active '+link_order if sort_attribute == 'notification_muted' else 'inactive' }}" href="{{url_for('watchlist.index', sort='notification_muted', order=link_order, tag=active_tag_uuid)}}"><i data-feather="volume-2" style="vertical-align: bottom; width: 14px; height: 14px;  margin-right: 4px;"></i><span class='arrow {{link_order}}'></span></a>
+                </th>
+                <th><a class="{{ 'active '+link_order if sort_attribute == 'label' else 'inactive' }}" href="{{url_for('watchlist.index', sort='label', order=link_order, tag=active_tag_uuid)}}">Website <span class='arrow {{link_order}}'></span></a></th>
+             {%- if any_has_restock_price_processor -%}
+                <th>Restock &amp; Price</th>
+             {%- endif -%}
+                <th><a class="{{ 'active '+link_order if sort_attribute == 'last_checked' else 'inactive' }}" href="{{url_for('watchlist.index', sort='last_checked', order=link_order, tag=active_tag_uuid)}}"><span class="hide-on-mobile">Last</span> Checked <span class='arrow {{link_order}}'></span></a></th>
+                <th><a class="{{ 'active '+link_order if sort_attribute == 'last_changed' else 'inactive' }}" href="{{url_for('watchlist.index', sort='last_changed', order=link_order, tag=active_tag_uuid)}}"><span class="hide-on-mobile">Last</span> Changed <span class='arrow {{link_order}}'></span></a></th>
+                <th class="empty-cell"></th>
+            </tr>
+            </thead>
+            <tbody>
+            {%- if not watches|length -%}
+            <tr>
+                <td colspan="{{ cols_required }}" style="text-wrap: wrap;">No website watches configured, please add a URL in the box above, or <a href="{{ url_for('imports.import_page')}}" >import a list</a>.</td>
+            </tr>
+            {%- endif -%}
+
+            {%- for watch in (watches|sort(attribute=sort_attribute, reverse=sort_order == 'asc'))|pagination_slice(skip=pagination.skip) -%}
+                {%- set checking_now = is_checking_now(watch) -%}
+                {%- set history_n = watch.history_n -%}
+                {%- set favicon = watch.get_favicon_filename() -%}
+                {%- set system_use_url_watchlist = datastore.data['settings']['application']['ui'].get('use_page_title_in_list')  -%}
+                {#  Class settings mirrored in changedetectionio/static/js/realtime.js for the frontend #}
+                {%- set row_classes = [
+                    loop.cycle('pure-table-odd', 'pure-table-even'),
+                    'processor-' ~ watch['processor'],
+                    'has-error' if watch.compile_error_texts()|length > 2 else '',
+                    'paused' if watch.paused is defined and watch.paused != False else '',
+                    'unviewed' if watch.has_unviewed else '',
+                    'has-restock-info' if watch.has_restock_info else 'no-restock-info',
+                    'has-favicon' if favicon else '',
+                    'in-stock' if watch.has_restock_info and watch['restock']['in_stock'] else '',
+                    'not-in-stock' if watch.has_restock_info and not watch['restock']['in_stock'] else '',
+                    'queued' if watch.uuid in queued_uuids else '',
+                    'checking-now' if checking_now else '',
+                    'notification_muted' if watch.notification_muted else '',
+                    'single-history' if history_n == 1 else '',
+                    'multiple-history' if history_n >= 2 else '',
+                    'use-html-title' if system_use_url_watchlist else 'no-html-title',
+                ] -%}
+            <tr id="{{ watch.uuid }}" data-watch-uuid="{{ watch.uuid }}" class="{{ row_classes | reject('equalto', '') | join(' ') }}">
+                <td class="inline checkbox-uuid" ><div><input name="uuids"  type="checkbox" value="{{ watch.uuid}} " > <span class="counter-i">{{ loop.index+pagination.skip }}</span></div></td>
+                <td class="inline watch-controls">
+                    <div>
+                    <a class="ajax-op state-off pause-toggle" data-op="pause" href="{{url_for('watchlist.index', op='pause', uuid=watch.uuid, tag=active_tag_uuid)}}"><img src="{{url_for('static_content', group='images', filename='pause.svg')}}" alt="Pause checks" title="Pause checks" class="icon icon-pause" ></a>
+                    <a class="ajax-op state-on pause-toggle"  data-op="pause" style="display: none" href="{{url_for('watchlist.index', op='pause', uuid=watch.uuid, tag=active_tag_uuid)}}"><img src="{{url_for('static_content', group='images', filename='play.svg')}}" alt="UnPause checks" title="UnPause checks" class="icon icon-unpause" ></a>
+                    <a class="ajax-op state-off mute-toggle" data-op="mute" href="{{url_for('watchlist.index', op='mute', uuid=watch.uuid, tag=active_tag_uuid)}}"><img src="{{url_for('static_content', group='images', filename='bell-off.svg')}}" alt="Mute notification" title="Mute notification" class="icon icon-mute" ></a>
+                    <a class="ajax-op state-on mute-toggle" data-op="mute"  style="display: none" href="{{url_for('watchlist.index', op='mute', uuid=watch.uuid, tag=active_tag_uuid)}}"><img src="{{url_for('static_content', group='images', filename='bell-off.svg')}}" alt="UnMute notification" title="UnMute notification" class="icon icon-mute" ></a>
+                    </div>
+                </td>
+
+                <td class="title-col inline">
+                    <div class="flex-wrapper">
+                    {% if 'favicons_enabled' not in ui_settings or ui_settings['favicons_enabled'] %}
+                        <div>{# A page might have hundreds of these images, set IMG options for lazy loading, don't set SRC if we dont have it so it doesnt fetch the placeholder'  #}
+                            <img alt="Favicon thumbnail" class="favicon" loading="lazy" decoding="async" fetchpriority="low" {% if favicon %} src="{{url_for('static_content', group='favicon', filename=watch.uuid)}}" {% else %} src='data:image/svg+xml;utf8,%3Csvg xmlns="http://www.w3.org/2000/svg" width="7.087" height="7.087" viewBox="0 0 7.087 7.087"%3E%3Ccircle cx="3.543" cy="3.543" r="3.279" stroke="%23e1e1e1" stroke-width="0.45" fill="none" opacity="0.74"/%3E%3C/svg%3E' {%  endif %} />
+                        </div>
+                    {%  endif %}
+                        <div>
+                        <span class="watch-title">
+                            {% if system_use_url_watchlist or watch.get('use_page_title_in_list') %}
+                                {{ watch.label }}
+                            {% else %}
+                                {{ watch.get('title') or watch.link }}
+                            {% endif %}
+                           <a class="external" target="_blank" rel="noopener" href="{{ watch.link.replace('source:','') }}">&nbsp;</a>
+                        </span>
+                            <div class="error-text" style="display:none;">{{ watch.compile_error_texts(has_proxies=datastore.proxy_list) }}</div>
+                            {%- if watch['processor'] == 'text_json_diff'  -%}
+                                {%- if watch['has_ldjson_price_data'] and not watch['track_ldjson_price_data']  -%}
+                                <div class="ldjson-price-track-offer">Switch to Restock & Price watch mode? <a href="{{url_for('price_data_follower.accept', uuid=watch.uuid)}}" class="pure-button button-xsmall">Yes</a> <a href="{{url_for('price_data_follower.reject', uuid=watch.uuid)}}" class="">No</a></div>
+                                {%- endif -%}
+                            {%- endif -%}
+                            {%- if watch['processor'] == 'restock_diff' -%}
+                                <span class="tracking-ldjson-price-data" title="Automatically following embedded price information"><img src="{{url_for('static_content', group='images', filename='price-tag-icon.svg')}}"  class="status-icon price-follow-tag-icon" > Price</span>
+                            {%- endif -%}
+                            {%- for watch_tag_uuid, watch_tag in datastore.get_all_tags_for_watch(watch['uuid']).items() -%}
+                              <span class="watch-tag-list">{{ watch_tag.title }}</span>
+                            {%- endfor -%}
+                        </div>
+                    <div class="status-icons">
+                            <a class="link-spread" href="{{url_for('ui.form_share_put_watch', uuid=watch.uuid)}}"><img src="{{url_for('static_content', group='images', filename='spread.svg')}}" class="status-icon icon icon-spread" title="Create a link to share watch config with others" ></a>
+                            {%- if watch.get_fetch_backend == "html_webdriver"
+                                 or ( watch.get_fetch_backend == "system" and system_default_fetcher == 'html_webdriver'  )
+                                 or "extra_browser_" in watch.get_fetch_backend
+                            -%}
+                            <img class="status-icon" src="{{url_for('static_content', group='images', filename='google-chrome-icon.png')}}" alt="Using a Chrome browser" title="Using a Chrome browser" >
+                            {%- endif -%}
+                            {%- if watch.is_pdf  -%}<img class="status-icon" src="{{url_for('static_content', group='images', filename='pdf-icon.svg')}}" alt="Converting PDF to text" >{%- endif -%}
+                            {%- if watch.has_browser_steps -%}<img class="status-icon status-browsersteps" src="{{url_for('static_content', group='images', filename='steps.svg')}}" alt="Browser Steps is enabled" >{%- endif -%}
+
+                    </div>
+                    </div>
+                </td>
+{%- if any_has_restock_price_processor -%}
+                <td class="restock-and-price">
+                    {%- if watch['processor'] == 'restock_diff'  -%}
+                        {%- if watch.has_restock_info -%}
+                            <span class="restock-label {{'in-stock' if watch['restock']['in_stock'] else 'not-in-stock' }}" title="Detecting restock and price">
+                                <!-- maybe some object watch['processor'][restock_diff] or.. -->
+                                 {%- if watch['restock']['in_stock']-%}  In stock {%- else-%}  Not in stock {%- endif -%}
+                            </span>
+                        {%- endif -%}
+
+                        {%- if watch.get('restock') and watch['restock']['price'] != None -%}
+                            {%- if watch['restock']['price'] != None -%}
+                                <span class="restock-label price" title="Price">
+                                {{ watch['restock']['price']|format_number_locale }} {{ watch['restock']['currency'] }}
+                                </span>
+                            {%- endif -%}
+                        {%- elif not watch.has_restock_info -%}
+                            <span class="restock-label error">No information</span>
+                        {%- endif -%}
+                    {%- endif -%}
+                </td>
+{%- endif -%}
+            {#last_checked becomes fetch-start-time#}
+                <td class="last-checked" data-timestamp="{{ watch.last_checked }}" data-fetchduration={{ watch.fetch_time }} data-eta_complete="{{ watch.last_checked+watch.fetch_time }}" >
+                    <div class="spinner-wrapper" style="display:none;" >
+                        <span class="spinner"></span><span>&nbsp;Checking now</span>
+                    </div>
+                    <span class="innertext">{{watch|format_last_checked_time|safe}}</span>
+                </td>
+                <td class="last-changed" data-timestamp="{{ watch.last_changed }}">{%- if watch.history_n >=2 and watch.last_changed >0 -%}
+                    {{watch.last_changed|format_timestamp_timeago}}
+                    {%- else -%}
+                    Not yet
+                    {%- endif -%}
+                </td>
+                <td class="buttons">
+                    <div>
+                    {%- set target_attr = ' target="' ~ watch.uuid ~ '"' if datastore.data['settings']['application']['ui'].get('open_diff_in_new_tab') else '' -%}
+                    <a href="" class="already-in-queue-button recheck pure-button pure-button-primary" style="display: none;" disabled="disabled">Queued</a>
+                    <a href="{{ url_for('ui.form_watch_checknow', uuid=watch.uuid, tag=request.args.get('tag')) }}" data-op='recheck' class="ajax-op recheck pure-button pure-button-primary">Recheck</a>
+                    <a href="{{ url_for('ui.ui_edit.edit_page', uuid=watch.uuid, tag=active_tag_uuid)}}#general" class="pure-button pure-button-primary">Edit</a>
+                    <a href="{{ url_for('ui.ui_views.diff_history_page', uuid=watch.uuid)}}" {{target_attr}} class="pure-button pure-button-primary history-link" style="display: none;">History</a>
+                    <a href="{{ url_for('ui.ui_views.preview_page', uuid=watch.uuid)}}" {{target_attr}} class="pure-button pure-button-primary preview-link" style="display: none;">Preview</a>
+                    </div>
+                </td>
+            </tr>
+            {%- endfor -%}
+            </tbody>
+        </table>
+        <ul id="post-list-buttons">
+            <li id="post-list-with-errors" style="display: none;" >
+                <a href="{{url_for('watchlist.index', with_errors=1, tag=request.args.get('tag')) }}" class="pure-button button-tag button-error">With errors ({{ errored_count }})</a>
+            </li>
+            <li id="post-list-mark-views" style="display: none;" >
+                <a href="{{url_for('ui.mark_all_viewed',with_errors=request.args.get('with_errors',0)) }}" class="pure-button button-tag " id="mark-all-viewed">Mark all viewed</a>
+            </li>
+        {%-  if active_tag_uuid -%}
+            <li id="post-list-mark-views-tag">
+                <a href="{{url_for('ui.mark_all_viewed', tag=active_tag_uuid) }}" class="pure-button button-tag " id="mark-all-viewed">Mark all viewed in '{{active_tag.title}}'</a>
+            </li>
+        {%-  endif -%}
+            <li id="post-list-unread" style="display: none;" >
+                <a href="{{url_for('watchlist.index', unread=1, tag=request.args.get('tag')) }}" class="pure-button button-tag">Unread (<span id="unread-tab-counter">{{ unread_changes_count }}</span>)</a>
+            </li>
+            <li>
+               <a href="{{ url_for('ui.form_watch_checknow', tag=active_tag_uuid, with_errors=request.args.get('with_errors',0)) }}" class="pure-button button-tag" id="recheck-all">Recheck
+                all {% if active_tag_uuid %}  in '{{active_tag.title}}'{%endif%}</a>
+            </li>
+            <li>
+                <a href="{{ url_for('rss.feed', tag=active_tag_uuid, token=app_rss_token)}}"><img alt="RSS Feed" id="feed-icon" src="{{url_for('static_content', group='images', filename='generic_feed-icon.svg')}}" height="15"></a>
+            </li>
+        </ul>
+        {{ pagination.links }}
+    </div>
+    </form>
+</div>
+{%- endblock -%}

changedetectionio/changedetection.py

@@ -1,153 +0,0 @@
-#!/usr/bin/python3
-
-# Launch as a eventlet.wsgi server instance.
-
-from distutils.util import strtobool
-from json.decoder import JSONDecodeError
-
-import eventlet
-import eventlet.wsgi
-import getopt
-import os
-import signal
-import socket
-import sys
-
-from . import store, changedetection_app, content_fetcher
-from . import __version__
-
-# Only global so we can access it in the signal handler
-app = None
-datastore = None
-
-def sigterm_handler(_signo, _stack_frame):
-    global app
-    global datastore
-#    app.config.exit.set()
-    print('Shutdown: Got SIGTERM, DB saved to disk')
-    datastore.sync_to_json()
-#    raise SystemExit
-
-def main():
-    global datastore
-    global app
-
-    datastore_path = None
-    do_cleanup = False
-    host = ''
-    ipv6_enabled = False
-    port = os.environ.get('PORT') or 5000
-    ssl_mode = False
-
-    # On Windows, create and use a default path.
-    if os.name == 'nt':
-        datastore_path = os.path.expandvars(r'%APPDATA%\changedetection.io')
-        os.makedirs(datastore_path, exist_ok=True)
-    else:
-        # Must be absolute so that send_from_directory doesnt try to make it relative to backend/
-        datastore_path = os.path.join(os.getcwd(), "../datastore")
-
-    try:
-        opts, args = getopt.getopt(sys.argv[1:], "6Ccsd:h:p:", "port")
-    except getopt.GetoptError:
-        print('backend.py -s SSL enable -h [host] -p [port] -d [datastore path]')
-        sys.exit(2)
-
-    create_datastore_dir = False
-
-    for opt, arg in opts:
-        if opt == '-s':
-            ssl_mode = True
-
-        if opt == '-h':
-            host = arg
-
-        if opt == '-p':
-            port = int(arg)
-
-        if opt == '-d':
-            datastore_path = arg
-
-        if opt == '-6':
-            print ("Enabling IPv6 listen support")
-            ipv6_enabled = True
-
-        # Cleanup (remove text files that arent in the index)
-        if opt == '-c':
-            do_cleanup = True
-
-        # Create the datadir if it doesnt exist
-        if opt == '-C':
-            create_datastore_dir = True
-
-    # isnt there some @thingy to attach to each route to tell it, that this route needs a datastore
-    app_config = {'datastore_path': datastore_path}
-
-    if not os.path.isdir(app_config['datastore_path']):
-        if create_datastore_dir:
-            os.mkdir(app_config['datastore_path'])
-        else:
-            print(
-                "ERROR: Directory path for the datastore '{}' does not exist, cannot start, please make sure the directory exists or specify a directory with the -d option.\n"
-                "Or use the -C parameter to create the directory.".format(app_config['datastore_path']), file=sys.stderr)
-            sys.exit(2)
-
-    try:
-        datastore = store.ChangeDetectionStore(datastore_path=app_config['datastore_path'], version_tag=__version__)
-    except JSONDecodeError as e:
-        # Dont' start if the JSON DB looks corrupt
-        print ("ERROR: JSON DB or Proxy List JSON at '{}' appears to be corrupt, aborting".format(app_config['datastore_path']))
-        print(str(e))
-        return
-
-    app = changedetection_app(app_config, datastore)
-
-    signal.signal(signal.SIGTERM, sigterm_handler)
-
-    # Go into cleanup mode
-    if do_cleanup:
-        datastore.remove_unused_snapshots()
-
-    app.config['datastore_path'] = datastore_path
-
-
-    @app.context_processor
-    def inject_version():
-        return dict(right_sticky="v{}".format(datastore.data['version_tag']),
-                    new_version_available=app.config['NEW_VERSION_AVAILABLE'],
-                    has_password=datastore.data['settings']['application']['password'] != False
-                    )
-
-    # Monitored websites will not receive a Referer header when a user clicks on an outgoing link.
-    # @Note: Incompatible with password login (and maybe other features) for now, submit a PR!
-    @app.after_request
-    def hide_referrer(response):
-        if strtobool(os.getenv("HIDE_REFERER", 'false')):
-            response.headers["Referrer-Policy"] = "no-referrer"
-
-        return response
-
-    # Proxy sub-directory support
-    # Set environment var USE_X_SETTINGS=1 on this script
-    # And then in your proxy_pass settings
-    #
-    #         proxy_set_header Host "localhost";
-    #         proxy_set_header X-Forwarded-Prefix /app;
-
-    if os.getenv('USE_X_SETTINGS'):
-        print ("USE_X_SETTINGS is ENABLED\n")
-        from werkzeug.middleware.proxy_fix import ProxyFix
-        app.wsgi_app = ProxyFix(app.wsgi_app, x_prefix=1, x_host=1)
-
-    s_type = socket.AF_INET6 if ipv6_enabled else socket.AF_INET
-
-    if ssl_mode:
-        # @todo finalise SSL config, but this should get you in the right direction if you need it.
-        eventlet.wsgi.server(eventlet.wrap_ssl(eventlet.listen((host, port), s_type),
-                                               certfile='cert.pem',
-                                               keyfile='privkey.pem',
-                                               server_side=True), app)
-
-    else:
-        eventlet.wsgi.server(eventlet.listen((host, int(port)), s_type), app)
-

changedetectionio/conditions/__init__.py

@@ -0,0 +1,167 @@
+from json_logic.builtins import BUILTINS
+
+from .exceptions import EmptyConditionRuleRowNotUsable
+from .pluggy_interface import plugin_manager  # Import the pluggy plugin manager
+from . import default_plugin
+from loguru import logger
+# List of all supported JSON Logic operators
+operator_choices = [
+    (None, "Choose one - Operator"),
+    (">", "Greater Than"),
+    ("<", "Less Than"),
+    (">=", "Greater Than or Equal To"),
+    ("<=", "Less Than or Equal To"),
+    ("==", "Equals"),
+    ("!=", "Not Equals"),
+    ("in", "Contains"),
+]
+
+# Fields available in the rules
+field_choices = [
+    (None, "Choose one - Field"),
+]
+
+# The data we will feed the JSON Rules to see if it passes the test/conditions or not
+EXECUTE_DATA = {}
+
+
+# Define the extended operations dictionary
+CUSTOM_OPERATIONS = {
+    **BUILTINS,  # Include all standard operators
+}
+
+def filter_complete_rules(ruleset):
+    rules = [
+        rule for rule in ruleset
+        if all(value not in ("", False, "None", None) for value in [rule["operator"], rule["field"], rule["value"]])
+    ]
+    return rules
+
+def convert_to_jsonlogic(logic_operator: str, rule_dict: list):
+    """
+    Convert a structured rule dict into a JSON Logic rule.
+
+    :param rule_dict: Dictionary containing conditions.
+    :return: JSON Logic rule as a dictionary.
+    """
+
+
+    json_logic_conditions = []
+
+    for condition in rule_dict:
+        operator = condition["operator"]
+        field = condition["field"]
+        value = condition["value"]
+
+        if not operator or operator == 'None' or not value or not field:
+            raise EmptyConditionRuleRowNotUsable()
+
+        # Convert value to int/float if possible
+        try:
+            if isinstance(value, str) and "." in value and str != "None":
+                value = float(value)
+            else:
+                value = int(value)
+        except (ValueError, TypeError):
+            pass  # Keep as a string if conversion fails
+
+        # Handle different JSON Logic operators properly
+        if operator == "in":
+            json_logic_conditions.append({"in": [value, {"var": field}]})  # value first
+        elif operator in ("!", "!!", "-"):
+            json_logic_conditions.append({operator: [{"var": field}]})  # Unary operators
+        elif operator in ("min", "max", "cat"):
+            json_logic_conditions.append({operator: value})  # Multi-argument operators
+        else:
+            json_logic_conditions.append({operator: [{"var": field}, value]})  # Standard binary operators
+
+    return {logic_operator: json_logic_conditions} if len(json_logic_conditions) > 1 else json_logic_conditions[0]
+
+
+def execute_ruleset_against_all_plugins(current_watch_uuid: str, application_datastruct, ephemeral_data={} ):
+    """
+    Build our data and options by calling our plugins then pass it to jsonlogic and see if the conditions pass
+
+    :param ruleset: JSON Logic rule dictionary.
+    :param extracted_data: Dictionary containing the facts.   <-- maybe the app struct+uuid
+    :return: Dictionary of plugin results.
+    """
+    from json_logic import jsonLogic
+
+    EXECUTE_DATA = {}
+    result = True
+    
+    watch = application_datastruct['watching'].get(current_watch_uuid)
+
+    if watch and watch.get("conditions"):
+        logic_operator = "and" if watch.get("conditions_match_logic", "ALL") == "ALL" else "or"
+        complete_rules = filter_complete_rules(watch['conditions'])
+        if complete_rules:
+            # Give all plugins a chance to update the data dict again (that we will test the conditions against)
+            for plugin in plugin_manager.get_plugins():
+                try:
+                    import concurrent.futures
+                    import time
+                    
+                    with concurrent.futures.ThreadPoolExecutor() as executor:
+                        future = executor.submit(
+                            plugin.add_data,
+                            current_watch_uuid=current_watch_uuid,
+                            application_datastruct=application_datastruct,
+                            ephemeral_data=ephemeral_data
+                        )
+                        logger.debug(f"Trying plugin {plugin}....")
+
+                        # Set a timeout of 10 seconds
+                        try:
+                            new_execute_data = future.result(timeout=10)
+                            if new_execute_data and isinstance(new_execute_data, dict):
+                                EXECUTE_DATA.update(new_execute_data)
+
+                        except concurrent.futures.TimeoutError:
+                            # The plugin took too long, abort processing for this watch
+                            raise Exception(f"Plugin {plugin.__class__.__name__} took more than 10 seconds to run.")
+                except Exception as e:
+                    # Log the error but continue with the next plugin
+                    import logging
+                    logging.error(f"Error executing plugin {plugin.__class__.__name__}: {str(e)}")
+                    continue
+
+            # Create the ruleset
+            ruleset = convert_to_jsonlogic(logic_operator=logic_operator, rule_dict=complete_rules)
+            
+            # Pass the custom operations dictionary to jsonLogic
+            if not jsonLogic(logic=ruleset, data=EXECUTE_DATA, operations=CUSTOM_OPERATIONS):
+                result = False
+
+    return {'executed_data': EXECUTE_DATA, 'result': result}
+
+# Load plugins dynamically
+for plugin in plugin_manager.get_plugins():
+    new_ops = plugin.register_operators()
+    if isinstance(new_ops, dict):
+        CUSTOM_OPERATIONS.update(new_ops)
+
+    new_operator_choices = plugin.register_operator_choices()
+    if isinstance(new_operator_choices, list):
+        operator_choices.extend(new_operator_choices)
+
+    new_field_choices = plugin.register_field_choices()
+    if isinstance(new_field_choices, list):
+        field_choices.extend(new_field_choices)
+
+def collect_ui_edit_stats_extras(watch):
+    """Collect and combine HTML content from all plugins that implement ui_edit_stats_extras"""
+    extras_content = []
+    
+    for plugin in plugin_manager.get_plugins():
+        try:
+            content = plugin.ui_edit_stats_extras(watch=watch)
+            if content:
+                extras_content.append(content)
+        except Exception as e:
+            # Skip plugins that don't implement the hook or have errors
+            pass
+            
+    return "\n".join(extras_content) if extras_content else ""
+

changedetectionio/conditions/blueprint.py

@@ -0,0 +1,81 @@
+# Flask Blueprint Definition
+import json
+
+from flask import Blueprint
+
+from changedetectionio.conditions import execute_ruleset_against_all_plugins
+
+
+def construct_blueprint(datastore):
+    from changedetectionio.flask_app import login_optionally_required
+
+    conditions_blueprint = Blueprint('conditions', __name__, template_folder="templates")
+
+    @conditions_blueprint.route("/<string:watch_uuid>/verify-condition-single-rule", methods=['POST'])
+    @login_optionally_required
+    def verify_condition_single_rule(watch_uuid):
+        """Verify a single condition rule against the current snapshot"""
+        from changedetectionio.processors.text_json_diff import prepare_filter_prevew
+        from flask import request, jsonify
+        from copy import deepcopy
+
+        ephemeral_data = {}
+
+        # Get the watch data
+        watch = datastore.data['watching'].get(watch_uuid)
+        if not watch:
+            return jsonify({'status': 'error', 'message': 'Watch not found'}), 404
+
+        # First use prepare_filter_prevew to process the form data
+        # This will return text_after_filter which is after all current form settings are applied
+        # Create ephemeral data with the text from the current snapshot
+
+        try:
+            # Call prepare_filter_prevew to get a processed version of the content with current form settings
+            # We'll ignore the returned response and just use the datastore which is modified by the function
+
+            # this should apply all filters etc so then we can run the CONDITIONS against the final output text
+            result = prepare_filter_prevew(datastore=datastore,
+                                           form_data=request.form,
+                                           watch_uuid=watch_uuid)
+
+            ephemeral_data['text'] = result.get('after_filter', '')
+            # Create a temporary watch data structure with this single rule
+            tmp_watch_data = deepcopy(datastore.data['watching'].get(watch_uuid))
+
+            # Override the conditions in the temporary watch
+            rule_json = request.args.get("rule")
+            rule = json.loads(rule_json) if rule_json else None
+
+            # Should be key/value of field, operator, value
+            tmp_watch_data['conditions'] = [rule]
+            tmp_watch_data['conditions_match_logic'] = "ALL"  # Single rule, so use ALL
+
+            # Create a temporary application data structure for the rule check
+            temp_app_data = {
+                'watching': {
+                    watch_uuid: tmp_watch_data
+                }
+            }
+
+            # Execute the rule against the current snapshot with form data
+            result = execute_ruleset_against_all_plugins(
+                current_watch_uuid=watch_uuid,
+                application_datastruct=temp_app_data,
+                ephemeral_data=ephemeral_data
+            )
+
+            return jsonify({
+                'status': 'success',
+                'result': result.get('result'),
+                'data': result.get('executed_data'),
+                'message': 'Condition passes' if result else 'Condition does not pass'
+            })
+
+        except Exception as e:
+            return jsonify({
+                'status': 'error',
+                'message': f'Error verifying condition: {str(e)}'
+            }), 500
+
+    return conditions_blueprint

changedetectionio/conditions/default_plugin.py

@@ -0,0 +1,83 @@
+import re
+
+import pluggy
+from price_parser import Price
+from loguru import logger
+
+hookimpl = pluggy.HookimplMarker("changedetectionio_conditions")
+
+
+@hookimpl
+def register_operators():
+    def starts_with(_, text, prefix):
+        return text.lower().strip().startswith(str(prefix).strip().lower())
+
+    def ends_with(_, text, suffix):
+        return text.lower().strip().endswith(str(suffix).strip().lower())
+
+    def length_min(_, text, strlen):
+        return len(text) >= int(strlen)
+
+    def length_max(_, text, strlen):
+        return len(text) <= int(strlen)
+
+    # Custom function for case-insensitive regex matching
+    def contains_regex(_, text, pattern):
+        """Returns True if `text` contains `pattern` (case-insensitive regex match)."""
+        return bool(re.search(pattern, str(text), re.IGNORECASE))
+
+    # Custom function for NOT matching case-insensitive regex
+    def not_contains_regex(_, text, pattern):
+        """Returns True if `text` does NOT contain `pattern` (case-insensitive regex match)."""
+        return not bool(re.search(pattern, str(text), re.IGNORECASE))
+
+    def not_contains(_, text, pattern):
+        return not pattern in text
+
+    return {
+        "!in": not_contains,
+        "!contains_regex": not_contains_regex,
+        "contains_regex": contains_regex,
+        "ends_with": ends_with,
+        "length_max": length_max,
+        "length_min": length_min,
+        "starts_with": starts_with,
+    }
+
+@hookimpl
+def register_operator_choices():
+    return [
+        ("!in", "Does NOT Contain"),
+        ("starts_with", "Text Starts With"),
+        ("ends_with", "Text Ends With"),
+        ("length_min", "Length minimum"),
+        ("length_max", "Length maximum"),
+        ("contains_regex", "Text Matches Regex"),
+        ("!contains_regex", "Text Does NOT Match Regex"),
+    ]
+
+@hookimpl
+def register_field_choices():
+    return [
+        ("extracted_number", "Extracted number after 'Filters & Triggers'"),
+#        ("meta_description", "Meta Description"),
+#        ("meta_keywords", "Meta Keywords"),
+        ("page_filtered_text", "Page text after 'Filters & Triggers'"),
+        #("page_title", "Page <title>"), # actual page title <title>
+    ]
+
+@hookimpl
+def add_data(current_watch_uuid, application_datastruct, ephemeral_data):
+
+    res = {}
+    if 'text' in ephemeral_data:
+        res['page_filtered_text'] = ephemeral_data['text']
+
+        # Better to not wrap this in try/except so that the UI can see any errors
+        price = Price.fromstring(ephemeral_data.get('text'))
+        if price and price.amount != None:
+            # This is slightly misleading, it's extracting a PRICE not a Number..
+            res['extracted_number'] = float(price.amount)
+            logger.debug(f"Extracted number result: '{price}' - returning float({res['extracted_number']})")
+
+    return res

changedetectionio/conditions/exceptions.py

@@ -0,0 +1,6 @@
+class EmptyConditionRuleRowNotUsable(Exception):
+    def __init__(self):
+        super().__init__("One of the 'conditions' rulesets is incomplete, cannot run.")
+
+    def __str__(self):
+        return self.args[0]

changedetectionio/conditions/form.py

@@ -0,0 +1,44 @@
+# Condition Rule Form (for each rule row)
+from wtforms import Form, SelectField, StringField, validators
+from wtforms import validators
+
+class ConditionFormRow(Form):
+
+    # ✅ Ensure Plugins Are Loaded BEFORE Importing Choices
+    from changedetectionio.conditions import plugin_manager
+    from changedetectionio.conditions import operator_choices, field_choices
+    field = SelectField(
+        "Field",
+        choices=field_choices,
+        validators=[validators.Optional()]
+    )
+
+    operator = SelectField(
+        "Operator",
+        choices=operator_choices,
+        validators=[validators.Optional()]
+    )
+
+    value = StringField("Value", validators=[validators.Optional()], render_kw={"placeholder": "A value"})
+
+    def validate(self, extra_validators=None):
+        # First, run the default validators
+        if not super().validate(extra_validators):
+            return False
+
+        # Custom validation logic
+        # If any of the operator/field/value is set, then they must be all set
+        if any(value not in ("", False, "None", None) for value in [self.operator.data, self.field.data, self.value.data]):
+            if not self.operator.data or self.operator.data == 'None':
+                self.operator.errors.append("Operator is required.")
+                return False
+
+            if not self.field.data or self.field.data == 'None':
+                self.field.errors.append("Field is required.")
+                return False
+
+            if not self.value.data:
+                self.value.errors.append("Value is required.")
+                return False
+
+        return True  # Only return True if all conditions pass

changedetectionio/conditions/pluggy_interface.py

@@ -0,0 +1,74 @@
+import pluggy
+import os
+import importlib
+import sys
+from . import default_plugin
+
+# ✅ Ensure that the namespace in HookspecMarker matches PluginManager
+PLUGIN_NAMESPACE = "changedetectionio_conditions"
+
+hookspec = pluggy.HookspecMarker(PLUGIN_NAMESPACE)
+hookimpl = pluggy.HookimplMarker(PLUGIN_NAMESPACE)
+
+
+class ConditionsSpec:
+    """Hook specifications for extending JSON Logic conditions."""
+
+    @hookspec
+    def register_operators():
+        """Return a dictionary of new JSON Logic operators."""
+        pass
+
+    @hookspec
+    def register_operator_choices():
+        """Return a list of new operator choices."""
+        pass
+
+    @hookspec
+    def register_field_choices():
+        """Return a list of new field choices."""
+        pass
+
+    @hookspec
+    def add_data(current_watch_uuid, application_datastruct, ephemeral_data):
+        """Add to the datadict"""
+        pass
+        
+    @hookspec
+    def ui_edit_stats_extras(watch):
+        """Return HTML content to add to the stats tab in the edit view"""
+        pass
+
+# ✅ Set up Pluggy Plugin Manager
+plugin_manager = pluggy.PluginManager(PLUGIN_NAMESPACE)
+
+# ✅ Register hookspecs (Ensures they are detected)
+plugin_manager.add_hookspecs(ConditionsSpec)
+
+# ✅ Register built-in plugins manually
+plugin_manager.register(default_plugin, "default_plugin")
+
+# ✅ Load plugins from the plugins directory
+def load_plugins_from_directory():
+    plugins_dir = os.path.join(os.path.dirname(__file__), 'plugins')
+    if not os.path.exists(plugins_dir):
+        return
+        
+    # Get all Python files (excluding __init__.py)
+    for filename in os.listdir(plugins_dir):
+        if filename.endswith(".py") and filename != "__init__.py":
+            module_name = filename[:-3]  # Remove .py extension
+            module_path = f"changedetectionio.conditions.plugins.{module_name}"
+            
+            try:
+                module = importlib.import_module(module_path)
+                # Register the plugin with pluggy
+                plugin_manager.register(module, module_name)
+            except (ImportError, AttributeError) as e:
+                print(f"Error loading plugin {module_name}: {e}")
+
+# Load plugins from the plugins directory
+load_plugins_from_directory()
+
+# ✅ Discover installed plugins from external packages (if any)
+plugin_manager.load_setuptools_entrypoints(PLUGIN_NAMESPACE)

changedetectionio/conditions/plugins/__init__.py

@@ -0,0 +1 @@
+# Import plugins package to make them discoverable

changedetectionio/conditions/plugins/levenshtein_plugin.py

@@ -0,0 +1,119 @@
+import pluggy
+from loguru import logger
+
+LEVENSHTEIN_MAX_LEN_FOR_EDIT_STATS=100000
+
+# Support both plugin systems
+conditions_hookimpl = pluggy.HookimplMarker("changedetectionio_conditions")
+global_hookimpl = pluggy.HookimplMarker("changedetectionio")
+
+def levenshtein_ratio_recent_history(watch, incoming_text=None):
+    try:
+        from Levenshtein import ratio, distance
+        k = list(watch.history.keys())
+        a = None
+        b = None
+
+        # When called from ui_edit_stats_extras, we don't have incoming_text
+        if incoming_text is None:
+            a = watch.get_history_snapshot(timestamp=k[-1])  # Latest snapshot
+            b = watch.get_history_snapshot(timestamp=k[-2])  # Previous snapshot
+
+        # Needs atleast one snapshot
+        elif len(k) >= 1: # Should be atleast one snapshot to compare against
+            a = watch.get_history_snapshot(timestamp=k[-1]) # Latest saved snapshot
+            b = incoming_text if incoming_text else k[-2]
+
+        if a and b:
+            distance_value = distance(a, b)
+            ratio_value = ratio(a, b)
+            return {
+                'distance': distance_value,
+                'ratio': ratio_value,
+                'percent_similar': round(ratio_value * 100, 2)
+            }
+    except Exception as e:
+        logger.warning(f"Unable to calc similarity: {str(e)}")
+
+    return ''
+
+@conditions_hookimpl
+def register_operators():
+    pass
+
+@conditions_hookimpl
+def register_operator_choices():
+    pass
+
+
+@conditions_hookimpl
+def register_field_choices():
+    return [
+        ("levenshtein_ratio", "Levenshtein - Text similarity ratio"),
+        ("levenshtein_distance", "Levenshtein - Text change distance"),
+    ]
+
+@conditions_hookimpl
+def add_data(current_watch_uuid, application_datastruct, ephemeral_data):
+    res = {}
+    watch = application_datastruct['watching'].get(current_watch_uuid)
+    # ephemeral_data['text'] will be the current text after filters, they may have edited filters but not saved them yet etc
+
+    if watch and 'text' in ephemeral_data:
+        lev_data = levenshtein_ratio_recent_history(watch, ephemeral_data.get('text',''))
+        if isinstance(lev_data, dict):
+            res['levenshtein_ratio'] = lev_data.get('ratio', 0)
+            res['levenshtein_similarity'] = lev_data.get('percent_similar', 0)
+            res['levenshtein_distance'] = lev_data.get('distance', 0)
+
+    return res
+
+@global_hookimpl
+def ui_edit_stats_extras(watch):
+    """Add Levenshtein stats to the UI using the global plugin system"""
+    """Generate the HTML for Levenshtein stats - shared by both plugin systems"""
+    if len(watch.history.keys()) < 2:
+        return "<p>Not enough history to calculate Levenshtein metrics</p>"
+
+
+    # Protection against the algorithm getting stuck on huge documents
+    k = list(watch.history.keys())
+    if any(
+            len(watch.get_history_snapshot(timestamp=k[idx])) > LEVENSHTEIN_MAX_LEN_FOR_EDIT_STATS
+            for idx in (-1, -2)
+            if len(k) >= abs(idx)
+    ):
+        return "<p>Snapshot too large for edit statistics, skipping.</p>"
+
+    try:
+        lev_data = levenshtein_ratio_recent_history(watch)
+        if not lev_data or not isinstance(lev_data, dict):
+            return "<p>Unable to calculate Levenshtein metrics</p>"
+            
+        html = f"""
+        <div class="levenshtein-stats">
+            <h4>Levenshtein Text Similarity Details</h4>
+            <table class="pure-table">
+                <tbody>
+                    <tr>
+                        <td>Raw distance (edits needed)</td>
+                        <td>{lev_data['distance']}</td>
+                    </tr>
+                    <tr>
+                        <td>Similarity ratio</td>
+                        <td>{lev_data['ratio']:.4f}</td>
+                    </tr>
+                    <tr>
+                        <td>Percent similar</td>
+                        <td>{lev_data['percent_similar']}%</td>
+                    </tr>
+                </tbody>
+            </table>
+            <p style="font-size: 80%;">Levenshtein metrics compare the last two snapshots, measuring how many character edits are needed to transform one into the other.</p>
+        </div>
+        """
+        return html
+    except Exception as e:
+        logger.error(f"Error generating Levenshtein UI extras: {str(e)}")
+        return "<p>Error calculating Levenshtein metrics</p>"
+        

changedetectionio/conditions/plugins/wordcount_plugin.py

@@ -0,0 +1,82 @@
+import pluggy
+from loguru import logger
+
+# Support both plugin systems
+conditions_hookimpl = pluggy.HookimplMarker("changedetectionio_conditions")
+global_hookimpl = pluggy.HookimplMarker("changedetectionio")
+
+def count_words_in_history(watch, incoming_text=None):
+    """Count words in snapshot text"""
+    try:
+        if incoming_text is not None:
+            # When called from add_data with incoming text
+            return len(incoming_text.split())
+        elif watch.history.keys():
+            # When called from UI extras to count latest snapshot
+            latest_key = list(watch.history.keys())[-1]
+            latest_content = watch.get_history_snapshot(latest_key)
+            return len(latest_content.split())
+        return 0
+    except Exception as e:
+        logger.error(f"Error counting words: {str(e)}")
+        return 0
+
+# Implement condition plugin hooks
+@conditions_hookimpl
+def register_operators():
+    # No custom operators needed
+    return {}
+
+@conditions_hookimpl
+def register_operator_choices():
+    # No custom operator choices needed
+    return []
+
+@conditions_hookimpl
+def register_field_choices():
+    # Add a field that will be available in conditions
+    return [
+        ("word_count", "Word count of content"),
+    ]
+
+@conditions_hookimpl
+def add_data(current_watch_uuid, application_datastruct, ephemeral_data):
+    """Add word count data for conditions"""
+    result = {}
+    watch = application_datastruct['watching'].get(current_watch_uuid)
+    
+    if watch and 'text' in ephemeral_data:
+        word_count = count_words_in_history(watch, ephemeral_data['text'])
+        result['word_count'] = word_count
+    
+    return result
+
+def _generate_stats_html(watch):
+    """Generate the HTML content for the stats tab"""
+    word_count = count_words_in_history(watch)
+    
+    html = f"""
+    <div class="word-count-stats">
+        <h4>Content Analysis</h4>
+        <table class="pure-table">
+            <tbody>
+                <tr>
+                    <td>Word count (latest snapshot)</td>
+                    <td>{word_count}</td>
+                </tr>
+            </tbody>
+        </table>
+        <p style="font-size: 80%;">Word count is a simple measure of content length, calculated by splitting text on whitespace.</p>
+    </div>
+    """
+    return html
+
+@conditions_hookimpl
+def ui_edit_stats_extras(watch):
+    """Add word count stats to the UI through conditions plugin system"""
+    return _generate_stats_html(watch)
+
+@global_hookimpl
+def ui_edit_stats_extras(watch):
+    """Add word count stats to the UI using the global plugin system"""
+    return _generate_stats_html(watch)

changedetectionio/content_fetcher.py

@@ -1,751 +0,0 @@
-from abc import abstractmethod
-from distutils.util import strtobool
-from urllib.parse import urlparse
-import chardet
-import hashlib
-import json
-import logging
-import os
-import requests
-import sys
-import time
-import urllib.parse
-
-visualselector_xpath_selectors = 'div,span,form,table,tbody,tr,td,a,p,ul,li,h1,h2,h3,h4, header, footer, section, article, aside, details, main, nav, section, summary'
-
-
-class Non200ErrorCodeReceived(Exception):
-    def __init__(self, status_code, url, screenshot=None, xpath_data=None, page_html=None):
-        # Set this so we can use it in other parts of the app
-        self.status_code = status_code
-        self.url = url
-        self.screenshot = screenshot
-        self.xpath_data = xpath_data
-        self.page_text = None
-
-        if page_html:
-            from changedetectionio import html_tools
-            self.page_text = html_tools.html_to_text(page_html)
-        return
-
-
-class checksumFromPreviousCheckWasTheSame(Exception):
-    def __init__(self):
-        return
-
-
-class JSActionExceptions(Exception):
-    def __init__(self, status_code, url, screenshot, message=''):
-        self.status_code = status_code
-        self.url = url
-        self.screenshot = screenshot
-        self.message = message
-        return
-
-
-class BrowserStepsStepTimout(Exception):
-    def __init__(self, step_n):
-        self.step_n = step_n
-        return
-
-
-class PageUnloadable(Exception):
-    def __init__(self, status_code, url, message, screenshot=False):
-        # Set this so we can use it in other parts of the app
-        self.status_code = status_code
-        self.url = url
-        self.screenshot = screenshot
-        self.message = message
-        return
-
-
-class EmptyReply(Exception):
-    def __init__(self, status_code, url, screenshot=None):
-        # Set this so we can use it in other parts of the app
-        self.status_code = status_code
-        self.url = url
-        self.screenshot = screenshot
-        return
-
-
-class ScreenshotUnavailable(Exception):
-    def __init__(self, status_code, url, page_html=None):
-        # Set this so we can use it in other parts of the app
-        self.status_code = status_code
-        self.url = url
-        if page_html:
-            from html_tools import html_to_text
-            self.page_text = html_to_text(page_html)
-        return
-
-
-class ReplyWithContentButNoText(Exception):
-    def __init__(self, status_code, url, screenshot=None, has_filters=False, html_content=''):
-        # Set this so we can use it in other parts of the app
-        self.status_code = status_code
-        self.url = url
-        self.screenshot = screenshot
-        self.has_filters = has_filters
-        self.html_content = html_content
-        return
-
-
-class Fetcher():
-    browser_steps = None
-    browser_steps_screenshot_path = None
-    content = None
-    error = None
-    fetcher_description = "No description"
-    headers = {}
-    status_code = None
-    webdriver_js_execute_code = None
-    xpath_data = None
-    xpath_element_js = ""
-    instock_data = None
-    instock_data_js = ""
-
-    # Will be needed in the future by the VisualSelector, always get this where possible.
-    screenshot = False
-    system_http_proxy = os.getenv('HTTP_PROXY')
-    system_https_proxy = os.getenv('HTTPS_PROXY')
-
-    # Time ONTOP of the system defined env minimum time
-    render_extract_delay = 0
-
-    def __init__(self):
-        from pkg_resources import resource_string
-        # The code that scrapes elements and makes a list of elements/size/position to click on in the VisualSelector
-        self.xpath_element_js = resource_string(__name__, "res/xpath_element_scraper.js").decode('utf-8')
-        self.instock_data_js = resource_string(__name__, "res/stock-not-in-stock.js").decode('utf-8')
-
-    @abstractmethod
-    def get_error(self):
-        return self.error
-
-    @abstractmethod
-    def run(self,
-            url,
-            timeout,
-            request_headers,
-            request_body,
-            request_method,
-            ignore_status_codes=False,
-            current_include_filters=None,
-            is_binary=False):
-        # Should set self.error, self.status_code and self.content
-        pass
-
-    @abstractmethod
-    def quit(self):
-        return
-
-    @abstractmethod
-    def get_last_status_code(self):
-        return self.status_code
-
-    @abstractmethod
-    def screenshot_step(self, step_n):
-        return None
-
-    @abstractmethod
-    # Return true/false if this checker is ready to run, in the case it needs todo some special config check etc
-    def is_ready(self):
-        return True
-
-    def get_all_headers(self):
-        """
-        Get all headers but ensure all keys are lowercase
-        :return:
-        """
-        return {k.lower(): v for k, v in self.headers.items()}
-
-    def browser_steps_get_valid_steps(self):
-        if self.browser_steps is not None and len(self.browser_steps):
-            valid_steps = filter(
-                lambda s: (s['operation'] and len(s['operation']) and s['operation'] != 'Choose one' and s['operation'] != 'Goto site'),
-                self.browser_steps)
-
-            return valid_steps
-
-        return None
-
-    def iterate_browser_steps(self):
-        from changedetectionio.blueprint.browser_steps.browser_steps import steppable_browser_interface
-        from playwright._impl._api_types import TimeoutError
-        from jinja2 import Environment
-        jinja2_env = Environment(extensions=['jinja2_time.TimeExtension'])
-
-        step_n = 0
-
-        if self.browser_steps is not None and len(self.browser_steps):
-            interface = steppable_browser_interface()
-            interface.page = self.page
-            valid_steps = self.browser_steps_get_valid_steps()
-
-            for step in valid_steps:
-                step_n += 1
-                print(">> Iterating check - browser Step n {} - {}...".format(step_n, step['operation']))
-                self.screenshot_step("before-" + str(step_n))
-                self.save_step_html("before-" + str(step_n))
-                try:
-                    optional_value = step['optional_value']
-                    selector = step['selector']
-                    # Support for jinja2 template in step values, with date module added
-                    if '{%' in step['optional_value'] or '{{' in step['optional_value']:
-                        optional_value = str(jinja2_env.from_string(step['optional_value']).render())
-                    if '{%' in step['selector'] or '{{' in step['selector']:
-                        selector = str(jinja2_env.from_string(step['selector']).render())
-
-                    getattr(interface, "call_action")(action_name=step['operation'],
-                                                      selector=selector,
-                                                      optional_value=optional_value)
-                    self.screenshot_step(step_n)
-                    self.save_step_html(step_n)
-                except TimeoutError as e:
-                    print(str(e))
-                    # Stop processing here
-                    raise BrowserStepsStepTimout(step_n=step_n)
-
-    # It's always good to reset these
-    def delete_browser_steps_screenshots(self):
-        import glob
-        if self.browser_steps_screenshot_path is not None:
-            dest = os.path.join(self.browser_steps_screenshot_path, 'step_*.jpeg')
-            files = glob.glob(dest)
-            for f in files:
-                if os.path.isfile(f):
-                    os.unlink(f)
-
-
-#   Maybe for the future, each fetcher provides its own diff output, could be used for text, image
-#   the current one would return javascript output (as we use JS to generate the diff)
-#
-def available_fetchers():
-    # See the if statement at the bottom of this file for how we switch between playwright and webdriver
-    import inspect
-    p = []
-    for name, obj in inspect.getmembers(sys.modules[__name__], inspect.isclass):
-        if inspect.isclass(obj):
-            # @todo html_ is maybe better as fetcher_ or something
-            # In this case, make sure to edit the default one in store.py and fetch_site_status.py
-            if name.startswith('html_'):
-                t = tuple([name, obj.fetcher_description])
-                p.append(t)
-
-    return p
-
-
-class base_html_playwright(Fetcher):
-    fetcher_description = "Playwright {}/Javascript".format(
-        os.getenv("PLAYWRIGHT_BROWSER_TYPE", 'chromium').capitalize()
-    )
-    if os.getenv("PLAYWRIGHT_DRIVER_URL"):
-        fetcher_description += " via '{}'".format(os.getenv("PLAYWRIGHT_DRIVER_URL"))
-
-    browser_type = ''
-    command_executor = ''
-
-    # Configs for Proxy setup
-    # In the ENV vars, is prefixed with "playwright_proxy_", so it is for example "playwright_proxy_server"
-    playwright_proxy_settings_mappings = ['bypass', 'server', 'username', 'password']
-
-    proxy = None
-
-    def __init__(self, proxy_override=None):
-        super().__init__()
-        # .strip('"') is going to save someone a lot of time when they accidently wrap the env value
-        self.browser_type = os.getenv("PLAYWRIGHT_BROWSER_TYPE", 'chromium').strip('"')
-        self.command_executor = os.getenv(
-            "PLAYWRIGHT_DRIVER_URL",
-            'ws://playwright-chrome:3000'
-        ).strip('"')
-
-        # If any proxy settings are enabled, then we should setup the proxy object
-        proxy_args = {}
-        for k in self.playwright_proxy_settings_mappings:
-            v = os.getenv('playwright_proxy_' + k, False)
-            if v:
-                proxy_args[k] = v.strip('"')
-
-        if proxy_args:
-            self.proxy = proxy_args
-
-        # allow per-watch proxy selection override
-        if proxy_override:
-            self.proxy = {'server': proxy_override}
-
-        if self.proxy:
-            # Playwright needs separate username and password values
-            parsed = urlparse(self.proxy.get('server'))
-            if parsed.username:
-                self.proxy['username'] = parsed.username
-                self.proxy['password'] = parsed.password
-
-    def screenshot_step(self, step_n=''):
-        screenshot = self.page.screenshot(type='jpeg', full_page=True, quality=85)
-
-        if self.browser_steps_screenshot_path is not None:
-            destination = os.path.join(self.browser_steps_screenshot_path, 'step_{}.jpeg'.format(step_n))
-            logging.debug("Saving step screenshot to {}".format(destination))
-            with open(destination, 'wb') as f:
-                f.write(screenshot)
-
-    def save_step_html(self, step_n):
-        content = self.page.content()
-        destination = os.path.join(self.browser_steps_screenshot_path, 'step_{}.html'.format(step_n))
-        logging.debug("Saving step HTML to {}".format(destination))
-        with open(destination, 'w') as f:
-            f.write(content)
-
-    def run_fetch_browserless_puppeteer(self,
-            url,
-            timeout,
-            request_headers,
-            request_body,
-            request_method,
-            ignore_status_codes=False,
-            current_include_filters=None,
-            is_binary=False):
-
-        from pkg_resources import resource_string
-
-        extra_wait_ms = (int(os.getenv("WEBDRIVER_DELAY_BEFORE_CONTENT_READY", 5)) + self.render_extract_delay) * 1000
-
-        self.xpath_element_js = self.xpath_element_js.replace('%ELEMENTS%', visualselector_xpath_selectors)
-        code = resource_string(__name__, "res/puppeteer_fetch.js").decode('utf-8')
-        # In the future inject this is a proper JS package
-        code = code.replace('%xpath_scrape_code%', self.xpath_element_js)
-        code = code.replace('%instock_scrape_code%', self.instock_data_js)
-
-        from requests.exceptions import ConnectTimeout, ReadTimeout
-        wait_browserless_seconds = 240
-
-        browserless_function_url = os.getenv('BROWSERLESS_FUNCTION_URL')
-        from urllib.parse import urlparse
-        if not browserless_function_url:
-            # Convert/try to guess from PLAYWRIGHT_DRIVER_URL
-            o = urlparse(os.getenv('PLAYWRIGHT_DRIVER_URL'))
-            browserless_function_url = o._replace(scheme="http")._replace(path="function").geturl()
-
-
-        # Append proxy connect string
-        if self.proxy:
-            # Remove username/password if it exists in the URL or you will receive "ERR_NO_SUPPORTED_PROXIES" error
-            # Actual authentication handled by Puppeteer/node
-            o = urlparse(self.proxy.get('server'))
-            proxy_url = urllib.parse.quote(o._replace(netloc="{}:{}".format(o.hostname, o.port)).geturl())
-            browserless_function_url = f"{browserless_function_url}&--proxy-server={proxy_url}"
-
-        try:
-            amp = '&' if '?' in browserless_function_url else '?'
-            response = requests.request(
-                method="POST",
-                json={
-                    "code": code,
-                    "context": {
-                        # Very primitive disk cache - USE WITH EXTREME CAUTION
-                        # Run browserless container  with -e "FUNCTION_BUILT_INS=[\"fs\",\"crypto\"]"
-                        'disk_cache_dir': os.getenv("PUPPETEER_DISK_CACHE", False), # or path to disk cache ending in /, ie /tmp/cache/
-                        'execute_js': self.webdriver_js_execute_code,
-                        'extra_wait_ms': extra_wait_ms,
-                        'include_filters': current_include_filters,
-                        'req_headers': request_headers,
-                        'screenshot_quality': int(os.getenv("PLAYWRIGHT_SCREENSHOT_QUALITY", 72)),
-                        'url': url,
-                        'user_agent': {k.lower(): v for k, v in request_headers.items()}.get('user-agent', None),
-                        'proxy_username': self.proxy.get('username', '') if self.proxy else False,
-                        'proxy_password': self.proxy.get('password', '') if self.proxy and self.proxy.get('username') else False,
-                        'no_cache_list': [
-                            'twitter',
-                            '.pdf'
-                        ],
-                        # Could use https://github.com/easylist/easylist here, or install a plugin
-                        'block_url_list': [
-                            'adnxs.com',
-                            'analytics.twitter.com',
-                            'doubleclick.net',
-                            'google-analytics.com',
-                            'googletagmanager',
-                            'trustpilot.com'
-                        ]
-                    }
-                },
-                # @todo /function needs adding ws:// to http:// rebuild this
-                url=browserless_function_url+f"{amp}--disable-features=AudioServiceOutOfProcess&dumpio=true&--disable-remote-fonts",
-                timeout=wait_browserless_seconds)
-
-        except ReadTimeout:
-            raise PageUnloadable(url=url, status_code=None, message=f"No response from browserless in {wait_browserless_seconds}s")
-        except ConnectTimeout:
-            raise PageUnloadable(url=url, status_code=None, message=f"Timed out connecting to browserless, retrying..")
-        else:
-            # 200 Here means that the communication to browserless worked only, not the page state
-            if response.status_code == 200:
-                import base64
-
-                x = response.json()
-                if not x.get('screenshot'):
-                    # https://github.com/puppeteer/puppeteer/blob/v1.0.0/docs/troubleshooting.md#tips
-                    # https://github.com/puppeteer/puppeteer/issues/1834
-                    # https://github.com/puppeteer/puppeteer/issues/1834#issuecomment-381047051
-                    # Check your memory is shared and big enough
-                    raise ScreenshotUnavailable(url=url, status_code=None)
-
-                if not x.get('content', '').strip():
-                    raise EmptyReply(url=url, status_code=None)
-
-                if x.get('status_code', 200) != 200 and not ignore_status_codes:
-                    raise Non200ErrorCodeReceived(url=url, status_code=x.get('status_code', 200), page_html=x['content'])
-
-                self.content = x.get('content')
-                self.headers = x.get('headers')
-                self.instock_data = x.get('instock_data')
-                self.screenshot = base64.b64decode(x.get('screenshot'))
-                self.status_code = x.get('status_code')
-                self.xpath_data = x.get('xpath_data')
-
-            else:
-                # Some other error from browserless
-                raise PageUnloadable(url=url, status_code=None, message=response.content.decode('utf-8'))
-
-    def run(self,
-            url,
-            timeout,
-            request_headers,
-            request_body,
-            request_method,
-            ignore_status_codes=False,
-            current_include_filters=None,
-            is_binary=False):
-
-        # For now, USE_EXPERIMENTAL_PUPPETEER_FETCH is not supported by watches with BrowserSteps (for now!)
-        has_browser_steps = self.browser_steps and list(filter(
-                lambda s: (s['operation'] and len(s['operation']) and s['operation'] != 'Choose one' and s['operation'] != 'Goto site'),
-                self.browser_steps))
-
-        if not has_browser_steps and os.getenv('USE_EXPERIMENTAL_PUPPETEER_FETCH'):
-            if strtobool(os.getenv('USE_EXPERIMENTAL_PUPPETEER_FETCH')):
-                # Temporary backup solution until we rewrite the playwright code
-                return self.run_fetch_browserless_puppeteer(
-                    url,
-                    timeout,
-                    request_headers,
-                    request_body,
-                    request_method,
-                    ignore_status_codes,
-                    current_include_filters,
-                    is_binary)
-
-        from playwright.sync_api import sync_playwright
-        import playwright._impl._api_types
-
-        self.delete_browser_steps_screenshots()
-        response = None
-
-        with sync_playwright() as p:
-            browser_type = getattr(p, self.browser_type)
-
-            # Seemed to cause a connection Exception even tho I can see it connect
-            # self.browser = browser_type.connect(self.command_executor, timeout=timeout*1000)
-            # 60,000 connection timeout only
-            browser = browser_type.connect_over_cdp(self.command_executor, timeout=60000)
-
-            # SOCKS5 with authentication is not supported (yet)
-            # https://github.com/microsoft/playwright/issues/10567
-
-            # Set user agent to prevent Cloudflare from blocking the browser
-            # Use the default one configured in the App.py model that's passed from fetch_site_status.py
-            context = browser.new_context(
-                user_agent={k.lower(): v for k, v in request_headers.items()}.get('user-agent', None),
-                proxy=self.proxy,
-                # This is needed to enable JavaScript execution on GitHub and others
-                bypass_csp=True,
-                # Should be `allow` or `block` - sites like YouTube can transmit large amounts of data via Service Workers
-                service_workers=os.getenv('PLAYWRIGHT_SERVICE_WORKERS', 'allow'),
-                # Should never be needed
-                accept_downloads=False
-            )
-
-            self.page = context.new_page()
-            if len(request_headers):
-                context.set_extra_http_headers(request_headers)
-
-            # Listen for all console events and handle errors
-            self.page.on("console", lambda msg: print(f"Playwright console: Watch URL: {url} {msg.type}: {msg.text} {msg.args}"))
-
-            # Re-use as much code from browser steps as possible so its the same
-            from changedetectionio.blueprint.browser_steps.browser_steps import steppable_browser_interface
-            browsersteps_interface = steppable_browser_interface()
-            browsersteps_interface.page = self.page
-
-            response = browsersteps_interface.action_goto_url(value=url)
-            self.headers = response.all_headers()
-
-            if response is None:
-                context.close()
-                browser.close()
-                print("Content Fetcher > Response object was none")
-                raise EmptyReply(url=url, status_code=None)
-
-            try:
-                if self.webdriver_js_execute_code is not None and len(self.webdriver_js_execute_code):
-                    browsersteps_interface.action_execute_js(value=self.webdriver_js_execute_code, selector=None)
-            except playwright._impl._api_types.TimeoutError as e:
-                context.close()
-                browser.close()
-                # This can be ok, we will try to grab what we could retrieve
-                pass
-            except Exception as e:
-                print("Content Fetcher > Other exception when executing custom JS code", str(e))
-                context.close()
-                browser.close()
-                raise PageUnloadable(url=url, status_code=None, message=str(e))
-
-            extra_wait = int(os.getenv("WEBDRIVER_DELAY_BEFORE_CONTENT_READY", 5)) + self.render_extract_delay
-            self.page.wait_for_timeout(extra_wait * 1000)
-
-
-            self.status_code = response.status
-
-            if self.status_code != 200 and not ignore_status_codes:
-                raise Non200ErrorCodeReceived(url=url, status_code=self.status_code)
-
-            if len(self.page.content().strip()) == 0:
-                context.close()
-                browser.close()
-                print("Content Fetcher > Content was empty")
-                raise EmptyReply(url=url, status_code=response.status)
-
-            # Run Browser Steps here
-            if self.browser_steps_get_valid_steps():
-                self.iterate_browser_steps()
-                
-            self.page.wait_for_timeout(extra_wait * 1000)
-
-            # So we can find an element on the page where its selector was entered manually (maybe not xPath etc)
-            if current_include_filters is not None:
-                self.page.evaluate("var include_filters={}".format(json.dumps(current_include_filters)))
-            else:
-                self.page.evaluate("var include_filters=''")
-
-            self.xpath_data = self.page.evaluate(
-                "async () => {" + self.xpath_element_js.replace('%ELEMENTS%', visualselector_xpath_selectors) + "}")
-            self.instock_data = self.page.evaluate("async () => {" + self.instock_data_js + "}")
-
-            self.content = self.page.content()
-            # Bug 3 in Playwright screenshot handling
-            # Some bug where it gives the wrong screenshot size, but making a request with the clip set first seems to solve it
-            # JPEG is better here because the screenshots can be very very large
-
-            # Screenshots also travel via the ws:// (websocket) meaning that the binary data is base64 encoded
-            # which will significantly increase the IO size between the server and client, it's recommended to use the lowest
-            # acceptable screenshot quality here
-            try:
-                # The actual screenshot
-                self.screenshot = self.page.screenshot(type='jpeg', full_page=True,
-                                                       quality=int(os.getenv("PLAYWRIGHT_SCREENSHOT_QUALITY", 72)))
-            except Exception as e:
-                context.close()
-                browser.close()
-                raise ScreenshotUnavailable(url=url, status_code=response.status_code)
-
-            context.close()
-            browser.close()
-
-
-class base_html_webdriver(Fetcher):
-    if os.getenv("WEBDRIVER_URL"):
-        fetcher_description = "WebDriver Chrome/Javascript via '{}'".format(os.getenv("WEBDRIVER_URL"))
-    else:
-        fetcher_description = "WebDriver Chrome/Javascript"
-
-    command_executor = ''
-
-    # Configs for Proxy setup
-    # In the ENV vars, is prefixed with "webdriver_", so it is for example "webdriver_sslProxy"
-    selenium_proxy_settings_mappings = ['proxyType', 'ftpProxy', 'httpProxy', 'noProxy',
-                                        'proxyAutoconfigUrl', 'sslProxy', 'autodetect',
-                                        'socksProxy', 'socksVersion', 'socksUsername', 'socksPassword']
-    proxy = None
-
-    def __init__(self, proxy_override=None):
-        super().__init__()
-        from selenium.webdriver.common.proxy import Proxy as SeleniumProxy
-
-        # .strip('"') is going to save someone a lot of time when they accidently wrap the env value
-        self.command_executor = os.getenv("WEBDRIVER_URL", 'http://browser-chrome:4444/wd/hub').strip('"')
-
-        # If any proxy settings are enabled, then we should setup the proxy object
-        proxy_args = {}
-        for k in self.selenium_proxy_settings_mappings:
-            v = os.getenv('webdriver_' + k, False)
-            if v:
-                proxy_args[k] = v.strip('"')
-
-        # Map back standard HTTP_ and HTTPS_PROXY to webDriver httpProxy/sslProxy
-        if not proxy_args.get('webdriver_httpProxy') and self.system_http_proxy:
-            proxy_args['httpProxy'] = self.system_http_proxy
-        if not proxy_args.get('webdriver_sslProxy') and self.system_https_proxy:
-            proxy_args['httpsProxy'] = self.system_https_proxy
-
-        # Allows override the proxy on a per-request basis
-        if proxy_override is not None:
-            proxy_args['httpProxy'] = proxy_override
-
-        if proxy_args:
-            self.proxy = SeleniumProxy(raw=proxy_args)
-
-    def run(self,
-            url,
-            timeout,
-            request_headers,
-            request_body,
-            request_method,
-            ignore_status_codes=False,
-            current_include_filters=None,
-            is_binary=False):
-
-        from selenium import webdriver
-        from selenium.webdriver.chrome.options import Options as ChromeOptions
-        from selenium.common.exceptions import WebDriverException
-        # request_body, request_method unused for now, until some magic in the future happens.
-
-        options = ChromeOptions()
-        if self.proxy:
-            options.proxy = self.proxy
-
-        self.driver = webdriver.Remote(
-            command_executor=self.command_executor,
-            options=options)
-
-        try:
-            self.driver.get(url)
-        except WebDriverException as e:
-            # Be sure we close the session window
-            self.quit()
-            raise
-
-        self.driver.set_window_size(1280, 1024)
-        self.driver.implicitly_wait(int(os.getenv("WEBDRIVER_DELAY_BEFORE_CONTENT_READY", 5)))
-
-        if self.webdriver_js_execute_code is not None:
-            self.driver.execute_script(self.webdriver_js_execute_code)
-            # Selenium doesn't automatically wait for actions as good as Playwright, so wait again
-            self.driver.implicitly_wait(int(os.getenv("WEBDRIVER_DELAY_BEFORE_CONTENT_READY", 5)))
-
-        # @todo - how to check this? is it possible?
-        self.status_code = 200
-        # @todo somehow we should try to get this working for WebDriver
-        # raise EmptyReply(url=url, status_code=r.status_code)
-
-        # @todo - dom wait loaded?
-        time.sleep(int(os.getenv("WEBDRIVER_DELAY_BEFORE_CONTENT_READY", 5)) + self.render_extract_delay)
-        self.content = self.driver.page_source
-        self.headers = {}
-
-        self.screenshot = self.driver.get_screenshot_as_png()
-
-    # Does the connection to the webdriver work? run a test connection.
-    def is_ready(self):
-        from selenium import webdriver
-        from selenium.webdriver.chrome.options import Options as ChromeOptions
-
-        self.driver = webdriver.Remote(
-            command_executor=self.command_executor,
-            options=ChromeOptions())
-
-        # driver.quit() seems to cause better exceptions
-        self.quit()
-        return True
-
-    def quit(self):
-        if self.driver:
-            try:
-                self.driver.quit()
-            except Exception as e:
-                print("Content Fetcher > Exception in chrome shutdown/quit" + str(e))
-
-
-# "html_requests" is listed as the default fetcher in store.py!
-class html_requests(Fetcher):
-    fetcher_description = "Basic fast Plaintext/HTTP Client"
-
-    def __init__(self, proxy_override=None):
-        self.proxy_override = proxy_override
-
-    def run(self,
-            url,
-            timeout,
-            request_headers,
-            request_body,
-            request_method,
-            ignore_status_codes=False,
-            current_include_filters=None,
-            is_binary=False):
-
-        # Make requests use a more modern looking user-agent
-        if not {k.lower(): v for k, v in request_headers.items()}.get('user-agent', None):
-            request_headers['User-Agent'] = os.getenv("DEFAULT_SETTINGS_HEADERS_USERAGENT",
-                                                      'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/87.0.4280.66 Safari/537.36')
-
-        proxies = {}
-
-        # Allows override the proxy on a per-request basis
-
-        # https://requests.readthedocs.io/en/latest/user/advanced/#socks
-        # Should also work with `socks5://user:pass@host:port` type syntax.
-
-        if self.proxy_override:
-            proxies = {'http': self.proxy_override, 'https': self.proxy_override, 'ftp': self.proxy_override}
-        else:
-            if self.system_http_proxy:
-                proxies['http'] = self.system_http_proxy
-            if self.system_https_proxy:
-                proxies['https'] = self.system_https_proxy
-
-        r = requests.request(method=request_method,
-                             data=request_body,
-                             url=url,
-                             headers=request_headers,
-                             timeout=timeout,
-                             proxies=proxies,
-                             verify=False)
-
-        # If the response did not tell us what encoding format to expect, Then use chardet to override what `requests` thinks.
-        # For example - some sites don't tell us it's utf-8, but return utf-8 content
-        # This seems to not occur when using webdriver/selenium, it seems to detect the text encoding more reliably.
-        # https://github.com/psf/requests/issues/1604 good info about requests encoding detection
-        if not is_binary:
-            # Don't run this for PDF (and requests identified as binary) takes a _long_ time
-            if not r.headers.get('content-type') or not 'charset=' in r.headers.get('content-type'):
-                encoding = chardet.detect(r.content)['encoding']
-                if encoding:
-                    r.encoding = encoding
-
-        if not r.content or not len(r.content):
-            raise EmptyReply(url=url, status_code=r.status_code)
-
-        # @todo test this
-        # @todo maybe you really want to test zero-byte return pages?
-        if r.status_code != 200 and not ignore_status_codes:
-            # maybe check with content works?
-            raise Non200ErrorCodeReceived(url=url, status_code=r.status_code, page_html=r.text)
-
-        self.status_code = r.status_code
-        if is_binary:
-            # Binary files just return their checksum until we add something smarter
-            self.content = hashlib.md5(r.content).hexdigest()
-        else:
-            self.content = r.text
-
-        self.headers = r.headers
-        self.raw_content = r.content
-
-
-# Decide which is the 'real' HTML webdriver, this is more a system wide config
-# rather than site-specific.
-use_playwright_as_chrome_fetcher = os.getenv('PLAYWRIGHT_DRIVER_URL', False)
-if use_playwright_as_chrome_fetcher:
-    html_webdriver = base_html_playwright
-else:
-    html_webdriver = base_html_webdriver

changedetectionio/content_fetchers/__init__.py

@@ -0,0 +1,64 @@
+import sys
+from changedetectionio.strtobool import strtobool
+from loguru import logger
+from changedetectionio.content_fetchers.exceptions import BrowserStepsStepException
+import os
+
+# Visual Selector scraper - 'Button' is there because some sites have <button>OUT OF STOCK</button>.
+visualselector_xpath_selectors = 'div,span,form,table,tbody,tr,td,a,p,ul,li,h1,h2,h3,h4,header,footer,section,article,aside,details,main,nav,section,summary,button'
+
+SCREENSHOT_MAX_HEIGHT_DEFAULT = 20000
+SCREENSHOT_DEFAULT_QUALITY = 40
+
+# Maximum total height for the final image (When in stitch mode).
+# We limit this to 16000px due to the huge amount of RAM that was being used
+# Example: 16000 × 1400 × 3 = 67,200,000 bytes ≈ 64.1 MB (not including buffers in PIL etc)
+SCREENSHOT_MAX_TOTAL_HEIGHT = int(os.getenv("SCREENSHOT_MAX_HEIGHT", SCREENSHOT_MAX_HEIGHT_DEFAULT))
+
+# The size at which we will switch to stitching method, when below this (and
+# MAX_TOTAL_HEIGHT which can be set by a user) we will use the default
+# screenshot method.
+SCREENSHOT_SIZE_STITCH_THRESHOLD = 8000
+
+# available_fetchers() will scan this implementation looking for anything starting with html_
+# this information is used in the form selections
+from changedetectionio.content_fetchers.requests import fetcher as html_requests
+
+
+import importlib.resources
+XPATH_ELEMENT_JS = importlib.resources.files("changedetectionio.content_fetchers.res").joinpath('xpath_element_scraper.js').read_text(encoding='utf-8')
+INSTOCK_DATA_JS = importlib.resources.files("changedetectionio.content_fetchers.res").joinpath('stock-not-in-stock.js').read_text(encoding='utf-8')
+FAVICON_FETCHER_JS = importlib.resources.files("changedetectionio.content_fetchers.res").joinpath('favicon-fetcher.js').read_text(encoding='utf-8')
+
+
+def available_fetchers():
+    # See the if statement at the bottom of this file for how we switch between playwright and webdriver
+    import inspect
+    p = []
+    for name, obj in inspect.getmembers(sys.modules[__name__], inspect.isclass):
+        if inspect.isclass(obj):
+            # @todo html_ is maybe better as fetcher_ or something
+            # In this case, make sure to edit the default one in store.py and fetch_site_status.py
+            if name.startswith('html_'):
+                t = tuple([name, obj.fetcher_description])
+                p.append(t)
+
+    return p
+
+
+# Decide which is the 'real' HTML webdriver, this is more a system wide config
+# rather than site-specific.
+use_playwright_as_chrome_fetcher = os.getenv('PLAYWRIGHT_DRIVER_URL', False)
+if use_playwright_as_chrome_fetcher:
+    # @note - For now, browser steps always uses playwright
+    if not strtobool(os.getenv('FAST_PUPPETEER_CHROME_FETCHER', 'False')):
+        logger.debug('Using Playwright library as fetcher')
+        from .playwright import fetcher as html_webdriver
+    else:
+        logger.debug('Using direct Python Puppeteer library as fetcher')
+        from .puppeteer import fetcher as html_webdriver
+
+else:
+    logger.debug("Falling back to selenium as fetcher")
+    from .webdriver_selenium import fetcher as html_webdriver
+

changedetectionio/content_fetchers/base.py

@@ -0,0 +1,190 @@
+import os
+from abc import abstractmethod
+from loguru import logger
+
+from changedetectionio.content_fetchers import BrowserStepsStepException
+
+
+def manage_user_agent(headers, current_ua=''):
+    """
+    Basic setting of user-agent
+
+    NOTE!!!!!! The service that does the actual Chrome fetching should handle any anti-robot techniques
+    THERE ARE MANY WAYS THAT IT CAN BE DETECTED AS A ROBOT!!
+    This does not take care of
+    - Scraping of 'navigator' (platform, productSub, vendor, oscpu etc etc) browser object (navigator.appVersion) etc
+    - TCP/IP fingerprint JA3 etc
+    - Graphic rendering fingerprinting
+    - Your IP being obviously in a pool of bad actors
+    - Too many requests
+    - Scraping of SCH-UA browser replies (thanks google!!)
+    - Scraping of ServiceWorker, new window calls etc
+
+    See https://filipvitas.medium.com/how-to-set-user-agent-header-with-puppeteer-js-and-not-fail-28c7a02165da
+    Puppeteer requests https://github.com/dgtlmoon/pyppeteerstealth
+
+    :param page:
+    :param headers:
+    :return:
+    """
+    # Ask it what the user agent is, if its obviously ChromeHeadless, switch it to the default
+    ua_in_custom_headers = headers.get('User-Agent')
+    if ua_in_custom_headers:
+        return ua_in_custom_headers
+
+    if not ua_in_custom_headers and current_ua:
+        current_ua = current_ua.replace('HeadlessChrome', 'Chrome')
+        return current_ua
+
+    return None
+
+
+class Fetcher():
+    browser_connection_is_custom = None
+    browser_connection_url = None
+    browser_steps = None
+    browser_steps_screenshot_path = None
+    content = None
+    error = None
+    fetcher_description = "No description"
+    headers = {}
+    favicon_blob = None
+    instock_data = None
+    instock_data_js = ""
+    status_code = None
+    webdriver_js_execute_code = None
+    xpath_data = None
+    xpath_element_js = ""
+
+    # Will be needed in the future by the VisualSelector, always get this where possible.
+    screenshot = False
+    system_http_proxy = os.getenv('HTTP_PROXY')
+    system_https_proxy = os.getenv('HTTPS_PROXY')
+
+    # Time ONTOP of the system defined env minimum time
+    render_extract_delay = 0
+
+    def clear_content(self):
+        """
+        Explicitly clear all content from memory to free up heap space.
+        Call this after content has been saved to disk.
+        """
+        self.content = None
+        if hasattr(self, 'raw_content'):
+            self.raw_content = None
+        self.screenshot = None
+        self.xpath_data = None
+        # Keep headers and status_code as they're small
+
+    @abstractmethod
+    def get_error(self):
+        return self.error
+
+    @abstractmethod
+    async def run(self,
+                  fetch_favicon=True,
+                  current_include_filters=None,
+                  empty_pages_are_a_change=False,
+                  ignore_status_codes=False,
+                  is_binary=False,
+                  request_body=None,
+                  request_headers=None,
+                  request_method=None,
+                  timeout=None,
+                  url=None,
+                  ):
+        # Should set self.error, self.status_code and self.content
+        pass
+
+    @abstractmethod
+    def quit(self, watch=None):
+        return
+
+    @abstractmethod
+    def get_last_status_code(self):
+        return self.status_code
+
+    @abstractmethod
+    def screenshot_step(self, step_n):
+        if self.browser_steps_screenshot_path and not os.path.isdir(self.browser_steps_screenshot_path):
+            logger.debug(f"> Creating data dir {self.browser_steps_screenshot_path}")
+            os.mkdir(self.browser_steps_screenshot_path)
+        return None
+
+    @abstractmethod
+    # Return true/false if this checker is ready to run, in the case it needs todo some special config check etc
+    def is_ready(self):
+        return True
+
+    def get_all_headers(self):
+        """
+        Get all headers but ensure all keys are lowercase
+        :return:
+        """
+        return {k.lower(): v for k, v in self.headers.items()}
+
+    def browser_steps_get_valid_steps(self):
+        if self.browser_steps is not None and len(self.browser_steps):
+            valid_steps = list(filter(
+                lambda s: (s['operation'] and len(s['operation']) and s['operation'] != 'Choose one'),
+                self.browser_steps))
+
+            # Just incase they selected Goto site by accident with older JS
+            if valid_steps and valid_steps[0]['operation'] == 'Goto site':
+                del(valid_steps[0])
+
+            return valid_steps
+
+        return None
+
+    async def iterate_browser_steps(self, start_url=None):
+        from changedetectionio.blueprint.browser_steps.browser_steps import steppable_browser_interface
+        from playwright._impl._errors import TimeoutError, Error
+        from changedetectionio.jinja2_custom import render as jinja_render
+        step_n = 0
+
+        if self.browser_steps is not None and len(self.browser_steps):
+            interface = steppable_browser_interface(start_url=start_url)
+            interface.page = self.page
+            valid_steps = self.browser_steps_get_valid_steps()
+
+            for step in valid_steps:
+                step_n += 1
+                logger.debug(f">> Iterating check - browser Step n {step_n} - {step['operation']}...")
+                await self.screenshot_step("before-" + str(step_n))
+                await self.save_step_html("before-" + str(step_n))
+
+                try:
+                    optional_value = step['optional_value']
+                    selector = step['selector']
+                    # Support for jinja2 template in step values, with date module added
+                    if '{%' in step['optional_value'] or '{{' in step['optional_value']:
+                        optional_value = jinja_render(template_str=step['optional_value'])
+                    if '{%' in step['selector'] or '{{' in step['selector']:
+                        selector = jinja_render(template_str=step['selector'])
+
+                    await getattr(interface, "call_action")(action_name=step['operation'],
+                                                      selector=selector,
+                                                      optional_value=optional_value)
+                    await self.screenshot_step(step_n)
+                    await self.save_step_html(step_n)
+                except (Error, TimeoutError) as e:
+                    logger.debug(str(e))
+                    # Stop processing here
+                    raise BrowserStepsStepException(step_n=step_n, original_e=e)
+
+    # It's always good to reset these
+    def delete_browser_steps_screenshots(self):
+        import glob
+        if self.browser_steps_screenshot_path is not None:
+            dest = os.path.join(self.browser_steps_screenshot_path, 'step_*.jpeg')
+            files = glob.glob(dest)
+            for f in files:
+                if os.path.isfile(f):
+                    os.unlink(f)
+
+    def save_step_html(self, step_n):
+        if self.browser_steps_screenshot_path and not os.path.isdir(self.browser_steps_screenshot_path):
+            logger.debug(f"> Creating data dir {self.browser_steps_screenshot_path}")
+            os.mkdir(self.browser_steps_screenshot_path)
+        pass

changedetectionio/content_fetchers/exceptions/__init__.py

@@ -0,0 +1,97 @@
+from loguru import logger
+
+class Non200ErrorCodeReceived(Exception):
+    def __init__(self, status_code, url, screenshot=None, xpath_data=None, page_html=None):
+        # Set this so we can use it in other parts of the app
+        self.status_code = status_code
+        self.url = url
+        self.screenshot = screenshot
+        self.xpath_data = xpath_data
+        self.page_text = None
+
+        if page_html:
+            from changedetectionio import html_tools
+            self.page_text = html_tools.html_to_text(page_html)
+        return
+
+
+class checksumFromPreviousCheckWasTheSame(Exception):
+    def __init__(self):
+        return
+
+
+class JSActionExceptions(Exception):
+    def __init__(self, status_code, url, screenshot, message=''):
+        self.status_code = status_code
+        self.url = url
+        self.screenshot = screenshot
+        self.message = message
+        return
+
+class BrowserConnectError(Exception):
+    msg = ''
+    def __init__(self, msg):
+        self.msg = msg
+        logger.error(f"Browser connection error {msg}")
+        return
+
+class BrowserFetchTimedOut(Exception):
+    msg = ''
+    def __init__(self, msg):
+        self.msg = msg
+        logger.error(f"Browser processing took too long - {msg}")
+        return
+
+class BrowserStepsStepException(Exception):
+    def __init__(self, step_n, original_e):
+        self.step_n = step_n
+        self.original_e = original_e
+        logger.debug(f"Browser Steps exception at step {self.step_n} {str(original_e)}")
+        return
+
+
+# @todo - make base Exception class that announces via logger()
+class PageUnloadable(Exception):
+    def __init__(self, status_code=None, url='', message='', screenshot=False):
+        # Set this so we can use it in other parts of the app
+        self.status_code = status_code
+        self.url = url
+        self.screenshot = screenshot
+        self.message = message
+        return
+
+class BrowserStepsInUnsupportedFetcher(Exception):
+    def __init__(self, url):
+        self.url = url
+        return
+
+class EmptyReply(Exception):
+    def __init__(self, status_code, url, screenshot=None):
+        # Set this so we can use it in other parts of the app
+        self.status_code = status_code
+        self.url = url
+        self.screenshot = screenshot
+        return
+
+
+class ScreenshotUnavailable(Exception):
+    def __init__(self, status_code, url, page_html=None):
+        # Set this so we can use it in other parts of the app
+        self.status_code = status_code
+        self.url = url
+        if page_html:
+            from changedetectionio.html_tools import html_to_text
+            self.page_text = html_to_text(page_html)
+        return
+
+
+class ReplyWithContentButNoText(Exception):
+    def __init__(self, status_code, url, screenshot=None, has_filters=False, html_content='', xpath_data=None):
+        # Set this so we can use it in other parts of the app
+        self.status_code = status_code
+        self.url = url
+        self.screenshot = screenshot
+        self.has_filters = has_filters
+        self.html_content = html_content
+        self.xpath_data = xpath_data
+        return

changedetectionio/content_fetchers/playwright.py

@@ -0,0 +1,333 @@
+import json
+import os
+from urllib.parse import urlparse
+
+from loguru import logger
+
+from changedetectionio.content_fetchers import SCREENSHOT_MAX_HEIGHT_DEFAULT, visualselector_xpath_selectors, \
+    SCREENSHOT_SIZE_STITCH_THRESHOLD, SCREENSHOT_MAX_TOTAL_HEIGHT, XPATH_ELEMENT_JS, INSTOCK_DATA_JS, FAVICON_FETCHER_JS
+from changedetectionio.content_fetchers.base import Fetcher, manage_user_agent
+from changedetectionio.content_fetchers.exceptions import PageUnloadable, Non200ErrorCodeReceived, EmptyReply, ScreenshotUnavailable
+
+async def capture_full_page_async(page):
+    import os
+    import time
+    from multiprocessing import Process, Pipe
+
+    start = time.time()
+
+    page_height = await page.evaluate("document.documentElement.scrollHeight")
+    page_width = await page.evaluate("document.documentElement.scrollWidth")
+    original_viewport = page.viewport_size
+
+    logger.debug(f"Playwright viewport size {page.viewport_size} page height {page_height} page width {page_width}")
+
+    # Use an approach similar to puppeteer: set a larger viewport and take screenshots in chunks
+    step_size = SCREENSHOT_SIZE_STITCH_THRESHOLD # Size that won't cause GPU to overflow
+    screenshot_chunks = []
+    y = 0
+
+    if page_height > page.viewport_size['height']:
+        if page_height < step_size:
+            step_size = page_height # Incase page is bigger than default viewport but smaller than proposed step size
+        logger.debug(f"Setting bigger viewport to step through large page width W{page.viewport_size['width']}xH{step_size} because page_height > viewport_size")
+        # Set viewport to a larger size to capture more content at once
+        await page.set_viewport_size({'width': page.viewport_size['width'], 'height': step_size})
+
+    # Capture screenshots in chunks up to the max total height
+    while y < min(page_height, SCREENSHOT_MAX_TOTAL_HEIGHT):
+        await page.request_gc()
+        await page.evaluate(f"window.scrollTo(0, {y})")
+        await page.request_gc()
+        screenshot_chunks.append(await page.screenshot(
+            type="jpeg",
+            full_page=False,
+            quality=int(os.getenv("SCREENSHOT_QUALITY", 72))
+        ))
+        y += step_size
+        await page.request_gc()
+
+    # Restore original viewport size
+    await page.set_viewport_size({'width': original_viewport['width'], 'height': original_viewport['height']})
+
+    # If we have multiple chunks, stitch them together
+    if len(screenshot_chunks) > 1:
+        from changedetectionio.content_fetchers.screenshot_handler import stitch_images_worker
+        logger.debug(f"Screenshot stitching {len(screenshot_chunks)} chunks together")
+        parent_conn, child_conn = Pipe()
+        p = Process(target=stitch_images_worker, args=(child_conn, screenshot_chunks, page_height, SCREENSHOT_MAX_TOTAL_HEIGHT))
+        p.start()
+        screenshot = parent_conn.recv_bytes()
+        p.join()
+        logger.debug(
+            f"Screenshot (chunked/stitched) - Page height: {page_height} Capture height: {SCREENSHOT_MAX_TOTAL_HEIGHT} - Stitched together in {time.time() - start:.2f}s")
+        # Explicit cleanup
+        del screenshot_chunks
+        del p
+        del parent_conn, child_conn
+        screenshot_chunks = None
+        return screenshot
+
+    logger.debug(
+        f"Screenshot Page height: {page_height} Capture height: {SCREENSHOT_MAX_TOTAL_HEIGHT} - Stitched together in {time.time() - start:.2f}s")
+
+    return screenshot_chunks[0]
+
+class fetcher(Fetcher):
+    fetcher_description = "Playwright {}/Javascript".format(
+        os.getenv("PLAYWRIGHT_BROWSER_TYPE", 'chromium').capitalize()
+    )
+    if os.getenv("PLAYWRIGHT_DRIVER_URL"):
+        fetcher_description += " via '{}'".format(os.getenv("PLAYWRIGHT_DRIVER_URL"))
+
+    browser_type = ''
+    command_executor = ''
+
+    # Configs for Proxy setup
+    # In the ENV vars, is prefixed with "playwright_proxy_", so it is for example "playwright_proxy_server"
+    playwright_proxy_settings_mappings = ['bypass', 'server', 'username', 'password']
+
+    proxy = None
+
+    def __init__(self, proxy_override=None, custom_browser_connection_url=None):
+        super().__init__()
+
+        self.browser_type = os.getenv("PLAYWRIGHT_BROWSER_TYPE", 'chromium').strip('"')
+
+        if custom_browser_connection_url:
+            self.browser_connection_is_custom = True
+            self.browser_connection_url = custom_browser_connection_url
+        else:
+            # Fallback to fetching from system
+            # .strip('"') is going to save someone a lot of time when they accidently wrap the env value
+            self.browser_connection_url = os.getenv("PLAYWRIGHT_DRIVER_URL", 'ws://playwright-chrome:3000').strip('"')
+
+        # If any proxy settings are enabled, then we should setup the proxy object
+        proxy_args = {}
+        for k in self.playwright_proxy_settings_mappings:
+            v = os.getenv('playwright_proxy_' + k, False)
+            if v:
+                proxy_args[k] = v.strip('"')
+
+        if proxy_args:
+            self.proxy = proxy_args
+
+        # allow per-watch proxy selection override
+        if proxy_override:
+            self.proxy = {'server': proxy_override}
+
+        if self.proxy:
+            # Playwright needs separate username and password values
+            parsed = urlparse(self.proxy.get('server'))
+            if parsed.username:
+                self.proxy['username'] = parsed.username
+                self.proxy['password'] = parsed.password
+
+    async def screenshot_step(self, step_n=''):
+        super().screenshot_step(step_n=step_n)
+        screenshot = await capture_full_page_async(page=self.page)
+
+
+        if self.browser_steps_screenshot_path is not None:
+            destination = os.path.join(self.browser_steps_screenshot_path, 'step_{}.jpeg'.format(step_n))
+            logger.debug(f"Saving step screenshot to {destination}")
+            with open(destination, 'wb') as f:
+                f.write(screenshot)
+
+    async def save_step_html(self, step_n):
+        super().save_step_html(step_n=step_n)
+        content = await self.page.content()
+        destination = os.path.join(self.browser_steps_screenshot_path, 'step_{}.html'.format(step_n))
+        logger.debug(f"Saving step HTML to {destination}")
+        with open(destination, 'w') as f:
+            f.write(content)
+
+    async def run(self,
+                  fetch_favicon=True,
+                  current_include_filters=None,
+                  empty_pages_are_a_change=False,
+                  ignore_status_codes=False,
+                  is_binary=False,
+                  request_body=None,
+                  request_headers=None,
+                  request_method=None,
+                  timeout=None,
+                  url=None,
+                  ):
+
+        from playwright.async_api import async_playwright
+        import playwright._impl._errors
+        import time
+        self.delete_browser_steps_screenshots()
+        response = None
+
+        async with async_playwright() as p:
+            browser_type = getattr(p, self.browser_type)
+
+            # Seemed to cause a connection Exception even tho I can see it connect
+            # self.browser = browser_type.connect(self.command_executor, timeout=timeout*1000)
+            # 60,000 connection timeout only
+            browser = await browser_type.connect_over_cdp(self.browser_connection_url, timeout=60000)
+
+            # SOCKS5 with authentication is not supported (yet)
+            # https://github.com/microsoft/playwright/issues/10567
+
+            # Set user agent to prevent Cloudflare from blocking the browser
+            # Use the default one configured in the App.py model that's passed from fetch_site_status.py
+            context = await browser.new_context(
+                accept_downloads=False,  # Should never be needed
+                bypass_csp=True,  # This is needed to enable JavaScript execution on GitHub and others
+                extra_http_headers=request_headers,
+                ignore_https_errors=True,
+                proxy=self.proxy,
+                service_workers=os.getenv('PLAYWRIGHT_SERVICE_WORKERS', 'allow'), # Should be `allow` or `block` - sites like YouTube can transmit large amounts of data via Service Workers
+                user_agent=manage_user_agent(headers=request_headers),
+            )
+
+            self.page = await context.new_page()
+
+            # Listen for all console events and handle errors
+            self.page.on("console", lambda msg: logger.debug(f"Playwright console: Watch URL: {url} {msg.type}: {msg.text} {msg.args}"))
+
+            # Re-use as much code from browser steps as possible so its the same
+            from changedetectionio.blueprint.browser_steps.browser_steps import steppable_browser_interface
+            browsersteps_interface = steppable_browser_interface(start_url=url)
+            browsersteps_interface.page = self.page
+
+            response = await browsersteps_interface.action_goto_url(value=url)
+
+            if response is None:
+                await context.close()
+                await browser.close()
+                logger.debug("Content Fetcher > Response object from the browser communication was none")
+                raise EmptyReply(url=url, status_code=None)
+
+            # In async_playwright, all_headers() returns a coroutine
+            try:
+                self.headers = await response.all_headers()
+            except TypeError:
+                # Fallback for sync version
+                self.headers = response.all_headers()
+
+            try:
+                if self.webdriver_js_execute_code is not None and len(self.webdriver_js_execute_code):
+                    await browsersteps_interface.action_execute_js(value=self.webdriver_js_execute_code, selector=None)
+            except playwright._impl._errors.TimeoutError as e:
+                await context.close()
+                await browser.close()
+                # This can be ok, we will try to grab what we could retrieve
+                pass
+            except Exception as e:
+                logger.debug(f"Content Fetcher > Other exception when executing custom JS code {str(e)}")
+                await context.close()
+                await browser.close()
+                raise PageUnloadable(url=url, status_code=None, message=str(e))
+
+            extra_wait = int(os.getenv("WEBDRIVER_DELAY_BEFORE_CONTENT_READY", 5)) + self.render_extract_delay
+            await self.page.wait_for_timeout(extra_wait * 1000)
+
+            try:
+                self.status_code = response.status
+            except Exception as e:
+                # https://github.com/dgtlmoon/changedetection.io/discussions/2122#discussioncomment-8241962
+                logger.critical(f"Response from the browser/Playwright did not have a status_code! Response follows.")
+                logger.critical(response)
+                await context.close()
+                await browser.close()
+                raise PageUnloadable(url=url, status_code=None, message=str(e))
+
+            if fetch_favicon:
+                try:
+                    self.favicon_blob = await self.page.evaluate(FAVICON_FETCHER_JS)
+                    await self.page.request_gc()
+                except Exception as e:
+                    logger.error(f"Error fetching FavIcon info {str(e)}, continuing.")
+
+            if self.status_code != 200 and not ignore_status_codes:
+                screenshot = await capture_full_page_async(self.page)
+                raise Non200ErrorCodeReceived(url=url, status_code=self.status_code, screenshot=screenshot)
+
+            if not empty_pages_are_a_change and len((await self.page.content()).strip()) == 0:
+                logger.debug("Content Fetcher > Content was empty, empty_pages_are_a_change = False")
+                await context.close()
+                await browser.close()
+                raise EmptyReply(url=url, status_code=response.status)
+
+            # Run Browser Steps here
+            if self.browser_steps_get_valid_steps():
+                await self.iterate_browser_steps(start_url=url)
+
+            await self.page.wait_for_timeout(extra_wait * 1000)
+
+            now = time.time()
+            # So we can find an element on the page where its selector was entered manually (maybe not xPath etc)
+            if current_include_filters is not None:
+                await self.page.evaluate("var include_filters={}".format(json.dumps(current_include_filters)))
+            else:
+                await self.page.evaluate("var include_filters=''")
+            await self.page.request_gc()
+
+            # request_gc before and after evaluate to free up memory
+            # @todo browsersteps etc
+            MAX_TOTAL_HEIGHT = int(os.getenv("SCREENSHOT_MAX_HEIGHT", SCREENSHOT_MAX_HEIGHT_DEFAULT))
+            self.xpath_data = await self.page.evaluate(XPATH_ELEMENT_JS, {
+                "visualselector_xpath_selectors": visualselector_xpath_selectors,
+                "max_height": MAX_TOTAL_HEIGHT
+            })
+            await self.page.request_gc()
+
+            self.instock_data = await self.page.evaluate(INSTOCK_DATA_JS)
+            await self.page.request_gc()
+
+            self.content = await self.page.content()
+            await self.page.request_gc()
+            logger.debug(f"Scrape xPath element data in browser done in {time.time() - now:.2f}s")
+
+
+            # Bug 3 in Playwright screenshot handling
+            # Some bug where it gives the wrong screenshot size, but making a request with the clip set first seems to solve it
+            # JPEG is better here because the screenshots can be very very large
+
+            # Screenshots also travel via the ws:// (websocket) meaning that the binary data is base64 encoded
+            # which will significantly increase the IO size between the server and client, it's recommended to use the lowest
+            # acceptable screenshot quality here
+            try:
+                # The actual screenshot - this always base64 and needs decoding! horrible! huge CPU usage
+                self.screenshot = await capture_full_page_async(page=self.page)
+
+            except Exception as e:
+                # It's likely the screenshot was too long/big and something crashed
+                raise ScreenshotUnavailable(url=url, status_code=self.status_code)
+            finally:
+                # Request garbage collection one more time before closing
+                try:
+                    await self.page.request_gc()
+                except:
+                    pass
+                
+                # Clean up resources properly
+                try:
+                    await self.page.request_gc()
+                except:
+                    pass
+
+                try:
+                    await self.page.close()
+                except:
+                    pass
+                self.page = None
+
+                try:
+                    await context.close()
+                except:
+                    pass
+                context = None
+
+                try:
+                    await browser.close()
+                except:
+                    pass
+                browser = None
+
+
+

changedetectionio/content_fetchers/puppeteer.py

@@ -0,0 +1,386 @@
+import asyncio
+import json
+import os
+import websockets.exceptions
+from urllib.parse import urlparse
+
+from loguru import logger
+
+from changedetectionio.content_fetchers import SCREENSHOT_MAX_HEIGHT_DEFAULT, visualselector_xpath_selectors, \
+    SCREENSHOT_SIZE_STITCH_THRESHOLD, SCREENSHOT_DEFAULT_QUALITY, XPATH_ELEMENT_JS, INSTOCK_DATA_JS, \
+    SCREENSHOT_MAX_TOTAL_HEIGHT, FAVICON_FETCHER_JS
+from changedetectionio.content_fetchers.base import Fetcher, manage_user_agent
+from changedetectionio.content_fetchers.exceptions import PageUnloadable, Non200ErrorCodeReceived, EmptyReply, BrowserFetchTimedOut, \
+    BrowserConnectError
+
+
+# Bug 3 in Playwright screenshot handling
+# Some bug where it gives the wrong screenshot size, but making a request with the clip set first seems to solve it
+
+# Screenshots also travel via the ws:// (websocket) meaning that the binary data is base64 encoded
+# which will significantly increase the IO size between the server and client, it's recommended to use the lowest
+# acceptable screenshot quality here
+async def capture_full_page(page):
+    import os
+    import time
+    from multiprocessing import Process, Pipe
+
+    start = time.time()
+
+    page_height = await page.evaluate("document.documentElement.scrollHeight")
+    page_width = await page.evaluate("document.documentElement.scrollWidth")
+    original_viewport = page.viewport
+
+    logger.debug(f"Puppeteer viewport size {page.viewport} page height {page_height} page width {page_width}")
+
+    # Bug 3 in Playwright screenshot handling
+    # Some bug where it gives the wrong screenshot size, but making a request with the clip set first seems to solve it
+    # JPEG is better here because the screenshots can be very very large
+
+    # Screenshots also travel via the ws:// (websocket) meaning that the binary data is base64 encoded
+    # which will significantly increase the IO size between the server and client, it's recommended to use the lowest
+    # acceptable screenshot quality here
+
+
+    step_size = SCREENSHOT_SIZE_STITCH_THRESHOLD # Something that will not cause the GPU to overflow when taking the screenshot
+    screenshot_chunks = []
+    y = 0
+    if page_height > page.viewport['height']:
+        if page_height < step_size:
+            step_size = page_height # Incase page is bigger than default viewport but smaller than proposed step size
+        await page.setViewport({'width': page.viewport['width'], 'height': step_size})
+
+    while y < min(page_height, SCREENSHOT_MAX_TOTAL_HEIGHT):
+        # better than scrollTo incase they override it in the page
+        await page.evaluate(
+            """(y) => {
+                document.documentElement.scrollTop = y;
+                document.body.scrollTop = y;
+            }""",
+            y
+        )
+
+        screenshot_chunks.append(await page.screenshot(type_='jpeg',
+                                                       fullPage=False,
+                                                       quality=int(os.getenv("SCREENSHOT_QUALITY", 72))))
+        y += step_size
+
+    await page.setViewport({'width': original_viewport['width'], 'height': original_viewport['height']})
+
+    if len(screenshot_chunks) > 1:
+        from changedetectionio.content_fetchers.screenshot_handler import stitch_images_worker
+        logger.debug(f"Screenshot stitching {len(screenshot_chunks)} chunks together")
+        parent_conn, child_conn = Pipe()
+        p = Process(target=stitch_images_worker, args=(child_conn, screenshot_chunks, page_height, SCREENSHOT_MAX_TOTAL_HEIGHT))
+        p.start()
+        screenshot = parent_conn.recv_bytes()
+        p.join()
+        logger.debug(
+            f"Screenshot (chunked/stitched) - Page height: {page_height} Capture height: {SCREENSHOT_MAX_TOTAL_HEIGHT} - Stitched together in {time.time() - start:.2f}s")
+
+        screenshot_chunks = None
+        return screenshot
+
+    logger.debug(
+        f"Screenshot Page height: {page_height} Capture height: {SCREENSHOT_MAX_TOTAL_HEIGHT} - Stitched together in {time.time() - start:.2f}s")
+    return screenshot_chunks[0]
+
+
+class fetcher(Fetcher):
+    fetcher_description = "Puppeteer/direct {}/Javascript".format(
+        os.getenv("PLAYWRIGHT_BROWSER_TYPE", 'chromium').capitalize()
+    )
+    if os.getenv("PLAYWRIGHT_DRIVER_URL"):
+        fetcher_description += " via '{}'".format(os.getenv("PLAYWRIGHT_DRIVER_URL"))
+
+    browser_type = ''
+    command_executor = ''
+
+    proxy = None
+
+    def __init__(self, proxy_override=None, custom_browser_connection_url=None):
+        super().__init__()
+
+        if custom_browser_connection_url:
+            self.browser_connection_is_custom = True
+            self.browser_connection_url = custom_browser_connection_url
+        else:
+            # Fallback to fetching from system
+            # .strip('"') is going to save someone a lot of time when they accidently wrap the env value
+            self.browser_connection_url = os.getenv("PLAYWRIGHT_DRIVER_URL", 'ws://playwright-chrome:3000').strip('"')
+
+        # allow per-watch proxy selection override
+        # @todo check global too?
+        if proxy_override:
+            # Playwright needs separate username and password values
+            parsed = urlparse(proxy_override)
+            if parsed:
+                self.proxy = {'username': parsed.username, 'password': parsed.password}
+                # Add the proxy server chrome start option, the username and password never gets added here
+                # (It always goes in via await self.page.authenticate(self.proxy))
+
+                # @todo filter some injection attack?
+                # check scheme when no scheme
+                proxy_url = parsed.scheme + "://" if parsed.scheme else 'http://'
+                r = "?" if not '?' in self.browser_connection_url else '&'
+                port = ":"+str(parsed.port) if parsed.port else ''
+                q = "?"+parsed.query if parsed.query else ''
+                proxy_url += f"{parsed.hostname}{port}{parsed.path}{q}"
+                self.browser_connection_url += f"{r}--proxy-server={proxy_url}"
+
+    # def screenshot_step(self, step_n=''):
+    #     screenshot = self.page.screenshot(type='jpeg', full_page=True, quality=85)
+    #
+    #     if self.browser_steps_screenshot_path is not None:
+    #         destination = os.path.join(self.browser_steps_screenshot_path, 'step_{}.jpeg'.format(step_n))
+    #         logger.debug(f"Saving step screenshot to {destination}")
+    #         with open(destination, 'wb') as f:
+    #             f.write(screenshot)
+    #
+    # def save_step_html(self, step_n):
+    #     content = self.page.content()
+    #     destination = os.path.join(self.browser_steps_screenshot_path, 'step_{}.html'.format(step_n))
+    #     logger.debug(f"Saving step HTML to {destination}")
+    #     with open(destination, 'w') as f:
+    #         f.write(content)
+
+    async def fetch_page(self,
+                         current_include_filters,
+                         empty_pages_are_a_change,
+                         fetch_favicon,
+                         ignore_status_codes,
+                         is_binary,
+                         request_body,
+                         request_headers,
+                         request_method,
+                         timeout,
+                         url,
+                         ):
+        import re
+        self.delete_browser_steps_screenshots()
+
+        n = int(os.getenv("WEBDRIVER_DELAY_BEFORE_CONTENT_READY", 5)) + self.render_extract_delay
+        extra_wait = min(n, 15)
+
+        logger.debug(f"Extra wait set to {extra_wait}s, requested was {n}s.")
+
+        from pyppeteer import Pyppeteer
+        pyppeteer_instance = Pyppeteer()
+
+        # Connect directly using the specified browser_ws_endpoint
+        # @todo timeout
+        try:
+            browser = await pyppeteer_instance.connect(browserWSEndpoint=self.browser_connection_url,
+                                                       ignoreHTTPSErrors=True
+                                                       )
+        except websockets.exceptions.InvalidStatusCode as e:
+            raise BrowserConnectError(msg=f"Error while trying to connect the browser, Code {e.status_code} (check your access, whitelist IP, password etc)")
+        except websockets.exceptions.InvalidURI:
+            raise BrowserConnectError(msg=f"Error connecting to the browser, check your browser connection address (should be ws:// or wss://")
+        except Exception as e:
+            raise BrowserConnectError(msg=f"Error connecting to the browser - Exception '{str(e)}'")
+
+        # more reliable is to just request a new page
+        self.page = await browser.newPage()
+        
+        # Add console handler to capture console.log from favicon fetcher
+        #self.page.on('console', lambda msg: logger.debug(f"Browser console [{msg.type}]: {msg.text}"))
+
+        if '--window-size' in self.browser_connection_url:
+            # Be sure the viewport is always the window-size, this is often not the same thing
+            match = re.search(r'--window-size=(\d+),(\d+)', self.browser_connection_url)
+            if match:
+                logger.debug(f"Setting viewport to same as --window-size in browser connection URL {int(match.group(1))},{int(match.group(2))}")
+                await self.page.setViewport({
+                    "width": int(match.group(1)),
+                    "height": int(match.group(2))
+                })
+                logger.debug(f"Puppeteer viewport size {self.page.viewport}")
+
+        try:
+            from pyppeteerstealth import inject_evasions_into_page
+        except ImportError:
+            logger.debug("pyppeteerstealth module not available, skipping")
+            pass
+        else:
+            # I tried hooking events via self.page.on(Events.Page.DOMContentLoaded, inject_evasions_requiring_obj_to_page)
+            # But I could never get it to fire reliably, so we just inject it straight after
+            await inject_evasions_into_page(self.page)
+
+        # This user agent is similar to what was used when tweaking the evasions in inject_evasions_into_page(..)
+        user_agent = None
+        if request_headers and request_headers.get('User-Agent'):
+            # Request_headers should now be CaaseInsensitiveDict
+            # Remove it so it's not sent again with headers after
+            user_agent = request_headers.pop('User-Agent').strip()
+            await self.page.setUserAgent(user_agent)
+
+        if not user_agent:
+            # Attempt to strip 'HeadlessChrome' etc
+            await self.page.setUserAgent(manage_user_agent(headers=request_headers, current_ua=await self.page.evaluate('navigator.userAgent')))
+
+        await self.page.setBypassCSP(True)
+        if request_headers:
+            await self.page.setExtraHTTPHeaders(request_headers)
+
+        # SOCKS5 with authentication is not supported (yet)
+        # https://github.com/microsoft/playwright/issues/10567
+        self.page.setDefaultNavigationTimeout(0)
+        await self.page.setCacheEnabled(True)
+        if self.proxy and self.proxy.get('username'):
+            # Setting Proxy-Authentication header is deprecated, and doing so can trigger header change errors from Puppeteer
+            # https://github.com/puppeteer/puppeteer/issues/676 ?
+            # https://help.brightdata.com/hc/en-us/articles/12632549957649-Proxy-Manager-How-to-Guides#h_01HAKWR4Q0AFS8RZTNYWRDFJC2
+            # https://cri.dev/posts/2020-03-30-How-to-solve-Puppeteer-Chrome-Error-ERR_INVALID_ARGUMENT/
+            await self.page.authenticate(self.proxy)
+
+        # Re-use as much code from browser steps as possible so its the same
+        # from changedetectionio.blueprint.browser_steps.browser_steps import steppable_browser_interface
+
+        # not yet used here, we fallback to playwright when browsersteps is required
+        #            browsersteps_interface = steppable_browser_interface()
+        #            browsersteps_interface.page = self.page
+
+        async def handle_frame_navigation(event):
+            logger.debug(f"Frame navigated: {event}")
+            w = extra_wait - 2 if extra_wait > 4 else 2
+            logger.debug(f"Waiting {w} seconds before calling Page.stopLoading...")
+            await asyncio.sleep(w)
+            logger.debug("Issuing stopLoading command...")
+            await self.page._client.send('Page.stopLoading')
+            logger.debug("stopLoading command sent!")
+
+        self.page._client.on('Page.frameStartedNavigating', lambda event: asyncio.create_task(handle_frame_navigation(event)))
+        self.page._client.on('Page.frameStartedLoading', lambda event: asyncio.create_task(handle_frame_navigation(event)))
+        self.page._client.on('Page.frameStoppedLoading', lambda event: logger.debug(f"Frame stopped loading: {event}"))
+
+        response = None
+        attempt=0
+        while not response:
+            logger.debug(f"Attempting page fetch {url} attempt {attempt}")
+            response = await self.page.goto(url)
+            await asyncio.sleep(1 + extra_wait)
+            if response:
+                break
+            if not response:
+                logger.warning("Page did not fetch! trying again!")
+            if response is None and attempt>=2:
+                await self.page.close()
+                await browser.close()
+                logger.warning(f"Content Fetcher > Response object was none (as in, the response from the browser was empty, not just the content) exiting attmpt {attempt}")
+                raise EmptyReply(url=url, status_code=None)
+            attempt+=1
+
+        self.headers = response.headers
+
+        try:
+            if self.webdriver_js_execute_code is not None and len(self.webdriver_js_execute_code):
+                await self.page.evaluate(self.webdriver_js_execute_code)
+        except Exception as e:
+            logger.warning("Got exception when running evaluate on custom JS code")
+            logger.error(str(e))
+            await self.page.close()
+            await browser.close()
+            # This can be ok, we will try to grab what we could retrieve
+            raise PageUnloadable(url=url, status_code=None, message=str(e))
+
+        try:
+            self.status_code = response.status
+        except Exception as e:
+            # https://github.com/dgtlmoon/changedetection.io/discussions/2122#discussioncomment-8241962
+            logger.critical(f"Response from the browser/Playwright did not have a status_code! Response follows.")
+            logger.critical(response)
+            await self.page.close()
+            await browser.close()
+            raise PageUnloadable(url=url, status_code=None, message=str(e))
+
+        if fetch_favicon:
+            try:
+                self.favicon_blob = await self.page.evaluate(FAVICON_FETCHER_JS)
+            except Exception as e:
+                logger.error(f"Error fetching FavIcon info {str(e)}, continuing.")
+
+        if self.status_code != 200 and not ignore_status_codes:
+            screenshot = await capture_full_page(page=self.page)
+
+            raise Non200ErrorCodeReceived(url=url, status_code=self.status_code, screenshot=screenshot)
+
+        content = await self.page.content
+
+        if not empty_pages_are_a_change and len(content.strip()) == 0:
+            logger.error("Content Fetcher > Content was empty (empty_pages_are_a_change is False), closing browsers")
+            await self.page.close()
+            await browser.close()
+            raise EmptyReply(url=url, status_code=response.status)
+
+        # Run Browser Steps here
+        # @todo not yet supported, we switch to playwright in this case
+        #            if self.browser_steps_get_valid_steps():
+        #                self.iterate_browser_steps()
+
+
+        # So we can find an element on the page where its selector was entered manually (maybe not xPath etc)
+        # Setup the xPath/VisualSelector scraper
+        if current_include_filters:
+            js = json.dumps(current_include_filters)
+            await self.page.evaluate(f"var include_filters={js}")
+        else:
+            await self.page.evaluate(f"var include_filters=''")
+
+        MAX_TOTAL_HEIGHT = int(os.getenv("SCREENSHOT_MAX_HEIGHT", SCREENSHOT_MAX_HEIGHT_DEFAULT))
+        self.xpath_data = await self.page.evaluate(XPATH_ELEMENT_JS, {
+            "visualselector_xpath_selectors": visualselector_xpath_selectors,
+            "max_height": MAX_TOTAL_HEIGHT
+        })
+        if not self.xpath_data:
+            raise Exception(f"Content Fetcher > xPath scraper failed. Please report this URL so we can fix it :)")
+
+        self.instock_data = await self.page.evaluate(INSTOCK_DATA_JS)
+
+        self.content = await self.page.content
+
+        self.screenshot = await capture_full_page(page=self.page)
+
+        # It's good to log here in the case that the browser crashes on shutting down but we still get the data we need
+        logger.success(f"Fetching '{url}' complete, closing page")
+        await self.page.close()
+        logger.success(f"Fetching '{url}' complete, closing browser")
+        await browser.close()
+        logger.success(f"Fetching '{url}' complete, exiting puppeteer fetch.")
+
+    async def main(self, **kwargs):
+        await self.fetch_page(**kwargs)
+
+    async def run(self,
+                  fetch_favicon=True,
+                  current_include_filters=None,
+                  empty_pages_are_a_change=False,
+                  ignore_status_codes=False,
+                  is_binary=False,
+                  request_body=None,
+                  request_headers=None,
+                  request_method=None,
+                  timeout=None,
+                  url=None,
+                  ):
+
+        #@todo make update_worker async which could run any of these content_fetchers within memory and time constraints
+        max_time = int(os.getenv('PUPPETEER_MAX_PROCESSING_TIMEOUT_SECONDS', 180))
+
+        # Now we run this properly in async context since we're called from async worker
+        try:
+            await asyncio.wait_for(self.main(
+                current_include_filters=current_include_filters,
+                empty_pages_are_a_change=empty_pages_are_a_change,
+                fetch_favicon=fetch_favicon,
+                ignore_status_codes=ignore_status_codes,
+                is_binary=is_binary,
+                request_body=request_body,
+                request_headers=request_headers,
+                request_method=request_method,
+                timeout=timeout,
+                url=url,
+            ), timeout=max_time
+            )
+        except asyncio.TimeoutError:
+            raise (BrowserFetchTimedOut(msg=f"Browser connected but was unable to process the page in {max_time} seconds."))

changedetectionio/content_fetchers/requests.py

@@ -0,0 +1,151 @@
+from loguru import logger
+import hashlib
+import os
+import asyncio
+from changedetectionio import strtobool
+from changedetectionio.content_fetchers.exceptions import BrowserStepsInUnsupportedFetcher, EmptyReply, Non200ErrorCodeReceived
+from changedetectionio.content_fetchers.base import Fetcher
+
+
+# "html_requests" is listed as the default fetcher in store.py!
+class fetcher(Fetcher):
+    fetcher_description = "Basic fast Plaintext/HTTP Client"
+
+    def __init__(self, proxy_override=None, custom_browser_connection_url=None):
+        super().__init__()
+        self.proxy_override = proxy_override
+        # browser_connection_url is none because its always 'launched locally'
+
+    def _run_sync(self,
+            url,
+            timeout,
+            request_headers,
+            request_body,
+            request_method,
+            ignore_status_codes=False,
+            current_include_filters=None,
+            is_binary=False,
+            empty_pages_are_a_change=False):
+        """Synchronous version of run - the original requests implementation"""
+
+        import chardet
+        import requests
+        from requests.exceptions import ProxyError, ConnectionError, RequestException
+
+        if self.browser_steps_get_valid_steps():
+            raise BrowserStepsInUnsupportedFetcher(url=url)
+
+        proxies = {}
+
+        # Allows override the proxy on a per-request basis
+        # https://requests.readthedocs.io/en/latest/user/advanced/#socks
+        # Should also work with `socks5://user:pass@host:port` type syntax.
+
+        if self.proxy_override:
+            proxies = {'http': self.proxy_override, 'https': self.proxy_override, 'ftp': self.proxy_override}
+        else:
+            if self.system_http_proxy:
+                proxies['http'] = self.system_http_proxy
+            if self.system_https_proxy:
+                proxies['https'] = self.system_https_proxy
+
+        session = requests.Session()
+
+
+        if strtobool(os.getenv('ALLOW_FILE_URI', 'false')) and url.startswith('file://'):
+            from requests_file import FileAdapter
+            session.mount('file://', FileAdapter())
+        try:
+            r = session.request(method=request_method,
+                                data=request_body.encode('utf-8') if type(request_body) is str else request_body,
+                                url=url,
+                                headers=request_headers,
+                                timeout=timeout,
+                                proxies=proxies,
+                                verify=False)
+        except Exception as e:
+            msg = str(e)
+            if proxies and 'SOCKSHTTPSConnectionPool' in msg:
+                msg = f"Proxy connection failed? {msg}"
+            raise Exception(msg) from e
+
+        # If the response did not tell us what encoding format to expect, Then use chardet to override what `requests` thinks.
+        # For example - some sites don't tell us it's utf-8, but return utf-8 content
+        # This seems to not occur when using webdriver/selenium, it seems to detect the text encoding more reliably.
+        # https://github.com/psf/requests/issues/1604 good info about requests encoding detection
+        if not is_binary:
+            # Don't run this for PDF (and requests identified as binary) takes a _long_ time
+            if not r.headers.get('content-type') or not 'charset=' in r.headers.get('content-type'):
+                encoding = chardet.detect(r.content)['encoding']
+                if encoding:
+                    r.encoding = encoding
+
+        self.headers = r.headers
+
+        if not r.content or not len(r.content):
+            logger.debug(f"Requests returned empty content for '{url}'")
+            if not empty_pages_are_a_change:
+                raise EmptyReply(url=url, status_code=r.status_code)
+            else:
+                logger.debug(f"URL {url} gave zero byte content reply with Status Code {r.status_code}, but empty_pages_are_a_change = True")
+
+        # @todo test this
+        # @todo maybe you really want to test zero-byte return pages?
+        if r.status_code != 200 and not ignore_status_codes:
+            # maybe check with content works?
+            raise Non200ErrorCodeReceived(url=url, status_code=r.status_code, page_html=r.text)
+
+        self.status_code = r.status_code
+        if is_binary:
+            # Binary files just return their checksum until we add something smarter
+            self.content = hashlib.md5(r.content).hexdigest()
+        else:
+            self.content = r.text
+
+        self.raw_content = r.content
+
+    async def run(self,
+                  fetch_favicon=True,
+                  current_include_filters=None,
+                  empty_pages_are_a_change=False,
+                  ignore_status_codes=False,
+                  is_binary=False,
+                  request_body=None,
+                  request_headers=None,
+                  request_method=None,
+                  timeout=None,
+                  url=None,
+                  ):
+        """Async wrapper that runs the synchronous requests code in a thread pool"""
+        
+        loop = asyncio.get_event_loop()
+        
+        # Run the synchronous _run_sync in a thread pool to avoid blocking the event loop
+        await loop.run_in_executor(
+            None,  # Use default ThreadPoolExecutor
+            lambda: self._run_sync(
+                url=url,
+                timeout=timeout,
+                request_headers=request_headers,
+                request_body=request_body,
+                request_method=request_method,
+                ignore_status_codes=ignore_status_codes,
+                current_include_filters=current_include_filters,
+                is_binary=is_binary,
+                empty_pages_are_a_change=empty_pages_are_a_change
+            )
+        )
+
+    def quit(self, watch=None):
+
+        # In case they switched to `requests` fetcher from something else
+        # Then the screenshot could be old, in any case, it's not used here.
+        # REMOVE_REQUESTS_OLD_SCREENSHOTS - Mainly used for testing
+        if strtobool(os.getenv("REMOVE_REQUESTS_OLD_SCREENSHOTS", 'true')):
+            screenshot = watch.get_screenshot()
+            if screenshot:
+                try:
+                    os.unlink(screenshot)
+                except Exception as e:
+                    logger.warning(f"Failed to unlink screenshot: {screenshot} - {e}")
+

changedetectionio/content_fetchers/res/__init__.py

@@ -0,0 +1 @@
+# resources for browser injection/scraping

changedetectionio/content_fetchers/res/favicon-fetcher.js

@@ -0,0 +1,101 @@
+(async () => {
+  // Define the function inside the IIFE for console testing
+  window.getFaviconAsBlob = async function() {
+    const links = Array.from(document.querySelectorAll(
+      'link[rel~="apple-touch-icon"], link[rel~="icon"]'
+    ));
+
+    const icons = links.map(link => {
+      const sizesStr = link.getAttribute('sizes');
+      let size = 0;
+      if (sizesStr) {
+        const [w] = sizesStr.split('x').map(Number);
+        if (!isNaN(w)) size = w;
+      } else {
+        size = 16;
+      }
+      return {
+        size,
+        rel: link.getAttribute('rel'),
+        href: link.href,
+        hasSizes: !!sizesStr
+      };
+    });
+
+    // If no icons found, add fallback favicon.ico
+    if (icons.length === 0) {
+      icons.push({
+        size: 16,
+        rel: 'icon',
+        href: '/favicon.ico',
+        hasSizes: false
+      });
+    }
+
+    // sort preference: highest resolution first, then apple-touch-icon, then regular icons
+    icons.sort((a, b) => {
+      // First priority: actual size (highest first)
+      if (a.size !== b.size) {
+        return b.size - a.size;
+      }
+      
+      // Second priority: apple-touch-icon over regular icon
+      const isAppleA = /apple-touch-icon/.test(a.rel);
+      const isAppleB = /apple-touch-icon/.test(b.rel);
+      if (isAppleA && !isAppleB) return -1;
+      if (!isAppleA && isAppleB) return 1;
+      
+      // Third priority: icons with no size attribute (fallback icons) last
+      const hasNoSizeA = !a.hasSizes;
+      const hasNoSizeB = !b.hasSizes;
+      if (hasNoSizeA && !hasNoSizeB) return 1;
+      if (!hasNoSizeA && hasNoSizeB) return -1;
+      
+      return 0;
+    });
+
+    const timeoutMs = 2000;
+
+    for (const icon of icons) {
+      try {
+        const controller = new AbortController();
+        const timeout = setTimeout(() => controller.abort(), timeoutMs);
+
+        const resp = await fetch(icon.href, {
+          signal: controller.signal,
+          redirect: 'follow'
+        });
+
+        clearTimeout(timeout);
+
+        if (!resp.ok) {
+          continue;
+        }
+
+        const blob = await resp.blob();
+
+        // Convert blob to base64
+        const reader = new FileReader();
+        return await new Promise(resolve => {
+          reader.onloadend = () => {
+            resolve({
+              url: icon.href,
+              base64: reader.result.split(",")[1]
+            });
+          };
+          reader.readAsDataURL(blob);
+        });
+
+      } catch (e) {
+        continue;
+      }
+    }
+
+    // nothing found
+    return null;
+  };
+
+  // Auto-execute and return result for page.evaluate()
+  return await window.getFaviconAsBlob();
+})();
+

changedetectionio/content_fetchers/res/stock-not-in-stock.js

@@ -0,0 +1,239 @@
+async () => {
+
+    function isItemInStock() {
+        // @todo Pass these in so the same list can be used in non-JS fetchers
+        const outOfStockTexts = [
+            ' أخبرني عندما يتوفر',
+            '0 in stock',
+            'actuellement indisponible',
+            'agotado',
+            'article épuisé',
+            'artikel zurzeit vergriffen',
+            'as soon as stock is available',
+            'aucune offre n\'est disponible',
+            'ausverkauft', // sold out
+            'available for back order',
+            'awaiting stock',
+            'back in stock soon',
+            'back-order or out of stock',
+            'backordered',
+            'backorder',
+            'benachrichtigt mich', // notify me
+            'binnenkort leverbaar', // coming soon
+            'brak na stanie',
+            'brak w magazynie',
+            'coming soon',
+            'currently have any tickets for this',
+            'currently unavailable',
+            'dieser artikel ist bald wieder verfügbar',
+            'dostępne wkrótce',
+            'en rupture',
+            'esgotado',
+            'in kürze lieferbar',
+            'indisponible',
+            'indisponível',
+            'isn\'t in stock right now',
+            'isnt in stock right now',
+            'isn’t in stock right now',
+            'item is no longer available',
+            'let me know when it\'s available',
+            'mail me when available',
+            'message if back in stock',
+            'mevcut değil',
+            'more on order',
+            'nachricht bei',
+            'nicht auf lager',
+            'nicht lagernd',
+            'nicht lieferbar',
+            'nicht verfügbar',
+            'nicht vorrätig',
+            'nicht mehr lieferbar',
+            'nicht zur verfügung',
+            'nie znaleziono produktów',
+            'niet beschikbaar',
+            'niet leverbaar',
+            'niet op voorraad',
+            'no disponible',
+            'no featured offers available',
+            'no longer available',
+            'no longer in stock',
+            'no tickets available',
+            'non disponibile',
+            'non disponible',
+            'not available',
+            'not currently available',
+            'not in stock',
+            'notify me when available',
+            'notify me',
+            'notify when available',
+            'não disponível',
+            'não estamos a aceitar encomendas',
+            'out of stock',
+            'out-of-stock',
+            'plus disponible',
+            'prodotto esaurito',
+            'produkt niedostępny',
+            'rupture',
+            'sold out',
+            'sold-out',
+            'stok habis',
+            'stok kosong',
+            'stok varian ini habis',
+            'stokta yok',
+            'temporarily out of stock',
+            'temporarily unavailable',
+            'there were no search results for',
+            'this item is currently unavailable',
+            'tickets unavailable',
+            'tidak dijual',
+            'tidak tersedia',
+            'tijdelijk uitverkocht',
+            'tiket tidak tersedia',
+            'to subscribe to back in stock',
+            'tükendi',
+            'unavailable nearby',
+            'unavailable tickets',
+            'vergriffen',
+            'vorbestellen',
+            'vorbestellung ist bald möglich',
+            'we couldn\'t find any products that match',
+            'we do not currently have an estimate of when this product will be back in stock.',
+            'we don\'t currently have any',
+            'we don\'t know when or if this item will be back in stock.',
+            'we were not able to find a match',
+            'when this arrives in stock',
+            'when this item is available to order',
+            'zur zeit nicht an lager',
+            'épuisé',
+            '品切れ',
+            '已售',
+            '已售完',
+            '품절'
+        ];
+
+
+        const vh = Math.max(document.documentElement.clientHeight || 0, window.innerHeight || 0);
+
+        function getElementBaseText(element) {
+            // .textContent can include text from children which may give the wrong results
+            // scan only immediate TEXT_NODEs, which will be a child of the element
+            var text = "";
+            for (var i = 0; i < element.childNodes.length; ++i)
+                if (element.childNodes[i].nodeType === Node.TEXT_NODE)
+                    text += element.childNodes[i].textContent;
+            return text.toLowerCase().trim();
+        }
+
+        const negateOutOfStockRegex = new RegExp('^([0-9] in stock|add to cart|in stock|arrives approximately)', 'ig');
+        // The out-of-stock or in-stock-text is generally always above-the-fold
+        // and often below-the-fold is a list of related products that may or may not contain trigger text
+        // so it's good to filter to just the 'above the fold' elements
+        // and it should be atleast 100px from the top to ignore items in the toolbar, sometimes menu items like "Coming soon" exist
+
+        function elementIsInEyeBallRange(element) {
+            // outside the 'fold' or some weird text in the heading area
+            // .getBoundingClientRect() was causing a crash in chrome 119, can only be run on contentVisibility != hidden
+            // Note: theres also an automated test that places the 'out of stock' text fairly low down
+            // Skip text that could be in the header area
+            if (element.getBoundingClientRect().bottom + window.scrollY <= 300 ) {
+                return false;
+            }
+            // Skip text that could be much further down (like a list of "you may like" products that have 'sold out' in there
+            if (element.getBoundingClientRect().bottom + window.scrollY >= 1300 ) {
+                return false;
+            }
+            return true;
+        }
+
+// @todo - if it's SVG or IMG, go into image diff mode
+
+        function collectVisibleElements(parent, visibleElements) {
+            if (!parent) return; // Base case: if parent is null or undefined, return
+
+            // Add the parent itself to the visible elements array if it's of the specified types
+            visibleElements.push(parent);
+
+            // Iterate over the parent's children
+            const children = parent.children;
+            for (let i = 0; i < children.length; i++) {
+                const child = children[i];
+                if (
+                    child.nodeType === Node.ELEMENT_NODE &&
+                    window.getComputedStyle(child).display !== 'none' &&
+                    window.getComputedStyle(child).visibility !== 'hidden' &&
+                    child.offsetWidth >= 0 &&
+                    child.offsetHeight >= 0 &&
+                    window.getComputedStyle(child).contentVisibility !== 'hidden'
+                ) {
+                    // If the child is an element and is visible, recursively collect visible elements
+                    collectVisibleElements(child, visibleElements);
+                }
+            }
+        }
+
+        const elementsToScan = [];
+        collectVisibleElements(document.body, elementsToScan);
+
+        var elementText = "";
+
+        // REGEXS THAT REALLY MEAN IT'S IN STOCK
+        for (let i = elementsToScan.length - 1; i >= 0; i--) {
+            const element = elementsToScan[i];
+
+            if (!elementIsInEyeBallRange(element)) {
+                continue
+            }
+
+            elementText = "";
+            try {
+                if (element.tagName.toLowerCase() === "input") {
+                    elementText = element.value.toLowerCase().trim();
+                } else {
+                    elementText = getElementBaseText(element);
+                }
+            } catch (e) {
+                console.warn('stock-not-in-stock.js scraper - handling element for gettext failed', e);
+            }
+            if (elementText.length) {
+                // try which ones could mean its in stock
+                if (negateOutOfStockRegex.test(elementText) && !elementText.includes('(0 products)')) {
+                    console.log(`Negating/overriding 'Out of Stock' back to "Possibly in stock" found "${elementText}"`)
+                    element.style.border = "2px solid green"; // highlight the element that was detected as in stock
+                    return 'Possibly in stock';
+                }
+            }
+        }
+
+        // OTHER STUFF THAT COULD BE THAT IT'S OUT OF STOCK
+        for (let i = elementsToScan.length - 1; i >= 0; i--) {
+            const element = elementsToScan[i];
+
+            if (!elementIsInEyeBallRange(element)) {
+                continue
+            }
+            elementText = "";
+            if (element.tagName.toLowerCase() === "input") {
+                elementText = element.value.toLowerCase().trim();
+            } else {
+                elementText = getElementBaseText(element);
+            }
+
+            if (elementText.length) {
+                // and these mean its out of stock
+                for (const outOfStockText of outOfStockTexts) {
+                    if (elementText.includes(outOfStockText)) {
+                        console.log(`Selected 'Out of Stock' - found text "${outOfStockText}" - "${elementText}" - offset top ${element.getBoundingClientRect().top}, page height is ${vh}`)
+                        element.style.border = "2px solid red"; // highlight the element that was detected as out of stock
+                        return outOfStockText; // item is out of stock
+                    }
+                }
+            }
+        }
+
+        console.log(`Returning 'Possibly in stock' - cant' find any useful matching text`)
+        return 'Possibly in stock'; // possibly in stock, cant decide otherwise.
+    }
+
+// returns the element text that makes it think it's out of stock
+    return isItemInStock().trim()
+}

changedetectionio/content_fetchers/res/xpath_element_scraper.js

@@ -0,0 +1,284 @@
+async (options) => {
+
+    let visualselector_xpath_selectors = options.visualselector_xpath_selectors
+    let max_height = options.max_height
+
+    var scroll_y = 0;
+    try {
+        scroll_y = +document.documentElement.scrollTop || document.body.scrollTop
+    } catch (e) {
+        console.log(e);
+    }
+
+// Include the getXpath script directly, easier than fetching
+    function getxpath(e) {
+        var n = e;
+        if (n && n.id) return '//*[@id="' + n.id + '"]';
+        for (var o = []; n && Node.ELEMENT_NODE === n.nodeType;) {
+            for (var i = 0, r = !1, d = n.previousSibling; d;) d.nodeType !== Node.DOCUMENT_TYPE_NODE && d.nodeName === n.nodeName && i++, d = d.previousSibling;
+            for (d = n.nextSibling; d;) {
+                if (d.nodeName === n.nodeName) {
+                    r = !0;
+                    break
+                }
+                d = d.nextSibling
+            }
+            o.push((n.prefix ? n.prefix + ":" : "") + n.localName + (i || r ? "[" + (i + 1) + "]" : "")), n = n.parentNode
+        }
+        return o.length ? "/" + o.reverse().join("/") : ""
+    }
+
+    const findUpTag = (el) => {
+        let r = el
+        chained_css = [];
+        depth = 0;
+
+        //  Strategy 1: If it's an input, with name, and there's only one, prefer that
+        if (el.name !== undefined && el.name.length) {
+            var proposed = el.tagName + "[name=\"" + CSS.escape(el.name) + "\"]";
+            var proposed_element = window.document.querySelectorAll(proposed);
+            if (proposed_element.length) {
+                if (proposed_element.length === 1) {
+                    return proposed;
+                } else {
+                    // Some sites change ID but name= stays the same, we can hit it if we know the index
+                    // Find all the elements that match and work out the input[n]
+                    var n = Array.from(proposed_element).indexOf(el);
+                    // Return a Playwright selector for nthinput[name=zipcode]
+                    return proposed + " >> nth=" + n;
+                }
+            }
+        }
+
+        // Strategy 2: Keep going up until we hit an ID tag, imagine it's like  #list-widget div h4
+        while (r.parentNode) {
+            if (depth === 5) {
+                break;
+            }
+            if ('' !== r.id) {
+                chained_css.unshift("#" + CSS.escape(r.id));
+                final_selector = chained_css.join(' > ');
+                // Be sure theres only one, some sites have multiples of the same ID tag :-(
+                if (window.document.querySelectorAll(final_selector).length === 1) {
+                    return final_selector;
+                }
+                return null;
+            } else {
+                chained_css.unshift(r.tagName.toLowerCase());
+            }
+            r = r.parentNode;
+            depth += 1;
+        }
+        return null;
+    }
+
+
+// @todo - if it's SVG or IMG, go into image diff mode
+
+    var size_pos = [];
+// after page fetch, inject this JS
+// build a map of all elements and their positions (maybe that only include text?)
+    var bbox;
+    console.log(`Scanning for "${visualselector_xpath_selectors}"`);
+
+    function collectVisibleElements(parent, visibleElements) {
+        if (!parent) return; // Base case: if parent is null or undefined, return
+
+
+        // Add the parent itself to the visible elements array if it's of the specified types
+        const tagName = parent.tagName.toLowerCase();
+        if (visualselector_xpath_selectors.split(',').includes(tagName)) {
+            visibleElements.push(parent);
+        }
+
+        // Iterate over the parent's children
+        const children = parent.children;
+        for (let i = 0; i < children.length; i++) {
+            const child = children[i];
+            const computedStyle = window.getComputedStyle(child);
+
+            if (
+                child.nodeType === Node.ELEMENT_NODE &&
+                computedStyle.display !== 'none' &&
+                computedStyle.visibility !== 'hidden' &&
+                child.offsetWidth >= 0 &&
+                child.offsetHeight >= 0 &&
+                computedStyle.contentVisibility !== 'hidden'
+            ) {
+                // If the child is an element and is visible, recursively collect visible elements
+                collectVisibleElements(child, visibleElements);
+            }
+        }
+    }
+
+// Create an array to hold the visible elements
+    const visibleElementsArray = [];
+
+// Call collectVisibleElements with the starting parent element
+    collectVisibleElements(document.body, visibleElementsArray);
+
+
+    visibleElementsArray.forEach(function (element) {
+
+        bbox = element.getBoundingClientRect();
+
+        // Skip really small ones, and where width or height ==0
+        if (bbox['width'] * bbox['height'] < 10) {
+            return
+        }
+
+        // Don't include elements that are offset from canvas
+        if (bbox['top'] + scroll_y < 0 || bbox['left'] < 0) {
+            return
+        }
+
+        // @todo the getXpath kind of sucks, it doesnt know when there is for example just one ID sometimes
+        // it should not traverse when we know we can anchor off just an ID one level up etc..
+        // maybe, get current class or id, keep traversing up looking for only class or id until there is just one match
+
+        // 1st primitive - if it has class, try joining it all and select, if theres only one.. well thats us.
+        xpath_result = false;
+        try {
+            var d = findUpTag(element);
+            if (d) {
+                xpath_result = d;
+            }
+        } catch (e) {
+            console.log(e);
+        }
+        // You could swap it and default to getXpath and then try the smarter one
+        // default back to the less intelligent one
+        if (!xpath_result) {
+            try {
+                // I've seen on FB and eBay that this doesnt work
+                // ReferenceError: getXPath is not defined at eval (eval at evaluate (:152:29), <anonymous>:67:20) at UtilityScript.evaluate (<anonymous>:159:18) at UtilityScript.<anonymous> (<anonymous>:1:44)
+                xpath_result = getxpath(element);
+            } catch (e) {
+                console.log(e);
+                return
+            }
+        }
+
+        let label = "not-interesting" // A placeholder, the actual labels for training are done by hand for now
+
+        let text = element.textContent.trim().slice(0, 30).trim();
+        while (/\n{2,}|\t{2,}/.test(text)) {
+            text = text.replace(/\n{2,}/g, '\n').replace(/\t{2,}/g, '\t')
+        }
+
+        // Try to identify any possible currency amounts "Sale: 4000" or "Sale now 3000 Kc", can help with the training.
+        const hasDigitCurrency = (/\d/.test(text.slice(0, 6)) || /\d/.test(text.slice(-6))) && /([€£$¥₩₹]|USD|AUD|EUR|Kč|kr|SEK|,–)/.test(text);
+        const computedStyle = window.getComputedStyle(element);
+
+        if (Math.floor(bbox['top']) + scroll_y > max_height) {
+            return
+        }
+
+        size_pos.push({
+            xpath: xpath_result,
+            width: Math.round(bbox['width']),
+            height: Math.round(bbox['height']),
+            left: Math.floor(bbox['left']),
+            top: Math.floor(bbox['top']) + scroll_y,
+            // tagName used by Browser Steps
+            tagName: (element.tagName) ? element.tagName.toLowerCase() : '',
+            // tagtype used by Browser Steps
+            tagtype: (element.tagName.toLowerCase() === 'input' && element.type) ? element.type.toLowerCase() : '',
+            isClickable: computedStyle.cursor === "pointer",
+            // Used by the keras trainer
+            fontSize: computedStyle.getPropertyValue('font-size'),
+            fontWeight: computedStyle.getPropertyValue('font-weight'),
+            hasDigitCurrency: hasDigitCurrency,
+            label: label,
+        });
+
+    });
+
+
+// Inject the current one set in the include_filters, which may be a CSS rule
+// used for displaying the current one in VisualSelector, where its not one we generated.
+    if (include_filters.length) {
+        let results;
+        // Foreach filter, go and find it on the page and add it to the results so we can visualise it again
+        for (const f of include_filters) {
+            bbox = false;
+
+            if (!f.length) {
+                console.log("xpath_element_scraper: Empty filter, skipping");
+                continue;
+            }
+
+            try {
+                // is it xpath?
+                if (f.startsWith('/') || f.startsWith('xpath')) {
+                    var qry_f = f.replace(/xpath(:|\d:)/, '')
+                    console.log("[xpath] Scanning for included filter " + qry_f)
+                    let xpathResult = document.evaluate(qry_f, document, null, XPathResult.ORDERED_NODE_SNAPSHOT_TYPE, null);
+                    results = [];
+                    for (let i = 0; i < xpathResult.snapshotLength; i++) {
+                        results.push(xpathResult.snapshotItem(i));
+                    }
+                } else {
+                    console.log("[css] Scanning for included filter " + f)
+                    console.log("[css] Scanning for included filter " + f);
+                    results = document.querySelectorAll(f);
+                }
+            } catch (e) {
+                // Maybe catch DOMException and alert?
+                console.log("xpath_element_scraper: Exception selecting element from filter " + f);
+                console.log(e);
+            }
+
+            if (results != null && results.length) {
+
+                // Iterate over the results
+                results.forEach(node => {
+                    // Try to resolve //something/text() back to its /something so we can atleast get the bounding box
+                    try {
+                        if (typeof node.nodeName == 'string' && node.nodeName === '#text') {
+                            node = node.parentElement
+                        }
+                    } catch (e) {
+                        console.log(e)
+                        console.log("xpath_element_scraper: #text resolver")
+                    }
+
+                    // #1231 - IN the case XPath attribute filter is applied, we will have to traverse up and find the element.
+                    if (typeof node.getBoundingClientRect == 'function') {
+                        bbox = node.getBoundingClientRect();
+                        console.log("xpath_element_scraper: Got filter element, scroll from top was " + scroll_y)
+                    } else {
+                        try {
+                            // Try and see we can find its ownerElement
+                            bbox = node.ownerElement.getBoundingClientRect();
+                            console.log("xpath_element_scraper: Got filter by ownerElement element, scroll from top was " + scroll_y)
+                        } catch (e) {
+                            console.log(e)
+                            console.log("xpath_element_scraper: error looking up node.ownerElement")
+                        }
+                    }
+
+                    if (bbox && bbox['width'] > 0 && bbox['height'] > 0) {
+                        size_pos.push({
+                            xpath: f,
+                            width: parseInt(bbox['width']),
+                            height: parseInt(bbox['height']),
+                            left: parseInt(bbox['left']),
+                            top: parseInt(bbox['top']) + scroll_y,
+                            highlight_as_custom_filter: true
+                        });
+                    }
+                });
+            }
+        }
+    }
+
+// Sort the elements so we find the smallest one first, in other words, we find the smallest one matching in that area
+// so that we dont select the wrapping element by mistake and be unable to select what we want
+    size_pos.sort((a, b) => (a.width * a.height > b.width * b.height) ? 1 : -1)
+
+// browser_width required for proper scaling in the frontend
+    // Return as a string to save playwright for juggling thousands of objects
+    return JSON.stringify({'size_pos': size_pos, 'browser_width': window.innerWidth});
+}
+

changedetectionio/content_fetchers/screenshot_handler.py

@@ -0,0 +1,73 @@
+# Pages with a vertical height longer than this will use the 'stitch together' method.
+
+# - Many GPUs have a max texture size of 16384x16384px (or lower on older devices).
+# - If a page is taller than ~8000–10000px, it risks exceeding GPU memory limits.
+# - This is especially important on headless Chromium, where Playwright may fail to allocate a massive full-page buffer.
+
+from loguru import logger
+
+from changedetectionio.content_fetchers import SCREENSHOT_MAX_HEIGHT_DEFAULT, SCREENSHOT_DEFAULT_QUALITY
+
+
+def stitch_images_worker(pipe_conn, chunks_bytes, original_page_height, capture_height):
+    import os
+    import io
+    from PIL import Image, ImageDraw, ImageFont
+
+    try:
+
+        # Load images from byte chunks
+        images = [Image.open(io.BytesIO(b)) for b in chunks_bytes]
+        total_height = sum(im.height for im in images)
+        max_width = max(im.width for im in images)
+
+        # Create stitched image
+        stitched = Image.new('RGB', (max_width, total_height))
+        y_offset = 0
+        for im in images:
+            stitched.paste(im, (0, y_offset))
+            y_offset += im.height
+
+        # Draw caption on top (overlaid, not extending canvas)
+        draw = ImageDraw.Draw(stitched)
+
+        if original_page_height > capture_height:
+            caption_text = f"WARNING: Screenshot was {original_page_height}px but trimmed to {capture_height}px because it was too long"
+            padding = 10
+            font_size = 35
+            font_color = (255, 0, 0)
+            background_color = (255, 255, 255)
+
+
+            # Try to load a proper font
+            try:
+                font = ImageFont.truetype("arial.ttf", font_size)
+            except IOError:
+                font = ImageFont.load_default()
+
+            bbox = draw.textbbox((0, 0), caption_text, font=font)
+            text_width = bbox[2] - bbox[0]
+            text_height = bbox[3] - bbox[1]
+
+            # Draw white rectangle background behind text
+            rect_top = 0
+            rect_bottom = text_height + 2 * padding
+            draw.rectangle([(0, rect_top), (max_width, rect_bottom)], fill=background_color)
+
+            # Draw text centered horizontally, 10px padding from top of the rectangle
+            text_x = (max_width - text_width) // 2
+            text_y = padding
+            draw.text((text_x, text_y), caption_text, font=font, fill=font_color)
+
+        # Encode and send image
+        output = io.BytesIO()
+        stitched.save(output, format="JPEG", quality=int(os.getenv("SCREENSHOT_QUALITY", SCREENSHOT_DEFAULT_QUALITY)))
+        pipe_conn.send_bytes(output.getvalue())
+
+        stitched.close()
+    except Exception as e:
+        pipe_conn.send(f"error:{e}")
+    finally:
+        pipe_conn.close()
+
+

changedetectionio/content_fetchers/webdriver_selenium.py

@@ -0,0 +1,143 @@
+import os
+import time
+
+from loguru import logger
+from changedetectionio.content_fetchers.base import Fetcher
+
+
+class fetcher(Fetcher):
+    if os.getenv("WEBDRIVER_URL"):
+        fetcher_description = f"WebDriver Chrome/Javascript via \"{os.getenv('WEBDRIVER_URL', '')}\""
+    else:
+        fetcher_description = "WebDriver Chrome/Javascript"
+
+    proxy = None
+    proxy_url = None
+
+    def __init__(self, proxy_override=None, custom_browser_connection_url=None):
+        super().__init__()
+        from urllib.parse import urlparse
+        from selenium.webdriver.common.proxy import Proxy
+
+        # .strip('"') is going to save someone a lot of time when they accidently wrap the env value
+        if not custom_browser_connection_url:
+            self.browser_connection_url = os.getenv("WEBDRIVER_URL", 'http://browser-chrome:4444/wd/hub').strip('"')
+        else:
+            self.browser_connection_is_custom = True
+            self.browser_connection_url = custom_browser_connection_url
+
+        ##### PROXY SETUP #####
+
+        proxy_sources = [
+            self.system_http_proxy,
+            self.system_https_proxy,
+            os.getenv('webdriver_proxySocks'),
+            os.getenv('webdriver_socksProxy'),
+            os.getenv('webdriver_proxyHttp'),
+            os.getenv('webdriver_httpProxy'),
+            os.getenv('webdriver_proxyHttps'),
+            os.getenv('webdriver_httpsProxy'),
+            os.getenv('webdriver_sslProxy'),
+            proxy_override,  # last one should override
+        ]
+        # The built in selenium proxy handling is super unreliable!!! so we just grab which ever proxy setting we can find and throw it in --proxy-server=
+        for k in filter(None, proxy_sources):
+            if not k:
+                continue
+            self.proxy_url = k.strip()
+
+    async def run(self,
+                  fetch_favicon=True,
+                  current_include_filters=None,
+                  empty_pages_are_a_change=False,
+                  ignore_status_codes=False,
+                  is_binary=False,
+                  request_body=None,
+                  request_headers=None,
+                  request_method=None,
+                  timeout=None,
+                  url=None,
+                  ):
+
+        import asyncio
+
+        # Wrap the entire selenium operation in a thread executor
+        def _run_sync():
+            from selenium.webdriver.chrome.options import Options as ChromeOptions
+            # request_body, request_method unused for now, until some magic in the future happens.
+
+            options = ChromeOptions()
+
+            # Load Chrome options from env
+            CHROME_OPTIONS = [
+                line.strip()
+                for line in os.getenv("CHROME_OPTIONS", "").strip().splitlines()
+                if line.strip()
+            ]
+
+            for opt in CHROME_OPTIONS:
+                options.add_argument(opt)
+
+            # 1. proxy_config /Proxy(proxy_config) selenium object is REALLY unreliable
+            # 2. selenium-wire cant be used because the websocket version conflicts with pypeteer-ng
+            # 3. selenium only allows ONE runner at a time by default!
+            # 4. driver must use quit() or it will continue to block/hold the selenium process!!
+
+            if self.proxy_url:
+                options.add_argument(f'--proxy-server={self.proxy_url}')
+
+            from selenium.webdriver.remote.remote_connection import RemoteConnection
+            from selenium.webdriver.remote.webdriver import WebDriver as RemoteWebDriver
+            driver = None
+            try:
+                # Create the RemoteConnection and set timeout (e.g., 30 seconds)
+                remote_connection = RemoteConnection(
+                    self.browser_connection_url,
+                )
+                remote_connection.set_timeout(30)  # seconds
+
+                # Now create the driver with the RemoteConnection
+                driver = RemoteWebDriver(
+                    command_executor=remote_connection,
+                    options=options
+                )
+
+                driver.set_page_load_timeout(int(os.getenv("WEBDRIVER_PAGELOAD_TIMEOUT", 45)))
+            except Exception as e:
+                if driver:
+                    driver.quit()
+                raise e
+
+            try:
+                driver.get(url)
+
+                if not "--window-size" in os.getenv("CHROME_OPTIONS", ""):
+                    driver.set_window_size(1280, 1024)
+
+                driver.implicitly_wait(int(os.getenv("WEBDRIVER_DELAY_BEFORE_CONTENT_READY", 5)))
+
+                if self.webdriver_js_execute_code is not None:
+                    driver.execute_script(self.webdriver_js_execute_code)
+                    # Selenium doesn't automatically wait for actions as good as Playwright, so wait again
+                    driver.implicitly_wait(int(os.getenv("WEBDRIVER_DELAY_BEFORE_CONTENT_READY", 5)))
+
+                # @todo - how to check this? is it possible?
+                self.status_code = 200
+                # @todo somehow we should try to get this working for WebDriver
+                # raise EmptyReply(url=url, status_code=r.status_code)
+
+                # @todo - dom wait loaded?
+                import time
+                time.sleep(int(os.getenv("WEBDRIVER_DELAY_BEFORE_CONTENT_READY", 5)) + self.render_extract_delay)
+                self.content = driver.page_source
+                self.headers = {}
+                self.screenshot = driver.get_screenshot_as_png()
+            except Exception as e:
+                driver.quit()
+                raise e
+
+            driver.quit()
+
+        # Run the selenium operations in a thread pool to avoid blocking the event loop
+        loop = asyncio.get_event_loop()
+        await loop.run_in_executor(None, _run_sync)

changedetectionio/custom_queue.py

@@ -0,0 +1,535 @@
+import queue
+import asyncio
+from blinker import signal
+from loguru import logger
+
+
+class NotificationQueue(queue.Queue):
+    """
+    Extended Queue that sends a 'notification_event' signal when notifications are added.
+    
+    This class extends the standard Queue and adds a signal emission after a notification
+    is put into the queue. The signal includes the watch UUID if available.
+    """
+    
+    def __init__(self, maxsize=0):
+        super().__init__(maxsize)
+        try:
+            self.notification_event_signal = signal('notification_event')
+        except Exception as e:
+            logger.critical(f"Exception creating notification_event signal: {e}")
+
+    def put(self, item, block=True, timeout=None):
+        # Call the parent's put method first
+        super().put(item, block, timeout)
+        
+        # After putting the notification in the queue, emit signal with watch UUID
+        try:
+            if self.notification_event_signal and isinstance(item, dict):
+                watch_uuid = item.get('uuid')
+                if watch_uuid:
+                    # Send the notification_event signal with the watch UUID
+                    self.notification_event_signal.send(watch_uuid=watch_uuid)
+                    logger.trace(f"NotificationQueue: Emitted notification_event signal for watch UUID {watch_uuid}")
+                else:
+                    # Send signal without UUID for system notifications
+                    self.notification_event_signal.send()
+                    logger.trace("NotificationQueue: Emitted notification_event signal for system notification")
+        except Exception as e:
+            logger.error(f"Exception emitting notification_event signal: {e}")
+
+class SignalPriorityQueue(queue.PriorityQueue):
+    """
+    Extended PriorityQueue that sends a signal when items with a UUID are added.
+    
+    This class extends the standard PriorityQueue and adds a signal emission
+    after an item is put into the queue. If the item contains a UUID, the signal
+    is sent with that UUID as a parameter.
+    """
+    
+    def __init__(self, maxsize=0):
+        super().__init__(maxsize)
+        try:
+            self.queue_length_signal = signal('queue_length')
+        except Exception as e:
+            logger.critical(f"Exception: {e}")
+
+    def put(self, item, block=True, timeout=None):
+        # Call the parent's put method first
+        super().put(item, block, timeout)
+        
+        # After putting the item in the queue, check if it has a UUID and emit signal
+        if hasattr(item, 'item') and isinstance(item.item, dict) and 'uuid' in item.item:
+            uuid = item.item['uuid']
+            # Get the signal and send it if it exists
+            watch_check_update = signal('watch_check_update')
+            if watch_check_update:
+                # Send the watch_uuid parameter
+                watch_check_update.send(watch_uuid=uuid)
+        
+        # Send queue_length signal with current queue size
+        try:
+
+            if self.queue_length_signal:
+                self.queue_length_signal.send(length=self.qsize())
+        except Exception as e:
+            logger.critical(f"Exception: {e}")
+
+    def get(self, block=True, timeout=None):
+        # Call the parent's get method first
+        item = super().get(block, timeout)
+        
+        # Send queue_length signal with current queue size
+        try:
+            if self.queue_length_signal:
+                self.queue_length_signal.send(length=self.qsize())
+        except Exception as e:
+            logger.critical(f"Exception: {e}")
+        return item
+    
+    def get_uuid_position(self, target_uuid):
+        """
+        Find the position of a watch UUID in the priority queue.
+        Optimized for large queues - O(n) complexity instead of O(n log n).
+        
+        Args:
+            target_uuid: The UUID to search for
+            
+        Returns:
+            dict: Contains position info or None if not found
+                - position: 0-based position in queue (0 = next to be processed)
+                - total_items: total number of items in queue
+                - priority: the priority value of the found item
+        """
+        with self.mutex:
+            queue_list = list(self.queue)
+            total_items = len(queue_list)
+            
+            if total_items == 0:
+                return {
+                    'position': None,
+                    'total_items': 0,
+                    'priority': None,
+                    'found': False
+                }
+            
+            # Find the target item and its priority first - O(n)
+            target_item = None
+            target_priority = None
+            
+            for item in queue_list:
+                if (hasattr(item, 'item') and 
+                    isinstance(item.item, dict) and 
+                    item.item.get('uuid') == target_uuid):
+                    target_item = item
+                    target_priority = item.priority
+                    break
+            
+            if target_item is None:
+                return {
+                    'position': None,
+                    'total_items': total_items,
+                    'priority': None,
+                    'found': False
+                }
+            
+            # Count how many items have higher priority (lower numbers) - O(n)
+            position = 0
+            for item in queue_list:
+                # Items with lower priority numbers are processed first
+                if item.priority < target_priority:
+                    position += 1
+                elif item.priority == target_priority and item != target_item:
+                    # For same priority, count items that come before this one
+                    # (Note: this is approximate since heap order isn't guaranteed for equal priorities)
+                    position += 1
+            
+            return {
+                'position': position,
+                'total_items': total_items,
+                'priority': target_priority,
+                'found': True
+            }
+    
+    def get_all_queued_uuids(self, limit=None, offset=0):
+        """
+        Get UUIDs currently in the queue with their positions.
+        For large queues, use limit/offset for pagination.
+        
+        Args:
+            limit: Maximum number of items to return (None = all)
+            offset: Number of items to skip (for pagination)
+        
+        Returns:
+            dict: Contains items and metadata
+                - items: List of dicts with uuid, position, and priority
+                - total_items: Total number of items in queue
+                - returned_items: Number of items returned
+                - has_more: Whether there are more items after this page
+        """
+        with self.mutex:
+            queue_list = list(self.queue)
+            total_items = len(queue_list)
+            
+            if total_items == 0:
+                return {
+                    'items': [],
+                    'total_items': 0,
+                    'returned_items': 0,
+                    'has_more': False
+                }
+            
+            # For very large queues, warn about performance
+            if total_items > 1000 and limit is None:
+                logger.warning(f"Getting all {total_items} queued items without limit - this may be slow")
+            
+            # Sort only if we need exact positions (expensive for large queues)
+            if limit is not None and limit <= 100:
+                # For small requests, we can afford to sort
+                queue_items = sorted(queue_list)
+                end_idx = min(offset + limit, len(queue_items)) if limit else len(queue_items)
+                items_to_process = queue_items[offset:end_idx]
+                
+                result = []
+                for position, item in enumerate(items_to_process, start=offset):
+                    if (hasattr(item, 'item') and 
+                        isinstance(item.item, dict) and 
+                        'uuid' in item.item):
+                        
+                        result.append({
+                            'uuid': item.item['uuid'],
+                            'position': position,
+                            'priority': item.priority
+                        })
+                
+                return {
+                    'items': result,
+                    'total_items': total_items,
+                    'returned_items': len(result),
+                    'has_more': (offset + len(result)) < total_items
+                }
+            else:
+                # For large requests, return items with approximate positions
+                # This is much faster O(n) instead of O(n log n)
+                result = []
+                processed = 0
+                skipped = 0
+                
+                for item in queue_list:
+                    if (hasattr(item, 'item') and 
+                        isinstance(item.item, dict) and 
+                        'uuid' in item.item):
+                        
+                        if skipped < offset:
+                            skipped += 1
+                            continue
+                        
+                        if limit and processed >= limit:
+                            break
+                        
+                        # Approximate position based on priority comparison
+                        approx_position = sum(1 for other in queue_list if other.priority < item.priority)
+                        
+                        result.append({
+                            'uuid': item.item['uuid'],
+                            'position': approx_position,  # Approximate
+                            'priority': item.priority
+                        })
+                        processed += 1
+                
+                return {
+                    'items': result,
+                    'total_items': total_items,
+                    'returned_items': len(result),
+                    'has_more': (offset + len(result)) < total_items,
+                    'note': 'Positions are approximate for performance with large queues'
+                }
+    
+    def get_queue_summary(self):
+        """
+        Get a quick summary of queue state without expensive operations.
+        O(n) complexity - fast even for large queues.
+        
+        Returns:
+            dict: Queue summary statistics
+        """
+        with self.mutex:
+            queue_list = list(self.queue)
+            total_items = len(queue_list)
+            
+            if total_items == 0:
+                return {
+                    'total_items': 0,
+                    'priority_breakdown': {},
+                    'immediate_items': 0,
+                    'clone_items': 0,
+                    'scheduled_items': 0
+                }
+            
+            # Count items by priority type - O(n)
+            immediate_items = 0  # priority 1
+            clone_items = 0      # priority 5  
+            scheduled_items = 0  # priority > 100 (timestamps)
+            priority_counts = {}
+            
+            for item in queue_list:
+                priority = item.priority
+                priority_counts[priority] = priority_counts.get(priority, 0) + 1
+                
+                if priority == 1:
+                    immediate_items += 1
+                elif priority == 5:
+                    clone_items += 1
+                elif priority > 100:
+                    scheduled_items += 1
+            
+            return {
+                'total_items': total_items,
+                'priority_breakdown': priority_counts,
+                'immediate_items': immediate_items,
+                'clone_items': clone_items,
+                'scheduled_items': scheduled_items,
+                'min_priority': min(priority_counts.keys()) if priority_counts else None,
+                'max_priority': max(priority_counts.keys()) if priority_counts else None
+            }
+
+
+class AsyncSignalPriorityQueue(asyncio.PriorityQueue):
+    """
+    Async version of SignalPriorityQueue that sends signals when items are added/removed.
+    
+    This class extends asyncio.PriorityQueue and maintains the same signal behavior
+    as the synchronous version for real-time UI updates.
+    """
+    
+    def __init__(self, maxsize=0):
+        super().__init__(maxsize)
+        try:
+            self.queue_length_signal = signal('queue_length')
+        except Exception as e:
+            logger.critical(f"Exception: {e}")
+
+    async def put(self, item):
+        # Call the parent's put method first
+        await super().put(item)
+        
+        # After putting the item in the queue, check if it has a UUID and emit signal
+        if hasattr(item, 'item') and isinstance(item.item, dict) and 'uuid' in item.item:
+            uuid = item.item['uuid']
+            # Get the signal and send it if it exists
+            watch_check_update = signal('watch_check_update')
+            if watch_check_update:
+                # Send the watch_uuid parameter
+                watch_check_update.send(watch_uuid=uuid)
+        
+        # Send queue_length signal with current queue size
+        try:
+            if self.queue_length_signal:
+                self.queue_length_signal.send(length=self.qsize())
+        except Exception as e:
+            logger.critical(f"Exception: {e}")
+
+    async def get(self):
+        # Call the parent's get method first
+        item = await super().get()
+        
+        # Send queue_length signal with current queue size
+        try:
+            if self.queue_length_signal:
+                self.queue_length_signal.send(length=self.qsize())
+        except Exception as e:
+            logger.critical(f"Exception: {e}")
+        return item
+    
+    @property
+    def queue(self):
+        """
+        Provide compatibility with sync PriorityQueue.queue access
+        Returns the internal queue for template access
+        """
+        return self._queue if hasattr(self, '_queue') else []
+    
+    def get_uuid_position(self, target_uuid):
+        """
+        Find the position of a watch UUID in the async priority queue.
+        Optimized for large queues - O(n) complexity instead of O(n log n).
+        
+        Args:
+            target_uuid: The UUID to search for
+            
+        Returns:
+            dict: Contains position info or None if not found
+                - position: 0-based position in queue (0 = next to be processed)
+                - total_items: total number of items in queue
+                - priority: the priority value of the found item
+        """
+        queue_list = list(self._queue)
+        total_items = len(queue_list)
+        
+        if total_items == 0:
+            return {
+                'position': None,
+                'total_items': 0,
+                'priority': None,
+                'found': False
+            }
+        
+        # Find the target item and its priority first - O(n)
+        target_item = None
+        target_priority = None
+        
+        for item in queue_list:
+            if (hasattr(item, 'item') and 
+                isinstance(item.item, dict) and 
+                item.item.get('uuid') == target_uuid):
+                target_item = item
+                target_priority = item.priority
+                break
+        
+        if target_item is None:
+            return {
+                'position': None,
+                'total_items': total_items,
+                'priority': None,
+                'found': False
+            }
+        
+        # Count how many items have higher priority (lower numbers) - O(n)
+        position = 0
+        for item in queue_list:
+            if item.priority < target_priority:
+                position += 1
+            elif item.priority == target_priority and item != target_item:
+                position += 1
+        
+        return {
+            'position': position,
+            'total_items': total_items,
+            'priority': target_priority,
+            'found': True
+        }
+    
+    def get_all_queued_uuids(self, limit=None, offset=0):
+        """
+        Get UUIDs currently in the async queue with their positions.
+        For large queues, use limit/offset for pagination.
+        
+        Args:
+            limit: Maximum number of items to return (None = all)
+            offset: Number of items to skip (for pagination)
+        
+        Returns:
+            dict: Contains items and metadata (same structure as sync version)
+        """
+        queue_list = list(self._queue)
+        total_items = len(queue_list)
+        
+        if total_items == 0:
+            return {
+                'items': [],
+                'total_items': 0,
+                'returned_items': 0,
+                'has_more': False
+            }
+        
+        # Same logic as sync version but without mutex
+        if limit is not None and limit <= 100:
+            queue_items = sorted(queue_list)
+            end_idx = min(offset + limit, len(queue_items)) if limit else len(queue_items)
+            items_to_process = queue_items[offset:end_idx]
+            
+            result = []
+            for position, item in enumerate(items_to_process, start=offset):
+                if (hasattr(item, 'item') and 
+                    isinstance(item.item, dict) and 
+                    'uuid' in item.item):
+                    
+                    result.append({
+                        'uuid': item.item['uuid'],
+                        'position': position,
+                        'priority': item.priority
+                    })
+            
+            return {
+                'items': result,
+                'total_items': total_items,
+                'returned_items': len(result),
+                'has_more': (offset + len(result)) < total_items
+            }
+        else:
+            # Fast approximate positions for large queues
+            result = []
+            processed = 0
+            skipped = 0
+            
+            for item in queue_list:
+                if (hasattr(item, 'item') and 
+                    isinstance(item.item, dict) and 
+                    'uuid' in item.item):
+                    
+                    if skipped < offset:
+                        skipped += 1
+                        continue
+                    
+                    if limit and processed >= limit:
+                        break
+                    
+                    approx_position = sum(1 for other in queue_list if other.priority < item.priority)
+                    
+                    result.append({
+                        'uuid': item.item['uuid'],
+                        'position': approx_position,
+                        'priority': item.priority
+                    })
+                    processed += 1
+            
+            return {
+                'items': result,
+                'total_items': total_items,
+                'returned_items': len(result),
+                'has_more': (offset + len(result)) < total_items,
+                'note': 'Positions are approximate for performance with large queues'
+            }
+    
+    def get_queue_summary(self):
+        """
+        Get a quick summary of async queue state.
+        O(n) complexity - fast even for large queues.
+        """
+        queue_list = list(self._queue)
+        total_items = len(queue_list)
+        
+        if total_items == 0:
+            return {
+                'total_items': 0,
+                'priority_breakdown': {},
+                'immediate_items': 0,
+                'clone_items': 0,
+                'scheduled_items': 0
+            }
+        
+        immediate_items = 0
+        clone_items = 0
+        scheduled_items = 0
+        priority_counts = {}
+        
+        for item in queue_list:
+            priority = item.priority
+            priority_counts[priority] = priority_counts.get(priority, 0) + 1
+            
+            if priority == 1:
+                immediate_items += 1
+            elif priority == 5:
+                clone_items += 1
+            elif priority > 100:
+                scheduled_items += 1
+        
+        return {
+            'total_items': total_items,
+            'priority_breakdown': priority_counts,
+            'immediate_items': immediate_items,
+            'clone_items': clone_items,
+            'scheduled_items': scheduled_items,
+            'min_priority': min(priority_counts.keys()) if priority_counts else None,
+            'max_priority': max(priority_counts.keys()) if priority_counts else None
+        }

changedetectionio/diff.py

@@ -1,62 +0,0 @@
-# used for the notifications, the front-end is using a JS library
-
-import difflib
-
-
-def same_slicer(l, a, b):
-    if a == b:
-        return [l[a]]
-    else:
-        return l[a:b]
-
-# like .compare but a little different output
-def customSequenceMatcher(before, after, include_equal=False, include_removed=True, include_added=True, include_replaced=True, include_change_type_prefix=True):
-    cruncher = difflib.SequenceMatcher(isjunk=lambda x: x in " \\t", a=before, b=after)
-
-    # @todo Line-by-line mode instead of buncghed, including `after` that is not in `before` (maybe unset?)
-    for tag, alo, ahi, blo, bhi in cruncher.get_opcodes():
-        if include_equal and tag == 'equal':
-            g = before[alo:ahi]
-            yield g
-        elif include_removed and tag == 'delete':
-            row_prefix = "(removed) " if include_change_type_prefix else ''
-            g = [ row_prefix + i for i in same_slicer(before, alo, ahi)]
-            yield g
-        elif include_replaced and tag == 'replace':
-            row_prefix = "(changed) " if include_change_type_prefix else ''
-            g = [row_prefix + i for i in same_slicer(before, alo, ahi)]
-            row_prefix = "(into) " if include_change_type_prefix else ''
-            g += [row_prefix + i for i in same_slicer(after, blo, bhi)]
-            yield g
-        elif include_added and tag == 'insert':
-            row_prefix = "(added) " if include_change_type_prefix else ''
-            g = [row_prefix + i for i in same_slicer(after, blo, bhi)]
-            yield g
-
-# only_differences - only return info about the differences, no context
-# line_feed_sep could be "<br>" or "<li>" or "\n" etc
-def render_diff(previous_version_file_contents, newest_version_file_contents, include_equal=False, include_removed=True, include_added=True, include_replaced=True, line_feed_sep="\n", include_change_type_prefix=True, patch_format=False):
-
-    newest_version_file_contents = [line.rstrip() for line in newest_version_file_contents.splitlines()]
-
-    if previous_version_file_contents:
-        previous_version_file_contents = [line.rstrip() for line in previous_version_file_contents.splitlines()]
-    else:
-        previous_version_file_contents = ""
-
-    if patch_format:
-        patch = difflib.unified_diff(previous_version_file_contents, newest_version_file_contents)
-        return line_feed_sep.join(patch)
-
-    rendered_diff = customSequenceMatcher(before=previous_version_file_contents,
-                                          after=newest_version_file_contents,
-                                          include_equal=include_equal,
-                                          include_removed=include_removed,
-                                          include_added=include_added,
-                                          include_replaced=include_replaced,
-                                          include_change_type_prefix=include_change_type_prefix)
-
-    # Recursively join lists
-    f = lambda L: line_feed_sep.join([f(x) if type(x) is list else x for x in L])
-    p= f(rendered_diff)
-    return p

changedetectionio/diff/__init__.py

@@ -0,0 +1,462 @@
+"""
+Diff rendering module for change detection.
+
+This module provides functions for rendering differences between text content,
+with support for various output formats and tokenization strategies.
+"""
+
+import difflib
+from typing import List, Iterator, Union
+import diff_match_patch as dmp_module
+import re
+
+from .tokenizers import TOKENIZERS, tokenize_words_and_html
+from ..notification_service import CUSTOM_LINEBREAK_PLACEHOLDER
+
+# Remember! gmail, outlook etc dont support <style> must be inline.
+# Gmail: strips <ins> and <del> tags entirely.
+# This is for the WHOLE line background style
+REMOVED_STYLE = "background-color: #fadad7; color: #b30000;"
+ADDED_STYLE = "background-color: #eaf2c2; color: #406619;"
+HTML_REMOVED_STYLE = REMOVED_STYLE  # Export alias for handler.py
+HTML_ADDED_STYLE = ADDED_STYLE      # Export alias for handler.py
+
+# Darker backgrounds for nested highlighting (changed parts within lines)
+REMOVED_INNER_STYLE = "background-color: #ff867a; color: #111;"
+ADDED_INNER_STYLE = "background-color: #b2e841; color: #444;"
+HTML_CHANGED_STYLE = REMOVED_STYLE
+HTML_CHANGED_INTO_STYLE = ADDED_STYLE
+
+# Placemarker constants - these get replaced by apply_service_tweaks() in handler.py
+# Something that cant get escaped to HTML by accident
+REMOVED_PLACEMARKER_OPEN = '@removed_PLACEMARKER_OPEN'
+REMOVED_PLACEMARKER_CLOSED = '@removed_PLACEMARKER_CLOSED'
+
+ADDED_PLACEMARKER_OPEN = '@added_PLACEMARKER_OPEN'
+ADDED_PLACEMARKER_CLOSED = '@added_PLACEMARKER_CLOSED'
+
+CHANGED_PLACEMARKER_OPEN = '@changed_PLACEMARKER_OPEN'
+CHANGED_PLACEMARKER_CLOSED = '@changed_PLACEMARKER_CLOSED'
+
+CHANGED_INTO_PLACEMARKER_OPEN = '@changed_into_PLACEMARKER_OPEN'
+CHANGED_INTO_PLACEMARKER_CLOSED = '@changed_into_PLACEMARKER_CLOSED'
+
+# Compiled regex patterns for performance
+WHITESPACE_NORMALIZE_RE = re.compile(r'\s+')
+
+
+def render_inline_word_diff(before_line: str, after_line: str, ignore_junk: bool = False, markdown_style: str = None, tokenizer: str = 'words_and_html') -> tuple[str, bool]:
+    """
+    Render word-level differences between two lines inline using diff-match-patch library.
+
+    Args:
+        before_line: Original line text
+        after_line: Modified line text
+        ignore_junk: Ignore whitespace-only changes
+        markdown_style: Unused (kept for backwards compatibility)
+        tokenizer: Name of tokenizer to use from TOKENIZERS registry (default: 'words_and_html')
+
+    Returns:
+        tuple[str, bool]: (diff output with inline word-level highlighting, has_changes flag)
+    """
+    # Normalize whitespace if ignore_junk is enabled
+    if ignore_junk:
+        # Normalize whitespace: replace multiple spaces/tabs with single space
+        before_normalized = WHITESPACE_NORMALIZE_RE.sub(' ', before_line)
+        after_normalized = WHITESPACE_NORMALIZE_RE.sub(' ', after_line)
+    else:
+        before_normalized = before_line
+        after_normalized = after_line
+
+    # Use diff-match-patch with word-level tokenization
+    # Strategy: Use linesToChars to treat words as atomic units
+    dmp = dmp_module.diff_match_patch()
+
+    # Get the tokenizer function from the registry
+    tokenizer_func = TOKENIZERS.get(tokenizer, tokenize_words_and_html)
+
+    # Tokenize both lines using the selected tokenizer
+    before_tokens = tokenizer_func(before_normalized)
+    after_tokens = tokenizer_func(after_normalized or ' ')
+
+    # Create mappings for linesToChars (using it for word-mode)
+    # Join tokens with newline so each "line" is a token
+    before_text = '\n'.join(before_tokens)
+    after_text = '\n'.join(after_tokens)
+
+    # Use linesToChars for word-mode diffing
+    lines_result = dmp.diff_linesToChars(before_text, after_text)
+    line_before, line_after, line_array = lines_result
+
+    # Perform diff on the encoded strings
+    diffs = dmp.diff_main(line_before, line_after, False)
+
+    # Convert back to original text
+    dmp.diff_charsToLines(diffs, line_array)
+
+    # Remove the newlines we added for tokenization
+    diffs = [(op, text.replace('\n', '')) for op, text in diffs]
+
+    # DON'T apply semantic cleanup here - it would break token boundaries
+    # (e.g., "63" -> "66" would become "6" + "3" vs "6" + "6")
+    # We want to preserve the tokenizer's word boundaries
+
+    # Check if there are any changes
+    has_changes = any(op != 0 for op, _ in diffs)
+
+    if ignore_junk and not has_changes:
+        return after_line, False
+
+    # Check if the whole line is replaced (no unchanged content)
+    whole_line_replaced = not any(op == 0 and text.strip() for op, text in diffs)
+
+    # Build the output using placemarkers
+    # When whole line is replaced, wrap entire removed content once and entire added content once
+    if whole_line_replaced:
+        removed_tokens = []
+        added_tokens = []
+
+        for op, text in diffs:
+            if op == 0:  # Equal (e.g., whitespace tokens in common positions)
+                # Include in both removed and added to preserve spacing
+                removed_tokens.append(text)
+                added_tokens.append(text)
+            elif op == -1:  # Deletion
+                removed_tokens.append(text)
+            elif op == 1:  # Insertion
+                added_tokens.append(text)
+
+        # Join all tokens and wrap the entire string once for removed, once for added
+        result_parts = []
+
+        if removed_tokens:
+            removed_full = ''.join(removed_tokens).rstrip()
+            trailing_removed = ''.join(removed_tokens)[len(removed_full):] if len(''.join(removed_tokens)) > len(removed_full) else ''
+            result_parts.append(f'{CHANGED_PLACEMARKER_OPEN}{removed_full}{CHANGED_PLACEMARKER_CLOSED}{trailing_removed}')
+
+        if added_tokens:
+            if result_parts:  # Add newline between removed and added
+                result_parts.append('\n')
+            added_full = ''.join(added_tokens).rstrip()
+            trailing_added = ''.join(added_tokens)[len(added_full):] if len(''.join(added_tokens)) > len(added_full) else ''
+            result_parts.append(f'{CHANGED_INTO_PLACEMARKER_OPEN}{added_full}{CHANGED_INTO_PLACEMARKER_CLOSED}{trailing_added}')
+
+        return ''.join(result_parts), has_changes
+    else:
+        # Inline changes within the line
+        result_parts = []
+        for op, text in diffs:
+            if op == 0:  # Equal
+                result_parts.append(text)
+            elif op == 1:  # Insertion
+                # Don't wrap empty content (e.g., whitespace-only tokens after rstrip)
+                content = text.rstrip()
+                trailing = text[len(content):] if len(text) > len(content) else ''
+                if content:
+                    result_parts.append(f'{ADDED_PLACEMARKER_OPEN}{content}{ADDED_PLACEMARKER_CLOSED}{trailing}')
+                else:
+                    result_parts.append(trailing)
+            elif op == -1:  # Deletion
+                # Don't wrap empty content (e.g., whitespace-only tokens after rstrip)
+                content = text.rstrip()
+                trailing = text[len(content):] if len(text) > len(content) else ''
+                if content:
+                    result_parts.append(f'{REMOVED_PLACEMARKER_OPEN}{content}{REMOVED_PLACEMARKER_CLOSED}{trailing}')
+                else:
+                    result_parts.append(trailing)
+
+        return ''.join(result_parts), has_changes
+
+
+def render_nested_line_diff(before_line: str, after_line: str, ignore_junk: bool = False, tokenizer: str = 'words_and_html') -> tuple[str, str, bool]:
+    """
+    Render line-level differences with nested highlighting for changed parts.
+
+    Returns two separate lines:
+    - Before line: light red background with dark red on removed parts
+    - After line: light green background with dark green on added parts
+
+    Args:
+        before_line: Original line text
+        after_line: Modified line text
+        ignore_junk: Ignore whitespace-only changes
+        tokenizer: Name of tokenizer to use from TOKENIZERS registry
+
+    Returns:
+        tuple[str, str, bool]: (before_with_highlights, after_with_highlights, has_changes)
+    """
+    # Normalize whitespace if ignore_junk is enabled
+    if ignore_junk:
+        before_normalized = WHITESPACE_NORMALIZE_RE.sub(' ', before_line)
+        after_normalized = WHITESPACE_NORMALIZE_RE.sub(' ', after_line)
+    else:
+        before_normalized = before_line
+        after_normalized = after_line
+
+    # Use diff-match-patch with word-level tokenization
+    dmp = dmp_module.diff_match_patch()
+
+    # Get the tokenizer function from the registry
+    tokenizer_func = TOKENIZERS.get(tokenizer, tokenize_words_and_html)
+
+    # Tokenize both lines
+    before_tokens = tokenizer_func(before_normalized)
+    after_tokens = tokenizer_func(after_normalized or ' ')
+
+    # Create mappings for linesToChars
+    before_text = '\n'.join(before_tokens)
+    after_text = '\n'.join(after_tokens)
+
+    # Use linesToChars for word-mode diffing
+    lines_result = dmp.diff_linesToChars(before_text, after_text)
+    line_before, line_after, line_array = lines_result
+
+    # Perform diff on the encoded strings
+    diffs = dmp.diff_main(line_before, line_after, False)
+
+    # Convert back to original text
+    dmp.diff_charsToLines(diffs, line_array)
+
+    # Remove the newlines we added for tokenization
+    diffs = [(op, text.replace('\n', '')) for op, text in diffs]
+
+    # DON'T apply semantic cleanup here - it would break token boundaries
+    # (e.g., "63" -> "66" would become "6" + "3" vs "6" + "6")
+    # We want to preserve the tokenizer's word boundaries
+
+    # Check if there are any changes
+    has_changes = any(op != 0 for op, _ in diffs)
+
+    if ignore_junk and not has_changes:
+        return before_line, after_line, False
+
+    # Build the before line (with nested highlighting for removed parts)
+    before_parts = []
+    for op, text in diffs:
+        if op == 0:  # Equal
+            before_parts.append(text)
+        elif op == -1:  # Deletion (in before)
+            before_parts.append(f'<span style="{REMOVED_INNER_STYLE}">{text}</span>')
+        # Skip insertions (op == 1) for the before line
+
+    before_content = ''.join(before_parts)
+
+    # Build the after line (with nested highlighting for added parts)
+    after_parts = []
+    for op, text in diffs:
+        if op == 0:  # Equal
+            after_parts.append(text)
+        elif op == 1:  # Insertion (in after)
+            after_parts.append(f'<span style="{ADDED_INNER_STYLE}">{text}</span>')
+        # Skip deletions (op == -1) for the after line
+
+    after_content = ''.join(after_parts)
+
+    # Wrap content with placemarkers (inner HTML highlighting is preserved)
+    before_html = f'{CHANGED_PLACEMARKER_OPEN}{before_content}{CHANGED_PLACEMARKER_CLOSED}'
+    after_html = f'{CHANGED_INTO_PLACEMARKER_OPEN}{after_content}{CHANGED_INTO_PLACEMARKER_CLOSED}'
+
+    return before_html, after_html, has_changes
+
+
+def same_slicer(lst: List[str], start: int, end: int) -> List[str]:
+    """Return a slice of the list, or a single element if start == end."""
+    return lst[start:end] if start != end else [lst[start]]
+
+def customSequenceMatcher(
+    before: List[str],
+    after: List[str],
+    include_equal: bool = False,
+    include_removed: bool = True,
+    include_added: bool = True,
+    include_replaced: bool = True,
+    include_change_type_prefix: bool = True,
+    word_diff: bool = False,
+    context_lines: int = 0,
+    case_insensitive: bool = False,
+    ignore_junk: bool = False,
+    tokenizer: str = 'words_and_html'
+) -> Iterator[List[str]]:
+    """
+    Compare two sequences and yield differences based on specified parameters.
+
+    Args:
+        before (List[str]): Original sequence
+        after (List[str]): Modified sequence
+        include_equal (bool): Include unchanged parts
+        include_removed (bool): Include removed parts
+        include_added (bool): Include added parts
+        include_replaced (bool): Include replaced parts
+        include_change_type_prefix (bool): Add prefixes to indicate change types
+        word_diff (bool): Use word-level diffing for replaced lines (controls inline rendering)
+        context_lines (int): Number of unchanged lines to show around changes (like grep -C)
+        case_insensitive (bool): Perform case-insensitive comparison
+        ignore_junk (bool): Ignore whitespace-only changes
+        tokenizer (str): Name of tokenizer to use from TOKENIZERS registry (default: 'words_and_html')
+
+    Yields:
+        List[str]: Differences between sequences
+    """
+    # Prepare sequences for comparison (lowercase if case-insensitive, normalize whitespace if ignore_junk)
+    def prepare_line(line):
+        if case_insensitive:
+            line = line.lower()
+        if ignore_junk:
+            # Normalize whitespace: replace multiple spaces/tabs with single space
+            line = WHITESPACE_NORMALIZE_RE.sub(' ', line)
+        return line
+
+    compare_before = [prepare_line(line) for line in before]
+    compare_after = [prepare_line(line) for line in after]
+
+    cruncher = difflib.SequenceMatcher(isjunk=lambda x: x in " \t", a=compare_before, b=compare_after)
+
+    # When context_lines is set and include_equal is False, we need to track which equal lines to include
+    if context_lines > 0 and not include_equal:
+        opcodes = list(cruncher.get_opcodes())
+        # Mark equal ranges that should be included based on context
+        included_equal_ranges = set()
+
+        for i, (tag, alo, ahi, blo, bhi) in enumerate(opcodes):
+            if tag != 'equal':
+                # Include context lines before this change
+                for j in range(max(0, i - 1), i):
+                    if opcodes[j][0] == 'equal':
+                        prev_alo, prev_ahi = opcodes[j][1], opcodes[j][2]
+                        # Include last N lines of the previous equal block
+                        context_start = max(prev_alo, prev_ahi - context_lines)
+                        for line_num in range(context_start, prev_ahi):
+                            included_equal_ranges.add(line_num)
+
+                # Include context lines after this change
+                for j in range(i + 1, min(len(opcodes), i + 2)):
+                    if opcodes[j][0] == 'equal':
+                        next_alo, next_ahi = opcodes[j][1], opcodes[j][2]
+                        # Include first N lines of the next equal block
+                        context_end = min(next_ahi, next_alo + context_lines)
+                        for line_num in range(next_alo, context_end):
+                            included_equal_ranges.add(line_num)
+
+    # Remember! gmail, outlook etc dont support <style> must be inline.
+    # Gmail: strips <ins> and <del> tags entirely.
+    for tag, alo, ahi, blo, bhi in cruncher.get_opcodes():
+        if tag == 'equal':
+            if include_equal:
+                yield before[alo:ahi]
+            elif context_lines > 0:
+                # Only include equal lines that are in the context range
+                context_lines_to_include = [before[i] for i in range(alo, ahi) if i in included_equal_ranges]
+                if context_lines_to_include:
+                    yield context_lines_to_include
+        elif include_removed and tag == 'delete':
+            if include_change_type_prefix:
+                yield [f'{REMOVED_PLACEMARKER_OPEN}{line}{REMOVED_PLACEMARKER_CLOSED}' for line in same_slicer(before, alo, ahi)]
+            else:
+                yield same_slicer(before, alo, ahi)
+        elif include_replaced and tag == 'replace':
+            before_lines = same_slicer(before, alo, ahi)
+            after_lines = same_slicer(after, blo, bhi)
+
+            # Use inline word-level diff for single line replacements when word_diff is enabled
+            if word_diff and len(before_lines) == 1 and len(after_lines) == 1:
+                inline_diff, has_changes = render_inline_word_diff(before_lines[0], after_lines[0], ignore_junk=ignore_junk, tokenizer=tokenizer)
+                # Check if there are any actual changes (not just whitespace when ignore_junk is enabled)
+                if ignore_junk and not has_changes:
+                    # No real changes, skip this line
+                    continue
+                yield [inline_diff]
+            else:
+                # Fall back to line-level diff for multi-line changes
+                if include_change_type_prefix:
+                    yield [f'{CHANGED_PLACEMARKER_OPEN}{line}{CHANGED_PLACEMARKER_CLOSED}' for line in before_lines] + \
+                          [f'{CHANGED_INTO_PLACEMARKER_OPEN}{line}{CHANGED_INTO_PLACEMARKER_CLOSED}' for line in after_lines]
+                else:
+                    yield before_lines + after_lines
+        elif include_added and tag == 'insert':
+            if include_change_type_prefix:
+                yield [f'{ADDED_PLACEMARKER_OPEN}{line}{ADDED_PLACEMARKER_CLOSED}' for line in same_slicer(after, blo, bhi)]
+            else:
+                yield same_slicer(after, blo, bhi)
+
+def render_diff(
+    previous_version_file_contents: str,
+    newest_version_file_contents: str,
+    include_equal: bool = False,
+    include_removed: bool = True,
+    include_added: bool = True,
+    include_replaced: bool = True,
+    include_change_type_prefix: bool = True,
+    patch_format: bool = False,
+    word_diff: bool = True,
+    context_lines: int = 0,
+    case_insensitive: bool = False,
+    ignore_junk: bool = False,
+    tokenizer: str = 'words_and_html'
+) -> str:
+    """
+    Render the difference between two file contents.
+
+    Args:
+        previous_version_file_contents (str): Original file contents
+        newest_version_file_contents (str): Modified file contents
+        include_equal (bool): Include unchanged parts
+        include_removed (bool): Include removed parts
+        include_added (bool): Include added parts
+        include_replaced (bool): Include replaced parts
+        include_change_type_prefix (bool): Add prefixes to indicate change types
+        patch_format (bool): Use patch format for output
+        word_diff (bool): Use word-level diffing for replaced lines (controls inline rendering)
+        context_lines (int): Number of unchanged lines to show around changes (like grep -C)
+        case_insensitive (bool): Perform case-insensitive comparison, By default the test_json_diff/process.py is case sensitive, so this follows same logic
+        ignore_junk (bool): Ignore whitespace-only changes
+        tokenizer (str): Name of tokenizer to use from TOKENIZERS registry (default: 'words_and_html')
+
+    Returns:
+        str: Rendered difference
+    """
+    newest_lines = [line.rstrip() for line in newest_version_file_contents.splitlines()]
+    previous_lines = [line.rstrip() for line in previous_version_file_contents.splitlines()] if previous_version_file_contents else []
+
+    if patch_format:
+        patch = difflib.unified_diff(previous_lines, newest_lines)
+        return CUSTOM_LINEBREAK_PLACEHOLDER.join(patch)
+
+    rendered_diff = customSequenceMatcher(
+        before=previous_lines,
+        after=newest_lines,
+        include_equal=include_equal,
+        include_removed=include_removed,
+        include_added=include_added,
+        include_replaced=include_replaced,
+        include_change_type_prefix=include_change_type_prefix,
+        word_diff=word_diff,
+        context_lines=context_lines,
+        case_insensitive=case_insensitive,
+        ignore_junk=ignore_junk,
+        tokenizer=tokenizer
+    )
+
+    def flatten(lst: List[Union[str, List[str]]]) -> str:
+        result = []
+        for x in lst:
+            if isinstance(x, list):
+                result.extend(x)
+            else:
+                result.append(x)
+        return CUSTOM_LINEBREAK_PLACEHOLDER.join(result)
+
+    return flatten(rendered_diff)
+
+
+# Export main public API
+__all__ = [
+    'render_diff',
+    'customSequenceMatcher',
+    'render_inline_word_diff',
+    'render_nested_line_diff',
+    'TOKENIZERS',
+    'REMOVED_STYLE',
+    'ADDED_STYLE',
+    'REMOVED_INNER_STYLE',
+    'ADDED_INNER_STYLE',
+]

changedetectionio/diff/tokenizers/__init__.py

@@ -0,0 +1,23 @@
+"""
+Tokenizers for diff operations.
+
+This module provides various tokenization strategies for use with the diff system.
+New tokenizers can be easily added by:
+1. Creating a new module in this directory
+2. Importing and registering it in the TOKENIZERS dictionary below
+"""
+
+from .natural_text import tokenize_words
+from .words_and_html import tokenize_words_and_html
+
+# Tokenizer registry - maps tokenizer names to functions
+TOKENIZERS = {
+    'words': tokenize_words,
+    'words_and_html': tokenize_words_and_html,
+}
+
+__all__ = [
+    'tokenize_words',
+    'tokenize_words_and_html',
+    'TOKENIZERS',
+]

changedetectionio/diff/tokenizers/natural_text.py

@@ -0,0 +1,44 @@
+"""
+Simple word tokenizer using whitespace boundaries.
+
+This is a simpler tokenizer that treats all whitespace as token boundaries
+without special handling for HTML tags or other markup.
+"""
+
+from typing import List
+
+
+def tokenize_words(text: str) -> List[str]:
+    """
+    Split text into words using simple whitespace boundaries.
+
+    This is a simpler tokenizer that treats all whitespace as token boundaries
+    without special handling for HTML tags.
+
+    Args:
+        text: Input text to tokenize
+
+    Returns:
+        List of tokens (words and whitespace)
+
+    Examples:
+        >>> tokenize_words("Hello world")
+        ['Hello', ' ', 'world']
+        >>> tokenize_words("one  two")
+        ['one', ' ', ' ', 'two']
+    """
+    tokens = []
+    current = ''
+
+    for char in text:
+        if char.isspace():
+            if current:
+                tokens.append(current)
+                current = ''
+            tokens.append(char)
+        else:
+            current += char
+
+    if current:
+        tokens.append(current)
+    return tokens

changedetectionio/diff/tokenizers/words_and_html.py

@@ -0,0 +1,61 @@
+"""
+Tokenizer that preserves HTML tags as atomic units while splitting on whitespace.
+
+This tokenizer is specifically designed for HTML content where:
+- HTML tags should remain intact (e.g., '<p>', '<a href="...">')
+- Whitespace tokens are preserved for accurate diff reconstruction
+- Words are split on whitespace boundaries
+"""
+
+from typing import List
+
+
+def tokenize_words_and_html(text: str) -> List[str]:
+    """
+    Split text into words and boundaries (spaces, HTML tags).
+
+    This tokenizer preserves HTML tags as atomic units while splitting on whitespace.
+    Useful for content that contains HTML markup.
+
+    Args:
+        text: Input text to tokenize
+
+    Returns:
+        List of tokens (words, spaces, HTML tags)
+
+    Examples:
+        >>> tokenize_words_and_html("<p>Hello world</p>")
+        ['<p>', 'Hello', ' ', 'world', '</p>']
+        >>> tokenize_words_and_html("<a href='test.com'>link</a>")
+        ['<a href=\\'test.com\\'>', 'link', '</a>']
+    """
+    tokens = []
+    current = ''
+    in_tag = False
+
+    for char in text:
+        if char == '<':
+            # Start of HTML tag
+            if current:
+                tokens.append(current)
+                current = ''
+            current = '<'
+            in_tag = True
+        elif char == '>' and in_tag:
+            # End of HTML tag
+            current += '>'
+            tokens.append(current)
+            current = ''
+            in_tag = False
+        elif char.isspace() and not in_tag:
+            # Space outside of tag
+            if current:
+                tokens.append(current)
+                current = ''
+            tokens.append(char)
+        else:
+            current += char
+
+    if current:
+        tokens.append(current)
+    return tokens