Ensure JSON is always nicely formatted

.github/actions/extract-memory-report/action.yml

@@ -1,51 +0,0 @@
-name: 'Extract Memory Test Report'
-description: 'Extracts and displays memory test report from a container'
-inputs:
-  container-name:
-    description: 'Name of the container to extract logs from'
-    required: true
-  python-version:
-    description: 'Python version for artifact naming'
-    required: true
-  output-dir:
-    description: 'Directory to store output logs'
-    required: false
-    default: 'output-logs'
-
-runs:
-  using: "composite"
-  steps:
-    - name: Create output directory
-      shell: bash
-      run: |
-        mkdir -p ${{ inputs.output-dir }}
-
-    - name: Dump container log
-      shell: bash
-      run: |
-        echo "Disabled for now"
-#        return
-#        docker logs ${{ inputs.container-name }} > ${{ inputs.output-dir }}/${{ inputs.container-name }}-stdout-${{ inputs.python-version }}.txt 2>&1 || echo "Could not get stdout"
-#        docker logs ${{ inputs.container-name }} 2> ${{ inputs.output-dir }}/${{ inputs.container-name }}-stderr-${{ inputs.python-version }}.txt || echo "Could not get stderr"
-
-    - name: Extract and display memory test report
-      shell: bash
-      run: |
-        echo "Disabled for now"
-#        echo "Extracting test-memory.log from container..."
-#        docker cp ${{ inputs.container-name }}:/app/changedetectionio/test-memory.log ${{ inputs.output-dir }}/test-memory-${{ inputs.python-version }}.log || echo "test-memory.log not found in container"
-#
-#        echo "=== Top 10 Highest Peak Memory Tests ==="
-#        if [ -f ${{ inputs.output-dir }}/test-memory-${{ inputs.python-version }}.log ]; then
-#          grep "Peak memory:" ${{ inputs.output-dir }}/test-memory-${{ inputs.python-version }}.log | \
-#            sed 's/.*Peak memory: //' | \
-#            paste -d'|' - <(grep "Peak memory:" ${{ inputs.output-dir }}/test-memory-${{ inputs.python-version }}.log) | \
-#            sort -t'|' -k1 -nr | \
-#            cut -d'|' -f2 | \
-#            head -10
-#          echo ""
-#          echo "=== Full Memory Test Report ==="
-#          cat ${{ inputs.output-dir }}/test-memory-${{ inputs.python-version }}.log
-#        else
-#          echo "No memory log available"
-#        fi

.github/workflows/containers.yml

@@ -45,14 +45,6 @@ jobs:
         with:
           python-version: 3.11
 
-      - name: Cache pip packages
-        uses: actions/cache@v4
-        with:
-          path: ~/.cache/pip
-          key: ${{ runner.os }}-pip-${{ hashFiles('requirements.txt') }}
-          restore-keys: |
-            ${{ runner.os }}-pip-
-
       - name: Install dependencies
         run: |
           python -m pip install --upgrade pip

.github/workflows/pypi-release.yml

@@ -21,20 +21,20 @@ jobs:
     - name: Build a binary wheel and a source tarball
       run: python3 -m build
     - name: Store the distribution packages
-      uses: actions/upload-artifact@v5
+      uses: actions/upload-artifact@v4
       with:
         name: python-package-distributions
         path: dist/
 
 
   test-pypi-package:
-    name: Test the built package works basically.
+    name: Test the built 📦 package works basically.
     runs-on: ubuntu-latest
     needs:
     - build
     steps:
     - name: Download all the dists
-      uses: actions/download-artifact@v6
+      uses: actions/download-artifact@v5
       with:
         name: python-package-distributions
         path: dist/
@@ -42,39 +42,18 @@ jobs:
       uses: actions/setup-python@v6
       with:
         python-version: '3.11'
-
     - name: Test that the basic pip built package runs without error
       run: |
         set -ex
         ls -alR 
         
-        # Install the first wheel found in dist/
-        WHEEL=$(find dist -type f -name "*.whl" -print -quit)
-        echo Installing $WHEEL
-        python3 -m pip install --upgrade pip
-        python3 -m pip install "$WHEEL"
+        # Find and install the first .whl file
+        find dist -type f -name "*.whl" -exec pip3 install {} \; -quit
         changedetection.io -d /tmp -p 10000 &
         
         sleep 3
         curl --retry-connrefused --retry 6 http://127.0.0.1:10000/static/styles/pure-min.css >/dev/null
         curl --retry-connrefused --retry 6 http://127.0.0.1:10000/ >/dev/null
-        
-        # --- API test ---
-        # This also means that the docs/api-spec.yml was shipped and could be read
-        test -f /tmp/url-watches.json
-        API_KEY=$(jq -r '.. | .api_access_token? // empty' /tmp/url-watches.json)
-        echo Test API KEY is $API_KEY
-        curl -X POST "http://127.0.0.1:10000/api/v1/watch" \
-          -H "x-api-key: ${API_KEY}" \
-          -H "Content-Type: application/json" \
-          --show-error --fail \
-          --retry 6 --retry-delay 1 --retry-connrefused \
-          -d '{
-            "url": "https://example.com",
-            "title": "Example Site Monitor",
-            "time_between_check": { "hours": 1 }
-          }'
-          
         killall changedetection.io
 
 
@@ -93,7 +72,7 @@ jobs:
 
     steps:
     - name: Download all the dists
-      uses: actions/download-artifact@v6
+      uses: actions/download-artifact@v5
       with:
         name: python-package-distributions
         path: dist/

.github/workflows/test-container-build.yml

@@ -50,14 +50,6 @@ jobs:
           with:
             python-version: 3.11
 
-        - name: Cache pip packages
-          uses: actions/cache@v4
-          with:
-            path: ~/.cache/pip
-            key: ${{ runner.os }}-pip-${{ hashFiles('requirements.txt') }}
-            restore-keys: |
-              ${{ runner.os }}-pip-
-
         # Just test that the build works, some libraries won't compile on ARM/rPi etc
         - name: Set up QEMU
           uses: docker/setup-qemu-action@v3
@@ -82,5 +74,5 @@ jobs:
             file: ${{ matrix.dockerfile }}
             platforms: ${{ matrix.platform }}
             cache-from: type=gha
-            cache-to: type=gha,mode=max
+            cache-to: type=gha,mode=min
 

.github/workflows/test-only.yml

@@ -21,8 +21,6 @@ jobs:
           python3 -c "from openapi_spec_validator import validate_spec; import yaml; validate_spec(yaml.safe_load(open('docs/api-spec.yaml')))"
 
   test-application-3-10:
-    # Only run on push to master (including PR merges)
-    if: github.event_name == 'push' && github.ref == 'refs/heads/master'
     needs: lint-code
     uses: ./.github/workflows/test-stack-reusable-workflow.yml
     with:
@@ -30,15 +28,12 @@ jobs:
 
 
   test-application-3-11:
-    # Always run
     needs: lint-code
     uses: ./.github/workflows/test-stack-reusable-workflow.yml
     with:
       python-version: '3.11'
 
   test-application-3-12:
-    # Only run on push to master (including PR merges)
-    if: github.event_name == 'push' && github.ref == 'refs/heads/master'
     needs: lint-code
     uses: ./.github/workflows/test-stack-reusable-workflow.yml
     with:
@@ -46,8 +41,6 @@ jobs:
       skip-pypuppeteer: true
 
   test-application-3-13:
-    # Only run on push to master (including PR merges)
-    if: github.event_name == 'push' && github.ref == 'refs/heads/master'
     needs: lint-code
     uses: ./.github/workflows/test-stack-reusable-workflow.yml
     with:

.github/workflows/test-stack-reusable-workflow.yml

@@ -15,294 +15,138 @@ on:
         default: false
 
 jobs:
-  # Build the Docker image once and share it with all test jobs
-  build:
+  test-application:
     runs-on: ubuntu-latest
     env:
       PYTHON_VERSION: ${{ inputs.python-version }}
     steps:
       - uses: actions/checkout@v5
 
+      # Mainly just for link/flake8
       - name: Set up Python ${{ env.PYTHON_VERSION }}
         uses: actions/setup-python@v6
         with:
           python-version: ${{ env.PYTHON_VERSION }}
 
-      - name: Cache pip packages
-        uses: actions/cache@v4
-        with:
-          path: ~/.cache/pip
-          key: ${{ runner.os }}-pip-py${{ env.PYTHON_VERSION }}-${{ hashFiles('requirements.txt') }}
-          restore-keys: |
-            ${{ runner.os }}-pip-py${{ env.PYTHON_VERSION }}-
-            ${{ runner.os }}-pip-
-
       - name: Build changedetection.io container for testing under Python ${{ env.PYTHON_VERSION }}
         run: |
           echo "---- Building for Python ${{ env.PYTHON_VERSION }} -----"
+          # Build a changedetection.io container and start testing inside
           docker build --build-arg PYTHON_VERSION=${{ env.PYTHON_VERSION }} --build-arg LOGGER_LEVEL=TRACE -t test-changedetectionio .
-          docker run test-changedetectionio bash -c 'pip list'
+          # Debug info
+          docker run test-changedetectionio  bash -c 'pip list'         
 
       - name: We should be Python ${{ env.PYTHON_VERSION }} ...
+        run: |         
+          docker run test-changedetectionio  bash -c 'python3 --version'
+
+      - name: Spin up ancillary testable services
         run: |
-          docker run test-changedetectionio bash -c 'python3 --version'
+          
+          docker network create changedet-network
+          
+          # Selenium
+          docker run --network changedet-network -d --hostname selenium  -p 4444:4444 --rm --shm-size="2g"  selenium/standalone-chrome:4
+          
+          # SocketPuppetBrowser + Extra for custom browser test
+          docker run --network changedet-network -d -e "LOG_LEVEL=TRACE" --cap-add=SYS_ADMIN --name sockpuppetbrowser --hostname sockpuppetbrowser --rm -p 3000:3000 dgtlmoon/sockpuppetbrowser:latest                    
+          docker run --network changedet-network -d -e "LOG_LEVEL=TRACE" --cap-add=SYS_ADMIN --name sockpuppetbrowser-custom-url --hostname sockpuppetbrowser-custom-url  -p 3001:3000 --rm dgtlmoon/sockpuppetbrowser:latest
 
-      - name: Save Docker image
+      - name: Spin up ancillary SMTP+Echo message test server
         run: |
-          docker save test-changedetectionio -o /tmp/test-changedetectionio.tar
+          # Debug SMTP server/echo message back server
+          docker run --network changedet-network -d -p 11025:11025 -p 11080:11080  --hostname mailserver test-changedetectionio  bash -c 'pip3 install aiosmtpd && python changedetectionio/tests/smtp/smtp-test-server.py'
+          docker ps
 
-      - name: Upload Docker image artifact
-        uses: actions/upload-artifact@v5
-        with:
-          name: test-changedetectionio-${{ env.PYTHON_VERSION }}
-          path: /tmp/test-changedetectionio.tar
-          retention-days: 1
-
-  # Unit tests (lightweight, no ancillary services needed)
-  unit-tests:
-    runs-on: ubuntu-latest
-    needs: build
-    timeout-minutes: 10
-    env:
-      PYTHON_VERSION: ${{ inputs.python-version }}
-    steps:
-      - uses: actions/checkout@v5
-
-      - name: Download Docker image artifact
-        uses: actions/download-artifact@v6
-        with:
-          name: test-changedetectionio-${{ env.PYTHON_VERSION }}
-          path: /tmp
-
-      - name: Load Docker image
+      - name: Show docker container state and other debug info
         run: |
-          docker load -i /tmp/test-changedetectionio.tar
+          set -x
+          echo "Running processes in docker..."
+          docker ps
 
       - name: Run Unit Tests
         run: |
-          docker run test-changedetectionio bash -c 'python3 -m unittest changedetectionio.tests.unit.test_notification_diff'
-          docker run test-changedetectionio bash -c 'python3 -m unittest changedetectionio.tests.unit.test_watch_model'
-          docker run test-changedetectionio bash -c 'python3 -m unittest changedetectionio.tests.unit.test_jinja2_security'
-          docker run test-changedetectionio bash -c 'python3 -m unittest changedetectionio.tests.unit.test_semver'
+          # Unit tests
+          docker run test-changedetectionio  bash -c 'python3 -m unittest changedetectionio.tests.unit.test_notification_diff'
+          docker run test-changedetectionio  bash -c 'python3 -m unittest changedetectionio.tests.unit.test_watch_model'
+          docker run test-changedetectionio  bash -c 'python3 -m unittest changedetectionio.tests.unit.test_jinja2_security'
+          docker run test-changedetectionio  bash -c 'python3 -m unittest changedetectionio.tests.unit.test_semver'
 
-  # Basic pytest tests with ancillary services
-  basic-tests:
-    runs-on: ubuntu-latest
-    needs: build
-    timeout-minutes: 25
-    env:
-      PYTHON_VERSION: ${{ inputs.python-version }}
-    steps:
-      - uses: actions/checkout@v5
-
-      - name: Download Docker image artifact
-        uses: actions/download-artifact@v6
-        with:
-          name: test-changedetectionio-${{ env.PYTHON_VERSION }}
-          path: /tmp
-
-      - name: Load Docker image
+      - name: Test built container with Pytest (generally as requests/plaintext fetching)
         run: |
-          docker load -i /tmp/test-changedetectionio.tar
+          # All tests
+          echo "run test with pytest"
+          # The default pytest logger_level is TRACE
+          # To change logger_level for pytest(test/conftest.py),
+          # append the docker option. e.g. '-e LOGGER_LEVEL=DEBUG'
+          docker run --name test-cdio-basic-tests --network changedet-network  test-changedetectionio  bash -c 'cd changedetectionio && ./run_basic_tests.sh'
 
-      - name: Test built container with Pytest
+# PLAYWRIGHT/NODE-> CDP
+      - name: Playwright and SocketPuppetBrowser - Specific tests in built container
         run: |
-          docker network inspect changedet-network >/dev/null 2>&1 || docker network create changedet-network
-          docker run --name test-cdio-basic-tests --network changedet-network test-changedetectionio bash -c 'cd changedetectionio && ./run_basic_tests.sh'
+          # Playwright via Sockpuppetbrowser fetch
+          # tests/visualselector/test_fetch_data.py will do browser steps  
+          docker run --rm -e "FLASK_SERVER_NAME=cdio" -e "PLAYWRIGHT_DRIVER_URL=ws://sockpuppetbrowser:3000" --network changedet-network --hostname=cdio test-changedetectionio  bash -c 'cd changedetectionio;pytest  -vv --capture=tee-sys --showlocals --tb=long --live-server-host=0.0.0.0 --live-server-port=5004 tests/fetchers/test_content.py'
+          docker run --rm -e "FLASK_SERVER_NAME=cdio" -e "PLAYWRIGHT_DRIVER_URL=ws://sockpuppetbrowser:3000" --network changedet-network --hostname=cdio test-changedetectionio  bash -c 'cd changedetectionio;pytest  -vv --capture=tee-sys --showlocals --tb=long --live-server-host=0.0.0.0 --live-server-port=5004 tests/test_errorhandling.py'
+          docker run --rm -e "FLASK_SERVER_NAME=cdio" -e "PLAYWRIGHT_DRIVER_URL=ws://sockpuppetbrowser:3000" --network changedet-network --hostname=cdio test-changedetectionio  bash -c 'cd changedetectionio;pytest  -vv --capture=tee-sys --showlocals --tb=long --live-server-host=0.0.0.0 --live-server-port=5004 tests/visualselector/test_fetch_data.py'
+          docker run --rm -e "FLASK_SERVER_NAME=cdio" -e "PLAYWRIGHT_DRIVER_URL=ws://sockpuppetbrowser:3000" --network changedet-network --hostname=cdio test-changedetectionio  bash -c 'cd changedetectionio;pytest  -vv --capture=tee-sys --showlocals --tb=long --live-server-host=0.0.0.0 --live-server-port=5004 tests/fetchers/test_custom_js_before_content.py'
 
-      - name: Extract memory report and logs
-        if: always()
-        uses: ./.github/actions/extract-memory-report
-        with:
-          container-name: test-cdio-basic-tests
-          python-version: ${{ env.PYTHON_VERSION }}
 
-      - name: Store test artifacts
-        if: always()
-        uses: actions/upload-artifact@v5
-        with:
-          name: test-cdio-basic-tests-output-py${{ env.PYTHON_VERSION }}
-          path: output-logs
+      - name: Playwright and SocketPuppetBrowser - Headers and requests
+        run: |       
+          # Settings headers playwright tests - Call back in from Sockpuppetbrowser, check headers
+          docker run --name "changedet" --hostname changedet --rm -e "FLASK_SERVER_NAME=changedet" -e "PLAYWRIGHT_DRIVER_URL=ws://sockpuppetbrowser:3000?dumpio=true" --network changedet-network test-changedetectionio  bash -c 'find .; cd changedetectionio; pytest --live-server-host=0.0.0.0  --live-server-port=5004 tests/test_request.py; pwd;find .'
 
-  # Playwright tests
-  playwright-tests:
-    runs-on: ubuntu-latest
-    needs: build
-    timeout-minutes: 10
-    env:
-      PYTHON_VERSION: ${{ inputs.python-version }}
-    steps:
-      - uses: actions/checkout@v5
+      - name: Playwright and SocketPuppetBrowser - Restock detection
+        run: |                            
+          # restock detection via playwright - added name=changedet here so that playwright and sockpuppetbrowser can connect to it
+          docker run --rm --name "changedet" -e "FLASK_SERVER_NAME=changedet" -e "PLAYWRIGHT_DRIVER_URL=ws://sockpuppetbrowser:3000" --network changedet-network test-changedetectionio  bash -c 'cd changedetectionio;pytest --live-server-port=5004 --live-server-host=0.0.0.0 tests/restock/test_restock.py'
 
-      - name: Download Docker image artifact
-        uses: actions/download-artifact@v6
-        with:
-          name: test-changedetectionio-${{ env.PYTHON_VERSION }}
-          path: /tmp
-
-      - name: Load Docker image
+# STRAIGHT TO CDP
+      - name: Pyppeteer and SocketPuppetBrowser - Specific tests in built container
+        if: ${{ inputs.skip-pypuppeteer == false }}
         run: |
-          docker load -i /tmp/test-changedetectionio.tar
+          # Playwright via Sockpuppetbrowser fetch 
+          docker run --rm -e "FLASK_SERVER_NAME=cdio" -e "FAST_PUPPETEER_CHROME_FETCHER=True" -e "PLAYWRIGHT_DRIVER_URL=ws://sockpuppetbrowser:3000" --network changedet-network --hostname=cdio test-changedetectionio  bash -c 'cd changedetectionio;pytest --live-server-host=0.0.0.0 --live-server-port=5004 tests/fetchers/test_content.py'
+          docker run --rm -e "FLASK_SERVER_NAME=cdio" -e "FAST_PUPPETEER_CHROME_FETCHER=True" -e "PLAYWRIGHT_DRIVER_URL=ws://sockpuppetbrowser:3000" --network changedet-network --hostname=cdio test-changedetectionio  bash -c 'cd changedetectionio;pytest --live-server-host=0.0.0.0 --live-server-port=5004 tests/test_errorhandling.py'
+          docker run --rm -e "FLASK_SERVER_NAME=cdio" -e "FAST_PUPPETEER_CHROME_FETCHER=True" -e "PLAYWRIGHT_DRIVER_URL=ws://sockpuppetbrowser:3000" --network changedet-network --hostname=cdio test-changedetectionio  bash -c 'cd changedetectionio;pytest --live-server-host=0.0.0.0 --live-server-port=5004 tests/visualselector/test_fetch_data.py'
+          docker run --rm -e "FLASK_SERVER_NAME=cdio" -e "FAST_PUPPETEER_CHROME_FETCHER=True" -e "PLAYWRIGHT_DRIVER_URL=ws://sockpuppetbrowser:3000" --network changedet-network --hostname=cdio test-changedetectionio  bash -c 'cd changedetectionio;pytest --live-server-host=0.0.0.0 --live-server-port=5004 tests/fetchers/test_custom_js_before_content.py'
 
-      - name: Spin up ancillary services
+      - name: Pyppeteer and SocketPuppetBrowser - Headers and requests checks
+        if: ${{ inputs.skip-pypuppeteer == false }}
         run: |
-          docker network create changedet-network
-          docker run --network changedet-network -d -e "LOG_LEVEL=TRACE" --cap-add=SYS_ADMIN --name sockpuppetbrowser --hostname sockpuppetbrowser --rm -p 3000:3000 dgtlmoon/sockpuppetbrowser:latest
-          docker run --network changedet-network -d -e "LOG_LEVEL=TRACE" --cap-add=SYS_ADMIN --name sockpuppetbrowser-custom-url --hostname sockpuppetbrowser-custom-url -p 3001:3000 --rm dgtlmoon/sockpuppetbrowser:latest
+          # Settings headers playwright tests - Call back in from Sockpuppetbrowser, check headers
+          docker run --name "changedet" --hostname changedet --rm  -e "FAST_PUPPETEER_CHROME_FETCHER=True" -e "FLASK_SERVER_NAME=changedet" -e "PLAYWRIGHT_DRIVER_URL=ws://sockpuppetbrowser:3000?dumpio=true" --network changedet-network test-changedetectionio  bash -c 'cd changedetectionio; pytest --live-server-host=0.0.0.0  --live-server-port=5004 tests/test_request.py'
 
-      - name: Playwright - Specific tests in built container
-        run: |
-          docker run --rm -e "FLASK_SERVER_NAME=cdio" -e "PLAYWRIGHT_DRIVER_URL=ws://sockpuppetbrowser:3000" --network changedet-network --hostname=cdio test-changedetectionio bash -c 'cd changedetectionio;pytest -vv --capture=tee-sys --showlocals --tb=long --live-server-host=0.0.0.0 --live-server-port=5004 tests/fetchers/test_content.py'
-          docker run --rm -e "FLASK_SERVER_NAME=cdio" -e "PLAYWRIGHT_DRIVER_URL=ws://sockpuppetbrowser:3000" --network changedet-network --hostname=cdio test-changedetectionio bash -c 'cd changedetectionio;pytest -vv --capture=tee-sys --showlocals --tb=long --live-server-host=0.0.0.0 --live-server-port=5004 tests/test_errorhandling.py'
-          docker run --rm -e "FLASK_SERVER_NAME=cdio" -e "PLAYWRIGHT_DRIVER_URL=ws://sockpuppetbrowser:3000" --network changedet-network --hostname=cdio test-changedetectionio bash -c 'cd changedetectionio;pytest -vv --capture=tee-sys --showlocals --tb=long --live-server-host=0.0.0.0 --live-server-port=5004 tests/visualselector/test_fetch_data.py'
-          docker run --rm -e "FLASK_SERVER_NAME=cdio" -e "PLAYWRIGHT_DRIVER_URL=ws://sockpuppetbrowser:3000" --network changedet-network --hostname=cdio test-changedetectionio bash -c 'cd changedetectionio;pytest -vv --capture=tee-sys --showlocals --tb=long --live-server-host=0.0.0.0 --live-server-port=5004 tests/fetchers/test_custom_js_before_content.py'
-
-      - name: Playwright - Headers and requests
-        run: |
-          docker run --name "changedet" --hostname changedet --rm -e "FLASK_SERVER_NAME=changedet" -e "PLAYWRIGHT_DRIVER_URL=ws://sockpuppetbrowser:3000?dumpio=true" --network changedet-network test-changedetectionio bash -c 'find .; cd changedetectionio; pytest --live-server-host=0.0.0.0 --live-server-port=5004 tests/test_request.py; pwd;find .'
-
-      - name: Playwright - Restock detection
-        run: |
-          docker run --rm --name "changedet" -e "FLASK_SERVER_NAME=changedet" -e "PLAYWRIGHT_DRIVER_URL=ws://sockpuppetbrowser:3000" --network changedet-network test-changedetectionio bash -c 'cd changedetectionio;pytest --live-server-port=5004 --live-server-host=0.0.0.0 tests/restock/test_restock.py'
-
-  # Pyppeteer tests
-  pyppeteer-tests:
-    runs-on: ubuntu-latest
-    needs: build
-    if: ${{ inputs.skip-pypuppeteer == false }}
-    timeout-minutes: 10
-    env:
-      PYTHON_VERSION: ${{ inputs.python-version }}
-    steps:
-      - uses: actions/checkout@v5
-
-      - name: Download Docker image artifact
-        uses: actions/download-artifact@v6
-        with:
-          name: test-changedetectionio-${{ env.PYTHON_VERSION }}
-          path: /tmp
-
-      - name: Load Docker image
-        run: |
-          docker load -i /tmp/test-changedetectionio.tar
-
-      - name: Spin up ancillary services
-        run: |
-          docker network create changedet-network
-          docker run --network changedet-network -d -e "LOG_LEVEL=TRACE" --cap-add=SYS_ADMIN --name sockpuppetbrowser --hostname sockpuppetbrowser --rm -p 3000:3000 dgtlmoon/sockpuppetbrowser:latest
-
-      - name: Pyppeteer - Specific tests in built container
-        run: |
-          docker run --rm -e "FLASK_SERVER_NAME=cdio" -e "FAST_PUPPETEER_CHROME_FETCHER=True" -e "PLAYWRIGHT_DRIVER_URL=ws://sockpuppetbrowser:3000" --network changedet-network --hostname=cdio test-changedetectionio bash -c 'cd changedetectionio;pytest --live-server-host=0.0.0.0 --live-server-port=5004 tests/fetchers/test_content.py'
-          docker run --rm -e "FLASK_SERVER_NAME=cdio" -e "FAST_PUPPETEER_CHROME_FETCHER=True" -e "PLAYWRIGHT_DRIVER_URL=ws://sockpuppetbrowser:3000" --network changedet-network --hostname=cdio test-changedetectionio bash -c 'cd changedetectionio;pytest --live-server-host=0.0.0.0 --live-server-port=5004 tests/test_errorhandling.py'
-          docker run --rm -e "FLASK_SERVER_NAME=cdio" -e "FAST_PUPPETEER_CHROME_FETCHER=True" -e "PLAYWRIGHT_DRIVER_URL=ws://sockpuppetbrowser:3000" --network changedet-network --hostname=cdio test-changedetectionio bash -c 'cd changedetectionio;pytest --live-server-host=0.0.0.0 --live-server-port=5004 tests/visualselector/test_fetch_data.py'
-          docker run --rm -e "FLASK_SERVER_NAME=cdio" -e "FAST_PUPPETEER_CHROME_FETCHER=True" -e "PLAYWRIGHT_DRIVER_URL=ws://sockpuppetbrowser:3000" --network changedet-network --hostname=cdio test-changedetectionio bash -c 'cd changedetectionio;pytest --live-server-host=0.0.0.0 --live-server-port=5004 tests/fetchers/test_custom_js_before_content.py'
-
-      - name: Pyppeteer - Headers and requests checks
-        run: |
-          docker run --name "changedet" --hostname changedet --rm -e "FAST_PUPPETEER_CHROME_FETCHER=True" -e "FLASK_SERVER_NAME=changedet" -e "PLAYWRIGHT_DRIVER_URL=ws://sockpuppetbrowser:3000?dumpio=true" --network changedet-network test-changedetectionio bash -c 'cd changedetectionio; pytest --live-server-host=0.0.0.0 --live-server-port=5004 tests/test_request.py'
-
-      - name: Pyppeteer - Restock detection
-        run: |
-          docker run --rm --name "changedet" -e "FLASK_SERVER_NAME=changedet" -e "FAST_PUPPETEER_CHROME_FETCHER=True" -e "PLAYWRIGHT_DRIVER_URL=ws://sockpuppetbrowser:3000" --network changedet-network test-changedetectionio bash -c 'cd changedetectionio;pytest --live-server-port=5004 --live-server-host=0.0.0.0 tests/restock/test_restock.py'
-
-  # Selenium tests
-  selenium-tests:
-    runs-on: ubuntu-latest
-    needs: build
-    timeout-minutes: 10
-    env:
-      PYTHON_VERSION: ${{ inputs.python-version }}
-    steps:
-      - uses: actions/checkout@v5
-
-      - name: Download Docker image artifact
-        uses: actions/download-artifact@v6
-        with:
-          name: test-changedetectionio-${{ env.PYTHON_VERSION }}
-          path: /tmp
-
-      - name: Load Docker image
-        run: |
-          docker load -i /tmp/test-changedetectionio.tar
-
-      - name: Spin up ancillary services
-        run: |
-          docker network create changedet-network
-          docker run --network changedet-network -d --hostname selenium -p 4444:4444 --rm --shm-size="2g" selenium/standalone-chrome:4
-          sleep 3
-
-      - name: Specific tests for headers and requests checks with Selenium
-        run: |
-
-          docker run --name "changedet" --hostname changedet --rm -e "FLASK_SERVER_NAME=changedet" -e "WEBDRIVER_URL=http://selenium:4444/wd/hub" --network changedet-network test-changedetectionio bash -c 'cd changedetectionio; pytest --live-server-host=0.0.0.0 --live-server-port=5004 tests/test_request.py'
+      - name: Pyppeteer and SocketPuppetBrowser - Restock detection
+        if: ${{ inputs.skip-pypuppeteer == false }}
+        run: |                            
+          # restock detection via playwright - added name=changedet here so that playwright and sockpuppetbrowser can connect to it
+          docker run --rm --name "changedet" -e "FLASK_SERVER_NAME=changedet"  -e "FAST_PUPPETEER_CHROME_FETCHER=True"  -e "PLAYWRIGHT_DRIVER_URL=ws://sockpuppetbrowser:3000" --network changedet-network test-changedetectionio  bash -c 'cd changedetectionio;pytest --live-server-port=5004 --live-server-host=0.0.0.0 tests/restock/test_restock.py'
 
+# SELENIUM
       - name: Specific tests in built container for Selenium
         run: |
-          docker run --rm -e "WEBDRIVER_URL=http://selenium:4444/wd/hub" --network changedet-network test-changedetectionio bash -c 'cd changedetectionio;pytest tests/fetchers/test_content.py && pytest tests/test_errorhandling.py'
+          # Selenium fetch
+          docker run --rm -e "WEBDRIVER_URL=http://selenium:4444/wd/hub" --network changedet-network test-changedetectionio  bash -c 'cd changedetectionio;pytest tests/fetchers/test_content.py && pytest tests/test_errorhandling.py'
 
-
-  # SMTP tests
-  smtp-tests:
-    runs-on: ubuntu-latest
-    needs: build
-    timeout-minutes: 10
-    env:
-      PYTHON_VERSION: ${{ inputs.python-version }}
-    steps:
-      - uses: actions/checkout@v5
-
-      - name: Download Docker image artifact
-        uses: actions/download-artifact@v6
-        with:
-          name: test-changedetectionio-${{ env.PYTHON_VERSION }}
-          path: /tmp
-
-      - name: Load Docker image
+      - name: Specific tests in built container for headers and requests checks with Selenium
         run: |
-          docker load -i /tmp/test-changedetectionio.tar
-
-      - name: Spin up SMTP test server
-        run: |
-          docker network create changedet-network
-          docker run --network changedet-network -d -p 11025:11025 -p 11080:11080 --hostname mailserver test-changedetectionio bash -c 'pip3 install aiosmtpd && python changedetectionio/tests/smtp/smtp-test-server.py'
+          docker run --name "changedet" --hostname changedet --rm -e "FLASK_SERVER_NAME=changedet" -e "WEBDRIVER_URL=http://selenium:4444/wd/hub" --network changedet-network test-changedetectionio  bash -c 'cd changedetectionio; pytest --live-server-host=0.0.0.0  --live-server-port=5004 tests/test_request.py'
 
+# OTHER STUFF
       - name: Test SMTP notification mime types
         run: |
+          # SMTP content types - needs the 'Debug SMTP server/echo message back server' container from above
+          # "mailserver" hostname defined above
           docker run --rm --network changedet-network test-changedetectionio bash -c 'cd changedetectionio;pytest tests/smtp/test_notification_smtp.py'
 
-  # Proxy tests
-  proxy-tests:
-    runs-on: ubuntu-latest
-    needs: build
-    timeout-minutes: 10
-    env:
-      PYTHON_VERSION: ${{ inputs.python-version }}
-    steps:
-      - uses: actions/checkout@v5
-
-      - name: Download Docker image artifact
-        uses: actions/download-artifact@v6
-        with:
-          name: test-changedetectionio-${{ env.PYTHON_VERSION }}
-          path: /tmp
-
-      - name: Load Docker image
-        run: |
-          docker load -i /tmp/test-changedetectionio.tar
-
-      - name: Spin up services
-        run: |
-          docker network create changedet-network
-          docker run --network changedet-network -d --hostname selenium -p 4444:4444 --rm --shm-size="2g" selenium/standalone-chrome:4
-          docker run --network changedet-network -d -e "LOG_LEVEL=TRACE" --cap-add=SYS_ADMIN --name sockpuppetbrowser --hostname sockpuppetbrowser --rm -p 3000:3000 dgtlmoon/sockpuppetbrowser:latest
-          docker run --network changedet-network -d -e "LOG_LEVEL=TRACE" --cap-add=SYS_ADMIN --name sockpuppetbrowser-custom-url --hostname sockpuppetbrowser-custom-url -p 3001:3000 --rm dgtlmoon/sockpuppetbrowser:latest
-
-      - name: Test proxy Squid style interaction
+      # @todo Add a test via playwright/puppeteer
+      # squid with auth is tested in run_proxy_tests.sh -> tests/proxy_list/test_select_custom_proxy.py
+      - name: Test proxy squid style interaction
         run: |
           cd changedetectionio
           ./run_proxy_tests.sh
-          docker ps
           cd ..
 
       - name: Test proxy SOCKS5 style interaction
@@ -311,65 +155,28 @@ jobs:
           ./run_socks_proxy_tests.sh
           cd ..
 
-  # Custom browser URL tests
-  custom-browser-tests:
-    runs-on: ubuntu-latest
-    needs: build
-    timeout-minutes: 10
-    env:
-      PYTHON_VERSION: ${{ inputs.python-version }}
-    steps:
-      - uses: actions/checkout@v5
-
-      - name: Download Docker image artifact
-        uses: actions/download-artifact@v6
-        with:
-          name: test-changedetectionio-${{ env.PYTHON_VERSION }}
-          path: /tmp
-
-      - name: Load Docker image
-        run: |
-          docker load -i /tmp/test-changedetectionio.tar
-
-      - name: Spin up ancillary services
-        run: |
-          docker network create changedet-network
-          docker run --network changedet-network -d -e "LOG_LEVEL=TRACE" --cap-add=SYS_ADMIN --name sockpuppetbrowser --hostname sockpuppetbrowser --rm -p 3000:3000 dgtlmoon/sockpuppetbrowser:latest
-          docker run --network changedet-network -d -e "LOG_LEVEL=TRACE" --cap-add=SYS_ADMIN --name sockpuppetbrowser-custom-url --hostname sockpuppetbrowser-custom-url -p 3001:3000 --rm dgtlmoon/sockpuppetbrowser:latest
-
       - name: Test custom browser URL
         run: |
           cd changedetectionio
           ./run_custom_browser_url_tests.sh
+          cd ..
 
-  # Container startup tests
-  container-tests:
-    runs-on: ubuntu-latest
-    needs: build
-    timeout-minutes: 10
-    env:
-      PYTHON_VERSION: ${{ inputs.python-version }}
-    steps:
-      - uses: actions/checkout@v5
-
-      - name: Download Docker image artifact
-        uses: actions/download-artifact@v6
-        with:
-          name: test-changedetectionio-${{ env.PYTHON_VERSION }}
-          path: /tmp
-
-      - name: Load Docker image
+      - name: Test changedetection.io container starts+runs basically without error
         run: |
-          docker load -i /tmp/test-changedetectionio.tar
-
-      - name: Test container starts+runs basically without error
-        run: |
-          docker run --name test-changedetectionio -p 5556:5000 -d test-changedetectionio
+          docker run --name test-changedetectionio -p 5556:5000  -d test-changedetectionio
           sleep 3
-          curl --retry-connrefused --retry 6 -s http://localhost:5556 |grep -q checkbox-uuid
-          curl --retry-connrefused --retry 6 -s -g -6 "http://[::1]:5556"|grep -q checkbox-uuid
+          # Should return 0 (no error) when grep finds it
+          curl --retry-connrefused --retry 6  -s http://localhost:5556 |grep -q checkbox-uuid
+          
+          # and IPv6
+          curl --retry-connrefused --retry 6  -s -g -6 "http://[::1]:5556"|grep -q checkbox-uuid
+
+          # Check whether TRACE log is enabled.
+          # Also, check whether TRACE came from STDOUT
           docker logs test-changedetectionio 2>/dev/null | grep 'TRACE log is enabled' || exit 1
+          # Check whether DEBUG is came from STDOUT
           docker logs test-changedetectionio 2>/dev/null | grep 'DEBUG' || exit 1
+
           docker kill test-changedetectionio
 
       - name: Test HTTPS SSL mode
@@ -377,66 +184,78 @@ jobs:
           openssl req -x509 -newkey rsa:4096 -keyout privkey.pem -out cert.pem -days 365 -nodes -subj "/CN=localhost"
           docker run --name test-changedetectionio-ssl --rm -e SSL_CERT_FILE=cert.pem -e SSL_PRIVKEY_FILE=privkey.pem -p 5000:5000 -v ./cert.pem:/app/cert.pem -v ./privkey.pem:/app/privkey.pem -d test-changedetectionio
           sleep 3
+          # Should return 0 (no error) when grep finds it
+          # -k because its self-signed
           curl --retry-connrefused --retry 6 -k https://localhost:5000 -v|grep -q checkbox-uuid
+      
           docker kill test-changedetectionio-ssl
 
       - name: Test IPv6 Mode
         run: |
+          # IPv6 - :: bind to all interfaces inside container (like 0.0.0.0), ::1 would be localhost only
           docker run --name test-changedetectionio-ipv6 --rm -p 5000:5000 -e LISTEN_HOST=:: -d test-changedetectionio
           sleep 3
+          # Should return 0 (no error) when grep finds it on localhost
           curl --retry-connrefused --retry 6 http://[::1]:5000 -v|grep -q checkbox-uuid
           docker kill test-changedetectionio-ipv6
 
-  # Signal tests
-  signal-tests:
-    runs-on: ubuntu-latest
-    needs: build
-    timeout-minutes: 10
-    env:
-      PYTHON_VERSION: ${{ inputs.python-version }}
-    steps:
-      - uses: actions/checkout@v5
-
-      - name: Download Docker image artifact
-        uses: actions/download-artifact@v6
-        with:
-          name: test-changedetectionio-${{ env.PYTHON_VERSION }}
-          path: /tmp
-
-      - name: Load Docker image
-        run: |
-          docker load -i /tmp/test-changedetectionio.tar
-
-      - name: Test SIGTERM and SIGINT signal shutdown
+      - name: Test changedetection.io SIGTERM and SIGINT signal shutdown
         run: |
+          
           echo SIGINT Shutdown request test
           docker run --name sig-test -d test-changedetectionio
           sleep 3
           echo ">>> Sending SIGINT to sig-test container"
           docker kill --signal=SIGINT sig-test
           sleep 3
+          # invert the check (it should be not 0/not running)
           docker ps
+          # check signal catch(STDERR) log. Because of
+          # changedetectionio/__init__.py: logger.add(sys.stderr, level=logger_level)
           docker logs sig-test 2>&1 | grep 'Shutdown: Got Signal - SIGINT' || exit 1
           test -z "`docker ps|grep sig-test`"
-          if [ $? -ne 0 ]; then
+          if [ $? -ne 0 ]
+          then
             echo "Looks like container was running when it shouldnt be"
             docker ps
             exit 1
           fi
+          
+          # @todo - scan the container log to see the right "graceful shutdown" text exists 
           docker rm sig-test
-
+          
           echo SIGTERM Shutdown request test
           docker run --name sig-test -d test-changedetectionio
           sleep 3
           echo ">>> Sending SIGTERM to sig-test container"
           docker kill --signal=SIGTERM sig-test
           sleep 3
+          # invert the check (it should be not 0/not running)
           docker ps
+          # check signal catch(STDERR) log. Because of
+          # changedetectionio/__init__.py: logger.add(sys.stderr, level=logger_level)
           docker logs sig-test 2>&1 | grep 'Shutdown: Got Signal - SIGTERM' || exit 1
           test -z "`docker ps|grep sig-test`"
-          if [ $? -ne 0 ]; then
+          if [ $? -ne 0 ]
+          then
             echo "Looks like container was running when it shouldnt be"
             docker ps
             exit 1
           fi
+          
+          # @todo - scan the container log to see the right "graceful shutdown" text exists           
           docker rm sig-test
+
+      - name: Dump container log
+        if: always()
+        run: |
+          mkdir output-logs
+          docker logs test-cdio-basic-tests > output-logs/test-cdio-basic-tests-stdout-${{ env.PYTHON_VERSION }}.txt
+          docker logs test-cdio-basic-tests 2> output-logs/test-cdio-basic-tests-stderr-${{ env.PYTHON_VERSION }}.txt
+
+      - name: Store everything including test-datastore
+        if: always()
+        uses: actions/upload-artifact@v4
+        with:
+          name: test-cdio-basic-tests-output-py${{ env.PYTHON_VERSION }}
+          path: .

Dockerfile

@@ -34,27 +34,23 @@ ENV OPENSSL_LIB_DIR="/usr/lib/arm-linux-gnueabihf"
 ENV OPENSSL_INCLUDE_DIR="/usr/include/openssl"
 # Additional environment variables for cryptography Rust build
 ENV CRYPTOGRAPHY_DONT_BUILD_RUST=1
-RUN --mount=type=cache,id=pip,sharing=locked,target=/tmp/pip-cache \
-  pip install \
-  --prefer-binary \
-  --extra-index-url https://www.piwheels.org/simple \
-  --extra-index-url https://pypi.anaconda.org/ARM-software/simple \
-  --cache-dir=/tmp/pip-cache \
-  --target=/dependencies \
-  -r /requirements.txt
-
+RUN --mount=type=cache,target=/tmp/pip-cache \
+    pip install \
+    --extra-index-url https://www.piwheels.org/simple \
+    --extra-index-url https://pypi.anaconda.org/ARM-software/simple \
+    --cache-dir=/tmp/pip-cache \
+    --target=/dependencies \
+    -r /requirements.txt
 
 # Playwright is an alternative to Selenium
 # Excluded this package from requirements.txt to prevent arm/v6 and arm/v7 builds from failing
 # https://github.com/dgtlmoon/changedetection.io/pull/1067 also musl/alpine (not supported)
-RUN --mount=type=cache,id=pip,sharing=locked,target=/tmp/pip-cache \
-  pip install \
-  --prefer-binary \
-  --cache-dir=/tmp/pip-cache \
-  --target=/dependencies \
-  playwright~=1.48.0 \
-  || echo "WARN: Failed to install Playwright. The application can still run, but the Playwright option will be disabled."
-
+RUN --mount=type=cache,target=/tmp/pip-cache \
+    pip install \
+    --cache-dir=/tmp/pip-cache \
+    --target=/dependencies \
+    playwright~=1.48.0 \
+    || echo "WARN: Failed to install Playwright. The application can still run, but the Playwright option will be disabled."
 
 # Final image stage
 FROM python:${PYTHON_VERSION}-slim-bookworm

MANIFEST.in

@@ -1,9 +1,7 @@
 recursive-include changedetectionio/api *
-include docs/api-spec.yaml
 recursive-include changedetectionio/blueprint *
 recursive-include changedetectionio/conditions *
 recursive-include changedetectionio/content_fetchers *
-recursive-include changedetectionio/jinja2_custom *
 recursive-include changedetectionio/model *
 recursive-include changedetectionio/notification *
 recursive-include changedetectionio/processors *

changedetectionio/__init__.py

@@ -2,7 +2,7 @@
 
 # Read more https://github.com/dgtlmoon/changedetection.io/wiki
 
-__version__ = '0.50.39'
+__version__ = '0.50.20'
 
 from changedetectionio.strtobool import strtobool
 from json.decoder import JSONDecodeError

changedetectionio/api/Import.py

@@ -1,22 +1,9 @@
+import os
 from changedetectionio.strtobool import strtobool
 from flask_restful import abort, Resource
 from flask import request
-from functools import wraps
+import validators
 from . import auth, validate_openapi_request
-from ..validate_url import is_safe_valid_url
-
-
-def default_content_type(content_type='text/plain'):
-    """Decorator to set a default Content-Type header if none is provided."""
-    def decorator(f):
-        @wraps(f)
-        def wrapper(*args, **kwargs):
-            if not request.content_type:
-                # Set default content type in the request environment
-                request.environ['CONTENT_TYPE'] = content_type
-            return f(*args, **kwargs)
-        return wrapper
-    return decorator
 
 
 class Import(Resource):
@@ -25,7 +12,6 @@ class Import(Resource):
         self.datastore = kwargs['datastore']
 
     @auth.check_token
-    @default_content_type('text/plain') #3547 #3542
     @validate_openapi_request('importWatches')
     def post(self):
         """Import a list of watched URLs."""
@@ -49,13 +35,14 @@ class Import(Resource):
 
         urls = request.get_data().decode('utf8').splitlines()
         added = []
+        allow_simplehost = not strtobool(os.getenv('BLOCK_SIMPLEHOSTS', 'False'))
         for url in urls:
             url = url.strip()
             if not len(url):
                 continue
 
             # If hosts that only contain alphanumerics are allowed ("localhost" for example)
-            if not is_safe_valid_url(url):
+            if not validators.url(url, simple_host=allow_simplehost):
                 return f"Invalid or unsupported URL - {url}", 400
 
             if dedupe and self.datastore.url_exists(url):

changedetectionio/api/Watch.py

@@ -1,19 +1,17 @@
 import os
-
-from changedetectionio.validate_url import is_safe_valid_url
+from changedetectionio.strtobool import strtobool
 
 from flask_expects_json import expects_json
-from changedetectionio import queuedWatchMetaData, strtobool
+from changedetectionio import queuedWatchMetaData
 from changedetectionio import worker_handler
 from flask_restful import abort, Resource
 from flask import request, make_response, send_from_directory
+import validators
 from . import auth
 import copy
 
 # Import schemas from __init__.py
 from . import schema, schema_create_watch, schema_update_watch, validate_openapi_request
-from ..notification import valid_notification_formats
-from ..notification.handler import newline_re
 
 
 def validate_time_between_check_required(json_data):
@@ -123,10 +121,6 @@ class Watch(Resource):
         if validation_error:
             return validation_error, 400
 
-        # XSS etc protection
-        if request.json.get('url') and not is_safe_valid_url(request.json.get('url')):
-            return "Invalid URL", 400
-
         watch.update(request.json)
 
         return "OK", 200
@@ -183,103 +177,6 @@ class WatchSingleHistory(Resource):
 
         return response
 
-class WatchHistoryDiff(Resource):
-    def __init__(self, **kwargs):
-        # datastore is a black box dependency
-        self.datastore = kwargs['datastore']
-
-    @auth.check_token
-    @validate_openapi_request('getWatchHistoryDiff')
-    def get(self, uuid, from_timestamp, to_timestamp):
-        """Generate diff between two historical snapshots."""
-        from changedetectionio import diff
-        from changedetectionio.notification.handler import apply_service_tweaks
-
-        watch = self.datastore.data['watching'].get(uuid)
-        if not watch:
-            abort(404, message=f"No watch exists with the UUID of {uuid}")
-
-        if not len(watch.history):
-            abort(404, message=f"Watch found but no history exists for the UUID {uuid}")
-
-        history_keys = list(watch.history.keys())
-
-        # Handle 'latest' keyword for to_timestamp
-        if to_timestamp == 'latest':
-            to_timestamp = history_keys[-1]
-
-        # Handle 'previous' keyword for from_timestamp (second-most-recent)
-        if from_timestamp == 'previous':
-            if len(history_keys) < 2:
-                abort(404, message=f"Not enough history entries. Need at least 2 snapshots for 'previous'")
-            from_timestamp = history_keys[-2]
-
-        # Validate timestamps exist
-        if from_timestamp not in watch.history:
-            abort(404, message=f"From timestamp {from_timestamp} not found in watch history")
-        if to_timestamp not in watch.history:
-            abort(404, message=f"To timestamp {to_timestamp} not found in watch history")
-
-        # Get the format parameter (default to 'text')
-        output_format = request.args.get('format', 'text').lower()
-
-        # Validate format
-        if output_format not in valid_notification_formats.keys():
-            abort(400, message=f"Invalid format. Must be one of: {', '.join(valid_notification_formats.keys())}")
-
-        # Get the word_diff parameter (default to False - line-level mode)
-        word_diff = strtobool(request.args.get('word_diff', 'false'))
-
-        # Get the no_markup parameter (default to False)
-        no_markup = strtobool(request.args.get('no_markup', 'false'))
-
-        # Retrieve snapshot contents
-        from_version_file_contents = watch.get_history_snapshot(from_timestamp)
-        to_version_file_contents = watch.get_history_snapshot(to_timestamp)
-
-        # Get diff preferences (using defaults similar to the existing code)
-        diff_prefs = {
-            'diff_ignoreWhitespace': False,
-            'diff_changesOnly': True
-        }
-
-        # Generate the diff
-        content = diff.render_diff(
-            previous_version_file_contents=from_version_file_contents,
-            newest_version_file_contents=to_version_file_contents,
-            ignore_junk=diff_prefs.get('diff_ignoreWhitespace'),
-            include_equal=not diff_prefs.get('diff_changesOnly'),
-            word_diff=word_diff,
-        )
-
-        # Skip formatting if no_markup is set
-        if no_markup:
-            mimetype = "text/plain"
-        else:
-            # Apply formatting based on the requested format
-            if output_format == 'htmlcolor':
-                from changedetectionio.notification.handler import apply_html_color_to_body
-                content = apply_html_color_to_body(n_body=content)
-                mimetype = "text/html"
-            else:
-                # Apply service tweaks for text/html formats
-                # Pass empty URL and title as they're not used for the placeholder replacement we need
-                _, content, _ = apply_service_tweaks(
-                    url='',
-                    n_body=content,
-                    n_title='',
-                    requested_output_format=output_format
-                )
-                mimetype = "text/html" if output_format == 'html' else "text/plain"
-
-            if 'html' in output_format:
-                content = newline_re.sub('<br>\r\n', content)
-
-        response = make_response(content, 200)
-        response.mimetype = mimetype
-        return response
-
-
 class WatchFavicon(Resource):
     def __init__(self, **kwargs):
         # datastore is a black box dependency
@@ -329,7 +226,9 @@ class CreateWatch(Resource):
         json_data = request.get_json()
         url = json_data['url'].strip()
 
-        if not is_safe_valid_url(url):
+        # If hosts that only contain alphanumerics are allowed ("localhost" for example)
+        allow_simplehost = not strtobool(os.getenv('BLOCK_SIMPLEHOSTS', 'False'))
+        if not validators.url(url, simple_host=allow_simplehost):
             return "Invalid or unsupported URL", 400
 
         if json_data.get('proxy'):

changedetectionio/api/__init__.py

@@ -37,10 +37,6 @@ def get_openapi_spec():
     from openapi_core import OpenAPI  # Lazy import - saves ~10.7 MB on startup
 
     spec_path = os.path.join(os.path.dirname(__file__), '../../docs/api-spec.yaml')
-    if not os.path.exists(spec_path):
-        # Possibly for pip3 packages
-        spec_path = os.path.join(os.path.dirname(__file__), '../docs/api-spec.yaml')
-
     with open(spec_path, 'r') as f:
         spec_dict = yaml.safe_load(f)
     _openapi_spec = OpenAPI.from_dict(spec_dict)
@@ -51,7 +47,6 @@ def validate_openapi_request(operation_id):
     def decorator(f):
         @functools.wraps(f)
         def wrapper(*args, **kwargs):
-            from werkzeug.exceptions import BadRequest
             try:
                 # Skip OpenAPI validation for GET requests since they don't have request bodies
                 if request.method.upper() != 'GET':
@@ -62,6 +57,7 @@ def validate_openapi_request(operation_id):
                     openapi_request = FlaskOpenAPIRequest(request)
                     result = spec.unmarshal_request(openapi_request)
                     if result.errors:
+                        from werkzeug.exceptions import BadRequest
                         error_details = []
                         for error in result.errors:
                             error_details.append(str(error))
@@ -78,7 +74,7 @@ def validate_openapi_request(operation_id):
     return decorator
 
 # Import all API resources
-from .Watch import Watch, WatchHistory, WatchSingleHistory, WatchHistoryDiff, CreateWatch, WatchFavicon
+from .Watch import Watch, WatchHistory, WatchSingleHistory, CreateWatch, WatchFavicon
 from .Tags import Tags, Tag
 from .Import import Import
 from .SystemInfo import SystemInfo

changedetectionio/api/api_schema.py

@@ -96,10 +96,7 @@ def build_watch_json_schema(d):
                                                            "enum": ["html_requests", "html_webdriver"]
                                                            })
 
-    schema['properties']['processor'] = {"anyOf": [
-        {"type": "string", "enum": ["restock_diff", "text_json_diff"]},
-        {"type": "null"}
-    ]}
+
 
     # All headers must be key/value type dict
     schema['properties']['headers'] = {

changedetectionio/async_update_worker.py

@@ -334,10 +334,6 @@ async def async_update_worker(worker_id, q, notification_q, app, datastore):
                             if update_handler.fetcher.content or (not update_handler.fetcher.content and empty_pages_are_a_change):
                                 watch.save_last_fetched_html(contents=update_handler.fetcher.content, timestamp=int(fetch_start_time))
 
-                            # Explicitly delete large content variables to free memory IMMEDIATELY after saving
-                            # These are no longer needed after being saved to history
-                            del contents
-
                             # Send notifications on second+ check
                             if watch.history_n >= 2:
                                 logger.info(f"Change detected in UUID {uuid} - {watch['url']}")
@@ -376,12 +372,6 @@ async def async_update_worker(worker_id, q, notification_q, app, datastore):
                 datastore.update_watch(uuid=uuid, update_obj={'fetch_time': round(time.time() - fetch_start_time, 3),
                                                                'check_count': count})
 
-                # NOW clear fetcher content - after all processing is complete
-                # This is the last point where we need the fetcher data
-                if update_handler and hasattr(update_handler, 'fetcher') and update_handler.fetcher:
-                    update_handler.fetcher.clear_content()
-                    logger.debug(f"Cleared fetcher content for UUID {uuid}")
-
         except Exception as e:
             logger.error(f"Worker {worker_id} unexpected error processing {uuid}: {e}")
             logger.error(f"Worker {worker_id} traceback:", exc_info=True)
@@ -402,28 +392,7 @@ async def async_update_worker(worker_id, q, notification_q, app, datastore):
                         #logger.info(f"Worker {worker_id} sending completion signal for UUID {watch['uuid']}")
                         watch_check_update.send(watch_uuid=watch['uuid'])
 
-                    # Explicitly clean up update_handler and all its references
-                    if update_handler:
-                        # Clear fetcher content using the proper method
-                        if hasattr(update_handler, 'fetcher') and update_handler.fetcher:
-                            update_handler.fetcher.clear_content()
-
-                        # Clear processor references
-                        if hasattr(update_handler, 'content_processor'):
-                            update_handler.content_processor = None
-
-                        update_handler = None
-
-                    # Clear local contents variable if it still exists
-                    if 'contents' in locals():
-                        del contents
-
-                    # Note: We don't set watch = None here because:
-                    # 1. watch is just a local reference to datastore.data['watching'][uuid]
-                    # 2. Setting it to None doesn't affect the datastore
-                    # 3. GC can't collect the object anyway (still referenced by datastore)
-                    # 4. It would just cause confusion
-
+                    update_handler = None
                     logger.debug(f"Worker {worker_id} completed watch {uuid} in {time.time()-fetch_start_time:.2f}s")
                 except Exception as cleanup_error:
                     logger.error(f"Worker {worker_id} error during cleanup: {cleanup_error}")

changedetectionio/blueprint/browser_steps/browser_steps.py

@@ -6,7 +6,7 @@ from loguru import logger
 
 from changedetectionio.content_fetchers import SCREENSHOT_MAX_HEIGHT_DEFAULT
 from changedetectionio.content_fetchers.base import manage_user_agent
-from changedetectionio.jinja2_custom import render as jinja_render
+from changedetectionio.safe_jinja import render as jinja_render
 
 
 

changedetectionio/blueprint/check_proxies/__init__.py

@@ -33,7 +33,7 @@ def construct_blueprint(datastore: ChangeDetectionStore):
     def long_task(uuid, preferred_proxy):
         import time
         from changedetectionio.content_fetchers import exceptions as content_fetcher_exceptions
-        from changedetectionio.jinja2_custom import render as jinja_render
+        from changedetectionio.safe_jinja import render as jinja_render
 
         status = {'status': '', 'length': 0, 'text': ''}
 

changedetectionio/blueprint/rss/blueprint.py

@@ -1,6 +1,5 @@
 
-from changedetectionio.jinja2_custom import render as jinja_render
-from changedetectionio.notification.handler import apply_service_tweaks
+from changedetectionio.safe_jinja import render as jinja_render
 from changedetectionio.store import ChangeDetectionStore
 from feedgen.feed import FeedGenerator
 from flask import Blueprint, make_response, request, url_for, redirect
@@ -120,13 +119,10 @@ def construct_blueprint(datastore: ChangeDetectionStore):
 
                     html_diff = diff.render_diff(previous_version_file_contents=watch.get_history_snapshot(dates[-2]),
                                                  newest_version_file_contents=watch.get_history_snapshot(dates[-1]),
-                                                 include_equal=False
+                                                 include_equal=False,
+                                                 line_feed_sep="<br>",
+                                                 html_colour=html_colour_enable
                                                  )
-
-
-                    requested_output_format = 'htmlcolor' if html_colour_enable else 'html'
-                    html_diff = apply_service_tweaks(url='', n_body=html_diff, n_title=None, requested_output_format=requested_output_format)
-
                 except FileNotFoundError as e:
                     html_diff = f"History snapshot file for watch {watch.get('uuid')}@{watch.last_changed} - '{watch.get('title')} not found."
 

changedetectionio/blueprint/settings/__init__.py

@@ -119,7 +119,7 @@ def construct_blueprint(datastore: ChangeDetectionStore):
                                 hide_remove_pass=os.getenv("SALTED_PASS", False),
                                 min_system_recheck_seconds=int(os.getenv('MINIMUM_SECONDS_RECHECK_TIME', 3)),
                                 settings_application=datastore.data['settings']['application'],
-                                timezone_default_config=datastore.data['settings']['application'].get('scheduler_timezone_default'),
+                                timezone_default_config=datastore.data['settings']['application'].get('timezone'),
                                 utc_time=utc_time,
                                 )
 

changedetectionio/blueprint/settings/templates/settings.html

@@ -1,7 +1,7 @@
 {% extends 'base.html' %}
 
 {% block content %}
-{% from '_helpers.html' import render_field, render_checkbox_field, render_button, render_time_schedule_form, render_ternary_field, render_fieldlist_with_inline_errors %}
+{% from '_helpers.html' import render_field, render_checkbox_field, render_button, render_time_schedule_form, render_ternary_field %}
 {% from '_common_fields.html' import render_common_settings_form %}
 <script>
     const notification_base_url="{{url_for('ui.ui_notification.ajax_callback_send_notification_test', mode="global-settings")}}";
@@ -72,23 +72,25 @@
                         <span class="pure-form-message-inline">Allow access to view watch diff page when password is enabled (Good for sharing the diff page)
                         </span>
                     </div>
+                    <div class="pure-control-group">
+                        {{ render_checkbox_field(form.application.form.rss_hide_muted_watches) }}
+                    </div>
+                    <div class="pure-control-group">
+                        {{ render_field(form.application.form.rss_content_format) }}
+                        <span class="pure-form-message-inline">Love RSS? Does your reader support HTML? Set it here</span>
+                    </div>
                     <div class="pure-control-group">
                         {{ render_checkbox_field(form.application.form.empty_pages_are_a_change) }}
                         <span class="pure-form-message-inline">When a request returns no content, or the HTML does not contain any text, is this considered a change?</span>
                     </div>
-                    <div class="grey-form-border">
-                        <div class="pure-control-group">
-                            {{ render_checkbox_field(form.application.form.rss_hide_muted_watches) }}
-                        </div>
-                        <div class="pure-control-group">
-                            {{ render_field(form.application.form.rss_content_format) }}
-                            <span class="pure-form-message-inline">Love RSS? Does your reader support HTML? Set it here</span>
-                        </div>
-                        <div class="pure-control-group">
-                            {{ render_checkbox_field(form.application.form.rss_reader_mode) }}
-                            <span class="pure-form-message-inline">Transforms RSS/RDF feed watches into beautiful text only</span>
-                        </div>
+                {% if form.requests.proxy %}
+                    <div class="pure-control-group inline-radio">
+                        {{ render_field(form.requests.form.proxy, class="fetch-backend-proxy") }}
+                        <span class="pure-form-message-inline">
+                        Choose a default proxy for all watches
+                        </span>
                     </div>
+                {% endif %}
                 </fieldset>
             </div>
 
@@ -131,10 +133,6 @@
                     <span class="pure-form-message-inline">Number of concurrent workers to process watches. More workers = faster processing but higher memory usage.<br>
                     Currently running: <strong>{{ worker_info.count }}</strong> operational {{ worker_info.type }} workers{% if worker_info.active_workers > 0 %} ({{ worker_info.active_workers }} actively processing){% endif %}.</span>
                 </div>
-                <div class="pure-control-group">
-                    {{ render_field(form.requests.form.timeout) }}
-                    <span class="pure-form-message-inline">For regular plain requests (not chrome based), maximum number of seconds until timeout, 1-999.<br>
-                </div>
                 <div class="pure-control-group inline-radio">
                     {{ render_field(form.requests.form.default_ua) }}
                     <span class="pure-form-message-inline">
@@ -238,9 +236,11 @@ nav
                     <p><strong>UTC Time &amp Date from Server:</strong> <span id="utc-time" >{{ utc_time }}</span></p>
                     <p><strong>Local Time &amp Date in Browser:</strong> <span class="local-time" data-utc="{{ utc_time }}"></span></p>
                     <p>
-                       {{ render_field(form.application.form.scheduler_timezone_default) }}
+                       {{ render_field(form.application.form.timezone) }}
                         <datalist id="timezones" style="display: none;">
-                            {%- for timezone in available_timezones -%}<option value="{{ timezone }}">{{ timezone }}</option>{%- endfor -%}
+                            {% for tz_name in available_timezones %}
+                                <option value="{{ tz_name }}">{{ tz_name }}</option>
+                            {% endfor %}
                         </datalist>
                     </p>
                 </div>
@@ -314,27 +314,17 @@ nav
                <p><strong>Tip</strong>: "Residential" and "Mobile" proxy type can be more successfull than "Data Center" for blocked websites.
 
                 <div class="pure-control-group" id="extra-proxies-setting">
-                {{ render_fieldlist_with_inline_errors(form.requests.form.extra_proxies) }}
+                {{ render_field(form.requests.form.extra_proxies) }}
                 <span class="pure-form-message-inline">"Name" will be used for selecting the proxy in the Watch Edit settings</span><br>
                 <span class="pure-form-message-inline">SOCKS5 proxies with authentication are only supported with 'plain requests' fetcher, for other fetchers you should whitelist the IP access instead</span>
-                {% if form.requests.proxy %}
-                <div>
-                <br>
-                    <div class="inline-radio">
-                        {{ render_field(form.requests.form.proxy, class="fetch-backend-proxy") }}
-                        <span class="pure-form-message-inline">Choose a default proxy for all watches</span>
-                    </div>
-                </div>
-                {% endif %}
                 </div>
                 <div class="pure-control-group" id="extra-browsers-setting">
                     <p>
                     <span class="pure-form-message-inline"><i>Extra Browsers</i> can be attached to further defeat CAPTCHA's on websites that are particularly hard to scrape.</span><br>
                     <span class="pure-form-message-inline">Simply paste the connection address into the box, <a href="https://changedetection.io/tutorial/using-bright-datas-scraping-browser-pass-captchas-and-other-protection-when-monitoring">More instructions and examples here</a> </span>
                     </p>
-                    {{ render_fieldlist_with_inline_errors(form.requests.form.extra_browsers) }}
+                    {{ render_field(form.requests.form.extra_browsers) }}
                 </div>
-            
             </div>
             <div id="actions">
                 <div class="pure-control-group">

changedetectionio/blueprint/ui/__init__.py

@@ -76,14 +76,14 @@ def _handle_operations(op, uuids, datastore, worker_handler, update_q, queuedWat
 
     elif (op == 'notification-default'):
         from changedetectionio.notification import (
-            USE_SYSTEM_DEFAULT_NOTIFICATION_FORMAT_FOR_WATCH
+            default_notification_format_for_watch
         )
         for uuid in uuids:
             if datastore.data['watching'].get(uuid):
                 datastore.data['watching'][uuid]['notification_title'] = None
                 datastore.data['watching'][uuid]['notification_body'] = None
                 datastore.data['watching'][uuid]['notification_urls'] = []
-                datastore.data['watching'][uuid]['notification_format'] = USE_SYSTEM_DEFAULT_NOTIFICATION_FORMAT_FOR_WATCH
+                datastore.data['watching'][uuid]['notification_format'] = default_notification_format_for_watch
         if emit_flash:
             flash(f"{len(uuids)} watches set to use default notification settings")
 

changedetectionio/blueprint/ui/edit.py

@@ -187,7 +187,7 @@ def construct_blueprint(datastore: ChangeDetectionStore, update_q, queuedWatchMe
 
             tz_name = time_schedule_limit.get('timezone')
             if not tz_name:
-                tz_name = datastore.data['settings']['application'].get('scheduler_timezone_default', os.getenv('TZ', 'UTC').strip())
+                tz_name = datastore.data['settings']['application'].get('timezone', 'UTC')
 
             if time_schedule_limit and time_schedule_limit.get('enabled'):
                 try:
@@ -257,7 +257,7 @@ def construct_blueprint(datastore: ChangeDetectionStore, update_q, queuedWatchMe
                 'system_has_webdriver_configured': os.getenv('WEBDRIVER_URL'),
                 'ui_edit_stats_extras': collect_ui_edit_stats_extras(watch),
                 'visual_selector_data_ready': datastore.visualselector_data_is_ready(watch_uuid=uuid),
-                'timezone_default_config': datastore.data['settings']['application'].get('scheduler_timezone_default'),
+                'timezone_default_config': datastore.data['settings']['application'].get('timezone'),
                 'using_global_webdriver_wait': not default['webdriver_delay'],
                 'uuid': uuid,
                 'watch': watch,

changedetectionio/blueprint/ui/notification.py

@@ -2,7 +2,6 @@ from flask import Blueprint, request, make_response
 import random
 from loguru import logger
 
-from changedetectionio.notification_service import NotificationContextData, set_basic_notification_vars
 from changedetectionio.store import ChangeDetectionStore
 from changedetectionio.auth_decorator import login_optionally_required
 
@@ -20,7 +19,6 @@ def construct_blueprint(datastore: ChangeDetectionStore):
         import apprise
         from changedetectionio.notification.handler import process_notification
         from changedetectionio.notification.apprise_plugin.assets import apprise_asset
-        from changedetectionio.jinja2_custom import render as jinja_render
 
         from changedetectionio.notification.apprise_plugin.custom_handlers import apprise_http_custom_handler
 
@@ -39,7 +37,11 @@ def construct_blueprint(datastore: ChangeDetectionStore):
             return make_response("Error: You must have atleast one watch configured for 'test notification' to work", 400)
 
         watch = datastore.data['watching'].get(watch_uuid)
-        notification_urls = request.form.get('notification_urls','').strip().splitlines()
+
+        notification_urls = None
+
+        if request.form.get('notification_urls'):
+            notification_urls = request.form['notification_urls'].strip().splitlines()
 
         if not notification_urls:
             logger.debug("Test notification - Trying by group/tag in the edit form if available")
@@ -59,26 +61,20 @@ def construct_blueprint(datastore: ChangeDetectionStore):
             return 'Error: No Notification URLs set/found'
 
         for n_url in notification_urls:
-            # We are ONLY validating the apprise:// part here, convert all tags to something so as not to break apprise URLs
-            generic_notification_context_data = NotificationContextData()
-            generic_notification_context_data.set_random_for_validation()
-            n_url = jinja_render(template_str=n_url, **generic_notification_context_data).strip()
             if len(n_url.strip()):
                 if not apobj.add(n_url):
                     return f'Error:  {n_url} is not a valid AppRise URL.'
 
         try:
             # use the same as when it is triggered, but then override it with the form test values
-            n_object = NotificationContextData({
+            n_object = {
                 'watch_url': request.form.get('window_url', "https://changedetection.io"),
                 'notification_urls': notification_urls
-            })
+            }
 
             # Only use if present, if not set in n_object it should use the default system value
             if 'notification_format' in request.form and request.form['notification_format'].strip():
                 n_object['notification_format'] = request.form.get('notification_format', '').strip()
-            else:
-                n_object['notification_format'] = datastore.data['settings']['application'].get('notification_format')
 
             if 'notification_title' in request.form and request.form['notification_title'].strip():
                 n_object['notification_title'] = request.form.get('notification_title', '').strip()
@@ -95,43 +91,7 @@ def construct_blueprint(datastore: ChangeDetectionStore):
                 n_object['notification_body'] = "Test body"
 
             n_object['as_async'] = False
-
-            #  Same like in notification service, should be refactored
-            dates = []
-            trigger_text = ''
-            snapshot_contents = ''
-            if watch:
-                watch_history = watch.history
-                dates = list(watch_history.keys())
-                trigger_text = watch.get('trigger_text', [])
-                # Add text that was triggered
-                if len(dates):
-                    snapshot_contents = watch.get_history_snapshot(dates[-1])
-                else:
-                    snapshot_contents = "No snapshot/history available, the watch should fetch atleast once."
-
-                if len(trigger_text):
-                    from . import html_tools
-                    triggered_text = html_tools.get_triggered_text(content=snapshot_contents, trigger_text=trigger_text)
-                    if triggered_text:
-                        triggered_text = '\n'.join(triggered_text)
-
-            # Could be called as a 'test notification' with only 1 snapshot available
-            prev_snapshot = "Example text: example test\nExample text: change detection is cool\nExample text: some more examples\n"
-            current_snapshot = "Example text: example test\nExample text: change detection is fantastic\nExample text: even more examples\nExample text: a lot more examples"
-
-
-
-            if len(dates) > 1:
-                prev_snapshot = watch.get_history_snapshot(dates[-2])
-                current_snapshot = watch.get_history_snapshot(dates[-1])
-
-            n_object.update(set_basic_notification_vars(current_snapshot=current_snapshot,
-                                                        prev_snapshot=prev_snapshot,
-                                                        watch=watch,
-                                                        triggered_text=trigger_text))
-
-
+            n_object.update(watch.extra_notification_token_values())
             sent_obj = process_notification(n_object, datastore)
 
         except Exception as e:

changedetectionio/blueprint/ui/templates/diff-offscreen-options.html

@@ -1,14 +0,0 @@
-<ul id="highlightSnippetActions">
-    <li>
-        <button class="pure-button pure-button-primary" onclick="diffToJpeg()" title="Share diff as image">Share as Image</button>
-    </li>
-    <li>
-        <a class="pure-button pure-button-primary" data-mode="exact" href="javascript:void(0);">Ignore any lines matching</a>
-    </li>
-    <li>
-        <a class="pure-button pure-button-primary" data-mode="digit-regex" href="javascript:void(0);" >Ignore any lines matching excluding digits</a>
-    </li>
-</ul>
-
-<!--             if (/\d/.test(window.getSelection().toString())) { -->
-

changedetectionio/blueprint/ui/templates/edit.html

@@ -1,6 +1,6 @@
 {% extends 'base.html' %}
 {% block content %}
-{% from '_helpers.html' import render_field, render_checkbox_field, render_button, render_time_schedule_form, playwright_warning, only_playwright_type_watches_warning, highlight_trigger_ignored_explainer, render_conditions_fieldlist_of_formfields_as_table, render_ternary_field %}
+{% from '_helpers.html' import render_field, render_checkbox_field, render_button, render_time_schedule_form, playwright_warning, only_playwright_type_watches_warning, render_conditions_fieldlist_of_formfields_as_table, render_ternary_field %}
 {% from '_common_fields.html' import render_common_settings_form %}
 <script src="{{url_for('static_content', group='js', filename='tabs.js')}}" defer></script>
 <script src="{{url_for('static_content', group='js', filename='vis.js')}}" defer></script>
@@ -351,22 +351,21 @@ Math: {{ 1 + 1 }}") }}
                 </div>
               </div>
               <div id="text-preview" style="display: none;" >
-                <script>
-                    const preview_text_edit_filters_url="{{url_for('ui.ui_edit.watch_get_preview_rendered', uuid=uuid)}}";
-                </script>
-                <br>
-                {#<div id="text-preview-controls"><span id="text-preview-refresh" class="pure-button button-xsmall">Refresh</span></div>#}
-                <div class="minitabs-wrapper">
-                  <div class="minitabs-content">
-                      <div id="text-preview-inner" class="monospace-preview">
-                          <p>Loading...</p>
+                    <script>
+                        const preview_text_edit_filters_url="{{url_for('ui.ui_edit.watch_get_preview_rendered', uuid=uuid)}}";
+                    </script>
+                    <br>
+                    {#<div id="text-preview-controls"><span id="text-preview-refresh" class="pure-button button-xsmall">Refresh</span></div>#}
+                    <div class="minitabs-wrapper">
+                      <div class="minitabs-content">
+                          <div id="text-preview-inner" class="monospace-preview">
+                              <p>Loading...</p>
+                          </div>
+                          <div id="text-preview-before-inner" style="display: none;" class="monospace-preview">
+                              <p>Loading...</p>
+                          </div>
                       </div>
-                      <div id="text-preview-before-inner" style="display: none;" class="monospace-preview">
-                          <p>Loading...</p>
-                      </div>
-                  </div>
-                </div>
-                {{ highlight_trigger_ignored_explainer() }}
+                    </div>
             </div>
           </div>
         </div>

changedetectionio/blueprint/ui/views.py

@@ -1,152 +1,16 @@
 from flask import Blueprint, request, redirect, url_for, flash, render_template, make_response, send_from_directory, abort
 import os
 import time
-import re
 from loguru import logger
-from markupsafe import Markup
 
-from changedetectionio.diff import (
-    REMOVED_STYLE, ADDED_STYLE, REMOVED_INNER_STYLE, ADDED_INNER_STYLE,
-    REMOVED_PLACEMARKER_OPEN, REMOVED_PLACEMARKER_CLOSED,
-    ADDED_PLACEMARKER_OPEN, ADDED_PLACEMARKER_CLOSED,
-    CHANGED_PLACEMARKER_OPEN, CHANGED_PLACEMARKER_CLOSED,
-    CHANGED_INTO_PLACEMARKER_OPEN, CHANGED_INTO_PLACEMARKER_CLOSED
-)
-from changedetectionio.notification.handler import apply_html_color_to_body
 from changedetectionio.store import ChangeDetectionStore
 from changedetectionio.auth_decorator import login_optionally_required
-from changedetectionio import html_tools, diff, strtobool
+from changedetectionio import html_tools
 from changedetectionio import worker_handler
 
-def build_diff_cell_visualizer(content, resolution=100):
-    """
-    Build a visual cell grid for the diff visualizer.
-
-    Analyzes the content for placemarkers indicating changes and creates a
-    grid of cells representing the document, with each cell marked as:
-    - 'deletion' for removed content
-    - 'insertion' for added content
-    - 'mixed' for cells containing both deletions and insertions
-    - empty string for cells with no changes
-
-    Args:
-        content: The diff content with placemarkers
-        resolution: Number of cells to create (default 100)
-
-    Returns:
-        List of dicts with 'class' key for each cell's CSS class
-    """
-    if not content:
-        return [{'class': ''} for _ in range(resolution)]
-    now = time.time()
-    # Work with character positions for better accuracy
-    content_length = len(content)
-
-    if content_length == 0:
-        return [{'class': ''} for _ in range(resolution)]
-
-    chars_per_cell = max(1, content_length / resolution)
-
-    # Track change type for each cell
-    cell_data = {}
-
-    # Placemarkers to detect
-    change_markers = {
-        REMOVED_PLACEMARKER_OPEN: 'deletion',
-        ADDED_PLACEMARKER_OPEN: 'insertion',
-        CHANGED_PLACEMARKER_OPEN: 'deletion',
-        CHANGED_INTO_PLACEMARKER_OPEN: 'insertion',
-    }
-
-    # Find all occurrences of each marker
-    for marker, change_type in change_markers.items():
-        pos = 0
-        while True:
-            pos = content.find(marker, pos)
-            if pos == -1:
-                break
-
-            # Calculate which cell this marker falls into
-            cell_index = min(int(pos / chars_per_cell), resolution - 1)
-
-            if cell_index not in cell_data:
-                cell_data[cell_index] = change_type
-            elif cell_data[cell_index] != change_type:
-                # Mixed changes in this cell
-                cell_data[cell_index] = 'mixed'
-
-            pos += len(marker)
-
-    # Build the cell list
-    cells = []
-    for i in range(resolution):
-        change_type = cell_data.get(i, '')
-        cells.append({'class': change_type})
-
-    logger.debug(f"Built diff cell visualizer: {len([c for c in cells if c['class']])} cells with changes out of {resolution} in {time.time() - now:.2f}s")
-
-    return cells
-
-
 def construct_blueprint(datastore: ChangeDetectionStore, update_q, queuedWatchMetaData, watch_check_update):
     views_blueprint = Blueprint('ui_views', __name__, template_folder="../ui/templates")
-
-    # Diff display preferences configuration
-    DIFF_PREFERENCES_CONFIG = {
-        'diff_changesOnly': {'default': False, 'type': 'bool'},
-        'diff_ignoreWhitespace': {'default': False, 'type': 'bool'},
-        'diff_removed': {'default': True, 'type': 'bool'},
-        'diff_added': {'default': True, 'type': 'bool'},
-        'diff_replaced': {'default': True, 'type': 'bool'},
-        'diff_type': {'default': 'diffLines', 'type': 'value'},
-    }
-
-    @views_blueprint.app_template_filter('diff_unescape_difference_spans')
-    def diff_unescape_difference_spans(content):
-        """Emulate Jinja2's auto-escape, then selectively unescape our diff spans."""
-        from markupsafe import escape
-
-        if not content:
-            return Markup('')
-
-        # Step 1: Escape everything like Jinja2 would (this makes it XSS-safe)
-        escaped_content = escape(str(content))
-
-        # Step 2: Unescape only our exact diff spans generated by apply_html_color_to_body()
-        # Pattern matches the exact structure:
-        # <span style="{STYLE}" role="{ROLE}" aria-label="{LABEL}" title="{TITLE}">
-
-        # Unescape outer span opening tags with full attributes (role, aria-label, title)
-        # Matches removed/added/changed/changed_into spans
-        result = re.sub(
-            rf'&lt;span style=&#34;({re.escape(REMOVED_STYLE)}|{re.escape(ADDED_STYLE)})&#34; '
-            rf'role=&#34;(deletion|insertion|note)&#34; '
-            rf'aria-label=&#34;([^&]+?)&#34; '
-            rf'title=&#34;([^&]+?)&#34;&gt;',
-            r'<span style="\1" role="\2" aria-label="\3" title="\4">',
-            str(escaped_content),
-            flags=re.IGNORECASE
-        )
-
-        # Unescape inner span opening tags (without additional attributes)
-        # This matches the darker background styles for changed parts within lines
-        result = re.sub(
-            rf'&lt;span style=&#34;({re.escape(REMOVED_INNER_STYLE)}|{re.escape(ADDED_INNER_STYLE)})&#34;&gt;',
-            r'<span style="\1">',
-            result,
-            flags=re.IGNORECASE
-        )
-
-        # Unescape closing tags (but only as many as we opened)
-        open_count = result.count('<span style=')
-        close_count = str(escaped_content).count('&lt;/span&gt;')
-
-        # Replace up to the number of spans we opened
-        for _ in range(min(open_count, close_count)):
-            result = result.replace('&lt;/span&gt;', '</span>', 1)
-
-        return Markup(result)
-
+    
     @views_blueprint.route("/preview/<string:uuid>", methods=['GET'])
     @login_optionally_required
     def preview_page(uuid):
@@ -170,11 +34,7 @@ def construct_blueprint(datastore: ChangeDetectionStore, update_q, queuedWatchMe
         is_html_webdriver = False
         if (watch.get('fetch_backend') == 'system' and system_uses_webdriver) or watch.get('fetch_backend') == 'html_webdriver' or watch.get('fetch_backend', '').startswith('extra_browser_'):
             is_html_webdriver = True
-
         triggered_line_numbers = []
-        ignored_line_numbers = []
-        blocked_line_numbers = []
-
         if datastore.data['watching'][uuid].history_n == 0 and (watch.get_error_text() or watch.get_error_snapshot()):
             flash("Preview unavailable - No fetch/check completed or triggers not reached", "error")
         else:
@@ -190,39 +50,30 @@ def construct_blueprint(datastore: ChangeDetectionStore, update_q, queuedWatchMe
                 content = watch.get_history_snapshot(timestamp)
 
                 triggered_line_numbers = html_tools.strip_ignore_text(content=content,
-                                                                      wordlist=watch.get('trigger_text'),
-                                                                      mode='line numbers'
-                                                                      )
-                ignored_line_numbers = html_tools.strip_ignore_text(content=content,
-                                                                      wordlist=watch.get('ignore_text'),
-                                                                      mode='line numbers'
-                                                                      )
-                blocked_line_numbers = html_tools.strip_ignore_text(content=content,
-                                                                      wordlist=watch.get("text_should_not_be_present"),
+                                                                      wordlist=watch['trigger_text'],
                                                                       mode='line numbers'
                                                                       )
+
             except Exception as e:
                 content.append({'line': f"File doesnt exist or unable to read timestamp {timestamp}", 'classes': ''})
 
         output = render_template("preview.html",
                                  content=content,
-                                 current_diff_url=watch['url'],
                                  current_version=timestamp,
+                                 history_n=watch.history_n,
                                  extra_stylesheets=extra_stylesheets,
                                  extra_title=f" - Diff - {watch.label} @ {timestamp}",
-                                 highlight_ignored_line_numbers=ignored_line_numbers,
-                                 highlight_triggered_line_numbers=triggered_line_numbers,
-                                 highlight_blocked_line_numbers=blocked_line_numbers,
-                                 history_n=watch.history_n,
+                                 triggered_line_numbers=triggered_line_numbers,
+                                 current_diff_url=watch['url'],
+                                 screenshot=watch.get_screenshot(),
+                                 watch=watch,
+                                 uuid=uuid,
                                  is_html_webdriver=is_html_webdriver,
                                  last_error=watch['last_error'],
-                                 last_error_screenshot=watch.get_error_snapshot(),
                                  last_error_text=watch.get_error_text(),
-                                 screenshot=watch.get_screenshot(),
-                                 uuid=uuid,
-                                 versions=versions,
-                                 watch=watch,
-                                 )
+                                 last_error_screenshot=watch.get_error_snapshot(),
+                                 versions=versions
+                                )
 
         return output
 
@@ -284,6 +135,7 @@ def construct_blueprint(datastore: ChangeDetectionStore, update_q, queuedWatchMe
             extract_form = forms.extractDataForm(formdata=request.form,
                                                  data={'extract_regex': request.form.get('extract_regex', '')}
                                                  )
+
         history = watch.history
         dates = list(history.keys())
 
@@ -323,64 +175,34 @@ def construct_blueprint(datastore: ChangeDetectionStore, update_q, queuedWatchMe
 
         datastore.set_last_viewed(uuid, time.time())
 
-        d_removed_opt =  strtobool(request.args.get('diff_removed', 'off'))
-        d_added_opt = strtobool(request.args.get('diff_added', 'off'))
-        d_replaced_opt = strtobool(request.args.get('diff_replaced', 'off'))
-        d_unchanged = strtobool(request.args.get('diff_changesOnly', 'on'))
-
-        any_special_opts = d_removed_opt or d_added_opt or d_replaced_opt
-        content = diff.render_diff(previous_version_file_contents=from_version_file_contents,
-                                   newest_version_file_contents=to_version_file_contents,
-
-                                   include_replaced=d_replaced_opt or not any_special_opts,
-                                   include_added=d_added_opt or not any_special_opts,
-                                   include_removed=d_removed_opt or not any_special_opts,
-                                   include_equal=d_unchanged,
-
-                                   ignore_junk=request.args.get('ignoreWhitespace'),
-
-                                   word_diff=request.args.get('diff_type') == 'diffWords',
-                                   )
-
-        # Build cell grid visualizer before applying HTML color (so we can detect placemarkers)
-        diff_cell_grid = build_diff_cell_visualizer(content)
-
-        content = apply_html_color_to_body(n_body=content)
-        offscreen_content = render_template("diff-offscreen-options.html")
-
-        output = render_template("diff.html",
-                                 bottom_horizontal_offscreen_contents=offscreen_content,
-                                 content=content,
+        return render_template("diff.html",
                                  current_diff_url=watch['url'],
-                                 diff_cell_grid=diff_cell_grid,
-                                 diff_prefs=request.args,
-                                 extra_stylesheets=extra_stylesheets,
-                                 extra_title=f" - {watch.label} - History",
-                                 extract_form=extract_form,
                                  from_version=str(from_version),
-                                 #initial_scroll_line_number=100,
+                                 to_version=str(to_version),
+                                 extra_stylesheets=extra_stylesheets,
+                                 extra_title=f" - Diff - {watch.label}",
+                                 extract_form=extract_form,
                                  is_html_webdriver=is_html_webdriver,
                                  last_error=watch['last_error'],
                                  last_error_screenshot=watch.get_error_snapshot(),
                                  last_error_text=watch.get_error_text(),
+                                 left_sticky=True,
                                  newest=to_version_file_contents,
                                  newest_version_timestamp=dates[-1],
                                  password_enabled_and_share_is_off=password_enabled_and_share_is_off,
-                                 pure_menu_fixed=False,
+                                 from_version_file_contents=from_version_file_contents,
+                                 to_version_file_contents=to_version_file_contents,
                                  screenshot=screenshot_url,
-                                 to_version=str(to_version),
                                  uuid=uuid,
-                                 versions=dates,  # All except current/last
-                                 watch_a=watch,
+                                 versions=dates, # All except current/last
+                                 watch_a=watch
                                  )
-        return output
 
     @views_blueprint.route("/diff/<string:uuid>", methods=['GET'])
     @login_optionally_required
     def diff_history_page(uuid):
         return _render_diff_template(uuid)
 
-
     @views_blueprint.route("/form/add/quickwatch", methods=['POST'])
     @login_optionally_required
     def form_quick_watch_add():

changedetectionio/content_fetchers/base.py

@@ -64,18 +64,6 @@ class Fetcher():
     # Time ONTOP of the system defined env minimum time
     render_extract_delay = 0
 
-    def clear_content(self):
-        """
-        Explicitly clear all content from memory to free up heap space.
-        Call this after content has been saved to disk.
-        """
-        self.content = None
-        if hasattr(self, 'raw_content'):
-            self.raw_content = None
-        self.screenshot = None
-        self.xpath_data = None
-        # Keep headers and status_code as they're small
-
     @abstractmethod
     def get_error(self):
         return self.error
@@ -140,7 +128,7 @@ class Fetcher():
     async def iterate_browser_steps(self, start_url=None):
         from changedetectionio.blueprint.browser_steps.browser_steps import steppable_browser_interface
         from playwright._impl._errors import TimeoutError, Error
-        from changedetectionio.jinja2_custom import render as jinja_render
+        from changedetectionio.safe_jinja import render as jinja_render
         step_n = 0
 
         if self.browser_steps is not None and len(self.browser_steps):

changedetectionio/content_fetchers/requests.py

@@ -51,7 +51,6 @@ class fetcher(Fetcher):
 
         session = requests.Session()
 
-
         if strtobool(os.getenv('ALLOW_FILE_URI', 'false')) and url.startswith('file://'):
             from requests_file import FileAdapter
             session.mount('file://', FileAdapter())

changedetectionio/diff.py

@@ -0,0 +1,113 @@
+import difflib
+from typing import List, Iterator, Union
+
+REMOVED_STYLE = "background-color: #fadad7; color: #b30000;"
+ADDED_STYLE = "background-color: #eaf2c2; color: #406619;"
+
+def same_slicer(lst: List[str], start: int, end: int) -> List[str]:
+    """Return a slice of the list, or a single element if start == end."""
+    return lst[start:end] if start != end else [lst[start]]
+
+def customSequenceMatcher(
+    before: List[str],
+    after: List[str],
+    include_equal: bool = False,
+    include_removed: bool = True,
+    include_added: bool = True,
+    include_replaced: bool = True,
+    include_change_type_prefix: bool = True,
+    html_colour: bool = False
+) -> Iterator[List[str]]:
+    """
+    Compare two sequences and yield differences based on specified parameters.
+
+    Args:
+        before (List[str]): Original sequence
+        after (List[str]): Modified sequence
+        include_equal (bool): Include unchanged parts
+        include_removed (bool): Include removed parts
+        include_added (bool): Include added parts
+        include_replaced (bool): Include replaced parts
+        include_change_type_prefix (bool): Add prefixes to indicate change types
+        html_colour (bool): Use HTML background colors for differences
+
+    Yields:
+        List[str]: Differences between sequences
+    """
+    cruncher = difflib.SequenceMatcher(isjunk=lambda x: x in " \t", a=before, b=after)
+
+
+
+    for tag, alo, ahi, blo, bhi in cruncher.get_opcodes():
+        if include_equal and tag == 'equal':
+            yield before[alo:ahi]
+        elif include_removed and tag == 'delete':
+            if html_colour:
+                yield [f'<span style="{REMOVED_STYLE}">{line}</span>' for line in same_slicer(before, alo, ahi)]
+            else:
+                yield [f"(removed) {line}" for line in same_slicer(before, alo, ahi)] if include_change_type_prefix else same_slicer(before, alo, ahi)
+        elif include_replaced and tag == 'replace':
+            if html_colour:
+                yield [f'<span style="{REMOVED_STYLE}">{line}</span>' for line in same_slicer(before, alo, ahi)] + \
+                      [f'<span style="{ADDED_STYLE}">{line}</span>' for line in same_slicer(after, blo, bhi)]
+            else:
+                yield [f"(changed) {line}" for line in same_slicer(before, alo, ahi)] + \
+                      [f"(into) {line}" for line in same_slicer(after, blo, bhi)] if include_change_type_prefix else same_slicer(before, alo, ahi) + same_slicer(after, blo, bhi)
+        elif include_added and tag == 'insert':
+            if html_colour:
+                yield [f'<span style="{ADDED_STYLE}">{line}</span>' for line in same_slicer(after, blo, bhi)]
+            else:
+                yield [f"(added) {line}" for line in same_slicer(after, blo, bhi)] if include_change_type_prefix else same_slicer(after, blo, bhi)
+
+def render_diff(
+    previous_version_file_contents: str,
+    newest_version_file_contents: str,
+    include_equal: bool = False,
+    include_removed: bool = True,
+    include_added: bool = True,
+    include_replaced: bool = True,
+    line_feed_sep: str = "\n",
+    include_change_type_prefix: bool = True,
+    patch_format: bool = False,
+    html_colour: bool = False
+) -> str:
+    """
+    Render the difference between two file contents.
+
+    Args:
+        previous_version_file_contents (str): Original file contents
+        newest_version_file_contents (str): Modified file contents
+        include_equal (bool): Include unchanged parts
+        include_removed (bool): Include removed parts
+        include_added (bool): Include added parts
+        include_replaced (bool): Include replaced parts
+        line_feed_sep (str): Separator for lines in output
+        include_change_type_prefix (bool): Add prefixes to indicate change types
+        patch_format (bool): Use patch format for output
+        html_colour (bool): Use HTML background colors for differences
+
+    Returns:
+        str: Rendered difference
+    """
+    newest_lines = [line.rstrip() for line in newest_version_file_contents.splitlines()]
+    previous_lines = [line.rstrip() for line in previous_version_file_contents.splitlines()] if previous_version_file_contents else []
+
+    if patch_format:
+        patch = difflib.unified_diff(previous_lines, newest_lines)
+        return line_feed_sep.join(patch)
+
+    rendered_diff = customSequenceMatcher(
+        before=previous_lines,
+        after=newest_lines,
+        include_equal=include_equal,
+        include_removed=include_removed,
+        include_added=include_added,
+        include_replaced=include_replaced,
+        include_change_type_prefix=include_change_type_prefix,
+        html_colour=html_colour
+    )
+
+    def flatten(lst: List[Union[str, List[str]]]) -> str:
+        return line_feed_sep.join(flatten(x) if isinstance(x, list) else x for x in lst)
+
+    return flatten(rendered_diff)

changedetectionio/diff/__init__.py

@@ -1,475 +0,0 @@
-"""
-Diff rendering module for change detection.
-
-This module provides functions for rendering differences between text content,
-with support for various output formats and tokenization strategies.
-"""
-
-import difflib
-from typing import List, Iterator, Union
-from loguru import logger
-import diff_match_patch as dmp_module
-import re
-
-from .tokenizers import TOKENIZERS, tokenize_words_and_html
-
-# Remember! gmail, outlook etc dont support <style> must be inline.
-# Gmail: strips <ins> and <del> tags entirely.
-# This is for the WHOLE line background style
-REMOVED_STYLE = "background-color: #fadad7; color: #b30000;"
-ADDED_STYLE = "background-color: #eaf2c2; color: #406619;"
-HTML_REMOVED_STYLE = REMOVED_STYLE  # Export alias for handler.py
-HTML_ADDED_STYLE = ADDED_STYLE      # Export alias for handler.py
-
-# Darker backgrounds for nested highlighting (changed parts within lines)
-REMOVED_INNER_STYLE = "background-color: #ff867a; color: #111;"
-ADDED_INNER_STYLE = "background-color: #b2e841; color: #444;"
-HTML_CHANGED_STYLE = REMOVED_STYLE
-HTML_CHANGED_INTO_STYLE = ADDED_STYLE
-
-# Placemarker constants - these get replaced by apply_service_tweaks() in handler.py
-# Something that cant get escaped to HTML by accident
-REMOVED_PLACEMARKER_OPEN = '@removed_PLACEMARKER_OPEN'
-REMOVED_PLACEMARKER_CLOSED = '@removed_PLACEMARKER_CLOSED'
-
-ADDED_PLACEMARKER_OPEN = '@added_PLACEMARKER_OPEN'
-ADDED_PLACEMARKER_CLOSED = '@added_PLACEMARKER_CLOSED'
-
-CHANGED_PLACEMARKER_OPEN = '@changed_PLACEMARKER_OPEN'
-CHANGED_PLACEMARKER_CLOSED = '@changed_PLACEMARKER_CLOSED'
-
-CHANGED_INTO_PLACEMARKER_OPEN = '@changed_into_PLACEMARKER_OPEN'
-CHANGED_INTO_PLACEMARKER_CLOSED = '@changed_into_PLACEMARKER_CLOSED'
-
-# Compiled regex patterns for performance
-WHITESPACE_NORMALIZE_RE = re.compile(r'\s+')
-
-
-def render_inline_word_diff(before_line: str, after_line: str, ignore_junk: bool = False, markdown_style: str = None, tokenizer: str = 'words_and_html') -> tuple[str, bool]:
-    """
-    Render word-level differences between two lines inline using diff-match-patch library.
-
-    Args:
-        before_line: Original line text
-        after_line: Modified line text
-        ignore_junk: Ignore whitespace-only changes
-        markdown_style: Unused (kept for backwards compatibility)
-        tokenizer: Name of tokenizer to use from TOKENIZERS registry (default: 'words_and_html')
-
-    Returns:
-        tuple[str, bool]: (diff output with inline word-level highlighting, has_changes flag)
-    """
-    # Normalize whitespace if ignore_junk is enabled
-    if ignore_junk:
-        # Normalize whitespace: replace multiple spaces/tabs with single space
-        before_normalized = WHITESPACE_NORMALIZE_RE.sub(' ', before_line)
-        after_normalized = WHITESPACE_NORMALIZE_RE.sub(' ', after_line)
-    else:
-        before_normalized = before_line
-        after_normalized = after_line
-
-    # Use diff-match-patch with word-level tokenization
-    # Strategy: Use linesToChars to treat words as atomic units
-    dmp = dmp_module.diff_match_patch()
-
-    # Get the tokenizer function from the registry
-    tokenizer_func = TOKENIZERS.get(tokenizer, tokenize_words_and_html)
-
-    # Tokenize both lines using the selected tokenizer
-    before_tokens = tokenizer_func(before_normalized)
-    after_tokens = tokenizer_func(after_normalized or ' ')
-
-    # Create mappings for linesToChars (using it for word-mode)
-    # Join tokens with newline so each "line" is a token
-    before_text = '\n'.join(before_tokens)
-    after_text = '\n'.join(after_tokens)
-
-    # Use linesToChars for word-mode diffing
-    lines_result = dmp.diff_linesToChars(before_text, after_text)
-    line_before, line_after, line_array = lines_result
-
-    # Perform diff on the encoded strings
-    diffs = dmp.diff_main(line_before, line_after, False)
-
-    # Convert back to original text
-    dmp.diff_charsToLines(diffs, line_array)
-
-    # Remove the newlines we added for tokenization
-    diffs = [(op, text.replace('\n', '')) for op, text in diffs]
-
-    # DON'T apply semantic cleanup here - it would break token boundaries
-    # (e.g., "63" -> "66" would become "6" + "3" vs "6" + "6")
-    # We want to preserve the tokenizer's word boundaries
-
-    # Check if there are any changes
-    has_changes = any(op != 0 for op, _ in diffs)
-
-    if ignore_junk and not has_changes:
-        return after_line, False
-
-    # Check if the whole line is replaced (no unchanged content)
-    whole_line_replaced = not any(op == 0 and text.strip() for op, text in diffs)
-
-    # Build the output using placemarkers
-    # When whole line is replaced, wrap entire removed content once and entire added content once
-    if whole_line_replaced:
-        removed_tokens = []
-        added_tokens = []
-
-        for op, text in diffs:
-            if op == 0:  # Equal (e.g., whitespace tokens in common positions)
-                # Include in both removed and added to preserve spacing
-                removed_tokens.append(text)
-                added_tokens.append(text)
-            elif op == -1:  # Deletion
-                removed_tokens.append(text)
-            elif op == 1:  # Insertion
-                added_tokens.append(text)
-
-        # Join all tokens and wrap the entire string once for removed, once for added
-        result_parts = []
-
-        if removed_tokens:
-            removed_full = ''.join(removed_tokens).rstrip()
-            trailing_removed = ''.join(removed_tokens)[len(removed_full):] if len(''.join(removed_tokens)) > len(removed_full) else ''
-            result_parts.append(f'{CHANGED_PLACEMARKER_OPEN}{removed_full}{CHANGED_PLACEMARKER_CLOSED}{trailing_removed}')
-
-        if added_tokens:
-            if result_parts:  # Add newline between removed and added
-                result_parts.append('\n')
-            added_full = ''.join(added_tokens).rstrip()
-            trailing_added = ''.join(added_tokens)[len(added_full):] if len(''.join(added_tokens)) > len(added_full) else ''
-            result_parts.append(f'{CHANGED_INTO_PLACEMARKER_OPEN}{added_full}{CHANGED_INTO_PLACEMARKER_CLOSED}{trailing_added}')
-
-        return ''.join(result_parts), has_changes
-    else:
-        # Inline changes within the line
-        result_parts = []
-        for op, text in diffs:
-            if op == 0:  # Equal
-                result_parts.append(text)
-            elif op == 1:  # Insertion
-                # Don't wrap empty content (e.g., whitespace-only tokens after rstrip)
-                content = text.rstrip()
-                trailing = text[len(content):] if len(text) > len(content) else ''
-                if content:
-                    result_parts.append(f'{ADDED_PLACEMARKER_OPEN}{content}{ADDED_PLACEMARKER_CLOSED}{trailing}')
-                else:
-                    result_parts.append(trailing)
-            elif op == -1:  # Deletion
-                # Don't wrap empty content (e.g., whitespace-only tokens after rstrip)
-                content = text.rstrip()
-                trailing = text[len(content):] if len(text) > len(content) else ''
-                if content:
-                    result_parts.append(f'{REMOVED_PLACEMARKER_OPEN}{content}{REMOVED_PLACEMARKER_CLOSED}{trailing}')
-                else:
-                    result_parts.append(trailing)
-
-        return ''.join(result_parts), has_changes
-
-
-def render_nested_line_diff(before_line: str, after_line: str, ignore_junk: bool = False, tokenizer: str = 'words_and_html') -> tuple[str, str, bool]:
-    """
-    Render line-level differences with nested highlighting for changed parts.
-
-    Returns two separate lines:
-    - Before line: light red background with dark red on removed parts
-    - After line: light green background with dark green on added parts
-
-    Args:
-        before_line: Original line text
-        after_line: Modified line text
-        ignore_junk: Ignore whitespace-only changes
-        tokenizer: Name of tokenizer to use from TOKENIZERS registry
-
-    Returns:
-        tuple[str, str, bool]: (before_with_highlights, after_with_highlights, has_changes)
-    """
-    # Normalize whitespace if ignore_junk is enabled
-    if ignore_junk:
-        before_normalized = WHITESPACE_NORMALIZE_RE.sub(' ', before_line)
-        after_normalized = WHITESPACE_NORMALIZE_RE.sub(' ', after_line)
-    else:
-        before_normalized = before_line
-        after_normalized = after_line
-
-    # Use diff-match-patch with word-level tokenization
-    dmp = dmp_module.diff_match_patch()
-
-    # Get the tokenizer function from the registry
-    tokenizer_func = TOKENIZERS.get(tokenizer, tokenize_words_and_html)
-
-    # Tokenize both lines
-    before_tokens = tokenizer_func(before_normalized)
-    after_tokens = tokenizer_func(after_normalized or ' ')
-
-    # Create mappings for linesToChars
-    before_text = '\n'.join(before_tokens)
-    after_text = '\n'.join(after_tokens)
-
-    # Use linesToChars for word-mode diffing
-    lines_result = dmp.diff_linesToChars(before_text, after_text)
-    line_before, line_after, line_array = lines_result
-
-    # Perform diff on the encoded strings
-    diffs = dmp.diff_main(line_before, line_after, False)
-
-    # Convert back to original text
-    dmp.diff_charsToLines(diffs, line_array)
-
-    # Remove the newlines we added for tokenization
-    diffs = [(op, text.replace('\n', '')) for op, text in diffs]
-
-    # DON'T apply semantic cleanup here - it would break token boundaries
-    # (e.g., "63" -> "66" would become "6" + "3" vs "6" + "6")
-    # We want to preserve the tokenizer's word boundaries
-
-    # Check if there are any changes
-    has_changes = any(op != 0 for op, _ in diffs)
-
-    if ignore_junk and not has_changes:
-        return before_line, after_line, False
-
-    # Build the before line (with nested highlighting for removed parts)
-    before_parts = []
-    for op, text in diffs:
-        if op == 0:  # Equal
-            before_parts.append(text)
-        elif op == -1:  # Deletion (in before)
-            before_parts.append(f'<span style="{REMOVED_INNER_STYLE}">{text}</span>')
-        # Skip insertions (op == 1) for the before line
-
-    before_content = ''.join(before_parts)
-
-    # Build the after line (with nested highlighting for added parts)
-    after_parts = []
-    for op, text in diffs:
-        if op == 0:  # Equal
-            after_parts.append(text)
-        elif op == 1:  # Insertion (in after)
-            after_parts.append(f'<span style="{ADDED_INNER_STYLE}">{text}</span>')
-        # Skip deletions (op == -1) for the after line
-
-    after_content = ''.join(after_parts)
-
-    # Wrap content with placemarkers (inner HTML highlighting is preserved)
-    before_html = f'{CHANGED_PLACEMARKER_OPEN}{before_content}{CHANGED_PLACEMARKER_CLOSED}'
-    after_html = f'{CHANGED_INTO_PLACEMARKER_OPEN}{after_content}{CHANGED_INTO_PLACEMARKER_CLOSED}'
-
-    return before_html, after_html, has_changes
-
-
-def same_slicer(lst: List[str], start: int, end: int) -> List[str]:
-    """Return a slice of the list, or a single element if start == end."""
-    return lst[start:end] if start != end else [lst[start]]
-
-def customSequenceMatcher(
-    before: List[str],
-    after: List[str],
-    include_equal: bool = False,
-    include_removed: bool = True,
-    include_added: bool = True,
-    include_replaced: bool = True,
-    include_change_type_prefix: bool = True,
-    word_diff: bool = False,
-    context_lines: int = 0,
-    case_insensitive: bool = False,
-    ignore_junk: bool = False,
-    tokenizer: str = 'words_and_html'
-) -> Iterator[List[str]]:
-    """
-    Compare two sequences and yield differences based on specified parameters.
-
-    Args:
-        before (List[str]): Original sequence
-        after (List[str]): Modified sequence
-        include_equal (bool): Include unchanged parts
-        include_removed (bool): Include removed parts
-        include_added (bool): Include added parts
-        include_replaced (bool): Include replaced parts
-        include_change_type_prefix (bool): Add prefixes to indicate change types
-        word_diff (bool): Use word-level diffing for replaced lines (controls inline rendering)
-        context_lines (int): Number of unchanged lines to show around changes (like grep -C)
-        case_insensitive (bool): Perform case-insensitive comparison
-        ignore_junk (bool): Ignore whitespace-only changes
-        tokenizer (str): Name of tokenizer to use from TOKENIZERS registry (default: 'words_and_html')
-
-    Yields:
-        List[str]: Differences between sequences
-    """
-    # Prepare sequences for comparison (lowercase if case-insensitive, normalize whitespace if ignore_junk)
-    def prepare_line(line):
-        if case_insensitive:
-            line = line.lower()
-        if ignore_junk:
-            # Normalize whitespace: replace multiple spaces/tabs with single space
-            line = WHITESPACE_NORMALIZE_RE.sub(' ', line)
-        return line
-
-    compare_before = [prepare_line(line) for line in before]
-    compare_after = [prepare_line(line) for line in after]
-
-    cruncher = difflib.SequenceMatcher(isjunk=lambda x: x in " \t", a=compare_before, b=compare_after)
-
-    # When context_lines is set and include_equal is False, we need to track which equal lines to include
-    if context_lines > 0 and not include_equal:
-        opcodes = list(cruncher.get_opcodes())
-        # Mark equal ranges that should be included based on context
-        included_equal_ranges = set()
-
-        for i, (tag, alo, ahi, blo, bhi) in enumerate(opcodes):
-            if tag != 'equal':
-                # Include context lines before this change
-                for j in range(max(0, i - 1), i):
-                    if opcodes[j][0] == 'equal':
-                        prev_alo, prev_ahi = opcodes[j][1], opcodes[j][2]
-                        # Include last N lines of the previous equal block
-                        context_start = max(prev_alo, prev_ahi - context_lines)
-                        for line_num in range(context_start, prev_ahi):
-                            included_equal_ranges.add(line_num)
-
-                # Include context lines after this change
-                for j in range(i + 1, min(len(opcodes), i + 2)):
-                    if opcodes[j][0] == 'equal':
-                        next_alo, next_ahi = opcodes[j][1], opcodes[j][2]
-                        # Include first N lines of the next equal block
-                        context_end = min(next_ahi, next_alo + context_lines)
-                        for line_num in range(next_alo, context_end):
-                            included_equal_ranges.add(line_num)
-
-    # Remember! gmail, outlook etc dont support <style> must be inline.
-    # Gmail: strips <ins> and <del> tags entirely.
-    for tag, alo, ahi, blo, bhi in cruncher.get_opcodes():
-        if tag == 'equal':
-            if include_equal:
-                yield before[alo:ahi]
-            elif context_lines > 0:
-                # Only include equal lines that are in the context range
-                context_lines_to_include = [before[i] for i in range(alo, ahi) if i in included_equal_ranges]
-                if context_lines_to_include:
-                    yield context_lines_to_include
-        elif include_removed and tag == 'delete':
-            if include_change_type_prefix:
-                yield [f'{REMOVED_PLACEMARKER_OPEN}{line}{REMOVED_PLACEMARKER_CLOSED}' for line in same_slicer(before, alo, ahi)]
-            else:
-                yield same_slicer(before, alo, ahi)
-        elif include_replaced and tag == 'replace':
-            before_lines = same_slicer(before, alo, ahi)
-            after_lines = same_slicer(after, blo, bhi)
-
-            # Use inline word-level diff for single line replacements when word_diff is enabled
-            if word_diff and len(before_lines) == 1 and len(after_lines) == 1:
-                inline_diff, has_changes = render_inline_word_diff(before_lines[0], after_lines[0], ignore_junk=ignore_junk, tokenizer=tokenizer)
-                # Check if there are any actual changes (not just whitespace when ignore_junk is enabled)
-                if ignore_junk and not has_changes:
-                    # No real changes, skip this line
-                    continue
-                yield [inline_diff]
-            else:
-                # Fall back to line-level diff for multi-line changes
-                if include_change_type_prefix:
-                    yield [f'{CHANGED_PLACEMARKER_OPEN}{line}{CHANGED_PLACEMARKER_CLOSED}' for line in before_lines] + \
-                          [f'{CHANGED_INTO_PLACEMARKER_OPEN}{line}{CHANGED_INTO_PLACEMARKER_CLOSED}' for line in after_lines]
-                else:
-                    yield before_lines + after_lines
-        elif include_added and tag == 'insert':
-            if include_change_type_prefix:
-                yield [f'{ADDED_PLACEMARKER_OPEN}{line}{ADDED_PLACEMARKER_CLOSED}' for line in same_slicer(after, blo, bhi)]
-            else:
-                yield same_slicer(after, blo, bhi)
-
-def render_diff(
-    previous_version_file_contents: str,
-    newest_version_file_contents: str,
-    include_equal: bool = False,
-    include_removed: bool = True,
-    include_added: bool = True,
-    include_replaced: bool = True,
-    include_change_type_prefix: bool = True,
-    patch_format: bool = False,
-    word_diff: bool = True,
-    context_lines: int = 0,
-    case_insensitive: bool = False,
-    ignore_junk: bool = False,
-    tokenizer: str = 'words_and_html'
-) -> str:
-    """
-    Render the difference between two file contents.
-
-    Args:
-        previous_version_file_contents (str): Original file contents
-        newest_version_file_contents (str): Modified file contents
-        include_equal (bool): Include unchanged parts
-        include_removed (bool): Include removed parts
-        include_added (bool): Include added parts
-        include_replaced (bool): Include replaced parts
-        include_change_type_prefix (bool): Add prefixes to indicate change types
-        patch_format (bool): Use patch format for output
-        word_diff (bool): Use word-level diffing for replaced lines (controls inline rendering)
-        context_lines (int): Number of unchanged lines to show around changes (like grep -C)
-        case_insensitive (bool): Perform case-insensitive comparison, By default the test_json_diff/process.py is case sensitive, so this follows same logic
-        ignore_junk (bool): Ignore whitespace-only changes
-        tokenizer (str): Name of tokenizer to use from TOKENIZERS registry (default: 'words_and_html')
-
-    Returns:
-        str: Rendered difference
-    """
-    newest_lines = [line.rstrip() for line in newest_version_file_contents.splitlines()]
-    previous_lines = [line.rstrip() for line in previous_version_file_contents.splitlines()] if previous_version_file_contents else []
-    logger.debug(
-        f"diff options: "
-        f"include_equal={include_equal}, "
-        f"include_removed={include_removed}, "
-        f"include_added={include_added}, "
-        f"include_replaced={include_replaced}, "
-        f"include_change_type_prefix={include_change_type_prefix}, "
-        f"patch_format={patch_format}, "
-        f"word_diff={word_diff}, "
-        f"context_lines={context_lines}, "
-        f"case_insensitive={case_insensitive}, "
-        f"ignore_junk={ignore_junk}, "
-        f"tokenizer={tokenizer}"
-    )
-    if patch_format:
-        patch = difflib.unified_diff(previous_lines, newest_lines)
-        return "\n".join(patch)
-
-    rendered_diff = customSequenceMatcher(
-        before=previous_lines,
-        after=newest_lines,
-        include_equal=include_equal,
-        include_removed=include_removed,
-        include_added=include_added,
-        include_replaced=include_replaced,
-        include_change_type_prefix=include_change_type_prefix,
-        word_diff=word_diff,
-        context_lines=context_lines,
-        case_insensitive=case_insensitive,
-        ignore_junk=ignore_junk,
-        tokenizer=tokenizer
-    )
-
-    def flatten(lst: List[Union[str, List[str]]]) -> str:
-        result = []
-        for x in lst:
-            if isinstance(x, list):
-                result.extend(x)
-            else:
-                result.append(x)
-        return "\n".join(result)
-
-    return flatten(rendered_diff)
-
-
-# Export main public API
-__all__ = [
-    'render_diff',
-    'customSequenceMatcher',
-    'render_inline_word_diff',
-    'render_nested_line_diff',
-    'TOKENIZERS',
-    'REMOVED_STYLE',
-    'ADDED_STYLE',
-    'REMOVED_INNER_STYLE',
-    'ADDED_INNER_STYLE',
-]

changedetectionio/diff/tokenizers/__init__.py

@@ -1,23 +0,0 @@
-"""
-Tokenizers for diff operations.
-
-This module provides various tokenization strategies for use with the diff system.
-New tokenizers can be easily added by:
-1. Creating a new module in this directory
-2. Importing and registering it in the TOKENIZERS dictionary below
-"""
-
-from .natural_text import tokenize_words
-from .words_and_html import tokenize_words_and_html
-
-# Tokenizer registry - maps tokenizer names to functions
-TOKENIZERS = {
-    'words': tokenize_words,
-    'words_and_html': tokenize_words_and_html,
-}
-
-__all__ = [
-    'tokenize_words',
-    'tokenize_words_and_html',
-    'TOKENIZERS',
-]

changedetectionio/diff/tokenizers/natural_text.py

@@ -1,44 +0,0 @@
-"""
-Simple word tokenizer using whitespace boundaries.
-
-This is a simpler tokenizer that treats all whitespace as token boundaries
-without special handling for HTML tags or other markup.
-"""
-
-from typing import List
-
-
-def tokenize_words(text: str) -> List[str]:
-    """
-    Split text into words using simple whitespace boundaries.
-
-    This is a simpler tokenizer that treats all whitespace as token boundaries
-    without special handling for HTML tags.
-
-    Args:
-        text: Input text to tokenize
-
-    Returns:
-        List of tokens (words and whitespace)
-
-    Examples:
-        >>> tokenize_words("Hello world")
-        ['Hello', ' ', 'world']
-        >>> tokenize_words("one  two")
-        ['one', ' ', ' ', 'two']
-    """
-    tokens = []
-    current = ''
-
-    for char in text:
-        if char.isspace():
-            if current:
-                tokens.append(current)
-                current = ''
-            tokens.append(char)
-        else:
-            current += char
-
-    if current:
-        tokens.append(current)
-    return tokens

changedetectionio/diff/tokenizers/words_and_html.py

@@ -1,61 +0,0 @@
-"""
-Tokenizer that preserves HTML tags as atomic units while splitting on whitespace.
-
-This tokenizer is specifically designed for HTML content where:
-- HTML tags should remain intact (e.g., '<p>', '<a href="...">')
-- Whitespace tokens are preserved for accurate diff reconstruction
-- Words are split on whitespace boundaries
-"""
-
-from typing import List
-
-
-def tokenize_words_and_html(text: str) -> List[str]:
-    """
-    Split text into words and boundaries (spaces, HTML tags).
-
-    This tokenizer preserves HTML tags as atomic units while splitting on whitespace.
-    Useful for content that contains HTML markup.
-
-    Args:
-        text: Input text to tokenize
-
-    Returns:
-        List of tokens (words, spaces, HTML tags)
-
-    Examples:
-        >>> tokenize_words_and_html("<p>Hello world</p>")
-        ['<p>', 'Hello', ' ', 'world', '</p>']
-        >>> tokenize_words_and_html("<a href='test.com'>link</a>")
-        ['<a href=\\'test.com\\'>', 'link', '</a>']
-    """
-    tokens = []
-    current = ''
-    in_tag = False
-
-    for char in text:
-        if char == '<':
-            # Start of HTML tag
-            if current:
-                tokens.append(current)
-                current = ''
-            current = '<'
-            in_tag = True
-        elif char == '>' and in_tag:
-            # End of HTML tag
-            current += '>'
-            tokens.append(current)
-            current = ''
-            in_tag = False
-        elif char.isspace() and not in_tag:
-            # Space outside of tag
-            if current:
-                tokens.append(current)
-                current = ''
-            tokens.append(char)
-        else:
-            current += char
-
-    if current:
-        tokens.append(current)
-    return tokens

changedetectionio/flask_app.py

@@ -38,7 +38,7 @@ from loguru import logger
 
 from changedetectionio import __version__
 from changedetectionio import queuedWatchMetaData
-from changedetectionio.api import Watch, WatchHistory, WatchSingleHistory, WatchHistoryDiff, CreateWatch, Import, SystemInfo, Tag, Tags, Notifications, WatchFavicon
+from changedetectionio.api import Watch, WatchHistory, WatchSingleHistory, CreateWatch, Import, SystemInfo, Tag, Tags, Notifications, WatchFavicon
 from changedetectionio.api.Search import Search
 from .time_handler import is_within_schedule
 
@@ -133,11 +133,6 @@ def get_socketio_path():
     # Socket.IO will be available at {prefix}/socket.io/
     return prefix
 
-@app.template_global('is_safe_valid_url')
-def _is_safe_valid_url(test_url):
-    from .validate_url import is_safe_valid_url
-    return is_safe_valid_url(test_url)
-
 
 @app.template_filter('format_number_locale')
 def _jinja2_filter_format_number_locale(value: float) -> str:
@@ -307,9 +302,6 @@ def changedetection_app(config=None, datastore_o=None):
                 return login_manager.unauthorized()
 
 
-    watch_api.add_resource(WatchHistoryDiff,
-                           '/api/v1/watch/<string:uuid>/difference/<string:from_timestamp>/<string:to_timestamp>',
-                           resource_class_kwargs={'datastore': datastore})
     watch_api.add_resource(WatchSingleHistory,
                            '/api/v1/watch/<string:uuid>/history/<string:timestamp>',
                            resource_class_kwargs={'datastore': datastore, 'update_q': update_q})
@@ -390,7 +382,7 @@ def changedetection_app(config=None, datastore_o=None):
             # We would sometimes get login loop errors on sites hosted in sub-paths
 
             # note for the future:
-            #            if not is_safe_valid_url(next):
+            #            if not is_safe_url(next):
             #                return flask.abort(400)
             return redirect(url_for('watchlist.index'))
 
@@ -803,7 +795,7 @@ def ticker_thread_check_time_launch_checks():
             else:
                 time_schedule_limit = watch.get('time_schedule_limit')
                 logger.trace(f"{uuid} Time scheduler - Using watch settings (not global settings)")
-            tz_name = datastore.data['settings']['application'].get('scheduler_timezone_default', os.getenv('TZ', 'UTC').strip())
+            tz_name = datastore.data['settings']['application'].get('timezone', 'UTC')
 
             if time_schedule_limit and time_schedule_limit.get('enabled'):
                 try:

changedetectionio/forms.py

@@ -5,7 +5,6 @@ from wtforms.widgets.core import TimeInput
 
 from changedetectionio.blueprint.rss import RSS_FORMAT_TYPES
 from changedetectionio.conditions.form import ConditionFormRow
-from changedetectionio.notification_service import NotificationContextData
 from changedetectionio.strtobool import strtobool
 
 from wtforms import (
@@ -28,8 +27,11 @@ from wtforms.utils import unset_value
 
 from wtforms.validators import ValidationError
 
+from validators.url import url as url_validator
+
 from changedetectionio.widgets import TernaryNoneBooleanField
 
+
 # default
 # each select <option data-enabled="enabled-0-0"
 from changedetectionio.blueprint.browser_steps.browser_steps import browser_step_ui_config
@@ -467,16 +469,11 @@ class ValidateAppRiseServers(object):
         import apprise
         from .notification.apprise_plugin.assets import apprise_asset
         from .notification.apprise_plugin.custom_handlers import apprise_http_custom_handler  # noqa: F401
-        from changedetectionio.jinja2_custom import render as jinja_render
 
         apobj = apprise.Apprise(asset=apprise_asset)
 
         for server_url in field.data:
-            generic_notification_context_data = NotificationContextData()
-            # Make sure something is atleast in all those regular token fields
-            generic_notification_context_data.set_random_for_validation()
-
-            url = jinja_render(template_str=server_url.strip(), **generic_notification_context_data).strip()
+            url = server_url.strip()
             if url.startswith("#"):
                 continue
 
@@ -490,8 +487,9 @@ class ValidateJinja2Template(object):
     """
     def __call__(self, form, field):
         from changedetectionio import notification
-        from changedetectionio.jinja2_custom import create_jinja_env
+
         from jinja2 import BaseLoader, TemplateSyntaxError, UndefinedError
+        from jinja2.sandbox import ImmutableSandboxedEnvironment
         from jinja2.meta import find_undeclared_variables
         import jinja2.exceptions
 
@@ -499,11 +497,9 @@ class ValidateJinja2Template(object):
         joined_data = ' '.join(map(str, field.data)) if isinstance(field.data, list) else f"{field.data}"
 
         try:
-            # Use the shared helper to create a properly configured environment
-            jinja2_env = create_jinja_env(loader=BaseLoader)
-
-            # Add notification tokens for validation
-            jinja2_env.globals.update(NotificationContextData())
+            jinja2_env = ImmutableSandboxedEnvironment(loader=BaseLoader, extensions=['jinja2_time.TimeExtension'])
+            jinja2_env.globals.update(notification.valid_tokens)
+            # Extra validation tokens provided on the form_class(... extra_tokens={}) setup
             if hasattr(field, 'extra_notification_tokens'):
                 jinja2_env.globals.update(field.extra_notification_tokens)
 
@@ -515,7 +511,6 @@ class ValidateJinja2Template(object):
         except jinja2.exceptions.SecurityError as e:
             raise ValidationError(f"This is not a valid Jinja2 template: {e}") from e
 
-        # Check for undeclared variables
         ast = jinja2_env.parse(joined_data)
         undefined = ", ".join(find_undeclared_variables(ast))
         if undefined:
@@ -538,10 +533,19 @@ class validateURL(object):
 
 
 def validate_url(test_url):
-    from changedetectionio.validate_url import is_safe_valid_url
-    if not is_safe_valid_url(test_url):
+    # If hosts that only contain alphanumerics are allowed ("localhost" for example)
+    try:
+        url_validator(test_url, simple_host=allow_simplehost)
+    except validators.ValidationError:
+        #@todo check for xss
+        message = f"'{test_url}' is not a valid URL."
         # This should be wtforms.validators.
-        raise ValidationError('Watch protocol is not permitted or invalid URL format')
+        raise ValidationError(message)
+
+    from .model.Watch import is_safe_url
+    if not is_safe_url(test_url):
+        # This should be wtforms.validators.
+        raise ValidationError('Watch protocol is not permitted by SAFE_PROTOCOL_REGEX or incorrect URL format')
 
 
 class ValidateSinglePythonRegexString(object):
@@ -674,51 +678,6 @@ class ValidateCSSJSONXPATHInput(object):
                 except:
                     raise ValidationError("A system-error occurred when validating your jq expression")
 
-class ValidateSimpleURL:
-    """Validate that the value can be parsed by urllib.parse.urlparse() and has a scheme/netloc."""
-    def __init__(self, message=None):
-        self.message = message or "Invalid URL."
-
-    def __call__(self, form, field):
-        data = (field.data or "").strip()
-        if not data:
-            return  # empty is OK — pair with validators.Optional()
-        from urllib.parse import urlparse
-
-        parsed = urlparse(data)
-        if not parsed.scheme or not parsed.netloc:
-            raise ValidationError(self.message)
-
-class ValidateStartsWithRegex(object):
-    def __init__(self, regex, *, flags=0, message=None, allow_empty=True, split_lines=True):
-        # compile with given flags (we’ll pass re.IGNORECASE below)
-        self.pattern = re.compile(regex, flags) if isinstance(regex, str) else regex
-        self.message = message
-        self.allow_empty = allow_empty
-        self.split_lines = split_lines
-
-    def __call__(self, form, field):
-        data = field.data
-        if not data:
-            return
-
-        # normalize into list of lines
-        if isinstance(data, str) and self.split_lines:
-            lines = data.splitlines()
-        elif isinstance(data, (list, tuple)):
-            lines = data
-        else:
-            lines = [data]
-
-        for line in lines:
-            stripped = line.strip()
-            if not stripped:
-                if self.allow_empty:
-                    continue
-                raise ValidationError(self.message or "Empty value not allowed.")
-            if not self.pattern.match(stripped):
-                raise ValidationError(self.message or "Invalid value.")
-
 class quickWatchForm(Form):
     from . import processors
 
@@ -729,6 +688,7 @@ class quickWatchForm(Form):
     edit_and_watch_submit_button = SubmitField('Edit > Watch', render_kw={"class": "pure-button pure-button-primary"})
 
 
+
 # Common to a single watch and the global settings
 class commonSettingsForm(Form):
     from . import processors
@@ -741,21 +701,13 @@ class commonSettingsForm(Form):
 
     fetch_backend = RadioField(u'Fetch Method', choices=content_fetchers.available_fetchers(), validators=[ValidateContentFetcherIsReady()])
     notification_body = TextAreaField('Notification Body', default='{{ watch_url }} had a change.', validators=[validators.Optional(), ValidateJinja2Template()])
-    notification_format = SelectField('Notification format', choices=list(valid_notification_formats.items()))
+    notification_format = SelectField('Notification format', choices=valid_notification_formats.keys())
     notification_title = StringField('Notification Title', default='ChangeDetection.io Notification - {{ watch_url }}', validators=[validators.Optional(), ValidateJinja2Template()])
     notification_urls = StringListField('Notification URL List', validators=[validators.Optional(), ValidateAppRiseServers(), ValidateJinja2Template()])
     processor = RadioField( label=u"Processor - What do you want to achieve?", choices=processors.available_processors(), default="text_json_diff")
-    scheduler_timezone_default = StringField("Default timezone for watch check scheduler", render_kw={"list": "timezones"}, validators=[validateTimeZoneName()])
+    timezone = StringField("Timezone for watch schedule", render_kw={"list": "timezones"}, validators=[validateTimeZoneName()])
     webdriver_delay = IntegerField('Wait seconds before extracting text', validators=[validators.Optional(), validators.NumberRange(min=1, message="Should contain one or more seconds")])
 
-# Not true anymore but keep the validate_ hook for future use, we convert color tags
-#    def validate_notification_urls(self, field):
-#        """Validate that HTML Color format is not used with Telegram"""
-#        if self.notification_format.data == 'HTML Color' and field.data:
-#            for url in field.data:
-#                if url and ('tgram://' in url or 'discord://' in url or 'discord.com/api/webhooks' in url):
-#                    raise ValidationError('HTML Color format is not supported by Telegram and Discord. Please choose another Notification Format (Plain Text, HTML, or Markdown to HTML).')
-
 
 class importForm(Form):
     from . import processors
@@ -843,7 +795,7 @@ class processor_text_json_diff_form(commonSettingsForm):
         if not super().validate():
             return False
 
-        from changedetectionio.jinja2_custom import render as jinja_render
+        from changedetectionio.safe_jinja import render as jinja_render
         result = True
 
         # Fail form validation when a body is set for a GET
@@ -906,36 +858,23 @@ class processor_text_json_diff_form(commonSettingsForm):
     ):
         super().__init__(formdata, obj, prefix, data, meta, **kwargs)
         if kwargs and kwargs.get('default_system_settings'):
-            default_tz = kwargs.get('default_system_settings').get('application', {}).get('scheduler_timezone_default')
+            default_tz = kwargs.get('default_system_settings').get('application', {}).get('timezone')
             if default_tz:
                 self.time_schedule_limit.form.timezone.render_kw['placeholder'] = default_tz
 
 
 
 class SingleExtraProxy(Form):
+
     # maybe better to set some <script>var..
     proxy_name = StringField('Name', [validators.Optional()], render_kw={"placeholder": "Name"})
-    proxy_url = StringField('Proxy URL', [
-        validators.Optional(),
-        ValidateStartsWithRegex(
-            regex=r'^(https?|socks5)://',  # ✅ main pattern
-            flags=re.IGNORECASE,  # ✅ makes it case-insensitive
-            message='Proxy URLs must start with http://, https:// or socks5://',
-        ),
-        ValidateSimpleURL()
-    ], render_kw={"placeholder": "socks5:// or regular proxy http://user:pass@...:3128", "size":50})
+    proxy_url = StringField('Proxy URL', [validators.Optional()], render_kw={"placeholder": "socks5:// or regular proxy http://user:pass@...:3128", "size":50})
+    # @todo do the validation here instead
 
 class SingleExtraBrowser(Form):
     browser_name = StringField('Name', [validators.Optional()], render_kw={"placeholder": "Name"})
-    browser_connection_url = StringField('Browser connection URL', [
-        validators.Optional(),
-        ValidateStartsWithRegex(
-            regex=r'^(wss?|ws)://',
-            flags=re.IGNORECASE,
-            message='Browser URLs must start with wss:// or ws://'
-        ),
-        ValidateSimpleURL()
-    ], render_kw={"placeholder": "wss://brightdata... wss://oxylabs etc", "size":50})
+    browser_connection_url = StringField('Browser connection URL', [validators.Optional()], render_kw={"placeholder": "wss://brightdata... wss://oxylabs etc", "size":50})
+    # @todo do the validation here instead
 
 class DefaultUAInputForm(Form):
     html_requests = StringField('Plaintext requests', validators=[validators.Optional()], render_kw={"placeholder": "<default>"})
@@ -946,7 +885,7 @@ class DefaultUAInputForm(Form):
 class globalSettingsRequestForm(Form):
     time_between_check = RequiredFormField(TimeBetweenCheckForm)
     time_schedule_limit = FormField(ScheduleLimitForm)
-    proxy = RadioField('Default proxy')
+    proxy = RadioField('Proxy')
     jitter_seconds = IntegerField('Random jitter seconds ± check',
                                   render_kw={"style": "width: 5em;"},
                                   validators=[validators.NumberRange(min=0, message="Should contain zero or more seconds")])
@@ -955,12 +894,7 @@ class globalSettingsRequestForm(Form):
                           render_kw={"style": "width: 5em;"},
                           validators=[validators.NumberRange(min=1, max=50,
                                                              message="Should be between 1 and 50")])
-
-    timeout = IntegerField('Requests timeout in seconds',
-                           render_kw={"style": "width: 5em;"},
-                           validators=[validators.NumberRange(min=1, max=999,
-                                                              message="Should be between 1 and 999")])
-
+    
     extra_proxies = FieldList(FormField(SingleExtraProxy), min_entries=5)
     extra_browsers = FieldList(FormField(SingleExtraBrowser), min_entries=5)
 
@@ -1006,10 +940,6 @@ class globalSettingsApplicationForm(commonSettingsForm):
     strip_ignored_lines = BooleanField('Strip ignored lines')
     rss_hide_muted_watches = BooleanField('Hide muted watches from RSS feed', default=True,
                                       validators=[validators.Optional()])
-
-    rss_reader_mode = BooleanField('RSS reader mode ', default=False,
-                                      validators=[validators.Optional()])
-
     filter_failure_notification_threshold_attempts = IntegerField('Number of times the filter can be missing before sending a notification',
                                                                   render_kw={"style": "width: 5em;"},
                                                                   validators=[validators.NumberRange(min=0,

changedetectionio/html_tools.py

@@ -1,5 +1,3 @@
-from functools import lru_cache
-
 from loguru import logger
 from typing import List
 import html
@@ -15,6 +13,7 @@ TITLE_RE = re.compile(r"<title[^>]*>(.*?)</title>", re.I | re.S)
 META_CS  = re.compile(r'<meta[^>]+charset=["\']?\s*([a-z0-9_\-:+.]+)', re.I)
 META_CT  = re.compile(r'<meta[^>]+http-equiv=["\']?content-type["\']?[^>]*content=["\'][^>]*charset=([a-z0-9_\-:+.]+)', re.I)
 
+
 # 'price' , 'lowPrice', 'highPrice' are usually under here
 # All of those may or may not appear on different websites - I didnt find a way todo case-insensitive searching here
 LD_JSON_PRODUCT_OFFER_SELECTORS = ["json:$..offers", "json:$..Offers"]
@@ -23,9 +22,9 @@ class JSONNotFound(ValueError):
     def __init__(self, msg):
         ValueError.__init__(self, msg)
 
+
 # Doesn't look like python supports forward slash auto enclosure in re.findall
 # So convert it to inline flag "(?i)foobar" type configuration
-@lru_cache(maxsize=100)
 def perl_style_slash_enclosed_regex_to_options(regex):
 
     res = re.search(PERL_STYLE_REGEX, regex, re.IGNORECASE)
@@ -186,21 +185,8 @@ def xpath_filter(xpath_filter, html_content, append_pretty_line_formatting=False
     tree = html.fromstring(bytes(html_content, encoding='utf-8'), parser=parser)
     html_block = ""
 
-    # Build namespace map for XPath queries
-    namespaces = {'re': 'http://exslt.org/regular-expressions'}
-
-    # Handle default namespace in documents (common in RSS/Atom feeds, but can occur in any XML)
-    # XPath spec: unprefixed element names have no namespace, not the default namespace
-    # Solution: Register the default namespace with empty string prefix in elementpath
-    # This is primarily for RSS/Atom feeds but works for any XML with default namespace
-    if hasattr(tree, 'nsmap') and tree.nsmap and None in tree.nsmap:
-        # Register the default namespace with empty string prefix for elementpath
-        # This allows //title to match elements in the default namespace
-        namespaces[''] = tree.nsmap[None]
-
-    r = elementpath.select(tree, xpath_filter.strip(), namespaces=namespaces, parser=XPath3Parser)
-    #@note: //title/text() now works with default namespaces (fixed by registering '' prefix)
-    #@note: //title/text() wont work where <title>CDATA.. (use cdata_in_document_to_text first)
+    r = elementpath.select(tree, xpath_filter.strip(), namespaces={'re': 'http://exslt.org/regular-expressions'}, parser=XPath3Parser)
+    #@note: //title/text() wont work where <title>CDATA..
 
     if type(r) != list:
         r = [r]
@@ -235,19 +221,8 @@ def xpath1_filter(xpath_filter, html_content, append_pretty_line_formatting=Fals
     tree = html.fromstring(bytes(html_content, encoding='utf-8'), parser=parser)
     html_block = ""
 
-    # Build namespace map for XPath queries
-    namespaces = {'re': 'http://exslt.org/regular-expressions'}
-
-    # NOTE: lxml's native xpath() does NOT support empty string prefix for default namespace
-    # For documents with default namespace (RSS/Atom feeds), users must use:
-    #   - local-name(): //*[local-name()='title']/text()
-    #   - Or use xpath_filter (not xpath1_filter) which supports default namespaces
-    # XPath spec: unprefixed element names have no namespace, not the default namespace
-
-    r = tree.xpath(xpath_filter.strip(), namespaces=namespaces)
-    #@note: xpath1 (lxml) does NOT automatically handle default namespaces
-    #@note: Use //*[local-name()='element'] or switch to xpath_filter for default namespace support
-    #@note: //title/text() wont work where <title>CDATA.. (use cdata_in_document_to_text first)
+    r = tree.xpath(xpath_filter.strip(), namespaces={'re': 'http://exslt.org/regular-expressions'})
+    #@note: //title/text() wont work where <title>CDATA..
 
     for element in r:
         # When there's more than 1 match, then add the suffix to separate each line
@@ -432,13 +407,7 @@ def strip_ignore_text(content, wordlist, mode="content"):
     ignore_regex_multiline = []
     ignored_lines = []
 
-    if not content:
-        return ''
-
     for k in wordlist:
-        # Skip empty strings to avoid matching everything
-        if not k or not k.strip():
-            continue
         # Is it a regex?
         res = re.search(PERL_STYLE_REGEX, k, re.IGNORECASE)
         if res:

changedetectionio/jinja2_custom/__init__.py

@@ -1,22 +0,0 @@
-"""
-Jinja2 custom extensions and safe rendering utilities.
-"""
-from .extensions.TimeExtension import TimeExtension
-from .safe_jinja import (
-    render,
-    render_fully_escaped,
-    create_jinja_env,
-    JINJA2_MAX_RETURN_PAYLOAD_SIZE,
-    DEFAULT_JINJA2_EXTENSIONS,
-)
-from .plugins.regex import regex_replace
-
-__all__ = [
-    'TimeExtension',
-    'render',
-    'render_fully_escaped',
-    'create_jinja_env',
-    'JINJA2_MAX_RETURN_PAYLOAD_SIZE',
-    'DEFAULT_JINJA2_EXTENSIONS',
-    'regex_replace',
-]

changedetectionio/jinja2_custom/extensions/TimeExtension.py

@@ -1,221 +0,0 @@
-"""
-Jinja2 TimeExtension - Custom date/time handling for templates.
-
-This extension provides the {% now %} tag for Jinja2 templates, offering timezone-aware
-date/time formatting with support for time offsets.
-
-Why This Extension Exists:
-    The Arrow library has a now() function (arrow.now()), but Jinja2 templates cannot
-    directly call Python functions - they need extensions or filters to expose functionality.
-
-    This TimeExtension serves as a Jinja2-to-Arrow bridge that:
-
-    1. Makes Arrow accessible in templates - Jinja2 requires registering functions/tags
-       through extensions. You cannot use arrow.now() directly in a template.
-
-    2. Provides template-friendly syntax - Instead of complex Python code, you get clean tags:
-       {% now 'UTC' %}
-       {% now 'UTC' + 'hours=2' %}
-       {% now 'Europe/London', '%Y-%m-%d' %}
-
-    3. Adds convenience features on top of Arrow:
-       - Default timezone from environment variable (TZ) or config
-       - Default datetime format configuration
-       - Offset syntax parsing: 'hours=2,minutes=30' → shift(hours=2, minutes=30)
-       - Empty string timezone support to use configured defaults
-
-    4. Maintains security - Works within Jinja2's sandboxed environment so users
-       cannot access arbitrary Python code or objects.
-
-    Essentially, this is a Jinja2 wrapper around arrow.now() and arrow.shift() that
-    provides user-friendly template syntax while maintaining security.
-
-Basic Usage:
-    {% now 'UTC' %}
-    # Output: Wed, 09 Dec 2015 23:33:01
-
-Custom Format:
-    {% now 'UTC', '%Y-%m-%d %H:%M:%S' %}
-    # Output: 2015-12-09 23:33:01
-
-Timezone Support:
-    {% now 'America/New_York' %}
-    {% now 'Europe/London' %}
-    {% now '' %}  # Uses default timezone from environment.default_timezone
-
-Time Offsets (Addition):
-    {% now 'UTC' + 'hours=2' %}
-    {% now 'UTC' + 'hours=2,minutes=30' %}
-    {% now 'UTC' + 'days=1,hours=2,minutes=15,seconds=10' %}
-
-Time Offsets (Subtraction):
-    {% now 'UTC' - 'minutes=11' %}
-    {% now 'UTC' - 'days=2,minutes=33,seconds=1' %}
-
-Time Offsets with Custom Format:
-    {% now 'UTC' + 'hours=2', '%Y-%m-%d %H:%M:%S' %}
-    # Output: 2015-12-10 01:33:01
-
-Weekday Support (for finding next/previous weekday):
-    {% now 'UTC' + 'weekday=0' %}  # Next Monday (0=Monday, 6=Sunday)
-    {% now 'UTC' + 'weekday=4' %}  # Next Friday
-
-Configuration:
-    - Default timezone: Set via TZ environment variable or override environment.default_timezone
-    - Default format: '%a, %d %b %Y %H:%M:%S' (can be overridden via environment.datetime_format)
-
-Environment Customization:
-    from changedetectionio.jinja2_custom import create_jinja_env
-
-    jinja2_env = create_jinja_env()
-    jinja2_env.default_timezone = 'America/New_York'  # Override default timezone
-    jinja2_env.datetime_format = '%Y-%m-%d %H:%M'      # Override default format
-
-Supported Offset Parameters:
-    - years, months, weeks, days
-    - hours, minutes, seconds, microseconds
-    - weekday (0=Monday through 6=Sunday, must be integer)
-
-Note:
-    This extension uses the Arrow library for timezone-aware datetime handling.
-    All timezone names should be valid IANA timezone identifiers (e.g., 'America/New_York').
-"""
-import arrow
-
-from jinja2 import nodes
-from jinja2.ext import Extension
-import os
-
-class TimeExtension(Extension):
-    """
-    Jinja2 Extension providing the {% now %} tag for timezone-aware date/time rendering.
-
-    This extension adds two attributes to the Jinja2 environment:
-    - datetime_format: Default strftime format string (default: '%a, %d %b %Y %H:%M:%S')
-    - default_timezone: Default timezone for rendering (default: TZ env var or 'UTC')
-
-    Both can be overridden after environment creation by setting the attributes directly.
-    """
-
-    tags = {'now'}
-
-    def __init__(self, environment):
-        """Jinja2 Extension constructor."""
-        super().__init__(environment)
-
-        environment.extend(
-            datetime_format='%a, %d %b %Y %H:%M:%S',
-            default_timezone=os.getenv('TZ', 'UTC').strip()
-        )
-
-    def _datetime(self, timezone, operator, offset, datetime_format):
-        """
-        Get current datetime with time offset applied.
-
-        Args:
-            timezone: IANA timezone identifier (e.g., 'UTC', 'America/New_York') or empty string for default
-            operator: '+' for addition or '-' for subtraction
-            offset: Comma-separated offset parameters (e.g., 'hours=2,minutes=30')
-            datetime_format: strftime format string or None to use environment default
-
-        Returns:
-            Formatted datetime string with offset applied
-
-        Example:
-            _datetime('UTC', '+', 'hours=2,minutes=30', '%Y-%m-%d %H:%M:%S')
-            # Returns current time + 2.5 hours
-        """
-        # Use default timezone if none specified
-        if not timezone or timezone == '':
-            timezone = self.environment.default_timezone
-
-        d = arrow.now(timezone)
-
-        # parse shift params from offset and include operator
-        shift_params = {}
-        for param in offset.split(','):
-            interval, value = param.split('=')
-            shift_params[interval.strip()] = float(operator + value.strip())
-
-        # Fix weekday parameter can not be float
-        if 'weekday' in shift_params:
-            shift_params['weekday'] = int(shift_params['weekday'])
-
-        d = d.shift(**shift_params)
-
-        if datetime_format is None:
-            datetime_format = self.environment.datetime_format
-        return d.strftime(datetime_format)
-
-    def _now(self, timezone, datetime_format):
-        """
-        Get current datetime without any offset.
-
-        Args:
-            timezone: IANA timezone identifier (e.g., 'UTC', 'America/New_York') or empty string for default
-            datetime_format: strftime format string or None to use environment default
-
-        Returns:
-            Formatted datetime string for current time
-
-        Example:
-            _now('America/New_York', '%Y-%m-%d %H:%M:%S')
-            # Returns current time in New York timezone
-        """
-        # Use default timezone if none specified
-        if not timezone or timezone == '':
-            timezone = self.environment.default_timezone
-
-        if datetime_format is None:
-            datetime_format = self.environment.datetime_format
-        return arrow.now(timezone).strftime(datetime_format)
-
-    def parse(self, parser):
-        """
-        Parse the {% now %} tag and generate appropriate AST nodes.
-
-        This method is called by Jinja2 when it encounters a {% now %} tag.
-        It parses the tag syntax and determines whether to call _now() or _datetime()
-        based on whether offset operations (+ or -) are present.
-
-        Supported syntax:
-            {% now 'timezone' %}                              -> calls _now()
-            {% now 'timezone', 'format' %}                    -> calls _now()
-            {% now 'timezone' + 'offset' %}                   -> calls _datetime()
-            {% now 'timezone' + 'offset', 'format' %}         -> calls _datetime()
-            {% now 'timezone' - 'offset', 'format' %}         -> calls _datetime()
-
-        Args:
-            parser: Jinja2 parser instance
-
-        Returns:
-            nodes.Output: AST output node containing the formatted datetime string
-        """
-        lineno = next(parser.stream).lineno
-
-        node = parser.parse_expression()
-
-        if parser.stream.skip_if('comma'):
-            datetime_format = parser.parse_expression()
-        else:
-            datetime_format = nodes.Const(None)
-
-        if isinstance(node, nodes.Add):
-            call_method = self.call_method(
-                '_datetime',
-                [node.left, nodes.Const('+'), node.right, datetime_format],
-                lineno=lineno,
-            )
-        elif isinstance(node, nodes.Sub):
-            call_method = self.call_method(
-                '_datetime',
-                [node.left, nodes.Const('-'), node.right, datetime_format],
-                lineno=lineno,
-            )
-        else:
-            call_method = self.call_method(
-                '_now',
-                [node, datetime_format],
-                lineno=lineno,
-            )
-        return nodes.Output([call_method], lineno=lineno)

changedetectionio/jinja2_custom/plugins/__init__.py

@@ -1,6 +0,0 @@
-"""
-Jinja2 custom filter plugins for changedetection.io
-"""
-from .regex import regex_replace
-
-__all__ = ['regex_replace']

changedetectionio/jinja2_custom/plugins/regex.py

@@ -1,98 +0,0 @@
-"""
-Regex filter plugin for Jinja2 templates.
-
-Provides regex_replace filter for pattern-based string replacements in templates.
-"""
-import re
-import signal
-from loguru import logger
-
-
-def regex_replace(value: str, pattern: str, replacement: str = '', count: int = 0) -> str:
-    """
-    Replace occurrences of a regex pattern in a string.
-
-    Security: Protected against ReDoS (Regular Expression Denial of Service) attacks:
-    - Limits input value size to prevent excessive processing
-    - Uses timeout mechanism to prevent runaway regex operations
-    - Validates pattern complexity to prevent catastrophic backtracking
-
-    Args:
-        value: The input string to perform replacements on
-        pattern: The regex pattern to search for
-        replacement: The replacement string (default: '')
-        count: Maximum number of replacements (0 = replace all, default: 0)
-
-    Returns:
-        String with replacements applied, or original value on error
-
-    Example:
-        {{ "hello world" | regex_replace("world", "universe") }}
-        {{ diff | regex_replace("<td>([^<]+)</td><td>([^<]+)</td>", "Label1: \\1\\nLabel2: \\2") }}
-
-    Security limits:
-        - Maximum input size: 10MB
-        - Maximum pattern length: 500 characters
-        - Operation timeout: 10 seconds
-        - Dangerous nested quantifier patterns are rejected
-    """
-    # Security limits
-    MAX_INPUT_SIZE = 1024 * 1024 * 10 # 10MB max input size
-    MAX_PATTERN_LENGTH = 500  # Maximum regex pattern length
-    REGEX_TIMEOUT_SECONDS = 10  # Maximum time for regex operation
-
-    # Validate input sizes
-    value_str = str(value)
-    if len(value_str) > MAX_INPUT_SIZE:
-        logger.warning(f"regex_replace: Input too large ({len(value_str)} bytes), truncating")
-        value_str = value_str[:MAX_INPUT_SIZE]
-
-    if len(pattern) > MAX_PATTERN_LENGTH:
-        logger.warning(f"regex_replace: Pattern too long ({len(pattern)} chars), rejecting")
-        return value_str
-
-    # Check for potentially dangerous patterns (basic checks)
-    # Nested quantifiers like (a+)+ can cause catastrophic backtracking
-    dangerous_patterns = [
-        r'\([^)]*\+[^)]*\)\+',  # (x+)+
-        r'\([^)]*\*[^)]*\)\+',  # (x*)+
-        r'\([^)]*\+[^)]*\)\*',  # (x+)*
-        r'\([^)]*\*[^)]*\)\*',  # (x*)*
-    ]
-
-    for dangerous in dangerous_patterns:
-        if re.search(dangerous, pattern):
-            logger.warning(f"regex_replace: Potentially dangerous pattern detected: {pattern}")
-            return value_str
-
-    def timeout_handler(signum, frame):
-        raise TimeoutError("Regex operation timed out")
-
-    try:
-        # Set up timeout for regex operation (Unix-like systems only)
-        # This prevents ReDoS attacks
-        old_handler = None
-        if hasattr(signal, 'SIGALRM'):
-            old_handler = signal.signal(signal.SIGALRM, timeout_handler)
-            signal.alarm(REGEX_TIMEOUT_SECONDS)
-
-        try:
-            result = re.sub(pattern, replacement, value_str, count=count)
-        finally:
-            # Cancel the alarm
-            if hasattr(signal, 'SIGALRM'):
-                signal.alarm(0)
-                if old_handler is not None:
-                    signal.signal(signal.SIGALRM, old_handler)
-
-        return result
-
-    except TimeoutError:
-        logger.error(f"regex_replace: Regex operation timed out - possible ReDoS attack. Pattern: {pattern}")
-        return value_str
-    except re.error as e:
-        logger.warning(f"regex_replace: Invalid regex pattern: {e}")
-        return value_str
-    except Exception as e:
-        logger.error(f"regex_replace: Unexpected error: {e}")
-        return value_str

changedetectionio/jinja2_custom/safe_jinja.py

@@ -1,58 +0,0 @@
-"""
-Safe Jinja2 render with max payload sizes
-
-See https://jinja.palletsprojects.com/en/3.1.x/sandbox/#security-considerations
-"""
-
-import jinja2.sandbox
-import typing as t
-import os
-from .extensions.TimeExtension import TimeExtension
-from .plugins import regex_replace
-
-JINJA2_MAX_RETURN_PAYLOAD_SIZE = 1024 * int(os.getenv("JINJA2_MAX_RETURN_PAYLOAD_SIZE_KB", 1024 * 10))
-
-# Default extensions - can be overridden in create_jinja_env()
-DEFAULT_JINJA2_EXTENSIONS = [TimeExtension]
-
-def create_jinja_env(extensions=None, **kwargs) -> jinja2.sandbox.ImmutableSandboxedEnvironment:
-    """
-    Create a sandboxed Jinja2 environment with our custom extensions and default timezone.
-
-    Args:
-        extensions: List of extension classes to use (defaults to DEFAULT_JINJA2_EXTENSIONS)
-        **kwargs: Additional arguments to pass to ImmutableSandboxedEnvironment
-
-    Returns:
-        Configured Jinja2 environment
-    """
-    if extensions is None:
-        extensions = DEFAULT_JINJA2_EXTENSIONS
-
-    jinja2_env = jinja2.sandbox.ImmutableSandboxedEnvironment(
-        extensions=extensions,
-        **kwargs
-    )
-
-    # Get default timezone from environment variable
-    default_timezone = os.getenv('TZ', 'UTC').strip()
-    jinja2_env.default_timezone = default_timezone
-
-    # Register custom filters
-    jinja2_env.filters['regex_replace'] = regex_replace
-
-    return jinja2_env
-
-
-# This is used for notifications etc, so actually it's OK to send custom HTML such as <a href> etc, but it should limit what data is available.
-# (Which also limits available functions that could be called)
-def render(template_str, **args: t.Any) -> str:
-    jinja2_env = create_jinja_env()
-    output = jinja2_env.from_string(template_str).render(args)
-    return output[:JINJA2_MAX_RETURN_PAYLOAD_SIZE]
-
-def render_fully_escaped(content):
-    env = jinja2.sandbox.ImmutableSandboxedEnvironment(autoescape=True)
-    template = env.from_string("{{ some_html|e }}")
-    return template.render(some_html=content)
-

changedetectionio/model/App.py

@@ -1,5 +1,4 @@
 from os import getenv
-from copy import deepcopy
 
 from changedetectionio.blueprint.rss import RSS_FORMAT_TYPES
 
@@ -56,12 +55,11 @@ class model(dict):
                     'rss_access_token': None,
                     'rss_content_format': RSS_FORMAT_TYPES[0][0],
                     'rss_hide_muted_watches': True,
-                    'rss_reader_mode': False,
-                    'scheduler_timezone_default': None,  # Default IANA timezone name
                     'schema_version' : 0,
                     'shared_diff_access': False,
                     'strip_ignored_lines': False,
                     'tags': {}, #@todo use Tag.model initialisers
+                    'timezone': None, # Default IANA timezone name
                     'webdriver_delay': None , # Extra delay in seconds before extracting text
                     'ui': {
                         'use_page_title_in_list': True,
@@ -75,8 +73,7 @@ class model(dict):
 
     def __init__(self, *arg, **kw):
         super(model, self).__init__(*arg, **kw)
-        # CRITICAL: deepcopy to avoid sharing mutable objects between instances
-        self.update(deepcopy(self.base_config))
+        self.update(self.base_config)
 
 
 def parse_headers_from_text_file(filepath):

changedetectionio/model/Watch.py

@@ -1,24 +1,42 @@
 from blinker import signal
-from changedetectionio.validate_url import is_safe_valid_url
 
 from changedetectionio.strtobool import strtobool
-from changedetectionio.jinja2_custom import render as jinja_render
+from changedetectionio.safe_jinja import render as jinja_render
 from . import watch_base
 import os
 import re
 from pathlib import Path
 from loguru import logger
 
-from .. import jinja2_custom as safe_jinja
-from ..diff import ADDED_PLACEMARKER_OPEN
+from .. import safe_jinja
 from ..html_tools import TRANSLATE_WHITESPACE_TABLE
 
+# Allowable protocols, protects against javascript: etc
+# file:// is further checked by ALLOW_FILE_URI
+SAFE_PROTOCOL_REGEX='^(http|https|ftp|file):'
 FAVICON_RESAVE_THRESHOLD_SECONDS=86400
 
 
 minimum_seconds_recheck_time = int(os.getenv('MINIMUM_SECONDS_RECHECK_TIME', 3))
 mtable = {'seconds': 1, 'minutes': 60, 'hours': 3600, 'days': 86400, 'weeks': 86400 * 7}
 
+
+def is_safe_url(test_url):
+    # See https://github.com/dgtlmoon/changedetection.io/issues/1358
+
+    # Remove 'source:' prefix so we dont get 'source:javascript:' etc
+    # 'source:' is a valid way to tell us to return the source
+
+    r = re.compile(re.escape('source:'), re.IGNORECASE)
+    test_url = r.sub('', test_url)
+
+    pattern = re.compile(os.getenv('SAFE_PROTOCOL_REGEX', SAFE_PROTOCOL_REGEX), re.IGNORECASE)
+    if not pattern.match(test_url.strip()):
+        return False
+
+    return True
+
+
 class model(watch_base):
     __newest_history_key = None
     __history_n = 0
@@ -61,7 +79,7 @@ class model(watch_base):
     def link(self):
 
         url = self.get('url', '')
-        if not is_safe_valid_url(url):
+        if not is_safe_url(url):
             return 'DISABLED'
 
         ready_url = url
@@ -71,8 +89,9 @@ class model(watch_base):
                 ready_url = jinja_render(template_str=url)
             except Exception as e:
                 logger.critical(f"Invalid URL template for: '{url}' - {str(e)}")
-                from flask import flash, url_for
-                from markupsafe import Markup
+                from flask import (
+                    flash, Markup, url_for
+                )
                 message = Markup('<a href="{}#general">The URL {} is invalid and cannot be used, click to edit</a>'.format(
                     url_for('ui.ui_edit.edit_page', uuid=self.get('uuid')), self.get('url', '')))
                 flash(message, 'error')
@@ -82,7 +101,7 @@ class model(watch_base):
             ready_url=ready_url.replace('source:', '')
 
         # Also double check it after any Jinja2 formatting just incase
-        if not is_safe_valid_url(ready_url):
+        if not is_safe_url(ready_url):
             return 'DISABLED'
         return ready_url
 

changedetectionio/model/__init__.py

@@ -2,7 +2,7 @@ import os
 import uuid
 
 from changedetectionio import strtobool
-USE_SYSTEM_DEFAULT_NOTIFICATION_FORMAT_FOR_WATCH = 'System default'
+default_notification_format_for_watch = 'System default'
 CONDITIONS_MATCH_LOGIC_DEFAULT = 'ALL'
 
 class watch_base(dict):
@@ -44,7 +44,7 @@ class watch_base(dict):
             'method': 'GET',
             'notification_alert_count': 0,
             'notification_body': None,
-            'notification_format': USE_SYSTEM_DEFAULT_NOTIFICATION_FORMAT_FOR_WATCH,
+            'notification_format': default_notification_format_for_watch,
             'notification_muted': False,
             'notification_screenshot': False,  # Include the latest screenshot if available and supported by the apprise URL
             'notification_title': None,

changedetectionio/notification/__init__.py

@@ -1,16 +1,35 @@
-from changedetectionio.model import USE_SYSTEM_DEFAULT_NOTIFICATION_FORMAT_FOR_WATCH
+from changedetectionio.model import default_notification_format_for_watch
 
-default_notification_format = 'htmlcolor'
+ult_notification_format_for_watch = 'System default'
+default_notification_format = 'HTML Color'
 default_notification_body = '{{watch_url}} had a change.\n---\n{{diff}}\n---\n'
 default_notification_title = 'ChangeDetection.io Notification - {{watch_url}}'
 
 # The values (markdown etc) are from apprise NotifyFormat,
 # But to avoid importing the whole heavy module just use the same strings here.
 valid_notification_formats = {
-    'text': 'Plain Text',
-    'html': 'HTML',
-    'htmlcolor': 'HTML Color',
-    'markdown': 'Markdown to HTML',
+    'Text': 'text',
+    'Markdown': 'markdown',
+    'HTML': 'html',
+    'HTML Color': 'htmlcolor',
     # Used only for editing a watch (not for global)
-    USE_SYSTEM_DEFAULT_NOTIFICATION_FORMAT_FOR_WATCH: USE_SYSTEM_DEFAULT_NOTIFICATION_FORMAT_FOR_WATCH
+    default_notification_format_for_watch: default_notification_format_for_watch
+}
+
+
+valid_tokens = {
+    'base_url': '',
+    'current_snapshot': '',
+    'diff': '',
+    'diff_added': '',
+    'diff_full': '',
+    'diff_patch': '',
+    'diff_removed': '',
+    'diff_url': '',
+    'preview_url': '',
+    'triggered_text': '',
+    'watch_tag': '',
+    'watch_title': '',
+    'watch_url': '',
+    'watch_uuid': '',
 }

changedetectionio/notification/apprise_plugin/custom_handlers.py

@@ -1,61 +1,10 @@
-"""
-Custom Apprise HTTP Handlers with format= Parameter Support
-
-IMPORTANT: This module works around a limitation in Apprise's @notify decorator.
-
-THE PROBLEM:
--------------
-When using Apprise's @notify decorator to create custom notification handlers, the
-decorator creates a CustomNotifyPlugin that uses parse_url(..., simple=True) to parse
-URLs. This simple parsing mode does NOT extract the format= query parameter from the URL
-and set it as a top-level parameter that NotifyBase.__init__ can use to set notify_format.
-
-As a result:
-1. URL: post://example.com/webhook?format=html
-2. Apprise parses this and sees format=html in qsd (query string dictionary)
-3. But it does NOT extract it and pass it to NotifyBase.__init__
-4. NotifyBase defaults to notify_format=TEXT
-5. When you call apobj.notify(body="<html>...", body_format="html"):
-   - Apprise sees: input format = html, output format (notify_format) = text
-   - Apprise calls convert_between("html", "text", body)
-   - This strips all HTML tags, leaving only plain text
-6. Your custom handler receives stripped plain text instead of HTML
-
-THE SOLUTION:
--------------
-Instead of using the @notify decorator directly, we:
-1. Manually register custom plugins using plugins.N_MGR.add()
-2. Create a CustomHTTPHandler class that extends CustomNotifyPlugin
-3. Override __init__ to extract format= from qsd and set it as kwargs['format']
-4. Call NotifyBase.__init__ which properly sets notify_format from kwargs['format']
-5. Set up _default_args like CustomNotifyPlugin does for compatibility
-
-This ensures that when format=html is in the URL:
-- notify_format is set to HTML
-- Apprise sees: input format = html, output format = html
-- No conversion happens (convert_between returns content unchanged)
-- Your custom handler receives the original HTML intact
-
-TESTING:
---------
-To verify this works:
->>> apobj = apprise.Apprise()
->>> apobj.add('post://localhost:5005/test?format=html')
->>> for server in apobj:
-...     print(server.notify_format)  # Should print: html (not text)
->>> apobj.notify(body='<span>Test</span>', body_format='html')
-# Your handler should receive '<span>Test</span>' not 'Test'
-"""
-
 import json
 import re
 from urllib.parse import unquote_plus
 
 import requests
-from apprise import plugins
-from apprise.decorators.base import CustomNotifyPlugin
-from apprise.utils.parse import parse_url as apprise_parse_url, url_assembly
-from apprise.utils.logic import dict_full_update
+from apprise.decorators import notify
+from apprise.utils.parse import parse_url as apprise_parse_url
 from loguru import logger
 from requests.structures import CaseInsensitiveDict
 
@@ -63,66 +12,13 @@ SUPPORTED_HTTP_METHODS = {"get", "post", "put", "delete", "patch", "head"}
 
 
 def notify_supported_methods(func):
-    """Register custom HTTP method handlers that properly support format= parameter."""
     for method in SUPPORTED_HTTP_METHODS:
-        _register_http_handler(method, func)
-        _register_http_handler(f"{method}s", func)
+        func = notify(on=method)(func)
+        # Add support for https, for each supported http method
+        func = notify(on=f"{method}s")(func)
     return func
 
 
-def _register_http_handler(schema, send_func):
-    """Register a custom HTTP handler that extracts format= from URL query parameters."""
-
-    # Parse base URL
-    base_url = f"{schema}://"
-    base_args = apprise_parse_url(base_url, default_schema=schema, verify_host=False, simple=True)
-
-    class CustomHTTPHandler(CustomNotifyPlugin):
-        secure_protocol = schema
-        service_name = f"Custom HTTP - {schema.upper()}"
-        _base_args = base_args
-
-        def __init__(self, **kwargs):
-            # Extract format from qsd and set it as a top-level kwarg
-            # This allows NotifyBase.__init__ to properly set notify_format
-            if 'qsd' in kwargs and 'format' in kwargs['qsd']:
-                kwargs['format'] = kwargs['qsd']['format']
-
-            # Call NotifyBase.__init__ (skip CustomNotifyPlugin.__init__)
-            super(CustomNotifyPlugin, self).__init__(**kwargs)
-
-            # Set up _default_args like CustomNotifyPlugin does
-            self._default_args = {}
-            kwargs.pop("secure", None)
-            dict_full_update(self._default_args, self._base_args)
-            dict_full_update(self._default_args, kwargs)
-            self._default_args["url"] = url_assembly(**self._default_args)
-
-        __send = staticmethod(send_func)
-
-        def send(self, body, title="", notify_type="info", *args, **kwargs):
-            """Call the custom send function."""
-            try:
-                result = self.__send(
-                    body, title, notify_type,
-                    *args,
-                    meta=self._default_args,
-                    **kwargs
-                )
-                return True if result is None else bool(result)
-            except Exception as e:
-                self.logger.warning(f"Exception in custom HTTP handler: {e}")
-                return False
-
-    # Register the plugin
-    plugins.N_MGR.add(
-        plugin=CustomHTTPHandler,
-        schemas=schema,
-        send_func=send_func,
-        url=base_url,
-    )
-
-
 def _get_auth(parsed_url: dict) -> str | tuple[str, str]:
     user: str | None = parsed_url.get("user")
     password: str | None = parsed_url.get("password")
@@ -174,12 +70,9 @@ def apprise_http_custom_handler(
     title: str,
     notify_type: str,
     meta: dict,
-    body_format: str = None,
     *args,
     **kwargs,
 ) -> bool:
-
-
     url: str = meta.get("url")
     schema: str = meta.get("schema")
     method: str = re.sub(r"s$", "", schema).upper()
@@ -195,16 +88,25 @@ def apprise_http_custom_handler(
 
     url = re.sub(rf"^{schema}", "https" if schema.endswith("s") else "http", parsed_url.get("url"))
 
-    response = requests.request(
-        method=method,
-        url=url,
-        auth=auth,
-        headers=headers,
-        params=params,
-        data=body.encode("utf-8") if isinstance(body, str) else body,
-    )
+    try:
+        response = requests.request(
+            method=method,
+            url=url,
+            auth=auth,
+            headers=headers,
+            params=params,
+            data=body.encode("utf-8") if isinstance(body, str) else body,
+        )
 
-    response.raise_for_status()
+        response.raise_for_status()
 
-    logger.info(f"Successfully sent custom notification to {url}")
-    return True
+        logger.info(f"Successfully sent custom notification to {url}")
+        return True
+
+    except requests.RequestException as e:
+        logger.error(f"Remote host error while sending custom notification to {url}: {e}")
+        return False
+
+    except Exception as e:
+        logger.error(f"Unexpected error occurred while sending custom notification to {url}: {e}")
+        return False

changedetectionio/notification/apprise_plugin/discord.py

@@ -1,286 +0,0 @@
-"""
-Custom Discord plugin for changedetection.io
-Extends Apprise's Discord plugin to support custom colored embeds for removed/added content
-"""
-from apprise.plugins.discord import NotifyDiscord
-from apprise.decorators import notify
-from apprise.common import NotifyFormat
-from loguru import logger
-
-# Import placeholders from changedetection's diff module
-from ...diff import (
-    REMOVED_PLACEMARKER_OPEN,
-    REMOVED_PLACEMARKER_CLOSED,
-    ADDED_PLACEMARKER_OPEN,
-    ADDED_PLACEMARKER_CLOSED,
-    CHANGED_PLACEMARKER_OPEN,
-    CHANGED_PLACEMARKER_CLOSED,
-    CHANGED_INTO_PLACEMARKER_OPEN,
-    CHANGED_INTO_PLACEMARKER_CLOSED,
-)
-
-# Discord embed sidebar colors for different change types
-DISCORD_COLOR_UNCHANGED = 8421504   # Gray (#808080)
-DISCORD_COLOR_REMOVED = 16711680    # Red (#FF0000)
-DISCORD_COLOR_ADDED = 65280         # Green (#00FF00)
-DISCORD_COLOR_CHANGED = 16753920    # Orange (#FFA500)
-DISCORD_COLOR_CHANGED_INTO = 3447003  # Blue (#5865F2 - Discord blue)
-DISCORD_COLOR_WARNING = 16776960    # Yellow (#FFFF00)
-
-
-class NotifyDiscordCustom(NotifyDiscord):
-    """
-    Custom Discord notification handler that supports multiple colored embeds
-    for showing removed (red) and added (green) content separately.
-    """
-
-    def send(self, body, title="", notify_type=None, attach=None, **kwargs):
-        """
-        Override send method to create custom embeds with red/green colors
-        for removed/added content when placeholders are present.
-        """
-
-        # Check if body contains our diff placeholders
-        has_removed = REMOVED_PLACEMARKER_OPEN in body
-        has_added = ADDED_PLACEMARKER_OPEN in body
-        has_changed = CHANGED_PLACEMARKER_OPEN in body
-        has_changed_into = CHANGED_INTO_PLACEMARKER_OPEN in body
-
-        # If we have diff placeholders and we're in markdown/html format, create custom embeds
-        if (has_removed or has_added or has_changed or has_changed_into) and self.notify_format in (NotifyFormat.MARKDOWN, NotifyFormat.HTML):
-            return self._send_with_colored_embeds(body, title, notify_type, attach, **kwargs)
-
-        # Otherwise, use the parent class's default behavior
-        return super().send(body, title, notify_type, attach, **kwargs)
-
-    def _send_with_colored_embeds(self, body, title, notify_type, attach, **kwargs):
-        """
-        Send Discord message with embeds in the original diff order.
-        Preserves the sequence: unchanged -> removed -> added -> unchanged, etc.
-        """
-        from datetime import datetime, timezone
-
-        payload = {
-            "tts": self.tts,
-            "wait": self.tts is False,
-        }
-
-        if self.flags:
-            payload["flags"] = self.flags
-
-        # Acquire image_url
-        image_url = self.image_url(notify_type)
-
-        if self.avatar and (image_url or self.avatar_url):
-            payload["avatar_url"] = self.avatar_url if self.avatar_url else image_url
-
-        if self.user:
-            payload["username"] = self.user
-
-        # Associate our thread_id with our message
-        params = {"thread_id": self.thread_id} if self.thread_id else None
-
-        # Build embeds array preserving order
-        embeds = []
-
-        # Add title as plain bold text in message content (not an embed)
-        if title:
-            payload["content"] = f"**{title}**"
-
-        # Parse the body into ordered chunks
-        chunks = self._parse_body_into_chunks(body)
-
-        # Discord limits:
-        # - Max 10 embeds per message
-        # - Max 6000 characters total across all embeds
-        # - Max 4096 characters per embed description
-        max_embeds = 10
-        max_total_chars = 6000
-        max_embed_description = 4096
-
-        # All 10 embed slots are available for content
-        max_content_embeds = max_embeds
-
-        # Start character count
-        total_chars = 0
-
-        # Create embeds from chunks in order (no titles, just color coding)
-        for chunk_type, content in chunks:
-            if not content.strip():
-                continue
-
-            # Truncate individual embed description if needed
-            if len(content) > max_embed_description:
-                content = content[:max_embed_description - 3] + "..."
-
-            # Check if we're approaching the embed count limit
-            # We need room for the warning embed, so stop at max_content_embeds - 1
-            current_content_embeds = len(embeds)
-            if current_content_embeds >= max_content_embeds - 1:
-                # Add a truncation notice (this will be the 10th embed)
-                embeds.append({
-                    "description": "⚠️ Content truncated (Discord 10 embed limit reached) - Tip: Select 'Plain Text' or 'HTML' format for longer diffs",
-                    "color": DISCORD_COLOR_WARNING,
-                })
-                break
-
-            # Check if adding this embed would exceed total character limit
-            if total_chars + len(content) > max_total_chars:
-                # Add a truncation notice
-                remaining_chars = max_total_chars - total_chars
-                if remaining_chars > 100:
-                    # Add partial content if we have room
-                    truncated_content = content[:remaining_chars - 100] + "..."
-                    embeds.append({
-                        "description": truncated_content,
-                        "color": (DISCORD_COLOR_UNCHANGED if chunk_type == "unchanged"
-                                 else DISCORD_COLOR_REMOVED if chunk_type == "removed"
-                                 else DISCORD_COLOR_ADDED),
-                    })
-                embeds.append({
-                    "description": "⚠️ Content truncated (Discord 6000 char limit reached)\nTip: Select 'Plain Text' or 'HTML' format for longer diffs",
-                    "color": DISCORD_COLOR_WARNING,
-                })
-                break
-
-            if chunk_type == "unchanged":
-                embeds.append({
-                    "description": content,
-                    "color": DISCORD_COLOR_UNCHANGED,
-                })
-            elif chunk_type == "removed":
-                embeds.append({
-                    "description": content,
-                    "color": DISCORD_COLOR_REMOVED,
-                })
-            elif chunk_type == "added":
-                embeds.append({
-                    "description": content,
-                    "color": DISCORD_COLOR_ADDED,
-                })
-            elif chunk_type == "changed":
-                # Changed (old value) - use orange to distinguish from pure removal
-                embeds.append({
-                    "description": content,
-                    "color": DISCORD_COLOR_CHANGED,
-                })
-            elif chunk_type == "changed_into":
-                # Changed into (new value) - use blue to distinguish from pure addition
-                embeds.append({
-                    "description": content,
-                    "color": DISCORD_COLOR_CHANGED_INTO,
-                })
-
-            total_chars += len(content)
-
-        if embeds:
-            payload["embeds"] = embeds
-
-        # Send the payload using parent's _send method
-        if not self._send(payload, params=params):
-            return False
-
-        # Handle attachments if present
-        if attach and self.attachment_support:
-            payload.update({
-                "tts": False,
-                "wait": True,
-            })
-            payload.pop("embeds", None)
-            payload.pop("content", None)
-            payload.pop("allow_mentions", None)
-
-            for attachment in attach:
-                self.logger.info(f"Posting Discord Attachment {attachment.name}")
-                if not self._send(payload, params=params, attach=attachment):
-                    return False
-
-        return True
-
-    def _parse_body_into_chunks(self, body):
-        """
-        Parse the body into ordered chunks of (type, content) tuples.
-        Types: "unchanged", "removed", "added", "changed", "changed_into"
-        Preserves the original order of the diff.
-        """
-        chunks = []
-        position = 0
-
-        while position < len(body):
-            # Find the next marker
-            next_removed = body.find(REMOVED_PLACEMARKER_OPEN, position)
-            next_added = body.find(ADDED_PLACEMARKER_OPEN, position)
-            next_changed = body.find(CHANGED_PLACEMARKER_OPEN, position)
-            next_changed_into = body.find(CHANGED_INTO_PLACEMARKER_OPEN, position)
-
-            # Determine which marker comes first
-            if next_removed == -1 and next_added == -1 and next_changed == -1 and next_changed_into == -1:
-                # No more markers, rest is unchanged
-                if position < len(body):
-                    chunks.append(("unchanged", body[position:]))
-                break
-
-            # Find the earliest marker
-            next_marker_pos = None
-            next_marker_type = None
-
-            # Compare all marker positions to find the earliest
-            markers = []
-            if next_removed != -1:
-                markers.append((next_removed, "removed"))
-            if next_added != -1:
-                markers.append((next_added, "added"))
-            if next_changed != -1:
-                markers.append((next_changed, "changed"))
-            if next_changed_into != -1:
-                markers.append((next_changed_into, "changed_into"))
-
-            if markers:
-                next_marker_pos, next_marker_type = min(markers, key=lambda x: x[0])
-
-            # Add unchanged content before the marker
-            if next_marker_pos > position:
-                chunks.append(("unchanged", body[position:next_marker_pos]))
-
-            # Find the closing marker
-            if next_marker_type == "removed":
-                open_marker = REMOVED_PLACEMARKER_OPEN
-                close_marker = REMOVED_PLACEMARKER_CLOSED
-            elif next_marker_type == "added":
-                open_marker = ADDED_PLACEMARKER_OPEN
-                close_marker = ADDED_PLACEMARKER_CLOSED
-            elif next_marker_type == "changed":
-                open_marker = CHANGED_PLACEMARKER_OPEN
-                close_marker = CHANGED_PLACEMARKER_CLOSED
-            else:  # changed_into
-                open_marker = CHANGED_INTO_PLACEMARKER_OPEN
-                close_marker = CHANGED_INTO_PLACEMARKER_CLOSED
-
-            close_pos = body.find(close_marker, next_marker_pos)
-
-            if close_pos == -1:
-                # No closing marker, take rest as this type
-                content = body[next_marker_pos + len(open_marker):]
-                chunks.append((next_marker_type, content))
-                break
-            else:
-                # Extract content between markers
-                content = body[next_marker_pos + len(open_marker):close_pos]
-                chunks.append((next_marker_type, content))
-                position = close_pos + len(close_marker)
-
-        return chunks
-
-
-# Register the custom Discord handler with Apprise
-# This will override the built-in discord:// handler
-@notify(on="discord")
-def discord_custom_wrapper(body, title, notify_type, meta, body_format=None, *args, **kwargs):
-    """
-    Wrapper function to make the custom Discord handler work with Apprise's decorator system.
-    Note: This decorator approach may not work for overriding built-in plugins.
-    The class-based approach above is the proper way to extend NotifyDiscord.
-    """
-    logger.info("Custom Discord handler called")
-    # This is here for potential future use with decorator-based registration
-    return True

changedetectionio/notification/email_helpers.py

@@ -1,42 +0,0 @@
-def as_monospaced_html_email(content: str, title: str) -> str:
-    """
-    Wraps `content` in a minimal, email-safe HTML template
-    that forces monospace rendering across Gmail, Hotmail, Apple Mail, etc.
-
-    Args:
-        content: The body text (plain text or HTML-like).
-        title: The title plaintext
-    Returns:
-        A complete HTML document string suitable for sending as an email body.
-    """
-
-    # All line feed types should be removed and then this function should only be fed <br>'s
-    # Then it works with our <pre> styling without double linefeeds
-    content = content.translate(str.maketrans('', '', '\r\n'))
-
-    if title:
-        import html
-        title = html.escape(title)
-    else:
-        title = ''
-    # 2. Full email-safe HTML
-    html_email = f"""<!DOCTYPE html>
-<html lang="en">
-<head>
-  <meta charset="UTF-8">
-  <meta name="x-apple-disable-message-reformatting">
-  <meta name="viewport" content="width=device-width, initial-scale=1.0">
-  <!--[if mso]>
-    <style>
-      body, div, pre, td {{ font-family: "Courier New", Courier, monospace !important; }}
-    </style>
-  <![endif]-->
-  <title>{title}</title>
-</head>
-<body style="-webkit-text-size-adjust:100%;-ms-text-size-adjust:100%;">
-  <pre role="article" aria-roledescription="email" lang="en"
-       style="font-family: monospace, 'Courier New', Courier; font-size: 0.9rem;
-              white-space: pre-wrap; word-break: break-word;">{content}</pre>
-</body>
-</html>"""
-    return html_email

changedetectionio/notification/handler.py

@@ -1,331 +1,30 @@
 
 import time
-import re
 import apprise
-from apprise import NotifyFormat
 from loguru import logger
-from urllib.parse import urlparse
 from .apprise_plugin.assets import apprise_asset, APPRISE_AVATAR_URL
-from .email_helpers import as_monospaced_html_email
-from ..diff import HTML_REMOVED_STYLE, REMOVED_PLACEMARKER_OPEN, REMOVED_PLACEMARKER_CLOSED, ADDED_PLACEMARKER_OPEN, HTML_ADDED_STYLE, \
-    ADDED_PLACEMARKER_CLOSED, CHANGED_INTO_PLACEMARKER_OPEN, CHANGED_INTO_PLACEMARKER_CLOSED, CHANGED_PLACEMARKER_OPEN, \
-    CHANGED_PLACEMARKER_CLOSED, HTML_CHANGED_STYLE, HTML_CHANGED_INTO_STYLE
-import re
 
-from ..notification_service import NotificationContextData, add_rendered_diff_to_notification_vars
-
-newline_re = re.compile(r'\r\n|\r|\n')
-
-def markup_text_links_to_html(body):
-    """
-    Convert plaintext to HTML with clickable links.
-    Uses Jinja2's escape and Markup for XSS safety.
-    """
-    from linkify_it import LinkifyIt
-    from markupsafe import Markup, escape
-
-    linkify = LinkifyIt()
-
-    # Match URLs in the ORIGINAL text (before escaping)
-    matches = linkify.match(body)
-
-    if not matches:
-        # No URLs, just escape everything
-        return Markup(escape(body))
-
-    result = []
-    last_index = 0
-
-    # Process each URL match
-    for match in matches:
-        # Add escaped text before the URL
-        if match.index > last_index:
-            text_part = body[last_index:match.index]
-            result.append(escape(text_part))
-
-        # Add the link with escaped URL (both in href and display)
-        url = match.url
-        result.append(Markup(f'<a href="{escape(url)}">{escape(url)}</a>'))
-
-        last_index = match.last_index
-
-    # Add remaining escaped text
-    if last_index < len(body):
-        result.append(escape(body[last_index:]))
-
-    # Join all parts
-    return str(Markup(''.join(str(part) for part in result)))
-
-def notification_format_align_with_apprise(n_format : str):
-    """
-    Correctly align changedetection's formats with apprise's formats
-    Probably these are the same - but good to be sure.
-    These set the expected OUTPUT format type
-    :param n_format:
-    :return:
-    """
-
-    if n_format.startswith('html'):
-        # Apprise only knows 'html' not 'htmlcolor' etc, which shouldnt matter here
-        n_format = NotifyFormat.HTML.value
-    elif n_format.startswith('markdown'):
-        # probably the same but just to be safe
-        n_format = NotifyFormat.MARKDOWN.value
-    elif n_format.startswith('text'):
-        # probably the same but just to be safe
-        n_format = NotifyFormat.TEXT.value
-    else:
-        n_format = NotifyFormat.TEXT.value
-
-    return n_format
-
-
-def apply_html_color_to_body(n_body: str):
-    # https://github.com/dgtlmoon/changedetection.io/issues/821#issuecomment-1241837050
-    n_body = n_body.replace(REMOVED_PLACEMARKER_OPEN,
-                            f'<span style="{HTML_REMOVED_STYLE}" role="deletion" aria-label="Removed text" title="Removed text">')
-    n_body = n_body.replace(REMOVED_PLACEMARKER_CLOSED, f'</span>')
-    n_body = n_body.replace(ADDED_PLACEMARKER_OPEN,
-                            f'<span style="{HTML_ADDED_STYLE}" role="insertion" aria-label="Added text" title="Added text">')
-    n_body = n_body.replace(ADDED_PLACEMARKER_CLOSED, f'</span>')
-    # Handle changed/replaced lines (old → new)
-    n_body = n_body.replace(CHANGED_PLACEMARKER_OPEN,
-                            f'<span style="{HTML_CHANGED_STYLE}" role="note" aria-label="Changed text" title="Changed text">')
-    n_body = n_body.replace(CHANGED_PLACEMARKER_CLOSED, f'</span>')
-    n_body = n_body.replace(CHANGED_INTO_PLACEMARKER_OPEN,
-                            f'<span style="{HTML_CHANGED_INTO_STYLE}" role="note" aria-label="Changed into" title="Changed into">')
-    n_body = n_body.replace(CHANGED_INTO_PLACEMARKER_CLOSED, f'</span>')
-    return n_body
-
-def apply_discord_markdown_to_body(n_body):
-    """
-    Discord does not support <del> but it supports non-standard ~~strikethrough~~
-    :param n_body:
-    :return:
-    """
-    import re
-    # Define the mapping between your placeholders and markdown markers
-    replacements = [
-        (REMOVED_PLACEMARKER_OPEN, '~~', REMOVED_PLACEMARKER_CLOSED, '~~'),
-        (ADDED_PLACEMARKER_OPEN, '**', ADDED_PLACEMARKER_CLOSED, '**'),
-        (CHANGED_PLACEMARKER_OPEN, '~~', CHANGED_PLACEMARKER_CLOSED, '~~'),
-        (CHANGED_INTO_PLACEMARKER_OPEN, '**', CHANGED_INTO_PLACEMARKER_CLOSED, '**'),
-    ]
-    # So that the markdown gets added without any whitespace following it which would break it
-    for open_tag, open_md, close_tag, close_md in replacements:
-        # Regex: match opening tag, optional whitespace, capture the content, optional whitespace, then closing tag
-        pattern = re.compile(
-            re.escape(open_tag) + r'(\s*)(.*?)?(\s*)' + re.escape(close_tag),
-            flags=re.DOTALL
-        )
-        n_body = pattern.sub(lambda m: f"{m.group(1)}{open_md}{m.group(2)}{close_md}{m.group(3)}", n_body)
-    return n_body
-
-def apply_standard_markdown_to_body(n_body):
-    """
-    Apprise does not support ~~strikethrough~~ but it will convert <del> to HTML strikethrough.
-    :param n_body:
-    :return:
-    """
-    import re
-    # Define the mapping between your placeholders and markdown markers
-    replacements = [
-        (REMOVED_PLACEMARKER_OPEN, '<del>', REMOVED_PLACEMARKER_CLOSED, '</del>'),
-        (ADDED_PLACEMARKER_OPEN, '**', ADDED_PLACEMARKER_CLOSED, '**'),
-        (CHANGED_PLACEMARKER_OPEN, '<del>', CHANGED_PLACEMARKER_CLOSED, '</del>'),
-        (CHANGED_INTO_PLACEMARKER_OPEN, '**', CHANGED_INTO_PLACEMARKER_CLOSED, '**'),
-    ]
-
-    # So that the markdown gets added without any whitespace following it which would break it
-    for open_tag, open_md, close_tag, close_md in replacements:
-        # Regex: match opening tag, optional whitespace, capture the content, optional whitespace, then closing tag
-        pattern = re.compile(
-            re.escape(open_tag) + r'(\s*)(.*?)?(\s*)' + re.escape(close_tag),
-            flags=re.DOTALL
-        )
-        n_body = pattern.sub(lambda m: f"{m.group(1)}{open_md}{m.group(2)}{close_md}{m.group(3)}", n_body)
-    return n_body
-
-
-def replace_placemarkers_in_text(text, url, requested_output_format):
-    """
-    Replace diff placemarkers in text based on the URL service type and requested output format.
-    Used for both notification title and body to ensure consistent placeholder replacement.
-
-    :param text: The text to process
-    :param url: The notification URL (to detect service type)
-    :param requested_output_format: The output format (html, htmlcolor, markdown, text, etc.)
-    :return: Processed text with placemarkers replaced
-    """
-    if not text:
-        return text
-
-    if url.startswith('tgram://'):
-        # Telegram only supports a limited subset of HTML
-        # Use strikethrough for removed content, bold for added content
-        text = text.replace(REMOVED_PLACEMARKER_OPEN, '<s>')
-        text = text.replace(REMOVED_PLACEMARKER_CLOSED, '</s>')
-        text = text.replace(ADDED_PLACEMARKER_OPEN, '<b>')
-        text = text.replace(ADDED_PLACEMARKER_CLOSED, '</b>')
-        # Handle changed/replaced lines (old → new)
-        text = text.replace(CHANGED_PLACEMARKER_OPEN, '<s>')
-        text = text.replace(CHANGED_PLACEMARKER_CLOSED, '</s>')
-        text = text.replace(CHANGED_INTO_PLACEMARKER_OPEN, '<b>')
-        text = text.replace(CHANGED_INTO_PLACEMARKER_CLOSED, '</b>')
-    elif (url.startswith('discord://') or url.startswith('https://discordapp.com/api/webhooks')
-          or url.startswith('https://discord.com/api')) and requested_output_format == 'html':
-        # Discord doesn't support HTML, use Discord markdown
-        text = apply_discord_markdown_to_body(n_body=text)
-    elif requested_output_format == 'htmlcolor':
-        # https://github.com/dgtlmoon/changedetection.io/issues/821#issuecomment-1241837050
-        text = text.replace(REMOVED_PLACEMARKER_OPEN, f'<span style="{HTML_REMOVED_STYLE}" role="deletion" aria-label="Removed text" title="Removed text">')
-        text = text.replace(REMOVED_PLACEMARKER_CLOSED, f'</span>')
-        text = text.replace(ADDED_PLACEMARKER_OPEN, f'<span style="{HTML_ADDED_STYLE}" role="insertion" aria-label="Added text" title="Added text">')
-        text = text.replace(ADDED_PLACEMARKER_CLOSED, f'</span>')
-        # Handle changed/replaced lines (old → new)
-        text = text.replace(CHANGED_PLACEMARKER_OPEN, f'<span style="{HTML_CHANGED_STYLE}" role="note" aria-label="Changed text" title="Changed text">')
-        text = text.replace(CHANGED_PLACEMARKER_CLOSED, f'</span>')
-        text = text.replace(CHANGED_INTO_PLACEMARKER_OPEN, f'<span style="{HTML_CHANGED_INTO_STYLE}" role="note" aria-label="Changed into" title="Changed into">')
-        text = text.replace(CHANGED_INTO_PLACEMARKER_CLOSED, f'</span>')
-    elif requested_output_format == 'markdown':
-        # Markdown to HTML - Apprise will convert this to HTML
-        text = apply_standard_markdown_to_body(n_body=text)
-    else:
-        # plaintext, html, and default - use simple text markers
-        text = text.replace(REMOVED_PLACEMARKER_OPEN, '(removed) ')
-        text = text.replace(REMOVED_PLACEMARKER_CLOSED, '')
-        text = text.replace(ADDED_PLACEMARKER_OPEN, '(added) ')
-        text = text.replace(ADDED_PLACEMARKER_CLOSED, '')
-        text = text.replace(CHANGED_PLACEMARKER_OPEN, f'(changed) ')
-        text = text.replace(CHANGED_PLACEMARKER_CLOSED, f'')
-        text = text.replace(CHANGED_INTO_PLACEMARKER_OPEN, f'(into) ')
-        text = text.replace(CHANGED_INTO_PLACEMARKER_CLOSED, f'')
-
-    return text
-
-def apply_service_tweaks(url, n_body, n_title, requested_output_format):
-
-    # Re 323 - Limit discord length to their 2000 char limit total or it wont send.
-    # Because different notifications may require different pre-processing, run each sequentially :(
-    # 2000 bytes minus -
-    #     200 bytes for the overhead of the _entire_ json payload, 200 bytes for {tts, wait, content} etc headers
-    #     Length of URL - Incase they specify a longer custom avatar_url
-
-    if not n_body or not n_body.strip():
-        return url, n_body, n_title
-
-    # Normalize URL scheme to lowercase to prevent case-sensitivity issues
-    # e.g., "Discord://webhook" -> "discord://webhook", "TGRAM://bot123" -> "tgram://bot123"
-    scheme_separator_pos = url.find('://')
-    if scheme_separator_pos > 0:
-        url = url[:scheme_separator_pos].lower() + url[scheme_separator_pos:]
-
-    # So if no avatar_url is specified, add one so it can be correctly calculated into the total payload
-    parsed = urlparse(url)
-    k = '?' if not parsed.query else '&'
-    if url and not 'avatar_url' in url \
-            and not url.startswith('mail') \
-            and not url.startswith('post') \
-            and not url.startswith('get') \
-            and not url.startswith('delete') \
-            and not url.startswith('put'):
-        url += k + f"avatar_url={APPRISE_AVATAR_URL}"
-
-    # Replace placemarkers in title first (this was the missing piece causing the bug)
-    # Titles are ALWAYS plain text across all notification services (Discord embeds, Slack attachments,
-    # email Subject headers, etc.), so we always use 'text' format for title placemarker replacement
-    # Looking over apprise library it seems that all plugins only expect plain-text.
-    n_title = replace_placemarkers_in_text(n_title, url, 'text')
-
-    if url.startswith('tgram://'):
-        # Telegram only supports a limit subset of HTML, remove the '<br>' we place in.
-        # re https://github.com/dgtlmoon/changedetection.io/issues/555
-        # @todo re-use an existing library we have already imported to strip all non-allowed tags
-        n_body = n_body.replace('<br>', '\n')
-        n_body = n_body.replace('</br>', '\n')
-        n_body = newline_re.sub('\n', n_body)
-
-        # Replace placemarkers for body
-        n_body = replace_placemarkers_in_text(n_body, url, requested_output_format)
-
-        # real limit is 4096, but minus some for extra metadata
-        payload_max_size = 3600
-        body_limit = max(0, payload_max_size - len(n_title))
-        n_title = n_title[0:payload_max_size]
-        n_body = n_body[0:body_limit]
-
-    elif (url.startswith('discord://') or url.startswith('https://discordapp.com/api/webhooks')
-          or url.startswith('https://discord.com/api'))\
-            and 'html' in requested_output_format:
-        # Discord doesn't support HTML, replace <br> with newlines
-        n_body = n_body.strip().replace('<br>', '\n')
-        n_body = n_body.replace('</br>', '\n')
-        n_body = newline_re.sub('\n', n_body)
-
-        # Don't replace placeholders or truncate here - let the custom Discord plugin handle it
-        # The plugin will use embeds (6000 char limit across all embeds) if placeholders are present,
-        # or plain content (2000 char limit) otherwise
-
-        # Only do placeholder replacement if NOT using htmlcolor (which triggers embeds in custom plugin)
-        if requested_output_format == 'html':
-            # No diff placeholders, use Discord markdown for any other formatting
-            # Use Discord markdown: strikethrough for removed, bold for added
-            n_body = replace_placemarkers_in_text(n_body, url, requested_output_format)
-
-            # Apply 2000 char limit for plain content
-            payload_max_size = 1700
-            body_limit = max(0, payload_max_size - len(n_title))
-            n_title = n_title[0:payload_max_size]
-            n_body = n_body[0:body_limit]
-        # else: our custom Discord plugin will convert any placeholders left over into embeds with color bars
-
-    # Is not discord/tgram and they want htmlcolor
-    elif requested_output_format == 'htmlcolor':
-        n_body = replace_placemarkers_in_text(n_body, url, requested_output_format)
-        n_body = newline_re.sub('<br>\n', n_body)
-    elif requested_output_format == 'html':
-        n_body = replace_placemarkers_in_text(n_body, url, requested_output_format)
-        n_body = newline_re.sub('<br>\n', n_body)
-    elif requested_output_format == 'markdown':
-        # Markdown to HTML - Apprise will convert this to HTML
-        n_body = replace_placemarkers_in_text(n_body, url, requested_output_format)
-
-    else: #plaintext etc default
-        n_body = replace_placemarkers_in_text(n_body, url, requested_output_format)
-
-    return url, n_body, n_title
-
-
-def process_notification(n_object: NotificationContextData, datastore):
-    from changedetectionio.jinja2_custom import render as jinja_render
-    from . import USE_SYSTEM_DEFAULT_NOTIFICATION_FORMAT_FOR_WATCH, default_notification_format, valid_notification_formats
+def process_notification(n_object, datastore):
+    from changedetectionio.safe_jinja import render as jinja_render
+    from . import default_notification_format_for_watch, default_notification_format, valid_notification_formats
     # be sure its registered
     from .apprise_plugin.custom_handlers import apprise_http_custom_handler
-    # Register custom Discord plugin
-    from .apprise_plugin.discord import NotifyDiscordCustom
-
-    if not isinstance(n_object, NotificationContextData):
-        raise TypeError(f"Expected NotificationContextData, got {type(n_object)}")
 
     now = time.time()
     if n_object.get('notification_timestamp'):
         logger.trace(f"Time since queued {now-n_object['notification_timestamp']:.3f}s")
-
     # Insert variables into the notification content
     notification_parameters = create_notification_parameters(n_object, datastore)
 
-    requested_output_format = n_object.get('notification_format', default_notification_format)
-    logger.debug(f"Requested notification output format: '{requested_output_format}'")
+    n_format = valid_notification_formats.get(
+        n_object.get('notification_format', default_notification_format),
+        valid_notification_formats[default_notification_format],
+    )
 
     # If we arrived with 'System default' then look it up
-    if requested_output_format == USE_SYSTEM_DEFAULT_NOTIFICATION_FORMAT_FOR_WATCH:
+    if n_format == default_notification_format_for_watch and datastore.data['settings']['application'].get('notification_format') != default_notification_format_for_watch:
         # Initially text or whatever
-        requested_output_format = datastore.data['settings']['application'].get('notification_format', default_notification_format)
-
-    requested_output_format_original = requested_output_format
-
-    # Now clean it up so it fits perfectly with apprise
-    requested_output_format = notification_format_align_with_apprise(n_format=requested_output_format)
+        n_format = datastore.data['settings']['application'].get('notification_format', valid_notification_formats[default_notification_format])
 
     logger.trace(f"Complete notification body including Jinja and placeholders calculated in  {time.time() - now:.2f}s")
 
@@ -340,117 +39,99 @@ def process_notification(n_object: NotificationContextData, datastore):
 
     apobj = apprise.Apprise(debug=True, asset=apprise_asset)
 
-    # Override Apprise's built-in Discord plugin with our custom one
-    # This allows us to use colored embeds for diff content
-    # First remove the built-in discord plugin, then add our custom one
-    apprise.plugins.N_MGR.remove('discord')
-    apprise.plugins.N_MGR.add(NotifyDiscordCustom, schemas='discord')
-
     if not n_object.get('notification_urls'):
         return None
 
-    n_object.update(add_rendered_diff_to_notification_vars(
-        notification_scan_text=n_object.get('notification_body', '')+n_object.get('notification_title', ''),
-        current_snapshot=n_object.get('current_snapshot'),
-        prev_snapshot=n_object.get('prev_snapshot'),
-        # Should always be false for 'text' mode or its too hard to read
-        # But otherwise, this could be some setting
-        word_diff=False if requested_output_format_original == 'text' else True,
-        )
-    )
-
-    with (apprise.LogCapture(level=apprise.logging.DEBUG) as logs):
+    with apprise.LogCapture(level=apprise.logging.DEBUG) as logs:
         for url in n_object['notification_urls']:
 
+            # Get the notification body from datastore
             n_body = jinja_render(template_str=n_object.get('notification_body', ''), **notification_parameters)
+            if n_object.get('notification_format', '').startswith('HTML'):
+                n_body = n_body.replace("\n", '<br>')
+
             n_title = jinja_render(template_str=n_object.get('notification_title', ''), **notification_parameters)
 
-            if n_object.get('markup_text_links_to_html_links'):
-                n_body = markup_text_links_to_html(body=n_body)
-
             url = url.strip()
-            if not url or url.startswith('#'):
-                logger.debug(f"Skipping commented out or empty notification URL - '{url}'")
+            if url.startswith('#'):
+                logger.trace(f"Skipping commented out notification URL - {url}")
                 continue
 
-            logger.info(f">> Process Notification: AppRise start notifying '{url}'")
+            if not url:
+                logger.warning(f"Process Notification: skipping empty notification URL.")
+                continue
+
+            logger.info(f">> Process Notification: AppRise notifying {url}")
             url = jinja_render(template_str=url, **notification_parameters)
 
-            # If it's a plaintext document, and they want HTML type email/alerts, so it needs to be escaped
-            watch_mime_type = n_object.get('watch_mime_type')
-            if watch_mime_type and 'text/' in watch_mime_type.lower() and not 'html' in watch_mime_type.lower():
-                if 'html' in requested_output_format:
-                    from markupsafe import escape
-                    n_body = str(escape(n_body))
+            # Re 323 - Limit discord length to their 2000 char limit total or it wont send.
+            # Because different notifications may require different pre-processing, run each sequentially :(
+            # 2000 bytes minus -
+            #     200 bytes for the overhead of the _entire_ json payload, 200 bytes for {tts, wait, content} etc headers
+            #     Length of URL - Incase they specify a longer custom avatar_url
 
-            if 'html' in requested_output_format:
-                # Since the n_body is always some kind of text from the 'diff' engine, attempt to preserve whitespaces that get sent to the HTML output
-                # But only where its more than 1 consecutive whitespace, otherwise "and this" becomes "and&nbsp;this" etc which is too much.
-                n_body = n_body.replace('  ', '&nbsp;&nbsp;')
+            # So if no avatar_url is specified, add one so it can be correctly calculated into the total payload
+            k = '?' if not '?' in url else '&'
+            if not 'avatar_url' in url \
+                    and not url.startswith('mail') \
+                    and not url.startswith('post') \
+                    and not url.startswith('get') \
+                    and not url.startswith('delete') \
+                    and not url.startswith('put'):
+                url += k + f"avatar_url={APPRISE_AVATAR_URL}"
 
-            (url, n_body, n_title) = apply_service_tweaks(url=url, n_body=n_body, n_title=n_title, requested_output_format=requested_output_format_original)
+            if url.startswith('tgram://'):
+                # Telegram only supports a limit subset of HTML, remove the '<br>' we place in.
+                # re https://github.com/dgtlmoon/changedetection.io/issues/555
+                # @todo re-use an existing library we have already imported to strip all non-allowed tags
+                n_body = n_body.replace('<br>', '\n')
+                n_body = n_body.replace('</br>', '\n')
+                # real limit is 4096, but minus some for extra metadata
+                payload_max_size = 3600
+                body_limit = max(0, payload_max_size - len(n_title))
+                n_title = n_title[0:payload_max_size]
+                n_body = n_body[0:body_limit]
 
-            apprise_input_format = "NO-THANKS-WE-WILL-MANAGE-ALL-OF-THIS"
+            elif url.startswith('discord://') or url.startswith('https://discordapp.com/api/webhooks') or url.startswith(
+                    'https://discord.com/api'):
+                # real limit is 2000, but minus some for extra metadata
+                payload_max_size = 1700
+                body_limit = max(0, payload_max_size - len(n_title))
+                n_title = n_title[0:payload_max_size]
+                n_body = n_body[0:body_limit]
 
-            if not 'format=' in url:
-                parsed_url = urlparse(url)
-                prefix_add_to_url = '?' if not parsed_url.query else '&'
-
-                # THIS IS THE TRICK HOW TO DISABLE APPRISE DOING WEIRD AUTO-CONVERSION WITH BREAKING BR TAGS ETC
-                if 'html' in requested_output_format:
-                    url = f"{url}{prefix_add_to_url}format={NotifyFormat.HTML.value}"
-                    apprise_input_format = NotifyFormat.HTML.value
-                elif 'text' in requested_output_format:
-                    url = f"{url}{prefix_add_to_url}format={NotifyFormat.TEXT.value}"
-                    apprise_input_format = NotifyFormat.TEXT.value
-
-                elif requested_output_format == NotifyFormat.MARKDOWN.value:
-                    # Convert markdown to HTML ourselves since not all plugins do this
-                    from apprise.conversion import markdown_to_html
-                    # Make sure there are paragraph breaks around horizontal rules
-                    n_body = n_body.replace('---', '\n\n---\n\n')
-                    n_body = markdown_to_html(n_body)
-                    url = f"{url}{prefix_add_to_url}format={NotifyFormat.HTML.value}"
-                    requested_output_format = NotifyFormat.HTML.value
-                    apprise_input_format = NotifyFormat.HTML.value  # Changed from MARKDOWN to HTML
-
-            else:
-                # ?format was IN the apprise URL, they are kind of on their own here, we will try our best
-                if 'format=html' in url:
-                    n_body = newline_re.sub('<br>\r\n', n_body)
-                    # This will also prevent apprise from doing conversion
-                    apprise_input_format = NotifyFormat.HTML.value
-                    requested_output_format = NotifyFormat.HTML.value
-                elif 'format=text' in url:
-                    apprise_input_format = NotifyFormat.TEXT.value
-                    requested_output_format = NotifyFormat.TEXT.value
+            elif url.startswith('mailto'):
+                # Apprise will default to HTML, so we need to override it
+                # So that whats' generated in n_body is in line with what is going to be sent.
+                # https://github.com/caronc/apprise/issues/633#issuecomment-1191449321
+                if not 'format=' in url and (n_format == 'Text' or n_format == 'Markdown'):
+                    prefix = '?' if not '?' in url else '&'
+                    # Apprise format is lowercase text https://github.com/caronc/apprise/issues/633
+                    n_format = n_format.lower()
+                    url = f"{url}{prefix}format={n_format}"
+                # If n_format == HTML, then apprise email should default to text/html and we should be sending HTML only
 
+            apobj.add(url)
 
             sent_objs.append({'title': n_title,
                               'body': n_body,
-                              'url': url})
-            apobj.add(url)
-
-            # Since the output is always based on the plaintext of the 'diff' engine, wrap it nicely.
-            # It should always be similar to the 'history' part of the UI.
-            if url.startswith('mail') and 'html' in requested_output_format:
-                if not '<pre' in n_body and not '<body' in n_body: # No custom HTML-ish body was setup already
-                    n_body = as_monospaced_html_email(content=n_body, title=n_title)
+                              'url': url,
+                              'body_format': n_format})
 
+        # Blast off the notifications tht are set in .add()
         apobj.notify(
             title=n_title,
             body=n_body,
-            # `body_format` Tell apprise what format the INPUT is in, specify a wrong/bad type and it will force skip conversion in apprise
-            # &format= in URL Tell apprise what format the OUTPUT should be in (it can convert between)
-            body_format=apprise_input_format,
+            body_format=n_format,
             # False is not an option for AppRise, must be type None
             attach=n_object.get('screenshot', None)
         )
 
+
         # Returns empty string if nothing found, multi-line string otherwise
         log_value = logs.getvalue()
 
-        if log_value and ('WARNING' in log_value or 'ERROR' in log_value):
+        if log_value and 'WARNING' in log_value or 'ERROR' in log_value:
             logger.critical(log_value)
             raise Exception(log_value)
 
@@ -460,15 +141,17 @@ def process_notification(n_object: NotificationContextData, datastore):
 
 # Notification title + body content parameters get created here.
 # ( Where we prepare the tokens in the notification to be replaced with actual values )
-def create_notification_parameters(n_object: NotificationContextData, datastore):
-    if not isinstance(n_object, NotificationContextData):
-        raise TypeError(f"Expected NotificationContextData, got {type(n_object)}")
+def create_notification_parameters(n_object, datastore):
+    from copy import deepcopy
+    from . import valid_tokens
 
-    watch = datastore.data['watching'].get(n_object['uuid'])
-    if watch:
-        watch_title = datastore.data['watching'][n_object['uuid']].label
+    # in the case we send a test notification from the main settings, there is no UUID.
+    uuid = n_object['uuid'] if 'uuid' in n_object else ''
+
+    if uuid:
+        watch_title = datastore.data['watching'][uuid].label
         tag_list = []
-        tags = datastore.get_all_tags_for_watch(n_object['uuid'])
+        tags = datastore.get_all_tags_for_watch(uuid)
         if tags:
             for tag_uuid, tag in tags.items():
                 tag_list.append(tag.get('title'))
@@ -483,10 +166,14 @@ def create_notification_parameters(n_object: NotificationContextData, datastore)
 
     watch_url = n_object['watch_url']
 
-    diff_url = "{}/diff/{}".format(base_url, n_object['uuid'])
-    preview_url = "{}/preview/{}".format(base_url, n_object['uuid'])
+    diff_url = "{}/diff/{}".format(base_url, uuid)
+    preview_url = "{}/preview/{}".format(base_url, uuid)
 
-    n_object.update(
+    # Not sure deepcopy is needed here, but why not
+    tokens = deepcopy(valid_tokens)
+
+    # Valid_tokens also used as a field validator
+    tokens.update(
         {
             'base_url': base_url,
             'diff_url': diff_url,
@@ -494,10 +181,13 @@ def create_notification_parameters(n_object: NotificationContextData, datastore)
             'watch_tag': watch_tag if watch_tag is not None else '',
             'watch_title': watch_title if watch_title is not None else '',
             'watch_url': watch_url,
-            'watch_uuid': n_object['uuid'],
+            'watch_uuid': uuid,
         })
 
-    if watch:
-        n_object.update(datastore.data['watching'].get(n_object['uuid']).extra_notification_token_values())
+    # n_object will contain diff, diff_added etc etc
+    tokens.update(n_object)
 
-    return n_object
+    if uuid:
+        tokens.update(datastore.data['watching'].get(uuid).extra_notification_token_values())
+
+    return tokens

changedetectionio/notification_service.py

@@ -6,150 +6,9 @@ Extracted from update_worker.py to provide standalone notification functionality
 for both sync and async workers
 """
 
-from loguru import logger
 import time
+from loguru import logger
 
-from changedetectionio.notification import default_notification_format, valid_notification_formats
-
-
-
-# What is passed around as notification context, also used as the complete list of valid {{ tokens }}
-class NotificationContextData(dict):
-    def __init__(self, initial_data=None, **kwargs):
-        super().__init__({
-            'base_url': None,
-            'current_snapshot': None,
-            'diff': None,
-            'diff_clean': None,
-            'diff_added': None,
-            'diff_added_clean': None,
-            'diff_full': None,
-            'diff_full_clean': None,
-            'diff_patch': None,
-            'diff_removed': None,
-            'diff_removed_clean': None,
-            'diff_url': None,
-            'markup_text_links_to_html_links': False, # If automatic conversion of plaintext to HTML should happen
-            'notification_timestamp': time.time(),
-            'preview_url': None,
-            'screenshot': None,
-            'triggered_text': None,
-            'uuid': 'XXXXXXXX-XXXX-XXXX-XXXX-XXXXXXXXXXXX',  # Converted to 'watch_uuid' in create_notification_parameters
-            'watch_mime_type': None,
-            'watch_tag': None,
-            'watch_title': None,
-            'watch_url': 'https://WATCH-PLACE-HOLDER/',
-        })
-
-        # Apply any initial data passed in
-        self.update({'watch_uuid': self.get('uuid')})
-        if initial_data:
-            self.update(initial_data)
-
-        # Apply any keyword arguments
-        if kwargs:
-            self.update(kwargs)
-
-        n_format = self.get('notification_format')
-        if n_format and not valid_notification_formats.get(n_format):
-            raise ValueError(f'Invalid notification format: "{n_format}"')
-
-    def set_random_for_validation(self):
-        import random, string
-        """Randomly fills all dict keys with random strings (for validation/testing). 
-        So we can test the output in the notification body
-        """
-        for key in self.keys():
-            if key in ['uuid', 'time', 'watch_uuid']:
-                continue
-            rand_str = 'RANDOM-PLACEHOLDER-'+''.join(random.choices(string.ascii_letters + string.digits, k=12))
-            self[key] = rand_str
-
-    def __setitem__(self, key, value):
-        if key == 'notification_format' and isinstance(value, str) and not value.startswith('RANDOM-PLACEHOLDER-'):
-            if not valid_notification_formats.get(value):
-                raise ValueError(f'Invalid notification format: "{value}"')
-
-        super().__setitem__(key, value)
-
-def add_rendered_diff_to_notification_vars(notification_scan_text:str, prev_snapshot:str, current_snapshot:str, word_diff:bool):
-    """
-    Efficiently renders only the diff placeholders that are actually used in the notification text.
-
-    Scans the notification template for diff placeholder usage (diff, diff_added, diff_clean, etc.)
-    and only renders those specific variants, avoiding expensive render_diff() calls for unused placeholders.
-    Uses LRU caching to avoid duplicate renders when multiple placeholders share the same arguments.
-
-    Args:
-        notification_scan_text: The notification template text to scan for placeholders
-        prev_snapshot: Previous version of content for diff comparison
-        current_snapshot: Current version of content for diff comparison
-        word_diff: Whether to use word-level (True) or line-level (False) diffing
-
-    Returns:
-        dict: Only the diff placeholders that were found in notification_scan_text, with rendered content
-    """
-    from changedetectionio import diff
-    import re
-    from functools import lru_cache
-
-    now = time.time()
-
-    # Define specifications for each diff variant
-    diff_specs = {
-        'diff': {'word_diff': word_diff},
-        'diff_clean': {'word_diff': word_diff, 'include_change_type_prefix': False},
-        'diff_added': {'word_diff': word_diff, 'include_removed': False},
-        'diff_added_clean': {'word_diff': word_diff, 'include_removed': False, 'include_change_type_prefix': False},
-        'diff_full': {'word_diff': word_diff, 'include_equal': True},
-        'diff_full_clean': {'word_diff': word_diff, 'include_equal': True, 'include_change_type_prefix': False},
-        'diff_patch': {'word_diff': word_diff, 'patch_format': True},
-        'diff_removed': {'word_diff': word_diff, 'include_added': False},
-        'diff_removed_clean': {'word_diff': word_diff, 'include_added': False, 'include_change_type_prefix': False},
-    }
-
-    # Memoize render_diff to avoid duplicate renders with same kwargs
-    @lru_cache(maxsize=4)
-    def cached_render(kwargs_tuple):
-        return diff.render_diff(prev_snapshot, current_snapshot, **dict(kwargs_tuple))
-
-    ret = {}
-    rendered_count = 0
-    # Only check and render diff keys that exist in NotificationContextData
-    for key in NotificationContextData().keys():
-        if key.startswith('diff') and key in diff_specs:
-            # Check if this placeholder is actually used in the notification text
-            pattern = rf"(?<![A-Za-z0-9_]){re.escape(key)}(?![A-Za-z0-9_])"
-            if re.search(pattern, notification_scan_text, re.IGNORECASE):
-                kwargs = diff_specs[key]
-                # Convert dict to sorted tuple for cache key (handles duplicate kwarg combinations)
-                ret[key] = cached_render(tuple(sorted(kwargs.items())))
-                rendered_count += 1
-
-    if rendered_count:
-        logger.trace(f"Rendered {rendered_count} diff placeholder(s) {sorted(ret.keys())} in {time.time() - now:.3f}s")
-
-    return ret
-
-def set_basic_notification_vars(current_snapshot, prev_snapshot, watch, triggered_text):
-
-    n_object = {
-        'current_snapshot': current_snapshot,
-        'prev_snapshot': prev_snapshot,
-        'screenshot': watch.get_screenshot() if watch and watch.get('notification_screenshot') else None,
-        'triggered_text': triggered_text,
-        'uuid': watch.get('uuid') if watch else None,
-        'watch_url': watch.get('url') if watch else None,
-        'watch_uuid': watch.get('uuid') if watch else None,
-        'watch_mime_type': watch.get('content-type')
-    }
-
-    # The \n's in the content from the above will get converted to <br> etc depending on the notification format
-
-    if watch:
-        n_object.update(watch.extra_notification_token_values())
-
-    return n_object
 
 class NotificationService:
     """
@@ -161,18 +20,18 @@ class NotificationService:
         self.datastore = datastore
         self.notification_q = notification_q
     
-    def queue_notification_for_watch(self, n_object: NotificationContextData, watch):
+    def queue_notification_for_watch(self, n_object, watch):
         """
         Queue a notification for a watch with full diff rendering and template variables
         """
-        from changedetectionio.notification import USE_SYSTEM_DEFAULT_NOTIFICATION_FORMAT_FOR_WATCH
-
-        if not isinstance(n_object, NotificationContextData):
-            raise TypeError(f"Expected NotificationContextData, got {type(n_object)}")
+        from changedetectionio import diff
+        from changedetectionio.notification import default_notification_format_for_watch
 
         dates = []
         trigger_text = ''
 
+        now = time.time()
+
         if watch:
             watch_history = watch.history
             dates = list(watch_history.keys())
@@ -185,16 +44,29 @@ class NotificationService:
             snapshot_contents = "No snapshot/history available, the watch should fetch atleast once."
 
         # If we ended up here with "System default"
-        if n_object.get('notification_format') == USE_SYSTEM_DEFAULT_NOTIFICATION_FORMAT_FOR_WATCH:
+        if n_object.get('notification_format') == default_notification_format_for_watch:
             n_object['notification_format'] = self.datastore.data['settings']['application'].get('notification_format')
 
+        html_colour_enable = False
+        # HTML needs linebreak, but MarkDown and Text can use a linefeed
+        if n_object.get('notification_format') == 'HTML':
+            line_feed_sep = "<br>"
+            # Snapshot will be plaintext on the disk, convert to some kind of HTML
+            snapshot_contents = snapshot_contents.replace('\n', line_feed_sep)
+        elif n_object.get('notification_format') == 'HTML Color':
+            line_feed_sep = "<br>"
+            # Snapshot will be plaintext on the disk, convert to some kind of HTML
+            snapshot_contents = snapshot_contents.replace('\n', line_feed_sep)
+            html_colour_enable = True
+        else:
+            line_feed_sep = "\n"
 
         triggered_text = ''
         if len(trigger_text):
             from . import html_tools
             triggered_text = html_tools.get_triggered_text(content=snapshot_contents, trigger_text=trigger_text)
             if triggered_text:
-                triggered_text = '\n'.join(triggered_text)
+                triggered_text = line_feed_sep.join(triggered_text)
 
         # Could be called as a 'test notification' with only 1 snapshot available
         prev_snapshot = "Example text: example test\nExample text: change detection is cool\nExample text: some more examples\n"
@@ -204,12 +76,24 @@ class NotificationService:
             prev_snapshot = watch.get_history_snapshot(dates[-2])
             current_snapshot = watch.get_history_snapshot(dates[-1])
 
+        n_object.update({
+            'current_snapshot': snapshot_contents,
+            'diff': diff.render_diff(prev_snapshot, current_snapshot, line_feed_sep=line_feed_sep, html_colour=html_colour_enable),
+            'diff_added': diff.render_diff(prev_snapshot, current_snapshot, include_removed=False, line_feed_sep=line_feed_sep),
+            'diff_full': diff.render_diff(prev_snapshot, current_snapshot, include_equal=True, line_feed_sep=line_feed_sep, html_colour=html_colour_enable),
+            'diff_patch': diff.render_diff(prev_snapshot, current_snapshot, line_feed_sep=line_feed_sep, patch_format=True),
+            'diff_removed': diff.render_diff(prev_snapshot, current_snapshot, include_added=False, line_feed_sep=line_feed_sep),
+            'notification_timestamp': now,
+            'screenshot': watch.get_screenshot() if watch and watch.get('notification_screenshot') else None,
+            'triggered_text': triggered_text,
+            'uuid': watch.get('uuid') if watch else None,
+            'watch_url': watch.get('url') if watch else None,
+        })
 
-        n_object.update(set_basic_notification_vars(current_snapshot=current_snapshot,
-                                                    prev_snapshot=prev_snapshot,
-                                                    watch=watch,
-                                                    triggered_text=triggered_text))
+        if watch:
+            n_object.update(watch.extra_notification_token_values())
 
+        logger.trace(f"Main rendered notification placeholders (diff_added etc) calculated in {time.time()-now:.3f}s")
         logger.debug("Queued notification for sending")
         self.notification_q.put(n_object)
 
@@ -219,7 +103,7 @@ class NotificationService:
         Individual watch settings > Tag settings > Global settings
         """
         from changedetectionio.notification import (
-            USE_SYSTEM_DEFAULT_NOTIFICATION_FORMAT_FOR_WATCH,
+            default_notification_format_for_watch,
             default_notification_body,
             default_notification_title
         )
@@ -227,7 +111,7 @@ class NotificationService:
         # Would be better if this was some kind of Object where Watch can reference the parent datastore etc
         v = watch.get(var_name)
         if v and not watch.get('notification_muted'):
-            if var_name == 'notification_format' and v == USE_SYSTEM_DEFAULT_NOTIFICATION_FORMAT_FOR_WATCH:
+            if var_name == 'notification_format' and v == default_notification_format_for_watch:
                 return self.datastore.data['settings']['application'].get('notification_format')
 
             return v
@@ -244,7 +128,7 @@ class NotificationService:
 
         # Otherwise could be defaults
         if var_name == 'notification_format':
-            return USE_SYSTEM_DEFAULT_NOTIFICATION_FORMAT_FOR_WATCH
+            return default_notification_format_for_watch
         if var_name == 'notification_body':
             return default_notification_body
         if var_name == 'notification_title':
@@ -256,7 +140,7 @@ class NotificationService:
         """
         Send notification when content changes are detected
         """
-        n_object = NotificationContextData()
+        n_object = {}
         watch = self.datastore.data['watching'].get(watch_uuid)
         if not watch:
             return
@@ -299,25 +183,11 @@ class NotificationService:
         if not watch:
             return
 
-        filter_list = ", ".join(watch['include_filters'])
-        # @todo - This could be a markdown template on the disk, apprise will convert the markdown to HTML+Plaintext parts in the email, and then 'markup_text_links_to_html_links' is not needed
-        body = f"""Hello,
-
-Your configured CSS/xPath filters of '{filter_list}' for {{{{watch_url}}}} did not appear on the page after {threshold} attempts.
-
-It's possible the page changed layout and the filter needs updating ( Try the 'Visual Selector' tab )
-
-Edit link: {{{{base_url}}}}/edit/{{{{watch_uuid}}}}
-
-Thanks - Your omniscient changedetection.io installation.
-"""
-
-        n_object = NotificationContextData({
-            'notification_title': 'Changedetection.io - Alert - CSS/xPath filter was not present in the page',
-            'notification_body': body,
-            'notification_format': self._check_cascading_vars('notification_format', watch),
-        })
-        n_object['markup_text_links_to_html_links'] = n_object.get('notification_format').startswith('html')
+        n_object = {'notification_title': 'Changedetection.io - Alert - CSS/xPath filter was not present in the page',
+                    'notification_body': "Your configured CSS/xPath filters of '{}' for {{{{watch_url}}}} did not appear on the page after {} attempts, did the page change layout?\n\nLink: {{{{base_url}}}}/edit/{{{{watch_uuid}}}}\n\nThanks - Your omniscient changedetection.io installation :)\n".format(
+                        ", ".join(watch['include_filters']),
+                        threshold),
+                    'notification_format': 'text'}
 
         if len(watch['notification_urls']):
             n_object['notification_urls'] = watch['notification_urls']
@@ -345,28 +215,12 @@ Thanks - Your omniscient changedetection.io installation.
         if not watch:
             return
         threshold = self.datastore.data['settings']['application'].get('filter_failure_notification_threshold_attempts')
-
-        step = step_n + 1
-        # @todo - This could be a markdown template on the disk, apprise will convert the markdown to HTML+Plaintext parts in the email, and then 'markup_text_links_to_html_links' is not needed
-
-        # {{{{ }}}} because this will be Jinja2 {{ }} tokens
-        body = f"""Hello,
-        
-Your configured browser step at position {step} for the web page watch {{{{watch_url}}}} did not appear on the page after {threshold} attempts, did the page change layout?
-
-The element may have moved and needs editing, or does it need a delay added?
-
-Edit link: {{{{base_url}}}}/edit/{{{{watch_uuid}}}}
-
-Thanks - Your omniscient changedetection.io installation.
-"""
-
-        n_object = NotificationContextData({
-            'notification_title': f"Changedetection.io - Alert - Browser step at position {step} could not be run",
-            'notification_body': body,
-            'notification_format': self._check_cascading_vars('notification_format', watch),
-        })
-        n_object['markup_text_links_to_html_links'] = n_object.get('notification_format').startswith('html')
+        n_object = {'notification_title': "Changedetection.io - Alert - Browser step at position {} could not be run".format(step_n+1),
+                    'notification_body': "Your configured browser step at position {} for {{{{watch_url}}}} "
+                                         "did not appear on the page after {} attempts, did the page change layout? "
+                                         "Does it need a delay added?\n\nLink: {{{{base_url}}}}/edit/{{{{watch_uuid}}}}\n\n"
+                                         "Thanks - Your omniscient changedetection.io installation :)\n".format(step_n+1, threshold),
+                    'notification_format': 'text'}
 
         if len(watch['notification_urls']):
             n_object['notification_urls'] = watch['notification_urls']

changedetectionio/processors/__init__.py

@@ -91,8 +91,6 @@ class difference_detection_processor():
             else:
                 logger.debug("Skipping adding proxy data when custom Browser endpoint is specified. ")
 
-        logger.debug(f"Using proxy '{proxy_url}' for {self.watch['uuid']}")
-
         # Now call the fetcher (playwright/requests/etc) with arguments that only a fetcher would need.
         # When browser_connection_url is None, it method should default to working out whats the best defaults (os env vars etc)
         self.fetcher = fetcher_obj(proxy_override=proxy_url,
@@ -104,7 +102,7 @@ class difference_detection_processor():
             self.fetcher.browser_steps_screenshot_path = os.path.join(self.datastore.datastore_path, self.watch.get('uuid'))
 
         # Tweak the base config with the per-watch ones
-        from changedetectionio.jinja2_custom import render as jinja_render
+        from changedetectionio.safe_jinja import render as jinja_render
         request_headers = CaseInsensitiveDict()
 
         ua = self.datastore.data['settings']['requests'].get('default_ua')

changedetectionio/processors/magic.py

@@ -64,31 +64,24 @@ class guess_stream_type():
         # Remove whitespace between < and tag name for robust detection (handles '< html', '<\nhtml', etc.)
         test_content_normalized = re.sub(r'<\s+', '<', test_content)
 
-        # Use puremagic for lightweight MIME detection (saves ~14MB vs python-magic)
+        # Magic will sometimes call text/plain as text/html!
         magic_result = None
         try:
-            import puremagic
+            import magic
 
-            # puremagic needs bytes, so encode if we have a string
-            content_bytes = content[:200].encode('utf-8') if isinstance(content, str) else content[:200]
-
-            # puremagic returns a list of PureMagic objects with confidence scores
-            detections = puremagic.magic_string(content_bytes)
-            if detections:
-                # Get the highest confidence detection
-                mime = detections[0].mime_type
-                logger.debug(f"Guessing mime type, original content_type '{http_content_header}', mime type detected '{mime}'")
-                if mime and "/" in mime:
-                    magic_result = mime
-                    # Ignore generic/fallback mime types
-                    if mime in ['application/octet-stream', 'application/x-empty', 'binary']:
-                        logger.debug(f"Ignoring generic mime type '{mime}' from puremagic library")
-                    # Trust puremagic for non-text types immediately
-                    elif mime not in ['text/html', 'text/plain']:
-                        magic_content_header = mime
+            mime = magic.from_buffer(content[:200], mime=True) # Send the original content
+            logger.debug(f"Guessing mime type, original content_type '{http_content_header}', mime type detected '{mime}'")
+            if mime and "/" in mime:
+                magic_result = mime
+                # Ignore generic/fallback mime types from magic
+                if mime in ['application/octet-stream', 'application/x-empty', 'binary']:
+                    logger.debug(f"Ignoring generic mime type '{mime}' from magic library")
+                # Trust magic for non-text types immediately
+                elif mime not in ['text/html', 'text/plain']:
+                    magic_content_header = mime
 
         except Exception as e:
-            logger.warning(f"Error getting a more precise mime type from 'puremagic' library ({str(e)}), using content-based detection")
+            logger.error(f"Error getting a more precise mime type from 'magic' library ({str(e)}), using content-based detection")
 
         # Content-based detection (most reliable for text formats)
         # Check for HTML patterns first - if found, override magic's text/plain
@@ -101,21 +94,24 @@ class guess_stream_type():
             self.is_rss = True
         elif any(s in http_content_header for s in JSON_CONTENT_TYPES):
             self.is_json = True
-        elif 'pdf' in magic_content_header:
-            self.is_pdf = True
-        elif has_html_patterns or http_content_header == 'text/html':
-            self.is_html = True
-        elif any(s in magic_content_header for s in JSON_CONTENT_TYPES):
-            self.is_json = True
-        # magic will call a rss document 'xml'
-        # Rarely do endpoints give the right header, usually just text/xml, so we check also for <rss
-        # This also triggers the automatic CDATA text parser so the RSS goes back a nice content list
-        elif '<rss' in test_content_normalized or '<feed' in test_content_normalized or any(s in magic_content_header for s in RSS_XML_CONTENT_TYPES) or '<rdf:' in test_content_normalized:
-            self.is_rss = True
         elif any(s in http_content_header for s in XML_CONTENT_TYPES):
             # Only mark as generic XML if not already detected as RSS
             if not self.is_rss:
                 self.is_xml = True
+        elif 'pdf' in magic_content_header:
+            self.is_pdf = True
+###
+        elif has_html_patterns or http_content_header == 'text/html':
+            self.is_html = True
+        # If magic says text/plain and we found no HTML patterns, trust it
+        elif magic_result == 'text/plain':
+            self.is_plaintext = True
+            logger.debug(f"Trusting magic's text/plain result (no HTML patterns detected)")
+        elif any(s in magic_content_header for s in JSON_CONTENT_TYPES):
+            self.is_json = True
+        # magic will call a rss document 'xml'
+        elif '<rss' in test_content_normalized or '<feed' in test_content_normalized or any(s in magic_content_header for s in RSS_XML_CONTENT_TYPES):
+            self.is_rss = True
         elif test_content_normalized.startswith('<?xml') or any(s in magic_content_header for s in XML_CONTENT_TYPES):
             # Generic XML that's not RSS/Atom (RSS/Atom checked above)
             self.is_xml = True
@@ -126,8 +122,4 @@ class guess_stream_type():
         # Only trust magic for 'text' if no other patterns matched
         elif 'text' in magic_content_header:
             self.is_plaintext = True
-        # If magic says text/plain and we found no HTML patterns, trust it
-        elif magic_result == 'text/plain':
-            self.is_plaintext = True
-            logger.debug(f"Trusting magic's text/plain result (no HTML patterns detected)")
 

changedetectionio/processors/text_json_diff/__init__.py

@@ -32,7 +32,7 @@ def prepare_filter_prevew(datastore, watch_uuid, form_data):
     '''Used by @app.route("/edit/<string:uuid>/preview-rendered", methods=['POST'])'''
     from changedetectionio import forms, html_tools
     from changedetectionio.model.Watch import model as watch_model
-    from concurrent.futures import ThreadPoolExecutor
+    from concurrent.futures import ProcessPoolExecutor
     from copy import deepcopy
     from flask import request
     import brotli
@@ -45,7 +45,6 @@ def prepare_filter_prevew(datastore, watch_uuid, form_data):
     text_before_filter = ''
     trigger_line_numbers = []
     ignore_line_numbers = []
-    blocked_line_numbers = []
 
     tmp_watch = deepcopy(datastore.data['watching'].get(watch_uuid))
 
@@ -77,16 +76,13 @@ def prepare_filter_prevew(datastore, watch_uuid, form_data):
             update_handler.fetcher.headers['content-type'] = tmp_watch.get('content-type')
 
             # Process our watch with filters and the HTML from disk, and also a blank watch with no filters but also with the same HTML from disk
-            # Do this as parallel threads (not processes) to avoid pickle issues with Lock objects
-            try:
-                with ThreadPoolExecutor(max_workers=2) as executor:
-                    future1 = executor.submit(_task, tmp_watch, update_handler)
-                    future2 = executor.submit(_task, blank_watch_no_filters, update_handler)
+            # Do this as a parallel process because it could take some time
+            with ProcessPoolExecutor(max_workers=2) as executor:
+                future1 = executor.submit(_task, tmp_watch, update_handler)
+                future2 = executor.submit(_task, blank_watch_no_filters, update_handler)
 
-                    text_after_filter = future1.result()
-                    text_before_filter = future2.result()
-            except Exception as e:
-                x=1
+                text_after_filter = future1.result()
+                text_before_filter = future2.result()
 
     try:
         trigger_line_numbers = html_tools.strip_ignore_text(content=text_after_filter,
@@ -105,23 +101,14 @@ def prepare_filter_prevew(datastore, watch_uuid, form_data):
     except Exception as e:
         text_before_filter = f"Error: {str(e)}"
 
-    try:
-        blocked_line_numbers = html_tools.strip_ignore_text(content=text_after_filter,
-                                                           wordlist=tmp_watch.get('text_should_not_be_present', []) + datastore.data['settings']['application'].get('text_should_not_be_present', []),
-                                                           mode='line numbers'
-                                                           )
-    except Exception as e:
-        text_before_filter = f"Error: {str(e)}"
-
     logger.trace(f"Parsed in {time.time() - now:.3f}s")
 
     return ({
-        'after_filter': text_after_filter,
-        'before_filter': text_before_filter.decode('utf-8') if isinstance(text_before_filter, bytes) else text_before_filter,
-        'blocked_line_numbers': blocked_line_numbers,
-        'duration': time.time() - now,
-        'ignore_line_numbers': ignore_line_numbers,
-        'trigger_line_numbers': trigger_line_numbers,
+            'after_filter': text_after_filter,
+            'before_filter': text_before_filter.decode('utf-8') if isinstance(text_before_filter, bytes) else text_before_filter,
+            'duration': time.time() - now,
+            'trigger_line_numbers': trigger_line_numbers,
+            'ignore_line_numbers': ignore_line_numbers,
         })
 
 

changedetectionio/processors/text_json_diff/processor.py

@@ -10,7 +10,7 @@ from changedetectionio.conditions import execute_ruleset_against_all_plugins
 from changedetectionio.processors import difference_detection_processor
 from changedetectionio.html_tools import PERL_STYLE_REGEX, cdata_in_document_to_text, TRANSLATE_WHITESPACE_TABLE
 from changedetectionio import html_tools, content_fetchers
-from changedetectionio.blueprint.price_data_follower import PRICE_DATA_TRACK_ACCEPT
+from changedetectionio.blueprint.price_data_follower import PRICE_DATA_TRACK_ACCEPT, PRICE_DATA_TRACK_REJECT
 from loguru import logger
 
 from changedetectionio.processors.magic import guess_stream_type
@@ -228,21 +228,8 @@ class ContentProcessor:
         self.datastore = datastore
 
     def preprocess_rss(self, content):
-        """
-        Convert CDATA/comments in RSS to usable text.
-
-        Supports two RSS processing modes:
-        - 'default': Inline CDATA replacement (original behavior)
-        - 'formatted': Format RSS items with title, link, guid, pubDate, and description (CDATA unmarked)
-        """
-        from changedetectionio import rss_tools
-        rss_mode = self.datastore.data["settings"]["application"].get("rss_reader_mode")
-        if rss_mode:
-            # Format RSS items nicely with CDATA content unmarked and converted to text
-            return rss_tools.format_rss_items(content)
-        else:
-            # Default: Original inline CDATA replacement
-            return cdata_in_document_to_text(html_content=content)
+        """Convert CDATA/comments in RSS to usable text."""
+        return cdata_in_document_to_text(html_content=content)
 
     def preprocess_pdf(self, raw_content):
         """Convert PDF to HTML using external tool."""
@@ -324,13 +311,13 @@ class ContentProcessor:
                     append_pretty_line_formatting=not self.watch.is_source_type_url
                 )
 
-        # Raise error if filter returned nothing
-        if not filtered_content.strip():
-            raise FilterNotFoundInResponse(
-                msg=self.filter_config.include_filters,
-                screenshot=self.fetcher.screenshot,
-                xpath_data=self.fetcher.xpath_data
-            )
+            # Raise error if filter returned nothing
+            if not filtered_content.strip():
+                raise FilterNotFoundInResponse(
+                    msg=self.filter_config.include_filters,
+                    screenshot=self.fetcher.screenshot,
+                    xpath_data=self.fetcher.xpath_data
+                )
 
         return filtered_content
 
@@ -397,11 +384,6 @@ class perform_site_check(difference_detection_processor):
         # RSS preprocessing
         if stream_content_type.is_rss:
             content = content_processor.preprocess_rss(content)
-            if self.datastore.data["settings"]["application"].get("rss_reader_mode"):
-                # Now just becomes regular HTML that can have xpath/CSS applied (first of the set etc)
-                stream_content_type.is_rss = False
-                stream_content_type.is_html = True
-                self.fetcher.content = content
 
         # PDF preprocessing
         if watch.is_pdf or stream_content_type.is_pdf:
@@ -467,7 +449,6 @@ class perform_site_check(difference_detection_processor):
                 c = ChecksumCalculator.calculate(text_content_before_ignored_filter, ignore_whitespace=True)
                 return False, {'previous_md5': c}, text_content_before_ignored_filter.encode('utf-8')
 
-
         # === EMPTY PAGE CHECK ===
         empty_pages_are_a_change = self.datastore.data['settings']['application'].get('empty_pages_are_a_change', False)
         if not stream_content_type.is_json and not empty_pages_are_a_change and len(stripped_text.strip()) == 0:
@@ -557,20 +538,6 @@ class perform_site_check(difference_detection_processor):
             else:
                 logger.debug(f"check_unique_lines: UUID {watch.get('uuid')} had unique content")
 
-        # Note: Explicit cleanup is only needed here because text_json_diff handles
-        # large strings (100KB-300KB for RSS/HTML). The other processors work with
-        # small strings and don't need this.
-        #
-        # Python would clean these up automatically, but explicit `del` frees memory
-        # immediately rather than waiting for function return, reducing peak memory usage.
-        del content
-        if 'html_content' in locals() and html_content is not stripped_text:
-            del html_content
-        if 'text_content_before_ignored_filter' in locals() and text_content_before_ignored_filter is not stripped_text:
-            del text_content_before_ignored_filter
-        if 'text_for_checksuming' in locals() and text_for_checksuming is not stripped_text:
-            del text_for_checksuming
-
         return changed_detected, update_obj, stripped_text
 
     def _apply_diff_filtering(self, watch, stripped_text, text_before_filter):
@@ -584,6 +551,7 @@ class perform_site_check(difference_detection_processor):
             include_added=watch.get('filter_text_added', True),
             include_removed=watch.get('filter_text_removed', True),
             include_replaced=watch.get('filter_text_replaced', True),
+            line_feed_sep="\n",
             include_change_type_prefix=False
         )
 

changedetectionio/pytest.ini

@@ -1,5 +1,5 @@
 [pytest]
-addopts = --no-start-live-server --live-server-port=0
+addopts = --no-start-live-server --live-server-port=5005
 #testpaths = tests pytest_invenio
 #live_server_scope = function
 

changedetectionio/rss_tools.py

@@ -1,130 +0,0 @@
-"""
-RSS/Atom feed processing tools for changedetection.io
-"""
-
-from loguru import logger
-import re
-
-
-def cdata_in_document_to_text(html_content: str, render_anchor_tag_content=False) -> str:
-    """
-    Process CDATA sections in HTML/XML content - inline replacement.
-
-    Args:
-        html_content: The HTML/XML content to process
-        render_anchor_tag_content: Whether to render anchor tag content
-
-    Returns:
-        Processed HTML/XML content with CDATA sections replaced inline
-    """
-    from xml.sax.saxutils import escape as xml_escape
-    from .html_tools import html_to_text
-
-    pattern = '<!\[CDATA\[(\s*(?:.(?<!\]\]>)\s*)*)\]\]>'
-
-    def repl(m):
-        text = m.group(1)
-        return xml_escape(html_to_text(html_content=text, render_anchor_tag_content=render_anchor_tag_content)).strip()
-
-    return re.sub(pattern, repl, html_content)
-
-
-def format_rss_items(rss_content: str, render_anchor_tag_content=False) -> str:
-    """
-    Format RSS/Atom feed items in a readable text format using feedparser.
-
-    Converts RSS <item> or Atom <entry> elements to formatted text with:
-    - <title> → <h1>Title</h1>
-    - <link> → Link: [url]
-    - <guid> → Guid: [id]
-    - <pubDate> → PubDate: [date]
-    - <description> or <content> → Raw HTML content (CDATA and entities automatically handled)
-
-    Args:
-        rss_content: The RSS/Atom feed content
-        render_anchor_tag_content: Whether to render anchor tag content in descriptions (unused, kept for compatibility)
-
-    Returns:
-        Formatted HTML content ready for html_to_text conversion
-    """
-    try:
-        import feedparser
-        from xml.sax.saxutils import escape as xml_escape
-
-        # Parse the feed - feedparser handles all RSS/Atom variants, CDATA, entity unescaping, etc.
-        feed = feedparser.parse(rss_content)
-
-        formatted_items = []
-
-        # Determine feed type for appropriate labels when fields are missing
-        # feedparser sets feed.version to things like 'rss20', 'atom10', etc.
-        is_atom = feed.version and 'atom' in feed.version
-
-        for entry in feed.entries:
-            item_parts = []
-
-            # Title - feedparser handles CDATA and entity unescaping automatically
-            if hasattr(entry, 'title') and entry.title:
-                item_parts.append(f'<h1>{xml_escape(entry.title)}</h1>')
-
-            # Link
-            if hasattr(entry, 'link') and entry.link:
-                item_parts.append(f'Link: {xml_escape(entry.link)}<br>')
-
-            # GUID/ID
-            if hasattr(entry, 'id') and entry.id:
-                item_parts.append(f'Guid: {xml_escape(entry.id)}<br>')
-
-            # Date - feedparser normalizes all date field names to 'published'
-            if hasattr(entry, 'published') and entry.published:
-                item_parts.append(f'PubDate: {xml_escape(entry.published)}<br>')
-
-            # Description/Content - feedparser handles CDATA and entity unescaping automatically
-            # Only add "Summary:" label for Atom <summary> tags
-            content = None
-            add_label = False
-
-            if hasattr(entry, 'content') and entry.content:
-                # Atom <content> - no label, just content
-                content = entry.content[0].value if entry.content[0].value else None
-            elif hasattr(entry, 'summary'):
-                # Could be RSS <description> or Atom <summary>
-                # feedparser maps both to entry.summary
-                content = entry.summary if entry.summary else None
-                # Only add "Summary:" label for Atom feeds (which use <summary> tag)
-                if is_atom:
-                    add_label = True
-
-            # Add content with or without label
-            if content:
-                if add_label:
-                    item_parts.append(f'Summary:<br>{content}')
-                else:
-                    item_parts.append(content)
-            else:
-                # No content - just show <none>
-                item_parts.append('&lt;none&gt;')
-
-            # Join all parts of this item
-            if item_parts:
-                formatted_items.append('\n'.join(item_parts))
-
-        # Wrap each item in a div with classes (first, last, item-N)
-        items_html = []
-        total_items = len(formatted_items)
-        for idx, item in enumerate(formatted_items):
-            classes = ['rss-item']
-            if idx == 0:
-                classes.append('first')
-            if idx == total_items - 1:
-                classes.append('last')
-            classes.append(f'item-{idx + 1}')
-
-            class_str = ' '.join(classes)
-            items_html.append(f'<div class="{class_str}">{item}</div>')
-        return '<html><body>\n'+"\n<br><br>".join(items_html)+'\n</body></html>'
-
-    except Exception as e:
-        logger.warning(f"Error formatting RSS items: {str(e)}")
-        # Fall back to original content
-        return rss_content

changedetectionio/run_basic_tests.sh

@@ -11,29 +11,32 @@ set -e
 
 SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )
 
-# REMOVE_REQUESTS_OLD_SCREENSHOTS disabled so that we can write a screenshot and send it in test_notifications.py without a real browser
-REMOVE_REQUESTS_OLD_SCREENSHOTS=false pytest -n 30 --dist load  tests/test_*.py
+find tests/test_*py -type f|while read test_name
+do
+  echo "TEST RUNNING $test_name"
+  # REMOVE_REQUESTS_OLD_SCREENSHOTS disabled so that we can write a screenshot and send it in test_notifications.py without a real browser
+  REMOVE_REQUESTS_OLD_SCREENSHOTS=false pytest $test_name
+done
 
-#time pytest -n auto --dist loadfile -vv --tb=long tests/test_*.py
 echo "RUNNING WITH BASE_URL SET"
 
 # Now re-run some tests with BASE_URL enabled
 # Re #65 - Ability to include a link back to the installation, in the notification.
 export BASE_URL="https://really-unique-domain.io"
-REMOVE_REQUESTS_OLD_SCREENSHOTS=false pytest -vv --maxfail=1 tests/test_notification.py
+REMOVE_REQUESTS_OLD_SCREENSHOTS=false pytest tests/test_notification.py
 
 
 # Re-run with HIDE_REFERER set - could affect login
 export HIDE_REFERER=True
-pytest -vv -s --maxfail=1 tests/test_access_control.py
+pytest tests/test_access_control.py
 
 # Re-run a few tests that will trigger brotli based storage
 export SNAPSHOT_BROTLI_COMPRESSION_THRESHOLD=5
-pytest -vv -s --maxfail=1 tests/test_access_control.py
+pytest tests/test_access_control.py
 REMOVE_REQUESTS_OLD_SCREENSHOTS=false pytest tests/test_notification.py
-pytest -vv -s --maxfail=1 tests/test_backend.py
-pytest -vv -s --maxfail=1 tests/test_rss.py
-pytest -vv -s --maxfail=1 tests/test_unique_lines.py
+pytest tests/test_backend.py
+pytest tests/test_rss.py
+pytest tests/test_unique_lines.py
 
 # Try high concurrency
 FETCH_WORKERS=130 pytest  tests/test_history_consistency.py -v -l

changedetectionio/run_custom_browser_url_tests.sh

@@ -6,8 +6,6 @@
 
 # enable debug
 set -x
-docker network inspect changedet-network >/dev/null 2>&1 || docker network create changedet-network
-docker run --network changedet-network -d --hostname selenium  -p 4444:4444 --rm --shm-size="2g"  selenium/standalone-chrome:4
 
 # A extra browser is configured, but we never chose to use it, so it should NOT show in the logs
 docker run --rm -e "PLAYWRIGHT_DRIVER_URL=ws://sockpuppetbrowser:3000" --network changedet-network test-changedetectionio  bash -c 'cd changedetectionio;pytest tests/custom_browser_url/test_custom_browser_url.py::test_request_not_via_custom_browser_url'

changedetectionio/run_proxy_tests.sh

@@ -19,13 +19,12 @@ docker run --network changedet-network -d \
   -v `pwd`/tests/proxy_list/squid-passwords.txt:/etc/squid3/passwords \
   ubuntu/squid:4.13-21.10_edge
 
-sleep 5
+
 ## 2nd test actually choose the preferred proxy from proxies.json
-# This will force a request via "proxy-two"
 docker run --network changedet-network \
-  -v `pwd`/tests/proxy_list/proxies.json-example:/tmp/proxies.json \
+  -v `pwd`/tests/proxy_list/proxies.json-example:/app/changedetectionio/test-datastore/proxies.json \
   test-changedetectionio \
-  bash -c 'cd changedetectionio && pytest -s tests/proxy_list/test_multiple_proxy.py --datastore-path /tmp'
+  bash -c 'cd changedetectionio && pytest tests/proxy_list/test_multiple_proxy.py'
 
 set +e
 echo "- Looking for chosen.changedetection.io request in squid-one - it should NOT be here"
@@ -49,10 +48,8 @@ fi
 # Test the UI configurable proxies
 docker run --network changedet-network \
   test-changedetectionio \
-  bash -c 'cd changedetectionio && pytest tests/proxy_list/test_select_custom_proxy.py --datastore-path /tmp'
+  bash -c 'cd changedetectionio && pytest tests/proxy_list/test_select_custom_proxy.py'
 
-# Give squid proxies a moment to flush their logs
-sleep 2
 
 # Should see a request for one.changedetection.io in there
 echo "- Looking for .changedetection.io request in squid-custom"
@@ -66,10 +63,7 @@ fi
 # Test "no-proxy" option
 docker run --network changedet-network \
   test-changedetectionio \
-  bash -c 'cd changedetectionio && pytest tests/proxy_list/test_noproxy.py --datastore-path /tmp'
-
-# Give squid proxies a moment to flush their logs
-sleep 2
+  bash -c 'cd changedetectionio && pytest tests/proxy_list/test_noproxy.py'
 
 # We need to handle grep returning 1
 set +e
@@ -86,8 +80,6 @@ for c in $(echo "squid-one squid-two squid-custom"); do
   fi
 done
 
-echo "docker ps output"
-docker ps
 
 docker kill squid-one squid-two squid-custom
 
@@ -96,19 +88,19 @@ docker kill squid-one squid-two squid-custom
 # Requests
 docker run --network changedet-network \
   test-changedetectionio \
-  bash -c 'cd changedetectionio && pytest tests/proxy_list/test_proxy_noconnect.py --datastore-path /tmp'
+  bash -c 'cd changedetectionio && pytest tests/proxy_list/test_proxy_noconnect.py'
 
 # Playwright
 docker run --network changedet-network \
   test-changedetectionio \
-  bash -c 'cd changedetectionio && PLAYWRIGHT_DRIVER_URL=ws://sockpuppetbrowser:3000 pytest tests/proxy_list/test_proxy_noconnect.py --datastore-path /tmp'
+  bash -c 'cd changedetectionio && PLAYWRIGHT_DRIVER_URL=ws://sockpuppetbrowser:3000 pytest tests/proxy_list/test_proxy_noconnect.py'
 
 # Puppeteer fast
 docker run --network changedet-network \
   test-changedetectionio \
-  bash -c 'cd changedetectionio && FAST_PUPPETEER_CHROME_FETCHER=1 PLAYWRIGHT_DRIVER_URL=ws://sockpuppetbrowser:3000 pytest tests/proxy_list/test_proxy_noconnect.py --datastore-path /tmp'
+  bash -c 'cd changedetectionio && FAST_PUPPETEER_CHROME_FETCHER=1 PLAYWRIGHT_DRIVER_URL=ws://sockpuppetbrowser:3000 pytest tests/proxy_list/test_proxy_noconnect.py'
 
 # Selenium
 docker run --network changedet-network \
   test-changedetectionio \
-  bash -c 'cd changedetectionio && WEBDRIVER_URL=http://selenium:4444/wd/hub pytest tests/proxy_list/test_proxy_noconnect.py --datastore-path /tmp'
+  bash -c 'cd changedetectionio && WEBDRIVER_URL=http://selenium:4444/wd/hub pytest tests/proxy_list/test_proxy_noconnect.py'

changedetectionio/run_socks_proxy_tests.sh

@@ -5,7 +5,6 @@ set -e
 # enable debug
 set -x
 
-docker network inspect changedet-network >/dev/null 2>&1 || docker network create changedet-network
 
 # SOCKS5 related - start simple Socks5 proxy server
 # SOCKSTEST=xyz should show in the logs of this service to confirm it fetched
@@ -15,13 +14,13 @@ docker run --network changedet-network -d --hostname socks5proxy-noauth --rm -p
 echo "---------------------------------- SOCKS5 -------------------"
 # SOCKS5 related - test from proxies.json
 docker run --network changedet-network \
-  -v `pwd`/tests/proxy_socks5/proxies.json-example:/tmp/proxies.json \
+  -v `pwd`/tests/proxy_socks5/proxies.json-example:/app/changedetectionio/test-datastore/proxies.json \
   --rm \
   -e "FLASK_SERVER_NAME=cdio" \
   --hostname cdio \
   -e "SOCKSTEST=proxiesjson" \
   test-changedetectionio \
-  bash -c 'cd changedetectionio && pytest --live-server-host=0.0.0.0 --live-server-port=5004  -s tests/proxy_socks5/test_socks5_proxy_sources.py  --datastore-path /tmp'
+  bash -c 'cd changedetectionio && pytest --live-server-host=0.0.0.0 --live-server-port=5004  -s tests/proxy_socks5/test_socks5_proxy_sources.py'
 
 # SOCKS5 related - by manually entering in UI
 docker run --network changedet-network \
@@ -30,18 +29,18 @@ docker run --network changedet-network \
   --hostname cdio \
   -e "SOCKSTEST=manual" \
   test-changedetectionio \
-  bash -c 'cd changedetectionio && pytest --live-server-host=0.0.0.0 --live-server-port=5004  -s tests/proxy_socks5/test_socks5_proxy.py --datastore-path /tmp'
+  bash -c 'cd changedetectionio && pytest --live-server-host=0.0.0.0 --live-server-port=5004  -s tests/proxy_socks5/test_socks5_proxy.py'
 
 # SOCKS5 related - test from proxies.json via playwright - NOTE- PLAYWRIGHT DOESNT SUPPORT AUTHENTICATING PROXY
 docker run --network changedet-network \
   -e "SOCKSTEST=manual-playwright" \
   --hostname cdio \
   -e "FLASK_SERVER_NAME=cdio" \
-  -v `pwd`/tests/proxy_socks5/proxies.json-example-noauth:/tmp/proxies.json \
+  -v `pwd`/tests/proxy_socks5/proxies.json-example-noauth:/app/changedetectionio/test-datastore/proxies.json \
   -e "PLAYWRIGHT_DRIVER_URL=ws://sockpuppetbrowser:3000" \
   --rm \
   test-changedetectionio \
-  bash -c 'cd changedetectionio && pytest --live-server-host=0.0.0.0 --live-server-port=5004  -s tests/proxy_socks5/test_socks5_proxy_sources.py --datastore-path /tmp'
+  bash -c 'cd changedetectionio && pytest --live-server-host=0.0.0.0 --live-server-port=5004  -s tests/proxy_socks5/test_socks5_proxy_sources.py'
 
 echo "socks5 server logs"
 docker logs socks5proxy

changedetectionio/safe_jinja.py

@@ -0,0 +1,24 @@
+"""
+Safe Jinja2 render with max payload sizes
+
+See https://jinja.palletsprojects.com/en/3.1.x/sandbox/#security-considerations
+"""
+
+import jinja2.sandbox
+import typing as t
+import os
+
+JINJA2_MAX_RETURN_PAYLOAD_SIZE = 1024 * int(os.getenv("JINJA2_MAX_RETURN_PAYLOAD_SIZE_KB", 1024 * 10))
+
+# This is used for notifications etc, so actually it's OK to send custom HTML such as <a href> etc, but it should limit what data is available.
+# (Which also limits available functions that could be called)
+def render(template_str, **args: t.Any) -> str:
+    jinja2_env = jinja2.sandbox.ImmutableSandboxedEnvironment(extensions=['jinja2_time.TimeExtension'])
+    output = jinja2_env.from_string(template_str).render(args)
+    return output[:JINJA2_MAX_RETURN_PAYLOAD_SIZE]
+
+def render_fully_escaped(content):
+    env = jinja2.sandbox.ImmutableSandboxedEnvironment(autoescape=True)
+    template = env.from_string("{{ some_html|e }}")
+    return template.render(some_html=content)
+

changedetectionio/static/js/diff-overview.js

@@ -27,45 +27,16 @@ $(document).ready(function () {
         }
     }
 
-    const article = $('#difference')[0];
+    const article = $('.highlightable-filter')[0];
 
     // We could also add the  'touchend' event for touch devices, but since
     // most iOS/Android browsers already show a dialog when you select
     // text (often with a Share option) we'll skip that
-    if (article) {
-        article.addEventListener('mouseup', dragTextHandler, false);
-        article.addEventListener('mousedown', clean, false);
-    }
-
-
-    $('#highlightSnippetActions button').bind('click', function (e) {
-        if (!window.getSelection().toString().trim().length) {
-            alert('Oops no text selected!');
-            return;
-        }
-
-        $.ajax({
-            type: "POST",
-            url: highlight_submit_ignore_url,
-            data: {'mode': $(this).data('mode'), 'selection': window.getSelection().toString()},
-            statusCode: {
-                400: function () {
-                    // More than likely the CSRF token was lost when the server restarted
-                    alert("There was a problem processing the request, please reload the page.");
-                }
-            }
-        }).done(function (data) {
-            // @todo some feedback
-            $("#highlightSnippet").html(data);
-            clean();
-        }).fail(function (data) {
-            console.log(data);
-            alert('There was an error communicating with the server.');
-        })
-    });
+    article.addEventListener('mouseup', dragTextHandler, false);
+    article.addEventListener('mousedown', clean, false);
 
     function clean(event) {
-        $('#bottom-horizontal-offscreen').hide();
+      $("#highlightSnippet").remove();
     }
 
     // Listen for Escape key press
@@ -80,9 +51,46 @@ $(document).ready(function () {
 
         // Check if any text was selected
         if (window.getSelection().toString().length > 0) {
-            $('#bottom-horizontal-offscreen').show();
-        } else {
-            clean();
+
+            // Find out how much (if any) user has scrolled
+            var scrollTop = (window.pageYOffset !== undefined) ? window.pageYOffset : (document.documentElement || document.body.parentNode || document.body).scrollTop;
+
+            // Get cursor position
+            const posX = event.clientX;
+            const posY = event.clientY + 20 + scrollTop;
+
+            // Append HTML to the body, create the "Tweet Selection" dialog
+            document.body.insertAdjacentHTML('beforeend', '<div id="highlightSnippet" style="position: absolute; top: ' + posY + 'px; left: ' + posX + 'px;"><div class="pure-form-message-inline" style="font-size: 70%">Ignore any change on any line which contains the selected text.</div><br><a data-mode="exact" href="javascript:void(0);" class="pure-button button-secondary button-xsmall">Ignore exact text</a>&nbsp;</div>');
+
+            if (/\d/.test(window.getSelection().toString())) {
+                // Offer regex replacement
+                document.getElementById("highlightSnippet").insertAdjacentHTML('beforeend', '<a data-mode="digit-regex"  href="javascript:void(0);" class="pure-button button-secondary button-xsmall">Ignore text including number changes</a>');
+            }
+
+            $('#highlightSnippet a').bind('click', function (e) {
+                if(!window.getSelection().toString().trim().length) {
+                    alert('Oops no text selected!');
+                    return;
+                }
+
+                $.ajax({
+                    type: "POST",
+                    url: highlight_submit_ignore_url,
+                    data: {'mode': $(this).data('mode'), 'selection': window.getSelection().toString()},
+                    statusCode: {
+                        400: function () {
+                            // More than likely the CSRF token was lost when the server restarted
+                            alert("There was a problem processing the request, please reload the page.");
+                        }
+                    }
+                }).done(function (data) {
+                    $("#highlightSnippet").html(data)
+                }).fail(function (data) {
+                    console.log(data);
+                    alert('There was an error communicating with the server.');
+                });
+            });
+
         }
     }
 

changedetectionio/static/js/diff-render.js

@@ -1,152 +1,115 @@
 $(document).ready(function () {
-
-    // Find all <span> elements inside pre#difference
-    var inputs = $('#difference span').toArray();
-    inputs.current = 0;
-
-    // Setup visual minimap of difference locations (cells are pre-built in Python)
-    var $visualizer = $('#cell-diff-jump-visualiser');
-    var $difference = $('#difference');
-    var $cells = $visualizer.find('> div');
-    var visualizerResolutionCells = $cells.length;
-    var cellHeight;
-
-    if ($difference.length && visualizerResolutionCells > 0) {
-        var docHeight = $difference[0].scrollHeight;
-        cellHeight = docHeight / visualizerResolutionCells;
-
-        // Add click handlers to pre-built cells
-        $cells.each(function(i) {
-            $(this).data('cellIndex', i);
-            $(this).on('click', function() {
-                var cellIndex = $(this).data('cellIndex');
-                var targetPositionInDifference = cellIndex * cellHeight;
-                var viewportHeight = $(window).height();
-
-                // Scroll so target is at viewport center (where eyes expect it)
-                window.scrollTo({
-                    top: $difference.offset().top + targetPositionInDifference - (viewportHeight / 2),
-                    behavior: "smooth"
-                });
-            });
-        });
-    }
+    var a = document.getElementById("a");
+    var b = document.getElementById("b");
+    var result = document.getElementById("result");
+    var inputs;
 
     $('#jump-next-diff').click(function () {
-        if (!inputs || inputs.length === 0) return;
 
-        // Find the next change after current scroll position
-        var currentScrollPos = $(window).scrollTop();
-        var viewportHeight = $(window).height();
-        var currentCenter = currentScrollPos + (viewportHeight / 2);
-
-        // Add small buffer (50px) to jump past changes already near center
-        var searchFromPosition = currentCenter + 50;
-
-        var nextElement = null;
-        for (var i = 0; i < inputs.length; i++) {
-            var elementTop = $(inputs[i]).offset().top;
-            if (elementTop > searchFromPosition) {
-                nextElement = inputs[i];
-                break;
-            }
-        }
-
-        // If no element found ahead, wrap to first element
-        if (!nextElement) {
-            nextElement = inputs[0];
-        }
-
-        // Scroll to position the element at viewport center
-        var elementTop = $(nextElement).offset().top;
-        var targetScrollPos = elementTop - (viewportHeight / 2);
+        var element = inputs[inputs.current];
+        var headerOffset = 80;
+        var elementPosition = element.getBoundingClientRect().top;
+        var offsetPosition = elementPosition - headerOffset + window.scrollY;
 
         window.scrollTo({
-            top: targetScrollPos,
+            top: offsetPosition,
             behavior: "smooth",
         });
+
+        inputs.current++;
+        if (inputs.current >= inputs.length) {
+            inputs.current = 0;
+        }
     });
 
-    // Track current scroll position in visualizer
-    function updateVisualizerPosition() {
-        if (!$difference.length || visualizerResolutionCells === 0) return;
+    function changed() {
+        // https://github.com/kpdecker/jsdiff/issues/389
+        // I would love to use `{ignoreWhitespace: true}` here but it breaks the formatting
+        options = {
+            ignoreWhitespace: document.getElementById("ignoreWhitespace").checked,
+        };
 
-        var scrollTop = $(window).scrollTop();
-        var viewportHeight = $(window).height();
-        var viewportCenter = scrollTop + (viewportHeight / 2);
-        var differenceTop = $difference.offset().top;
-        var differenceHeight = $difference[0].scrollHeight;
-        var positionInDifference = viewportCenter - differenceTop;
-
-        // Handle edge case: if we're at max scroll, show last cell
-        // This prevents shorter documents from never reaching 100%
-        var maxScrollTop = $(document).height() - viewportHeight;
-        var isAtBottom = scrollTop >= maxScrollTop - 10; // 10px tolerance
-
-        // Calculate which cell we're currently viewing
-        var currentCell;
-        if (isAtBottom) {
-            currentCell = visualizerResolutionCells - 1;
-        } else {
-            currentCell = Math.floor(positionInDifference / cellHeight);
-            currentCell = Math.max(0, Math.min(currentCell, visualizerResolutionCells - 1));
-        }
-
-        // Remove previous active marker and add to current cell
-        $visualizer.find('> div').removeClass('current-position');
-        $visualizer.find('> div').eq(currentCell).addClass('current-position');
-    }
-
-    // Recalculate cellHeight on window resize
-    function handleResize() {
-        if ($difference.length) {
-            var docHeight = $difference[0].scrollHeight;
-            cellHeight = docHeight / visualizerResolutionCells;
-            updateVisualizerPosition();
-        }
-    }
-
-    // Debounce scroll and resize events to reduce CPU usage
-    $(window).on('scroll', updateVisualizerPosition.debounce(5));
-    $(window).on('resize', handleResize.debounce(100));
-
-    // Initial scroll to specific line if requested
-    if (typeof initialScrollToLineNumber !== 'undefined' && initialScrollToLineNumber !== null && $difference.length) {
-        // Convert line number to text position and scroll to it
-        var diffText = $difference.text();
-        var lines = diffText.split('\n');
-
-        if (initialScrollToLineNumber > 0 && initialScrollToLineNumber <= lines.length) {
-            // Calculate character position of the target line
-            var charPosition = 0;
-            for (var i = 0; i < initialScrollToLineNumber - 1; i++) {
-                charPosition += lines[i].length + 1; // +1 for newline
+        var diff = Diff[window.diffType](a.textContent, b.textContent, options);
+        var fragment = document.createDocumentFragment();
+        for (var i = 0; i < diff.length; i++) {
+            if (diff[i].added && diff[i + 1] && diff[i + 1].removed) {
+                var swap = diff[i];
+                diff[i] = diff[i + 1];
+                diff[i + 1] = swap;
             }
 
-            // Estimate vertical position based on average line height
-            var totalChars = diffText.length;
-            var totalHeight = $difference[0].scrollHeight;
-            var estimatedTop = (charPosition / totalChars) * totalHeight;
-
-            // Scroll to position with line at viewport center
-            var viewportHeight = $(window).height();
-            setTimeout(function() {
-                window.scrollTo({
-                    top: $difference.offset().top + estimatedTop - (viewportHeight / 2),
-                    behavior: "smooth"
-                });
-            }, 100); // Small delay to ensure page is fully loaded
+            var node;
+            if (diff[i].removed) {
+                node = document.createElement("del");
+                node.classList.add("change");
+                const wrapper = node.appendChild(document.createElement("span"));
+                wrapper.appendChild(document.createTextNode(diff[i].value));
+            } else if (diff[i].added) {
+                node = document.createElement("ins");
+                node.classList.add("change");
+                const wrapper = node.appendChild(document.createElement("span"));
+                wrapper.appendChild(document.createTextNode(diff[i].value));
+            } else {
+                node = document.createTextNode(diff[i].value);
+            }
+            fragment.appendChild(node);
         }
+
+        result.textContent = "";
+        result.appendChild(fragment);
+
+        // For nice mouse-over hover/title information
+        const removed_current_option = $('#diff-version option:selected')
+        if (removed_current_option) {
+            $('del').each(function () {
+                $(this).prop('title', 'Removed '+removed_current_option[0].label);
+            });
+        }
+        const inserted_current_option = $('#current-version option:selected')
+        if (removed_current_option) {
+            $('ins').each(function () {
+                $(this).prop('title', 'Inserted '+inserted_current_option[0].label);
+            });
+        }
+        // Set the list of possible differences to jump to
+        inputs = document.querySelectorAll('#diff-ui .change')
+        // Set the "current" diff pointer
+        inputs.current = 0;
+        // Goto diff
+        $('#jump-next-diff').click();
     }
 
-    // Initial position update
-    if ($difference.length && cellHeight) {
-        updateVisualizerPosition();
+
+    onDiffTypeChange(
+        document.querySelector('#settings [name="diff_type"]:checked'),
+    );
+    changed();
+
+    a.onpaste = a.onchange = b.onpaste = b.onchange = changed;
+
+    if ("oninput" in a) {
+        a.oninput = b.oninput = changed;
+    } else {
+        a.onkeyup = b.onkeyup = changed;
     }
 
-    function changed() {
-        //$('#jump-next-diff').click();
+    function onDiffTypeChange(radio) {
+        window.diffType = radio.value;
+        // Not necessary
+        //	document.title = "Diff " + radio.value.slice(4);
     }
 
+    var radio = document.getElementsByName("diff_type");
+    for (var i = 0; i < radio.length; i++) {
+        radio[i].onchange = function (e) {
+            onDiffTypeChange(e.target);
+            changed();
+        };
+    }
+
+    document.getElementById("ignoreWhitespace").onchange = function (e) {
+        changed();
+    };
+
 });
 

changedetectionio/static/js/global-settings.js

@@ -29,7 +29,7 @@ $(document).ready(function () {
         $(this).text(new Date($(this).data("utc")).toLocaleString());
     })
 
-    const timezoneInput = $('#application-scheduler_timezone_default');
+    const timezoneInput = $('#application-timezone');
     if(timezoneInput.length) {
         const timezone = Intl.DateTimeFormat().resolvedOptions().timeZone;
         if (!timezoneInput.val().trim()) {

changedetectionio/static/js/notifications.js

@@ -14,10 +14,10 @@ $(document).ready(function () {
         e.preventDefault();
 
         data = {
-            notification_urls: $('textarea.notification-urls').val(),
-            notification_title: $('input.notification-title').val(),
-            notification_body: $('textarea.notification-body').val(),
-            notification_format: $('select.notification-format').val(),
+            notification_body: $('#notification_body').val(),
+            notification_format: $('#notification_format').val(),
+            notification_title: $('#notification_title').val(),
+            notification_urls: $('.notification-urls').val(),
             tags: $('#tags').val(),
             window_url: window.location.href,
         }

changedetectionio/static/js/plugins.js

@@ -62,12 +62,15 @@
             const textContent = $pre.text();
             const lines = textContent.split(/\r?\n/); // Handles both \n and \r\n line endings
 
-            // Build a map of line numbers to their configuration index
-            const lineConfigIndex = {};
+            // Build a map of line numbers to styles
+            const lineStyles = {};
 
-            configurations.forEach((config, index) =>
-                config.lines.forEach(lineNumber => lineConfigIndex[lineNumber] = index)
-            );
+            configurations.forEach(config => {
+                const {color, lines: lineNumbers} = config;
+                lineNumbers.forEach(lineNumber => {
+                    lineStyles[lineNumber] = color;
+                });
+            });
 
             // Function to escape HTML characters
             function escapeHtml(text) {
@@ -80,12 +83,11 @@
             const processedLines = lines.map((line, index) => {
                 const lineNumber = index + 1; // Line numbers start at 1
                 const escapedLine = escapeHtml(line);
-                const configIndex = lineConfigIndex[lineNumber];
+                const color = lineStyles[lineNumber];
 
-                if (configIndex !== undefined) {
-                    const config = configurations[configIndex];
+                if (color) {
                     // Wrap the line in a span with inline style
-                    return `<span title="${config.title}" style="background-color: ${config.color}">${escapedLine}</span>`;
+                    return `<span style="background-color: ${color}">${escapedLine}</span>`;
                 } else {
                     return escapedLine;
                 }
@@ -98,7 +100,6 @@
             $pre.html(newContent);
         });
     };
-
     $.fn.miniTabs = function (tabsConfig, options) {
         const settings = {
             tabClass: 'minitab',

changedetectionio/static/js/preview.js

@@ -53,21 +53,11 @@ $(document).ready(function () {
     if ($('#preview-version').length) {
         setupDateWidget();
     }
+
     $('#diff-col > pre').highlightLines([
         {
-            'color': 'var(--highlight-trigger-text-bg-color)',
-            'lines': triggered_line_numbers,
-            'title': "Triggers a change if this text appears, AND something changed in the document."
-        },
-        {
-            'color': 'var(--highlight-ignored-text-bg-color)',
-            'lines': ignored_line_numbers,
-            'title': "Ignored for calculating changes, but still shown."
-        },
-        {
-            'color': 'var(--highlight-blocked-text-bg-color)',
-            'lines': blocked_line_numbers,
-            'title': "No change-detection will occur because this text exists."
+            'color': '#ee0000',
+            'lines': triggered_line_numbers
         }
     ]);
 });

changedetectionio/static/js/realtime.js

@@ -2,13 +2,6 @@
 
 $(document).ready(function () {
 
-    function reapplyTableStripes() {
-        $('.watch-table tbody tr').each(function(index) {
-            $(this).removeClass('pure-table-odd pure-table-even');
-            $(this).addClass(index % 2 === 0 ? 'pure-table-odd' : 'pure-table-even');
-        });
-    }
-
     function bindSocketHandlerButtonsEvents(socket) {
         $('.ajax-op').on('click.socketHandlerNamespace', function (e) {
             e.preventDefault();
@@ -108,7 +101,6 @@ $(document).ready(function () {
             socket.on('watch_deleted', function (data) {
                 $('tr[data-watch-uuid="' + data.uuid + '"] td').fadeOut(500, function () {
                     $(this).closest('tr').remove();
-                    reapplyTableStripes();
                 });
             });
 

changedetectionio/static/js/snippet-to-image.js

@@ -1,430 +0,0 @@
-/**
- * snippet-to-image.js
- * Converts selected diff content to a shareable JPEG image with metadata
- */
-
-// Constants
-const IMAGE_PADDING = 5;
-const JPEG_QUALITY = 0.95;
-const CANVAS_SCALE = 1;
-const RENDER_DELAY_MS = 50;
-
-/**
- * Utility: Get the target URL from global watch_url or fallback to current URL
- */
-function getTargetUrl() {
-    return (typeof watch_url !== 'undefined' && watch_url) ? watch_url : window.location.href;
-}
-
-/**
- * Utility: Get formatted current date with timezone
- */
-function getFormattedDate() {
-    return new Date().toLocaleString(undefined, {
-        year: 'numeric',
-        month: 'long',
-        day: 'numeric',
-        hour: '2-digit',
-        minute: '2-digit',
-        second: '2-digit',
-        timeZoneName: 'short'
-    });
-}
-
-/**
- * Utility: Get version comparison info from the diff selectors
- */
-function getVersionInfo() {
-    const fromSelect = document.getElementById('diff-version');
-    const toSelect = document.getElementById('current-version');
-
-    if (!fromSelect || !toSelect) {
-        return '';
-    }
-
-    const fromOption = fromSelect.options[fromSelect.selectedIndex];
-    const toOption = toSelect.options[toSelect.selectedIndex];
-    const fromLabel = fromOption ? (fromOption.getAttribute('label') || fromOption.text) : 'Unknown';
-    const toLabel = toOption ? (toOption.getAttribute('label') || toOption.text) : 'Unknown';
-
-    return `<br>Change comparison from <strong>${fromLabel}</strong> to <strong>${toLabel}</strong><br>Monitored via automated content change detection on public webpages. Data reflects observed text updates, not editorial verification.<br>`;
-}
-
-/**
- * Helper: Find text node containing newline in a given direction
- */
-function findTextNodeWithNewline(node, searchBackwards = false) {
-    if (node.nodeType === Node.TEXT_NODE) {
-        const text = node.textContent;
-        const idx = searchBackwards ? text.lastIndexOf('\n') : text.indexOf('\n');
-        if (idx !== -1) {
-            return { node, offset: searchBackwards ? idx + 1 : idx };
-        }
-    } else {
-        const walker = document.createTreeWalker(node, NodeFilter.SHOW_TEXT);
-        let textNode;
-        while (textNode = walker.nextNode()) {
-            const text = textNode.textContent;
-            const idx = searchBackwards ? text.lastIndexOf('\n') : text.indexOf('\n');
-            if (idx !== -1) {
-                return { node: textNode, offset: searchBackwards ? idx + 1 : idx };
-            }
-        }
-    }
-    return null;
-}
-
-/**
- * Helper: Walk through siblings in a given direction to find line boundary
- */
-function findLineBoundary(node, container, searchBackwards = false) {
-    let currentNode = node;
-
-    while (currentNode && currentNode !== container) {
-        const sibling = searchBackwards ? currentNode.previousSibling : currentNode.nextSibling;
-        let currentSibling = sibling;
-
-        while (currentSibling) {
-            const result = findTextNodeWithNewline(currentSibling, searchBackwards);
-            if (result) {
-                return result;
-            }
-            currentSibling = searchBackwards ? currentSibling.previousSibling : currentSibling.nextSibling;
-        }
-
-        currentNode = currentNode.parentNode;
-    }
-
-    return null;
-}
-
-/**
- * Helper: Get the last text node in a container
- */
-function getLastTextNode(container) {
-    const walker = document.createTreeWalker(container, NodeFilter.SHOW_TEXT);
-    let lastNode = null;
-    let textNode;
-    while (textNode = walker.nextNode()) {
-        lastNode = textNode;
-    }
-    return lastNode;
-}
-
-/**
- * Expands a selection range to include complete lines
- * If a user selects partial text, this ensures full lines are captured
- */
-function expandRangeToFullLines(range, container) {
-    const newRange = range.cloneRange();
-
-    // Expand start to line beginning
-    if (newRange.startContainer.nodeType === Node.TEXT_NODE) {
-        const text = newRange.startContainer.textContent;
-        const lastNewline = text.lastIndexOf('\n', newRange.startOffset - 1);
-        if (lastNewline !== -1) {
-            newRange.setStart(newRange.startContainer, lastNewline + 1);
-        } else {
-            const lineStart = findLineBoundary(newRange.startContainer, container, true);
-            if (lineStart) {
-                newRange.setStart(lineStart.node, lineStart.offset);
-            } else {
-                newRange.setStart(container, 0);
-            }
-        }
-    }
-
-    // Expand end to line end
-    if (newRange.endContainer.nodeType === Node.TEXT_NODE) {
-        const text = newRange.endContainer.textContent;
-        const nextNewline = text.indexOf('\n', newRange.endOffset);
-        if (nextNewline !== -1) {
-            newRange.setEnd(newRange.endContainer, nextNewline);
-        } else {
-            const lineEnd = findLineBoundary(newRange.endContainer, container, false);
-            if (lineEnd) {
-                newRange.setEnd(lineEnd.node, lineEnd.offset);
-            } else {
-                const lastNode = getLastTextNode(container);
-                newRange.setEnd(
-                    lastNode || container,
-                    lastNode ? lastNode.textContent.length : container.childNodes.length
-                );
-            }
-        }
-    }
-
-    return newRange;
-}
-
-/**
- * Create a temporary element with the selected content styled for capture
- */
-function createCaptureElement(selectedFragment, originalElement) {
-    const originalStyles = window.getComputedStyle(originalElement);
-
-    // Create container with watermark background
-    const container = document.createElement("div");
-    container.innerHTML = `
-        <div style="
-            position: absolute;
-            left: -9999px;
-            top: 0;
-            padding: 2px;
-            background-color: transparent;
-        ">
-        <div style="
-            background-color: #ffffff;
-            width: ${originalElement.offsetWidth}px;
-            border: 1px solid #ccc;
-            border-radius: 4px;
-            overflow: hidden;
-        ">
-            <!-- Watermark background -->
-            <div style="
-                position: absolute;
-                top: 0;
-                left: 0;
-                width: 100%;
-                height: 100%;
-                overflow: hidden;
-                pointer-events: none;
-                z-index: 0;
-                background-image: url(&quot;data:image/svg+xml;utf8,<svg xmlns='http://www.w3.org/2000/svg' width='400' height='200' viewBox='0 0 400 200'><g font-family='Arial' font-size='18' font-weight='700' fill='%23e8e8e8' transform='rotate(-45 200 100)'><text x='0' y='40'>changedetection.io   changedetection.io   changedetection.io</text><text x='0' y='100'>changedetection.io   changedetection.io   changedetection.io</text><text x='0' y='160'>changedetection.io   changedetection.io   changedetection.io</text></g></svg>&quot;);
-                background-repeat: repeat;
-                background-size: 400px 200px;
-            "></div>
-
-            <!-- Content -->
-            <pre id="temp-capture-element" style="
-                position: relative;
-                z-index: 1;
-                white-space: ${originalStyles.whiteSpace};
-                font-family: ${originalStyles.fontFamily};
-                font-size: ${originalStyles.fontSize};
-                line-height: ${originalStyles.lineHeight};
-                color: ${originalStyles.color};
-                word-wrap: ${originalStyles.wordWrap};
-                overflow-wrap: ${originalStyles.overflowWrap};
-                background-color: transparent;
-                padding: ${IMAGE_PADDING}px;
-                border: ${originalStyles.border};
-                box-sizing: border-box;
-                margin: 0;
-            "></pre>
-        </div>
-        </div>
-    `;
-
-    const outerWrapper = container.firstElementChild;
-    const innerWrapper = outerWrapper.querySelector('div');
-    const tempElement = innerWrapper.querySelector('#temp-capture-element');
-    tempElement.appendChild(selectedFragment);
-
-    // Store innerWrapper for footer appending
-    outerWrapper._innerWrapper = innerWrapper;
-
-    return outerWrapper;
-}
-
-/**
- * Create footer with metadata (URL, date, version info)
- */
-function createFooter() {
-    const url = getTargetUrl();
-    const date = getFormattedDate();
-    const versionInfo = getVersionInfo();
-
-    const footer = document.createElement("div");
-    footer.innerHTML = `
-        <div style="
-            position: relative;
-            z-index: 1;
-            background-color: #1324fd;
-            color: #fff;
-            padding: 10px;
-            margin-top: 10px;
-            font-size: 12px;
-            font-family: Arial, sans-serif;
-            line-height: 1.5;
-            border-top: 1px solid #ccc;
-        ">
-            Source: <strong>${url}</strong><br>
-            Generated by changedetection.io at ${date}
-            ${versionInfo}
-        </div>
-    `;
-
-    return footer.firstElementChild;
-}
-
-/**
- * Add EXIF metadata to JPEG image
- */
-function addExifMetadata(jpegDataUrl) {
-    if (typeof piexif === 'undefined') {
-        return jpegDataUrl;
-    }
-
-    try {
-        const url = getTargetUrl();
-        const timestamp = new Date().toISOString();
-
-        const exifObj = {
-            "0th": {
-                [piexif.ImageIFD.Software]: "changedetection.io",
-                [piexif.ImageIFD.ImageDescription]: `Diff snapshot from ${url}`,
-                [piexif.ImageIFD.Copyright]: "Generated by changedetection.io"
-            },
-            "Exif": {
-                [piexif.ExifIFD.DateTimeOriginal]: timestamp,
-                [piexif.ExifIFD.UserComment]: `URL: ${url} | Captured: ${timestamp} | Source: changedetection.io`
-            }
-        };
-
-        const exifBytes = piexif.dump(exifObj);
-        return piexif.insert(exifBytes, jpegDataUrl);
-    } catch (error) {
-        console.warn("Failed to add EXIF metadata:", error);
-        return jpegDataUrl;
-    }
-}
-
-/**
- * Display or download the generated image
- */
-function displayImage(jpegDataUrl) {
-    const win = window.open();
-    if (win) {
-        win.document.write(`
-            <html>
-                <head><title>Diff Screenshot</title></head>
-                <body style="margin:0;">
-                    <img src="${jpegDataUrl}" alt="Diff Screenshot" style="max-width:100%;"/>
-                </body>
-            </html>
-        `);
-    } else {
-        // Fallback: trigger download if popup is blocked
-        const a = document.createElement("a");
-        a.href = jpegDataUrl;
-        a.download = "diff-snapshot-" + Date.now() + ".jpg";
-        a.click();
-    }
-}
-
-/**
- * Update button UI state
- */
-function setButtonState(button, isLoading, originalHtml = '') {
-    if (!button) return;
-
-    if (isLoading) {
-        button.innerHTML = 'Generating...';
-        button.style.opacity = "0.5";
-        button.style.pointerEvents = "none";
-    } else {
-        button.innerHTML = originalHtml;
-        button.style.opacity = "1";
-        button.style.pointerEvents = "auto";
-    }
-}
-
-/**
- * Main function: Convert selected diff text to a shareable JPEG image
- *
- * Features:
- * - Expands partial selections to full lines
- * - Preserves all diff highlighting and formatting
- * - Adds metadata footer with URL and version info
- * - Embeds EXIF metadata in the JPEG
- * - Opens in new window or downloads if popup blocked
- */
-async function diffToJpeg() {
-    // Validate dependencies
-    if (typeof html2canvas === 'undefined') {
-        alert("html2canvas library is not loaded yet. Please wait a moment and try again.");
-        return;
-    }
-
-    // Validate selection
-    const selection = window.getSelection();
-    if (!selection || selection.rangeCount === 0 || selection.isCollapsed) {
-        alert("Please select the text/lines you want to capture first by highlighting with your mouse.");
-        return;
-    }
-
-    const originalRange = selection.getRangeAt(0);
-    const differenceElement = document.getElementById("difference");
-
-    if (!differenceElement || !differenceElement.contains(originalRange.commonAncestorContainer)) {
-        alert("Please select text within the diff content.");
-        return;
-    }
-
-    // Setup UI state
-    const btn = document.getElementById("share-as-image-btn");
-    const originalBtnHtml = btn ? btn.innerHTML : '';
-    setButtonState(btn, true);
-
-    let tempElement = null;
-
-    try {
-        // Expand selection to full lines and clone content
-        const expandedRange = expandRangeToFullLines(originalRange, differenceElement);
-        const selectedFragment = expandedRange.cloneContents();
-
-        // Create temporary element with proper styling
-        tempElement = createCaptureElement(selectedFragment, differenceElement);
-        // Append footer to innerWrapper (inside the border), not outerWrapper
-        tempElement._innerWrapper.appendChild(createFooter());
-
-        // Add to DOM for rendering
-        document.body.appendChild(tempElement);
-
-        // Wait for rendering
-        await new Promise(resolve => setTimeout(resolve, RENDER_DELAY_MS));
-
-        // Capture to canvas
-        const canvas = await html2canvas(tempElement, {
-            scale: CANVAS_SCALE,
-            useCORS: true,
-            allowTaint: true,
-            logging: false,
-            backgroundColor: '#ffffff',
-            scrollX: 0,
-            scrollY: 0
-        });
-
-        // Validate canvas
-        if (canvas.width === 0 || canvas.height === 0) {
-            throw new Error("Canvas is empty - no content captured");
-        }
-
-        // Convert to JPEG
-        let jpeg = canvas.toDataURL("image/jpeg", JPEG_QUALITY);
-
-        if (jpeg === "data:," || jpeg.length < 100) {
-            throw new Error("Failed to generate image data");
-        }
-
-        // Add EXIF metadata
-        jpeg = addExifMetadata(jpeg);
-
-        // Display the image
-        displayImage(jpeg);
-
-        // Clear selection
-        selection.removeAllRanges();
-
-    } catch (error) {
-        console.error("Error generating image:", error);
-        alert("Failed to generate image: " + error.message);
-    } finally {
-        // Cleanup
-        if (tempElement && tempElement.parentNode) {
-            tempElement.parentNode.removeChild(tempElement);
-        }
-        setButtonState(btn, false, originalBtnHtml);
-    }
-}

changedetectionio/static/js/watch-settings.js

@@ -26,19 +26,12 @@ function request_textpreview_update() {
             .text(data['after_filter'])
             .highlightLines([
                 {
-                    'color': 'var(--highlight-trigger-text-bg-color)',
-                    'lines': data['trigger_line_numbers'],
-                    'title': "Triggers a change if this text appears, AND something changed in the document."
+                    'color': '#ee0000',
+                    'lines': data['trigger_line_numbers']
                 },
                 {
-                    'color': 'var(--highlight-ignored-text-bg-color)',
-                    'lines': data['ignore_line_numbers'],
-                    'title': "Ignored for calculating changes, but still shown."
-                },
-                {
-                    'color': 'var(--highlight-blocked-text-bg-color)',
-                    'lines': data['blocked_line_numbers'],
-                    'title': "No change-detection will occur because this text exists."
+                    'color': '#757575',
+                    'lines': data['ignore_line_numbers']
                 }
             ])
     }).fail(function (error) {

changedetectionio/static/styles/scss/diff.scss

@@ -1,3 +1,5 @@
+@use "parts/variables";
+
 #diff-ui {
 
   background: var(--color-background);
@@ -26,7 +28,7 @@
   }
 
   pre {
-    white-space: break-spaces;
+    white-space: pre-wrap;
   }
 }
 
@@ -63,7 +65,7 @@ ins {
   border-radius: 10px;
   margin-bottom: 1em;
   color: #fff;
-  font-size: 0.9rem;
+  font-size: 80%;
 
   label {
     margin-left: 1em;
@@ -128,6 +130,13 @@ td#diff-col div {
   }
 }
 
+#highlightSnippet {
+  background: var(--color-background);
+  padding: 1em;
+  border-radius: 5px;
+  background: var(--color-background);
+  box-shadow: 1px 1px 4px var(--color-shadow-jump);
+}
 
 // resets button margin to 0px
 .pure-form button.reset-margin {
@@ -139,99 +148,4 @@ td#diff-col div {
   align-items: center;
   gap: 4px;
   flex-wrap: wrap;
-}
-
-
-
-ul#highlightSnippetActions {
-  list-style-type: none;
-  display: flex;
-  align-items: center;
-  justify-content: center;
-  gap: 1.5rem;
-  flex-wrap: wrap;
-  padding: 0;
-  margin: 0;
-
-  li {
-    display: flex;
-    flex-direction: column;
-    align-items: center;
-    text-align: center;
-    padding: 0.5rem;
-    gap: 0.3rem;
-
-    button, a {
-      white-space: nowrap;
-    }
-  }
-
-  span {
-    font-size: 0.8rem;
-    color: var(--color-text-input-description);
-  }
-}
-
-#cell-diff-jump-visualiser {
-  display: flex;
-  flex-direction: row;
-  gap: 1px;
-  background: var(--color-background);
-  border-radius: 3px;
-  overflow-x: auto;
-  position: sticky;
-  top: 0;
-  z-index: 10;
-  padding-top: 1rem;
-  padding-bottom: 1rem;
-  justify-content: center;
-  > div {
-    flex: 1;
-    min-width: 1px;
-    max-width: 10px;
-    height: 10px;
-    background: var(--color-background-button-cancel);
-    opacity: 0.3;
-    border-radius: 1px;
-    transition: opacity 0.2s;
-    position: relative;
-
-    &.deletion {
-      background: #b30000; // Red for deletions
-      opacity: 1;
-    }
-
-    &.insertion {
-      background: #406619; // Green for insertions
-      opacity: 1;
-    }
-
-    &.note {
-      background: #406619; // Orange for changed/notes
-      opacity: 1;
-    }
-
-    &.mixed {
-      background: linear-gradient(to right,  #b30000 50%, #406619 50%); // Half red, half green
-      opacity: 1;
-    }
-
-    &.current-position::after {
-      content: '';
-      position: absolute;
-      bottom: -6px;
-      left: 50%;
-      transform: translateX(-50%);
-      width: 0;
-      height: 0;
-      border-left: 4px solid transparent;
-      border-right: 4px solid transparent;
-      border-bottom: 4px solid var(--color-text);
-    }
-
-    &:hover {
-      opacity: 0.8;
-      cursor: pointer;
-    }
-  }
-}
+}

changedetectionio/static/styles/scss/parts/_variables.scss

@@ -102,10 +102,6 @@
 
   --color-watch-table-error: var(--color-dark-red);
   --color-watch-table-row-text: var(--color-grey-100);
-
-  --highlight-trigger-text-bg-color: #1b98f8;
-  --highlight-ignored-text-bg-color: var(--color-grey-700);
-  --highlight-blocked-text-bg-color: rgb(202, 60, 60);
 }
 
 html[data-darkmode="true"] {

changedetectionio/static/styles/scss/styles.scss

@@ -329,18 +329,12 @@ a.pure-button-selected {
 .notifications-wrapper {
   padding-top: 0.5rem;
   #notification-test-log {
-    margin-top: 1rem;
-    padding: 1rem;
+    padding-top: 1rem;
     white-space: pre-wrap;
     word-break: break-word;
     overflow-wrap: break-word;
     max-width: 100%;
     box-sizing: border-box;
-    max-height: 12rem;
-    overflow-y: scroll;
-    border: 1px solid var(--color-border-notification);
-    border-radius: 5px;
-
   }
 }
 
@@ -350,7 +344,7 @@ label {
  }  
 }
 
-.grey-form-border {
+#notification-customisation {
   border: 1px solid var(--color-border-notification);
   padding: 0.5rem;
   border-radius: 5px;
@@ -1127,30 +1121,3 @@ ul {
   color: #fff;
   opacity: 0.8;
 }
-
-#bottom-horizontal-offscreen {
-  position: fixed;
-  bottom: 0;
-  left: 0;
-  right: 0;
-  width: 100%;
-  min-height: 50px;
-  max-height: 50vh; // Don't take more than 50% of viewport height
-  background: var(--color-background);
-  border-top: 1px solid var(--color-border-table-cell);
-  padding: 10px;
-  box-shadow: 0 -2px 10px rgba(0, 0, 0, 0.2);
-  z-index: 100;
-  overflow-y: auto; // Allow scrolling if content exceeds max-height
-
-  // Smooth transition when shown/hidden
-  transition: opacity 0.3s ease-in-out;
-
-  // When JavaScript removes display:none, ensure it scrolls into view
-  scroll-margin-bottom: 10px;
-
-  // Center contents horizontally
-  display: flex;
-  justify-content: center;
-  align-items: center;
-}

changedetectionio/store.py

@@ -1,13 +1,11 @@
 from changedetectionio.strtobool import strtobool
 
-from changedetectionio.validate_url import is_safe_valid_url
-
 from flask import (
     flash
 )
 
 from .html_tools import TRANSLATE_WHITESPACE_TABLE
-from .model import App, Watch, USE_SYSTEM_DEFAULT_NOTIFICATION_FORMAT_FOR_WATCH
+from . model import App, Watch
 from copy import deepcopy, copy
 from os import path, unlink
 from threading import Lock
@@ -42,24 +40,17 @@ class ChangeDetectionStore:
     needs_write_urgent = False
 
     __version_check = True
-    save_data_thread = None
 
     def __init__(self, datastore_path="/datastore", include_default_watches=True, version_tag="0.0.0"):
         # Should only be active for docker
         # logging.basicConfig(filename='/dev/stdout', level=logging.INFO)
-
-        self.needs_write = False
-        self.start_time = time.time()
-        self.stop_thread = False
-        self.reload_state(datastore_path=datastore_path, include_default_watches=include_default_watches, version_tag=version_tag)
-
-
-    def reload_state(self, datastore_path, include_default_watches, version_tag):
-        logger.info(f"Datastore path is '{datastore_path}'")
-
         self.__data = App.model()
         self.datastore_path = datastore_path
         self.json_store_path = os.path.join(self.datastore_path, "url-watches.json")
+        logger.info(f"Datastore path is '{self.json_store_path}'")
+        self.needs_write = False
+        self.start_time = time.time()
+        self.stop_thread = False
         # Base definition for all watchers
         # deepcopy part of #569 - not sure why its needed exactly
         self.generic_definition = deepcopy(Watch.model(datastore_path = datastore_path, default={}))
@@ -152,10 +143,7 @@ class ChangeDetectionStore:
         self.needs_write = True
 
         # Finally start the thread that will manage periodic data saves to JSON
-        # Only start if thread is not already running (reload_state might be called multiple times)
-        if not self.save_data_thread or not self.save_data_thread.is_alive():
-            self.save_data_thread = threading.Thread(target=self.save_datastore)
-            self.save_data_thread.start()
+        save_data_thread = threading.Thread(target=self.save_datastore).start()
 
     def rehydrate_entity(self, uuid, entity, processor_override=None):
         """Set the dict back to the dict Watch object"""
@@ -240,37 +228,26 @@ class ChangeDetectionStore:
         d['settings']['application']['active_base_url'] = active_base_url.strip('" ')
         return d
 
-    from pathlib import Path
-
-    def delete_path(self, path: Path):
-        import shutil
-        """Delete a file or directory tree, including the path itself."""
-        if not path.exists():
-            return
-        if path.is_file() or path.is_symlink():
-            path.unlink(missing_ok=True)  # deletes a file or symlink
-        else:
-            shutil.rmtree(path, ignore_errors=True)  # deletes dir *and* its contents
-
     # Delete a single watch by UUID
     def delete(self, uuid):
         import pathlib
+        import shutil
 
         with self.lock:
             if uuid == 'all':
                 self.__data['watching'] = {}
                 time.sleep(1) # Mainly used for testing to allow all items to flush before running next test
+
+                # GitHub #30 also delete history records
                 for uuid in self.data['watching']:
-                    path = pathlib.Path(
-                        os.path.join(self.datastore_path, uuid))
+                    path = pathlib.Path(os.path.join(self.datastore_path, uuid))
                     if os.path.exists(path):
-                        self.delete(uuid)
+                        shutil.rmtree(path)
 
             else:
                 path = pathlib.Path(os.path.join(self.datastore_path, uuid))
                 if os.path.exists(path):
-                    self.delete_path(path)
-
+                    shutil.rmtree(path)
                 del self.data['watching'][uuid]
 
         self.needs_write_urgent = True
@@ -353,10 +330,9 @@ class ChangeDetectionStore:
                 logger.error(f"Error fetching metadata for shared watch link {url} {str(e)}")
                 flash("Error fetching metadata for {}".format(url), 'error')
                 return False
-
-        if not is_safe_valid_url(url):
-            flash('Watch protocol is not permitted or invalid URL format', 'error')
-
+        from .model.Watch import is_safe_url
+        if not is_safe_url(url):
+            flash('Watch protocol is not permitted by SAFE_PROTOCOL_REGEX', 'error')
             return None
 
         if tag and type(tag) == str:
@@ -422,6 +398,7 @@ class ChangeDetectionStore:
             self.sync_to_json()
             return
         else:
+
             try:
                 # Re #286  - First write to a temp file, then confirm it looks OK and rename it
                 # This is a fairly basic strategy to deal with the case that the file is corrupted,
@@ -451,7 +428,7 @@ class ChangeDetectionStore:
                 logger.remove()
                 logger.add(sys.stderr)
 
-                logger.info(f"Shutting down datastore '{self.datastore_path}' thread")
+                logger.critical("Shutting down datastore thread")
                 return
 
             if self.needs_write or self.needs_write_urgent:
@@ -999,45 +976,6 @@ class ChangeDetectionStore:
         if self.data['settings']['application'].get('extract_title_as_title'):
             self.data['settings']['application']['ui']['use_page_title_in_list'] = self.data['settings']['application'].get('extract_title_as_title')
 
-    def update_21(self):
-        if self.data['settings']['application'].get('timezone'):
-            self.data['settings']['application']['scheduler_timezone_default'] = self.data['settings']['application'].get('timezone')
-            del self.data['settings']['application']['timezone']
-
-
-    # Some notification formats got the wrong name type
-    def update_23(self):
-
-        def re_run(formats):
-            sys_n_format = self.data['settings']['application'].get('notification_format')
-            key_exists_as_value = next((k for k, v in formats.items() if v == sys_n_format), None)
-            if key_exists_as_value:  # key of "Plain text"
-                logger.success(f"['settings']['application']['notification_format'] '{sys_n_format}' -> '{key_exists_as_value}'")
-                self.data['settings']['application']['notification_format'] = key_exists_as_value
-
-            for uuid, watch in self.data['watching'].items():
-                n_format = self.data['watching'][uuid].get('notification_format')
-                key_exists_as_value = next((k for k, v in formats.items() if v == n_format), None)
-                if key_exists_as_value and key_exists_as_value != USE_SYSTEM_DEFAULT_NOTIFICATION_FORMAT_FOR_WATCH:  # key of "Plain text"
-                    logger.success(f"['watching'][{uuid}]['notification_format'] '{n_format}' -> '{key_exists_as_value}'")
-                    self.data['watching'][uuid]['notification_format'] = key_exists_as_value  # should be 'text' or whatever
-
-            for uuid, tag in self.data['settings']['application']['tags'].items():
-                n_format = self.data['settings']['application']['tags'][uuid].get('notification_format')
-                key_exists_as_value = next((k for k, v in formats.items() if v == n_format), None)
-                if key_exists_as_value and key_exists_as_value != USE_SYSTEM_DEFAULT_NOTIFICATION_FORMAT_FOR_WATCH:  # key of "Plain text"
-                    logger.success(
-                        f"['settings']['application']['tags'][{uuid}]['notification_format'] '{n_format}' -> '{key_exists_as_value}'")
-                    self.data['settings']['application']['tags'][uuid][
-                        'notification_format'] = key_exists_as_value  # should be 'text' or whatever
-
-        from .notification import valid_notification_formats
-        formats = deepcopy(valid_notification_formats)
-        re_run(formats)
-        # And in previous versions, it was "text" instead of Plain text, Markdown instead of "Markdown to HTML"
-        formats['text'] = 'Text'
-        formats['markdown'] = 'Markdown'
-        re_run(formats)
 
     def add_notification_url(self, notification_url):
         

changedetectionio/templates/_common_fields.html

@@ -33,7 +33,7 @@
                                 <div id="notification-test-log" style="display: none;"><span class="pure-form-message-inline">Processing..</span></div>
                             </div>
                         </div>
-                        <div class="pure-control-group grey-form-border">
+                        <div id="notification-customisation" class="pure-control-group">
                             <div class="pure-control-group">
                                 {{ render_field(form.notification_title, class="m-d notification-title", placeholder=settings_application['notification_title']) }}
                                 <span class="pure-form-message-inline">Title for all notifications</span>
@@ -87,35 +87,19 @@
 									<tr>
                                         <td><code>{{ '{{diff}}' }}</code></td>
                                         <td>The diff output - only changes, additions, and removals</td>
-                                    </tr>
-									<tr>
-                                        <td><code>{{ '{{diff_clean}}' }}</code></td>
-                                        <td>The diff output - only changes, additions, and removals &dash; <i>Without (added) prefix or colors</i></td>
                                     </tr>
 									<tr>
                                         <td><code>{{ '{{diff_added}}' }}</code></td>
                                         <td>The diff output - only changes and additions</td>
-                                    </tr>
-									<tr>
-                                        <td><code>{{ '{{diff_added_clean}}' }}</code></td>
-                                        <td>The diff output - only changes and additions &dash; <i>Without (added) prefix or colors</i></td>
                                     </tr>
 									<tr>
                                         <td><code>{{ '{{diff_removed}}' }}</code></td>
                                         <td>The diff output - only changes and removals</td>
-                                    </tr>
-									<tr>
-                                        <td><code>{{ '{{diff_removed_clean}}' }}</code></td>
-                                        <td>The diff output - only changes and removals &dash; <i>Without (added) prefix or colors</i></td>
                                     </tr>
                                     <tr>
                                         <td><code>{{ '{{diff_full}}' }}</code></td>
                                         <td>The diff output - full difference output</td>
                                     </tr>
-                                    <tr>
-                                        <td><code>{{ '{{diff_full_clean}}' }}</code></td>
-                                        <td>The diff output - full difference output &dash; <i>Without (added) prefix or colors</i></td>
-                                    </tr>
                                     <tr>
                                         <td><code>{{ '{{diff_patch}}' }}</code></td>
                                         <td>The diff output - patch in unified format</td>
@@ -150,12 +134,6 @@
                                     <p>
                                         URL encoding, use <strong>|urlencode</strong>, for example - <code>gets://hook-website.com/test.php?title={{ '{{ watch_title|urlencode }}' }}</code>
                                     </p>
-                                    <p>
-                                        Regular-expression replace, use <strong>|regex_replace</strong>, for example -   <code>{{ "{{ \"hello world 123\" | regex_replace('[0-9]+', 'no-more-numbers') }}" }}</code>
-                                    </p>
-                                    <p>
-                                        For a complete reference of all Jinja2 built-in filters, users can refer to the <a href="https://jinja.palletsprojects.com/en/3.1.x/templates/#builtin-filters">https://jinja.palletsprojects.com/en/3.1.x/templates/#builtin-filters</a>
-                                    </p>
                                 </div>
                             </div>
                             <div class="pure-control-group">

changedetectionio/templates/_helpers.html

@@ -14,31 +14,13 @@
                 {% if field.errors is mapping and 'form' in field.errors %}
                     {#  and subfield form errors, such as used in RequiredFormField() for TimeBetweenCheckForm sub form #}
                     {% set errors = field.errors['form'] %}
-                    {% for error in errors %}
-                        <li>{{ error }}</li>
-                    {% endfor %}
-                {% elif field.type == 'FieldList' %}
-                    {# Handle FieldList of FormFields - errors is a list of dicts, one per entry #}
-                    {% for idx, entry_errors in field.errors|enumerate %}
-                        {% if entry_errors is mapping and entry_errors %}
-                            {# Only show entries that have actual errors #}
-                            <li><strong>Entry {{ idx + 1 }}:</strong>
-                                <ul>
-                                    {% for field_name, messages in entry_errors.items() %}
-                                        {% for message in messages %}
-                                            <li>{{ field_name }}: {{ message }}</li>
-                                        {% endfor %}
-                                    {% endfor %}
-                                </ul>
-                            </li>
-                        {% endif %}
-                    {% endfor %}
                 {% else %}
                     {#  regular list of errors with this field #}
-                    {% for error in field.errors %}
-                        <li>{{ error }}</li>
-                    {% endfor %}
+                    {% set errors = field.errors %}
                 {% endif %}
+                {% for error in errors %}
+                    <li>{{ error }}</li>
+                {% endfor %}
             </ul>
         {% endif %}
     </div>
@@ -111,39 +93,6 @@
   {{ field(**kwargs)|safe }}
 {% endmacro %}
 
-{% macro render_fieldlist_with_inline_errors(fieldlist) %}
-  {# Specialized macro for FieldList(FormField(...)) that renders errors inline with each field #}
-  <div {% if fieldlist.errors %} class="error" {% endif %}>{{ fieldlist.label }}</div>
-  <div {% if fieldlist.errors %} class="error" {% endif %}>
-    <ul id="{{ fieldlist.id }}">
-      {% for entry in fieldlist %}
-        <li {% if entry.errors %} class="error" {% endif %}>
-          <label for="{{ entry.id }}" {% if entry.errors %} class="error" {% endif %}>{{ fieldlist.label.text }}-{{ loop.index0 }}</label>
-          <table id="{{ entry.id }}" {% if entry.errors %} class="error" {% endif %}>
-            <tbody>
-              {% for subfield in entry %}
-                <tr {% if subfield.errors %} class="error" {% endif %}>
-                  <th {% if subfield.errors %} class="error" {% endif %}><label for="{{ subfield.id }}" {% if subfield.errors %} class="error" {% endif %}>{{ subfield.label.text }}</label></th>
-                  <td {% if subfield.errors %} class="error" {% endif %}>
-                    {{ subfield(**kwargs)|safe }}
-                    {% if subfield.errors %}
-                      <ul class="errors">
-                        {% for error in subfield.errors %}
-                          <li class="error">{{ error }}</li>
-                        {% endfor %}
-                      </ul>
-                    {% endif %}
-                  </td>
-                </tr>
-              {% endfor %}
-            </tbody>
-          </table>
-        </li>
-      {% endfor %}
-    </ul>
-  </div>
-{% endmacro %}
-
 {% macro render_conditions_fieldlist_of_formfields_as_table(fieldlist, table_id="rulesTable") %}
   <div class="fieldlist_formfields" id="{{ table_id }}">
     <div class="fieldlist-header">
@@ -266,7 +215,9 @@
             <li id="timezone-info">
                 {{ render_field(form.time_schedule_limit.timezone, placeholder=timezone_default_config) }} <span id="local-time-in-tz"></span>
                 <datalist id="timezones" style="display: none;">
-                    {%- for timezone in available_timezones -%}<option value="{{ timezone }}">{{ timezone }}</option>{%- endfor -%}
+                    {% for timezone in available_timezones %}
+                        <option value="{{ timezone }}">{{ timezone }}</option>
+                    {% endfor %}
                 </datalist>
             </li>
         </ul>
@@ -282,12 +233,4 @@
         <br>
     {% endif %}
 
-{% endmacro %}
-
-{% macro highlight_trigger_ignored_explainer() %}
-                <p>
-                    <span title="Triggers a change if this text appears, AND something changed in the document." style="background-color: var(--highlight-trigger-text-bg-color); color: #fff; padding: 4px; border-radius: 2px; margin-right: 4px;">Triggered text</span>
-                    <span title="Ignored for calculating changes, but still shown." style="background-color: var(--highlight-ignored-text-bg-color); color: #fff; padding: 4px; border-radius: 2px; margin-right: 4px;">Ignored text</span>
-                    <span title="No change-detection will occur because this text exists." style="background-color: var(--highlight-blocked-text-bg-color); color: #fff; padding: 4px; border-radius: 2px; margin-right: 4px;">Blocked text</span>
-                </p>
 {% endmacro %}

changedetectionio/templates/base.html

@@ -43,7 +43,7 @@
 
   <body class="{{extra_classes}}">
     <div class="header">
-    <div {% if pure_menu_fixed != False %}class="pure-menu-fixed"{% endif %} style="width: 100%;">
+    <div class="pure-menu-fixed" style="width: 100%;">
       <div class="home-menu pure-menu pure-menu-horizontal" id="nav-menu">
 
         {% if has_password and not current_user.is_authenticated %}
@@ -53,7 +53,7 @@
           <a class="pure-menu-heading" href="{{url_for('watchlist.index')}}">
             <strong>Change</strong>Detection.io</a>
         {% endif %}
-        {% if current_diff_url and is_safe_valid_url(current_diff_url) %}
+        {% if current_diff_url %}
           <a class="current-diff-url" href="{{ current_diff_url }}">
             <span style="max-width: 30%; overflow: hidden">{{ current_diff_url }}</span></a>
         {% else %}
@@ -152,6 +152,8 @@
     {% endif %}
     {% if left_sticky %}
       <div class="sticky-tab" id="left-sticky">
+        <a href="{{url_for('ui.ui_views.preview_page', uuid=uuid)}}">Show current snapshot</a><br>
+          Visualise <strong>triggers</strong> and <strong>ignored text</strong>
       </div>
     {% endif %}
     {% if right_sticky %}
@@ -236,11 +238,6 @@
 
     <div id="checking-now-fixed-tab" style="display: none;"><span class="spinner"></span><span>&nbsp;Checking now</span></div>
     <div id="realtime-conn-error" style="display:none">Real-time updates offline</div>
-    {% if bottom_horizontal_offscreen_contents %}
-        <div id="bottom-horizontal-offscreen" style="display:none">
-            {{ bottom_horizontal_offscreen_contents|safe }}
-        </div>
-    {% endif %}
   </body>
 
 </html>

changedetectionio/templates/diff.html

@@ -8,20 +8,12 @@
     {% endif %}
 
     const highlight_submit_ignore_url="{{url_for('ui.ui_edit.highlight_submit_ignore_url', uuid=uuid)}}";
-    const watch_url= {{watch_a.link|tojson}};
 
-    // Initial scroll position: if set, scroll to this line number in #difference on page load
-    const initialScrollToLineNumber = {{ initial_scroll_line_number|default('null') }};
 </script>
-<script src="https://cdn.jsdelivr.net/npm/html2canvas@1.4.1/dist/html2canvas.min.js"></script>
-<script src="{{url_for('static_content', group='js', filename='plugins.js')}}"></script>
-<script src="https://cdn.jsdelivr.net/npm/piexifjs@1.0.6/piexif.min.js"></script>
-<script src="{{url_for('static_content', group='js', filename='snippet-to-image.js')}}"></script>
 <script src="{{url_for('static_content', group='js', filename='diff-overview.js')}}" defer></script>
 
-
 <div id="settings">
-    <form class="pure-form " action="{{ url_for("ui.ui_views.diff_history_page", uuid=uuid) }}" method="GET" id="diff-form">
+    <form class="pure-form " action="" method="GET" id="diff-form">
         <fieldset class="diff-fieldset">
             {% if versions|length >= 1 %}
                 <strong>Compare</strong>
@@ -45,27 +37,26 @@
             {% endif %}
         </fieldset>
         <fieldset>
-                <strong>Style</strong>
+            <strong>Style</strong>
+            <label for="diffWords" class="pure-checkbox">
+                <input type="radio" name="diff_type" id="diffWords" value="diffWords"> Words</label>
+            <label for="diffLines" class="pure-checkbox">
+                <input type="radio" name="diff_type" id="diffLines" value="diffLines" checked=""> Lines</label>
 
-                <label for="diffWords" class="pure-checkbox">
-                    <input type="radio" name="diff_type" id="diffWords" value="diffWords" {% if diff_prefs.diff_type == 'diffWords' %}checked=""{% endif %}> Words</label>
-                <label for="diffLines" class="pure-checkbox">
-                    <input type="radio" name="diff_type" id="diffLines" value="diffLines" {% if diff_prefs.diff_type == 'diffLines' %}checked=""{% endif %}> Lines</label>
+            <label for="diffChars" class="pure-checkbox">
+                <input type="radio" name="diff_type" id="diffChars" value="diffChars"> Chars</label>
+            <!-- @todo - when mimetype is JSON, select this by default? -->
+            <label for="diffJson" class="pure-checkbox">
+                <input type="radio" name="diff_type" id="diffJson" value="diffJson"> JSON</label>
 
-                <label for="ignoreWhitespace" class="pure-checkbox" id="label-diff-ignorewhitespace">
-                    <input type="checkbox" id="ignoreWhitespace" name="ignoreWhitespace" {% if diff_prefs.ignoreWhitespace %}checked=""{% endif %}> Ignore Whitespace</label>
-
-                <label for="diff_changesOnly" class="pure-checkbox" id="label-diff-changes">
-                    <input type="checkbox" id="diff_changesOnly" name="diff_changesOnly" {% if diff_prefs.diff_changesOnly %}checked=""{% endif %}> Same/non-changed</label>
-
-                <label for="diff_removed" class="pure-checkbox" id="label-diff-removed">
-                    <input type="checkbox" id="diff_removed" name="diff_removed" {% if diff_prefs.diff_removed %}checked=""{% endif %}> Removed</label>
-                <label for="diff_added" class="pure-checkbox" id="label-diff-added">
-                    <input type="checkbox" id="diff_added" name="diff_added" {% if diff_prefs.diff_added %}checked=""{% endif %}> Added</label>
-                <label for="diff_replaced" class="pure-checkbox" id="label-diff-replaced">
-                    <input type="checkbox" id="diff_replaced"  name="diff_replaced" {% if diff_prefs.diff_replaced %}checked=""{% endif %}> Replaced</label>
+            <span>
+        <!-- https://github.com/kpdecker/jsdiff/issues/389 ? -->
+        <label for="ignoreWhitespace" class="pure-checkbox" id="label-diff-ignorewhitespace">
+            <input type="checkbox" id="ignoreWhitespace" name="ignoreWhitespace"> Ignore Whitespace</label>
+    </span>
         </fieldset>
     </form>
+
 </div>
 
 <div id="diff-jump">
@@ -96,21 +87,28 @@
         <img id="error-screenshot-img"  style="max-width: 80%" alt="Current error-ing screenshot from most recent request" >
     </div>
 
-    <div class="tab-pane-inner" id="text">
-        {% if password_enabled_and_share_is_off %}
-            <div class="tip">Pro-tip: You can enable <strong>"share access when password is enabled"</strong> from settings.
-            </div>
-        {% endif %}
-        <div id="cell-diff-jump-visualiser">
-            {%- for cell in diff_cell_grid -%}
-            <div{% if cell.class %} class="{{ cell.class }}"{% endif %}></div>
-            {%- endfor -%}
-        </div>
-        <div class="snapshot-age">{{ watch_a.snapshot_text_ctime|format_timestamp_timeago }}</div>
-        <pre id="difference" style="border-left: 2px solid #ddd;">{{ content| diff_unescape_difference_spans }}</pre>
-    </div>
+     <div class="tab-pane-inner" id="text">
+         {% if password_enabled_and_share_is_off %}
+           <div class="tip">Pro-tip: You can enable <strong>"share access when password is enabled"</strong> from settings</div>
+         {% endif %}
 
-    <div class="tab-pane-inner" id="screenshot">
+         <div class="snapshot-age">{{watch_a.snapshot_text_ctime|format_timestamp_timeago}}</div>
+
+         <table>
+             <tbody>
+             <tr>
+                 <!-- just proof of concept copied straight from github.com/kpdecker/jsdiff -->
+                 <td id="a" style="display: none;">{{from_version_file_contents}}</td>
+                 <td id="b" style="display: none;">{{to_version_file_contents}}</td>
+                 <td id="diff-col">
+                     <span id="result" class="highlightable-filter"></span>
+                 </td>
+             </tr>
+             </tbody>
+         </table>
+         Diff algorithm from the amazing <a href="https://github.com/kpdecker/jsdiff">github.com/kpdecker/jsdiff</a>
+     </div>
+     <div class="tab-pane-inner" id="screenshot">
          <div class="tip">
              For now, Differences are performed on text, not graphically, only the latest screenshot is available.
          </div>
@@ -161,6 +159,8 @@
 <script>
     const newest_version_timestamp = {{newest_version_timestamp}};
 </script>
+<script src="{{url_for('static_content', group='js', filename='diff.min.js')}}"></script>
+
 <script src="{{url_for('static_content', group='js', filename='diff-render.js')}}"></script>
 
 

changedetectionio/templates/preview.html

@@ -1,11 +1,9 @@
 {% extends 'base.html' %}
-{% from '_helpers.html' import highlight_trigger_ignored_explainer %}
+
 {% block content %}
     <script>
         const screenshot_url = "{{url_for('static_content', group='screenshot', filename=uuid)}}";
-        const triggered_line_numbers = {{ highlight_triggered_line_numbers|tojson }};
-        const ignored_line_numbers = {{ highlight_ignored_line_numbers|tojson }};
-        const blocked_line_numbers = {{ highlight_blocked_line_numbers|tojson }};
+        const triggered_line_numbers = {{ triggered_line_numbers|tojson }};
         {% if last_error_screenshot %}
             const error_screenshot_url = "{{url_for('static_content', group='screenshot', filename=uuid, error_screenshot=1) }}";
         {% endif %}
@@ -84,7 +82,6 @@
                 </tr>
                 </tbody>
             </table>
-            {{ highlight_trigger_ignored_explainer() }}
         </div>
 
         <div class="tab-pane-inner" id="screenshot">

changedetectionio/tests/conftest.py

@@ -4,14 +4,12 @@ import time
 from threading import Thread
 
 import pytest
-import arrow
 from changedetectionio import changedetection_app
 from changedetectionio import store
 import os
 import sys
 from loguru import logger
 
-from changedetectionio.flask_app import init_app_secret
 from changedetectionio.tests.util import live_server_setup, new_live_server_setup
 
 # https://github.com/pallets/flask/blob/1.1.2/examples/tutorial/tests/test_auth.py
@@ -31,17 +29,6 @@ def reportlog(pytestconfig):
     logger.remove(handler_id)
 
 
-@pytest.fixture
-def environment(mocker):
-    """Mock arrow.now() to return a fixed datetime for testing jinja2 time extension."""
-    # Fixed datetime: Wed, 09 Dec 2015 23:33:01 UTC
-    # This is calculated to match the test expectations when offsets are applied
-    fixed_datetime = arrow.Arrow(2015, 12, 9, 23, 33, 1, tzinfo='UTC')
-    # Patch arrow.now in the TimeExtension module where it's actually used
-    mocker.patch('changedetectionio.jinja2_custom.extensions.TimeExtension.arrow.now', return_value=fixed_datetime)
-    return fixed_datetime
-
-
 def format_memory_human(bytes_value):
     """Format memory in human-readable units (KB, MB, GB)"""
     if bytes_value < 1024:
@@ -88,6 +75,7 @@ def measure_memory_usage(request):
 def cleanup(datastore_path):
     import glob
     # Unlink test output files
+
     for g in ["*.txt", "*.json", "*.pdf"]:
         files = glob.glob(os.path.join(datastore_path, g))
         for f in files:
@@ -97,121 +85,34 @@ def cleanup(datastore_path):
             if os.path.isfile(f):
                 os.unlink(f)
 
-def pytest_addoption(parser):
-    """Add custom command-line options for pytest.
-
-    Provides --datastore-path option for specifying custom datastore location.
-    Note: Cannot use -d short option as it's reserved by pytest for debug mode.
-    """
-    parser.addoption(
-        "--datastore-path",
-        action="store",
-        default=None,
-        help="Custom datastore path for tests"
-    )
-
-@pytest.fixture(scope='session')
-def datastore_path(tmp_path_factory, request):
-    """Provide datastore path unique to this worker.
-
-    Supports custom path via --datastore-path/-d flag (mirrors main app).
-
-    CRITICAL for xdist isolation:
-    - Each WORKER gets its own directory
-    - Tests on same worker run SEQUENTIALLY and cleanup between tests
-    - No subdirectories needed since tests don't overlap on same worker
-    - Example: /tmp/test-datastore-gw0/ for worker gw0
-    """
-    # Check for custom path first (mirrors main app's -d flag)
-    custom_path = request.config.getoption("--datastore-path")
-    if custom_path:
-        # Ensure the directory exists
-        os.makedirs(custom_path, exist_ok=True)
-        logger.info(f"Using custom datastore path: {custom_path}")
-        return custom_path
-
-    # Otherwise use default tmp_path_factory logic
-    worker_id = getattr(request.config, 'workerinput', {}).get('workerid', 'master')
-    if worker_id == 'master':
-        path = tmp_path_factory.mktemp("test-datastore")
-    else:
-        path = tmp_path_factory.mktemp(f"test-datastore-{worker_id}")
-    return str(path)
-
-
 @pytest.fixture(scope='function', autouse=True)
-def prepare_test_function(live_server, datastore_path):
-    """Prepare each test with complete isolation.
+def prepare_test_function(live_server):
 
-    CRITICAL for xdist per-test isolation:
-    - Reuses the SAME datastore instance (so blueprint references stay valid)
-    - Clears all watches and state for a clean slate
-    - First watch will get uuid="first"
-    """
     routes = [rule.rule for rule in live_server.app.url_map.iter_rules()]
     if '/test-random-content-endpoint' not in routes:
         logger.debug("Setting up test URL routes")
         new_live_server_setup(live_server)
 
-    # CRITICAL: Point app to THIS test's unique datastore directory
-    live_server.app.config['TEST_DATASTORE_PATH'] = datastore_path
-
-    # CRITICAL: Get datastore and stop it from writing stale data
-    datastore = live_server.app.config.get('DATASTORE')
-
-    # Prevent background thread from writing during cleanup/reload
-    datastore.needs_write = False
-    datastore.needs_write_urgent = False
-
-    # CRITICAL: Clean up any files from previous tests
-    # This ensures a completely clean directory
-    cleanup(datastore_path)
-
-    # CRITICAL: Reload the EXISTING datastore instead of creating a new one
-    # This keeps blueprint references valid (they capture datastore at construction)
-    # reload_state() completely resets the datastore to a clean state
-
-    # Reload state with clean data (no default watches)
-    datastore.reload_state(
-        datastore_path=datastore_path,
-        include_default_watches=False,
-        version_tag=datastore.data.get('version_tag', '0.0.0')
-    )
-    live_server.app.secret_key = init_app_secret(datastore_path)
-    logger.debug(f"prepare_test_function: Reloaded datastore at {hex(id(datastore))}")
-    logger.debug(f"prepare_test_function: Path {datastore.datastore_path}")
 
     yield
-
-    # Cleanup: Clear watches again after test
-    try:
-        datastore.data['watching'] = {}
-        datastore.needs_write = True
-    except Exception as e:
-        logger.warning(f"Error during datastore cleanup: {e}")
-
-
-# So the app can also know which test name it was
-@pytest.fixture(autouse=True)
-def set_test_name(request):
-  """Automatically set TEST_NAME env var for every test"""
-  test_name = request.node.name
-  os.environ['PYTEST_CURRENT_TEST'] = test_name
-  yield
-  # Cleanup if needed
+    # Then cleanup/shutdown
+    live_server.app.config['DATASTORE'].data['watching']={}
+    time.sleep(0.3)
+    live_server.app.config['DATASTORE'].data['watching']={}
 
 
 @pytest.fixture(scope='session')
-def app(request, datastore_path):
-    """Create application once per worker (session).
+def app(request):
+    """Create application for the tests."""
+    datastore_path = "./test-datastore"
 
-    Note: Actual per-test isolation is handled by:
-    - prepare_test_function() recreates datastore and cleans directory
-    - All tests on same worker use same directory (cleaned between tests)
-    """
     # So they don't delay in fetching
     os.environ["MINIMUM_SECONDS_RECHECK_TIME"] = "0"
-    logger.debug(f"Testing with datastore_path={datastore_path}")
+    try:
+        os.mkdir(datastore_path)
+    except FileExistsError:
+        pass
+
     cleanup(datastore_path)
 
     app_config = {'datastore_path': datastore_path, 'disable_checkver' : True}
@@ -234,8 +135,6 @@ def app(request, datastore_path):
     # Disable CSRF while running tests
     app.config['WTF_CSRF_ENABLED'] = False
     app.config['STOP_THREADS'] = True
-    # Store datastore_path so Flask routes can access it
-    app.config['TEST_DATASTORE_PATH'] = datastore_path
 
     def teardown():
         # Stop all threads and services

changedetectionio/tests/custom_browser_url/test_custom_browser_url.py

@@ -73,13 +73,13 @@ def do_test(client, live_server, make_test_use_extra_browser=False):
 
 
 # Requires playwright to be installed
-def test_request_via_custom_browser_url(client, live_server, measure_memory_usage, datastore_path):
+def test_request_via_custom_browser_url(client, live_server, measure_memory_usage):
    #  live_server_setup(live_server) # Setup on conftest per function
     # We do this so we can grep the logs of the custom container and see if the request actually went through that container
     do_test(client, live_server, make_test_use_extra_browser=True)
 
 
-def test_request_not_via_custom_browser_url(client, live_server, measure_memory_usage, datastore_path):
+def test_request_not_via_custom_browser_url(client, live_server, measure_memory_usage):
    #  live_server_setup(live_server) # Setup on conftest per function
     # We do this so we can grep the logs of the custom container and see if the request actually went through that container
     do_test(client, live_server, make_test_use_extra_browser=False)

changedetectionio/tests/fetchers/test_content.py

@@ -8,7 +8,7 @@ import logging
 
 
 # Requires playwright to be installed
-def test_fetch_webdriver_content(client, live_server, measure_memory_usage, datastore_path):
+def test_fetch_webdriver_content(client, live_server, measure_memory_usage):
     #  live_server_setup(live_server) # Setup on conftest per function
 
     #####################

changedetectionio/tests/fetchers/test_custom_js_before_content.py

@@ -3,7 +3,7 @@ from flask import url_for
 from ..util import live_server_setup, wait_for_all_checks, extract_UUID_from_client
 
 
-def test_execute_custom_js(client, live_server, measure_memory_usage, datastore_path):
+def test_execute_custom_js(client, live_server, measure_memory_usage):
 
    #  live_server_setup(live_server) # Setup on conftest per function
     assert os.getenv('PLAYWRIGHT_DRIVER_URL'), "Needs PLAYWRIGHT_DRIVER_URL set for this test"

changedetectionio/tests/proxy_list/test_multiple_proxy.py

@@ -5,7 +5,7 @@ from flask import url_for
 from ..util import live_server_setup, wait_for_all_checks
 
 
-def test_preferred_proxy(client, live_server, measure_memory_usage, datastore_path):
+def test_preferred_proxy(client, live_server, measure_memory_usage):
    #  live_server_setup(live_server) # Setup on conftest per function
     url = "http://chosen.changedetection.io"
 

changedetectionio/tests/proxy_list/test_noproxy.py

@@ -5,7 +5,7 @@ from flask import url_for
 from ..util import live_server_setup, wait_for_all_checks, extract_UUID_from_client
 
 
-def test_noproxy_option(client, live_server, measure_memory_usage, datastore_path):
+def test_noproxy_option(client, live_server, measure_memory_usage):
    #  live_server_setup(live_server) # Setup on conftest per function
     # Run by run_proxy_tests.sh
     # Call this URL then scan the containers that it never went through them

changedetectionio/tests/proxy_list/test_proxy.py

@@ -5,7 +5,7 @@ from flask import url_for
 from ..util import live_server_setup, wait_for_all_checks, extract_UUID_from_client
 
 # just make a request, we will grep in the docker logs to see it actually got called
-def test_check_basic_change_detection_functionality(client, live_server, measure_memory_usage, datastore_path):
+def test_check_basic_change_detection_functionality(client, live_server, measure_memory_usage):
    #  live_server_setup(live_server) # Setup on conftest per function
     res = client.post(
         url_for("imports.import_page"),

changedetectionio/tests/proxy_list/test_proxy_noconnect.py

@@ -12,7 +12,7 @@ from ... import strtobool
 # FAST_PUPPETEER_CHROME_FETCHER=True PLAYWRIGHT_DRIVER_URL=ws://127.0.0.1:3000 pytest tests/proxy_list/test_proxy_noconnect.py
 # WEBDRIVER_URL=http://127.0.0.1:4444/wd/hub pytest tests/proxy_list/test_proxy_noconnect.py
 
-def test_proxy_noconnect_custom(client, live_server, measure_memory_usage, datastore_path):
+def test_proxy_noconnect_custom(client, live_server, measure_memory_usage):
    #  live_server_setup(live_server) # Setup on conftest per function
 
     # Goto settings, add our custom one

changedetectionio/tests/proxy_list/test_select_custom_proxy.py

@@ -6,7 +6,7 @@ from ..util import live_server_setup, wait_for_all_checks
 import os
 
 # just make a request, we will grep in the docker logs to see it actually got called
-def test_select_custom(client, live_server, measure_memory_usage, datastore_path):
+def test_select_custom(client, live_server, measure_memory_usage):
    #  live_server_setup(live_server) # Setup on conftest per function
 
     # Goto settings, add our custom one
@@ -49,39 +49,3 @@ def test_select_custom(client, live_server, measure_memory_usage, datastore_path
     #
     # Now we should see the request in the container logs for "squid-squid-custom" because it will be the only default
 
-
-def test_custom_proxy_validation(client, live_server, measure_memory_usage, datastore_path):
-    #  live_server_setup(live_server) # Setup on conftest per function
-
-    # Goto settings, add our custom one
-    res = client.post(
-        url_for("settings.settings_page"),
-        data={
-            "requests-time_between_check-minutes": 180,
-            "application-ignore_whitespace": "y",
-            "application-fetch_backend": 'html_requests',
-            "requests-extra_proxies-0-proxy_name": "custom-test-proxy",
-            "requests-extra_proxies-0-proxy_url": "xxxxhtt/333??p://test:awesome@squid-custom:3128",
-        },
-        follow_redirects=True
-    )
-
-    assert b"Settings updated." not in res.data
-    assert b'Proxy URLs must start with' in res.data
-
-
-    res = client.post(
-        url_for("settings.settings_page"),
-        data={
-            "requests-time_between_check-minutes": 180,
-            "application-ignore_whitespace": "y",
-            "application-fetch_backend": 'html_requests',
-            "requests-extra_proxies-0-proxy_name": "custom-test-proxy",
-            "requests-extra_proxies-0-proxy_url": "https://",
-        },
-        follow_redirects=True
-    )
-
-    assert b"Settings updated." not in res.data
-    assert b"Invalid URL." in res.data
-    

changedetectionio/tests/proxy_socks5/test_socks5_proxy.py

@@ -5,7 +5,7 @@ from flask import url_for
 from changedetectionio.tests.util import live_server_setup, wait_for_all_checks, extract_UUID_from_client, delete_all_watches
 
 
-def set_response(datastore_path):
+def set_response():
     import time
     data = """<html>
        <body>
@@ -15,13 +15,13 @@ def set_response(datastore_path):
      </html>
     """
 
-    with open(os.path.join(datastore_path, "endpoint-content.txt"), "w") as f:
+    with open("test-datastore/endpoint-content.txt", "w") as f:
         f.write(data)
     time.sleep(1)
 
-def test_socks5(client, live_server, measure_memory_usage, datastore_path):
+def test_socks5(client, live_server, measure_memory_usage):
    #  live_server_setup(live_server) # Setup on conftest per function
-    set_response(datastore_path)
+    set_response()
 
     # Setup a proxy
     res = client.post(

changedetectionio/tests/proxy_socks5/test_socks5_proxy_sources.py

@@ -4,7 +4,7 @@ from flask import url_for
 from changedetectionio.tests.util import live_server_setup, wait_for_all_checks
 
 
-def set_response(datastore_path):
+def set_response():
     import time
     data = """<html>
        <body>
@@ -14,15 +14,15 @@ def set_response(datastore_path):
      </html>
     """
 
-    with open(os.path.join(datastore_path, "endpoint-content.txt"), "w") as f:
+    with open("test-datastore/endpoint-content.txt", "w") as f:
         f.write(data)
     time.sleep(1)
 
 # should be proxies.json mounted from run_proxy_tests.sh already
 # -v `pwd`/tests/proxy_socks5/proxies.json-example:/app/changedetectionio/test-datastore/proxies.json
-def test_socks5_from_proxiesjson_file(client, live_server, measure_memory_usage, datastore_path):
+def test_socks5_from_proxiesjson_file(client, live_server, measure_memory_usage):
    #  live_server_setup(live_server) # Setup on conftest per function
-    set_response(datastore_path)
+    set_response()
     # Because the socks server should connect back to us
     test_url = url_for('test_endpoint', _external=True) + f"?socks-test-tag={os.getenv('SOCKSTEST', '')}"
     test_url = test_url.replace('localhost.localdomain', 'cdio')

changedetectionio/tests/restock/test_restock.py

@@ -11,7 +11,7 @@ from changedetectionio.notification import (
 )
 
 
-def set_original_response(datastore_path):
+def set_original_response():
     test_return_data = """<html>
        <body>
        <section id=header style="padding: 50px; height: 350px">This is the header which should be ignored always - <span>add to cart</span></section>
@@ -26,13 +26,13 @@ def set_original_response(datastore_path):
      </html>
     """
 
-    with open(os.path.join(datastore_path, "endpoint-content.txt"), "w") as f:
+    with open("test-datastore/endpoint-content.txt", "w") as f:
         f.write(test_return_data)
     return None
 
 
 
-def set_back_in_stock_response(datastore_path):
+def set_back_in_stock_response():
     test_return_data = """<html>
        <body>
      Some initial text<br>
@@ -45,14 +45,14 @@ def set_back_in_stock_response(datastore_path):
      </html>
     """
 
-    with open(os.path.join(datastore_path, "endpoint-content.txt"), "w") as f:
+    with open("test-datastore/endpoint-content.txt", "w") as f:
         f.write(test_return_data)
     return None
 
 # Add a site in paused mode, add an invalid filter, we should still have visual selector data ready
-def test_restock_detection(client, live_server, measure_memory_usage, datastore_path):
+def test_restock_detection(client, live_server, measure_memory_usage):
 
-    set_original_response(datastore_path=datastore_path)
+    set_original_response()
     #assert os.getenv('PLAYWRIGHT_DRIVER_URL'), "Needs PLAYWRIGHT_DRIVER_URL set for this test"
    #  live_server_setup(live_server) # Setup on conftest per function
     #####################
@@ -88,25 +88,24 @@ def test_restock_detection(client, live_server, measure_memory_usage, datastore_
     assert b'not-in-stock' in res.data # should be out of stock
 
     # Is it correctly shown as in stock
-    set_back_in_stock_response(datastore_path)
+    set_back_in_stock_response()
     client.get(url_for("ui.form_watch_checknow"), follow_redirects=True)
     wait_for_all_checks(client)
     res = client.get(url_for("watchlist.index"))
     assert b'not-in-stock' not in res.data
 
     # We should have a notification
-    notification_file = os.path.join(datastore_path, "notification.txt")
-    wait_for_notification_endpoint_output(datastore_path=datastore_path)
-    assert os.path.isfile(notification_file), "Notification received"
-    os.unlink(notification_file)
+    wait_for_notification_endpoint_output()
+    assert os.path.isfile("test-datastore/notification.txt"), "Notification received"
+    os.unlink("test-datastore/notification.txt")
 
     # Default behaviour is to only fire notification when it goes OUT OF STOCK -> IN STOCK
     # So here there should be no file, because we go IN STOCK -> OUT OF STOCK
-    set_original_response(datastore_path=datastore_path)
+    set_original_response()
     client.get(url_for("ui.form_watch_checknow"), follow_redirects=True)
     wait_for_all_checks(client)
     time.sleep(5)
-    assert not os.path.isfile(notification_file), "No notification should have fired when it went OUT OF STOCK by default"
+    assert not os.path.isfile("test-datastore/notification.txt"), "No notification should have fired when it went OUT OF STOCK by default"
 
     # BUT we should see that it correctly shows "not in stock"
     res = client.get(url_for("watchlist.index"))

changedetectionio/tests/smtp/smtp-test-server.py

@@ -1,110 +1,51 @@
 #!/usr/bin/env python3
-import threading
-import time
+import asyncio
 from aiosmtpd.controller import Controller
-from flask import Flask, Response
-from email import message_from_bytes
-from email.policy import default
+from aiosmtpd.smtp import SMTP
 
-# Accept a SMTP message and offer a way to retrieve the last message via HTTP
+# Accept a SMTP message and offer a way to retrieve the last message via TCP Socket
 
-last_received_message = b"SMTP Test Server - Nothing received yet."
-active_smtp_connections = 0
-smtp_lock = threading.Lock()
+last_received_message = b"Nothing"
 
 
 class CustomSMTPHandler:
     async def handle_DATA(self, server, session, envelope):
-        global last_received_message, active_smtp_connections
-
-        with smtp_lock:
-            active_smtp_connections += 1
-
-        try:
-            last_received_message = envelope.content
-            print('Receiving message from:', session.peer)
-            print('Message addressed from:', envelope.mail_from)
-            print('Message addressed to  :', envelope.rcpt_tos)
-            print('Message length        :', len(envelope.content))
-            print('*******************************')
-            print(envelope.content.decode('utf8'))
-            print('*******************************')
-
-            # Parse the email message
-            msg = message_from_bytes(envelope.content, policy=default)
-            with open('/tmp/last.eml', 'wb') as f:
-                f.write(envelope.content)
-
-            # Write parts to files based on content type
-            if msg.is_multipart():
-                for part in msg.walk():
-                    content_type = part.get_content_type()
-                    payload = part.get_payload(decode=True)
-
-                    if payload:
-                        if content_type == 'text/plain':
-                            with open('/tmp/last.txt', 'wb') as f:
-                                f.write(payload)
-                            print(f'Written text/plain part to /tmp/last.txt')
-                        elif content_type == 'text/html':
-                            with open('/tmp/last.html', 'wb') as f:
-                                f.write(payload)
-                            print(f'Written text/html part to /tmp/last.html')
-            else:
-                # Single part message
-                content_type = msg.get_content_type()
-                payload = msg.get_payload(decode=True)
-
-                if payload:
-                    if content_type == 'text/plain' or content_type.startswith('text/'):
-                        with open('/tmp/last.txt', 'wb') as f:
-                            f.write(payload)
-                        print(f'Written single part message to /tmp/last.txt')
-
-            return '250 Message accepted for delivery'
-        finally:
-            with smtp_lock:
-                active_smtp_connections -= 1
+        global last_received_message
+        last_received_message = envelope.content
+        print('Receiving message from:', session.peer)
+        print('Message addressed from:', envelope.mail_from)
+        print('Message addressed to  :', envelope.rcpt_tos)
+        print('Message length        :', len(envelope.content))
+        print(envelope.content.decode('utf8'))
+        return '250 Message accepted for delivery'
 
 
-# Simple Flask HTTP server to echo back the last SMTP message
-app = Flask(__name__)
+class EchoServerProtocol(asyncio.Protocol):
+    def connection_made(self, transport):
+        global last_received_message
+        self.transport = transport
+        peername = transport.get_extra_info('peername')
+        print('Incoming connection from {}'.format(peername))
+        self.transport.write(last_received_message)
+
+        last_received_message = b''
+        self.transport.close()
 
 
-@app.route('/')
-def echo_last_message():
-    global last_received_message, active_smtp_connections
-
-    # Wait for any in-progress SMTP connections to complete
-    max_wait = 5  # Maximum 5 seconds
-    wait_interval = 0.05  # Check every 50ms
-    elapsed = 0
-
-    while elapsed < max_wait:
-        with smtp_lock:
-            if active_smtp_connections == 0:
-                break
-        time.sleep(wait_interval)
-        elapsed += wait_interval
-
-    return Response(last_received_message, mimetype='text/plain')
-
-
-def run_flask():
-    app.run(host='0.0.0.0', port=11080, debug=False, use_reloader=False)
-
-
-if __name__ == "__main__":
+async def main():
     # Start the SMTP server
     controller = Controller(CustomSMTPHandler(), hostname='0.0.0.0', port=11025)
     controller.start()
 
-    # Start the HTTP server in a separate thread
-    flask_thread = threading.Thread(target=run_flask, daemon=True)
-    flask_thread.start()
+    # Start the TCP Echo server
+    loop = asyncio.get_running_loop()
+    server = await loop.create_server(
+        lambda: EchoServerProtocol(),
+        '0.0.0.0', 11080
+    )
+    async with server:
+        await server.serve_forever()
 
-    # Keep the main thread alive
-    try:
-        flask_thread.join()
-    except KeyboardInterrupt:
-        print("Shutting down...")
+
+if __name__ == "__main__":
+    asyncio.run(main())

changedetectionio/tests/smtp/test_notification_smtp.py

@@ -1,23 +1,18 @@
+import json
+import os
 import time
+import re
 from flask import url_for
-from email import message_from_string
-from email.policy import default as email_policy
-
-from changedetectionio.diff import HTML_REMOVED_STYLE, HTML_ADDED_STYLE, HTML_CHANGED_STYLE, REMOVED_PLACEMARKER_OPEN, \
-    CHANGED_PLACEMARKER_OPEN, ADDED_PLACEMARKER_OPEN
-from changedetectionio.notification_service import NotificationContextData
 from changedetectionio.tests.util import set_original_response, set_modified_response, set_more_modified_response, live_server_setup, \
     wait_for_all_checks, \
     set_longer_modified_response, delete_all_watches
-
+from changedetectionio.tests.util import extract_UUID_from_client
 import logging
-
+import base64
 
 # NOTE - RELIES ON mailserver as hostname running, see github build recipes
 smtp_test_server = 'mailserver'
 
-ALL_MARKUP_TOKENS = ''.join(f"TOKEN: '{t}'\n{{{{{t}}}}}\n" for t in NotificationContextData().keys())
-
 from changedetectionio.notification import (
     default_notification_body,
     default_notification_format,
@@ -28,23 +23,24 @@ from changedetectionio.notification import (
 
 
 def get_last_message_from_smtp_server():
-    import requests
-    time.sleep(1) # wait for any smtp connects to die off
-    port = 11080  # HTTP server port number
-    # Make HTTP GET request to Flask server
-    response = requests.get(f'http://{smtp_test_server}:{port}/')
-    data = response.text
+    import socket
+    port = 11080  # socket server port number
+
+    client_socket = socket.socket()  # instantiate
+    client_socket.connect((smtp_test_server, port))  # connect to the server
+
+    data = client_socket.recv(50024).decode()  # receive response
     logging.info("get_last_message_from_smtp_server..")
     logging.info(data)
+    client_socket.close()  # close the connection
     return data
 
 
 # Requires running the test SMTP server
 
-def test_check_notification_email_formats_default_HTML(client, live_server, measure_memory_usage, datastore_path):
+def test_check_notification_email_formats_default_HTML(client, live_server, measure_memory_usage):
     ##  live_server_setup(live_server) # Setup on conftest per function
-    set_original_response(datastore_path=datastore_path)
-
+    set_original_response()
 
     notification_url = f'mailto://changedetection@{smtp_test_server}:11025/?to=fff@home.com'
 
@@ -54,8 +50,8 @@ def test_check_notification_email_formats_default_HTML(client, live_server, meas
         url_for("settings.settings_page"),
         data={"application-notification_urls": notification_url,
               "application-notification_title": "fallback-title " + default_notification_title,
-              "application-notification_body": "some text\nfallback-body<br> " + default_notification_body,
-              "application-notification_format": 'html',
+              "application-notification_body": "fallback-body<br> " + default_notification_body,
+              "application-notification_format": 'HTML',
               "requests-time_between_check-minutes": 180,
               'application-fetch_backend': "html_requests"},
         follow_redirects=True
@@ -73,7 +69,7 @@ def test_check_notification_email_formats_default_HTML(client, live_server, meas
     assert b"Watch added" in res.data
 
     wait_for_all_checks(client)
-    set_longer_modified_response(datastore_path=datastore_path)
+    set_longer_modified_response()
     time.sleep(2)
 
     client.get(url_for("ui.form_watch_checknow"), follow_redirects=True)
@@ -81,273 +77,24 @@ def test_check_notification_email_formats_default_HTML(client, live_server, meas
 
     time.sleep(3)
 
-    msg_raw = get_last_message_from_smtp_server()
-    assert len(msg_raw) >= 1
+    msg = get_last_message_from_smtp_server()
+    assert len(msg) >= 1
 
-    # Parse the email properly using Python's email library
-    msg = message_from_string(msg_raw, policy=email_policy)
-
-    # The email should have two bodies (multipart/alternative with text/plain and text/html)
-    assert msg.is_multipart()
-    assert msg.get_content_type() == 'multipart/alternative'
-
-    # Get the parts
-    parts = list(msg.iter_parts())
-    assert len(parts) == 2
-
-    # First part should be text/plain (the auto-generated plaintext version)
-    text_part = parts[0]
-    assert text_part.get_content_type() == 'text/plain'
-    text_content = text_part.get_content()
-    assert '(added) So let\'s see what happens.\r\n' in text_content  # The plaintext part
-    assert 'fallback-body\r\n' in text_content  # The plaintext part
-
-    # Second part should be text/html
-    html_part = parts[1]
-    assert html_part.get_content_type() == 'text/html'
-    html_content = html_part.get_content()
-    assert 'some text<br>' in html_content  # We converted \n from the notification body
-    assert 'fallback-body<br>' in html_content  # kept the original <br>
-    assert '(added) So let\'s see what happens.<br>' in html_content  # the html part
+    # The email should have two bodies, and the text/html part should be <br>
+    assert 'Content-Type: text/plain' in msg
+    assert '(added) So let\'s see what happens.\r\n' in msg  # The plaintext part with \r\n
+    assert 'Content-Type: text/html' in msg
+    assert '(added) So let\'s see what happens.<br>' in msg  # the html part
     delete_all_watches(client)
 
 
-def test_check_notification_plaintext_format(client, live_server, measure_memory_usage, datastore_path):
-    set_original_response(datastore_path=datastore_path)
-
-
-    notification_url = f'mailto://changedetection@{smtp_test_server}:11025/?to=fff@home.com'
-
-    #####################
-    # Set this up for when we remove the notification from the watch, it should fallback with these details
-    res = client.post(
-        url_for("settings.settings_page"),
-        data={"application-notification_urls": notification_url,
-              "application-notification_title": "fallback-title {{watch_title}}  {{ diff_added.splitlines()[0] if diff_added else 'diff added didnt split' }}  " + default_notification_title,
-              "application-notification_body": f"some text\n" + default_notification_body + f"\nMore output test\n{ALL_MARKUP_TOKENS}",
-              "application-notification_format": 'text',
-              "requests-time_between_check-minutes": 180,
-              'application-fetch_backend': "html_requests"},
-        follow_redirects=True
-    )
-
-    assert b"Settings updated." in res.data
-
-    # Add a watch and trigger a HTTP POST
-    test_url = url_for('test_endpoint', _external=True)
-    uuid = client.application.config.get('DATASTORE').add_watch(url=test_url)
-    client.get(url_for("ui.form_watch_checknow"), follow_redirects=True)
-    time.sleep(2)
-
-    set_longer_modified_response(datastore_path=datastore_path)
-    client.get(url_for("ui.form_watch_checknow"), follow_redirects=True)
-    wait_for_all_checks(client)
-
-    time.sleep(3)
-
-    msg_raw = get_last_message_from_smtp_server()
-    assert len(msg_raw) >= 1
-    #time.sleep(60)
-    # Parse the email properly using Python's email library
-    msg = message_from_string(msg_raw, policy=email_policy)
-    # Subject/title got marked up
-    subject = msg['subject']
-    # Subject should always be plaintext and never marked up to anything else
-    assert REMOVED_PLACEMARKER_OPEN not in subject
-    assert CHANGED_PLACEMARKER_OPEN not in subject
-    assert ADDED_PLACEMARKER_OPEN not in subject
-    assert 'diff added didnt split' not in subject
-    assert '(changed) Which is across' in subject
-    assert 'PLACEMARKER' not in subject
-
-    # The email should be plain text only (not multipart)
-    assert not msg.is_multipart()
-    assert msg.get_content_type() == 'text/plain'
-
-    # Get the plain text content
-    text_content = msg.get_content()
-    assert '(added) So let\'s see what happens.\r\n' in text_content  # The plaintext part
-
-    # Should NOT contain HTML
-    assert '<br>' not in text_content  # We should not have HTML in plain text
-    delete_all_watches(client)
-
-
-
-def test_check_notification_html_color_format(client, live_server, measure_memory_usage, datastore_path):
-    set_original_response(datastore_path=datastore_path)
-
-
-    notification_url = f'mailto://changedetection@{smtp_test_server}:11025/?to=fff@home.com'
-
-    #####################
-    # Set this up for when we remove the notification from the watch, it should fallback with these details
-    res = client.post(
-        url_for("settings.settings_page"),
-        data={"application-notification_urls": notification_url,
-              "application-notification_title": "fallback-title {{watch_title}} - diff_added_lines_test : '{{ diff_added.splitlines()[0] if diff_added else 'diff added didnt split' }}' " + default_notification_title,
-              "application-notification_body": f"some text\n{default_notification_body}\nMore output test\n{ALL_MARKUP_TOKENS}",
-              "application-notification_format": 'htmlcolor',
-              "requests-time_between_check-minutes": 180,
-              'application-fetch_backend': "html_requests"},
-        follow_redirects=True
-    )
-
-    assert b"Settings updated." in res.data
-
-    # Add a watch and trigger a HTTP POST
-    test_url = url_for('test_endpoint', _external=True)
-    res = client.post(
-        url_for("ui.ui_views.form_quick_watch_add"),
-        data={"url": test_url, "tags": 'nice one'},
-        follow_redirects=True
-    )
-
-    assert b"Watch added" in res.data
-
-    wait_for_all_checks(client)
-    set_longer_modified_response(datastore_path=datastore_path)
-    time.sleep(2)
-
-    client.get(url_for("ui.form_watch_checknow"), follow_redirects=True)
-    wait_for_all_checks(client)
-
-    time.sleep(3)
-
-    msg_raw = get_last_message_from_smtp_server()
-    assert len(msg_raw) >= 1
-
-    # Parse the email properly using Python's email library
-    msg = message_from_string(msg_raw, policy=email_policy)
-    # Subject/title got marked up
-    subject = msg['subject']
-    # Subject should always be plaintext and never marked up to anything else
-    assert REMOVED_PLACEMARKER_OPEN not in subject
-    assert CHANGED_PLACEMARKER_OPEN not in subject
-    assert ADDED_PLACEMARKER_OPEN not in subject
-    assert 'diff added didnt split' not in subject
-    assert '(changed) Which is across' in subject
-    assert 'PLACEMARKER' not in subject
-    assert 'head title' in subject
-    assert "span" not in subject
-    assert 'background-color' not in subject
-
-
-    # The email should have two bodies (multipart/alternative with text/plain and text/html)
-    assert msg.is_multipart()
-    assert msg.get_content_type() == 'multipart/alternative'
-
-    # Get the parts
-    parts = list(msg.iter_parts())
-    assert len(parts) == 2
-
-    # First part should be text/plain (the auto-generated plaintext version)
-    text_part = parts[0]
-    assert text_part.get_content_type() == 'text/plain'
-    text_content = text_part.get_content()
-    assert 'So let\'s see what happens.\r\n' in text_content  # The plaintext part
-    assert '(added)' not in text_content # Because apprise only dumb converts the html to text
-
-    # Second part should be text/html with color styling
-    html_part = parts[1]
-    assert html_part.get_content_type() == 'text/html'
-    html_content = html_part.get_content()
-    assert HTML_CHANGED_STYLE or HTML_REMOVED_STYLE in html_content
-    assert HTML_ADDED_STYLE in html_content
-    assert '&lt;' not in html_content
-
-    assert 'some text<br>' in html_content
-    delete_all_watches(client)
-
-def test_check_notification_markdown_format(client, live_server, measure_memory_usage, datastore_path):
-    set_original_response(datastore_path=datastore_path)
-
-
-    notification_url = f'mailto://changedetection@{smtp_test_server}:11025/?to=fff@home.com'
-
-    #####################
-    # Set this up for when we remove the notification from the watch, it should fallback with these details
-    res = client.post(
-        url_for("settings.settings_page"),
-        data={"application-notification_urls": notification_url,
-              "application-notification_title": "fallback-title  diff_added_lines_test : '{{ diff_added.splitlines()[0] if diff_added else 'diff added didnt split' }}' " + default_notification_title,
-              "application-notification_body": "*header*\n\nsome text\n" + default_notification_body,
-              "application-notification_format": 'markdown',
-              "requests-time_between_check-minutes": 180,
-              'application-fetch_backend': "html_requests"},
-        follow_redirects=True
-    )
-
-    assert b"Settings updated." in res.data
-
-    # Add a watch and trigger a HTTP POST
-    test_url = url_for('test_endpoint', _external=True)
-    res = client.post(
-        url_for("ui.ui_views.form_quick_watch_add"),
-        data={"url": test_url, "tags": 'nice one'},
-        follow_redirects=True
-    )
-
-    assert b"Watch added" in res.data
-
-    wait_for_all_checks(client)
-    set_longer_modified_response(datastore_path=datastore_path)
-    time.sleep(2)
-
-    client.get(url_for("ui.form_watch_checknow"), follow_redirects=True)
-    wait_for_all_checks(client)
-
-    time.sleep(3)
-
-    msg_raw = get_last_message_from_smtp_server()
-    assert len(msg_raw) >= 1
-
-    # Parse the email properly using Python's email library
-    msg = message_from_string(msg_raw, policy=email_policy)
-
-    # The email should have two bodies (multipart/alternative with text/plain and text/html)
-    assert msg.is_multipart()
-    assert msg.get_content_type() == 'multipart/alternative'
-    subject = msg['subject']
-    # Subject should always be plaintext and never marked up to anything else
-    assert REMOVED_PLACEMARKER_OPEN not in subject
-    assert CHANGED_PLACEMARKER_OPEN not in subject
-    assert ADDED_PLACEMARKER_OPEN not in subject
-    assert 'diff added didnt split' not in subject
-    assert '(changed) Which is across' in subject
-
-
-    # Get the parts
-    parts = list(msg.iter_parts())
-    assert len(parts) == 2
-
-    # First part should be text/plain (the auto-generated plaintext version)
-    text_part = parts[0]
-    assert text_part.get_content_type() == 'text/plain'
-    text_content = text_part.get_content()
-    # We wont see anything in the "FALLBACK" text but that's OK (no added/strikethrough etc)
-    assert 'So let\'s see what happens.\r\n' in text_content  # The plaintext part
-
-
-    # Second part should be text/html and roughly converted from markdown to HTML
-    html_part = parts[1]
-    assert html_part.get_content_type() == 'text/html'
-    html_content = html_part.get_content()
-    assert '<p><em>header</em></p>' in html_content
-    assert '<strong>So let\'s see what happens.</strong><br />' in html_content # Additions are <strong> in markdown
-    # the '<br />' will come from apprises conversion, not from our code, we would rather use '<br>' correctly
-    # the '<br />' is actually a nice way to know if apprise done the conversion.
-
-    delete_all_watches(client)
-
-# Custom notification body with HTML, that is either sent as HTML or rendered to plaintext and sent
-def test_check_notification_email_formats_default_Text_override_HTML(client, live_server, measure_memory_usage, datastore_path):
+def test_check_notification_email_formats_default_Text_override_HTML(client, live_server, measure_memory_usage):
+    ##  live_server_setup(live_server) # Setup on conftest per function
 
     # HTML problems? see this
     # https://github.com/caronc/apprise/issues/633
 
-    set_original_response(datastore_path=datastore_path)
-
+    set_original_response()
     notification_url = f'mailto://changedetection@{smtp_test_server}:11025/?to=fff@home.com'
     notification_body = f"""<!DOCTYPE html>
 <html lang="en">
@@ -368,378 +115,7 @@ def test_check_notification_email_formats_default_Text_override_HTML(client, liv
         data={"application-notification_urls": notification_url,
               "application-notification_title": "fallback-title " + default_notification_title,
               "application-notification_body": notification_body,
-              "application-notification_format": 'text',
-              "requests-time_between_check-minutes": 180,
-              'application-fetch_backend': "html_requests"},
-        follow_redirects=True
-    )
-    assert b"Settings updated." in res.data
-
-    # Add a watch and trigger a HTTP POST
-    test_url = url_for('test_endpoint',content_type="text/html", _external=True)
-    res = client.post(
-        url_for("ui.ui_views.form_quick_watch_add"),
-        data={"url": test_url, "tags": 'nice one'},
-        follow_redirects=True
-    )
-
-    assert b"Watch added" in res.data
-
-    #################################### FIRST SITUATION, PLAIN TEXT NOTIFICATION IS WANTED BUT WE HAVE HTML IN OUR TEMPLATE AND CONTENT ##########
-    wait_for_all_checks(client)
-    set_longer_modified_response(datastore_path=datastore_path)
-    time.sleep(2)
-    client.get(url_for("ui.form_watch_checknow"), follow_redirects=True)
-    wait_for_all_checks(client)
-
-    time.sleep(3)
-    msg_raw = get_last_message_from_smtp_server()
-    assert len(msg_raw) >= 1
-    #    with open('/tmp/m.txt', 'w') as f:
-    #        f.write(msg_raw)
-
-    # Parse the email properly using Python's email library
-    msg = message_from_string(msg_raw, policy=email_policy)
-
-    # The email should not have two bodies, should be TEXT only
-    assert not msg.is_multipart()
-    assert msg.get_content_type() == 'text/plain'
-
-    # Get the plain text content
-    text_content = msg.get_content()
-    assert '(added) So let\'s see what happens.\r\n' in text_content  # The plaintext part
-    assert '<!DOCTYPE html>' in text_content # even tho they added html, they selected plaintext so it should have not got converted
-
-
-    #################################### SECOND SITUATION, HTML IS CORRECTLY PASSED THROUGH TO THE EMAIL ####################
-    set_original_response(datastore_path=datastore_path)
-
-    # Now override as HTML format
-    res = client.post(
-        url_for("ui.ui_edit.edit_page", uuid="first"),
-        data={
-            "url": test_url,
-            "notification_format": 'html',
-            'fetch_backend': "html_requests",
-            "time_between_check_use_default": "y"},
-        follow_redirects=True
-    )
-    assert b"Updated watch." in res.data
-    wait_for_all_checks(client)
-
-    time.sleep(3)
-    msg_raw = get_last_message_from_smtp_server()
-    assert len(msg_raw) >= 1
-
-    # Parse the email properly using Python's email library
-    msg = message_from_string(msg_raw, policy=email_policy)
-
-    # The email should have two bodies (multipart/alternative)
-    assert msg.is_multipart()
-    assert msg.get_content_type() == 'multipart/alternative'
-
-    # Get the parts
-    parts = list(msg.iter_parts())
-    assert len(parts) == 2
-
-    # First part should be text/plain
-    text_part = parts[0]
-    assert text_part.get_content_type() == 'text/plain'
-    text_content = text_part.get_content()
-    assert '(removed) So let\'s see what happens.\r\n' in text_content  # The plaintext part
-
-    # Second part should be text/html
-    html_part = parts[1]
-    assert html_part.get_content_type() == 'text/html'
-    html_content = html_part.get_content()
-    assert '(removed) So let\'s see what happens.' in html_content  # the html part
-    assert '&lt;!DOCTYPE html' not in html_content
-    assert '<!DOCTYPE html' in html_content # Our original template is working correctly
-
-    # https://github.com/dgtlmoon/changedetection.io/issues/2103
-    assert '<h1>Test</h1>' in html_content
-    assert '&lt;' not in html_content
-
-    delete_all_watches(client)
-
-def test_check_plaintext_document_plaintext_notification_smtp(client, live_server, measure_memory_usage, datastore_path):
-    """When following a plaintext document, notification in Plain Text format is sent correctly"""
-    import os
-
-    with open(os.path.join(datastore_path, "endpoint-content.txt"), "w") as f:
-        f.write("Some nice plain text\nwhich we add some extra data\nover here\n")
-
-    notification_url = f'mailto://changedetection@{smtp_test_server}:11025/?to=fff@home.com'
-    notification_body = f"""{default_notification_body}"""
-
-    #####################
-    # Set this up for when we remove the notification from the watch, it should fallback with these details
-    res = client.post(
-        url_for("settings.settings_page"),
-        data={"application-notification_urls": notification_url,
-              "application-notification_title": "fallback-title " + default_notification_title,
-              "application-notification_body": f"{notification_body}\nMore output test\n{ALL_MARKUP_TOKENS}",
-              "application-notification_format": 'text',
-              "requests-time_between_check-minutes": 180,
-              'application-fetch_backend': "html_requests"},
-        follow_redirects=True
-    )
-    assert b"Settings updated." in res.data
-
-    # Add our URL to the import page
-    test_url = url_for('test_endpoint', content_type="text/plain", _external=True)
-    uuid = client.application.config.get('DATASTORE').add_watch(url=test_url)
-    client.get(url_for("ui.form_watch_checknow"), follow_redirects=True)
-    wait_for_all_checks(client)
-
-    # Change the content
-    with open(os.path.join(datastore_path, "endpoint-content.txt"), "w") as f:
-        f.write("Some nice plain text\nwhich we add some extra data\nAnd let's talk about <title> tags\nover here\n")
-
-
-    time.sleep(1)
-    client.get(url_for("ui.form_watch_checknow"), follow_redirects=True)
-    wait_for_all_checks(client)
-
-    # Parse the email properly using Python's email library
-    msg = message_from_string(get_last_message_from_smtp_server(), policy=email_policy)
-
-    assert not msg.is_multipart()
-    assert msg.get_content_type() == 'text/plain'
-    body = msg.get_content()
-    # nothing is escaped, raw html stuff in text/plain
-    assert 'talk about <title> tags' in body
-    assert '(added)' in body
-    assert '<br' not in body
-    assert '&lt;' not in body
-    assert '<pre' not in body
-    delete_all_watches(client)
-
-def test_check_plaintext_document_html_notifications(client, live_server, measure_memory_usage, datastore_path):
-    """When following a plaintext document, notification in Plain Text format is sent correctly"""
-    import os
-
-    with open(os.path.join(datastore_path, "endpoint-content.txt"), "w") as f:
-        f.write("    Some nice plain text\nwhich we add some extra data\nover here\n")
-
-    notification_url = f'mailto://changedetection@{smtp_test_server}:11025/?to=fff@home.com'
-    notification_body = f"""{default_notification_body}"""
-
-    #####################
-    # Set this up for when we remove the notification from the watch, it should fallback with these details
-    res = client.post(
-        url_for("settings.settings_page"),
-        data={"application-notification_urls": notification_url,
-              "application-notification_title": "fallback-title " + default_notification_title,
-              "application-notification_body": f"{notification_body}\nMore output test\n{ALL_MARKUP_TOKENS}",
-              "application-notification_format": 'html',
-              "requests-time_between_check-minutes": 180,
-              'application-fetch_backend': "html_requests"},
-        follow_redirects=True
-    )
-    assert b"Settings updated." in res.data
-
-    # Add our URL to the import page
-    test_url = url_for('test_endpoint', content_type="text/plain", _external=True)
-    uuid = client.application.config.get('DATASTORE').add_watch(url=test_url)
-    client.get(url_for("ui.form_watch_checknow"), follow_redirects=True)
-    wait_for_all_checks(client)
-
-    # Change the content
-    with open(os.path.join(datastore_path, "endpoint-content.txt"), "w") as f:
-        f.write("    Some nice plain text\nwhich we add some extra data\nAnd let's talk about <title> tags\nover here\n")
-
-
-    time.sleep(2)
-    client.get(url_for("ui.form_watch_checknow"), follow_redirects=True)
-    wait_for_all_checks(client)
-
-    # Parse the email properly using Python's email library
-    msg = message_from_string(get_last_message_from_smtp_server(), policy=email_policy)
-
-
-    # The email should have two bodies (multipart/alternative)
-    assert msg.is_multipart()
-    assert msg.get_content_type() == 'multipart/alternative'
-
-    # Get the parts
-    parts = list(msg.iter_parts())
-    assert len(parts) == 2
-
-    # First part should be text/plain
-    text_part = parts[0]
-    assert text_part.get_content_type() == 'text/plain'
-    text_content = text_part.get_content()
-    html_part = parts[1]
-    assert html_part.get_content_type() == 'text/html'
-    html_content = html_part.get_content()
-
-
-    assert 'And let\'s talk about <title> tags\r\n' in text_content
-    assert '&lt;br' not in text_content
-    assert '<span' not in text_content
-
-
-    assert 'talk about <title>' not in html_content  # the html part, should have got marked up to &lt; etc
-    assert 'talk about &lt;title&gt;' in html_content
-    # Should be the HTML, but not HTML Color
-    assert 'background-color' not in html_content
-    assert '<br>(added) And let&#39;s talk about &lt;title&gt; tags<br>' in html_content
-    assert 'PLACEMARKER' not in html_content
-    assert '&lt;br' not in html_content
-    assert '<pre role="article"' in html_content # Should have got wrapped nicely in email_helpers.py
-
-    # And now for the whitespace retention
-    assert '&nbsp;&nbsp;&nbsp;&nbsp;Some nice plain text' in html_content
-    assert '(added) And let' in html_content # just to show a single whitespace didnt get touched
-    delete_all_watches(client)
-
-
-def test_check_plaintext_document_html_color_notifications(client, live_server, measure_memory_usage, datastore_path):
-    """When following a plaintext document, notification in Plain Text format is sent correctly"""
-    import os
-
-    with open(os.path.join(datastore_path, "endpoint-content.txt"), "w") as f:
-        f.write("Some nice plain text\nwhich we add some extra data\nover here\n")
-
-    notification_url = f'mailto://changedetection@{smtp_test_server}:11025/?to=fff@home.com'
-    notification_body = f"""{default_notification_body}"""
-
-    #####################
-    # Set this up for when we remove the notification from the watch, it should fallback with these details
-    res = client.post(
-        url_for("settings.settings_page"),
-        data={"application-notification_urls": notification_url,
-              "application-notification_title": "fallback-title " + default_notification_title,
-              "application-notification_body": f"{notification_body}\nMore output test\n{ALL_MARKUP_TOKENS}",
-              "application-notification_format": 'htmlcolor',
-              "requests-time_between_check-minutes": 180,
-              'application-fetch_backend': "html_requests"},
-        follow_redirects=True
-    )
-
-    assert b"Settings updated." in res.data
-
-    # Add our URL to the import page
-    test_url = url_for('test_endpoint', content_type="text/plain", _external=True)
-    uuid = client.application.config.get('DATASTORE').add_watch(url=test_url)
-    client.get(url_for("ui.form_watch_checknow"), follow_redirects=True)
-    wait_for_all_checks(client)
-
-    # Change the content
-    with open(os.path.join(datastore_path, "endpoint-content.txt"), "w") as f:
-        f.write("Some nice plain text\nwhich we add some extra data\nAnd let's talk about <title> tags\nover here\n")
-
-    time.sleep(1)
-    client.get(url_for("ui.form_watch_checknow"), follow_redirects=True)
-    wait_for_all_checks(client)
-
-    # Parse the email properly using Python's email library
-    msg = message_from_string(get_last_message_from_smtp_server(), policy=email_policy)
-
-    # The email should have two bodies (multipart/alternative)
-    assert msg.is_multipart()
-    assert msg.get_content_type() == 'multipart/alternative'
-
-    # Get the parts
-    parts = list(msg.iter_parts())
-    assert len(parts) == 2
-
-    # First part should be text/plain
-    text_part = parts[0]
-    assert text_part.get_content_type() == 'text/plain'
-    text_content = text_part.get_content()
-    html_part = parts[1]
-    assert html_part.get_content_type() == 'text/html'
-    html_content = html_part.get_content()
-
-
-    assert 'And let\'s talk about <title> tags\r\n' in text_content
-    assert '&lt;br' not in text_content
-    assert '<span' not in text_content
-
-    assert 'talk about <title>' not in html_content  # the html part, should have got marked up to &lt; etc
-    assert 'talk about &lt;title&gt;' in html_content
-    # Should be the HTML, but not HTML Color
-    assert 'background-color' in html_content
-    assert '(added) And let' not in html_content
-    assert '&lt;br' not in html_content
-    assert '<br>' in html_content
-    assert '<pre role="article"' in html_content # Should have got wrapped nicely in email_helpers.py
-    delete_all_watches(client)
-
-def test_check_html_document_plaintext_notification(client, live_server, measure_memory_usage, datastore_path):
-    """When following a HTML document, notification in Plain Text format is sent correctly"""
-    import os
-
-    with open(os.path.join(datastore_path, "endpoint-content.txt"), "w") as f:
-        f.write("<html><body>some stuff<br>and more stuff<br>and even more stuff<br></body></html>")
-
-    notification_url = f'mailto://changedetection@{smtp_test_server}:11025/?to=fff@home.com'
-    notification_body = f"""{default_notification_body}"""
-
-    #####################
-    # Set this up for when we remove the notification from the watch, it should fallback with these details
-    res = client.post(
-        url_for("settings.settings_page"),
-        data={"application-notification_urls": notification_url,
-              "application-notification_title": "fallback-title " + default_notification_title,
-              "application-notification_body": f"{notification_body}\nMore output test\n{ALL_MARKUP_TOKENS}",
-              "application-notification_format": 'text',
-              "requests-time_between_check-minutes": 180,
-              'application-fetch_backend': "html_requests"},
-        follow_redirects=True
-    )
-
-    assert b"Settings updated." in res.data
-
-    # Add our URL to the import page
-    test_url = url_for('test_endpoint', content_type="text/html", _external=True)
-    uuid = client.application.config.get('DATASTORE').add_watch(url=test_url)
-    client.get(url_for("ui.form_watch_checknow"), follow_redirects=True)
-    wait_for_all_checks(client)
-
-    with open(os.path.join(datastore_path, "endpoint-content.txt"), "w") as f:
-        f.write("<html><body>sxome stuff<br>and more stuff<br>lets slip this in<br>and this in<br>and even more stuff<br>&lt;tag&gt;</body></html>")
-
-    time.sleep(0.1)
-    client.get(url_for("ui.form_watch_checknow"), follow_redirects=True)
-    wait_for_all_checks(client)
-
-
-    # Parse the email properly using Python's email library
-    msg = message_from_string(get_last_message_from_smtp_server(), policy=email_policy)
-
-    assert not msg.is_multipart()
-    assert msg.get_content_type() == 'text/plain'
-    body = msg.get_content()
-    assert '<tag>' in body # Should have got converted from original HTML to plaintext
-    assert '(changed) some stuff\r\n' in body
-    assert 'PLACEMARKER' not in body
-    assert '(into) sxome stuff\r\n' in body
-    assert '(added) lets slip this in\r\n' in body
-    assert '(added) and this in\r\n' in body
-    assert '&nbsp;' not in body
-
-
-    delete_all_watches(client)
-
-
-def test_check_html_notification_with_apprise_format_is_html(client, live_server, measure_memory_usage, datastore_path):
-    ##  live_server_setup(live_server) # Setup on conftest per function
-    set_original_response(datastore_path=datastore_path)
-
-
-    notification_url = f'mailto://changedetection@{smtp_test_server}:11025/?to=fff@home.com&format=html'
-
-    #####################
-    # Set this up for when we remove the notification from the watch, it should fallback with these details
-    res = client.post(
-        url_for("settings.settings_page"),
-        data={"application-notification_urls": notification_url,
-              "application-notification_title": "fallback-title " + default_notification_title,
-              "application-notification_body": "some text\nfallback-body<br> " + default_notification_body,
-              "application-notification_format": 'html',
+              "application-notification_format": 'Text',
               "requests-time_between_check-minutes": 180,
               'application-fetch_backend': "html_requests"},
         follow_redirects=True
@@ -757,40 +133,49 @@ def test_check_html_notification_with_apprise_format_is_html(client, live_server
     assert b"Watch added" in res.data
 
     wait_for_all_checks(client)
-    set_longer_modified_response(datastore_path=datastore_path)
+    set_longer_modified_response()
     time.sleep(2)
-
     client.get(url_for("ui.form_watch_checknow"), follow_redirects=True)
     wait_for_all_checks(client)
 
     time.sleep(3)
+    msg = get_last_message_from_smtp_server()
+    assert len(msg) >= 1
+    #    with open('/tmp/m.txt', 'w') as f:
+    #        f.write(msg)
 
-    msg_raw = get_last_message_from_smtp_server()
-    assert len(msg_raw) >= 1
+    # The email should not have two bodies, should be TEXT only
 
-    # Parse the email properly using Python's email library
-    msg = message_from_string(msg_raw, policy=email_policy)
+    assert 'Content-Type: text/plain' in msg
+    assert '(added) So let\'s see what happens.\r\n' in msg  # The plaintext part with \r\n
 
-    # The email should have two bodies (multipart/alternative with text/plain and text/html)
-    assert msg.is_multipart()
-    assert msg.get_content_type() == 'multipart/alternative'
+    set_original_response()
+    # Now override as HTML format
+    res = client.post(
+        url_for("ui.ui_edit.edit_page", uuid="first"),
+        data={
+            "url": test_url,
+            "notification_format": 'HTML',
+            'fetch_backend': "html_requests",
+            "time_between_check_use_default": "y"},
+        follow_redirects=True
+    )
+    assert b"Updated watch." in res.data
+    wait_for_all_checks(client)
 
-    # Get the parts
-    parts = list(msg.iter_parts())
-    assert len(parts) == 2
+    time.sleep(3)
+    msg = get_last_message_from_smtp_server()
+    assert len(msg) >= 1
 
-    # First part should be text/plain (the auto-generated plaintext version)
-    text_part = parts[0]
-    assert text_part.get_content_type() == 'text/plain'
-    text_content = text_part.get_content()
-    assert '(added) So let\'s see what happens.\r\n' in text_content  # The plaintext part
-    assert 'fallback-body\r\n' in text_content  # The plaintext part
+    # The email should have two bodies, and the text/html part should be <br>
+    assert 'Content-Type: text/plain' in msg
+    assert '(removed) So let\'s see what happens.\r\n' in msg  # The plaintext part with \n
+    assert 'Content-Type: text/html' in msg
+    assert '(removed) So let\'s see what happens.<br>' in msg  # the html part
 
-    # Second part should be text/html
-    html_part = parts[1]
-    assert html_part.get_content_type() == 'text/html'
-    html_content = html_part.get_content()
-    assert 'some text<br>' in html_content  # We converted \n from the notification body
-    assert 'fallback-body<br>' in html_content  # kept the original <br>
-    assert '(added) So let\'s see what happens.<br>' in html_content  # the html part
-    delete_all_watches(client)
+    # https://github.com/dgtlmoon/changedetection.io/issues/2103
+    assert '<h1>Test</h1>' in msg
+    assert '&lt;' not in msg
+    assert 'Content-Type: text/html' in msg
+
+    delete_all_watches(client)

changedetectionio/tests/test_access_control.py

@@ -2,7 +2,7 @@ from .util import live_server_setup, wait_for_all_checks
 from flask import url_for
 import time
 
-def test_check_access_control(app, client, live_server, measure_memory_usage, datastore_path):
+def test_check_access_control(app, client, live_server, measure_memory_usage):
     # Still doesnt work, but this is closer.
    #  live_server_setup(live_server) # Setup on conftest per function
 

changedetectionio/tests/test_add_replace_remove_filter.py

@@ -1,16 +1,12 @@
 #!/usr/bin/env python3
 
 import os.path
-import os
 
 from flask import url_for
 from .util import live_server_setup, wait_for_all_checks, wait_for_notification_endpoint_output, delete_all_watches
 import time
 
-from ..diff import ADDED_PLACEMARKER_OPEN
-
-
-def set_original(datastore_path, excluding=None, add_line=None):
+def set_original(excluding=None, add_line=None):
     test_return_data = """<html>
      <body>
      <p>Some initial text</p>
@@ -36,16 +32,16 @@ def set_original(datastore_path, excluding=None, add_line=None):
 
         test_return_data = output
 
-    with open(os.path.join(datastore_path, "endpoint-content.txt"), "w") as f:
+    with open("test-datastore/endpoint-content.txt", "w") as f:
         f.write(test_return_data)
 
-# def test_setup(client, live_server, measure_memory_usage, datastore_path):
+# def test_setup(client, live_server, measure_memory_usage):
    #  live_server_setup(live_server) # Setup on conftest per function
 
-def test_check_removed_line_contains_trigger(client, live_server, measure_memory_usage, datastore_path):
+def test_check_removed_line_contains_trigger(client, live_server, measure_memory_usage):
 
     # Give the endpoint time to spin up
-    set_original(datastore_path=datastore_path)
+    set_original()
     # Add our URL to the import page
     test_url = url_for('test_endpoint', _external=True)
     uuid = client.application.config.get('DATASTORE').add_watch(url=test_url)
@@ -65,10 +61,9 @@ def test_check_removed_line_contains_trigger(client, live_server, measure_memory
               "time_between_check_use_default": "y"},
         follow_redirects=True
     )
-
     assert b"Updated watch." in res.data
     wait_for_all_checks(client)
-    set_original(excluding='Something irrelevant', datastore_path=datastore_path)
+    set_original(excluding='Something irrelevant')
 
     # A line thats not the trigger should not trigger anything
     res = client.get(url_for("ui.form_watch_checknow"), follow_redirects=True)
@@ -79,7 +74,7 @@ def test_check_removed_line_contains_trigger(client, live_server, measure_memory
     assert b'has-unread-changes' not in res.data
 
     # The trigger line is REMOVED,  this should trigger
-    set_original(excluding='The golden line', datastore_path=datastore_path)
+    set_original(excluding='The golden line')
 
     # Check in the processor here what's going on, its triggering empty-reply and no change.
     client.get(url_for("ui.form_watch_checknow"), follow_redirects=True)
@@ -94,7 +89,7 @@ def test_check_removed_line_contains_trigger(client, live_server, measure_memory
     time.sleep(0.2)
 
     time.sleep(1)
-    set_original(excluding=None, datastore_path=datastore_path)
+    set_original(excluding=None)
     client.get(url_for("ui.form_watch_checknow"), follow_redirects=True)
     wait_for_all_checks(client)
     time.sleep(1)
@@ -102,7 +97,7 @@ def test_check_removed_line_contains_trigger(client, live_server, measure_memory
     assert b'has-unread-changes' not in res.data
 
     # Remove it again, and we should get a trigger
-    set_original(excluding='The golden line', datastore_path=datastore_path)
+    set_original(excluding='The golden line')
     client.get(url_for("ui.form_watch_checknow"), follow_redirects=True)
     wait_for_all_checks(client)
     res = client.get(url_for("watchlist.index"))
@@ -111,7 +106,7 @@ def test_check_removed_line_contains_trigger(client, live_server, measure_memory
     delete_all_watches(client)
 
 
-def test_check_add_line_contains_trigger(client, live_server, measure_memory_usage, datastore_path):
+def test_check_add_line_contains_trigger(client, live_server, measure_memory_usage):
     
     delete_all_watches(client)
     time.sleep(1)
@@ -126,7 +121,6 @@ def test_check_add_line_contains_trigger(client, live_server, measure_memory_usa
               "application-notification_body": 'triggered text was -{{triggered_text}}- ### 网站监测 内容更新了 ####',
               # https://github.com/caronc/apprise/wiki/Notify_Custom_JSON#get-parameter-manipulation
               "application-notification_urls": test_notification_url,
-              "application-notification_format": 'text',
               "application-minutes_between_check": 180,
               "application-fetch_backend": "html_requests"
               },
@@ -134,7 +128,7 @@ def test_check_add_line_contains_trigger(client, live_server, measure_memory_usa
     )
     assert b'Settings updated' in res.data
 
-    set_original(datastore_path=datastore_path)
+    set_original()
     # Add our URL to the import page
     test_url = url_for('test_endpoint', _external=True)
     uuid = client.application.config.get('DATASTORE').add_watch(url=test_url)
@@ -157,7 +151,7 @@ def test_check_add_line_contains_trigger(client, live_server, measure_memory_usa
     )
     assert b"Updated watch." in res.data
     wait_for_all_checks(client)
-    set_original(excluding='Something irrelevant', datastore_path=datastore_path)
+    set_original(excluding='Something irrelevant')
 
     # A line thats not the trigger should not trigger anything
     res = client.get(url_for("ui.form_watch_checknow"), follow_redirects=True)
@@ -168,7 +162,7 @@ def test_check_add_line_contains_trigger(client, live_server, measure_memory_usa
     assert b'has-unread-changes' not in res.data
 
     # The trigger line is ADDED,  this should trigger
-    set_original(add_line='<p>Oh yes please</p>', datastore_path=datastore_path)
+    set_original(add_line='<p>Oh yes please</p>')
     client.get(url_for("ui.form_watch_checknow"), follow_redirects=True)
     wait_for_all_checks(client)
     res = client.get(url_for("watchlist.index"))
@@ -176,11 +170,10 @@ def test_check_add_line_contains_trigger(client, live_server, measure_memory_usa
     assert b'has-unread-changes' in res.data
 
     # Takes a moment for apprise to fire
-    wait_for_notification_endpoint_output(datastore_path=datastore_path)
-    assert os.path.isfile(os.path.join(datastore_path, "notification.txt")), "Notification fired because I can see the output file"
-    with open(os.path.join(datastore_path, "notification.txt"), 'rb') as f:
+    wait_for_notification_endpoint_output()
+    assert os.path.isfile("test-datastore/notification.txt"), "Notification fired because I can see the output file"
+    with open("test-datastore/notification.txt", 'rb') as f:
         response = f.read()
-        assert ADDED_PLACEMARKER_OPEN.encode('utf-8') not in response #  _apply_diff_filtering shouldnt add something here
         assert b'-Oh yes please' in response
         assert '网站监测 内容更新了'.encode('utf-8') in response
 

changedetectionio/tests/test_api.py

@@ -3,13 +3,12 @@
 import time
 from flask import url_for
 from .util import live_server_setup, wait_for_all_checks, delete_all_watches
-import os
 
 import json
 import uuid
 
 
-def set_original_response(datastore_path):
+def set_original_response():
     test_return_data = """<html>
        <body>
      Some initial text<br>
@@ -22,12 +21,12 @@ def set_original_response(datastore_path):
      </html>
     """
 
-    with open(os.path.join(datastore_path, "endpoint-content.txt"), "w") as f:
+    with open("test-datastore/endpoint-content.txt", "w") as f:
         f.write(test_return_data)
     return None
 
 
-def set_modified_response(datastore_path):
+def set_modified_response():
     test_return_data = """<html>
        <body>
      Some initial text<br>
@@ -40,7 +39,7 @@ def set_modified_response(datastore_path):
      </html>
     """
 
-    with open(os.path.join(datastore_path, "endpoint-content.txt"), "w") as f:
+    with open("test-datastore/endpoint-content.txt", "w") as f:
         f.write(test_return_data)
 
     return None
@@ -53,17 +52,17 @@ def is_valid_uuid(val):
         return False
 
 
-# def test_setup(client, live_server, measure_memory_usage, datastore_path):
+# def test_setup(client, live_server, measure_memory_usage):
    #  live_server_setup(live_server) # Setup on conftest per function
 
 
-def test_api_simple(client, live_server, measure_memory_usage, datastore_path):
+def test_api_simple(client, live_server, measure_memory_usage):
     
 
     api_key = live_server.app.config['DATASTORE'].data['settings']['application'].get('api_access_token')
 
     # Create a watch
-    set_original_response(datastore_path=datastore_path)
+    set_original_response()
 
     # Validate bad URL
     test_url = url_for('test_endpoint', _external=True )
@@ -112,7 +111,7 @@ def test_api_simple(client, live_server, measure_memory_usage, datastore_path):
     time.sleep(1)
     wait_for_all_checks(client)
 
-    set_modified_response(datastore_path=datastore_path)
+    set_modified_response()
     # Trigger recheck of all ?recheck_all=1
     client.get(
         url_for("createwatch", recheck_all='1'),
@@ -138,7 +137,6 @@ def test_api_simple(client, live_server, measure_memory_usage, datastore_path):
         url_for("watchhistory", uuid=watch_uuid),
         headers={'x-api-key': api_key},
     )
-    watch_history = res.json
     assert len(res.json) == 2, "Should have two history entries (the original and the changed)"
 
     # Fetch a snapshot by timestamp, check the right one was found
@@ -164,20 +162,6 @@ def test_api_simple(client, live_server, measure_memory_usage, datastore_path):
     assert b'which has this one new line' in res.data
     assert b'<div id' in res.data
 
-
-    # Fetch the difference between two versions
-    res = client.get(
-        url_for("watchhistorydiff", uuid=watch_uuid, from_timestamp='previous', to_timestamp='latest'),
-        headers={'x-api-key': api_key},
-    )
-    assert b'(changed) Which is across' in res.data
-    res = client.get(
-        url_for("watchhistorydiff", uuid=watch_uuid, from_timestamp='previous', to_timestamp='latest')+'?format=htmlcolor',
-        headers={'x-api-key': api_key},
-    )
-    assert b'aria-label="Changed text" title="Changed text">Which is across multiple lines' in res.data
-
-
     # Fetch the whole watch
     res = client.get(
         url_for("watch", uuid=watch_uuid),
@@ -246,10 +230,6 @@ def test_api_simple(client, live_server, measure_memory_usage, datastore_path):
     assert res.json.get('notification_muted') == 0
     ######################################################
 
-
-
-
-
     # Finally delete the watch
     res = client.delete(
         url_for("watch", uuid=watch_uuid),
@@ -264,7 +244,7 @@ def test_api_simple(client, live_server, measure_memory_usage, datastore_path):
     )
     assert len(res.json) == 0, "Watch list should be empty"
 
-def test_access_denied(client, live_server, measure_memory_usage, datastore_path):
+def test_access_denied(client, live_server, measure_memory_usage):
     # `config_api_token_enabled` Should be On by default
     res = client.get(
         url_for("createwatch")
@@ -309,11 +289,11 @@ def test_access_denied(client, live_server, measure_memory_usage, datastore_path
     )
     assert b"Settings updated." in res.data
 
-def test_api_watch_PUT_update(client, live_server, measure_memory_usage, datastore_path):
+def test_api_watch_PUT_update(client, live_server, measure_memory_usage):
 
     api_key = live_server.app.config['DATASTORE'].data['settings']['application'].get('api_access_token')
     # Create a watch
-    set_original_response(datastore_path=datastore_path)
+    set_original_response()
     test_url = url_for('test_endpoint', _external=True)
 
     # Create new
@@ -390,7 +370,7 @@ def test_api_watch_PUT_update(client, live_server, measure_memory_usage, datasto
 
     ######################################################
 
-    # HTTP PUT try a field that doesn't exist
+    # HTTP PUT try a field that doenst exist
 
     # HTTP PUT an update
     res = client.put(
@@ -403,30 +383,18 @@ def test_api_watch_PUT_update(client, live_server, measure_memory_usage, datasto
     # Message will come from `flask_expects_json`
     assert b'Additional properties are not allowed' in res.data
 
-
-    # Try a XSS URL
-    res = client.put(
-        url_for("watch", uuid=watch_uuid),
-        headers={'x-api-key': api_key, 'content-type': 'application/json'},
-        data=json.dumps({
-            'url': 'javascript:alert(document.domain)'
-        }),
-    )
-    assert res.status_code == 400
-
     # Cleanup everything
     delete_all_watches(client)
 
 
-def test_api_import(client, live_server, measure_memory_usage, datastore_path):
+def test_api_import(client, live_server, measure_memory_usage):
 
     api_key = live_server.app.config['DATASTORE'].data['settings']['application'].get('api_access_token')
 
     res = client.post(
         url_for("import") + "?tag=import-test",
         data='https://website1.com\r\nhttps://website2.com',
-        # We removed  'content-type': 'text/plain', the Import API should assume this if none is set #3547 #3542
-        headers={'x-api-key': api_key},
+        headers={'x-api-key': api_key, 'content-type': 'text/plain'},
         follow_redirects=True
     )
 
@@ -440,7 +408,7 @@ def test_api_import(client, live_server, measure_memory_usage, datastore_path):
     res = client.get(url_for('tags.tags_overview_page'))
     assert b'import-test' in res.data
 
-def test_api_conflict_UI_password(client, live_server, measure_memory_usage, datastore_path):
+def test_api_conflict_UI_password(client, live_server, measure_memory_usage):
 
     
     api_key = live_server.app.config['DATASTORE'].data['settings']['application'].get('api_access_token')
@@ -458,7 +426,7 @@ def test_api_conflict_UI_password(client, live_server, measure_memory_usage, dat
     assert b"Password protection enabled." in res.data
 
     # Create a watch
-    set_original_response(datastore_path=datastore_path)
+    set_original_response()
     test_url = url_for('test_endpoint', _external=True)
 
     # Create new

changedetectionio/tests/test_api_notifications.py

@@ -4,7 +4,7 @@ from flask import url_for
 from .util import live_server_setup
 import json
 
-def test_api_notifications_crud(client, live_server, measure_memory_usage, datastore_path):
+def test_api_notifications_crud(client, live_server, measure_memory_usage):
    #  live_server_setup(live_server) # Setup on conftest per function
     api_key = live_server.app.config['DATASTORE'].data['settings']['application'].get('api_access_token')