Add extra test

.github/actions/extract-memory-report/action.yml

@@ -0,0 +1,51 @@
+name: 'Extract Memory Test Report'
+description: 'Extracts and displays memory test report from a container'
+inputs:
+  container-name:
+    description: 'Name of the container to extract logs from'
+    required: true
+  python-version:
+    description: 'Python version for artifact naming'
+    required: true
+  output-dir:
+    description: 'Directory to store output logs'
+    required: false
+    default: 'output-logs'
+
+runs:
+  using: "composite"
+  steps:
+    - name: Create output directory
+      shell: bash
+      run: |
+        mkdir -p ${{ inputs.output-dir }}
+
+    - name: Dump container log
+      shell: bash
+      run: |
+        echo "Disabled for now"
+#        return
+#        docker logs ${{ inputs.container-name }} > ${{ inputs.output-dir }}/${{ inputs.container-name }}-stdout-${{ inputs.python-version }}.txt 2>&1 || echo "Could not get stdout"
+#        docker logs ${{ inputs.container-name }} 2> ${{ inputs.output-dir }}/${{ inputs.container-name }}-stderr-${{ inputs.python-version }}.txt || echo "Could not get stderr"
+
+    - name: Extract and display memory test report
+      shell: bash
+      run: |
+        echo "Disabled for now"
+#        echo "Extracting test-memory.log from container..."
+#        docker cp ${{ inputs.container-name }}:/app/changedetectionio/test-memory.log ${{ inputs.output-dir }}/test-memory-${{ inputs.python-version }}.log || echo "test-memory.log not found in container"
+#
+#        echo "=== Top 10 Highest Peak Memory Tests ==="
+#        if [ -f ${{ inputs.output-dir }}/test-memory-${{ inputs.python-version }}.log ]; then
+#          grep "Peak memory:" ${{ inputs.output-dir }}/test-memory-${{ inputs.python-version }}.log | \
+#            sed 's/.*Peak memory: //' | \
+#            paste -d'|' - <(grep "Peak memory:" ${{ inputs.output-dir }}/test-memory-${{ inputs.python-version }}.log) | \
+#            sort -t'|' -k1 -nr | \
+#            cut -d'|' -f2 | \
+#            head -10
+#          echo ""
+#          echo "=== Full Memory Test Report ==="
+#          cat ${{ inputs.output-dir }}/test-memory-${{ inputs.python-version }}.log
+#        else
+#          echo "No memory log available"
+#        fi

.github/workflows/containers.yml

@@ -45,6 +45,14 @@ jobs:
         with:
           python-version: 3.11
 
+      - name: Cache pip packages
+        uses: actions/cache@v4
+        with:
+          path: ~/.cache/pip
+          key: ${{ runner.os }}-pip-${{ hashFiles('requirements.txt') }}
+          restore-keys: |
+            ${{ runner.os }}-pip-
+
       - name: Install dependencies
         run: |
           python -m pip install --upgrade pip

.github/workflows/pypi-release.yml

@@ -21,7 +21,7 @@ jobs:
     - name: Build a binary wheel and a source tarball
       run: python3 -m build
     - name: Store the distribution packages
-      uses: actions/upload-artifact@v4
+      uses: actions/upload-artifact@v5
       with:
         name: python-package-distributions
         path: dist/
@@ -34,7 +34,7 @@ jobs:
     - build
     steps:
     - name: Download all the dists
-      uses: actions/download-artifact@v5
+      uses: actions/download-artifact@v6
       with:
         name: python-package-distributions
         path: dist/
@@ -93,7 +93,7 @@ jobs:
 
     steps:
     - name: Download all the dists
-      uses: actions/download-artifact@v5
+      uses: actions/download-artifact@v6
       with:
         name: python-package-distributions
         path: dist/

.github/workflows/test-container-build.yml

@@ -50,6 +50,14 @@ jobs:
           with:
             python-version: 3.11
 
+        - name: Cache pip packages
+          uses: actions/cache@v4
+          with:
+            path: ~/.cache/pip
+            key: ${{ runner.os }}-pip-${{ hashFiles('requirements.txt') }}
+            restore-keys: |
+              ${{ runner.os }}-pip-
+
         # Just test that the build works, some libraries won't compile on ARM/rPi etc
         - name: Set up QEMU
           uses: docker/setup-qemu-action@v3

.github/workflows/test-stack-reusable-workflow.yml

@@ -15,141 +15,294 @@ on:
         default: false
 
 jobs:
-  test-application:
+  # Build the Docker image once and share it with all test jobs
+  build:
     runs-on: ubuntu-latest
     env:
       PYTHON_VERSION: ${{ inputs.python-version }}
     steps:
       - uses: actions/checkout@v5
 
-      # Mainly just for link/flake8
       - name: Set up Python ${{ env.PYTHON_VERSION }}
         uses: actions/setup-python@v6
         with:
           python-version: ${{ env.PYTHON_VERSION }}
 
+      - name: Cache pip packages
+        uses: actions/cache@v4
+        with:
+          path: ~/.cache/pip
+          key: ${{ runner.os }}-pip-py${{ env.PYTHON_VERSION }}-${{ hashFiles('requirements.txt') }}
+          restore-keys: |
+            ${{ runner.os }}-pip-py${{ env.PYTHON_VERSION }}-
+            ${{ runner.os }}-pip-
+
       - name: Build changedetection.io container for testing under Python ${{ env.PYTHON_VERSION }}
         run: |
           echo "---- Building for Python ${{ env.PYTHON_VERSION }} -----"
-          # Build a changedetection.io container and start testing inside
           docker build --build-arg PYTHON_VERSION=${{ env.PYTHON_VERSION }} --build-arg LOGGER_LEVEL=TRACE -t test-changedetectionio .
-          # Debug info
-          docker run test-changedetectionio  bash -c 'pip list'         
+          docker run test-changedetectionio bash -c 'pip list'
 
       - name: We should be Python ${{ env.PYTHON_VERSION }} ...
-        run: |         
-          docker run test-changedetectionio  bash -c 'python3 --version'
-
-      - name: Spin up ancillary testable services
         run: |
-          
-          docker network create changedet-network
-          
-          # Selenium
-          docker run --network changedet-network -d --hostname selenium  -p 4444:4444 --rm --shm-size="2g"  selenium/standalone-chrome:4
-          
-          # SocketPuppetBrowser + Extra for custom browser test
-          docker run --network changedet-network -d -e "LOG_LEVEL=TRACE" --cap-add=SYS_ADMIN --name sockpuppetbrowser --hostname sockpuppetbrowser --rm -p 3000:3000 dgtlmoon/sockpuppetbrowser:latest                    
-          docker run --network changedet-network -d -e "LOG_LEVEL=TRACE" --cap-add=SYS_ADMIN --name sockpuppetbrowser-custom-url --hostname sockpuppetbrowser-custom-url  -p 3001:3000 --rm dgtlmoon/sockpuppetbrowser:latest
+          docker run test-changedetectionio bash -c 'python3 --version'
 
-      - name: Spin up ancillary SMTP+Echo message test server
+      - name: Save Docker image
         run: |
-          # Debug SMTP server/echo message back server, telnet 11080 to it should immediately bounce back the most recent message that tried to send (then you can see if cdio tried to send, the format, etc)
-          # 11025 is the SMTP port for testing
-          # apprise example would be 'mailto://changedetection@localhost:11025/?to=fff@home.com  (it will also echo to STDOUT)
-          # telnet localhost 11080
-          docker run --network changedet-network -d -p 11025:11025 -p 11080:11080  --hostname mailserver test-changedetectionio  bash -c 'pip3 install aiosmtpd && python changedetectionio/tests/smtp/smtp-test-server.py'
-          docker ps
+          docker save test-changedetectionio -o /tmp/test-changedetectionio.tar
 
-      - name: Show docker container state and other debug info
+      - name: Upload Docker image artifact
+        uses: actions/upload-artifact@v5
+        with:
+          name: test-changedetectionio-${{ env.PYTHON_VERSION }}
+          path: /tmp/test-changedetectionio.tar
+          retention-days: 1
+
+  # Unit tests (lightweight, no ancillary services needed)
+  unit-tests:
+    runs-on: ubuntu-latest
+    needs: build
+    timeout-minutes: 10
+    env:
+      PYTHON_VERSION: ${{ inputs.python-version }}
+    steps:
+      - uses: actions/checkout@v5
+
+      - name: Download Docker image artifact
+        uses: actions/download-artifact@v5
+        with:
+          name: test-changedetectionio-${{ env.PYTHON_VERSION }}
+          path: /tmp
+
+      - name: Load Docker image
         run: |
-          set -x
-          echo "Running processes in docker..."
-          docker ps
+          docker load -i /tmp/test-changedetectionio.tar
 
       - name: Run Unit Tests
         run: |
-          # Unit tests
-          docker run test-changedetectionio  bash -c 'python3 -m unittest changedetectionio.tests.unit.test_notification_diff'
-          docker run test-changedetectionio  bash -c 'python3 -m unittest changedetectionio.tests.unit.test_watch_model'
-          docker run test-changedetectionio  bash -c 'python3 -m unittest changedetectionio.tests.unit.test_jinja2_security'
-          docker run test-changedetectionio  bash -c 'python3 -m unittest changedetectionio.tests.unit.test_semver'
+          docker run test-changedetectionio bash -c 'python3 -m unittest changedetectionio.tests.unit.test_notification_diff'
+          docker run test-changedetectionio bash -c 'python3 -m unittest changedetectionio.tests.unit.test_watch_model'
+          docker run test-changedetectionio bash -c 'python3 -m unittest changedetectionio.tests.unit.test_jinja2_security'
+          docker run test-changedetectionio bash -c 'python3 -m unittest changedetectionio.tests.unit.test_semver'
 
-      - name: Test built container with Pytest (generally as requests/plaintext fetching)
+  # Basic pytest tests with ancillary services
+  basic-tests:
+    runs-on: ubuntu-latest
+    needs: build
+    timeout-minutes: 25
+    env:
+      PYTHON_VERSION: ${{ inputs.python-version }}
+    steps:
+      - uses: actions/checkout@v5
+
+      - name: Download Docker image artifact
+        uses: actions/download-artifact@v5
+        with:
+          name: test-changedetectionio-${{ env.PYTHON_VERSION }}
+          path: /tmp
+
+      - name: Load Docker image
         run: |
-          # All tests
-          echo "run test with pytest"
-          # The default pytest logger_level is TRACE
-          # To change logger_level for pytest(test/conftest.py),
-          # append the docker option. e.g. '-e LOGGER_LEVEL=DEBUG'
-          docker run --name test-cdio-basic-tests --network changedet-network  test-changedetectionio  bash -c 'cd changedetectionio && ./run_basic_tests.sh'
+          docker load -i /tmp/test-changedetectionio.tar
 
-# PLAYWRIGHT/NODE-> CDP
-      - name: Playwright and SocketPuppetBrowser - Specific tests in built container
+      - name: Test built container with Pytest
         run: |
-          # Playwright via Sockpuppetbrowser fetch
-          # tests/visualselector/test_fetch_data.py will do browser steps  
-          docker run --rm -e "FLASK_SERVER_NAME=cdio" -e "PLAYWRIGHT_DRIVER_URL=ws://sockpuppetbrowser:3000" --network changedet-network --hostname=cdio test-changedetectionio  bash -c 'cd changedetectionio;pytest  -vv --capture=tee-sys --showlocals --tb=long --live-server-host=0.0.0.0 --live-server-port=5004 tests/fetchers/test_content.py'
-          docker run --rm -e "FLASK_SERVER_NAME=cdio" -e "PLAYWRIGHT_DRIVER_URL=ws://sockpuppetbrowser:3000" --network changedet-network --hostname=cdio test-changedetectionio  bash -c 'cd changedetectionio;pytest  -vv --capture=tee-sys --showlocals --tb=long --live-server-host=0.0.0.0 --live-server-port=5004 tests/test_errorhandling.py'
-          docker run --rm -e "FLASK_SERVER_NAME=cdio" -e "PLAYWRIGHT_DRIVER_URL=ws://sockpuppetbrowser:3000" --network changedet-network --hostname=cdio test-changedetectionio  bash -c 'cd changedetectionio;pytest  -vv --capture=tee-sys --showlocals --tb=long --live-server-host=0.0.0.0 --live-server-port=5004 tests/visualselector/test_fetch_data.py'
-          docker run --rm -e "FLASK_SERVER_NAME=cdio" -e "PLAYWRIGHT_DRIVER_URL=ws://sockpuppetbrowser:3000" --network changedet-network --hostname=cdio test-changedetectionio  bash -c 'cd changedetectionio;pytest  -vv --capture=tee-sys --showlocals --tb=long --live-server-host=0.0.0.0 --live-server-port=5004 tests/fetchers/test_custom_js_before_content.py'
+          docker network inspect changedet-network >/dev/null 2>&1 || docker network create changedet-network
+          docker run --name test-cdio-basic-tests --network changedet-network test-changedetectionio bash -c 'cd changedetectionio && ./run_basic_tests.sh'
 
+      - name: Extract memory report and logs
+        if: always()
+        uses: ./.github/actions/extract-memory-report
+        with:
+          container-name: test-cdio-basic-tests
+          python-version: ${{ env.PYTHON_VERSION }}
 
-      - name: Playwright and SocketPuppetBrowser - Headers and requests
-        run: |       
-          # Settings headers playwright tests - Call back in from Sockpuppetbrowser, check headers
-          docker run --name "changedet" --hostname changedet --rm -e "FLASK_SERVER_NAME=changedet" -e "PLAYWRIGHT_DRIVER_URL=ws://sockpuppetbrowser:3000?dumpio=true" --network changedet-network test-changedetectionio  bash -c 'find .; cd changedetectionio; pytest --live-server-host=0.0.0.0  --live-server-port=5004 tests/test_request.py; pwd;find .'
+      - name: Store test artifacts
+        if: always()
+        uses: actions/upload-artifact@v5
+        with:
+          name: test-cdio-basic-tests-output-py${{ env.PYTHON_VERSION }}
+          path: output-logs
 
-      - name: Playwright and SocketPuppetBrowser - Restock detection
-        run: |                            
-          # restock detection via playwright - added name=changedet here so that playwright and sockpuppetbrowser can connect to it
-          docker run --rm --name "changedet" -e "FLASK_SERVER_NAME=changedet" -e "PLAYWRIGHT_DRIVER_URL=ws://sockpuppetbrowser:3000" --network changedet-network test-changedetectionio  bash -c 'cd changedetectionio;pytest --live-server-port=5004 --live-server-host=0.0.0.0 tests/restock/test_restock.py'
+  # Playwright tests
+  playwright-tests:
+    runs-on: ubuntu-latest
+    needs: build
+    timeout-minutes: 10
+    env:
+      PYTHON_VERSION: ${{ inputs.python-version }}
+    steps:
+      - uses: actions/checkout@v5
 
-# STRAIGHT TO CDP
-      - name: Pyppeteer and SocketPuppetBrowser - Specific tests in built container
-        if: ${{ inputs.skip-pypuppeteer == false }}
+      - name: Download Docker image artifact
+        uses: actions/download-artifact@v5
+        with:
+          name: test-changedetectionio-${{ env.PYTHON_VERSION }}
+          path: /tmp
+
+      - name: Load Docker image
         run: |
-          # Playwright via Sockpuppetbrowser fetch 
-          docker run --rm -e "FLASK_SERVER_NAME=cdio" -e "FAST_PUPPETEER_CHROME_FETCHER=True" -e "PLAYWRIGHT_DRIVER_URL=ws://sockpuppetbrowser:3000" --network changedet-network --hostname=cdio test-changedetectionio  bash -c 'cd changedetectionio;pytest --live-server-host=0.0.0.0 --live-server-port=5004 tests/fetchers/test_content.py'
-          docker run --rm -e "FLASK_SERVER_NAME=cdio" -e "FAST_PUPPETEER_CHROME_FETCHER=True" -e "PLAYWRIGHT_DRIVER_URL=ws://sockpuppetbrowser:3000" --network changedet-network --hostname=cdio test-changedetectionio  bash -c 'cd changedetectionio;pytest --live-server-host=0.0.0.0 --live-server-port=5004 tests/test_errorhandling.py'
-          docker run --rm -e "FLASK_SERVER_NAME=cdio" -e "FAST_PUPPETEER_CHROME_FETCHER=True" -e "PLAYWRIGHT_DRIVER_URL=ws://sockpuppetbrowser:3000" --network changedet-network --hostname=cdio test-changedetectionio  bash -c 'cd changedetectionio;pytest --live-server-host=0.0.0.0 --live-server-port=5004 tests/visualselector/test_fetch_data.py'
-          docker run --rm -e "FLASK_SERVER_NAME=cdio" -e "FAST_PUPPETEER_CHROME_FETCHER=True" -e "PLAYWRIGHT_DRIVER_URL=ws://sockpuppetbrowser:3000" --network changedet-network --hostname=cdio test-changedetectionio  bash -c 'cd changedetectionio;pytest --live-server-host=0.0.0.0 --live-server-port=5004 tests/fetchers/test_custom_js_before_content.py'
+          docker load -i /tmp/test-changedetectionio.tar
 
-      - name: Pyppeteer and SocketPuppetBrowser - Headers and requests checks
-        if: ${{ inputs.skip-pypuppeteer == false }}
+      - name: Spin up ancillary services
         run: |
-          # Settings headers playwright tests - Call back in from Sockpuppetbrowser, check headers
-          docker run --name "changedet" --hostname changedet --rm  -e "FAST_PUPPETEER_CHROME_FETCHER=True" -e "FLASK_SERVER_NAME=changedet" -e "PLAYWRIGHT_DRIVER_URL=ws://sockpuppetbrowser:3000?dumpio=true" --network changedet-network test-changedetectionio  bash -c 'cd changedetectionio; pytest --live-server-host=0.0.0.0  --live-server-port=5004 tests/test_request.py'
+          docker network create changedet-network
+          docker run --network changedet-network -d -e "LOG_LEVEL=TRACE" --cap-add=SYS_ADMIN --name sockpuppetbrowser --hostname sockpuppetbrowser --rm -p 3000:3000 dgtlmoon/sockpuppetbrowser:latest
+          docker run --network changedet-network -d -e "LOG_LEVEL=TRACE" --cap-add=SYS_ADMIN --name sockpuppetbrowser-custom-url --hostname sockpuppetbrowser-custom-url -p 3001:3000 --rm dgtlmoon/sockpuppetbrowser:latest
 
-      - name: Pyppeteer and SocketPuppetBrowser - Restock detection
-        if: ${{ inputs.skip-pypuppeteer == false }}
-        run: |                            
-          # restock detection via playwright - added name=changedet here so that playwright and sockpuppetbrowser can connect to it
-          docker run --rm --name "changedet" -e "FLASK_SERVER_NAME=changedet"  -e "FAST_PUPPETEER_CHROME_FETCHER=True"  -e "PLAYWRIGHT_DRIVER_URL=ws://sockpuppetbrowser:3000" --network changedet-network test-changedetectionio  bash -c 'cd changedetectionio;pytest --live-server-port=5004 --live-server-host=0.0.0.0 tests/restock/test_restock.py'
+      - name: Playwright - Specific tests in built container
+        run: |
+          docker run --rm -e "FLASK_SERVER_NAME=cdio" -e "PLAYWRIGHT_DRIVER_URL=ws://sockpuppetbrowser:3000" --network changedet-network --hostname=cdio test-changedetectionio bash -c 'cd changedetectionio;pytest -vv --capture=tee-sys --showlocals --tb=long --live-server-host=0.0.0.0 --live-server-port=5004 tests/fetchers/test_content.py'
+          docker run --rm -e "FLASK_SERVER_NAME=cdio" -e "PLAYWRIGHT_DRIVER_URL=ws://sockpuppetbrowser:3000" --network changedet-network --hostname=cdio test-changedetectionio bash -c 'cd changedetectionio;pytest -vv --capture=tee-sys --showlocals --tb=long --live-server-host=0.0.0.0 --live-server-port=5004 tests/test_errorhandling.py'
+          docker run --rm -e "FLASK_SERVER_NAME=cdio" -e "PLAYWRIGHT_DRIVER_URL=ws://sockpuppetbrowser:3000" --network changedet-network --hostname=cdio test-changedetectionio bash -c 'cd changedetectionio;pytest -vv --capture=tee-sys --showlocals --tb=long --live-server-host=0.0.0.0 --live-server-port=5004 tests/visualselector/test_fetch_data.py'
+          docker run --rm -e "FLASK_SERVER_NAME=cdio" -e "PLAYWRIGHT_DRIVER_URL=ws://sockpuppetbrowser:3000" --network changedet-network --hostname=cdio test-changedetectionio bash -c 'cd changedetectionio;pytest -vv --capture=tee-sys --showlocals --tb=long --live-server-host=0.0.0.0 --live-server-port=5004 tests/fetchers/test_custom_js_before_content.py'
+
+      - name: Playwright - Headers and requests
+        run: |
+          docker run --name "changedet" --hostname changedet --rm -e "FLASK_SERVER_NAME=changedet" -e "PLAYWRIGHT_DRIVER_URL=ws://sockpuppetbrowser:3000?dumpio=true" --network changedet-network test-changedetectionio bash -c 'find .; cd changedetectionio; pytest --live-server-host=0.0.0.0 --live-server-port=5004 tests/test_request.py; pwd;find .'
+
+      - name: Playwright - Restock detection
+        run: |
+          docker run --rm --name "changedet" -e "FLASK_SERVER_NAME=changedet" -e "PLAYWRIGHT_DRIVER_URL=ws://sockpuppetbrowser:3000" --network changedet-network test-changedetectionio bash -c 'cd changedetectionio;pytest --live-server-port=5004 --live-server-host=0.0.0.0 tests/restock/test_restock.py'
+
+  # Pyppeteer tests
+  pyppeteer-tests:
+    runs-on: ubuntu-latest
+    needs: build
+    if: ${{ inputs.skip-pypuppeteer == false }}
+    timeout-minutes: 10
+    env:
+      PYTHON_VERSION: ${{ inputs.python-version }}
+    steps:
+      - uses: actions/checkout@v5
+
+      - name: Download Docker image artifact
+        uses: actions/download-artifact@v5
+        with:
+          name: test-changedetectionio-${{ env.PYTHON_VERSION }}
+          path: /tmp
+
+      - name: Load Docker image
+        run: |
+          docker load -i /tmp/test-changedetectionio.tar
+
+      - name: Spin up ancillary services
+        run: |
+          docker network create changedet-network
+          docker run --network changedet-network -d -e "LOG_LEVEL=TRACE" --cap-add=SYS_ADMIN --name sockpuppetbrowser --hostname sockpuppetbrowser --rm -p 3000:3000 dgtlmoon/sockpuppetbrowser:latest
+
+      - name: Pyppeteer - Specific tests in built container
+        run: |
+          docker run --rm -e "FLASK_SERVER_NAME=cdio" -e "FAST_PUPPETEER_CHROME_FETCHER=True" -e "PLAYWRIGHT_DRIVER_URL=ws://sockpuppetbrowser:3000" --network changedet-network --hostname=cdio test-changedetectionio bash -c 'cd changedetectionio;pytest --live-server-host=0.0.0.0 --live-server-port=5004 tests/fetchers/test_content.py'
+          docker run --rm -e "FLASK_SERVER_NAME=cdio" -e "FAST_PUPPETEER_CHROME_FETCHER=True" -e "PLAYWRIGHT_DRIVER_URL=ws://sockpuppetbrowser:3000" --network changedet-network --hostname=cdio test-changedetectionio bash -c 'cd changedetectionio;pytest --live-server-host=0.0.0.0 --live-server-port=5004 tests/test_errorhandling.py'
+          docker run --rm -e "FLASK_SERVER_NAME=cdio" -e "FAST_PUPPETEER_CHROME_FETCHER=True" -e "PLAYWRIGHT_DRIVER_URL=ws://sockpuppetbrowser:3000" --network changedet-network --hostname=cdio test-changedetectionio bash -c 'cd changedetectionio;pytest --live-server-host=0.0.0.0 --live-server-port=5004 tests/visualselector/test_fetch_data.py'
+          docker run --rm -e "FLASK_SERVER_NAME=cdio" -e "FAST_PUPPETEER_CHROME_FETCHER=True" -e "PLAYWRIGHT_DRIVER_URL=ws://sockpuppetbrowser:3000" --network changedet-network --hostname=cdio test-changedetectionio bash -c 'cd changedetectionio;pytest --live-server-host=0.0.0.0 --live-server-port=5004 tests/fetchers/test_custom_js_before_content.py'
+
+      - name: Pyppeteer - Headers and requests checks
+        run: |
+          docker run --name "changedet" --hostname changedet --rm -e "FAST_PUPPETEER_CHROME_FETCHER=True" -e "FLASK_SERVER_NAME=changedet" -e "PLAYWRIGHT_DRIVER_URL=ws://sockpuppetbrowser:3000?dumpio=true" --network changedet-network test-changedetectionio bash -c 'cd changedetectionio; pytest --live-server-host=0.0.0.0 --live-server-port=5004 tests/test_request.py'
+
+      - name: Pyppeteer - Restock detection
+        run: |
+          docker run --rm --name "changedet" -e "FLASK_SERVER_NAME=changedet" -e "FAST_PUPPETEER_CHROME_FETCHER=True" -e "PLAYWRIGHT_DRIVER_URL=ws://sockpuppetbrowser:3000" --network changedet-network test-changedetectionio bash -c 'cd changedetectionio;pytest --live-server-port=5004 --live-server-host=0.0.0.0 tests/restock/test_restock.py'
+
+  # Selenium tests
+  selenium-tests:
+    runs-on: ubuntu-latest
+    needs: build
+    timeout-minutes: 10
+    env:
+      PYTHON_VERSION: ${{ inputs.python-version }}
+    steps:
+      - uses: actions/checkout@v5
+
+      - name: Download Docker image artifact
+        uses: actions/download-artifact@v5
+        with:
+          name: test-changedetectionio-${{ env.PYTHON_VERSION }}
+          path: /tmp
+
+      - name: Load Docker image
+        run: |
+          docker load -i /tmp/test-changedetectionio.tar
+
+      - name: Spin up ancillary services
+        run: |
+          docker network create changedet-network
+          docker run --network changedet-network -d --hostname selenium -p 4444:4444 --rm --shm-size="2g" selenium/standalone-chrome:4
+          sleep 3
+
+      - name: Specific tests for headers and requests checks with Selenium
+        run: |
+
+          docker run --name "changedet" --hostname changedet --rm -e "FLASK_SERVER_NAME=changedet" -e "WEBDRIVER_URL=http://selenium:4444/wd/hub" --network changedet-network test-changedetectionio bash -c 'cd changedetectionio; pytest --live-server-host=0.0.0.0 --live-server-port=5004 tests/test_request.py'
 
-# SELENIUM
       - name: Specific tests in built container for Selenium
         run: |
-          # Selenium fetch
-          docker run --rm -e "WEBDRIVER_URL=http://selenium:4444/wd/hub" --network changedet-network test-changedetectionio  bash -c 'cd changedetectionio;pytest tests/fetchers/test_content.py && pytest tests/test_errorhandling.py'
+          docker run --rm -e "WEBDRIVER_URL=http://selenium:4444/wd/hub" --network changedet-network test-changedetectionio bash -c 'cd changedetectionio;pytest tests/fetchers/test_content.py && pytest tests/test_errorhandling.py'
 
-      - name: Specific tests in built container for headers and requests checks with Selenium
+
+  # SMTP tests
+  smtp-tests:
+    runs-on: ubuntu-latest
+    needs: build
+    timeout-minutes: 10
+    env:
+      PYTHON_VERSION: ${{ inputs.python-version }}
+    steps:
+      - uses: actions/checkout@v5
+
+      - name: Download Docker image artifact
+        uses: actions/download-artifact@v5
+        with:
+          name: test-changedetectionio-${{ env.PYTHON_VERSION }}
+          path: /tmp
+
+      - name: Load Docker image
         run: |
-          docker run --name "changedet" --hostname changedet --rm -e "FLASK_SERVER_NAME=changedet" -e "WEBDRIVER_URL=http://selenium:4444/wd/hub" --network changedet-network test-changedetectionio  bash -c 'cd changedetectionio; pytest --live-server-host=0.0.0.0  --live-server-port=5004 tests/test_request.py'
+          docker load -i /tmp/test-changedetectionio.tar
+
+      - name: Spin up SMTP test server
+        run: |
+          docker network create changedet-network
+          docker run --network changedet-network -d -p 11025:11025 -p 11080:11080 --hostname mailserver test-changedetectionio bash -c 'pip3 install aiosmtpd && python changedetectionio/tests/smtp/smtp-test-server.py'
 
-# OTHER STUFF
       - name: Test SMTP notification mime types
         run: |
-          # SMTP content types - needs the 'Debug SMTP server/echo message back server' container from above
-          # "mailserver" hostname defined above
           docker run --rm --network changedet-network test-changedetectionio bash -c 'cd changedetectionio;pytest tests/smtp/test_notification_smtp.py'
 
-      # @todo Add a test via playwright/puppeteer
-      # squid with auth is tested in run_proxy_tests.sh -> tests/proxy_list/test_select_custom_proxy.py
-      - name: Test proxy squid style interaction
+  # Proxy tests
+  proxy-tests:
+    runs-on: ubuntu-latest
+    needs: build
+    timeout-minutes: 10
+    env:
+      PYTHON_VERSION: ${{ inputs.python-version }}
+    steps:
+      - uses: actions/checkout@v5
+
+      - name: Download Docker image artifact
+        uses: actions/download-artifact@v5
+        with:
+          name: test-changedetectionio-${{ env.PYTHON_VERSION }}
+          path: /tmp
+
+      - name: Load Docker image
+        run: |
+          docker load -i /tmp/test-changedetectionio.tar
+
+      - name: Spin up services
+        run: |
+          docker network create changedet-network
+          docker run --network changedet-network -d --hostname selenium -p 4444:4444 --rm --shm-size="2g" selenium/standalone-chrome:4
+          docker run --network changedet-network -d -e "LOG_LEVEL=TRACE" --cap-add=SYS_ADMIN --name sockpuppetbrowser --hostname sockpuppetbrowser --rm -p 3000:3000 dgtlmoon/sockpuppetbrowser:latest
+          docker run --network changedet-network -d -e "LOG_LEVEL=TRACE" --cap-add=SYS_ADMIN --name sockpuppetbrowser-custom-url --hostname sockpuppetbrowser-custom-url -p 3001:3000 --rm dgtlmoon/sockpuppetbrowser:latest
+
+      - name: Test proxy Squid style interaction
         run: |
           cd changedetectionio
           ./run_proxy_tests.sh
+          docker ps
           cd ..
 
       - name: Test proxy SOCKS5 style interaction
@@ -158,28 +311,65 @@ jobs:
           ./run_socks_proxy_tests.sh
           cd ..
 
+  # Custom browser URL tests
+  custom-browser-tests:
+    runs-on: ubuntu-latest
+    needs: build
+    timeout-minutes: 10
+    env:
+      PYTHON_VERSION: ${{ inputs.python-version }}
+    steps:
+      - uses: actions/checkout@v5
+
+      - name: Download Docker image artifact
+        uses: actions/download-artifact@v5
+        with:
+          name: test-changedetectionio-${{ env.PYTHON_VERSION }}
+          path: /tmp
+
+      - name: Load Docker image
+        run: |
+          docker load -i /tmp/test-changedetectionio.tar
+
+      - name: Spin up ancillary services
+        run: |
+          docker network create changedet-network
+          docker run --network changedet-network -d -e "LOG_LEVEL=TRACE" --cap-add=SYS_ADMIN --name sockpuppetbrowser --hostname sockpuppetbrowser --rm -p 3000:3000 dgtlmoon/sockpuppetbrowser:latest
+          docker run --network changedet-network -d -e "LOG_LEVEL=TRACE" --cap-add=SYS_ADMIN --name sockpuppetbrowser-custom-url --hostname sockpuppetbrowser-custom-url -p 3001:3000 --rm dgtlmoon/sockpuppetbrowser:latest
+
       - name: Test custom browser URL
         run: |
           cd changedetectionio
           ./run_custom_browser_url_tests.sh
-          cd ..
 
-      - name: Test changedetection.io container starts+runs basically without error
+  # Container startup tests
+  container-tests:
+    runs-on: ubuntu-latest
+    needs: build
+    timeout-minutes: 10
+    env:
+      PYTHON_VERSION: ${{ inputs.python-version }}
+    steps:
+      - uses: actions/checkout@v5
+
+      - name: Download Docker image artifact
+        uses: actions/download-artifact@v5
+        with:
+          name: test-changedetectionio-${{ env.PYTHON_VERSION }}
+          path: /tmp
+
+      - name: Load Docker image
         run: |
-          docker run --name test-changedetectionio -p 5556:5000  -d test-changedetectionio
+          docker load -i /tmp/test-changedetectionio.tar
+
+      - name: Test container starts+runs basically without error
+        run: |
+          docker run --name test-changedetectionio -p 5556:5000 -d test-changedetectionio
           sleep 3
-          # Should return 0 (no error) when grep finds it
-          curl --retry-connrefused --retry 6  -s http://localhost:5556 |grep -q checkbox-uuid
-          
-          # and IPv6
-          curl --retry-connrefused --retry 6  -s -g -6 "http://[::1]:5556"|grep -q checkbox-uuid
-
-          # Check whether TRACE log is enabled.
-          # Also, check whether TRACE came from STDOUT
+          curl --retry-connrefused --retry 6 -s http://localhost:5556 |grep -q checkbox-uuid
+          curl --retry-connrefused --retry 6 -s -g -6 "http://[::1]:5556"|grep -q checkbox-uuid
           docker logs test-changedetectionio 2>/dev/null | grep 'TRACE log is enabled' || exit 1
-          # Check whether DEBUG is came from STDOUT
           docker logs test-changedetectionio 2>/dev/null | grep 'DEBUG' || exit 1
-
           docker kill test-changedetectionio
 
       - name: Test HTTPS SSL mode
@@ -187,102 +377,66 @@ jobs:
           openssl req -x509 -newkey rsa:4096 -keyout privkey.pem -out cert.pem -days 365 -nodes -subj "/CN=localhost"
           docker run --name test-changedetectionio-ssl --rm -e SSL_CERT_FILE=cert.pem -e SSL_PRIVKEY_FILE=privkey.pem -p 5000:5000 -v ./cert.pem:/app/cert.pem -v ./privkey.pem:/app/privkey.pem -d test-changedetectionio
           sleep 3
-          # Should return 0 (no error) when grep finds it
-          # -k because its self-signed
           curl --retry-connrefused --retry 6 -k https://localhost:5000 -v|grep -q checkbox-uuid
-      
           docker kill test-changedetectionio-ssl
 
       - name: Test IPv6 Mode
         run: |
-          # IPv6 - :: bind to all interfaces inside container (like 0.0.0.0), ::1 would be localhost only
           docker run --name test-changedetectionio-ipv6 --rm -p 5000:5000 -e LISTEN_HOST=:: -d test-changedetectionio
           sleep 3
-          # Should return 0 (no error) when grep finds it on localhost
           curl --retry-connrefused --retry 6 http://[::1]:5000 -v|grep -q checkbox-uuid
           docker kill test-changedetectionio-ipv6
 
-      - name: Test changedetection.io SIGTERM and SIGINT signal shutdown
+  # Signal tests
+  signal-tests:
+    runs-on: ubuntu-latest
+    needs: build
+    timeout-minutes: 10
+    env:
+      PYTHON_VERSION: ${{ inputs.python-version }}
+    steps:
+      - uses: actions/checkout@v5
+
+      - name: Download Docker image artifact
+        uses: actions/download-artifact@v5
+        with:
+          name: test-changedetectionio-${{ env.PYTHON_VERSION }}
+          path: /tmp
+
+      - name: Load Docker image
+        run: |
+          docker load -i /tmp/test-changedetectionio.tar
+
+      - name: Test SIGTERM and SIGINT signal shutdown
         run: |
-          
           echo SIGINT Shutdown request test
           docker run --name sig-test -d test-changedetectionio
           sleep 3
           echo ">>> Sending SIGINT to sig-test container"
           docker kill --signal=SIGINT sig-test
           sleep 3
-          # invert the check (it should be not 0/not running)
           docker ps
-          # check signal catch(STDERR) log. Because of
-          # changedetectionio/__init__.py: logger.add(sys.stderr, level=logger_level)
           docker logs sig-test 2>&1 | grep 'Shutdown: Got Signal - SIGINT' || exit 1
           test -z "`docker ps|grep sig-test`"
-          if [ $? -ne 0 ]
-          then
+          if [ $? -ne 0 ]; then
             echo "Looks like container was running when it shouldnt be"
             docker ps
             exit 1
           fi
-          
-          # @todo - scan the container log to see the right "graceful shutdown" text exists 
           docker rm sig-test
-          
+
           echo SIGTERM Shutdown request test
           docker run --name sig-test -d test-changedetectionio
           sleep 3
           echo ">>> Sending SIGTERM to sig-test container"
           docker kill --signal=SIGTERM sig-test
           sleep 3
-          # invert the check (it should be not 0/not running)
           docker ps
-          # check signal catch(STDERR) log. Because of
-          # changedetectionio/__init__.py: logger.add(sys.stderr, level=logger_level)
           docker logs sig-test 2>&1 | grep 'Shutdown: Got Signal - SIGTERM' || exit 1
           test -z "`docker ps|grep sig-test`"
-          if [ $? -ne 0 ]
-          then
+          if [ $? -ne 0 ]; then
             echo "Looks like container was running when it shouldnt be"
             docker ps
             exit 1
           fi
-          
-          # @todo - scan the container log to see the right "graceful shutdown" text exists           
           docker rm sig-test
-
-      - name: Dump container log
-        if: always()
-        run: |
-          mkdir output-logs
-          docker logs test-cdio-basic-tests > output-logs/test-cdio-basic-tests-stdout-${{ env.PYTHON_VERSION }}.txt
-          docker logs test-cdio-basic-tests 2> output-logs/test-cdio-basic-tests-stderr-${{ env.PYTHON_VERSION }}.txt
-
-      - name: Extract and display memory test report
-        if: always()
-        run: |
-          # Extract test-memory.log from the container
-          echo "Extracting test-memory.log from container..."
-          docker cp test-cdio-basic-tests:/app/changedetectionio/test-memory.log output-logs/test-memory-${{ env.PYTHON_VERSION }}.log || echo "test-memory.log not found in container"
-
-          # Display the memory log contents for immediate visibility in workflow output
-          echo "=== Top 10 Highest Peak Memory Tests ==="
-          if [ -f output-logs/test-memory-${{ env.PYTHON_VERSION }}.log ]; then
-            # Sort by peak memory value (extract number before MB and sort numerically, reverse order)
-            grep "Peak memory:" output-logs/test-memory-${{ env.PYTHON_VERSION }}.log | \
-              sed 's/.*Peak memory: //' | \
-              paste -d'|' - <(grep "Peak memory:" output-logs/test-memory-${{ env.PYTHON_VERSION }}.log) | \
-              sort -t'|' -k1 -nr | \
-              cut -d'|' -f2 | \
-              head -10
-            echo ""
-            echo "=== Full Memory Test Report ==="
-            cat output-logs/test-memory-${{ env.PYTHON_VERSION }}.log
-          else
-            echo "No memory log available"
-          fi
-
-      - name: Store everything including test-datastore
-        if: always()
-        uses: actions/upload-artifact@v4
-        with:
-          name: test-cdio-basic-tests-output-py${{ env.PYTHON_VERSION }}
-          path: .

Dockerfile

@@ -36,6 +36,7 @@ ENV OPENSSL_INCLUDE_DIR="/usr/include/openssl"
 ENV CRYPTOGRAPHY_DONT_BUILD_RUST=1
 RUN --mount=type=cache,target=/tmp/pip-cache \
     pip install \
+    --prefer-binary \
     --extra-index-url https://www.piwheels.org/simple \
     --extra-index-url https://pypi.anaconda.org/ARM-software/simple \
     --cache-dir=/tmp/pip-cache \
@@ -47,6 +48,7 @@ RUN --mount=type=cache,target=/tmp/pip-cache \
 # https://github.com/dgtlmoon/changedetection.io/pull/1067 also musl/alpine (not supported)
 RUN --mount=type=cache,target=/tmp/pip-cache \
     pip install \
+    --prefer-binary \
     --cache-dir=/tmp/pip-cache \
     --target=/dependencies \
     playwright~=1.48.0 \

changedetectionio/__init__.py

@@ -2,7 +2,7 @@
 
 # Read more https://github.com/dgtlmoon/changedetection.io/wiki
 
-__version__ = '0.50.28'
+__version__ = '0.50.33'
 
 from changedetectionio.strtobool import strtobool
 from json.decoder import JSONDecodeError

changedetectionio/api/Import.py

@@ -1,9 +1,22 @@
-import os
 from changedetectionio.strtobool import strtobool
 from flask_restful import abort, Resource
 from flask import request
-import validators
+from functools import wraps
 from . import auth, validate_openapi_request
+from ..validate_url import is_safe_valid_url
+
+
+def default_content_type(content_type='text/plain'):
+    """Decorator to set a default Content-Type header if none is provided."""
+    def decorator(f):
+        @wraps(f)
+        def wrapper(*args, **kwargs):
+            if not request.content_type:
+                # Set default content type in the request environment
+                request.environ['CONTENT_TYPE'] = content_type
+            return f(*args, **kwargs)
+        return wrapper
+    return decorator
 
 
 class Import(Resource):
@@ -12,6 +25,7 @@ class Import(Resource):
         self.datastore = kwargs['datastore']
 
     @auth.check_token
+    @default_content_type('text/plain') #3547 #3542
     @validate_openapi_request('importWatches')
     def post(self):
         """Import a list of watched URLs."""
@@ -35,14 +49,13 @@ class Import(Resource):
 
         urls = request.get_data().decode('utf8').splitlines()
         added = []
-        allow_simplehost = not strtobool(os.getenv('BLOCK_SIMPLEHOSTS', 'False'))
         for url in urls:
             url = url.strip()
             if not len(url):
                 continue
 
             # If hosts that only contain alphanumerics are allowed ("localhost" for example)
-            if not validators.url(url, simple_host=allow_simplehost):
+            if not is_safe_valid_url(url):
                 return f"Invalid or unsupported URL - {url}", 400
 
             if dedupe and self.datastore.url_exists(url):

changedetectionio/api/Watch.py

@@ -1,12 +1,12 @@
 import os
-from changedetectionio.strtobool import strtobool
+
+from changedetectionio.validate_url import is_safe_valid_url
 
 from flask_expects_json import expects_json
 from changedetectionio import queuedWatchMetaData
 from changedetectionio import worker_handler
 from flask_restful import abort, Resource
 from flask import request, make_response, send_from_directory
-import validators
 from . import auth
 import copy
 
@@ -121,6 +121,10 @@ class Watch(Resource):
         if validation_error:
             return validation_error, 400
 
+        # XSS etc protection
+        if request.json.get('url') and not is_safe_valid_url(request.json.get('url')):
+            return "Invalid URL", 400
+
         watch.update(request.json)
 
         return "OK", 200
@@ -226,9 +230,7 @@ class CreateWatch(Resource):
         json_data = request.get_json()
         url = json_data['url'].strip()
 
-        # If hosts that only contain alphanumerics are allowed ("localhost" for example)
-        allow_simplehost = not strtobool(os.getenv('BLOCK_SIMPLEHOSTS', 'False'))
-        if not validators.url(url, simple_host=allow_simplehost):
+        if not is_safe_valid_url(url):
             return "Invalid or unsupported URL", 400
 
         if json_data.get('proxy'):

changedetectionio/blueprint/rss/blueprint.py

@@ -1,5 +1,6 @@
 
 from changedetectionio.jinja2_custom import render as jinja_render
+from changedetectionio.notification.handler import apply_service_tweaks
 from changedetectionio.store import ChangeDetectionStore
 from feedgen.feed import FeedGenerator
 from flask import Blueprint, make_response, request, url_for, redirect
@@ -120,9 +121,13 @@ def construct_blueprint(datastore: ChangeDetectionStore):
                     html_diff = diff.render_diff(previous_version_file_contents=watch.get_history_snapshot(dates[-2]),
                                                  newest_version_file_contents=watch.get_history_snapshot(dates[-1]),
                                                  include_equal=False,
-                                                 line_feed_sep="<br>",
-                                                 html_colour=html_colour_enable
+                                                 line_feed_sep="<br>"
                                                  )
+
+
+                    requested_output_format = 'htmlcolor' if html_colour_enable else 'html'
+                    html_diff = apply_service_tweaks(url='', n_body=html_diff, n_title=None, requested_output_format=requested_output_format)
+
                 except FileNotFoundError as e:
                     html_diff = f"History snapshot file for watch {watch.get('uuid')}@{watch.last_changed} - '{watch.get('title')} not found."
 

changedetectionio/blueprint/settings/templates/settings.html

@@ -240,9 +240,7 @@ nav
                     <p>
                        {{ render_field(form.application.form.scheduler_timezone_default) }}
                         <datalist id="timezones" style="display: none;">
-                            {% for tz_name in available_timezones %}
-                                <option value="{{ tz_name }}">{{ tz_name }}</option>
-                            {% endfor %}
+                            {%- for timezone in available_timezones -%}<option value="{{ timezone }}">{{ timezone }}</option>{%- endfor -%}
                         </datalist>
                     </p>
                 </div>

changedetectionio/blueprint/ui/__init__.py

@@ -76,14 +76,14 @@ def _handle_operations(op, uuids, datastore, worker_handler, update_q, queuedWat
 
     elif (op == 'notification-default'):
         from changedetectionio.notification import (
-            default_notification_format_for_watch
+            USE_SYSTEM_DEFAULT_NOTIFICATION_FORMAT_FOR_WATCH
         )
         for uuid in uuids:
             if datastore.data['watching'].get(uuid):
                 datastore.data['watching'][uuid]['notification_title'] = None
                 datastore.data['watching'][uuid]['notification_body'] = None
                 datastore.data['watching'][uuid]['notification_urls'] = []
-                datastore.data['watching'][uuid]['notification_format'] = default_notification_format_for_watch
+                datastore.data['watching'][uuid]['notification_format'] = USE_SYSTEM_DEFAULT_NOTIFICATION_FORMAT_FOR_WATCH
         if emit_flash:
             flash(f"{len(uuids)} watches set to use default notification settings")
 

changedetectionio/content_fetchers/base.py

@@ -75,7 +75,6 @@ class Fetcher():
         self.screenshot = None
         self.xpath_data = None
         # Keep headers and status_code as they're small
-        logger.trace("Fetcher content cleared from memory")
 
     @abstractmethod
     def get_error(self):

changedetectionio/diff.py

@@ -1,8 +1,32 @@
 import difflib
 from typing import List, Iterator, Union
 
-REMOVED_STYLE = "background-color: #fadad7; color: #b30000;"
-ADDED_STYLE = "background-color: #eaf2c2; color: #406619;"
+# https://github.com/dgtlmoon/changedetection.io/issues/821#issuecomment-1241837050
+#HTML_ADDED_STYLE = "background-color: #d2f7c2; color: #255d00;"
+#HTML_CHANGED_INTO_STYLE = "background-color: #dafbe1; color: #116329;"
+#HTML_CHANGED_STYLE = "background-color: #ffd6cc; color: #7a2000;"
+#HTML_REMOVED_STYLE = "background-color: #ffebe9; color: #82071e;"
+
+# @todo - In the future we can make this configurable
+HTML_ADDED_STYLE = "background-color: #eaf2c2; color: #406619"
+HTML_REMOVED_STYLE = "background-color: #fadad7; color: #b30000"
+HTML_CHANGED_STYLE = HTML_REMOVED_STYLE
+HTML_CHANGED_INTO_STYLE = HTML_ADDED_STYLE
+
+
+# These get set to html or telegram type or discord compatible or whatever in handler.py
+# Something that cant get escaped to HTML by accident
+REMOVED_PLACEMARKER_OPEN = '@removed_PLACEMARKER_OPEN'
+REMOVED_PLACEMARKER_CLOSED = '@removed_PLACEMARKER_CLOSED'
+
+ADDED_PLACEMARKER_OPEN = '@added_PLACEMARKER_OPEN'
+ADDED_PLACEMARKER_CLOSED = '@added_PLACEMARKER_CLOSED'
+
+CHANGED_PLACEMARKER_OPEN = '@changed_PLACEMARKER_OPEN'
+CHANGED_PLACEMARKER_CLOSED = '@changed_PLACEMARKER_CLOSED'
+
+CHANGED_INTO_PLACEMARKER_OPEN = '@changed_into_PLACEMARKER_OPEN'
+CHANGED_INTO_PLACEMARKER_CLOSED = '@changed_into_PLACEMARKER_CLOSED'
 
 def same_slicer(lst: List[str], start: int, end: int) -> List[str]:
     """Return a slice of the list, or a single element if start == end."""
@@ -15,8 +39,7 @@ def customSequenceMatcher(
     include_removed: bool = True,
     include_added: bool = True,
     include_replaced: bool = True,
-    include_change_type_prefix: bool = True,
-    html_colour: bool = False
+    include_change_type_prefix: bool = True
 ) -> Iterator[List[str]]:
     """
     Compare two sequences and yield differences based on specified parameters.
@@ -29,8 +52,6 @@ def customSequenceMatcher(
         include_added (bool): Include added parts
         include_replaced (bool): Include replaced parts
         include_change_type_prefix (bool): Add prefixes to indicate change types
-        html_colour (bool): Use HTML background colors for differences
-
     Yields:
         List[str]: Differences between sequences
     """
@@ -42,22 +63,22 @@ def customSequenceMatcher(
         if include_equal and tag == 'equal':
             yield before[alo:ahi]
         elif include_removed and tag == 'delete':
-            if html_colour:
-                yield [f'<span style="{REMOVED_STYLE}">{line}</span>' for line in same_slicer(before, alo, ahi)]
+            if include_change_type_prefix:
+                yield [f'{REMOVED_PLACEMARKER_OPEN}{line}{REMOVED_PLACEMARKER_CLOSED}' for line in same_slicer(before, alo, ahi)]
             else:
-                yield [f"(removed) {line}" for line in same_slicer(before, alo, ahi)] if include_change_type_prefix else same_slicer(before, alo, ahi)
+                yield same_slicer(before, alo, ahi)
         elif include_replaced and tag == 'replace':
-            if html_colour:
-                yield [f'<span style="{REMOVED_STYLE}">{line}</span>' for line in same_slicer(before, alo, ahi)] + \
-                      [f'<span style="{ADDED_STYLE}">{line}</span>' for line in same_slicer(after, blo, bhi)]
+            if include_change_type_prefix:
+                yield [f'{CHANGED_PLACEMARKER_OPEN}{line}{CHANGED_PLACEMARKER_CLOSED}' for line in same_slicer(before, alo, ahi)] + \
+                      [f'{CHANGED_INTO_PLACEMARKER_OPEN}{line}{CHANGED_INTO_PLACEMARKER_CLOSED}' for line in same_slicer(after, blo, bhi)]
             else:
-                yield [f"(changed) {line}" for line in same_slicer(before, alo, ahi)] + \
-                      [f"(into) {line}" for line in same_slicer(after, blo, bhi)] if include_change_type_prefix else same_slicer(before, alo, ahi) + same_slicer(after, blo, bhi)
+                yield same_slicer(before, alo, ahi) + same_slicer(after, blo, bhi)
         elif include_added and tag == 'insert':
-            if html_colour:
-                yield [f'<span style="{ADDED_STYLE}">{line}</span>' for line in same_slicer(after, blo, bhi)]
+            if include_change_type_prefix:
+                yield [f'{ADDED_PLACEMARKER_OPEN}{line}{ADDED_PLACEMARKER_CLOSED}' for line in same_slicer(after, blo, bhi)]
             else:
-                yield [f"(added) {line}" for line in same_slicer(after, blo, bhi)] if include_change_type_prefix else same_slicer(after, blo, bhi)
+                yield same_slicer(after, blo, bhi)
+
 
 def render_diff(
     previous_version_file_contents: str,
@@ -68,8 +89,7 @@ def render_diff(
     include_replaced: bool = True,
     line_feed_sep: str = "\n",
     include_change_type_prefix: bool = True,
-    patch_format: bool = False,
-    html_colour: bool = False
+    patch_format: bool = False
 ) -> str:
     """
     Render the difference between two file contents.
@@ -84,8 +104,6 @@ def render_diff(
         line_feed_sep (str): Separator for lines in output
         include_change_type_prefix (bool): Add prefixes to indicate change types
         patch_format (bool): Use patch format for output
-        html_colour (bool): Use HTML background colors for differences
-
     Returns:
         str: Rendered difference
     """
@@ -103,8 +121,7 @@ def render_diff(
         include_removed=include_removed,
         include_added=include_added,
         include_replaced=include_replaced,
-        include_change_type_prefix=include_change_type_prefix,
-        html_colour=html_colour
+        include_change_type_prefix=include_change_type_prefix
     )
 
     def flatten(lst: List[Union[str, List[str]]]) -> str:

changedetectionio/flask_app.py

@@ -133,6 +133,11 @@ def get_socketio_path():
     # Socket.IO will be available at {prefix}/socket.io/
     return prefix
 
+@app.template_global('is_safe_valid_url')
+def _is_safe_valid_url(test_url):
+    from .validate_url import is_safe_valid_url
+    return is_safe_valid_url(test_url)
+
 
 @app.template_filter('format_number_locale')
 def _jinja2_filter_format_number_locale(value: float) -> str:
@@ -382,7 +387,7 @@ def changedetection_app(config=None, datastore_o=None):
             # We would sometimes get login loop errors on sites hosted in sub-paths
 
             # note for the future:
-            #            if not is_safe_url(next):
+            #            if not is_safe_valid_url(next):
             #                return flask.abort(400)
             return redirect(url_for('watchlist.index'))
 

changedetectionio/forms.py

@@ -28,11 +28,8 @@ from wtforms.utils import unset_value
 
 from wtforms.validators import ValidationError
 
-from validators.url import url as url_validator
-
 from changedetectionio.widgets import TernaryNoneBooleanField
 
-
 # default
 # each select <option data-enabled="enabled-0-0"
 from changedetectionio.blueprint.browser_steps.browser_steps import browser_step_ui_config
@@ -541,19 +538,10 @@ class validateURL(object):
 
 
 def validate_url(test_url):
-    # If hosts that only contain alphanumerics are allowed ("localhost" for example)
-    try:
-        url_validator(test_url, simple_host=allow_simplehost)
-    except validators.ValidationError:
-        #@todo check for xss
-        message = f"'{test_url}' is not a valid URL."
+    from changedetectionio.validate_url import is_safe_valid_url
+    if not is_safe_valid_url(test_url):
         # This should be wtforms.validators.
-        raise ValidationError(message)
-
-    from .model.Watch import is_safe_url
-    if not is_safe_url(test_url):
-        # This should be wtforms.validators.
-        raise ValidationError('Watch protocol is not permitted by SAFE_PROTOCOL_REGEX or incorrect URL format')
+        raise ValidationError('Watch protocol is not permitted or invalid URL format')
 
 
 class ValidateSinglePythonRegexString(object):
@@ -741,7 +729,6 @@ class quickWatchForm(Form):
     edit_and_watch_submit_button = SubmitField('Edit > Watch', render_kw={"class": "pure-button pure-button-primary"})
 
 
-
 # Common to a single watch and the global settings
 class commonSettingsForm(Form):
     from . import processors
@@ -754,13 +741,21 @@ class commonSettingsForm(Form):
 
     fetch_backend = RadioField(u'Fetch Method', choices=content_fetchers.available_fetchers(), validators=[ValidateContentFetcherIsReady()])
     notification_body = TextAreaField('Notification Body', default='{{ watch_url }} had a change.', validators=[validators.Optional(), ValidateJinja2Template()])
-    notification_format = SelectField('Notification format', choices=valid_notification_formats.keys())
+    notification_format = SelectField('Notification format', choices=list(valid_notification_formats.items()))
     notification_title = StringField('Notification Title', default='ChangeDetection.io Notification - {{ watch_url }}', validators=[validators.Optional(), ValidateJinja2Template()])
     notification_urls = StringListField('Notification URL List', validators=[validators.Optional(), ValidateAppRiseServers(), ValidateJinja2Template()])
     processor = RadioField( label=u"Processor - What do you want to achieve?", choices=processors.available_processors(), default="text_json_diff")
     scheduler_timezone_default = StringField("Default timezone for watch check scheduler", render_kw={"list": "timezones"}, validators=[validateTimeZoneName()])
     webdriver_delay = IntegerField('Wait seconds before extracting text', validators=[validators.Optional(), validators.NumberRange(min=1, message="Should contain one or more seconds")])
 
+# Not true anymore but keep the validate_ hook for future use, we convert color tags
+#    def validate_notification_urls(self, field):
+#        """Validate that HTML Color format is not used with Telegram"""
+#        if self.notification_format.data == 'HTML Color' and field.data:
+#            for url in field.data:
+#                if url and ('tgram://' in url or 'discord://' in url or 'discord.com/api/webhooks' in url):
+#                    raise ValidationError('HTML Color format is not supported by Telegram and Discord. Please choose another Notification Format (Plain Text, HTML, or Markdown to HTML).')
+
 
 class importForm(Form):
     from . import processors

changedetectionio/html_tools.py

@@ -1,3 +1,5 @@
+from functools import lru_cache
+
 from loguru import logger
 from typing import List
 import html
@@ -13,7 +15,6 @@ TITLE_RE = re.compile(r"<title[^>]*>(.*?)</title>", re.I | re.S)
 META_CS  = re.compile(r'<meta[^>]+charset=["\']?\s*([a-z0-9_\-:+.]+)', re.I)
 META_CT  = re.compile(r'<meta[^>]+http-equiv=["\']?content-type["\']?[^>]*content=["\'][^>]*charset=([a-z0-9_\-:+.]+)', re.I)
 
-
 # 'price' , 'lowPrice', 'highPrice' are usually under here
 # All of those may or may not appear on different websites - I didnt find a way todo case-insensitive searching here
 LD_JSON_PRODUCT_OFFER_SELECTORS = ["json:$..offers", "json:$..Offers"]
@@ -22,9 +23,9 @@ class JSONNotFound(ValueError):
     def __init__(self, msg):
         ValueError.__init__(self, msg)
 
-
 # Doesn't look like python supports forward slash auto enclosure in re.findall
 # So convert it to inline flag "(?i)foobar" type configuration
+@lru_cache(maxsize=100)
 def perl_style_slash_enclosed_regex_to_options(regex):
 
     res = re.search(PERL_STYLE_REGEX, regex, re.IGNORECASE)
@@ -185,8 +186,21 @@ def xpath_filter(xpath_filter, html_content, append_pretty_line_formatting=False
     tree = html.fromstring(bytes(html_content, encoding='utf-8'), parser=parser)
     html_block = ""
 
-    r = elementpath.select(tree, xpath_filter.strip(), namespaces={'re': 'http://exslt.org/regular-expressions'}, parser=XPath3Parser)
-    #@note: //title/text() wont work where <title>CDATA..
+    # Build namespace map for XPath queries
+    namespaces = {'re': 'http://exslt.org/regular-expressions'}
+
+    # Handle default namespace in documents (common in RSS/Atom feeds, but can occur in any XML)
+    # XPath spec: unprefixed element names have no namespace, not the default namespace
+    # Solution: Register the default namespace with empty string prefix in elementpath
+    # This is primarily for RSS/Atom feeds but works for any XML with default namespace
+    if hasattr(tree, 'nsmap') and tree.nsmap and None in tree.nsmap:
+        # Register the default namespace with empty string prefix for elementpath
+        # This allows //title to match elements in the default namespace
+        namespaces[''] = tree.nsmap[None]
+
+    r = elementpath.select(tree, xpath_filter.strip(), namespaces=namespaces, parser=XPath3Parser)
+    #@note: //title/text() now works with default namespaces (fixed by registering '' prefix)
+    #@note: //title/text() wont work where <title>CDATA.. (use cdata_in_document_to_text first)
 
     if type(r) != list:
         r = [r]
@@ -221,8 +235,19 @@ def xpath1_filter(xpath_filter, html_content, append_pretty_line_formatting=Fals
     tree = html.fromstring(bytes(html_content, encoding='utf-8'), parser=parser)
     html_block = ""
 
-    r = tree.xpath(xpath_filter.strip(), namespaces={'re': 'http://exslt.org/regular-expressions'})
-    #@note: //title/text() wont work where <title>CDATA..
+    # Build namespace map for XPath queries
+    namespaces = {'re': 'http://exslt.org/regular-expressions'}
+
+    # NOTE: lxml's native xpath() does NOT support empty string prefix for default namespace
+    # For documents with default namespace (RSS/Atom feeds), users must use:
+    #   - local-name(): //*[local-name()='title']/text()
+    #   - Or use xpath_filter (not xpath1_filter) which supports default namespaces
+    # XPath spec: unprefixed element names have no namespace, not the default namespace
+
+    r = tree.xpath(xpath_filter.strip(), namespaces=namespaces)
+    #@note: xpath1 (lxml) does NOT automatically handle default namespaces
+    #@note: Use //*[local-name()='element'] or switch to xpath_filter for default namespace support
+    #@note: //title/text() wont work where <title>CDATA.. (use cdata_in_document_to_text first)
 
     for element in r:
         # When there's more than 1 match, then add the suffix to separate each line

changedetectionio/jinja2_custom/__init__.py

@@ -9,6 +9,7 @@ from .safe_jinja import (
     JINJA2_MAX_RETURN_PAYLOAD_SIZE,
     DEFAULT_JINJA2_EXTENSIONS,
 )
+from .plugins.regex import regex_replace
 
 __all__ = [
     'TimeExtension',
@@ -17,4 +18,5 @@ __all__ = [
     'create_jinja_env',
     'JINJA2_MAX_RETURN_PAYLOAD_SIZE',
     'DEFAULT_JINJA2_EXTENSIONS',
+    'regex_replace',
 ]

changedetectionio/jinja2_custom/plugins/__init__.py

@@ -0,0 +1,6 @@
+"""
+Jinja2 custom filter plugins for changedetection.io
+"""
+from .regex import regex_replace
+
+__all__ = ['regex_replace']

changedetectionio/jinja2_custom/plugins/regex.py

@@ -0,0 +1,98 @@
+"""
+Regex filter plugin for Jinja2 templates.
+
+Provides regex_replace filter for pattern-based string replacements in templates.
+"""
+import re
+import signal
+from loguru import logger
+
+
+def regex_replace(value: str, pattern: str, replacement: str = '', count: int = 0) -> str:
+    """
+    Replace occurrences of a regex pattern in a string.
+
+    Security: Protected against ReDoS (Regular Expression Denial of Service) attacks:
+    - Limits input value size to prevent excessive processing
+    - Uses timeout mechanism to prevent runaway regex operations
+    - Validates pattern complexity to prevent catastrophic backtracking
+
+    Args:
+        value: The input string to perform replacements on
+        pattern: The regex pattern to search for
+        replacement: The replacement string (default: '')
+        count: Maximum number of replacements (0 = replace all, default: 0)
+
+    Returns:
+        String with replacements applied, or original value on error
+
+    Example:
+        {{ "hello world" | regex_replace("world", "universe") }}
+        {{ diff | regex_replace("<td>([^<]+)</td><td>([^<]+)</td>", "Label1: \\1\\nLabel2: \\2") }}
+
+    Security limits:
+        - Maximum input size: 10MB
+        - Maximum pattern length: 500 characters
+        - Operation timeout: 10 seconds
+        - Dangerous nested quantifier patterns are rejected
+    """
+    # Security limits
+    MAX_INPUT_SIZE = 1024 * 1024 * 10 # 10MB max input size
+    MAX_PATTERN_LENGTH = 500  # Maximum regex pattern length
+    REGEX_TIMEOUT_SECONDS = 10  # Maximum time for regex operation
+
+    # Validate input sizes
+    value_str = str(value)
+    if len(value_str) > MAX_INPUT_SIZE:
+        logger.warning(f"regex_replace: Input too large ({len(value_str)} bytes), truncating")
+        value_str = value_str[:MAX_INPUT_SIZE]
+
+    if len(pattern) > MAX_PATTERN_LENGTH:
+        logger.warning(f"regex_replace: Pattern too long ({len(pattern)} chars), rejecting")
+        return value_str
+
+    # Check for potentially dangerous patterns (basic checks)
+    # Nested quantifiers like (a+)+ can cause catastrophic backtracking
+    dangerous_patterns = [
+        r'\([^)]*\+[^)]*\)\+',  # (x+)+
+        r'\([^)]*\*[^)]*\)\+',  # (x*)+
+        r'\([^)]*\+[^)]*\)\*',  # (x+)*
+        r'\([^)]*\*[^)]*\)\*',  # (x*)*
+    ]
+
+    for dangerous in dangerous_patterns:
+        if re.search(dangerous, pattern):
+            logger.warning(f"regex_replace: Potentially dangerous pattern detected: {pattern}")
+            return value_str
+
+    def timeout_handler(signum, frame):
+        raise TimeoutError("Regex operation timed out")
+
+    try:
+        # Set up timeout for regex operation (Unix-like systems only)
+        # This prevents ReDoS attacks
+        old_handler = None
+        if hasattr(signal, 'SIGALRM'):
+            old_handler = signal.signal(signal.SIGALRM, timeout_handler)
+            signal.alarm(REGEX_TIMEOUT_SECONDS)
+
+        try:
+            result = re.sub(pattern, replacement, value_str, count=count)
+        finally:
+            # Cancel the alarm
+            if hasattr(signal, 'SIGALRM'):
+                signal.alarm(0)
+                if old_handler is not None:
+                    signal.signal(signal.SIGALRM, old_handler)
+
+        return result
+
+    except TimeoutError:
+        logger.error(f"regex_replace: Regex operation timed out - possible ReDoS attack. Pattern: {pattern}")
+        return value_str
+    except re.error as e:
+        logger.warning(f"regex_replace: Invalid regex pattern: {e}")
+        return value_str
+    except Exception as e:
+        logger.error(f"regex_replace: Unexpected error: {e}")
+        return value_str

changedetectionio/jinja2_custom/safe_jinja.py

@@ -8,13 +8,13 @@ import jinja2.sandbox
 import typing as t
 import os
 from .extensions.TimeExtension import TimeExtension
+from .plugins import regex_replace
 
 JINJA2_MAX_RETURN_PAYLOAD_SIZE = 1024 * int(os.getenv("JINJA2_MAX_RETURN_PAYLOAD_SIZE_KB", 1024 * 10))
 
 # Default extensions - can be overridden in create_jinja_env()
 DEFAULT_JINJA2_EXTENSIONS = [TimeExtension]
 
-
 def create_jinja_env(extensions=None, **kwargs) -> jinja2.sandbox.ImmutableSandboxedEnvironment:
     """
     Create a sandboxed Jinja2 environment with our custom extensions and default timezone.
@@ -38,6 +38,9 @@ def create_jinja_env(extensions=None, **kwargs) -> jinja2.sandbox.ImmutableSandb
     default_timezone = os.getenv('TZ', 'UTC').strip()
     jinja2_env.default_timezone = default_timezone
 
+    # Register custom filters
+    jinja2_env.filters['regex_replace'] = regex_replace
+
     return jinja2_env
 
 

changedetectionio/model/App.py

@@ -1,4 +1,5 @@
 from os import getenv
+from copy import deepcopy
 
 from changedetectionio.blueprint.rss import RSS_FORMAT_TYPES
 
@@ -74,7 +75,8 @@ class model(dict):
 
     def __init__(self, *arg, **kw):
         super(model, self).__init__(*arg, **kw)
-        self.update(self.base_config)
+        # CRITICAL: deepcopy to avoid sharing mutable objects between instances
+        self.update(deepcopy(self.base_config))
 
 
 def parse_headers_from_text_file(filepath):

changedetectionio/model/Watch.py

@@ -1,4 +1,5 @@
 from blinker import signal
+from changedetectionio.validate_url import is_safe_valid_url
 
 from changedetectionio.strtobool import strtobool
 from changedetectionio.jinja2_custom import render as jinja_render
@@ -9,34 +10,15 @@ from pathlib import Path
 from loguru import logger
 
 from .. import jinja2_custom as safe_jinja
+from ..diff import ADDED_PLACEMARKER_OPEN
 from ..html_tools import TRANSLATE_WHITESPACE_TABLE
 
-# Allowable protocols, protects against javascript: etc
-# file:// is further checked by ALLOW_FILE_URI
-SAFE_PROTOCOL_REGEX='^(http|https|ftp|file):'
 FAVICON_RESAVE_THRESHOLD_SECONDS=86400
 
 
 minimum_seconds_recheck_time = int(os.getenv('MINIMUM_SECONDS_RECHECK_TIME', 3))
 mtable = {'seconds': 1, 'minutes': 60, 'hours': 3600, 'days': 86400, 'weeks': 86400 * 7}
 
-
-def is_safe_url(test_url):
-    # See https://github.com/dgtlmoon/changedetection.io/issues/1358
-
-    # Remove 'source:' prefix so we dont get 'source:javascript:' etc
-    # 'source:' is a valid way to tell us to return the source
-
-    r = re.compile(re.escape('source:'), re.IGNORECASE)
-    test_url = r.sub('', test_url)
-
-    pattern = re.compile(os.getenv('SAFE_PROTOCOL_REGEX', SAFE_PROTOCOL_REGEX), re.IGNORECASE)
-    if not pattern.match(test_url.strip()):
-        return False
-
-    return True
-
-
 class model(watch_base):
     __newest_history_key = None
     __history_n = 0
@@ -79,7 +61,7 @@ class model(watch_base):
     def link(self):
 
         url = self.get('url', '')
-        if not is_safe_url(url):
+        if not is_safe_valid_url(url):
             return 'DISABLED'
 
         ready_url = url
@@ -89,9 +71,8 @@ class model(watch_base):
                 ready_url = jinja_render(template_str=url)
             except Exception as e:
                 logger.critical(f"Invalid URL template for: '{url}' - {str(e)}")
-                from flask import (
-                    flash, Markup, url_for
-                )
+                from flask import flash, url_for
+                from markupsafe import Markup
                 message = Markup('<a href="{}#general">The URL {} is invalid and cannot be used, click to edit</a>'.format(
                     url_for('ui.ui_edit.edit_page', uuid=self.get('uuid')), self.get('url', '')))
                 flash(message, 'error')
@@ -101,7 +82,7 @@ class model(watch_base):
             ready_url=ready_url.replace('source:', '')
 
         # Also double check it after any Jinja2 formatting just incase
-        if not is_safe_url(ready_url):
+        if not is_safe_valid_url(ready_url):
             return 'DISABLED'
         return ready_url
 

changedetectionio/model/__init__.py

@@ -2,7 +2,7 @@ import os
 import uuid
 
 from changedetectionio import strtobool
-default_notification_format_for_watch = 'System default'
+USE_SYSTEM_DEFAULT_NOTIFICATION_FORMAT_FOR_WATCH = 'System default'
 CONDITIONS_MATCH_LOGIC_DEFAULT = 'ALL'
 
 class watch_base(dict):
@@ -44,7 +44,7 @@ class watch_base(dict):
             'method': 'GET',
             'notification_alert_count': 0,
             'notification_body': None,
-            'notification_format': default_notification_format_for_watch,
+            'notification_format': USE_SYSTEM_DEFAULT_NOTIFICATION_FORMAT_FOR_WATCH,
             'notification_muted': False,
             'notification_screenshot': False,  # Include the latest screenshot if available and supported by the apprise URL
             'notification_title': None,

changedetectionio/notification/__init__.py

@@ -1,17 +1,16 @@
-from changedetectionio.model import default_notification_format_for_watch
+from changedetectionio.model import USE_SYSTEM_DEFAULT_NOTIFICATION_FORMAT_FOR_WATCH
 
-default_notification_format = 'HTML Color'
+default_notification_format = 'htmlcolor'
 default_notification_body = '{{watch_url}} had a change.\n---\n{{diff}}\n---\n'
 default_notification_title = 'ChangeDetection.io Notification - {{watch_url}}'
 
 # The values (markdown etc) are from apprise NotifyFormat,
 # But to avoid importing the whole heavy module just use the same strings here.
 valid_notification_formats = {
-    'Text': 'text',
-    'Markdown': 'markdown',
-    'HTML': 'html',
-    'HTML Color': 'htmlcolor',
+    'text': 'Plain Text',
+    'html': 'HTML',
+    'htmlcolor': 'HTML Color',
+    'markdown': 'Markdown to HTML',
     # Used only for editing a watch (not for global)
-    default_notification_format_for_watch: default_notification_format_for_watch
+    USE_SYSTEM_DEFAULT_NOTIFICATION_FORMAT_FOR_WATCH: USE_SYSTEM_DEFAULT_NOTIFICATION_FORMAT_FOR_WATCH
 }
-

changedetectionio/notification/apprise_plugin/custom_handlers.py

@@ -1,10 +1,61 @@
+"""
+Custom Apprise HTTP Handlers with format= Parameter Support
+
+IMPORTANT: This module works around a limitation in Apprise's @notify decorator.
+
+THE PROBLEM:
+-------------
+When using Apprise's @notify decorator to create custom notification handlers, the
+decorator creates a CustomNotifyPlugin that uses parse_url(..., simple=True) to parse
+URLs. This simple parsing mode does NOT extract the format= query parameter from the URL
+and set it as a top-level parameter that NotifyBase.__init__ can use to set notify_format.
+
+As a result:
+1. URL: post://example.com/webhook?format=html
+2. Apprise parses this and sees format=html in qsd (query string dictionary)
+3. But it does NOT extract it and pass it to NotifyBase.__init__
+4. NotifyBase defaults to notify_format=TEXT
+5. When you call apobj.notify(body="<html>...", body_format="html"):
+   - Apprise sees: input format = html, output format (notify_format) = text
+   - Apprise calls convert_between("html", "text", body)
+   - This strips all HTML tags, leaving only plain text
+6. Your custom handler receives stripped plain text instead of HTML
+
+THE SOLUTION:
+-------------
+Instead of using the @notify decorator directly, we:
+1. Manually register custom plugins using plugins.N_MGR.add()
+2. Create a CustomHTTPHandler class that extends CustomNotifyPlugin
+3. Override __init__ to extract format= from qsd and set it as kwargs['format']
+4. Call NotifyBase.__init__ which properly sets notify_format from kwargs['format']
+5. Set up _default_args like CustomNotifyPlugin does for compatibility
+
+This ensures that when format=html is in the URL:
+- notify_format is set to HTML
+- Apprise sees: input format = html, output format = html
+- No conversion happens (convert_between returns content unchanged)
+- Your custom handler receives the original HTML intact
+
+TESTING:
+--------
+To verify this works:
+>>> apobj = apprise.Apprise()
+>>> apobj.add('post://localhost:5005/test?format=html')
+>>> for server in apobj:
+...     print(server.notify_format)  # Should print: html (not text)
+>>> apobj.notify(body='<span>Test</span>', body_format='html')
+# Your handler should receive '<span>Test</span>' not 'Test'
+"""
+
 import json
 import re
 from urllib.parse import unquote_plus
 
 import requests
-from apprise.decorators import notify
-from apprise.utils.parse import parse_url as apprise_parse_url
+from apprise import plugins
+from apprise.decorators.base import CustomNotifyPlugin
+from apprise.utils.parse import parse_url as apprise_parse_url, url_assembly
+from apprise.utils.logic import dict_full_update
 from loguru import logger
 from requests.structures import CaseInsensitiveDict
 
@@ -12,13 +63,66 @@ SUPPORTED_HTTP_METHODS = {"get", "post", "put", "delete", "patch", "head"}
 
 
 def notify_supported_methods(func):
+    """Register custom HTTP method handlers that properly support format= parameter."""
     for method in SUPPORTED_HTTP_METHODS:
-        func = notify(on=method)(func)
-        # Add support for https, for each supported http method
-        func = notify(on=f"{method}s")(func)
+        _register_http_handler(method, func)
+        _register_http_handler(f"{method}s", func)
     return func
 
 
+def _register_http_handler(schema, send_func):
+    """Register a custom HTTP handler that extracts format= from URL query parameters."""
+
+    # Parse base URL
+    base_url = f"{schema}://"
+    base_args = apprise_parse_url(base_url, default_schema=schema, verify_host=False, simple=True)
+
+    class CustomHTTPHandler(CustomNotifyPlugin):
+        secure_protocol = schema
+        service_name = f"Custom HTTP - {schema.upper()}"
+        _base_args = base_args
+
+        def __init__(self, **kwargs):
+            # Extract format from qsd and set it as a top-level kwarg
+            # This allows NotifyBase.__init__ to properly set notify_format
+            if 'qsd' in kwargs and 'format' in kwargs['qsd']:
+                kwargs['format'] = kwargs['qsd']['format']
+
+            # Call NotifyBase.__init__ (skip CustomNotifyPlugin.__init__)
+            super(CustomNotifyPlugin, self).__init__(**kwargs)
+
+            # Set up _default_args like CustomNotifyPlugin does
+            self._default_args = {}
+            kwargs.pop("secure", None)
+            dict_full_update(self._default_args, self._base_args)
+            dict_full_update(self._default_args, kwargs)
+            self._default_args["url"] = url_assembly(**self._default_args)
+
+        __send = staticmethod(send_func)
+
+        def send(self, body, title="", notify_type="info", *args, **kwargs):
+            """Call the custom send function."""
+            try:
+                result = self.__send(
+                    body, title, notify_type,
+                    *args,
+                    meta=self._default_args,
+                    **kwargs
+                )
+                return True if result is None else bool(result)
+            except Exception as e:
+                self.logger.warning(f"Exception in custom HTTP handler: {e}")
+                return False
+
+    # Register the plugin
+    plugins.N_MGR.add(
+        plugin=CustomHTTPHandler,
+        schemas=schema,
+        send_func=send_func,
+        url=base_url,
+    )
+
+
 def _get_auth(parsed_url: dict) -> str | tuple[str, str]:
     user: str | None = parsed_url.get("user")
     password: str | None = parsed_url.get("password")
@@ -74,6 +178,8 @@ def apprise_http_custom_handler(
     *args,
     **kwargs,
 ) -> bool:
+
+
     url: str = meta.get("url")
     schema: str = meta.get("schema")
     method: str = re.sub(r"s$", "", schema).upper()
@@ -89,25 +195,16 @@ def apprise_http_custom_handler(
 
     url = re.sub(rf"^{schema}", "https" if schema.endswith("s") else "http", parsed_url.get("url"))
 
-    try:
-        response = requests.request(
-            method=method,
-            url=url,
-            auth=auth,
-            headers=headers,
-            params=params,
-            data=body.encode("utf-8") if isinstance(body, str) else body,
-        )
+    response = requests.request(
+        method=method,
+        url=url,
+        auth=auth,
+        headers=headers,
+        params=params,
+        data=body.encode("utf-8") if isinstance(body, str) else body,
+    )
 
-        response.raise_for_status()
+    response.raise_for_status()
 
-        logger.info(f"Successfully sent custom notification to {url}")
-        return True
-
-    except requests.RequestException as e:
-        logger.error(f"Remote host error while sending custom notification to {url}: {e}")
-        return False
-
-    except Exception as e:
-        logger.error(f"Unexpected error occurred while sending custom notification to {url}: {e}")
-        return False
+    logger.info(f"Successfully sent custom notification to {url}")
+    return True

changedetectionio/notification/apprise_plugin/discord.py

@@ -0,0 +1,286 @@
+"""
+Custom Discord plugin for changedetection.io
+Extends Apprise's Discord plugin to support custom colored embeds for removed/added content
+"""
+from apprise.plugins.discord import NotifyDiscord
+from apprise.decorators import notify
+from apprise.common import NotifyFormat
+from loguru import logger
+
+# Import placeholders from changedetection's diff module
+from ...diff import (
+    REMOVED_PLACEMARKER_OPEN,
+    REMOVED_PLACEMARKER_CLOSED,
+    ADDED_PLACEMARKER_OPEN,
+    ADDED_PLACEMARKER_CLOSED,
+    CHANGED_PLACEMARKER_OPEN,
+    CHANGED_PLACEMARKER_CLOSED,
+    CHANGED_INTO_PLACEMARKER_OPEN,
+    CHANGED_INTO_PLACEMARKER_CLOSED,
+)
+
+# Discord embed sidebar colors for different change types
+DISCORD_COLOR_UNCHANGED = 8421504   # Gray (#808080)
+DISCORD_COLOR_REMOVED = 16711680    # Red (#FF0000)
+DISCORD_COLOR_ADDED = 65280         # Green (#00FF00)
+DISCORD_COLOR_CHANGED = 16753920    # Orange (#FFA500)
+DISCORD_COLOR_CHANGED_INTO = 3447003  # Blue (#5865F2 - Discord blue)
+DISCORD_COLOR_WARNING = 16776960    # Yellow (#FFFF00)
+
+
+class NotifyDiscordCustom(NotifyDiscord):
+    """
+    Custom Discord notification handler that supports multiple colored embeds
+    for showing removed (red) and added (green) content separately.
+    """
+
+    def send(self, body, title="", notify_type=None, attach=None, **kwargs):
+        """
+        Override send method to create custom embeds with red/green colors
+        for removed/added content when placeholders are present.
+        """
+
+        # Check if body contains our diff placeholders
+        has_removed = REMOVED_PLACEMARKER_OPEN in body
+        has_added = ADDED_PLACEMARKER_OPEN in body
+        has_changed = CHANGED_PLACEMARKER_OPEN in body
+        has_changed_into = CHANGED_INTO_PLACEMARKER_OPEN in body
+
+        # If we have diff placeholders and we're in markdown/html format, create custom embeds
+        if (has_removed or has_added or has_changed or has_changed_into) and self.notify_format in (NotifyFormat.MARKDOWN, NotifyFormat.HTML):
+            return self._send_with_colored_embeds(body, title, notify_type, attach, **kwargs)
+
+        # Otherwise, use the parent class's default behavior
+        return super().send(body, title, notify_type, attach, **kwargs)
+
+    def _send_with_colored_embeds(self, body, title, notify_type, attach, **kwargs):
+        """
+        Send Discord message with embeds in the original diff order.
+        Preserves the sequence: unchanged -> removed -> added -> unchanged, etc.
+        """
+        from datetime import datetime, timezone
+
+        payload = {
+            "tts": self.tts,
+            "wait": self.tts is False,
+        }
+
+        if self.flags:
+            payload["flags"] = self.flags
+
+        # Acquire image_url
+        image_url = self.image_url(notify_type)
+
+        if self.avatar and (image_url or self.avatar_url):
+            payload["avatar_url"] = self.avatar_url if self.avatar_url else image_url
+
+        if self.user:
+            payload["username"] = self.user
+
+        # Associate our thread_id with our message
+        params = {"thread_id": self.thread_id} if self.thread_id else None
+
+        # Build embeds array preserving order
+        embeds = []
+
+        # Add title as plain bold text in message content (not an embed)
+        if title:
+            payload["content"] = f"**{title}**"
+
+        # Parse the body into ordered chunks
+        chunks = self._parse_body_into_chunks(body)
+
+        # Discord limits:
+        # - Max 10 embeds per message
+        # - Max 6000 characters total across all embeds
+        # - Max 4096 characters per embed description
+        max_embeds = 10
+        max_total_chars = 6000
+        max_embed_description = 4096
+
+        # All 10 embed slots are available for content
+        max_content_embeds = max_embeds
+
+        # Start character count
+        total_chars = 0
+
+        # Create embeds from chunks in order (no titles, just color coding)
+        for chunk_type, content in chunks:
+            if not content.strip():
+                continue
+
+            # Truncate individual embed description if needed
+            if len(content) > max_embed_description:
+                content = content[:max_embed_description - 3] + "..."
+
+            # Check if we're approaching the embed count limit
+            # We need room for the warning embed, so stop at max_content_embeds - 1
+            current_content_embeds = len(embeds)
+            if current_content_embeds >= max_content_embeds - 1:
+                # Add a truncation notice (this will be the 10th embed)
+                embeds.append({
+                    "description": "⚠️ Content truncated (Discord 10 embed limit reached) - Tip: Select 'Plain Text' or 'HTML' format for longer diffs",
+                    "color": DISCORD_COLOR_WARNING,
+                })
+                break
+
+            # Check if adding this embed would exceed total character limit
+            if total_chars + len(content) > max_total_chars:
+                # Add a truncation notice
+                remaining_chars = max_total_chars - total_chars
+                if remaining_chars > 100:
+                    # Add partial content if we have room
+                    truncated_content = content[:remaining_chars - 100] + "..."
+                    embeds.append({
+                        "description": truncated_content,
+                        "color": (DISCORD_COLOR_UNCHANGED if chunk_type == "unchanged"
+                                 else DISCORD_COLOR_REMOVED if chunk_type == "removed"
+                                 else DISCORD_COLOR_ADDED),
+                    })
+                embeds.append({
+                    "description": "⚠️ Content truncated (Discord 6000 char limit reached)\nTip: Select 'Plain Text' or 'HTML' format for longer diffs",
+                    "color": DISCORD_COLOR_WARNING,
+                })
+                break
+
+            if chunk_type == "unchanged":
+                embeds.append({
+                    "description": content,
+                    "color": DISCORD_COLOR_UNCHANGED,
+                })
+            elif chunk_type == "removed":
+                embeds.append({
+                    "description": content,
+                    "color": DISCORD_COLOR_REMOVED,
+                })
+            elif chunk_type == "added":
+                embeds.append({
+                    "description": content,
+                    "color": DISCORD_COLOR_ADDED,
+                })
+            elif chunk_type == "changed":
+                # Changed (old value) - use orange to distinguish from pure removal
+                embeds.append({
+                    "description": content,
+                    "color": DISCORD_COLOR_CHANGED,
+                })
+            elif chunk_type == "changed_into":
+                # Changed into (new value) - use blue to distinguish from pure addition
+                embeds.append({
+                    "description": content,
+                    "color": DISCORD_COLOR_CHANGED_INTO,
+                })
+
+            total_chars += len(content)
+
+        if embeds:
+            payload["embeds"] = embeds
+
+        # Send the payload using parent's _send method
+        if not self._send(payload, params=params):
+            return False
+
+        # Handle attachments if present
+        if attach and self.attachment_support:
+            payload.update({
+                "tts": False,
+                "wait": True,
+            })
+            payload.pop("embeds", None)
+            payload.pop("content", None)
+            payload.pop("allow_mentions", None)
+
+            for attachment in attach:
+                self.logger.info(f"Posting Discord Attachment {attachment.name}")
+                if not self._send(payload, params=params, attach=attachment):
+                    return False
+
+        return True
+
+    def _parse_body_into_chunks(self, body):
+        """
+        Parse the body into ordered chunks of (type, content) tuples.
+        Types: "unchanged", "removed", "added", "changed", "changed_into"
+        Preserves the original order of the diff.
+        """
+        chunks = []
+        position = 0
+
+        while position < len(body):
+            # Find the next marker
+            next_removed = body.find(REMOVED_PLACEMARKER_OPEN, position)
+            next_added = body.find(ADDED_PLACEMARKER_OPEN, position)
+            next_changed = body.find(CHANGED_PLACEMARKER_OPEN, position)
+            next_changed_into = body.find(CHANGED_INTO_PLACEMARKER_OPEN, position)
+
+            # Determine which marker comes first
+            if next_removed == -1 and next_added == -1 and next_changed == -1 and next_changed_into == -1:
+                # No more markers, rest is unchanged
+                if position < len(body):
+                    chunks.append(("unchanged", body[position:]))
+                break
+
+            # Find the earliest marker
+            next_marker_pos = None
+            next_marker_type = None
+
+            # Compare all marker positions to find the earliest
+            markers = []
+            if next_removed != -1:
+                markers.append((next_removed, "removed"))
+            if next_added != -1:
+                markers.append((next_added, "added"))
+            if next_changed != -1:
+                markers.append((next_changed, "changed"))
+            if next_changed_into != -1:
+                markers.append((next_changed_into, "changed_into"))
+
+            if markers:
+                next_marker_pos, next_marker_type = min(markers, key=lambda x: x[0])
+
+            # Add unchanged content before the marker
+            if next_marker_pos > position:
+                chunks.append(("unchanged", body[position:next_marker_pos]))
+
+            # Find the closing marker
+            if next_marker_type == "removed":
+                open_marker = REMOVED_PLACEMARKER_OPEN
+                close_marker = REMOVED_PLACEMARKER_CLOSED
+            elif next_marker_type == "added":
+                open_marker = ADDED_PLACEMARKER_OPEN
+                close_marker = ADDED_PLACEMARKER_CLOSED
+            elif next_marker_type == "changed":
+                open_marker = CHANGED_PLACEMARKER_OPEN
+                close_marker = CHANGED_PLACEMARKER_CLOSED
+            else:  # changed_into
+                open_marker = CHANGED_INTO_PLACEMARKER_OPEN
+                close_marker = CHANGED_INTO_PLACEMARKER_CLOSED
+
+            close_pos = body.find(close_marker, next_marker_pos)
+
+            if close_pos == -1:
+                # No closing marker, take rest as this type
+                content = body[next_marker_pos + len(open_marker):]
+                chunks.append((next_marker_type, content))
+                break
+            else:
+                # Extract content between markers
+                content = body[next_marker_pos + len(open_marker):close_pos]
+                chunks.append((next_marker_type, content))
+                position = close_pos + len(close_marker)
+
+        return chunks
+
+
+# Register the custom Discord handler with Apprise
+# This will override the built-in discord:// handler
+@notify(on="discord")
+def discord_custom_wrapper(body, title, notify_type, meta, body_format=None, *args, **kwargs):
+    """
+    Wrapper function to make the custom Discord handler work with Apprise's decorator system.
+    Note: This decorator approach may not work for overriding built-in plugins.
+    The class-based approach above is the proper way to extend NotifyDiscord.
+    """
+    logger.info("Custom Discord handler called")
+    # This is here for potential future use with decorator-based registration
+    return True

changedetectionio/notification/email_helpers.py

@@ -0,0 +1,42 @@
+def as_monospaced_html_email(content: str, title: str) -> str:
+    """
+    Wraps `content` in a minimal, email-safe HTML template
+    that forces monospace rendering across Gmail, Hotmail, Apple Mail, etc.
+
+    Args:
+        content: The body text (plain text or HTML-like).
+        title: The title plaintext
+    Returns:
+        A complete HTML document string suitable for sending as an email body.
+    """
+
+    # All line feed types should be removed and then this function should only be fed <br>'s
+    # Then it works with our <pre> styling without double linefeeds
+    content = content.translate(str.maketrans('', '', '\r\n'))
+
+    if title:
+        import html
+        title = html.escape(title)
+    else:
+        title = ''
+    # 2. Full email-safe HTML
+    html_email = f"""<!DOCTYPE html>
+<html lang="en">
+<head>
+  <meta charset="UTF-8">
+  <meta name="x-apple-disable-message-reformatting">
+  <meta name="viewport" content="width=device-width, initial-scale=1.0">
+  <!--[if mso]>
+    <style>
+      body, div, pre, td {{ font-family: "Courier New", Courier, monospace !important; }}
+    </style>
+  <![endif]-->
+  <title>{title}</title>
+</head>
+<body style="-webkit-text-size-adjust:100%;-ms-text-size-adjust:100%;">
+  <pre role="article" aria-roledescription="email" lang="en"
+       style="font-family: monospace, 'Courier New', Courier; font-size: 0.8em;
+              white-space: pre-wrap; word-break: break-word;">{content}</pre>
+</body>
+</html>"""
+    return html_email

changedetectionio/notification/handler.py

@@ -6,7 +6,12 @@ from loguru import logger
 from urllib.parse import urlparse
 from .apprise_plugin.assets import apprise_asset, APPRISE_AVATAR_URL
 from .apprise_plugin.custom_handlers import SUPPORTED_HTTP_METHODS
-from ..notification_service import NotificationContextData
+from .email_helpers import as_monospaced_html_email
+from ..diff import HTML_REMOVED_STYLE, REMOVED_PLACEMARKER_OPEN, REMOVED_PLACEMARKER_CLOSED, ADDED_PLACEMARKER_OPEN, HTML_ADDED_STYLE, \
+    ADDED_PLACEMARKER_CLOSED, CHANGED_INTO_PLACEMARKER_OPEN, CHANGED_INTO_PLACEMARKER_CLOSED, CHANGED_PLACEMARKER_OPEN, \
+    CHANGED_PLACEMARKER_CLOSED, HTML_CHANGED_STYLE, HTML_CHANGED_INTO_STYLE
+from ..notification_service import NotificationContextData, CUSTOM_LINEBREAK_PLACEHOLDER
+
 
 
 def markup_text_links_to_html(body):
@@ -53,17 +58,18 @@ def notification_format_align_with_apprise(n_format : str):
     """
     Correctly align changedetection's formats with apprise's formats
     Probably these are the same - but good to be sure.
+    These set the expected OUTPUT format type
     :param n_format:
     :return:
     """
 
-    if n_format.lower().startswith('html'):
+    if n_format.startswith('html'):
         # Apprise only knows 'html' not 'htmlcolor' etc, which shouldnt matter here
         n_format = NotifyFormat.HTML.value
-    elif n_format.lower().startswith('markdown'):
+    elif n_format.startswith('markdown'):
         # probably the same but just to be safe
         n_format = NotifyFormat.MARKDOWN.value
-    elif n_format.lower().startswith('text'):
+    elif n_format.startswith('text'):
         # probably the same but just to be safe
         n_format = NotifyFormat.TEXT.value
     else:
@@ -71,11 +77,175 @@ def notification_format_align_with_apprise(n_format : str):
 
     return n_format
 
+def apply_discord_markdown_to_body(n_body):
+    """
+    Discord does not support <del> but it supports non-standard ~~strikethrough~~
+    :param n_body:
+    :return:
+    """
+    import re
+    # Define the mapping between your placeholders and markdown markers
+    replacements = [
+        (REMOVED_PLACEMARKER_OPEN, '~~', REMOVED_PLACEMARKER_CLOSED, '~~'),
+        (ADDED_PLACEMARKER_OPEN, '**', ADDED_PLACEMARKER_CLOSED, '**'),
+        (CHANGED_PLACEMARKER_OPEN, '~~', CHANGED_PLACEMARKER_CLOSED, '~~'),
+        (CHANGED_INTO_PLACEMARKER_OPEN, '**', CHANGED_INTO_PLACEMARKER_CLOSED, '**'),
+    ]
+    # So that the markdown gets added without any whitespace following it which would break it
+    for open_tag, open_md, close_tag, close_md in replacements:
+        # Regex: match opening tag, optional whitespace, capture the content, optional whitespace, then closing tag
+        pattern = re.compile(
+            re.escape(open_tag) + r'(\s*)(.*?)?(\s*)' + re.escape(close_tag),
+            flags=re.DOTALL
+        )
+        n_body = pattern.sub(lambda m: f"{m.group(1)}{open_md}{m.group(2)}{close_md}{m.group(3)}", n_body)
+    return n_body
+
+def apply_standard_markdown_to_body(n_body):
+    """
+    Apprise does not support ~~strikethrough~~ but it will convert <del> to HTML strikethrough.
+    :param n_body:
+    :return:
+    """
+    import re
+    # Define the mapping between your placeholders and markdown markers
+    replacements = [
+        (REMOVED_PLACEMARKER_OPEN, '<del>', REMOVED_PLACEMARKER_CLOSED, '</del>'),
+        (ADDED_PLACEMARKER_OPEN, '**', ADDED_PLACEMARKER_CLOSED, '**'),
+        (CHANGED_PLACEMARKER_OPEN, '<del>', CHANGED_PLACEMARKER_CLOSED, '</del>'),
+        (CHANGED_INTO_PLACEMARKER_OPEN, '**', CHANGED_INTO_PLACEMARKER_CLOSED, '**'),
+    ]
+
+    # So that the markdown gets added without any whitespace following it which would break it
+    for open_tag, open_md, close_tag, close_md in replacements:
+        # Regex: match opening tag, optional whitespace, capture the content, optional whitespace, then closing tag
+        pattern = re.compile(
+            re.escape(open_tag) + r'(\s*)(.*?)?(\s*)' + re.escape(close_tag),
+            flags=re.DOTALL
+        )
+        n_body = pattern.sub(lambda m: f"{m.group(1)}{open_md}{m.group(2)}{close_md}{m.group(3)}", n_body)
+    return n_body
+
+
+def apply_service_tweaks(url, n_body, n_title, requested_output_format):
+
+    # Re 323 - Limit discord length to their 2000 char limit total or it wont send.
+    # Because different notifications may require different pre-processing, run each sequentially :(
+    # 2000 bytes minus -
+    #     200 bytes for the overhead of the _entire_ json payload, 200 bytes for {tts, wait, content} etc headers
+    #     Length of URL - Incase they specify a longer custom avatar_url
+
+    if not n_body or not n_body.strip():
+        return url, n_body, n_title
+
+    # So if no avatar_url is specified, add one so it can be correctly calculated into the total payload
+    parsed = urlparse(url)
+    k = '?' if not parsed.query else '&'
+    if url and not 'avatar_url' in url \
+            and not url.startswith('mail') \
+            and not url.startswith('post') \
+            and not url.startswith('get') \
+            and not url.startswith('delete') \
+            and not url.startswith('put'):
+        url += k + f"avatar_url={APPRISE_AVATAR_URL}"
+
+    if url.startswith('tgram://'):
+        # Telegram only supports a limit subset of HTML, remove the '<br>' we place in.
+        # re https://github.com/dgtlmoon/changedetection.io/issues/555
+        # @todo re-use an existing library we have already imported to strip all non-allowed tags
+        n_body = n_body.replace('<br>', '\n')
+        n_body = n_body.replace('</br>', '\n')
+        n_body = n_body.replace(CUSTOM_LINEBREAK_PLACEHOLDER, '\n')
+
+        # Use strikethrough for removed content, bold for added content
+        n_body = n_body.replace(REMOVED_PLACEMARKER_OPEN, '<s>')
+        n_body = n_body.replace(REMOVED_PLACEMARKER_CLOSED, '</s>')
+        n_body = n_body.replace(ADDED_PLACEMARKER_OPEN, '<b>')
+        n_body = n_body.replace(ADDED_PLACEMARKER_CLOSED, '</b>')
+        # Handle changed/replaced lines (old → new)
+        n_body = n_body.replace(CHANGED_PLACEMARKER_OPEN, '<s>')
+        n_body = n_body.replace(CHANGED_PLACEMARKER_CLOSED, '</s>')
+        n_body = n_body.replace(CHANGED_INTO_PLACEMARKER_OPEN, '<b>')
+        n_body = n_body.replace(CHANGED_INTO_PLACEMARKER_CLOSED, '</b>')
+
+        # real limit is 4096, but minus some for extra metadata
+        payload_max_size = 3600
+        body_limit = max(0, payload_max_size - len(n_title))
+        n_title = n_title[0:payload_max_size]
+        n_body = n_body[0:body_limit]
+
+    elif (url.startswith('discord://') or url.startswith('https://discordapp.com/api/webhooks')
+          or url.startswith('https://discord.com/api'))\
+            and 'html' in requested_output_format:
+        # Discord doesn't support HTML, replace <br> with newlines
+        n_body = n_body.strip().replace('<br>', '\n')
+        n_body = n_body.replace('</br>', '\n')
+        n_body = n_body.replace(CUSTOM_LINEBREAK_PLACEHOLDER, '\n')
+
+        # Don't replace placeholders or truncate here - let the custom Discord plugin handle it
+        # The plugin will use embeds (6000 char limit across all embeds) if placeholders are present,
+        # or plain content (2000 char limit) otherwise
+
+        # Only do placeholder replacement if NOT using htmlcolor (which triggers embeds in custom plugin)
+        if requested_output_format == 'html':
+            # No diff placeholders, use Discord markdown for any other formatting
+            # Use Discord markdown: strikethrough for removed, bold for added
+            n_body = apply_discord_markdown_to_body(n_body=n_body)
+
+            # Apply 2000 char limit for plain content
+            payload_max_size = 1700
+            body_limit = max(0, payload_max_size - len(n_title))
+            n_title = n_title[0:payload_max_size]
+            n_body = n_body[0:body_limit]
+        # else: our custom Discord plugin will convert any placeholders left over into embeds with color bars
+
+    # Is not discord/tgram and they want htmlcolor
+    elif requested_output_format == 'htmlcolor':
+        # https://github.com/dgtlmoon/changedetection.io/issues/821#issuecomment-1241837050
+        n_body = n_body.replace(REMOVED_PLACEMARKER_OPEN, f'<span style="{HTML_REMOVED_STYLE}" role="deletion" aria-label="Removed text" title="Removed text">')
+        n_body = n_body.replace(REMOVED_PLACEMARKER_CLOSED, f'</span>')
+        n_body = n_body.replace(ADDED_PLACEMARKER_OPEN, f'<span style="{HTML_ADDED_STYLE}" role="insertion" aria-label="Added text" title="Added text">')
+        n_body = n_body.replace(ADDED_PLACEMARKER_CLOSED, f'</span>')
+        # Handle changed/replaced lines (old → new)
+        n_body = n_body.replace(CHANGED_PLACEMARKER_OPEN, f'<span style="{HTML_CHANGED_STYLE}" role="note" aria-label="Changed text" title="Changed text">')
+        n_body = n_body.replace(CHANGED_PLACEMARKER_CLOSED, f'</span>')
+        n_body = n_body.replace(CHANGED_INTO_PLACEMARKER_OPEN, f'<span style="{HTML_CHANGED_INTO_STYLE}" role="note" aria-label="Changed into" title="Changed into">')
+        n_body = n_body.replace(CHANGED_INTO_PLACEMARKER_CLOSED, f'</span>')
+        n_body = n_body.replace('\n', f'{CUSTOM_LINEBREAK_PLACEHOLDER}\n')
+    elif requested_output_format == 'html':
+        n_body = n_body.replace(REMOVED_PLACEMARKER_OPEN, '(removed) ')
+        n_body = n_body.replace(REMOVED_PLACEMARKER_CLOSED, '')
+        n_body = n_body.replace(ADDED_PLACEMARKER_OPEN, '(added) ')
+        n_body = n_body.replace(ADDED_PLACEMARKER_CLOSED, '')
+        n_body = n_body.replace(CHANGED_PLACEMARKER_OPEN, f'(changed) ')
+        n_body = n_body.replace(CHANGED_PLACEMARKER_CLOSED, f'')
+        n_body = n_body.replace(CHANGED_INTO_PLACEMARKER_OPEN, f'(into) ')
+        n_body = n_body.replace(CHANGED_INTO_PLACEMARKER_CLOSED, f'')
+        n_body = n_body.replace('\n', f'{CUSTOM_LINEBREAK_PLACEHOLDER}\n')
+    elif requested_output_format == 'markdown':
+        # Markdown to HTML - Apprise will convert this to HTML
+        n_body = apply_standard_markdown_to_body(n_body=n_body)
+
+    else: #plaintext etc default
+        n_body = n_body.replace(REMOVED_PLACEMARKER_OPEN, '(removed) ')
+        n_body = n_body.replace(REMOVED_PLACEMARKER_CLOSED, '')
+        n_body = n_body.replace(ADDED_PLACEMARKER_OPEN, '(added) ')
+        n_body = n_body.replace(ADDED_PLACEMARKER_CLOSED, '')
+        n_body = n_body.replace(CHANGED_PLACEMARKER_OPEN, f'(changed) ')
+        n_body = n_body.replace(CHANGED_PLACEMARKER_CLOSED, f'')
+        n_body = n_body.replace(CHANGED_INTO_PLACEMARKER_OPEN, f'(into) ')
+        n_body = n_body.replace(CHANGED_INTO_PLACEMARKER_CLOSED, f'')
+
+    return url, n_body, n_title
+
+
 def process_notification(n_object: NotificationContextData, datastore):
     from changedetectionio.jinja2_custom import render as jinja_render
-    from . import default_notification_format_for_watch, default_notification_format, valid_notification_formats
+    from . import USE_SYSTEM_DEFAULT_NOTIFICATION_FORMAT_FOR_WATCH, default_notification_format, valid_notification_formats
     # be sure its registered
     from .apprise_plugin.custom_handlers import apprise_http_custom_handler
+    # Register custom Discord plugin
+    from .apprise_plugin.discord import NotifyDiscordCustom
 
     if not isinstance(n_object, NotificationContextData):
         raise TypeError(f"Expected NotificationContextData, got {type(n_object)}")
@@ -87,17 +257,18 @@ def process_notification(n_object: NotificationContextData, datastore):
     # Insert variables into the notification content
     notification_parameters = create_notification_parameters(n_object, datastore)
 
-    n_format = valid_notification_formats.get(
-        n_object.get('notification_format', default_notification_format),
-        valid_notification_formats[default_notification_format],
-    )
+    requested_output_format = n_object.get('notification_format', default_notification_format)
+    logger.debug(f"Requested notification output format: '{requested_output_format}'")
 
     # If we arrived with 'System default' then look it up
-    if n_format == default_notification_format_for_watch and datastore.data['settings']['application'].get('notification_format') != default_notification_format_for_watch:
+    if requested_output_format == USE_SYSTEM_DEFAULT_NOTIFICATION_FORMAT_FOR_WATCH:
         # Initially text or whatever
-        n_format = datastore.data['settings']['application'].get('notification_format', valid_notification_formats[default_notification_format]).lower()
+        requested_output_format = datastore.data['settings']['application'].get('notification_format', default_notification_format)
 
-    n_format = notification_format_align_with_apprise(n_format=n_format)
+    requested_output_format_original = requested_output_format
+
+    # Now clean it up so it fits perfectly with apprise
+    requested_output_format = notification_format_align_with_apprise(n_format=requested_output_format)
 
     logger.trace(f"Complete notification body including Jinja and placeholders calculated in  {time.time() - now:.2f}s")
 
@@ -112,21 +283,24 @@ def process_notification(n_object: NotificationContextData, datastore):
 
     apobj = apprise.Apprise(debug=True, asset=apprise_asset)
 
+    # Override Apprise's built-in Discord plugin with our custom one
+    # This allows us to use colored embeds for diff content
+    # First remove the built-in discord plugin, then add our custom one
+    apprise.plugins.N_MGR.remove('discord')
+    apprise.plugins.N_MGR.add(NotifyDiscordCustom, schemas='discord')
+
     if not n_object.get('notification_urls'):
         return None
 
-    with apprise.LogCapture(level=apprise.logging.DEBUG) as logs:
+    with (apprise.LogCapture(level=apprise.logging.DEBUG) as logs):
         for url in n_object['notification_urls']:
 
             # Get the notification body from datastore
             n_body = jinja_render(template_str=n_object.get('notification_body', ''), **notification_parameters)
 
-            if n_object.get('markup_text_to_html'):
+            if n_object.get('markup_text_links_to_html_links'):
                 n_body = markup_text_links_to_html(body=n_body)
 
-            if n_format == NotifyFormat.HTML.value:
-                n_body = n_body.replace("\n", '<br>')
-
             n_title = jinja_render(template_str=n_object.get('notification_title', ''), **notification_parameters)
 
             url = url.strip()
@@ -141,88 +315,84 @@ def process_notification(n_object: NotificationContextData, datastore):
             logger.info(f">> Process Notification: AppRise notifying {url}")
             url = jinja_render(template_str=url, **notification_parameters)
 
-            # Re 323 - Limit discord length to their 2000 char limit total or it wont send.
-            # Because different notifications may require different pre-processing, run each sequentially :(
-            # 2000 bytes minus -
-            #     200 bytes for the overhead of the _entire_ json payload, 200 bytes for {tts, wait, content} etc headers
-            #     Length of URL - Incase they specify a longer custom avatar_url
+            # If it's a plaintext document, and they want HTML type email/alerts, so it needs to be escaped
+            watch_mime_type = n_object.get('watch_mime_type')
+            if watch_mime_type and 'text/' in watch_mime_type.lower() and not 'html' in watch_mime_type.lower():
+                if 'html' in requested_output_format:
+                    from markupsafe import escape
+                    n_body = str(escape(n_body))
 
-            # So if no avatar_url is specified, add one so it can be correctly calculated into the total payload
-            parsed = urlparse(url)
-            k = '?' if not parsed.query else '&'
-            if not 'avatar_url' in url \
-                    and not url.startswith('mail') \
-                    and not url.startswith('post') \
-                    and not url.startswith('get') \
-                    and not url.startswith('delete') \
-                    and not url.startswith('put'):
-                url += k + f"avatar_url={APPRISE_AVATAR_URL}"
+            if 'html' in requested_output_format:
+                # Since the n_body is always some kind of text from the 'diff' engine, attempt to preserve whitespaces that get sent to the HTML output
+                # But only where its more than 1 consecutive whitespace, otherwise "and this" becomes "and&nbsp;this" etc which is too much.
+                n_body = n_body.replace('  ', '&nbsp;&nbsp;')
 
-            if url.startswith('tgram://'):
-                # Telegram only supports a limit subset of HTML, remove the '<br>' we place in.
-                # re https://github.com/dgtlmoon/changedetection.io/issues/555
-                # @todo re-use an existing library we have already imported to strip all non-allowed tags
-                n_body = n_body.replace('<br>', '\n')
-                n_body = n_body.replace('</br>', '\n')
-                # real limit is 4096, but minus some for extra metadata
-                payload_max_size = 3600
-                body_limit = max(0, payload_max_size - len(n_title))
-                n_title = n_title[0:payload_max_size]
-                n_body = n_body[0:body_limit]
+            (url, n_body, n_title) = apply_service_tweaks(url=url, n_body=n_body, n_title=n_title, requested_output_format=requested_output_format_original)
 
-            elif url.startswith('discord://') or url.startswith('https://discordapp.com/api/webhooks') or url.startswith(
-                    'https://discord.com/api'):
-                # real limit is 2000, but minus some for extra metadata
-                payload_max_size = 1700
-                body_limit = max(0, payload_max_size - len(n_title))
-                n_title = n_title[0:payload_max_size]
-                n_body = n_body[0:body_limit]
+            apprise_input_format = "NO-THANKS-WE-WILL-MANAGE-ALL-OF-THIS"
 
-            # Add format parameter to mailto URLs to ensure proper text/html handling
-            # https://github.com/caronc/apprise/issues/633#issuecomment-1191449321
-            # Note: Custom handlers (post://, get://, etc.) don't need this as we handle them
-            # differently by passing an invalid body_format to prevent HTML conversion
-            if not 'format=' in url and url.startswith(('mailto', 'mailtos')):
-                parsed = urlparse(url)
-                prefix = '?' if not parsed.query else '&'
-                # Apprise format is already lowercase from notification_format_align_with_apprise()
-                url = f"{url}{prefix}format={n_format}"
+            if not 'format=' in url:
+                parsed_url = urlparse(url)
+                prefix_add_to_url = '?' if not parsed_url.query else '&'
 
-            apobj.add(url)
+                # THIS IS THE TRICK HOW TO DISABLE APPRISE DOING WEIRD AUTO-CONVERSION WITH BREAKING BR TAGS ETC
+                if 'html' in requested_output_format:
+                    url = f"{url}{prefix_add_to_url}format={NotifyFormat.HTML.value}"
+                    apprise_input_format = NotifyFormat.HTML.value
+                elif 'text' in requested_output_format:
+                    url = f"{url}{prefix_add_to_url}format={NotifyFormat.TEXT.value}"
+                    apprise_input_format = NotifyFormat.TEXT.value
+
+                elif requested_output_format == NotifyFormat.MARKDOWN.value:
+                    # Convert markdown to HTML ourselves since not all plugins do this
+                    from apprise.conversion import markdown_to_html
+                    # Make sure there are paragraph breaks around horizontal rules
+                    n_body = n_body.replace('---', '\n\n---\n\n')
+                    n_body = markdown_to_html(n_body)
+                    url = f"{url}{prefix_add_to_url}format={NotifyFormat.HTML.value}"
+                    requested_output_format = NotifyFormat.HTML.value
+                    apprise_input_format = NotifyFormat.HTML.value  # Changed from MARKDOWN to HTML
+
+                # Could have arrived at any stage, so we dont end up running .escape on it
+                if 'html' in requested_output_format:
+                    n_body = n_body.replace(CUSTOM_LINEBREAK_PLACEHOLDER, '<br>\r\n')
+                else:
+                    # texty types
+                    n_body = n_body.replace(CUSTOM_LINEBREAK_PLACEHOLDER, '\r\n')
+
+            else:
+                # ?format was IN the apprise URL, they are kind of on their own here, we will try our best
+                if 'format=html' in url:
+                    n_body = n_body.replace(CUSTOM_LINEBREAK_PLACEHOLDER, '<br>\r\n')
+                    # This will also prevent apprise from doing conversion
+                    apprise_input_format = NotifyFormat.HTML.value
+                    requested_output_format = NotifyFormat.HTML.value
+                elif 'format=text' in url:
+                    n_body = n_body.replace(CUSTOM_LINEBREAK_PLACEHOLDER, '\r\n')
+                    apprise_input_format = NotifyFormat.TEXT.value
+                    requested_output_format = NotifyFormat.TEXT.value
 
             sent_objs.append({'title': n_title,
                               'body': n_body,
-                              'url': url,
-                              'body_format': n_format})
+                              'url': url})
+            apobj.add(url)
 
-        # Blast off the notifications tht are set in .add()
-        # Check if we have any custom HTTP handlers (post://, get://, etc.)
-        # These handlers created with @notify decorator don't handle format conversion properly
-        # and will strip HTML if we pass a valid format. So we pass an invalid format string
-        # to prevent Apprise from converting HTML->TEXT
-
-        # Create list of custom handler protocols (both http and https versions)
-        custom_handler_protocols = [f"{method}://" for method in SUPPORTED_HTTP_METHODS]
-        custom_handler_protocols += [f"{method}s://" for method in SUPPORTED_HTTP_METHODS]
-
-        has_custom_handler = any(
-            url.startswith(tuple(custom_handler_protocols))
-            for url in n_object['notification_urls']
-        )
-
-        # If we have custom handlers, use invalid format to prevent conversion
-        # Otherwise use the proper format
-        notify_format = 'raw-no-convert' if has_custom_handler else n_format
+            # Since the output is always based on the plaintext of the 'diff' engine, wrap it nicely.
+            # It should always be similar to the 'history' part of the UI.
+            if url.startswith('mail') and 'html' in requested_output_format:
+                if not '<pre' in n_body and not '<body' in n_body: # No custom HTML-ish body was setup already
+                    n_body = as_monospaced_html_email(content=n_body, title=n_title)
 
         apobj.notify(
             title=n_title,
             body=n_body,
-            body_format=notify_format,
+            # `body_format` Tell apprise what format the INPUT is in, specify a wrong/bad type and it will force skip conversion in apprise
+            # &format= in URL Tell apprise what format the OUTPUT should be in (it can convert between)
+            body_format=apprise_input_format,
             # False is not an option for AppRise, must be type None
             attach=n_object.get('screenshot', None)
         )
 
-
         # Returns empty string if nothing found, multi-line string otherwise
         log_value = logs.getvalue()
 

changedetectionio/notification_service.py

@@ -9,29 +9,35 @@ for both sync and async workers
 from loguru import logger
 import time
 
-from changedetectionio.notification import default_notification_format
+from changedetectionio.model import USE_SYSTEM_DEFAULT_NOTIFICATION_FORMAT_FOR_WATCH
+from changedetectionio.notification import default_notification_format, valid_notification_formats
+
+# This gets modified on notification time (handler.py) depending on the required notification output
+CUSTOM_LINEBREAK_PLACEHOLDER='@BR@'
+
 
 # What is passed around as notification context, also used as the complete list of valid {{ tokens }}
 class NotificationContextData(dict):
     def __init__(self, initial_data=None, **kwargs):
         super().__init__({
+            'base_url': None,
             'current_snapshot': None,
             'diff': None,
             'diff_added': None,
             'diff_full': None,
             'diff_patch': None,
             'diff_removed': None,
+            'diff_url': None,
+            'markup_text_links_to_html_links': False, # If automatic conversion of plaintext to HTML should happen
             'notification_timestamp': time.time(),
+            'preview_url': None,
             'screenshot': None,
             'triggered_text': None,
             'uuid': 'XXXXXXXX-XXXX-XXXX-XXXX-XXXXXXXXXXXX',  # Converted to 'watch_uuid' in create_notification_parameters
-            'watch_url': 'https://WATCH-PLACE-HOLDER/',
-            'base_url': None,
-            'diff_url': None,
-            'preview_url': None,
+            'watch_mime_type': None,
             'watch_tag': None,
             'watch_title': None,
-            'markup_text_to_html': False, # If automatic conversion of plaintext to HTML should happen
+            'watch_url': 'https://WATCH-PLACE-HOLDER/',
         })
 
         # Apply any initial data passed in
@@ -43,15 +49,28 @@ class NotificationContextData(dict):
         if kwargs:
             self.update(kwargs)
 
+        n_format = self.get('notification_format')
+        if n_format and not valid_notification_formats.get(n_format):
+            raise ValueError(f'Invalid notification format: "{n_format}"')
+
     def set_random_for_validation(self):
         import random, string
-        """Randomly fills all dict keys with random strings (for validation/testing)."""
+        """Randomly fills all dict keys with random strings (for validation/testing). 
+        So we can test the output in the notification body
+        """
         for key in self.keys():
             if key in ['uuid', 'time', 'watch_uuid']:
                 continue
             rand_str = 'RANDOM-PLACEHOLDER-'+''.join(random.choices(string.ascii_letters + string.digits, k=12))
             self[key] = rand_str
 
+    def __setitem__(self, key, value):
+        if key == 'notification_format' and isinstance(value, str) and not value.startswith('RANDOM-PLACEHOLDER-'):
+            if not valid_notification_formats.get(value):
+                raise ValueError(f'Invalid notification format: "{value}"')
+
+        super().__setitem__(key, value)
+
 class NotificationService:
     """
     Standalone notification service that handles all notification functionality
@@ -67,7 +86,7 @@ class NotificationService:
         Queue a notification for a watch with full diff rendering and template variables
         """
         from changedetectionio import diff
-        from changedetectionio.notification import default_notification_format_for_watch
+        from changedetectionio.notification import USE_SYSTEM_DEFAULT_NOTIFICATION_FORMAT_FOR_WATCH
 
         if not isinstance(n_object, NotificationContextData):
             raise TypeError(f"Expected NotificationContextData, got {type(n_object)}")
@@ -89,29 +108,16 @@ class NotificationService:
             snapshot_contents = "No snapshot/history available, the watch should fetch atleast once."
 
         # If we ended up here with "System default"
-        if n_object.get('notification_format') == default_notification_format_for_watch:
+        if n_object.get('notification_format') == USE_SYSTEM_DEFAULT_NOTIFICATION_FORMAT_FOR_WATCH:
             n_object['notification_format'] = self.datastore.data['settings']['application'].get('notification_format')
 
-        html_colour_enable = False
-        # HTML needs linebreak, but MarkDown and Text can use a linefeed
-        if n_object.get('notification_format') == 'HTML':
-            line_feed_sep = "<br>"
-            # Snapshot will be plaintext on the disk, convert to some kind of HTML
-            snapshot_contents = snapshot_contents.replace('\n', line_feed_sep)
-        elif n_object.get('notification_format') == 'HTML Color':
-            line_feed_sep = "<br>"
-            # Snapshot will be plaintext on the disk, convert to some kind of HTML
-            snapshot_contents = snapshot_contents.replace('\n', line_feed_sep)
-            html_colour_enable = True
-        else:
-            line_feed_sep = "\n"
 
         triggered_text = ''
         if len(trigger_text):
             from . import html_tools
             triggered_text = html_tools.get_triggered_text(content=snapshot_contents, trigger_text=trigger_text)
             if triggered_text:
-                triggered_text = line_feed_sep.join(triggered_text)
+                triggered_text = CUSTOM_LINEBREAK_PLACEHOLDER.join(triggered_text)
 
         # Could be called as a 'test notification' with only 1 snapshot available
         prev_snapshot = "Example text: example test\nExample text: change detection is cool\nExample text: some more examples\n"
@@ -123,16 +129,17 @@ class NotificationService:
 
         n_object.update({
             'current_snapshot': snapshot_contents,
-            'diff': diff.render_diff(prev_snapshot, current_snapshot, line_feed_sep=line_feed_sep, html_colour=html_colour_enable),
-            'diff_added': diff.render_diff(prev_snapshot, current_snapshot, include_removed=False, line_feed_sep=line_feed_sep, html_colour=html_colour_enable),
-            'diff_full': diff.render_diff(prev_snapshot, current_snapshot, include_equal=True, line_feed_sep=line_feed_sep, html_colour=html_colour_enable),
-            'diff_patch': diff.render_diff(prev_snapshot, current_snapshot, line_feed_sep=line_feed_sep, patch_format=True),
-            'diff_removed': diff.render_diff(prev_snapshot, current_snapshot, include_added=False, line_feed_sep=line_feed_sep, html_colour=html_colour_enable),
+            'diff': diff.render_diff(prev_snapshot, current_snapshot, line_feed_sep=CUSTOM_LINEBREAK_PLACEHOLDER),
+            'diff_added': diff.render_diff(prev_snapshot, current_snapshot, include_removed=False, line_feed_sep=CUSTOM_LINEBREAK_PLACEHOLDER),
+            'diff_full': diff.render_diff(prev_snapshot, current_snapshot, include_equal=True, line_feed_sep=CUSTOM_LINEBREAK_PLACEHOLDER),
+            'diff_patch': diff.render_diff(prev_snapshot, current_snapshot, line_feed_sep=CUSTOM_LINEBREAK_PLACEHOLDER, patch_format=True),
+            'diff_removed': diff.render_diff(prev_snapshot, current_snapshot, include_added=False, line_feed_sep=CUSTOM_LINEBREAK_PLACEHOLDER),
             'screenshot': watch.get_screenshot() if watch and watch.get('notification_screenshot') else None,
             'triggered_text': triggered_text,
             'uuid': watch.get('uuid') if watch else None,
             'watch_url': watch.get('url') if watch else None,
             'watch_uuid': watch.get('uuid') if watch else None,
+            'watch_mime_type': watch.get('content-type')
         })
 
         if watch:
@@ -148,7 +155,7 @@ class NotificationService:
         Individual watch settings > Tag settings > Global settings
         """
         from changedetectionio.notification import (
-            default_notification_format_for_watch,
+            USE_SYSTEM_DEFAULT_NOTIFICATION_FORMAT_FOR_WATCH,
             default_notification_body,
             default_notification_title
         )
@@ -156,7 +163,7 @@ class NotificationService:
         # Would be better if this was some kind of Object where Watch can reference the parent datastore etc
         v = watch.get(var_name)
         if v and not watch.get('notification_muted'):
-            if var_name == 'notification_format' and v == default_notification_format_for_watch:
+            if var_name == 'notification_format' and v == USE_SYSTEM_DEFAULT_NOTIFICATION_FORMAT_FOR_WATCH:
                 return self.datastore.data['settings']['application'].get('notification_format')
 
             return v
@@ -173,7 +180,7 @@ class NotificationService:
 
         # Otherwise could be defaults
         if var_name == 'notification_format':
-            return default_notification_format_for_watch
+            return USE_SYSTEM_DEFAULT_NOTIFICATION_FORMAT_FOR_WATCH
         if var_name == 'notification_body':
             return default_notification_body
         if var_name == 'notification_title':
@@ -228,9 +235,8 @@ class NotificationService:
         if not watch:
             return
 
-        n_format = self.datastore.data['settings']['application'].get('notification_format', default_notification_format)
         filter_list = ", ".join(watch['include_filters'])
-        # @todo - This could be a markdown template on the disk, apprise will convert the markdown to HTML+Plaintext parts in the email, and then 'markup_text_to_html' is not needed
+        # @todo - This could be a markdown template on the disk, apprise will convert the markdown to HTML+Plaintext parts in the email, and then 'markup_text_links_to_html_links' is not needed
         body = f"""Hello,
 
 Your configured CSS/xPath filters of '{filter_list}' for {{{{watch_url}}}} did not appear on the page after {threshold} attempts.
@@ -245,9 +251,9 @@ Thanks - Your omniscient changedetection.io installation.
         n_object = NotificationContextData({
             'notification_title': 'Changedetection.io - Alert - CSS/xPath filter was not present in the page',
             'notification_body': body,
-            'notification_format': n_format,
-            'markup_text_to_html': n_format.lower().startswith('html')
+            'notification_format': self._check_cascading_vars('notification_format', watch),
         })
+        n_object['markup_text_links_to_html_links'] = n_object.get('notification_format').startswith('html')
 
         if len(watch['notification_urls']):
             n_object['notification_urls'] = watch['notification_urls']
@@ -275,9 +281,9 @@ Thanks - Your omniscient changedetection.io installation.
         if not watch:
             return
         threshold = self.datastore.data['settings']['application'].get('filter_failure_notification_threshold_attempts')
-        n_format = self.datastore.data['settings']['application'].get('notification_format', default_notification_format).lower()
+
         step = step_n + 1
-        # @todo - This could be a markdown template on the disk, apprise will convert the markdown to HTML+Plaintext parts in the email, and then 'markup_text_to_html' is not needed
+        # @todo - This could be a markdown template on the disk, apprise will convert the markdown to HTML+Plaintext parts in the email, and then 'markup_text_links_to_html_links' is not needed
 
         # {{{{ }}}} because this will be Jinja2 {{ }} tokens
         body = f"""Hello,
@@ -294,9 +300,9 @@ Thanks - Your omniscient changedetection.io installation.
         n_object = NotificationContextData({
             'notification_title': f"Changedetection.io - Alert - Browser step at position {step} could not be run",
             'notification_body': body,
-            'notification_format': n_format,
-            'markup_text_to_html': n_format.lower().startswith('html')
+            'notification_format': self._check_cascading_vars('notification_format', watch),
         })
+        n_object['markup_text_links_to_html_links'] = n_object.get('notification_format').startswith('html')
 
         if len(watch['notification_urls']):
             n_object['notification_urls'] = watch['notification_urls']

changedetectionio/processors/__init__.py

@@ -91,6 +91,8 @@ class difference_detection_processor():
             else:
                 logger.debug("Skipping adding proxy data when custom Browser endpoint is specified. ")
 
+        logger.debug(f"Using proxy '{proxy_url}' for {self.watch['uuid']}")
+
         # Now call the fetcher (playwright/requests/etc) with arguments that only a fetcher would need.
         # When browser_connection_url is None, it method should default to working out whats the best defaults (os env vars etc)
         self.fetcher = fetcher_obj(proxy_override=proxy_url,

changedetectionio/processors/magic.py

@@ -88,7 +88,7 @@ class guess_stream_type():
                         magic_content_header = mime
 
         except Exception as e:
-            logger.error(f"Error getting a more precise mime type from 'puremagic' library ({str(e)}), using content-based detection")
+            logger.warning(f"Error getting a more precise mime type from 'puremagic' library ({str(e)}), using content-based detection")
 
         # Content-based detection (most reliable for text formats)
         # Check for HTML patterns first - if found, override magic's text/plain

changedetectionio/processors/text_json_diff/__init__.py

@@ -32,7 +32,7 @@ def prepare_filter_prevew(datastore, watch_uuid, form_data):
     '''Used by @app.route("/edit/<string:uuid>/preview-rendered", methods=['POST'])'''
     from changedetectionio import forms, html_tools
     from changedetectionio.model.Watch import model as watch_model
-    from concurrent.futures import ProcessPoolExecutor
+    from concurrent.futures import ThreadPoolExecutor
     from copy import deepcopy
     from flask import request
     import brotli
@@ -76,13 +76,16 @@ def prepare_filter_prevew(datastore, watch_uuid, form_data):
             update_handler.fetcher.headers['content-type'] = tmp_watch.get('content-type')
 
             # Process our watch with filters and the HTML from disk, and also a blank watch with no filters but also with the same HTML from disk
-            # Do this as a parallel process because it could take some time
-            with ProcessPoolExecutor(max_workers=2) as executor:
-                future1 = executor.submit(_task, tmp_watch, update_handler)
-                future2 = executor.submit(_task, blank_watch_no_filters, update_handler)
+            # Do this as parallel threads (not processes) to avoid pickle issues with Lock objects
+            try:
+                with ThreadPoolExecutor(max_workers=2) as executor:
+                    future1 = executor.submit(_task, tmp_watch, update_handler)
+                    future2 = executor.submit(_task, blank_watch_no_filters, update_handler)
 
-                text_after_filter = future1.result()
-                text_before_filter = future2.result()
+                    text_after_filter = future1.result()
+                    text_before_filter = future2.result()
+            except Exception as e:
+                x=1
 
     try:
         trigger_line_numbers = html_tools.strip_ignore_text(content=text_after_filter,

changedetectionio/processors/text_json_diff/processor.py

@@ -7,6 +7,7 @@ import re
 import urllib3
 
 from changedetectionio.conditions import execute_ruleset_against_all_plugins
+from changedetectionio.diff import ADDED_PLACEMARKER_OPEN
 from changedetectionio.processors import difference_detection_processor
 from changedetectionio.html_tools import PERL_STYLE_REGEX, cdata_in_document_to_text, TRANSLATE_WHITESPACE_TABLE
 from changedetectionio import html_tools, content_fetchers

changedetectionio/pytest.ini

@@ -1,5 +1,5 @@
 [pytest]
-addopts = --no-start-live-server --live-server-port=5005
+addopts = --no-start-live-server --live-server-port=0
 #testpaths = tests pytest_invenio
 #live_server_scope = function
 

changedetectionio/run_basic_tests.sh

@@ -11,19 +11,16 @@ set -e
 
 SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )
 
-find tests/test_*py -type f|while read test_name
-do
-  echo "TEST RUNNING $test_name"
-  # REMOVE_REQUESTS_OLD_SCREENSHOTS disabled so that we can write a screenshot and send it in test_notifications.py without a real browser
-  REMOVE_REQUESTS_OLD_SCREENSHOTS=false pytest -vv -s --maxfail=1 --tb=long $test_name
-done
+# REMOVE_REQUESTS_OLD_SCREENSHOTS disabled so that we can write a screenshot and send it in test_notifications.py without a real browser
+REMOVE_REQUESTS_OLD_SCREENSHOTS=false pytest -n 30 --dist load  tests/test_*.py
 
+#time pytest -n auto --dist loadfile -vv --tb=long tests/test_*.py
 echo "RUNNING WITH BASE_URL SET"
 
 # Now re-run some tests with BASE_URL enabled
 # Re #65 - Ability to include a link back to the installation, in the notification.
 export BASE_URL="https://really-unique-domain.io"
-REMOVE_REQUESTS_OLD_SCREENSHOTS=false pytest -vv -s --maxfail=1 tests/test_notification.py
+REMOVE_REQUESTS_OLD_SCREENSHOTS=false pytest -vv --maxfail=1 tests/test_notification.py
 
 
 # Re-run with HIDE_REFERER set - could affect login

changedetectionio/run_custom_browser_url_tests.sh

@@ -6,6 +6,8 @@
 
 # enable debug
 set -x
+docker network inspect changedet-network >/dev/null 2>&1 || docker network create changedet-network
+docker run --network changedet-network -d --hostname selenium  -p 4444:4444 --rm --shm-size="2g"  selenium/standalone-chrome:4
 
 # A extra browser is configured, but we never chose to use it, so it should NOT show in the logs
 docker run --rm -e "PLAYWRIGHT_DRIVER_URL=ws://sockpuppetbrowser:3000" --network changedet-network test-changedetectionio  bash -c 'cd changedetectionio;pytest tests/custom_browser_url/test_custom_browser_url.py::test_request_not_via_custom_browser_url'

changedetectionio/run_proxy_tests.sh

@@ -19,12 +19,13 @@ docker run --network changedet-network -d \
   -v `pwd`/tests/proxy_list/squid-passwords.txt:/etc/squid3/passwords \
   ubuntu/squid:4.13-21.10_edge
 
-
+sleep 5
 ## 2nd test actually choose the preferred proxy from proxies.json
+# This will force a request via "proxy-two"
 docker run --network changedet-network \
-  -v `pwd`/tests/proxy_list/proxies.json-example:/app/changedetectionio/test-datastore/proxies.json \
+  -v `pwd`/tests/proxy_list/proxies.json-example:/tmp/proxies.json \
   test-changedetectionio \
-  bash -c 'cd changedetectionio && pytest tests/proxy_list/test_multiple_proxy.py'
+  bash -c 'cd changedetectionio && pytest -s tests/proxy_list/test_multiple_proxy.py --datastore-path /tmp'
 
 set +e
 echo "- Looking for chosen.changedetection.io request in squid-one - it should NOT be here"
@@ -48,8 +49,10 @@ fi
 # Test the UI configurable proxies
 docker run --network changedet-network \
   test-changedetectionio \
-  bash -c 'cd changedetectionio && pytest tests/proxy_list/test_select_custom_proxy.py'
+  bash -c 'cd changedetectionio && pytest tests/proxy_list/test_select_custom_proxy.py --datastore-path /tmp'
 
+# Give squid proxies a moment to flush their logs
+sleep 2
 
 # Should see a request for one.changedetection.io in there
 echo "- Looking for .changedetection.io request in squid-custom"
@@ -63,7 +66,10 @@ fi
 # Test "no-proxy" option
 docker run --network changedet-network \
   test-changedetectionio \
-  bash -c 'cd changedetectionio && pytest tests/proxy_list/test_noproxy.py'
+  bash -c 'cd changedetectionio && pytest tests/proxy_list/test_noproxy.py --datastore-path /tmp'
+
+# Give squid proxies a moment to flush their logs
+sleep 2
 
 # We need to handle grep returning 1
 set +e
@@ -80,6 +86,8 @@ for c in $(echo "squid-one squid-two squid-custom"); do
   fi
 done
 
+echo "docker ps output"
+docker ps
 
 docker kill squid-one squid-two squid-custom
 
@@ -88,19 +96,19 @@ docker kill squid-one squid-two squid-custom
 # Requests
 docker run --network changedet-network \
   test-changedetectionio \
-  bash -c 'cd changedetectionio && pytest tests/proxy_list/test_proxy_noconnect.py'
+  bash -c 'cd changedetectionio && pytest tests/proxy_list/test_proxy_noconnect.py --datastore-path /tmp'
 
 # Playwright
 docker run --network changedet-network \
   test-changedetectionio \
-  bash -c 'cd changedetectionio && PLAYWRIGHT_DRIVER_URL=ws://sockpuppetbrowser:3000 pytest tests/proxy_list/test_proxy_noconnect.py'
+  bash -c 'cd changedetectionio && PLAYWRIGHT_DRIVER_URL=ws://sockpuppetbrowser:3000 pytest tests/proxy_list/test_proxy_noconnect.py --datastore-path /tmp'
 
 # Puppeteer fast
 docker run --network changedet-network \
   test-changedetectionio \
-  bash -c 'cd changedetectionio && FAST_PUPPETEER_CHROME_FETCHER=1 PLAYWRIGHT_DRIVER_URL=ws://sockpuppetbrowser:3000 pytest tests/proxy_list/test_proxy_noconnect.py'
+  bash -c 'cd changedetectionio && FAST_PUPPETEER_CHROME_FETCHER=1 PLAYWRIGHT_DRIVER_URL=ws://sockpuppetbrowser:3000 pytest tests/proxy_list/test_proxy_noconnect.py --datastore-path /tmp'
 
 # Selenium
 docker run --network changedet-network \
   test-changedetectionio \
-  bash -c 'cd changedetectionio && WEBDRIVER_URL=http://selenium:4444/wd/hub pytest tests/proxy_list/test_proxy_noconnect.py'
+  bash -c 'cd changedetectionio && WEBDRIVER_URL=http://selenium:4444/wd/hub pytest tests/proxy_list/test_proxy_noconnect.py --datastore-path /tmp'

changedetectionio/run_socks_proxy_tests.sh

@@ -5,6 +5,7 @@ set -e
 # enable debug
 set -x
 
+docker network inspect changedet-network >/dev/null 2>&1 || docker network create changedet-network
 
 # SOCKS5 related - start simple Socks5 proxy server
 # SOCKSTEST=xyz should show in the logs of this service to confirm it fetched
@@ -14,13 +15,13 @@ docker run --network changedet-network -d --hostname socks5proxy-noauth --rm -p
 echo "---------------------------------- SOCKS5 -------------------"
 # SOCKS5 related - test from proxies.json
 docker run --network changedet-network \
-  -v `pwd`/tests/proxy_socks5/proxies.json-example:/app/changedetectionio/test-datastore/proxies.json \
+  -v `pwd`/tests/proxy_socks5/proxies.json-example:/tmp/proxies.json \
   --rm \
   -e "FLASK_SERVER_NAME=cdio" \
   --hostname cdio \
   -e "SOCKSTEST=proxiesjson" \
   test-changedetectionio \
-  bash -c 'cd changedetectionio && pytest --live-server-host=0.0.0.0 --live-server-port=5004  -s tests/proxy_socks5/test_socks5_proxy_sources.py'
+  bash -c 'cd changedetectionio && pytest --live-server-host=0.0.0.0 --live-server-port=5004  -s tests/proxy_socks5/test_socks5_proxy_sources.py  --datastore-path /tmp'
 
 # SOCKS5 related - by manually entering in UI
 docker run --network changedet-network \
@@ -29,18 +30,18 @@ docker run --network changedet-network \
   --hostname cdio \
   -e "SOCKSTEST=manual" \
   test-changedetectionio \
-  bash -c 'cd changedetectionio && pytest --live-server-host=0.0.0.0 --live-server-port=5004  -s tests/proxy_socks5/test_socks5_proxy.py'
+  bash -c 'cd changedetectionio && pytest --live-server-host=0.0.0.0 --live-server-port=5004  -s tests/proxy_socks5/test_socks5_proxy.py --datastore-path /tmp'
 
 # SOCKS5 related - test from proxies.json via playwright - NOTE- PLAYWRIGHT DOESNT SUPPORT AUTHENTICATING PROXY
 docker run --network changedet-network \
   -e "SOCKSTEST=manual-playwright" \
   --hostname cdio \
   -e "FLASK_SERVER_NAME=cdio" \
-  -v `pwd`/tests/proxy_socks5/proxies.json-example-noauth:/app/changedetectionio/test-datastore/proxies.json \
+  -v `pwd`/tests/proxy_socks5/proxies.json-example-noauth:/tmp/proxies.json \
   -e "PLAYWRIGHT_DRIVER_URL=ws://sockpuppetbrowser:3000" \
   --rm \
   test-changedetectionio \
-  bash -c 'cd changedetectionio && pytest --live-server-host=0.0.0.0 --live-server-port=5004  -s tests/proxy_socks5/test_socks5_proxy_sources.py'
+  bash -c 'cd changedetectionio && pytest --live-server-host=0.0.0.0 --live-server-port=5004  -s tests/proxy_socks5/test_socks5_proxy_sources.py --datastore-path /tmp'
 
 echo "socks5 server logs"
 docker logs socks5proxy

changedetectionio/static/js/realtime.js

@@ -2,6 +2,13 @@
 
 $(document).ready(function () {
 
+    function reapplyTableStripes() {
+        $('.watch-table tbody tr').each(function(index) {
+            $(this).removeClass('pure-table-odd pure-table-even');
+            $(this).addClass(index % 2 === 0 ? 'pure-table-odd' : 'pure-table-even');
+        });
+    }
+
     function bindSocketHandlerButtonsEvents(socket) {
         $('.ajax-op').on('click.socketHandlerNamespace', function (e) {
             e.preventDefault();
@@ -101,6 +108,7 @@ $(document).ready(function () {
             socket.on('watch_deleted', function (data) {
                 $('tr[data-watch-uuid="' + data.uuid + '"] td').fadeOut(500, function () {
                     $(this).closest('tr').remove();
+                    reapplyTableStripes();
                 });
             });
 

changedetectionio/store.py

@@ -1,11 +1,13 @@
 from changedetectionio.strtobool import strtobool
 
+from changedetectionio.validate_url import is_safe_valid_url
+
 from flask import (
     flash
 )
 
 from .html_tools import TRANSLATE_WHITESPACE_TABLE
-from . model import App, Watch
+from .model import App, Watch, USE_SYSTEM_DEFAULT_NOTIFICATION_FORMAT_FOR_WATCH
 from copy import deepcopy, copy
 from os import path, unlink
 from threading import Lock
@@ -40,17 +42,24 @@ class ChangeDetectionStore:
     needs_write_urgent = False
 
     __version_check = True
+    save_data_thread = None
 
     def __init__(self, datastore_path="/datastore", include_default_watches=True, version_tag="0.0.0"):
         # Should only be active for docker
         # logging.basicConfig(filename='/dev/stdout', level=logging.INFO)
-        self.__data = App.model()
-        self.datastore_path = datastore_path
-        self.json_store_path = os.path.join(self.datastore_path, "url-watches.json")
-        logger.info(f"Datastore path is '{self.json_store_path}'")
+
         self.needs_write = False
         self.start_time = time.time()
         self.stop_thread = False
+        self.reload_state(datastore_path=datastore_path, include_default_watches=include_default_watches, version_tag=version_tag)
+
+
+    def reload_state(self, datastore_path, include_default_watches, version_tag):
+        logger.info(f"Datastore path is '{datastore_path}'")
+
+        self.__data = App.model()
+        self.datastore_path = datastore_path
+        self.json_store_path = os.path.join(self.datastore_path, "url-watches.json")
         # Base definition for all watchers
         # deepcopy part of #569 - not sure why its needed exactly
         self.generic_definition = deepcopy(Watch.model(datastore_path = datastore_path, default={}))
@@ -143,7 +152,10 @@ class ChangeDetectionStore:
         self.needs_write = True
 
         # Finally start the thread that will manage periodic data saves to JSON
-        save_data_thread = threading.Thread(target=self.save_datastore).start()
+        # Only start if thread is not already running (reload_state might be called multiple times)
+        if not self.save_data_thread or not self.save_data_thread.is_alive():
+            self.save_data_thread = threading.Thread(target=self.save_datastore)
+            self.save_data_thread.start()
 
     def rehydrate_entity(self, uuid, entity, processor_override=None):
         """Set the dict back to the dict Watch object"""
@@ -340,9 +352,10 @@ class ChangeDetectionStore:
                 logger.error(f"Error fetching metadata for shared watch link {url} {str(e)}")
                 flash("Error fetching metadata for {}".format(url), 'error')
                 return False
-        from .model.Watch import is_safe_url
-        if not is_safe_url(url):
-            flash('Watch protocol is not permitted by SAFE_PROTOCOL_REGEX', 'error')
+
+        if not is_safe_valid_url(url):
+            flash('Watch protocol is not permitted or invalid URL format', 'error')
+
             return None
 
         if tag and type(tag) == str:
@@ -408,7 +421,6 @@ class ChangeDetectionStore:
             self.sync_to_json()
             return
         else:
-
             try:
                 # Re #286  - First write to a temp file, then confirm it looks OK and rename it
                 # This is a fairly basic strategy to deal with the case that the file is corrupted,
@@ -438,7 +450,7 @@ class ChangeDetectionStore:
                 logger.remove()
                 logger.add(sys.stderr)
 
-                logger.critical("Shutting down datastore thread")
+                logger.info(f"Shutting down datastore '{self.datastore_path}' thread")
                 return
 
             if self.needs_write or self.needs_write_urgent:
@@ -987,10 +999,35 @@ class ChangeDetectionStore:
             self.data['settings']['application']['ui']['use_page_title_in_list'] = self.data['settings']['application'].get('extract_title_as_title')
 
     def update_21(self):
-        self.data['settings']['application']['scheduler_timezone_default'] = self.data['settings']['application'].get('timezone')
-        del self.data['settings']['application']['timezone']
+        if self.data['settings']['application'].get('timezone'):
+            self.data['settings']['application']['scheduler_timezone_default'] = self.data['settings']['application'].get('timezone')
+            del self.data['settings']['application']['timezone']
 
 
+    # Some notification formats got the wrong name type
+    def update_22(self):
+        from .notification import valid_notification_formats
+
+        sys_n_format = self.data['settings']['application'].get('notification_format')
+        key_exists_as_value = next((k for k, v in valid_notification_formats.items() if v == sys_n_format), None)
+        if key_exists_as_value: # key of "Plain text"
+            logger.success(f"['settings']['application']['notification_format'] '{sys_n_format}' -> '{key_exists_as_value}'")
+            self.data['settings']['application']['notification_format'] = key_exists_as_value
+
+        for uuid, watch in self.data['watching'].items():
+            n_format = self.data['watching'][uuid].get('notification_format')
+            key_exists_as_value = next((k for k, v in valid_notification_formats.items() if v == n_format), None)
+            if key_exists_as_value and key_exists_as_value != USE_SYSTEM_DEFAULT_NOTIFICATION_FORMAT_FOR_WATCH:  # key of "Plain text"
+                logger.success(f"['watching'][{uuid}]['notification_format'] '{n_format}' -> '{key_exists_as_value}'")
+                self.data['watching'][uuid]['notification_format'] = key_exists_as_value # should be 'text' or whatever
+
+        for uuid, tag in self.data['settings']['application']['tags'].items():
+            n_format = self.data['settings']['application']['tags'][uuid].get('notification_format')
+            key_exists_as_value = next((k for k, v in valid_notification_formats.items() if v == n_format), None)
+            if key_exists_as_value and key_exists_as_value != USE_SYSTEM_DEFAULT_NOTIFICATION_FORMAT_FOR_WATCH:  # key of "Plain text"
+                logger.success(f"['settings']['application']['tags'][{uuid}]['notification_format'] '{n_format}' -> '{key_exists_as_value}'")
+                self.data['settings']['application']['tags'][uuid]['notification_format'] = key_exists_as_value # should be 'text' or whatever
+
     def add_notification_url(self, notification_url):
         
         logger.debug(f">>> Adding new notification_url - '{notification_url}'")

changedetectionio/templates/_common_fields.html

@@ -134,6 +134,12 @@
                                     <p>
                                         URL encoding, use <strong>|urlencode</strong>, for example - <code>gets://hook-website.com/test.php?title={{ '{{ watch_title|urlencode }}' }}</code>
                                     </p>
+                                    <p>
+                                        Regular-expression replace, use <strong>|regex_replace</strong>, for example -   <code>{{ "{{ \"hello world 123\" | regex_replace('[0-9]+', 'no-more-numbers') }}" }}</code>
+                                    </p>
+                                    <p>
+                                        For a complete reference of all Jinja2 built-in filters, users can refer to the <a href="https://jinja.palletsprojects.com/en/3.1.x/templates/#builtin-filters">https://jinja.palletsprojects.com/en/3.1.x/templates/#builtin-filters</a>
+                                    </p>
                                 </div>
                             </div>
                             <div class="pure-control-group">

changedetectionio/templates/_helpers.html

@@ -266,9 +266,7 @@
             <li id="timezone-info">
                 {{ render_field(form.time_schedule_limit.timezone, placeholder=timezone_default_config) }} <span id="local-time-in-tz"></span>
                 <datalist id="timezones" style="display: none;">
-                    {% for timezone in available_timezones %}
-                        <option value="{{ timezone }}">{{ timezone }}</option>
-                    {% endfor %}
+                    {%- for timezone in available_timezones -%}<option value="{{ timezone }}">{{ timezone }}</option>{%- endfor -%}
                 </datalist>
             </li>
         </ul>

changedetectionio/templates/base.html

@@ -53,7 +53,7 @@
           <a class="pure-menu-heading" href="{{url_for('watchlist.index')}}">
             <strong>Change</strong>Detection.io</a>
         {% endif %}
-        {% if current_diff_url %}
+        {% if current_diff_url and is_safe_valid_url(current_diff_url) %}
           <a class="current-diff-url" href="{{ current_diff_url }}">
             <span style="max-width: 30%; overflow: hidden">{{ current_diff_url }}</span></a>
         {% else %}

changedetectionio/tests/conftest.py

@@ -11,6 +11,7 @@ import os
 import sys
 from loguru import logger
 
+from changedetectionio.flask_app import init_app_secret
 from changedetectionio.tests.util import live_server_setup, new_live_server_setup
 
 # https://github.com/pallets/flask/blob/1.1.2/examples/tutorial/tests/test_auth.py
@@ -87,7 +88,6 @@ def measure_memory_usage(request):
 def cleanup(datastore_path):
     import glob
     # Unlink test output files
-
     for g in ["*.txt", "*.json", "*.pdf"]:
         files = glob.glob(os.path.join(datastore_path, g))
         for f in files:
@@ -97,34 +97,121 @@ def cleanup(datastore_path):
             if os.path.isfile(f):
                 os.unlink(f)
 
-@pytest.fixture(scope='function', autouse=True)
-def prepare_test_function(live_server):
+def pytest_addoption(parser):
+    """Add custom command-line options for pytest.
 
+    Provides --datastore-path option for specifying custom datastore location.
+    Note: Cannot use -d short option as it's reserved by pytest for debug mode.
+    """
+    parser.addoption(
+        "--datastore-path",
+        action="store",
+        default=None,
+        help="Custom datastore path for tests"
+    )
+
+@pytest.fixture(scope='session')
+def datastore_path(tmp_path_factory, request):
+    """Provide datastore path unique to this worker.
+
+    Supports custom path via --datastore-path/-d flag (mirrors main app).
+
+    CRITICAL for xdist isolation:
+    - Each WORKER gets its own directory
+    - Tests on same worker run SEQUENTIALLY and cleanup between tests
+    - No subdirectories needed since tests don't overlap on same worker
+    - Example: /tmp/test-datastore-gw0/ for worker gw0
+    """
+    # Check for custom path first (mirrors main app's -d flag)
+    custom_path = request.config.getoption("--datastore-path")
+    if custom_path:
+        # Ensure the directory exists
+        os.makedirs(custom_path, exist_ok=True)
+        logger.info(f"Using custom datastore path: {custom_path}")
+        return custom_path
+
+    # Otherwise use default tmp_path_factory logic
+    worker_id = getattr(request.config, 'workerinput', {}).get('workerid', 'master')
+    if worker_id == 'master':
+        path = tmp_path_factory.mktemp("test-datastore")
+    else:
+        path = tmp_path_factory.mktemp(f"test-datastore-{worker_id}")
+    return str(path)
+
+
+@pytest.fixture(scope='function', autouse=True)
+def prepare_test_function(live_server, datastore_path):
+    """Prepare each test with complete isolation.
+
+    CRITICAL for xdist per-test isolation:
+    - Reuses the SAME datastore instance (so blueprint references stay valid)
+    - Clears all watches and state for a clean slate
+    - First watch will get uuid="first"
+    """
     routes = [rule.rule for rule in live_server.app.url_map.iter_rules()]
     if '/test-random-content-endpoint' not in routes:
         logger.debug("Setting up test URL routes")
         new_live_server_setup(live_server)
 
+    # CRITICAL: Point app to THIS test's unique datastore directory
+    live_server.app.config['TEST_DATASTORE_PATH'] = datastore_path
+
+    # CRITICAL: Get datastore and stop it from writing stale data
+    datastore = live_server.app.config.get('DATASTORE')
+
+    # Prevent background thread from writing during cleanup/reload
+    datastore.needs_write = False
+    datastore.needs_write_urgent = False
+
+    # CRITICAL: Clean up any files from previous tests
+    # This ensures a completely clean directory
+    cleanup(datastore_path)
+
+    # CRITICAL: Reload the EXISTING datastore instead of creating a new one
+    # This keeps blueprint references valid (they capture datastore at construction)
+    # reload_state() completely resets the datastore to a clean state
+
+    # Reload state with clean data (no default watches)
+    datastore.reload_state(
+        datastore_path=datastore_path,
+        include_default_watches=False,
+        version_tag=datastore.data.get('version_tag', '0.0.0')
+    )
+    live_server.app.secret_key = init_app_secret(datastore_path)
+    logger.debug(f"prepare_test_function: Reloaded datastore at {hex(id(datastore))}")
+    logger.debug(f"prepare_test_function: Path {datastore.datastore_path}")
 
     yield
-    # Then cleanup/shutdown
-    live_server.app.config['DATASTORE'].data['watching']={}
-    time.sleep(0.3)
-    live_server.app.config['DATASTORE'].data['watching']={}
+
+    # Cleanup: Clear watches again after test
+    try:
+        datastore.data['watching'] = {}
+        datastore.needs_write = True
+    except Exception as e:
+        logger.warning(f"Error during datastore cleanup: {e}")
+
+
+# So the app can also know which test name it was
+@pytest.fixture(autouse=True)
+def set_test_name(request):
+  """Automatically set TEST_NAME env var for every test"""
+  test_name = request.node.name
+  os.environ['PYTEST_CURRENT_TEST'] = test_name
+  yield
+  # Cleanup if needed
 
 
 @pytest.fixture(scope='session')
-def app(request):
-    """Create application for the tests."""
-    datastore_path = "./test-datastore"
+def app(request, datastore_path):
+    """Create application once per worker (session).
 
+    Note: Actual per-test isolation is handled by:
+    - prepare_test_function() recreates datastore and cleans directory
+    - All tests on same worker use same directory (cleaned between tests)
+    """
     # So they don't delay in fetching
     os.environ["MINIMUM_SECONDS_RECHECK_TIME"] = "0"
-    try:
-        os.mkdir(datastore_path)
-    except FileExistsError:
-        pass
-
+    logger.debug(f"Testing with datastore_path={datastore_path}")
     cleanup(datastore_path)
 
     app_config = {'datastore_path': datastore_path, 'disable_checkver' : True}
@@ -147,6 +234,8 @@ def app(request):
     # Disable CSRF while running tests
     app.config['WTF_CSRF_ENABLED'] = False
     app.config['STOP_THREADS'] = True
+    # Store datastore_path so Flask routes can access it
+    app.config['TEST_DATASTORE_PATH'] = datastore_path
 
     def teardown():
         # Stop all threads and services

changedetectionio/tests/custom_browser_url/test_custom_browser_url.py

@@ -73,13 +73,13 @@ def do_test(client, live_server, make_test_use_extra_browser=False):
 
 
 # Requires playwright to be installed
-def test_request_via_custom_browser_url(client, live_server, measure_memory_usage):
+def test_request_via_custom_browser_url(client, live_server, measure_memory_usage, datastore_path):
    #  live_server_setup(live_server) # Setup on conftest per function
     # We do this so we can grep the logs of the custom container and see if the request actually went through that container
     do_test(client, live_server, make_test_use_extra_browser=True)
 
 
-def test_request_not_via_custom_browser_url(client, live_server, measure_memory_usage):
+def test_request_not_via_custom_browser_url(client, live_server, measure_memory_usage, datastore_path):
    #  live_server_setup(live_server) # Setup on conftest per function
     # We do this so we can grep the logs of the custom container and see if the request actually went through that container
     do_test(client, live_server, make_test_use_extra_browser=False)

changedetectionio/tests/fetchers/test_content.py

@@ -8,7 +8,7 @@ import logging
 
 
 # Requires playwright to be installed
-def test_fetch_webdriver_content(client, live_server, measure_memory_usage):
+def test_fetch_webdriver_content(client, live_server, measure_memory_usage, datastore_path):
     #  live_server_setup(live_server) # Setup on conftest per function
 
     #####################

changedetectionio/tests/fetchers/test_custom_js_before_content.py

@@ -3,7 +3,7 @@ from flask import url_for
 from ..util import live_server_setup, wait_for_all_checks, extract_UUID_from_client
 
 
-def test_execute_custom_js(client, live_server, measure_memory_usage):
+def test_execute_custom_js(client, live_server, measure_memory_usage, datastore_path):
 
    #  live_server_setup(live_server) # Setup on conftest per function
     assert os.getenv('PLAYWRIGHT_DRIVER_URL'), "Needs PLAYWRIGHT_DRIVER_URL set for this test"

changedetectionio/tests/proxy_list/test_multiple_proxy.py

@@ -5,7 +5,7 @@ from flask import url_for
 from ..util import live_server_setup, wait_for_all_checks
 
 
-def test_preferred_proxy(client, live_server, measure_memory_usage):
+def test_preferred_proxy(client, live_server, measure_memory_usage, datastore_path):
    #  live_server_setup(live_server) # Setup on conftest per function
     url = "http://chosen.changedetection.io"
 

changedetectionio/tests/proxy_list/test_noproxy.py

@@ -5,7 +5,7 @@ from flask import url_for
 from ..util import live_server_setup, wait_for_all_checks, extract_UUID_from_client
 
 
-def test_noproxy_option(client, live_server, measure_memory_usage):
+def test_noproxy_option(client, live_server, measure_memory_usage, datastore_path):
    #  live_server_setup(live_server) # Setup on conftest per function
     # Run by run_proxy_tests.sh
     # Call this URL then scan the containers that it never went through them

changedetectionio/tests/proxy_list/test_proxy.py

@@ -5,7 +5,7 @@ from flask import url_for
 from ..util import live_server_setup, wait_for_all_checks, extract_UUID_from_client
 
 # just make a request, we will grep in the docker logs to see it actually got called
-def test_check_basic_change_detection_functionality(client, live_server, measure_memory_usage):
+def test_check_basic_change_detection_functionality(client, live_server, measure_memory_usage, datastore_path):
    #  live_server_setup(live_server) # Setup on conftest per function
     res = client.post(
         url_for("imports.import_page"),

changedetectionio/tests/proxy_list/test_proxy_noconnect.py

@@ -12,7 +12,7 @@ from ... import strtobool
 # FAST_PUPPETEER_CHROME_FETCHER=True PLAYWRIGHT_DRIVER_URL=ws://127.0.0.1:3000 pytest tests/proxy_list/test_proxy_noconnect.py
 # WEBDRIVER_URL=http://127.0.0.1:4444/wd/hub pytest tests/proxy_list/test_proxy_noconnect.py
 
-def test_proxy_noconnect_custom(client, live_server, measure_memory_usage):
+def test_proxy_noconnect_custom(client, live_server, measure_memory_usage, datastore_path):
    #  live_server_setup(live_server) # Setup on conftest per function
 
     # Goto settings, add our custom one

changedetectionio/tests/proxy_list/test_select_custom_proxy.py

@@ -6,7 +6,7 @@ from ..util import live_server_setup, wait_for_all_checks
 import os
 
 # just make a request, we will grep in the docker logs to see it actually got called
-def test_select_custom(client, live_server, measure_memory_usage):
+def test_select_custom(client, live_server, measure_memory_usage, datastore_path):
    #  live_server_setup(live_server) # Setup on conftest per function
 
     # Goto settings, add our custom one
@@ -50,7 +50,7 @@ def test_select_custom(client, live_server, measure_memory_usage):
     # Now we should see the request in the container logs for "squid-squid-custom" because it will be the only default
 
 
-def test_custom_proxy_validation(client, live_server, measure_memory_usage):
+def test_custom_proxy_validation(client, live_server, measure_memory_usage, datastore_path):
     #  live_server_setup(live_server) # Setup on conftest per function
 
     # Goto settings, add our custom one

changedetectionio/tests/proxy_socks5/test_socks5_proxy.py

@@ -5,7 +5,7 @@ from flask import url_for
 from changedetectionio.tests.util import live_server_setup, wait_for_all_checks, extract_UUID_from_client, delete_all_watches
 
 
-def set_response():
+def set_response(datastore_path):
     import time
     data = """<html>
        <body>
@@ -15,13 +15,13 @@ def set_response():
      </html>
     """
 
-    with open("test-datastore/endpoint-content.txt", "w") as f:
+    with open(os.path.join(datastore_path, "endpoint-content.txt"), "w") as f:
         f.write(data)
     time.sleep(1)
 
-def test_socks5(client, live_server, measure_memory_usage):
+def test_socks5(client, live_server, measure_memory_usage, datastore_path):
    #  live_server_setup(live_server) # Setup on conftest per function
-    set_response()
+    set_response(datastore_path)
 
     # Setup a proxy
     res = client.post(

changedetectionio/tests/proxy_socks5/test_socks5_proxy_sources.py

@@ -4,7 +4,7 @@ from flask import url_for
 from changedetectionio.tests.util import live_server_setup, wait_for_all_checks
 
 
-def set_response():
+def set_response(datastore_path):
     import time
     data = """<html>
        <body>
@@ -14,15 +14,15 @@ def set_response():
      </html>
     """
 
-    with open("test-datastore/endpoint-content.txt", "w") as f:
+    with open(os.path.join(datastore_path, "endpoint-content.txt"), "w") as f:
         f.write(data)
     time.sleep(1)
 
 # should be proxies.json mounted from run_proxy_tests.sh already
 # -v `pwd`/tests/proxy_socks5/proxies.json-example:/app/changedetectionio/test-datastore/proxies.json
-def test_socks5_from_proxiesjson_file(client, live_server, measure_memory_usage):
+def test_socks5_from_proxiesjson_file(client, live_server, measure_memory_usage, datastore_path):
    #  live_server_setup(live_server) # Setup on conftest per function
-    set_response()
+    set_response(datastore_path)
     # Because the socks server should connect back to us
     test_url = url_for('test_endpoint', _external=True) + f"?socks-test-tag={os.getenv('SOCKSTEST', '')}"
     test_url = test_url.replace('localhost.localdomain', 'cdio')

changedetectionio/tests/restock/test_restock.py

@@ -11,7 +11,7 @@ from changedetectionio.notification import (
 )
 
 
-def set_original_response():
+def set_original_response(datastore_path):
     test_return_data = """<html>
        <body>
        <section id=header style="padding: 50px; height: 350px">This is the header which should be ignored always - <span>add to cart</span></section>
@@ -26,13 +26,13 @@ def set_original_response():
      </html>
     """
 
-    with open("test-datastore/endpoint-content.txt", "w") as f:
+    with open(os.path.join(datastore_path, "endpoint-content.txt"), "w") as f:
         f.write(test_return_data)
     return None
 
 
 
-def set_back_in_stock_response():
+def set_back_in_stock_response(datastore_path):
     test_return_data = """<html>
        <body>
      Some initial text<br>
@@ -45,14 +45,14 @@ def set_back_in_stock_response():
      </html>
     """
 
-    with open("test-datastore/endpoint-content.txt", "w") as f:
+    with open(os.path.join(datastore_path, "endpoint-content.txt"), "w") as f:
         f.write(test_return_data)
     return None
 
 # Add a site in paused mode, add an invalid filter, we should still have visual selector data ready
-def test_restock_detection(client, live_server, measure_memory_usage):
+def test_restock_detection(client, live_server, measure_memory_usage, datastore_path):
 
-    set_original_response()
+    set_original_response(datastore_path=datastore_path)
     #assert os.getenv('PLAYWRIGHT_DRIVER_URL'), "Needs PLAYWRIGHT_DRIVER_URL set for this test"
    #  live_server_setup(live_server) # Setup on conftest per function
     #####################
@@ -88,24 +88,25 @@ def test_restock_detection(client, live_server, measure_memory_usage):
     assert b'not-in-stock' in res.data # should be out of stock
 
     # Is it correctly shown as in stock
-    set_back_in_stock_response()
+    set_back_in_stock_response(datastore_path)
     client.get(url_for("ui.form_watch_checknow"), follow_redirects=True)
     wait_for_all_checks(client)
     res = client.get(url_for("watchlist.index"))
     assert b'not-in-stock' not in res.data
 
     # We should have a notification
-    wait_for_notification_endpoint_output()
-    assert os.path.isfile("test-datastore/notification.txt"), "Notification received"
-    os.unlink("test-datastore/notification.txt")
+    notification_file = os.path.join(datastore_path, "notification.txt")
+    wait_for_notification_endpoint_output(datastore_path=datastore_path)
+    assert os.path.isfile(notification_file), "Notification received"
+    os.unlink(notification_file)
 
     # Default behaviour is to only fire notification when it goes OUT OF STOCK -> IN STOCK
     # So here there should be no file, because we go IN STOCK -> OUT OF STOCK
-    set_original_response()
+    set_original_response(datastore_path=datastore_path)
     client.get(url_for("ui.form_watch_checknow"), follow_redirects=True)
     wait_for_all_checks(client)
     time.sleep(5)
-    assert not os.path.isfile("test-datastore/notification.txt"), "No notification should have fired when it went OUT OF STOCK by default"
+    assert not os.path.isfile(notification_file), "No notification should have fired when it went OUT OF STOCK by default"
 
     # BUT we should see that it correctly shows "not in stock"
     res = client.get(url_for("watchlist.index"))

changedetectionio/tests/smtp/smtp-test-server.py

@@ -1,51 +1,110 @@
 #!/usr/bin/env python3
-import asyncio
+import threading
+import time
 from aiosmtpd.controller import Controller
-from aiosmtpd.smtp import SMTP
+from flask import Flask, Response
+from email import message_from_bytes
+from email.policy import default
 
-# Accept a SMTP message and offer a way to retrieve the last message via TCP Socket
+# Accept a SMTP message and offer a way to retrieve the last message via HTTP
 
-last_received_message = b"Nothing"
+last_received_message = b"Nothing received yet."
+active_smtp_connections = 0
+smtp_lock = threading.Lock()
 
 
 class CustomSMTPHandler:
     async def handle_DATA(self, server, session, envelope):
-        global last_received_message
-        last_received_message = envelope.content
-        print('Receiving message from:', session.peer)
-        print('Message addressed from:', envelope.mail_from)
-        print('Message addressed to  :', envelope.rcpt_tos)
-        print('Message length        :', len(envelope.content))
-        print(envelope.content.decode('utf8'))
-        return '250 Message accepted for delivery'
+        global last_received_message, active_smtp_connections
+
+        with smtp_lock:
+            active_smtp_connections += 1
+
+        try:
+            last_received_message = envelope.content
+            print('Receiving message from:', session.peer)
+            print('Message addressed from:', envelope.mail_from)
+            print('Message addressed to  :', envelope.rcpt_tos)
+            print('Message length        :', len(envelope.content))
+            print('*******************************')
+            print(envelope.content.decode('utf8'))
+            print('*******************************')
+
+            # Parse the email message
+            msg = message_from_bytes(envelope.content, policy=default)
+            with open('/tmp/last.eml', 'wb') as f:
+                f.write(envelope.content)
+
+            # Write parts to files based on content type
+            if msg.is_multipart():
+                for part in msg.walk():
+                    content_type = part.get_content_type()
+                    payload = part.get_payload(decode=True)
+
+                    if payload:
+                        if content_type == 'text/plain':
+                            with open('/tmp/last.txt', 'wb') as f:
+                                f.write(payload)
+                            print(f'Written text/plain part to /tmp/last.txt')
+                        elif content_type == 'text/html':
+                            with open('/tmp/last.html', 'wb') as f:
+                                f.write(payload)
+                            print(f'Written text/html part to /tmp/last.html')
+            else:
+                # Single part message
+                content_type = msg.get_content_type()
+                payload = msg.get_payload(decode=True)
+
+                if payload:
+                    if content_type == 'text/plain' or content_type.startswith('text/'):
+                        with open('/tmp/last.txt', 'wb') as f:
+                            f.write(payload)
+                        print(f'Written single part message to /tmp/last.txt')
+
+            return '250 Message accepted for delivery'
+        finally:
+            with smtp_lock:
+                active_smtp_connections -= 1
 
 
-class EchoServerProtocol(asyncio.Protocol):
-    def connection_made(self, transport):
-        global last_received_message
-        self.transport = transport
-        peername = transport.get_extra_info('peername')
-        print('Incoming connection from {}'.format(peername))
-        self.transport.write(last_received_message)
-
-        last_received_message = b''
-        self.transport.close()
+# Simple Flask HTTP server to echo back the last SMTP message
+app = Flask(__name__)
 
 
-async def main():
+@app.route('/')
+def echo_last_message():
+    global last_received_message, active_smtp_connections
+
+    # Wait for any in-progress SMTP connections to complete
+    max_wait = 5  # Maximum 5 seconds
+    wait_interval = 0.05  # Check every 50ms
+    elapsed = 0
+
+    while elapsed < max_wait:
+        with smtp_lock:
+            if active_smtp_connections == 0:
+                break
+        time.sleep(wait_interval)
+        elapsed += wait_interval
+
+    return Response(last_received_message, mimetype='text/plain')
+
+
+def run_flask():
+    app.run(host='0.0.0.0', port=11080, debug=False, use_reloader=False)
+
+
+if __name__ == "__main__":
     # Start the SMTP server
     controller = Controller(CustomSMTPHandler(), hostname='0.0.0.0', port=11025)
     controller.start()
 
-    # Start the TCP Echo server
-    loop = asyncio.get_running_loop()
-    server = await loop.create_server(
-        lambda: EchoServerProtocol(),
-        '0.0.0.0', 11080
-    )
-    async with server:
-        await server.serve_forever()
+    # Start the HTTP server in a separate thread
+    flask_thread = threading.Thread(target=run_flask, daemon=True)
+    flask_thread.start()
 
-
-if __name__ == "__main__":
-    asyncio.run(main())
+    # Keep the main thread alive
+    try:
+        flask_thread.join()
+    except KeyboardInterrupt:
+        print("Shutting down...")

changedetectionio/tests/smtp/test_notification_smtp.py

@@ -1,22 +1,22 @@
-import json
-import os
 import time
-import re
 from flask import url_for
 from email import message_from_string
 from email.policy import default as email_policy
 
-from changedetectionio.diff import REMOVED_STYLE, ADDED_STYLE
+from changedetectionio.diff import HTML_REMOVED_STYLE, HTML_ADDED_STYLE, HTML_CHANGED_STYLE
+from changedetectionio.notification_service import NotificationContextData, CUSTOM_LINEBREAK_PLACEHOLDER
 from changedetectionio.tests.util import set_original_response, set_modified_response, set_more_modified_response, live_server_setup, \
     wait_for_all_checks, \
     set_longer_modified_response, delete_all_watches
-from changedetectionio.tests.util import extract_UUID_from_client
+
 import logging
-import base64
+
 
 # NOTE - RELIES ON mailserver as hostname running, see github build recipes
 smtp_test_server = 'mailserver'
 
+ALL_MARKUP_TOKENS = ''.join(f"TOKEN: '{t}'\n{{{{{t}}}}}\n" for t in NotificationContextData().keys())
+
 from changedetectionio.notification import (
     default_notification_body,
     default_notification_format,
@@ -27,24 +27,23 @@ from changedetectionio.notification import (
 
 
 def get_last_message_from_smtp_server():
-    import socket
-    port = 11080  # socket server port number
-
-    client_socket = socket.socket()  # instantiate
-    client_socket.connect((smtp_test_server, port))  # connect to the server
-
-    data = client_socket.recv(50024).decode()  # receive response
+    import requests
+    time.sleep(1) # wait for any smtp connects to die off
+    port = 11080  # HTTP server port number
+    # Make HTTP GET request to Flask server
+    response = requests.get(f'http://{smtp_test_server}:{port}/')
+    data = response.text
     logging.info("get_last_message_from_smtp_server..")
     logging.info(data)
-    client_socket.close()  # close the connection
     return data
 
 
 # Requires running the test SMTP server
 
-def test_check_notification_email_formats_default_HTML(client, live_server, measure_memory_usage):
+def test_check_notification_email_formats_default_HTML(client, live_server, measure_memory_usage, datastore_path):
     ##  live_server_setup(live_server) # Setup on conftest per function
-    set_original_response()
+    set_original_response(datastore_path=datastore_path)
+
 
     notification_url = f'mailto://changedetection@{smtp_test_server}:11025/?to=fff@home.com'
 
@@ -55,7 +54,7 @@ def test_check_notification_email_formats_default_HTML(client, live_server, meas
         data={"application-notification_urls": notification_url,
               "application-notification_title": "fallback-title " + default_notification_title,
               "application-notification_body": "some text\nfallback-body<br> " + default_notification_body,
-              "application-notification_format": 'HTML',
+              "application-notification_format": 'html',
               "requests-time_between_check-minutes": 180,
               'application-fetch_backend': "html_requests"},
         follow_redirects=True
@@ -73,7 +72,7 @@ def test_check_notification_email_formats_default_HTML(client, live_server, meas
     assert b"Watch added" in res.data
 
     wait_for_all_checks(client)
-    set_longer_modified_response()
+    set_longer_modified_response(datastore_path=datastore_path)
     time.sleep(2)
 
     client.get(url_for("ui.form_watch_checknow"), follow_redirects=True)
@@ -101,6 +100,7 @@ def test_check_notification_email_formats_default_HTML(client, live_server, meas
     text_content = text_part.get_content()
     assert '(added) So let\'s see what happens.\r\n' in text_content  # The plaintext part
     assert 'fallback-body\r\n' in text_content  # The plaintext part
+    assert CUSTOM_LINEBREAK_PLACEHOLDER not in text_content
 
     # Second part should be text/html
     html_part = parts[1]
@@ -109,11 +109,13 @@ def test_check_notification_email_formats_default_HTML(client, live_server, meas
     assert 'some text<br>' in html_content  # We converted \n from the notification body
     assert 'fallback-body<br>' in html_content  # kept the original <br>
     assert '(added) So let\'s see what happens.<br>' in html_content  # the html part
+    assert CUSTOM_LINEBREAK_PLACEHOLDER not in html_content
     delete_all_watches(client)
 
 
-def test_check_notification_plaintext_format(client, live_server, measure_memory_usage):
-    set_original_response()
+def test_check_notification_plaintext_format(client, live_server, measure_memory_usage, datastore_path):
+    set_original_response(datastore_path=datastore_path)
+
 
     notification_url = f'mailto://changedetection@{smtp_test_server}:11025/?to=fff@home.com'
 
@@ -124,7 +126,7 @@ def test_check_notification_plaintext_format(client, live_server, measure_memory
         data={"application-notification_urls": notification_url,
               "application-notification_title": "fallback-title " + default_notification_title,
               "application-notification_body": "some text\n" + default_notification_body,
-              "application-notification_format": 'Text',
+              "application-notification_format": 'text',
               "requests-time_between_check-minutes": 180,
               'application-fetch_backend': "html_requests"},
         follow_redirects=True
@@ -134,18 +136,11 @@ def test_check_notification_plaintext_format(client, live_server, measure_memory
 
     # Add a watch and trigger a HTTP POST
     test_url = url_for('test_endpoint', _external=True)
-    res = client.post(
-        url_for("ui.ui_views.form_quick_watch_add"),
-        data={"url": test_url, "tags": 'nice one'},
-        follow_redirects=True
-    )
-
-    assert b"Watch added" in res.data
-
-    wait_for_all_checks(client)
-    set_longer_modified_response()
+    uuid = client.application.config.get('DATASTORE').add_watch(url=test_url)
+    client.get(url_for("ui.form_watch_checknow"), follow_redirects=True)
     time.sleep(2)
 
+    set_longer_modified_response(datastore_path=datastore_path)
     client.get(url_for("ui.form_watch_checknow"), follow_redirects=True)
     wait_for_all_checks(client)
 
@@ -171,8 +166,9 @@ def test_check_notification_plaintext_format(client, live_server, measure_memory
 
 
 
-def test_check_notification_html_color_format(client, live_server, measure_memory_usage):
-    set_original_response()
+def test_check_notification_html_color_format(client, live_server, measure_memory_usage, datastore_path):
+    set_original_response(datastore_path=datastore_path)
+
 
     notification_url = f'mailto://changedetection@{smtp_test_server}:11025/?to=fff@home.com'
 
@@ -182,8 +178,8 @@ def test_check_notification_html_color_format(client, live_server, measure_memor
         url_for("settings.settings_page"),
         data={"application-notification_urls": notification_url,
               "application-notification_title": "fallback-title " + default_notification_title,
-              "application-notification_body": "some text\n" + default_notification_body,
-              "application-notification_format": 'HTML Color',
+              "application-notification_body": f"some text\n{default_notification_body}\nMore output test\n{ALL_MARKUP_TOKENS}",
+              "application-notification_format": 'htmlcolor',
               "requests-time_between_check-minutes": 180,
               'application-fetch_backend': "html_requests"},
         follow_redirects=True
@@ -202,7 +198,7 @@ def test_check_notification_html_color_format(client, live_server, measure_memor
     assert b"Watch added" in res.data
 
     wait_for_all_checks(client)
-    set_longer_modified_response()
+    set_longer_modified_response(datastore_path=datastore_path)
     time.sleep(2)
 
     client.get(url_for("ui.form_watch_checknow"), follow_redirects=True)
@@ -235,20 +231,91 @@ def test_check_notification_html_color_format(client, live_server, measure_memor
     html_part = parts[1]
     assert html_part.get_content_type() == 'text/html'
     html_content = html_part.get_content()
-    assert REMOVED_STYLE in html_content
-    assert ADDED_STYLE in html_content
+    assert HTML_CHANGED_STYLE or HTML_REMOVED_STYLE in html_content
+    assert HTML_ADDED_STYLE in html_content
+    assert '&lt;' not in html_content
 
     assert 'some text<br>' in html_content
     delete_all_watches(client)
 
+def test_check_notification_markdown_format(client, live_server, measure_memory_usage, datastore_path):
+    set_original_response(datastore_path=datastore_path)
 
-def test_check_notification_email_formats_default_Text_override_HTML(client, live_server, measure_memory_usage):
-    ##  live_server_setup(live_server) # Setup on conftest per function
+
+    notification_url = f'mailto://changedetection@{smtp_test_server}:11025/?to=fff@home.com'
+
+    #####################
+    # Set this up for when we remove the notification from the watch, it should fallback with these details
+    res = client.post(
+        url_for("settings.settings_page"),
+        data={"application-notification_urls": notification_url,
+              "application-notification_title": "fallback-title " + default_notification_title,
+              "application-notification_body": "*header*\n\nsome text\n" + default_notification_body,
+              "application-notification_format": 'markdown',
+              "requests-time_between_check-minutes": 180,
+              'application-fetch_backend': "html_requests"},
+        follow_redirects=True
+    )
+
+    assert b"Settings updated." in res.data
+
+    # Add a watch and trigger a HTTP POST
+    test_url = url_for('test_endpoint', _external=True)
+    res = client.post(
+        url_for("ui.ui_views.form_quick_watch_add"),
+        data={"url": test_url, "tags": 'nice one'},
+        follow_redirects=True
+    )
+
+    assert b"Watch added" in res.data
+
+    wait_for_all_checks(client)
+    set_longer_modified_response(datastore_path=datastore_path)
+    time.sleep(2)
+
+    client.get(url_for("ui.form_watch_checknow"), follow_redirects=True)
+    wait_for_all_checks(client)
+
+    time.sleep(3)
+
+    msg_raw = get_last_message_from_smtp_server()
+    assert len(msg_raw) >= 1
+
+    # Parse the email properly using Python's email library
+    msg = message_from_string(msg_raw, policy=email_policy)
+
+    # The email should have two bodies (multipart/alternative with text/plain and text/html)
+    assert msg.is_multipart()
+    assert msg.get_content_type() == 'multipart/alternative'
+
+    # Get the parts
+    parts = list(msg.iter_parts())
+    assert len(parts) == 2
+
+    # First part should be text/plain (the auto-generated plaintext version)
+    text_part = parts[0]
+    assert text_part.get_content_type() == 'text/plain'
+    text_content = text_part.get_content()
+    # We wont see anything in the "FALLBACK" text but that's OK (no added/strikethrough etc)
+    assert 'So let\'s see what happens.\r\n' in text_content  # The plaintext part
+
+
+    # Second part should be text/html and roughly converted from markdown to HTML
+    html_part = parts[1]
+    assert html_part.get_content_type() == 'text/html'
+    html_content = html_part.get_content()
+    assert '<p><em>header</em></p>' in html_content
+    assert '<strong>So let\'s see what happens.</strong><br>' in html_content # Additions are <strong> in markdown
+    delete_all_watches(client)
+
+# Custom notification body with HTML, that is either sent as HTML or rendered to plaintext and sent
+def test_check_notification_email_formats_default_Text_override_HTML(client, live_server, measure_memory_usage, datastore_path):
 
     # HTML problems? see this
     # https://github.com/caronc/apprise/issues/633
 
-    set_original_response()
+    set_original_response(datastore_path=datastore_path)
+
     notification_url = f'mailto://changedetection@{smtp_test_server}:11025/?to=fff@home.com'
     notification_body = f"""<!DOCTYPE html>
 <html lang="en">
@@ -269,7 +336,7 @@ def test_check_notification_email_formats_default_Text_override_HTML(client, liv
         data={"application-notification_urls": notification_url,
               "application-notification_title": "fallback-title " + default_notification_title,
               "application-notification_body": notification_body,
-              "application-notification_format": 'Text',
+              "application-notification_format": 'text',
               "requests-time_between_check-minutes": 180,
               'application-fetch_backend': "html_requests"},
         follow_redirects=True
@@ -277,7 +344,7 @@ def test_check_notification_email_formats_default_Text_override_HTML(client, liv
     assert b"Settings updated." in res.data
 
     # Add a watch and trigger a HTTP POST
-    test_url = url_for('test_endpoint', _external=True)
+    test_url = url_for('test_endpoint',content_type="text/html", _external=True)
     res = client.post(
         url_for("ui.ui_views.form_quick_watch_add"),
         data={"url": test_url, "tags": 'nice one'},
@@ -286,8 +353,9 @@ def test_check_notification_email_formats_default_Text_override_HTML(client, liv
 
     assert b"Watch added" in res.data
 
+    #################################### FIRST SITUATION, PLAIN TEXT NOTIFICATION IS WANTED BUT WE HAVE HTML IN OUR TEMPLATE AND CONTENT ##########
     wait_for_all_checks(client)
-    set_longer_modified_response()
+    set_longer_modified_response(datastore_path=datastore_path)
     time.sleep(2)
     client.get(url_for("ui.form_watch_checknow"), follow_redirects=True)
     wait_for_all_checks(client)
@@ -308,14 +376,18 @@ def test_check_notification_email_formats_default_Text_override_HTML(client, liv
     # Get the plain text content
     text_content = msg.get_content()
     assert '(added) So let\'s see what happens.\r\n' in text_content  # The plaintext part
+    assert '<!DOCTYPE html>' in text_content # even tho they added html, they selected plaintext so it should have not got converted
+
+
+    #################################### SECOND SITUATION, HTML IS CORRECTLY PASSED THROUGH TO THE EMAIL ####################
+    set_original_response(datastore_path=datastore_path)
 
-    set_original_response()
     # Now override as HTML format
     res = client.post(
         url_for("ui.ui_edit.edit_page", uuid="first"),
         data={
             "url": test_url,
-            "notification_format": 'HTML',
+            "notification_format": 'html',
             'fetch_backend': "html_requests",
             "time_between_check_use_default": "y"},
         follow_redirects=True
@@ -348,10 +420,346 @@ def test_check_notification_email_formats_default_Text_override_HTML(client, liv
     html_part = parts[1]
     assert html_part.get_content_type() == 'text/html'
     html_content = html_part.get_content()
-    assert '(removed) So let\'s see what happens.<br>' in html_content  # the html part
+    assert '(removed) So let\'s see what happens.' in html_content  # the html part
+    assert '&lt;!DOCTYPE html' not in html_content
+    assert '<!DOCTYPE html' in html_content # Our original template is working correctly
 
     # https://github.com/dgtlmoon/changedetection.io/issues/2103
     assert '<h1>Test</h1>' in html_content
     assert '&lt;' not in html_content
 
     delete_all_watches(client)
+
+def test_check_plaintext_document_plaintext_notification_smtp(client, live_server, measure_memory_usage, datastore_path):
+    """When following a plaintext document, notification in Plain Text format is sent correctly"""
+    import os
+
+    with open(os.path.join(datastore_path, "endpoint-content.txt"), "w") as f:
+        f.write("Some nice plain text\nwhich we add some extra data\nover here\n")
+
+    notification_url = f'mailto://changedetection@{smtp_test_server}:11025/?to=fff@home.com'
+    notification_body = f"""{default_notification_body}"""
+
+    #####################
+    # Set this up for when we remove the notification from the watch, it should fallback with these details
+    res = client.post(
+        url_for("settings.settings_page"),
+        data={"application-notification_urls": notification_url,
+              "application-notification_title": "fallback-title " + default_notification_title,
+              "application-notification_body": f"{notification_body}\nMore output test\n{ALL_MARKUP_TOKENS}",
+              "application-notification_format": 'text',
+              "requests-time_between_check-minutes": 180,
+              'application-fetch_backend': "html_requests"},
+        follow_redirects=True
+    )
+    assert b"Settings updated." in res.data
+
+    # Add our URL to the import page
+    test_url = url_for('test_endpoint', content_type="text/plain", _external=True)
+    uuid = client.application.config.get('DATASTORE').add_watch(url=test_url)
+    client.get(url_for("ui.form_watch_checknow"), follow_redirects=True)
+    wait_for_all_checks(client)
+
+    # Change the content
+    with open(os.path.join(datastore_path, "endpoint-content.txt"), "w") as f:
+        f.write("Some nice plain text\nwhich we add some extra data\nAnd let's talk about <title> tags\nover here\n")
+
+
+    time.sleep(1)
+    client.get(url_for("ui.form_watch_checknow"), follow_redirects=True)
+    wait_for_all_checks(client)
+
+    # Parse the email properly using Python's email library
+    msg = message_from_string(get_last_message_from_smtp_server(), policy=email_policy)
+
+    assert not msg.is_multipart()
+    assert msg.get_content_type() == 'text/plain'
+    body = msg.get_content()
+    # nothing is escaped, raw html stuff in text/plain
+    assert 'talk about <title> tags' in body
+    assert '(added)' in body
+    assert '<br' not in body
+    assert '&lt;' not in body
+    assert '<pre' not in body
+    delete_all_watches(client)
+
+def test_check_plaintext_document_html_notifications(client, live_server, measure_memory_usage, datastore_path):
+    """When following a plaintext document, notification in Plain Text format is sent correctly"""
+    import os
+
+    with open(os.path.join(datastore_path, "endpoint-content.txt"), "w") as f:
+        f.write("    Some nice plain text\nwhich we add some extra data\nover here\n")
+
+    notification_url = f'mailto://changedetection@{smtp_test_server}:11025/?to=fff@home.com'
+    notification_body = f"""{default_notification_body}"""
+
+    #####################
+    # Set this up for when we remove the notification from the watch, it should fallback with these details
+    res = client.post(
+        url_for("settings.settings_page"),
+        data={"application-notification_urls": notification_url,
+              "application-notification_title": "fallback-title " + default_notification_title,
+              "application-notification_body": f"{notification_body}\nMore output test\n{ALL_MARKUP_TOKENS}",
+              "application-notification_format": 'html',
+              "requests-time_between_check-minutes": 180,
+              'application-fetch_backend': "html_requests"},
+        follow_redirects=True
+    )
+    assert b"Settings updated." in res.data
+
+    # Add our URL to the import page
+    test_url = url_for('test_endpoint', content_type="text/plain", _external=True)
+    uuid = client.application.config.get('DATASTORE').add_watch(url=test_url)
+    client.get(url_for("ui.form_watch_checknow"), follow_redirects=True)
+    wait_for_all_checks(client)
+
+    # Change the content
+    with open(os.path.join(datastore_path, "endpoint-content.txt"), "w") as f:
+        f.write("    Some nice plain text\nwhich we add some extra data\nAnd let's talk about <title> tags\nover here\n")
+
+
+    time.sleep(2)
+    client.get(url_for("ui.form_watch_checknow"), follow_redirects=True)
+    wait_for_all_checks(client)
+
+    # Parse the email properly using Python's email library
+    msg = message_from_string(get_last_message_from_smtp_server(), policy=email_policy)
+
+
+    # The email should have two bodies (multipart/alternative)
+    assert msg.is_multipart()
+    assert msg.get_content_type() == 'multipart/alternative'
+
+    # Get the parts
+    parts = list(msg.iter_parts())
+    assert len(parts) == 2
+
+    # First part should be text/plain
+    text_part = parts[0]
+    assert text_part.get_content_type() == 'text/plain'
+    text_content = text_part.get_content()
+    html_part = parts[1]
+    assert html_part.get_content_type() == 'text/html'
+    html_content = html_part.get_content()
+
+
+    assert 'And let\'s talk about <title> tags\r\n' in text_content
+    assert '&lt;br' not in text_content
+    assert '<span' not in text_content
+
+
+    assert 'talk about <title>' not in html_content  # the html part, should have got marked up to &lt; etc
+    assert 'talk about &lt;title&gt;' in html_content
+    # Should be the HTML, but not HTML Color
+    assert 'background-color' not in html_content
+    assert '<br>(added) And let&#39;s talk about &lt;title&gt; tags<br>' in html_content
+    assert '&lt;br' not in html_content
+    assert '<pre role="article"' in html_content # Should have got wrapped nicely in email_helpers.py
+
+    # And now for the whitespace retention
+    assert '&nbsp;&nbsp;&nbsp;&nbsp;Some nice plain text' in html_content
+    assert '(added) And let' in html_content # just to show a single whitespace didnt get touched
+    delete_all_watches(client)
+
+
+def test_check_plaintext_document_html_color_notifications(client, live_server, measure_memory_usage, datastore_path):
+    """When following a plaintext document, notification in Plain Text format is sent correctly"""
+    import os
+
+    with open(os.path.join(datastore_path, "endpoint-content.txt"), "w") as f:
+        f.write("Some nice plain text\nwhich we add some extra data\nover here\n")
+
+    notification_url = f'mailto://changedetection@{smtp_test_server}:11025/?to=fff@home.com'
+    notification_body = f"""{default_notification_body}"""
+
+    #####################
+    # Set this up for when we remove the notification from the watch, it should fallback with these details
+    res = client.post(
+        url_for("settings.settings_page"),
+        data={"application-notification_urls": notification_url,
+              "application-notification_title": "fallback-title " + default_notification_title,
+              "application-notification_body": f"{notification_body}\nMore output test\n{ALL_MARKUP_TOKENS}",
+              "application-notification_format": 'htmlcolor',
+              "requests-time_between_check-minutes": 180,
+              'application-fetch_backend': "html_requests"},
+        follow_redirects=True
+    )
+
+    assert b"Settings updated." in res.data
+
+    # Add our URL to the import page
+    test_url = url_for('test_endpoint', content_type="text/plain", _external=True)
+    uuid = client.application.config.get('DATASTORE').add_watch(url=test_url)
+    client.get(url_for("ui.form_watch_checknow"), follow_redirects=True)
+    wait_for_all_checks(client)
+
+    # Change the content
+    with open(os.path.join(datastore_path, "endpoint-content.txt"), "w") as f:
+        f.write("Some nice plain text\nwhich we add some extra data\nAnd let's talk about <title> tags\nover here\n")
+
+    time.sleep(1)
+    client.get(url_for("ui.form_watch_checknow"), follow_redirects=True)
+    wait_for_all_checks(client)
+
+    # Parse the email properly using Python's email library
+    msg = message_from_string(get_last_message_from_smtp_server(), policy=email_policy)
+
+    # The email should have two bodies (multipart/alternative)
+    assert msg.is_multipart()
+    assert msg.get_content_type() == 'multipart/alternative'
+
+    # Get the parts
+    parts = list(msg.iter_parts())
+    assert len(parts) == 2
+
+    # First part should be text/plain
+    text_part = parts[0]
+    assert text_part.get_content_type() == 'text/plain'
+    text_content = text_part.get_content()
+    html_part = parts[1]
+    assert html_part.get_content_type() == 'text/html'
+    html_content = html_part.get_content()
+
+
+    assert 'And let\'s talk about <title> tags\r\n' in text_content
+    assert '&lt;br' not in text_content
+    assert '<span' not in text_content
+
+    assert 'talk about <title>' not in html_content  # the html part, should have got marked up to &lt; etc
+    assert 'talk about &lt;title&gt;' in html_content
+    # Should be the HTML, but not HTML Color
+    assert 'background-color' in html_content
+    assert '(added) And let' not in html_content
+    assert '&lt;br' not in html_content
+    assert '<br>' in html_content
+    assert '<pre role="article"' in html_content # Should have got wrapped nicely in email_helpers.py
+    delete_all_watches(client)
+
+def test_check_html_document_plaintext_notification(client, live_server, measure_memory_usage, datastore_path):
+    """When following a HTML document, notification in Plain Text format is sent correctly"""
+    import os
+
+    with open(os.path.join(datastore_path, "endpoint-content.txt"), "w") as f:
+        f.write("<html><body>some stuff<br>and more stuff<br>and even more stuff<br></body></html>")
+
+    notification_url = f'mailto://changedetection@{smtp_test_server}:11025/?to=fff@home.com'
+    notification_body = f"""{default_notification_body}"""
+
+    #####################
+    # Set this up for when we remove the notification from the watch, it should fallback with these details
+    res = client.post(
+        url_for("settings.settings_page"),
+        data={"application-notification_urls": notification_url,
+              "application-notification_title": "fallback-title " + default_notification_title,
+              "application-notification_body": f"{notification_body}\nMore output test\n{ALL_MARKUP_TOKENS}",
+              "application-notification_format": 'text',
+              "requests-time_between_check-minutes": 180,
+              'application-fetch_backend': "html_requests"},
+        follow_redirects=True
+    )
+
+    assert b"Settings updated." in res.data
+
+    # Add our URL to the import page
+    test_url = url_for('test_endpoint', content_type="text/html", _external=True)
+    uuid = client.application.config.get('DATASTORE').add_watch(url=test_url)
+    client.get(url_for("ui.form_watch_checknow"), follow_redirects=True)
+    wait_for_all_checks(client)
+
+    with open(os.path.join(datastore_path, "endpoint-content.txt"), "w") as f:
+        f.write("<html><body>sxome stuff<br>and more stuff<br>lets slip this in<br>and this in<br>and even more stuff<br>&lt;tag&gt;</body></html>")
+
+    time.sleep(0.1)
+    client.get(url_for("ui.form_watch_checknow"), follow_redirects=True)
+    wait_for_all_checks(client)
+
+
+    # Parse the email properly using Python's email library
+    msg = message_from_string(get_last_message_from_smtp_server(), policy=email_policy)
+
+    assert not msg.is_multipart()
+    assert msg.get_content_type() == 'text/plain'
+    body = msg.get_content()
+
+    assert '<tag>' in body # Should have got converted from original HTML to plaintext
+    assert '(changed) some stuff\r\n' in body
+    assert '(into) sxome stuff\r\n' in body
+    assert '(added) lets slip this in\r\n' in body
+    assert '(added) and this in\r\n' in body
+    assert '&nbsp;' not in body
+
+
+    delete_all_watches(client)
+
+
+def test_check_html_notification_with_apprise_format_is_html(client, live_server, measure_memory_usage, datastore_path):
+    ##  live_server_setup(live_server) # Setup on conftest per function
+    set_original_response(datastore_path=datastore_path)
+
+
+    notification_url = f'mailto://changedetection@{smtp_test_server}:11025/?to=fff@home.com&format=html'
+
+    #####################
+    # Set this up for when we remove the notification from the watch, it should fallback with these details
+    res = client.post(
+        url_for("settings.settings_page"),
+        data={"application-notification_urls": notification_url,
+              "application-notification_title": "fallback-title " + default_notification_title,
+              "application-notification_body": "some text\nfallback-body<br> " + default_notification_body,
+              "application-notification_format": 'html',
+              "requests-time_between_check-minutes": 180,
+              'application-fetch_backend': "html_requests"},
+        follow_redirects=True
+    )
+    assert b"Settings updated." in res.data
+
+    # Add a watch and trigger a HTTP POST
+    test_url = url_for('test_endpoint', _external=True)
+    res = client.post(
+        url_for("ui.ui_views.form_quick_watch_add"),
+        data={"url": test_url, "tags": 'nice one'},
+        follow_redirects=True
+    )
+
+    assert b"Watch added" in res.data
+
+    wait_for_all_checks(client)
+    set_longer_modified_response(datastore_path=datastore_path)
+    time.sleep(2)
+
+    client.get(url_for("ui.form_watch_checknow"), follow_redirects=True)
+    wait_for_all_checks(client)
+
+    time.sleep(3)
+
+    msg_raw = get_last_message_from_smtp_server()
+    assert len(msg_raw) >= 1
+
+    # Parse the email properly using Python's email library
+    msg = message_from_string(msg_raw, policy=email_policy)
+
+    # The email should have two bodies (multipart/alternative with text/plain and text/html)
+    assert msg.is_multipart()
+    assert msg.get_content_type() == 'multipart/alternative'
+
+    # Get the parts
+    parts = list(msg.iter_parts())
+    assert len(parts) == 2
+
+    # First part should be text/plain (the auto-generated plaintext version)
+    text_part = parts[0]
+    assert text_part.get_content_type() == 'text/plain'
+    text_content = text_part.get_content()
+    assert '(added) So let\'s see what happens.\r\n' in text_content  # The plaintext part
+    assert 'fallback-body\r\n' in text_content  # The plaintext part
+    assert CUSTOM_LINEBREAK_PLACEHOLDER not in text_content
+
+    # Second part should be text/html
+    html_part = parts[1]
+    assert html_part.get_content_type() == 'text/html'
+    html_content = html_part.get_content()
+    assert 'some text<br>' in html_content  # We converted \n from the notification body
+    assert 'fallback-body<br>' in html_content  # kept the original <br>
+    assert '(added) So let\'s see what happens.<br>' in html_content  # the html part
+    assert CUSTOM_LINEBREAK_PLACEHOLDER not in html_content
+    delete_all_watches(client)

changedetectionio/tests/test_access_control.py

@@ -2,7 +2,7 @@ from .util import live_server_setup, wait_for_all_checks
 from flask import url_for
 import time
 
-def test_check_access_control(app, client, live_server, measure_memory_usage):
+def test_check_access_control(app, client, live_server, measure_memory_usage, datastore_path):
     # Still doesnt work, but this is closer.
    #  live_server_setup(live_server) # Setup on conftest per function
 

changedetectionio/tests/test_add_replace_remove_filter.py

@@ -1,12 +1,16 @@
 #!/usr/bin/env python3
 
 import os.path
+import os
 
 from flask import url_for
 from .util import live_server_setup, wait_for_all_checks, wait_for_notification_endpoint_output, delete_all_watches
 import time
 
-def set_original(excluding=None, add_line=None):
+from ..diff import ADDED_PLACEMARKER_OPEN
+
+
+def set_original(datastore_path, excluding=None, add_line=None):
     test_return_data = """<html>
      <body>
      <p>Some initial text</p>
@@ -32,16 +36,16 @@ def set_original(excluding=None, add_line=None):
 
         test_return_data = output
 
-    with open("test-datastore/endpoint-content.txt", "w") as f:
+    with open(os.path.join(datastore_path, "endpoint-content.txt"), "w") as f:
         f.write(test_return_data)
 
-# def test_setup(client, live_server, measure_memory_usage):
+# def test_setup(client, live_server, measure_memory_usage, datastore_path):
    #  live_server_setup(live_server) # Setup on conftest per function
 
-def test_check_removed_line_contains_trigger(client, live_server, measure_memory_usage):
+def test_check_removed_line_contains_trigger(client, live_server, measure_memory_usage, datastore_path):
 
     # Give the endpoint time to spin up
-    set_original()
+    set_original(datastore_path=datastore_path)
     # Add our URL to the import page
     test_url = url_for('test_endpoint', _external=True)
     uuid = client.application.config.get('DATASTORE').add_watch(url=test_url)
@@ -61,9 +65,10 @@ def test_check_removed_line_contains_trigger(client, live_server, measure_memory
               "time_between_check_use_default": "y"},
         follow_redirects=True
     )
+
     assert b"Updated watch." in res.data
     wait_for_all_checks(client)
-    set_original(excluding='Something irrelevant')
+    set_original(excluding='Something irrelevant', datastore_path=datastore_path)
 
     # A line thats not the trigger should not trigger anything
     res = client.get(url_for("ui.form_watch_checknow"), follow_redirects=True)
@@ -74,7 +79,7 @@ def test_check_removed_line_contains_trigger(client, live_server, measure_memory
     assert b'has-unread-changes' not in res.data
 
     # The trigger line is REMOVED,  this should trigger
-    set_original(excluding='The golden line')
+    set_original(excluding='The golden line', datastore_path=datastore_path)
 
     # Check in the processor here what's going on, its triggering empty-reply and no change.
     client.get(url_for("ui.form_watch_checknow"), follow_redirects=True)
@@ -89,7 +94,7 @@ def test_check_removed_line_contains_trigger(client, live_server, measure_memory
     time.sleep(0.2)
 
     time.sleep(1)
-    set_original(excluding=None)
+    set_original(excluding=None, datastore_path=datastore_path)
     client.get(url_for("ui.form_watch_checknow"), follow_redirects=True)
     wait_for_all_checks(client)
     time.sleep(1)
@@ -97,7 +102,7 @@ def test_check_removed_line_contains_trigger(client, live_server, measure_memory
     assert b'has-unread-changes' not in res.data
 
     # Remove it again, and we should get a trigger
-    set_original(excluding='The golden line')
+    set_original(excluding='The golden line', datastore_path=datastore_path)
     client.get(url_for("ui.form_watch_checknow"), follow_redirects=True)
     wait_for_all_checks(client)
     res = client.get(url_for("watchlist.index"))
@@ -106,7 +111,7 @@ def test_check_removed_line_contains_trigger(client, live_server, measure_memory
     delete_all_watches(client)
 
 
-def test_check_add_line_contains_trigger(client, live_server, measure_memory_usage):
+def test_check_add_line_contains_trigger(client, live_server, measure_memory_usage, datastore_path):
     
     delete_all_watches(client)
     time.sleep(1)
@@ -121,6 +126,7 @@ def test_check_add_line_contains_trigger(client, live_server, measure_memory_usa
               "application-notification_body": 'triggered text was -{{triggered_text}}- ### 网站监测 内容更新了 ####',
               # https://github.com/caronc/apprise/wiki/Notify_Custom_JSON#get-parameter-manipulation
               "application-notification_urls": test_notification_url,
+              "application-notification_format": 'text',
               "application-minutes_between_check": 180,
               "application-fetch_backend": "html_requests"
               },
@@ -128,7 +134,7 @@ def test_check_add_line_contains_trigger(client, live_server, measure_memory_usa
     )
     assert b'Settings updated' in res.data
 
-    set_original()
+    set_original(datastore_path=datastore_path)
     # Add our URL to the import page
     test_url = url_for('test_endpoint', _external=True)
     uuid = client.application.config.get('DATASTORE').add_watch(url=test_url)
@@ -151,7 +157,7 @@ def test_check_add_line_contains_trigger(client, live_server, measure_memory_usa
     )
     assert b"Updated watch." in res.data
     wait_for_all_checks(client)
-    set_original(excluding='Something irrelevant')
+    set_original(excluding='Something irrelevant', datastore_path=datastore_path)
 
     # A line thats not the trigger should not trigger anything
     res = client.get(url_for("ui.form_watch_checknow"), follow_redirects=True)
@@ -162,7 +168,7 @@ def test_check_add_line_contains_trigger(client, live_server, measure_memory_usa
     assert b'has-unread-changes' not in res.data
 
     # The trigger line is ADDED,  this should trigger
-    set_original(add_line='<p>Oh yes please</p>')
+    set_original(add_line='<p>Oh yes please</p>', datastore_path=datastore_path)
     client.get(url_for("ui.form_watch_checknow"), follow_redirects=True)
     wait_for_all_checks(client)
     res = client.get(url_for("watchlist.index"))
@@ -170,10 +176,11 @@ def test_check_add_line_contains_trigger(client, live_server, measure_memory_usa
     assert b'has-unread-changes' in res.data
 
     # Takes a moment for apprise to fire
-    wait_for_notification_endpoint_output()
-    assert os.path.isfile("test-datastore/notification.txt"), "Notification fired because I can see the output file"
-    with open("test-datastore/notification.txt", 'rb') as f:
+    wait_for_notification_endpoint_output(datastore_path=datastore_path)
+    assert os.path.isfile(os.path.join(datastore_path, "notification.txt")), "Notification fired because I can see the output file"
+    with open(os.path.join(datastore_path, "notification.txt"), 'rb') as f:
         response = f.read()
+        assert ADDED_PLACEMARKER_OPEN.encode('utf-8') not in response #  _apply_diff_filtering shouldnt add something here
         assert b'-Oh yes please' in response
         assert '网站监测 内容更新了'.encode('utf-8') in response
 

changedetectionio/tests/test_api.py

@@ -3,12 +3,13 @@
 import time
 from flask import url_for
 from .util import live_server_setup, wait_for_all_checks, delete_all_watches
+import os
 
 import json
 import uuid
 
 
-def set_original_response():
+def set_original_response(datastore_path):
     test_return_data = """<html>
        <body>
      Some initial text<br>
@@ -21,12 +22,12 @@ def set_original_response():
      </html>
     """
 
-    with open("test-datastore/endpoint-content.txt", "w") as f:
+    with open(os.path.join(datastore_path, "endpoint-content.txt"), "w") as f:
         f.write(test_return_data)
     return None
 
 
-def set_modified_response():
+def set_modified_response(datastore_path):
     test_return_data = """<html>
        <body>
      Some initial text<br>
@@ -39,7 +40,7 @@ def set_modified_response():
      </html>
     """
 
-    with open("test-datastore/endpoint-content.txt", "w") as f:
+    with open(os.path.join(datastore_path, "endpoint-content.txt"), "w") as f:
         f.write(test_return_data)
 
     return None
@@ -52,17 +53,17 @@ def is_valid_uuid(val):
         return False
 
 
-# def test_setup(client, live_server, measure_memory_usage):
+# def test_setup(client, live_server, measure_memory_usage, datastore_path):
    #  live_server_setup(live_server) # Setup on conftest per function
 
 
-def test_api_simple(client, live_server, measure_memory_usage):
+def test_api_simple(client, live_server, measure_memory_usage, datastore_path):
     
 
     api_key = live_server.app.config['DATASTORE'].data['settings']['application'].get('api_access_token')
 
     # Create a watch
-    set_original_response()
+    set_original_response(datastore_path=datastore_path)
 
     # Validate bad URL
     test_url = url_for('test_endpoint', _external=True )
@@ -111,7 +112,7 @@ def test_api_simple(client, live_server, measure_memory_usage):
     time.sleep(1)
     wait_for_all_checks(client)
 
-    set_modified_response()
+    set_modified_response(datastore_path=datastore_path)
     # Trigger recheck of all ?recheck_all=1
     client.get(
         url_for("createwatch", recheck_all='1'),
@@ -244,7 +245,7 @@ def test_api_simple(client, live_server, measure_memory_usage):
     )
     assert len(res.json) == 0, "Watch list should be empty"
 
-def test_access_denied(client, live_server, measure_memory_usage):
+def test_access_denied(client, live_server, measure_memory_usage, datastore_path):
     # `config_api_token_enabled` Should be On by default
     res = client.get(
         url_for("createwatch")
@@ -289,11 +290,11 @@ def test_access_denied(client, live_server, measure_memory_usage):
     )
     assert b"Settings updated." in res.data
 
-def test_api_watch_PUT_update(client, live_server, measure_memory_usage):
+def test_api_watch_PUT_update(client, live_server, measure_memory_usage, datastore_path):
 
     api_key = live_server.app.config['DATASTORE'].data['settings']['application'].get('api_access_token')
     # Create a watch
-    set_original_response()
+    set_original_response(datastore_path=datastore_path)
     test_url = url_for('test_endpoint', _external=True)
 
     # Create new
@@ -370,7 +371,7 @@ def test_api_watch_PUT_update(client, live_server, measure_memory_usage):
 
     ######################################################
 
-    # HTTP PUT try a field that doenst exist
+    # HTTP PUT try a field that doesn't exist
 
     # HTTP PUT an update
     res = client.put(
@@ -383,18 +384,30 @@ def test_api_watch_PUT_update(client, live_server, measure_memory_usage):
     # Message will come from `flask_expects_json`
     assert b'Additional properties are not allowed' in res.data
 
+
+    # Try a XSS URL
+    res = client.put(
+        url_for("watch", uuid=watch_uuid),
+        headers={'x-api-key': api_key, 'content-type': 'application/json'},
+        data=json.dumps({
+            'url': 'javascript:alert(document.domain)'
+        }),
+    )
+    assert res.status_code == 400
+
     # Cleanup everything
     delete_all_watches(client)
 
 
-def test_api_import(client, live_server, measure_memory_usage):
+def test_api_import(client, live_server, measure_memory_usage, datastore_path):
 
     api_key = live_server.app.config['DATASTORE'].data['settings']['application'].get('api_access_token')
 
     res = client.post(
         url_for("import") + "?tag=import-test",
         data='https://website1.com\r\nhttps://website2.com',
-        headers={'x-api-key': api_key, 'content-type': 'text/plain'},
+        # We removed  'content-type': 'text/plain', the Import API should assume this if none is set #3547 #3542
+        headers={'x-api-key': api_key},
         follow_redirects=True
     )
 
@@ -408,7 +421,7 @@ def test_api_import(client, live_server, measure_memory_usage):
     res = client.get(url_for('tags.tags_overview_page'))
     assert b'import-test' in res.data
 
-def test_api_conflict_UI_password(client, live_server, measure_memory_usage):
+def test_api_conflict_UI_password(client, live_server, measure_memory_usage, datastore_path):
 
     
     api_key = live_server.app.config['DATASTORE'].data['settings']['application'].get('api_access_token')
@@ -426,7 +439,7 @@ def test_api_conflict_UI_password(client, live_server, measure_memory_usage):
     assert b"Password protection enabled." in res.data
 
     # Create a watch
-    set_original_response()
+    set_original_response(datastore_path=datastore_path)
     test_url = url_for('test_endpoint', _external=True)
 
     # Create new

changedetectionio/tests/test_api_notifications.py

@@ -4,7 +4,7 @@ from flask import url_for
 from .util import live_server_setup
 import json
 
-def test_api_notifications_crud(client, live_server, measure_memory_usage):
+def test_api_notifications_crud(client, live_server, measure_memory_usage, datastore_path):
    #  live_server_setup(live_server) # Setup on conftest per function
     api_key = live_server.app.config['DATASTORE'].data['settings']['application'].get('api_access_token')
 

changedetectionio/tests/test_api_openapi.py

@@ -12,7 +12,7 @@ from flask import url_for
 from .util import live_server_setup, wait_for_all_checks
 
 
-def test_openapi_validation_invalid_content_type_on_create_watch(client, live_server, measure_memory_usage):
+def test_openapi_validation_invalid_content_type_on_create_watch(client, live_server, measure_memory_usage, datastore_path):
     """Test that creating a watch with invalid content-type triggers OpenAPI validation error."""
     api_key = live_server.app.config['DATASTORE'].data['settings']['application'].get('api_access_token')
 
@@ -29,7 +29,7 @@ def test_openapi_validation_invalid_content_type_on_create_watch(client, live_se
     assert b"OpenAPI validation failed" in res.data, "Should contain OpenAPI validation error message"
 
 
-def test_openapi_validation_missing_required_field_create_watch(client, live_server, measure_memory_usage):
+def test_openapi_validation_missing_required_field_create_watch(client, live_server, measure_memory_usage, datastore_path):
     """Test that creating a watch without required URL field triggers OpenAPI validation error."""
     api_key = live_server.app.config['DATASTORE'].data['settings']['application'].get('api_access_token')
 
@@ -46,7 +46,7 @@ def test_openapi_validation_missing_required_field_create_watch(client, live_ser
     assert b"OpenAPI validation failed" in res.data, "Should contain OpenAPI validation error message"
 
 
-def test_openapi_validation_invalid_field_in_request_body(client, live_server, measure_memory_usage):
+def test_openapi_validation_invalid_field_in_request_body(client, live_server, measure_memory_usage, datastore_path):
     """Test that including invalid fields triggers OpenAPI validation error."""
     api_key = live_server.app.config['DATASTORE'].data['settings']['application'].get('api_access_token')
 
@@ -83,7 +83,7 @@ def test_openapi_validation_invalid_field_in_request_body(client, live_server, m
     assert b"Additional properties are not allowed" in res.data, "Should contain validation error about additional properties"
 
 
-def test_openapi_validation_import_wrong_content_type(client, live_server, measure_memory_usage):
+def test_openapi_validation_import_wrong_content_type(client, live_server, measure_memory_usage, datastore_path):
     """Test that import endpoint with wrong content-type triggers OpenAPI validation error."""
     api_key = live_server.app.config['DATASTORE'].data['settings']['application'].get('api_access_token')
 
@@ -100,7 +100,7 @@ def test_openapi_validation_import_wrong_content_type(client, live_server, measu
     assert b"OpenAPI validation failed" in res.data, "Should contain OpenAPI validation error message"
 
 
-def test_openapi_validation_import_correct_content_type_succeeds(client, live_server, measure_memory_usage):
+def test_openapi_validation_import_correct_content_type_succeeds(client, live_server, measure_memory_usage, datastore_path):
     """Test that import endpoint with correct content-type succeeds (positive test)."""
     api_key = live_server.app.config['DATASTORE'].data['settings']['application'].get('api_access_token')
 
@@ -117,7 +117,7 @@ def test_openapi_validation_import_correct_content_type_succeeds(client, live_se
     assert len(res.json) == 2, "Should import 2 URLs"
 
 
-def test_openapi_validation_get_requests_bypass_validation(client, live_server, measure_memory_usage):
+def test_openapi_validation_get_requests_bypass_validation(client, live_server, measure_memory_usage, datastore_path):
     """Test that GET requests bypass OpenAPI validation entirely."""
     api_key = live_server.app.config['DATASTORE'].data['settings']['application'].get('api_access_token')
 
@@ -141,7 +141,7 @@ def test_openapi_validation_get_requests_bypass_validation(client, live_server,
     assert isinstance(res.json, dict), "Should return JSON dictionary for watch list"
 
 
-def test_openapi_validation_create_tag_missing_required_title(client, live_server, measure_memory_usage):
+def test_openapi_validation_create_tag_missing_required_title(client, live_server, measure_memory_usage, datastore_path):
     """Test that creating a tag without required title triggers OpenAPI validation error."""
     api_key = live_server.app.config['DATASTORE'].data['settings']['application'].get('api_access_token')
 
@@ -158,7 +158,7 @@ def test_openapi_validation_create_tag_missing_required_title(client, live_serve
     assert b"OpenAPI validation failed" in res.data, "Should contain OpenAPI validation error message"
 
 
-def test_openapi_validation_watch_update_allows_partial_updates(client, live_server, measure_memory_usage):
+def test_openapi_validation_watch_update_allows_partial_updates(client, live_server, measure_memory_usage, datastore_path):
     """Test that watch updates allow partial updates without requiring all fields (positive test)."""
     api_key = live_server.app.config['DATASTORE'].data['settings']['application'].get('api_access_token')
 

changedetectionio/tests/test_api_search.py

@@ -6,7 +6,7 @@ import time
 from .util import live_server_setup, wait_for_all_checks
 
 
-def test_api_search(client, live_server, measure_memory_usage):
+def test_api_search(client, live_server, measure_memory_usage, datastore_path):
    #  live_server_setup(live_server) # Setup on conftest per function
     api_key = live_server.app.config['DATASTORE'].data['settings']['application'].get('api_access_token')
 

changedetectionio/tests/test_api_tags.py

@@ -5,13 +5,14 @@ from .util import live_server_setup, wait_for_all_checks, set_original_response
 import json
 import time
 
-def test_api_tags_listing(client, live_server, measure_memory_usage):
+def test_api_tags_listing(client, live_server, measure_memory_usage, datastore_path):
    #  live_server_setup(live_server) # Setup on conftest per function
     api_key = live_server.app.config['DATASTORE'].data['settings']['application'].get('api_access_token')
     tag_title = 'Test Tag'
 
 
-    set_original_response()
+    set_original_response(datastore_path=datastore_path)
+
 
     res = client.get(
         url_for("tags"),

changedetectionio/tests/test_auth.py

@@ -5,7 +5,7 @@ from flask import url_for
 from .util import live_server_setup, wait_for_all_checks
 
 # test pages with http://username@password:foobar.com/ work
-def test_basic_auth(client, live_server, measure_memory_usage):
+def test_basic_auth(client, live_server, measure_memory_usage, datastore_path):
    #  live_server_setup(live_server) # Setup on conftest per function
 
 

changedetectionio/tests/test_automatic_follow_ldjson_price.py

@@ -3,9 +3,10 @@
 import time
 from flask import url_for
 from .util import live_server_setup, extract_UUID_from_client, wait_for_all_checks
+import os
 
 
-def set_response_with_ldjson():
+def set_response_with_ldjson(datastore_path):
     test_return_data = """<html>
        <body>
      Some initial text<br>
@@ -55,11 +56,11 @@ def set_response_with_ldjson():
      </html>
 """
 
-    with open("test-datastore/endpoint-content.txt", "w") as f:
+    with open(os.path.join(datastore_path, "endpoint-content.txt"), "w") as f:
         f.write(test_return_data)
     return None
 
-def set_response_without_ldjson():
+def set_response_without_ldjson(datastore_path):
     test_return_data = """<html>
        <body>
      Some initial text<br>
@@ -72,17 +73,17 @@ def set_response_without_ldjson():
      </html>
 """
 
-    with open("test-datastore/endpoint-content.txt", "w") as f:
+    with open(os.path.join(datastore_path, "endpoint-content.txt"), "w") as f:
         f.write(test_return_data)
     return None
 
-# def test_setup(client, live_server, measure_memory_usage):
+# def test_setup(client, live_server, measure_memory_usage, datastore_path):
    #  live_server_setup(live_server) # Setup on conftest per function
 
 # actually only really used by the distll.io importer, but could be handy too
-def test_check_ldjson_price_autodetect(client, live_server, measure_memory_usage):
+def test_check_ldjson_price_autodetect(client, live_server, measure_memory_usage, datastore_path):
     
-    set_response_with_ldjson()
+    set_response_with_ldjson(datastore_path=datastore_path)
 
     # Add our URL to the import page
     test_url = url_for('test_endpoint', _external=True)
@@ -121,7 +122,7 @@ def test_check_ldjson_price_autodetect(client, live_server, measure_memory_usage
 
     ##########################################################################################
     # And we shouldnt see the offer
-    set_response_without_ldjson()
+    set_response_without_ldjson(datastore_path=datastore_path)
 
     # Add our URL to the import page
     test_url = url_for('test_endpoint', _external=True)
@@ -151,7 +152,7 @@ def _test_runner_check_bad_format_ignored(live_server, client, has_ldjson_price_
     client.get(url_for("ui.form_delete", uuid="all"), follow_redirects=True)
 
 
-def test_bad_ldjson_is_correctly_ignored(client, live_server, measure_memory_usage):
+def test_bad_ldjson_is_correctly_ignored(client, live_server, measure_memory_usage, datastore_path):
     
     test_return_data = """
             <html>
@@ -181,7 +182,7 @@ def test_bad_ldjson_is_correctly_ignored(client, live_server, measure_memory_usa
             <div class="yes">Some extra stuff</div>
             </body></html>
      """
-    with open("test-datastore/endpoint-content.txt", "w") as f:
+    with open(os.path.join(datastore_path, "endpoint-content.txt"), "w") as f:
         f.write(test_return_data)
 
     _test_runner_check_bad_format_ignored(live_server=live_server, client=client, has_ldjson_price_data=True)
@@ -215,7 +216,7 @@ def test_bad_ldjson_is_correctly_ignored(client, live_server, measure_memory_usa
     #         <div class="yes">Some extra stuff</div>
     #         </body></html>
     #  """
-    # with open("test-datastore/endpoint-content.txt", "w") as f:
+    # with open(os.path.join(datastore_path, "endpoint-content.txt"), "w") as f:
     #     f.write(test_return_data)
     #
     # _test_runner_check_bad_format_ignored(live_server=live_server, client=client, has_ldjson_price_data=False)

changedetectionio/tests/test_backend.py

@@ -1,4 +1,5 @@
 #!/usr/bin/env python3
+import os
 
 import time
 from flask import url_for
@@ -16,8 +17,8 @@ def test_inscriptus():
     assert stripped_text_from_html == 'test!\nok man'
 
 
-def test_check_basic_change_detection_functionality(client, live_server, measure_memory_usage):
-    set_original_response()
+def test_check_basic_change_detection_functionality(client, live_server, measure_memory_usage, datastore_path):
+    set_original_response(datastore_path=datastore_path)
    #  live_server_setup(live_server) # Setup on conftest per function
 
     # Add our URL to the import page
@@ -60,7 +61,7 @@ def test_check_basic_change_detection_functionality(client, live_server, measure
     assert b'foobar-detection' not in res.data
 
     # Make a change
-    set_modified_response()
+    set_modified_response(datastore_path=datastore_path)
 
     # Force recheck
     res = client.get(url_for("ui.form_watch_checknow"), follow_redirects=True)
@@ -121,7 +122,7 @@ def test_check_basic_change_detection_functionality(client, live_server, measure
         assert b'test-endpoint' in res.data
 
     # Recheck it but only with a title change, content wasnt changed
-    set_original_response(extra_title=" and more")
+    set_original_response(datastore_path=datastore_path, extra_title=" and more")
 
     client.get(url_for("ui.form_watch_checknow"), follow_redirects=True)
     wait_for_all_checks(client)
@@ -167,7 +168,7 @@ def test_check_basic_change_detection_functionality(client, live_server, measure
 
 
 # Server says its plaintext, we should always treat it as plaintext, and then if they have a filter, try to apply that
-def test_requests_timeout(client, live_server, measure_memory_usage):
+def test_requests_timeout(client, live_server, measure_memory_usage, datastore_path):
     delay = 2
     test_url = url_for('test_endpoint', delay=delay, _external=True)
 
@@ -205,7 +206,7 @@ def test_requests_timeout(client, live_server, measure_memory_usage):
     res = client.get(url_for("watchlist.index"))
     assert b'Read timed out' not in res.data
 
-def test_non_text_mime_or_downloads(client, live_server, measure_memory_usage):
+def test_non_text_mime_or_downloads(client, live_server, measure_memory_usage, datastore_path):
     """
 
     https://github.com/dgtlmoon/changedetection.io/issues/3434
@@ -220,7 +221,7 @@ def test_non_text_mime_or_downloads(client, live_server, measure_memory_usage):
     :param measure_memory_usage:
     :return:
     """
-    with open("test-datastore/endpoint-content.txt", "w") as f:
+    with open(os.path.join(datastore_path, "endpoint-content.txt"), "w") as f:
         f.write("""some random text that should be split by line
 and not parsed with html_to_text
 this way we know that it correctly parsed as plain text
@@ -264,7 +265,7 @@ got it\r\n
     delete_all_watches(client)
 
 
-def test_standard_text_plain(client, live_server, measure_memory_usage):
+def test_standard_text_plain(client, live_server, measure_memory_usage, datastore_path):
     """
 
     https://github.com/dgtlmoon/changedetection.io/issues/3434
@@ -279,7 +280,7 @@ def test_standard_text_plain(client, live_server, measure_memory_usage):
     :param measure_memory_usage:
     :return:
     """
-    with open("test-datastore/endpoint-content.txt", "w") as f:
+    with open(os.path.join(datastore_path, "endpoint-content.txt"), "w") as f:
         f.write("""some random text that should be split by line
 and not parsed with html_to_text
 <title>Even this title should stay because we are just plain text</title>
@@ -325,9 +326,9 @@ got it\r\n
     delete_all_watches(client)
 
 # Server says its plaintext, we should always treat it as plaintext
-def test_plaintext_even_if_xml_content(client, live_server, measure_memory_usage):
+def test_plaintext_even_if_xml_content(client, live_server, measure_memory_usage, datastore_path):
 
-    with open("test-datastore/endpoint-content.txt", "w") as f:
+    with open(os.path.join(datastore_path, "endpoint-content.txt"), "w") as f:
         f.write("""<?xml version="1.0" encoding="utf-8"?>
 <resources xmlns:tools="http://schemas.android.com/tools">
     <!--Activity and fragment titles-->
@@ -353,10 +354,10 @@ def test_plaintext_even_if_xml_content(client, live_server, measure_memory_usage
     delete_all_watches(client)
 
 # Server says its plaintext, we should always treat it as plaintext, and then if they have a filter, try to apply that
-def test_plaintext_even_if_xml_content_and_can_apply_filters(client, live_server, measure_memory_usage):
+def test_plaintext_even_if_xml_content_and_can_apply_filters(client, live_server, measure_memory_usage, datastore_path):
 
 
-    with open("test-datastore/endpoint-content.txt", "w") as f:
+    with open(os.path.join(datastore_path, "endpoint-content.txt"), "w") as f:
         f.write("""<?xml version="1.0" encoding="utf-8"?>
 <resources xmlns:tools="http://schemas.android.com/tools">
     <!--Activity and fragment titles-->

changedetectionio/tests/test_backup.py

@@ -8,13 +8,11 @@ import re
 import time
 
 
-def test_backup(client, live_server, measure_memory_usage):
+def test_backup(client, live_server, measure_memory_usage, datastore_path):
    #  live_server_setup(live_server) # Setup on conftest per function
 
-    set_original_response()
+    set_original_response(datastore_path=datastore_path)
 
-    # Give the endpoint time to spin up
-    time.sleep(1)
 
     # Add our URL to the import page
     res = client.post(
@@ -31,7 +29,7 @@ def test_backup(client, live_server, measure_memory_usage):
         url_for("backups.request_backup"),
         follow_redirects=True
     )
-    time.sleep(2)
+    time.sleep(4)
 
     res = client.get(
         url_for("backups.index"),

changedetectionio/tests/test_basic_socketio.py

@@ -10,11 +10,12 @@ from .util import (
 )
 from loguru import logger
 
-def run_socketio_watch_update_test(client, live_server, password_mode=""):
+def run_socketio_watch_update_test(client, live_server, password_mode="", datastore_path=""):
     """Test that the socketio emits a watch update event when content changes"""
 
     # Set up the test server
-    set_original_response()
+    set_original_response(datastore_path=datastore_path)
+
 
     # Get the SocketIO instance from the app
     from changedetectionio.flask_app import app
@@ -47,7 +48,7 @@ def run_socketio_watch_update_test(client, live_server, password_mode=""):
     socketio_test_client.get_received()
 
     # Make a change to trigger an update
-    set_modified_response()
+    set_modified_response(datastore_path=datastore_path)
 
     # Force recheck
     res = client.get(url_for("ui.form_watch_checknow"), follow_redirects=True)
@@ -105,11 +106,11 @@ def run_socketio_watch_update_test(client, live_server, password_mode=""):
     # Clean up
     client.get(url_for("ui.form_delete", uuid="all"), follow_redirects=True)
 
-def test_everything(live_server, client):
+def test_everything(live_server, client, measure_memory_usage, datastore_path):
 
    #  live_server_setup(live_server) # Setup on conftest per function
 
-    run_socketio_watch_update_test(password_mode="", live_server=live_server, client=client)
+    run_socketio_watch_update_test(password_mode="", live_server=live_server, client=client, datastore_path=datastore_path)
 
     ############################ Password required auth check ##############################
 
@@ -124,7 +125,7 @@ def test_everything(live_server, client):
 
     assert b"Password protection enabled." in res.data
 
-    run_socketio_watch_update_test(password_mode="not logged in, should exit on connect", live_server=live_server, client=client)
+    run_socketio_watch_update_test(password_mode="not logged in, should exit on connect", live_server=live_server, client=client, datastore_path=datastore_path)
     res = client.post(
         url_for("login"),
         data={"password": "foobar"},
@@ -133,4 +134,4 @@ def test_everything(live_server, client):
 
     # Yes we are correctly logged in
     assert b"LOG OUT" in res.data
-    run_socketio_watch_update_test(password_mode="should be like normal", live_server=live_server, client=client)
+    run_socketio_watch_update_test(password_mode="should be like normal", live_server=live_server, client=client, datastore_path=datastore_path)

changedetectionio/tests/test_block_while_text_present.py

@@ -4,8 +4,9 @@ import time
 from flask import url_for
 from .util import live_server_setup, wait_for_all_checks, delete_all_watches
 from changedetectionio import html_tools
+import os
 
-def set_original_ignore_response():
+def set_original_ignore_response(datastore_path):
     test_return_data = """<html>
        <body>
      Some initial text<br>
@@ -17,11 +18,11 @@ def set_original_ignore_response():
 
     """
 
-    with open("test-datastore/endpoint-content.txt", "w") as f:
+    with open(os.path.join(datastore_path, "endpoint-content.txt"), "w") as f:
         f.write(test_return_data)
 
 
-def set_modified_original_ignore_response():
+def set_modified_original_ignore_response(datastore_path):
     test_return_data = """<html>
        <body>
      Some NEW nice initial text<br>
@@ -36,12 +37,12 @@ def set_modified_original_ignore_response():
 
     """
 
-    with open("test-datastore/endpoint-content.txt", "w") as f:
+    with open(os.path.join(datastore_path, "endpoint-content.txt"), "w") as f:
         f.write(test_return_data)
 
 
 # Is the same but includes ZZZZZ, 'ZZZZZ' is the last line in ignore_text
-def set_modified_response_minus_block_text():
+def set_modified_response_minus_block_text(datastore_path):
     test_return_data = """<html>
        <body>
      Some NEW nice initial text<br>
@@ -56,16 +57,16 @@ def set_modified_response_minus_block_text():
 
     """
 
-    with open("test-datastore/endpoint-content.txt", "w") as f:
+    with open(os.path.join(datastore_path, "endpoint-content.txt"), "w") as f:
         f.write(test_return_data)
 
 
-def test_check_block_changedetection_text_NOT_present(client, live_server, measure_memory_usage):
+def test_check_block_changedetection_text_NOT_present(client, live_server, measure_memory_usage, datastore_path):
 
    #  live_server_setup(live_server) # Setup on conftest per function
     # Use a mix of case in ZzZ to prove it works case-insensitive.
     ignore_text = "out of stoCk\r\nfoobar"
-    set_original_ignore_response()
+    set_original_ignore_response(datastore_path=datastore_path)
 
 
     # Add our URL to the import page
@@ -109,7 +110,7 @@ def test_check_block_changedetection_text_NOT_present(client, live_server, measu
     assert b'/test-endpoint' in res.data
 
     # The page changed, BUT the text is still there, just the rest of it changes, we should not see a change
-    set_modified_original_ignore_response()
+    set_modified_original_ignore_response(datastore_path=datastore_path)
 
     # Trigger a check
     client.get(url_for("ui.form_watch_checknow"), follow_redirects=True)
@@ -123,7 +124,7 @@ def test_check_block_changedetection_text_NOT_present(client, live_server, measu
 
     # 2548
     # Going back to the ORIGINAL should NOT trigger a change
-    set_original_ignore_response()
+    set_original_ignore_response(datastore_path=datastore_path)
     client.get(url_for("ui.form_watch_checknow"), follow_redirects=True)
     wait_for_all_checks(client)
     res = client.get(url_for("watchlist.index"))
@@ -131,10 +132,11 @@ def test_check_block_changedetection_text_NOT_present(client, live_server, measu
 
 
     # Now we set a change where the text is gone AND its different content, it should now trigger
-    set_modified_response_minus_block_text()
+    set_modified_response_minus_block_text(datastore_path=datastore_path)
     client.get(url_for("ui.form_watch_checknow"), follow_redirects=True)
     wait_for_all_checks(client)
     res = client.get(url_for("watchlist.index"))
+
     assert b'has-unread-changes' in res.data
 
 

changedetectionio/tests/test_clone.py

@@ -3,12 +3,13 @@
 import time
 from flask import url_for
 from .util import live_server_setup, wait_for_all_checks
+import os
 
 
-def test_clone_functionality(client, live_server, measure_memory_usage):
+def test_clone_functionality(client, live_server, measure_memory_usage, datastore_path):
 
    #  live_server_setup(live_server) # Setup on conftest per function
-    with open("test-datastore/endpoint-content.txt", "w") as f:
+    with open(os.path.join(datastore_path, "endpoint-content.txt"), "w") as f:
         f.write("<html><body>Some content</body></html>")
 
     test_url = url_for('test_endpoint', _external=True)

changedetectionio/tests/test_conditions.py

@@ -1,13 +1,14 @@
 #!/usr/bin/env python3
 import json
 import time
+import os
 
 from flask import url_for
 from .util import live_server_setup, wait_for_all_checks, delete_all_watches
 from ..model import CONDITIONS_MATCH_LOGIC_DEFAULT
 
 
-def set_original_response(number="50"):
+def set_original_response(datastore_path, number="50"):
     test_return_data = f"""<html>
        <body>
      <h1>Test Page for Conditions</h1>
@@ -17,10 +18,10 @@ def set_original_response(number="50"):
      </html>
     """
 
-    with open("test-datastore/endpoint-content.txt", "w") as f:
+    with open(os.path.join(datastore_path, "endpoint-content.txt"), "w") as f:
         f.write(test_return_data)
 
-def set_number_in_range_response(number="75"):
+def set_number_in_range_response(datastore_path, number="75"):
     test_return_data = f"""<html>
        <body>
      <h1>Test Page for Conditions</h1>
@@ -30,10 +31,10 @@ def set_number_in_range_response(number="75"):
      </html>
     """
 
-    with open("test-datastore/endpoint-content.txt", "w") as f:
+    with open(os.path.join(datastore_path, "endpoint-content.txt"), "w") as f:
         f.write(test_return_data)
 
-def set_number_out_of_range_response(number="150"):
+def set_number_out_of_range_response(datastore_path, number="150"):
     test_return_data = f"""<html>
        <body>
      <h1>Test Page for Conditions</h1>
@@ -43,18 +44,18 @@ def set_number_out_of_range_response(number="150"):
      </html>
     """
 
-    with open("test-datastore/endpoint-content.txt", "w") as f:
+    with open(os.path.join(datastore_path, "endpoint-content.txt"), "w") as f:
         f.write(test_return_data)
 
 
-# def test_setup(client, live_server, measure_memory_usage):
+# def test_setup(client, live_server, measure_memory_usage, datastore_path):
     """Test that both text and number conditions work together with AND logic."""
    #  live_server_setup(live_server) # Setup on conftest per function
 
-def test_conditions_with_text_and_number(client, live_server, measure_memory_usage):
+def test_conditions_with_text_and_number(client, live_server, measure_memory_usage, datastore_path):
     """Test that both text and number conditions work together with AND logic."""
     
-    set_original_response("50")
+    set_original_response(datastore_path=datastore_path, number="50")
     
 
     test_url = url_for('test_endpoint', _external=True)
@@ -114,7 +115,7 @@ def test_conditions_with_text_and_number(client, live_server, measure_memory_usa
     wait_for_all_checks(client)
 
     # Case 1
-    set_number_in_range_response("70.5")
+    set_number_in_range_response(datastore_path=datastore_path, number="70.5")
     client.get(url_for("ui.form_watch_checknow"), follow_redirects=True)
     wait_for_all_checks(client)
 
@@ -129,7 +130,7 @@ def test_conditions_with_text_and_number(client, live_server, measure_memory_usa
     client.get(url_for("ui.mark_all_viewed"), follow_redirects=True)
     time.sleep(0.2)
 
-    set_number_out_of_range_response("150.5")
+    set_number_out_of_range_response(datastore_path=datastore_path, number="150.5")
 
 
     client.get(url_for("ui.form_watch_checknow"), follow_redirects=True)
@@ -142,9 +143,9 @@ def test_conditions_with_text_and_number(client, live_server, measure_memory_usa
     delete_all_watches(client)
 
 # The 'validate' button next to each rule row
-def test_condition_validate_rule_row(client, live_server, measure_memory_usage):
+def test_condition_validate_rule_row(client, live_server, measure_memory_usage, datastore_path):
 
-    set_original_response("50")
+    set_original_response(datastore_path=datastore_path, number="50")
 
     test_url = url_for('test_endpoint', _external=True)
 
@@ -203,7 +204,7 @@ def test_condition_validate_rule_row(client, live_server, measure_memory_usage):
 
 
 # If there was only a change in the whitespacing, then we shouldnt have a change detected
-def test_wordcount_conditions_plugin(client, live_server, measure_memory_usage):
+def test_wordcount_conditions_plugin(client, live_server, measure_memory_usage, datastore_path):
     
 
     test_return_data = """<html>
@@ -216,7 +217,7 @@ def test_wordcount_conditions_plugin(client, live_server, measure_memory_usage):
      </html>
     """
 
-    with open("test-datastore/endpoint-content.txt", "w") as f:
+    with open(os.path.join(datastore_path, "endpoint-content.txt"), "w") as f:
         f.write(test_return_data)
 
     # Add our URL to the import page
@@ -242,10 +243,10 @@ def test_wordcount_conditions_plugin(client, live_server, measure_memory_usage):
     )
 
 # If there was only a change in the whitespacing, then we shouldnt have a change detected
-def test_lev_conditions_plugin(client, live_server, measure_memory_usage):
-    
+def test_lev_conditions_plugin(client, live_server, measure_memory_usage, datastore_path):
+    # This should break..
 
-    with open("test-datastore/endpoint-content.txt", "w") as f:
+    with open(os.path.join(datastore_path, "endpoint-content.txt"), "w") as f:
         f.write("""<html>
        <body>
      Some initial text<br>
@@ -297,7 +298,7 @@ def test_lev_conditions_plugin(client, live_server, measure_memory_usage):
 
 
     ############### Now change it a LITTLE bit...
-    with open("test-datastore/endpoint-content.txt", "w") as f:
+    with open(os.path.join(datastore_path, "endpoint-content.txt"), "w") as f:
         f.write("""<html>
        <body>
      Some initial text<br>
@@ -326,7 +327,7 @@ def test_lev_conditions_plugin(client, live_server, measure_memory_usage):
      </html>
     """
 
-    with open("test-datastore/endpoint-content.txt", "w") as f:
+    with open(os.path.join(datastore_path, "endpoint-content.txt"), "w") as f:
         f.write(test_return_data)
     res = client.get(url_for("ui.form_watch_checknow"), follow_redirects=True)
     assert b'Queued 1 watch for rechecking.' in res.data

changedetectionio/tests/test_css_selector.py

@@ -3,12 +3,13 @@
 import time
 from flask import url_for
 from .util import live_server_setup, wait_for_all_checks
+import os
 
 from ..html_tools import *
 
 
 
-def set_original_response():
+def set_original_response(datastore_path):
     test_return_data = """<html>
        <body>
      Some initial text<br>
@@ -21,11 +22,11 @@ def set_original_response():
      </html>
     """
 
-    with open("test-datastore/endpoint-content.txt", "w") as f:
+    with open(os.path.join(datastore_path, "endpoint-content.txt"), "w") as f:
         f.write(test_return_data)
     return None
 
-def set_modified_response():
+def set_modified_response(datastore_path):
     test_return_data = """<html>
        <body>
      Some initial text<br>
@@ -38,7 +39,7 @@ def set_modified_response():
      </html>
     """
 
-    with open("test-datastore/endpoint-content.txt", "w") as f:
+    with open(os.path.join(datastore_path, "endpoint-content.txt"), "w") as f:
         f.write(test_return_data)
 
     return None
@@ -69,12 +70,12 @@ def test_include_filters_output():
 
 
 # Tests the whole stack works with the CSS Filter
-def test_check_markup_include_filters_restriction(client, live_server, measure_memory_usage):
+def test_check_markup_include_filters_restriction(client, live_server, measure_memory_usage, datastore_path):
     sleep_time_for_fetch_thread = 3
 
     include_filters = "#sametext"
 
-    set_original_response()
+    set_original_response(datastore_path=datastore_path)
 
     # Give the endpoint time to spin up
     time.sleep(1)
@@ -105,7 +106,7 @@ def test_check_markup_include_filters_restriction(client, live_server, measure_m
     # Give the thread time to pick it up
     time.sleep(sleep_time_for_fetch_thread)
     #  Make a change
-    set_modified_response()
+    set_modified_response(datastore_path=datastore_path)
 
     # Trigger a check
     client.get(url_for("ui.form_watch_checknow"), follow_redirects=True)
@@ -119,11 +120,11 @@ def test_check_markup_include_filters_restriction(client, live_server, measure_m
 
 
 # Tests the whole stack works with the CSS Filter
-def test_check_multiple_filters(client, live_server, measure_memory_usage):
+def test_check_multiple_filters(client, live_server, measure_memory_usage, datastore_path):
     
     include_filters = "#blob-a\r\nxpath://*[contains(@id,'blob-b')]"
 
-    with open("test-datastore/endpoint-content.txt", "w") as f:
+    with open(os.path.join(datastore_path, "endpoint-content.txt"), "w") as f:
         f.write("""<html><body>
      <div id="blob-a">Blob A</div>
      <div id="blob-b">Blob B</div>
@@ -168,12 +169,12 @@ def test_check_multiple_filters(client, live_server, measure_memory_usage):
 # The filter exists, but did not contain anything useful
 # Mainly used when the filter contains just an IMG, this can happen when someone selects an image in the visual-selector
 # Tests fetcher can throw a "ReplyWithContentButNoText" exception after applying filter and extracting text
-def test_filter_is_empty_help_suggestion(client, live_server, measure_memory_usage):
+def test_filter_is_empty_help_suggestion(client, live_server, measure_memory_usage, datastore_path):
     
 
     include_filters = "#blob-a"
 
-    with open("test-datastore/endpoint-content.txt", "w") as f:
+    with open(os.path.join(datastore_path, "endpoint-content.txt"), "w") as f:
         f.write("""<html><body>
          <div id="blob-a">
            <img src="something.jpg">
@@ -216,7 +217,7 @@ def test_filter_is_empty_help_suggestion(client, live_server, measure_memory_usa
 
     ### Just an empty selector, no image
 
-    with open("test-datastore/endpoint-content.txt", "w") as f:
+    with open(os.path.join(datastore_path, "endpoint-content.txt"), "w") as f:
         f.write("""<html><body>
          <div id="blob-a">
            <!-- doo doo -->

changedetectionio/tests/test_datastore_isolation.py

@@ -0,0 +1,19 @@
+#!/usr/bin/env python3
+"""Test to verify client and live_server share the same datastore"""
+
+def test_client_and_live_server_share_datastore(client, live_server):
+    """Verify that client and live_server use the same app and datastore."""
+
+    # They should be the SAME object
+    assert client.application is live_server.app, "client.application and live_server.app should be the SAME object!"
+
+    # They should share the same datastore
+    client_datastore = client.application.config.get('DATASTORE')
+    server_datastore = live_server.app.config.get('DATASTORE')
+
+    assert client_datastore is server_datastore, \
+        f"Datastores are DIFFERENT objects! client={hex(id(client_datastore))} server={hex(id(server_datastore))}"
+
+    print(f"✓ client.application and live_server.app are the SAME object")
+    print(f"✓ Both use the same DATASTORE at {hex(id(client_datastore))}")
+    print(f"✓ Datastore path: {client_datastore.datastore_path}")

changedetectionio/tests/test_element_removal.py

@@ -1,6 +1,7 @@
 #!/usr/bin/env python3
 
 import time
+import os
 
 from flask import url_for
 
@@ -10,7 +11,7 @@ from .util import live_server_setup, wait_for_all_checks, delete_all_watches
 
 
 
-def set_response_with_multiple_index():
+def set_response_with_multiple_index(datastore_path):
     data= """<!DOCTYPE html>
 <html>
 <body>
@@ -36,11 +37,11 @@ def set_response_with_multiple_index():
 </body>
 </html>
 """
-    with open("test-datastore/endpoint-content.txt", "w") as f:
+    with open(os.path.join(datastore_path, "endpoint-content.txt"), "w") as f:
         f.write(data)
 
 
-def set_original_response():
+def set_original_response(datastore_path):
     test_return_data = """<html>
     <header>
     <h2>Header</h2>
@@ -65,11 +66,11 @@ def set_original_response():
      </html>
     """
 
-    with open("test-datastore/endpoint-content.txt", "w") as f:
+    with open(os.path.join(datastore_path, "endpoint-content.txt"), "w") as f:
         f.write(test_return_data)
 
 
-def set_modified_response():
+def set_modified_response(datastore_path):
     test_return_data = """<html>
     <header>
     <h2>Header changed</h2>
@@ -94,7 +95,7 @@ def set_modified_response():
      </html>
     """
 
-    with open("test-datastore/endpoint-content.txt", "w") as f:
+    with open(os.path.join(datastore_path, "endpoint-content.txt"), "w") as f:
         f.write(test_return_data)
 
 
@@ -146,10 +147,10 @@ across multiple lines
     )
 
 
-def test_element_removal_full(client, live_server, measure_memory_usage):
+def test_element_removal_full(client, live_server, measure_memory_usage, datastore_path):
     
 
-    set_original_response()
+    set_original_response(datastore_path=datastore_path)
 
 
     # Add our URL to the import page
@@ -194,7 +195,7 @@ def test_element_removal_full(client, live_server, measure_memory_usage):
     client.get(url_for("ui.ui_views.diff_history_page", uuid="first"))
 
     #  Make a change to header/footer/nav
-    set_modified_response()
+    set_modified_response(datastore_path=datastore_path)
 
     # Trigger a check
     res = client.get(url_for("ui.form_watch_checknow"), follow_redirects=True)
@@ -208,9 +209,9 @@ def test_element_removal_full(client, live_server, measure_memory_usage):
     assert b"unviewed" not in res.data
 
 # Re #2752
-def test_element_removal_nth_offset_no_shift(client, live_server, measure_memory_usage):
+def test_element_removal_nth_offset_no_shift(client, live_server, measure_memory_usage, datastore_path):
 
-    set_response_with_multiple_index()
+    set_response_with_multiple_index(datastore_path=datastore_path)
     subtractive_selectors_data = [
 ### css style ###
 """body > table > tr:nth-child(1) > th:nth-child(2)

changedetectionio/tests/test_encoding.py

@@ -5,26 +5,27 @@ import time
 from flask import url_for
 from .util import live_server_setup, wait_for_all_checks, extract_UUID_from_client
 import pytest
+import os
 
 
 
 
 
-def set_html_response():
+def set_html_response(datastore_path):
     test_return_data = """
 <html><body><span class="nav_second_img_text">
                   &nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;铸大国重器，挺制造脊梁，致力能源未来，赋能美好生活。
                                   </span>
 </body></html>
     """
-    with open("test-datastore/endpoint-content.txt", "w") as f:
+    with open(os.path.join(datastore_path, "endpoint-content.txt"), "w") as f:
         f.write(test_return_data)
     return None
 
 
 # In the case the server does not issue a charset= or doesnt have content_type header set
-def test_check_encoding_detection(client, live_server, measure_memory_usage):
-    set_html_response()
+def test_check_encoding_detection(client, live_server, measure_memory_usage, datastore_path):
+    set_html_response(datastore_path=datastore_path)
 
     # Add our URL to the import page
     test_url = url_for('test_endpoint', content_type="text/html", _external=True)
@@ -51,8 +52,8 @@ def test_check_encoding_detection(client, live_server, measure_memory_usage):
 
 
 # In the case the server does not issue a charset= or doesnt have content_type header set
-def test_check_encoding_detection_missing_content_type_header(client, live_server, measure_memory_usage):
-    set_html_response()
+def test_check_encoding_detection_missing_content_type_header(client, live_server, measure_memory_usage, datastore_path):
+    set_html_response(datastore_path=datastore_path)
 
     # Add our URL to the import page
     test_url = url_for('test_endpoint', _external=True)

changedetectionio/tests/test_errorhandling.py

@@ -1,6 +1,7 @@
 #!/usr/bin/env python3
 
 import time
+import os
 
 from flask import url_for
 from .util import live_server_setup, wait_for_all_checks, delete_all_watches
@@ -8,9 +9,9 @@ from .util import live_server_setup, wait_for_all_checks, delete_all_watches
 
 
 
-def _runner_test_http_errors(client, live_server, http_code, expected_text):
+def _runner_test_http_errors(client, live_server, http_code, expected_text, datastore_path):
 
-    with open("test-datastore/endpoint-content.txt", "w") as f:
+    with open(os.path.join(datastore_path, "endpoint-content.txt"), "w") as f:
         f.write("Now you going to get a {} error code\n".format(http_code))
 
 
@@ -46,17 +47,15 @@ def _runner_test_http_errors(client, live_server, http_code, expected_text):
     delete_all_watches(client)
 
 
-def test_http_error_handler(client, live_server, measure_memory_usage):
-    _runner_test_http_errors(client, live_server, 403, 'Access denied')
-    _runner_test_http_errors(client, live_server, 404, 'Page not found')
-    _runner_test_http_errors(client, live_server, 500, '(Internal server error) received')
-    _runner_test_http_errors(client, live_server, 400, 'Error - Request returned a HTTP error code 400')
+def test_http_error_handler(client, live_server, measure_memory_usage, datastore_path):
+    _runner_test_http_errors(client, live_server, 403, 'Access denied', datastore_path=datastore_path)
+    _runner_test_http_errors(client, live_server, 404, 'Page not found', datastore_path=datastore_path)
+    _runner_test_http_errors(client, live_server, 500, '(Internal server error) received', datastore_path=datastore_path)
+    _runner_test_http_errors(client, live_server, 400, 'Error - Request returned a HTTP error code 400', datastore_path=datastore_path)
     delete_all_watches(client)
 
 # Just to be sure error text is properly handled
-def test_DNS_errors(client, live_server, measure_memory_usage):
-    # Give the endpoint time to spin up
-    time.sleep(1)
+def test_DNS_errors(client, live_server, measure_memory_usage, datastore_path):
 
     # Add our URL to the import page
     res = client.post(
@@ -84,12 +83,9 @@ def test_DNS_errors(client, live_server, measure_memory_usage):
     delete_all_watches(client)
 
 # Re 1513
-def test_low_level_errors_clear_correctly(client, live_server, measure_memory_usage):
-    
-    # Give the endpoint time to spin up
-    time.sleep(1)
+def test_low_level_errors_clear_correctly(client, live_server, measure_memory_usage, datastore_path):
 
-    with open("test-datastore/endpoint-content.txt", "w") as f:
+    with open(os.path.join(datastore_path, "endpoint-content.txt"), "w") as f:
         f.write("<html><body><div id=here>Hello world</div></body></html>")
 
     # Add our URL to the import page

changedetectionio/tests/test_extract_csv.py

@@ -4,14 +4,15 @@ import time
 from flask import url_for
 from urllib.request import urlopen
 from .util import set_original_response, set_modified_response, live_server_setup, wait_for_all_checks
+import os
 
 sleep_time_for_fetch_thread = 3
 
 
 
-def test_check_extract_text_from_diff(client, live_server, measure_memory_usage):
+def test_check_extract_text_from_diff(client, live_server, measure_memory_usage, datastore_path):
     import time
-    with open("test-datastore/endpoint-content.txt", "w") as f:
+    with open(os.path.join(datastore_path, "endpoint-content.txt"), "w") as f:
         f.write("Now it's {} seconds since epoch, time flies!".format(str(time.time())))
 
    #  live_server_setup(live_server) # Setup on conftest per function
@@ -33,7 +34,7 @@ def test_check_extract_text_from_diff(client, live_server, measure_memory_usage)
         # Give the thread time to pick it up
         print("Bumping snapshot and checking.. ", n)
         last_date = str(time.time())
-        with open("test-datastore/endpoint-content.txt", "w") as f:
+        with open(os.path.join(datastore_path, "endpoint-content.txt"), "w") as f:
             f.write("Now it's {} seconds since epoch, time flies!".format(last_date))
 
         client.get(url_for("ui.form_watch_checknow"), follow_redirects=True)

changedetectionio/tests/test_extract_regex.py

@@ -3,11 +3,12 @@
 import time
 from flask import url_for
 from .util import live_server_setup, wait_for_all_checks, delete_all_watches
+import os
 
 from ..html_tools import *
 
 
-def set_original_response():
+def set_original_response(datastore_path):
     test_return_data = """<html>
        <body>
      Some initial text<br>
@@ -20,12 +21,12 @@ def set_original_response():
      </html>
     """
 
-    with open("test-datastore/endpoint-content.txt", "w") as f:
+    with open(os.path.join(datastore_path, "endpoint-content.txt"), "w") as f:
         f.write(test_return_data)
     return None
 
 
-def set_modified_response():
+def set_modified_response(datastore_path):
     test_return_data = """<html>
        <body>
      Some initial text<br>
@@ -39,13 +40,13 @@ def set_modified_response():
      </html>
     """
 
-    with open("test-datastore/endpoint-content.txt", "w") as f:
+    with open(os.path.join(datastore_path, "endpoint-content.txt"), "w") as f:
         f.write(test_return_data)
 
     return None
 
 
-def set_multiline_response():
+def set_multiline_response(datastore_path):
     test_return_data = """<html>
        <body>
      
@@ -61,18 +62,18 @@ def set_multiline_response():
      </html>
     """
 
-    with open("test-datastore/endpoint-content.txt", "w") as f:
+    with open(os.path.join(datastore_path, "endpoint-content.txt"), "w") as f:
         f.write(test_return_data)
 
     return None
 
 
-# def test_setup(client, live_server, measure_memory_usage):
+# def test_setup(client, live_server, measure_memory_usage, datastore_path):
    #  live_server_setup(live_server) # Setup on conftest per function
 
-def test_check_filter_multiline(client, live_server, measure_memory_usage):
+def test_check_filter_multiline(client, live_server, measure_memory_usage, datastore_path):
    ##  live_server_setup(live_server) # Setup on conftest per function
-    set_multiline_response()
+    set_multiline_response(datastore_path=datastore_path)
 
     # Add our URL to the import page
     test_url = url_for('test_endpoint', _external=True)
@@ -119,11 +120,11 @@ def test_check_filter_multiline(client, live_server, measure_memory_usage):
     # but the last one, which also says 'lines' shouldnt be here (non-greedy match checking)
     assert b'aaand something lines' not in res.data
 
-def test_check_filter_and_regex_extract(client, live_server, measure_memory_usage):
+def test_check_filter_and_regex_extract(client, live_server, measure_memory_usage, datastore_path):
     
     include_filters = ".changetext"
 
-    set_original_response()
+    set_original_response(datastore_path=datastore_path)
 
     # Add our URL to the import page
     test_url = url_for('test_endpoint', _external=True)
@@ -159,7 +160,7 @@ def test_check_filter_and_regex_extract(client, live_server, measure_memory_usag
     assert b'not at the start of the expression' not in res.data
 
     #  Make a change
-    set_modified_response()
+    set_modified_response(datastore_path=datastore_path)
 
     # Trigger a check
     client.get(url_for("ui.form_watch_checknow"), follow_redirects=True)
@@ -198,7 +199,7 @@ def test_check_filter_and_regex_extract(client, live_server, measure_memory_usag
 
 
 
-def test_regex_error_handling(client, live_server, measure_memory_usage):
+def test_regex_error_handling(client, live_server, measure_memory_usage, datastore_path):
 
     
 

changedetectionio/tests/test_filter_exist_changes.py

@@ -8,7 +8,7 @@ from .util import set_original_response, live_server_setup, wait_for_notificatio
 from changedetectionio.model import App
 
 
-def set_response_without_filter():
+def set_response_without_filter(datastore_path):
     test_return_data = """<html>
        <body>
      Some initial text<br>
@@ -20,12 +20,12 @@ def set_response_without_filter():
      </html>
     """
 
-    with open("test-datastore/endpoint-content.txt", "w") as f:
+    with open(os.path.join(datastore_path, "endpoint-content.txt"), "w") as f:
         f.write(test_return_data)
     return None
 
 
-def set_response_with_filter():
+def set_response_with_filter(datastore_path):
     test_return_data = """<html>
        <body>
      Some initial text<br>
@@ -37,11 +37,11 @@ def set_response_with_filter():
      </html>
     """
 
-    with open("test-datastore/endpoint-content.txt", "w") as f:
+    with open(os.path.join(datastore_path, "endpoint-content.txt"), "w") as f:
         f.write(test_return_data)
     return None
 
-def test_filter_doesnt_exist_then_exists_should_get_notification(client, live_server, measure_memory_usage):
+def test_filter_doesnt_exist_then_exists_should_get_notification(client, live_server, measure_memory_usage, datastore_path):
 #  Filter knowingly doesn't exist, like someone setting up a known filter to see if some cinema tickets are on sale again
 #  And the page has that filter available
 #  Then I should get a notification
@@ -50,7 +50,7 @@ def test_filter_doesnt_exist_then_exists_should_get_notification(client, live_se
 
     # Give the endpoint time to spin up
     time.sleep(1)
-    set_response_without_filter()
+    set_response_without_filter(datastore_path=datastore_path)
 
     # Add our URL to the import page
     test_url = url_for('test_endpoint', _external=True)
@@ -86,7 +86,7 @@ def test_filter_doesnt_exist_then_exists_should_get_notification(client, live_se
                                                    "Diff Full: {{diff_full}}\n"
                                                    "Diff as Patch: {{diff_patch}}\n"
                                                    ":-)",
-                              "notification_format": "Text"}
+                              "notification_format": 'text'}
 
     notification_form_data.update({
         "url": test_url,
@@ -105,20 +105,20 @@ def test_filter_doesnt_exist_then_exists_should_get_notification(client, live_se
         follow_redirects=True
     )
     assert b"Updated watch." in res.data
-    wait_for_notification_endpoint_output()
+    wait_for_notification_endpoint_output(datastore_path=datastore_path)
 
     # Shouldn't exist, shouldn't have fired
-    assert not os.path.isfile("test-datastore/notification.txt")
+    assert not os.path.isfile(os.path.join(datastore_path, "notification.txt"))
     # Now the filter should exist
-    set_response_with_filter()
+    set_response_with_filter(datastore_path=datastore_path)
     client.get(url_for("ui.form_watch_checknow"), follow_redirects=True)
 
-    wait_for_notification_endpoint_output()
+    wait_for_notification_endpoint_output(datastore_path=datastore_path)
 
-    assert os.path.isfile("test-datastore/notification.txt")
+    assert os.path.isfile(os.path.join(datastore_path, "notification.txt"))
 
-    with open("test-datastore/notification.txt", 'r') as f:
+    with open(os.path.join(datastore_path, "notification.txt"), 'r') as f:
         notification = f.read()
 
     assert 'Ticket now on sale' in notification
-    os.unlink("test-datastore/notification.txt")
+    os.unlink(os.path.join(datastore_path, "notification.txt"))

changedetectionio/tests/test_filter_failure_notification.py

@@ -5,7 +5,7 @@ from .util import set_original_response,  wait_for_all_checks, wait_for_notifica
 from ..notification import valid_notification_formats
 
 
-def set_response_with_filter():
+def set_response_with_filter(datastore_path):
     test_return_data = """<html>
        <body>
      Some initial text<br>
@@ -17,14 +17,14 @@ def set_response_with_filter():
      </html>
     """
 
-    with open("test-datastore/endpoint-content.txt", "w") as f:
+    with open(os.path.join(datastore_path, "endpoint-content.txt"), "w") as f:
         f.write(test_return_data)
     return None
 
-def run_filter_test(client, live_server, content_filter, app_notification_format):
+def run_filter_test(client, live_server, content_filter, app_notification_format, datastore_path):
 
     # Response WITHOUT the filter ID element
-    set_original_response()
+    set_original_response(datastore_path=datastore_path)
     live_server.app.config['DATASTORE'].data['settings']['application']['notification_format'] = app_notification_format
 
     # Goto the edit page, add our ignore text
@@ -38,10 +38,16 @@ def run_filter_test(client, live_server, content_filter, app_notification_format
         url_for("ui.form_delete", uuid="all"),
         follow_redirects=True
     )
-    if os.path.isfile("test-datastore/notification.txt"):
-        os.unlink("test-datastore/notification.txt")
+    notification_file = os.path.join(datastore_path, "notification.txt")
+    if os.path.isfile(notification_file):
+        os.unlink(notification_file)
 
     uuid = client.application.config.get('DATASTORE').add_watch(url=test_url)
+    res = client.get(url_for("watchlist.index"))
+
+    assert b'No website watches configured' not in res.data
+
+
     client.get(url_for("ui.form_watch_checknow"), follow_redirects=True)
     wait_for_all_checks(client)
 
@@ -63,7 +69,7 @@ def run_filter_test(client, live_server, content_filter, app_notification_format
                                        "Diff Full: {{diff_full}}\n"
                                        "Diff as Patch: {{diff_patch}}\n"
                                        ":-)",
-                  "notification_format": "Text",
+                  "notification_format": 'text',
                   "fetch_backend": "html_requests",
                   "filter_failure_notification_send": 'y',
                   "time_between_check_use_default": "y",
@@ -79,6 +85,7 @@ def run_filter_test(client, live_server, content_filter, app_notification_format
         data=watch_data,
         follow_redirects=True
     )
+
     assert b"Updated watch." in res.data
     wait_for_all_checks(client)
     assert live_server.app.config['DATASTORE'].data['watching'][uuid]['consecutive_filter_failures'] == 0, "No filter = No filter failure"
@@ -95,7 +102,7 @@ def run_filter_test(client, live_server, content_filter, app_notification_format
     # It should have checked once so far and given this error (because we hit SAVE)
 
     wait_for_all_checks(client)
-    assert not os.path.isfile("test-datastore/notification.txt")
+    assert not os.path.isfile(notification_file)
 
     # Hitting [save] would have triggered a recheck, and we have a filter, so this would be ONE failure
     assert live_server.app.config['DATASTORE'].data['watching'][uuid]['consecutive_filter_failures'] == 1, "Should have been checked once"
@@ -110,20 +117,20 @@ def run_filter_test(client, live_server, content_filter, app_notification_format
         wait_for_all_checks(client)
         res = client.get(url_for("watchlist.index"))
         assert b'Warning, no filters were found' in res.data
-        assert not os.path.isfile("test-datastore/notification.txt")
+        assert not os.path.isfile(notification_file)
         time.sleep(1)
-        
+
     assert live_server.app.config['DATASTORE'].data['watching'][uuid]['consecutive_filter_failures'] == 5
 
     time.sleep(2)
     # One more check should trigger the _FILTER_FAILURE_THRESHOLD_ATTEMPTS_DEFAULT threshold
     client.get(url_for("ui.form_watch_checknow"), follow_redirects=True)
     wait_for_all_checks(client)
-    wait_for_notification_endpoint_output()
+    wait_for_notification_endpoint_output(datastore_path=datastore_path)
 
     # Now it should exist and contain our "filter not found" alert
-    assert os.path.isfile("test-datastore/notification.txt")
-    with open("test-datastore/notification.txt", 'r') as f:
+    assert os.path.isfile(notification_file)
+    with open(notification_file, 'r') as f:
         notification = f.read()
 
     assert 'Your configured CSS/xPath filters' in notification
@@ -146,19 +153,19 @@ def run_filter_test(client, live_server, content_filter, app_notification_format
 
     # Remove it and prove that it doesn't trigger when not expected
     # It should register a change, but no 'filter not found'
-    os.unlink("test-datastore/notification.txt")
-    set_response_with_filter()
+    os.unlink(notification_file)
+    set_response_with_filter(datastore_path)
 
     # Try several times, it should NOT have 'filter not found'
     for i in range(0, ATTEMPT_THRESHOLD_SETTING + 2):
         client.get(url_for("ui.form_watch_checknow"), follow_redirects=True)
         wait_for_all_checks(client)
 
-    wait_for_notification_endpoint_output()
+    wait_for_notification_endpoint_output(datastore_path=datastore_path)
     # It should have sent a notification, but..
-    assert os.path.isfile("test-datastore/notification.txt")
+    assert os.path.isfile(notification_file)
     # but it should not contain the info about a failed filter (because there was none in this case)
-    with open("test-datastore/notification.txt", 'r') as f:
+    with open(notification_file, 'r') as f:
         notification = f.read()
     assert not 'CSS/xPath filter was not present in the page' in notification
 
@@ -170,22 +177,22 @@ def run_filter_test(client, live_server, content_filter, app_notification_format
         url_for("ui.form_delete", uuid="all"),
         follow_redirects=True
     )
-    os.unlink("test-datastore/notification.txt")
+    os.unlink(notification_file)
 
 
-def test_check_include_filters_failure_notification(client, live_server, measure_memory_usage):
+def test_check_include_filters_failure_notification(client, live_server, measure_memory_usage, datastore_path):
     #   #  live_server_setup(live_server) # Setup on conftest per function
-    run_filter_test(client=client, live_server=live_server, content_filter='#nope-doesnt-exist', app_notification_format=valid_notification_formats.get('HTML Color'))
+    run_filter_test(client=client, live_server=live_server, content_filter='#nope-doesnt-exist', app_notification_format=valid_notification_formats.get('htmlcolor'), datastore_path=datastore_path)
     # Check markup send conversion didnt affect plaintext preference
-    run_filter_test(client=client, live_server=live_server, content_filter='#nope-doesnt-exist', app_notification_format=valid_notification_formats.get('Text'))
+    run_filter_test(client=client, live_server=live_server, content_filter='#nope-doesnt-exist', app_notification_format=valid_notification_formats.get('text'), datastore_path=datastore_path)
 
-def test_check_xpath_filter_failure_notification(client, live_server, measure_memory_usage):
+def test_check_xpath_filter_failure_notification(client, live_server, measure_memory_usage, datastore_path):
     #   #  live_server_setup(live_server) # Setup on conftest per function
-    run_filter_test(client=client, live_server=live_server, content_filter='//*[@id="nope-doesnt-exist"]', app_notification_format=valid_notification_formats.get('HTML Color'))
+    run_filter_test(client=client, live_server=live_server, content_filter='//*[@id="nope-doesnt-exist"]', app_notification_format=valid_notification_formats.get('htmlcolor'), datastore_path=datastore_path)
 
 # Test that notification is never sent
 
-def test_basic_markup_from_text(client, live_server, measure_memory_usage):
+def test_basic_markup_from_text(client, live_server, measure_memory_usage, datastore_path):
     # Test the notification error templates convert to HTML if needed (link activate)
     from ..notification.handler import markup_text_links_to_html
     x = markup_text_links_to_html("hello https://google.com")

changedetectionio/tests/test_group.py

@@ -6,10 +6,10 @@ from .util import live_server_setup, wait_for_all_checks, extract_rss_token_from
 import os
 
 
-# def test_setup(client, live_server, measure_memory_usage):
+# def test_setup(client, live_server, measure_memory_usage, datastore_path):
    #  live_server_setup(live_server) # Setup on conftest per function
 
-def set_original_response():
+def set_original_response(datastore_path):
     test_return_data = """<html>
        <body>
      Some initial text<br>
@@ -20,11 +20,11 @@ def set_original_response():
      </html>
     """
 
-    with open("test-datastore/endpoint-content.txt", "w") as f:
+    with open(os.path.join(datastore_path, "endpoint-content.txt"), "w") as f:
         f.write(test_return_data)
     return None
 
-def set_modified_response():
+def set_modified_response(datastore_path):
     test_return_data = """<html>
        <body>
      Some initial text<br>
@@ -35,13 +35,13 @@ def set_modified_response():
      </html>
     """
 
-    with open("test-datastore/endpoint-content.txt", "w") as f:
+    with open(os.path.join(datastore_path, "endpoint-content.txt"), "w") as f:
         f.write(test_return_data)
     return None
 
-def test_setup_group_tag(client, live_server, measure_memory_usage):
+def test_setup_group_tag(client, live_server, measure_memory_usage, datastore_path):
     
-    set_original_response()
+    set_original_response(datastore_path=datastore_path)
 
     # Add a tag with some config, import a tag and it should roughly work
     res = client.post(
@@ -116,7 +116,7 @@ def test_setup_group_tag(client, live_server, measure_memory_usage):
     )
     assert b"1 Imported" in res.data
     wait_for_all_checks(client)
-    set_modified_response()
+    set_modified_response(datastore_path=datastore_path)
     res = client.get(url_for("ui.form_watch_checknow"), follow_redirects=True)
     wait_for_all_checks(client)
     rss_token = extract_rss_token_from_UI(client)
@@ -129,7 +129,7 @@ def test_setup_group_tag(client, live_server, measure_memory_usage):
     assert b"first-imported=1" in res.data
     delete_all_watches(client)
 
-def test_tag_import_singular(client, live_server, measure_memory_usage):
+def test_tag_import_singular(client, live_server, measure_memory_usage, datastore_path):
     
 
     test_url = url_for('test_endpoint', _external=True)
@@ -148,7 +148,7 @@ def test_tag_import_singular(client, live_server, measure_memory_usage):
     assert res.data.count(b'test-tag') == 1
     delete_all_watches(client)
 
-def test_tag_add_in_ui(client, live_server, measure_memory_usage):
+def test_tag_add_in_ui(client, live_server, measure_memory_usage, datastore_path):
     
 #
     res = client.post(
@@ -164,9 +164,9 @@ def test_tag_add_in_ui(client, live_server, measure_memory_usage):
 
     delete_all_watches(client)
 
-def test_group_tag_notification(client, live_server, measure_memory_usage):
+def test_group_tag_notification(client, live_server, measure_memory_usage, datastore_path):
     
-    set_original_response()
+    set_original_response(datastore_path=datastore_path)
 
     test_url = url_for('test_endpoint', _external=True)
     res = client.post(
@@ -195,7 +195,7 @@ def test_group_tag_notification(client, live_server, measure_memory_usage):
                                                    "Diff as Patch: {{diff_patch}}\n"
                                                    ":-)",
                               "notification_screenshot": True,
-                              "notification_format": "Text",
+                              "notification_format": 'text',
                               "title": "test-tag"}
 
     res = client.post(
@@ -207,16 +207,16 @@ def test_group_tag_notification(client, live_server, measure_memory_usage):
 
     wait_for_all_checks(client)
 
-    set_modified_response()
+    set_modified_response(datastore_path=datastore_path)
     client.get(url_for("ui.form_watch_checknow"), follow_redirects=True)
     time.sleep(3)
 
-    assert os.path.isfile("test-datastore/notification.txt")
+    assert os.path.isfile(os.path.join(datastore_path, "notification.txt"))
 
     # Verify what was sent as a notification, this file should exist
-    with open("test-datastore/notification.txt", "r") as f:
+    with open(os.path.join(datastore_path, "notification.txt"), "r") as f:
         notification_submission = f.read()
-    os.unlink("test-datastore/notification.txt")
+    os.unlink(os.path.join(datastore_path, "notification.txt"))
 
     # Did we see the URL that had a change, in the notification?
     # Diff was correctly executed
@@ -231,7 +231,7 @@ def test_group_tag_notification(client, live_server, measure_memory_usage):
     #@todo Test that each of multiple notifications with different settings
     delete_all_watches(client)
 
-def test_limit_tag_ui(client, live_server, measure_memory_usage):
+def test_limit_tag_ui(client, live_server, measure_memory_usage, datastore_path):
 
     test_url = url_for('test_random_content_endpoint', _external=True)
 
@@ -269,7 +269,7 @@ def test_limit_tag_ui(client, live_server, measure_memory_usage):
     res = client.get(url_for("tags.delete_all"), follow_redirects=True)
     assert b'All tags deleted' in res.data
 
-def test_clone_tag_on_import(client, live_server, measure_memory_usage):
+def test_clone_tag_on_import(client, live_server, measure_memory_usage, datastore_path):
     
     test_url = url_for('test_endpoint', _external=True)
     res = client.post(
@@ -294,7 +294,7 @@ def test_clone_tag_on_import(client, live_server, measure_memory_usage):
     assert res.data.count(b'another-tag') == 3
     delete_all_watches(client)
 
-def test_clone_tag_on_quickwatchform_add(client, live_server, measure_memory_usage):
+def test_clone_tag_on_quickwatchform_add(client, live_server, measure_memory_usage, datastore_path):
     
 
     test_url = url_for('test_endpoint', _external=True)
@@ -324,7 +324,7 @@ def test_clone_tag_on_quickwatchform_add(client, live_server, measure_memory_usa
     res = client.get(url_for("tags.delete_all"), follow_redirects=True)
     assert b'All tags deleted' in res.data
 
-def test_order_of_filters_tag_filter_and_watch_filter(client, live_server, measure_memory_usage):
+def test_order_of_filters_tag_filter_and_watch_filter(client, live_server, measure_memory_usage, datastore_path):
 
     # Add a tag with some config, import a tag and it should roughly work
     res = client.post(
@@ -378,7 +378,7 @@ def test_order_of_filters_tag_filter_and_watch_filter(client, live_server, measu
      </html>
     """
 
-    with open("test-datastore/endpoint-content.txt", "w") as f:
+    with open(os.path.join(datastore_path, "endpoint-content.txt"), "w") as f:
         f.write(d)
 
     test_url = url_for('test_endpoint', _external=True)

changedetectionio/tests/test_history_consistency.py

@@ -7,7 +7,7 @@ from flask import url_for
 from .util import wait_for_all_checks, delete_all_watches
 from urllib.parse import urlparse, parse_qs
 
-def test_consistent_history(client, live_server, measure_memory_usage):
+def test_consistent_history(client, live_server, measure_memory_usage, datastore_path):
    #  live_server_setup(live_server) # Setup on conftest per function
     workers = int(os.getenv("FETCH_WORKERS", 10))
     r = range(1, 10+workers)
@@ -80,9 +80,9 @@ def test_consistent_history(client, live_server, measure_memory_usage):
         assert '"default"' not in f.read(), "'default' probably shouldnt be here, it came from when the 'default' Watch vars were accidently being saved"
 
 
-def test_check_text_history_view(client, live_server, measure_memory_usage):
+def test_check_text_history_view(client, live_server, measure_memory_usage, datastore_path):
 
-    with open("test-datastore/endpoint-content.txt", "w") as f:
+    with open(os.path.join(datastore_path, "endpoint-content.txt"), "w") as f:
         f.write("<html>test-one</html>")
 
     # Add our URL to the import page
@@ -94,7 +94,7 @@ def test_check_text_history_view(client, live_server, measure_memory_usage):
     wait_for_all_checks(client)
 
     # Set second version, Make a change
-    with open("test-datastore/endpoint-content.txt", "w") as f:
+    with open(os.path.join(datastore_path, "endpoint-content.txt"), "w") as f:
         f.write("<html>test-two</html>")
 
     client.get(url_for("ui.form_watch_checknow"), follow_redirects=True)
@@ -105,7 +105,7 @@ def test_check_text_history_view(client, live_server, measure_memory_usage):
     assert b'test-two' in res.data
 
     # Set third version, Make a change
-    with open("test-datastore/endpoint-content.txt", "w") as f:
+    with open(os.path.join(datastore_path, "endpoint-content.txt"), "w") as f:
         f.write("<html>test-three</html>")
 
     client.get(url_for("ui.form_watch_checknow"), follow_redirects=True)

changedetectionio/tests/test_ignore.py

@@ -5,8 +5,9 @@ from flask import url_for
 from .util import live_server_setup, wait_for_all_checks
 from changedetectionio import html_tools
 from . util import  extract_UUID_from_client
+import os
 
-def set_original_ignore_response():
+def set_original_ignore_response(datastore_path):
     test_return_data = """<html>
        <body>
      Some initial text<br>
@@ -19,13 +20,13 @@ def set_original_ignore_response():
 
     """
 
-    with open("test-datastore/endpoint-content.txt", "w") as f:
+    with open(os.path.join(datastore_path, "endpoint-content.txt"), "w") as f:
         f.write(test_return_data)
 
 
-def test_ignore(client, live_server, measure_memory_usage):
+def test_ignore(client, live_server, measure_memory_usage, datastore_path):
    #  live_server_setup(live_server) # Setup on conftest per function
-    set_original_ignore_response()
+    set_original_ignore_response(datastore_path)
     test_url = url_for('test_endpoint', _external=True)
     uuid = client.application.config.get('DATASTORE').add_watch(url=test_url)
     client.get(url_for("ui.form_watch_checknow"), follow_redirects=True)
@@ -55,9 +56,9 @@ def test_ignore(client, live_server, measure_memory_usage):
     assert b'csrftoken' in res.data
 
 
-def test_strip_ignore_lines(client, live_server, measure_memory_usage):
+def test_strip_ignore_lines(client, live_server, measure_memory_usage, datastore_path):
    #  live_server_setup(live_server) # Setup on conftest per function
-    set_original_ignore_response()
+    set_original_ignore_response(datastore_path)
 
 
     # Goto the settings page, add our ignore text

changedetectionio/tests/test_ignore_text.py

@@ -4,6 +4,7 @@ import time
 from flask import url_for
 from .util import live_server_setup, wait_for_all_checks, delete_all_watches
 from changedetectionio import html_tools
+import os
 
 
 
@@ -31,7 +32,7 @@ def test_strip_text_func():
     stripped_content = html_tools.strip_ignore_text(test_content, ignore)
     assert stripped_content == "Some initial text\n\nWhich is across multiple lines\n\n\n\nSo let's see what happens."
 
-def set_original_ignore_response(ver_stamp="123"):
+def set_original_ignore_response(datastore_path, ver_stamp="123"):
     test_return_data = f"""<html>
        <body>
      Some initial text<br>
@@ -44,11 +45,11 @@ def set_original_ignore_response(ver_stamp="123"):
 
     """
 
-    with open("test-datastore/endpoint-content.txt", "w") as f:
+    with open(os.path.join(datastore_path, "endpoint-content.txt"), "w") as f:
         f.write(test_return_data)
 
 
-def set_modified_original_ignore_response(ver_stamp="123"):
+def set_modified_original_ignore_response(datastore_path, ver_stamp="123"):
     test_return_data = f"""<html>
        <body>
      Some NEW nice initial text<br>
@@ -63,12 +64,12 @@ def set_modified_original_ignore_response(ver_stamp="123"):
 
     """
 
-    with open("test-datastore/endpoint-content.txt", "w") as f:
+    with open(os.path.join(datastore_path, "endpoint-content.txt"), "w") as f:
         f.write(test_return_data)
 
 
 # Is the same but includes ZZZZZ, 'ZZZZZ' is the last line in ignore_text
-def set_modified_ignore_response(ver_stamp="123"):
+def set_modified_ignore_response(datastore_path, ver_stamp="123"):
     test_return_data = f"""<html>
        <body>
      Some initial text<br>
@@ -82,17 +83,17 @@ def set_modified_ignore_response(ver_stamp="123"):
 
     """
 
-    with open("test-datastore/endpoint-content.txt", "w") as f:
+    with open(os.path.join(datastore_path, "endpoint-content.txt"), "w") as f:
         f.write(test_return_data)
 
 
 # Ignore text now just removes it entirely, is a LOT more simpler code this way
 
-def test_check_ignore_text_functionality(client, live_server, measure_memory_usage):
+def test_check_ignore_text_functionality(client, live_server, measure_memory_usage, datastore_path):
 
     # Use a mix of case in ZzZ to prove it works case-insensitive.
     ignore_text = "XXXXX\r\nYYYYY\r\nzZzZZ\r\nnew ignore stuff"
-    set_original_ignore_response()
+    set_original_ignore_response(datastore_path=datastore_path)
 
 
     # Add our URL to the import page
@@ -130,7 +131,7 @@ def test_check_ignore_text_functionality(client, live_server, measure_memory_usa
     assert b'/test-endpoint' in res.data
 
     #  Make a change
-    set_modified_ignore_response()
+    set_modified_ignore_response(datastore_path=datastore_path)
 
     # Trigger a check
     client.get(url_for("ui.form_watch_checknow"), follow_redirects=True)
@@ -145,7 +146,7 @@ def test_check_ignore_text_functionality(client, live_server, measure_memory_usa
 
 
     # Just to be sure.. set a regular modified change..
-    set_modified_original_ignore_response()
+    set_modified_original_ignore_response(datastore_path=datastore_path)
     client.get(url_for("ui.form_watch_checknow"), follow_redirects=True)
     wait_for_all_checks(client)
 
@@ -162,10 +163,10 @@ def test_check_ignore_text_functionality(client, live_server, measure_memory_usa
     delete_all_watches(client)
 
 # When adding some ignore text, it should not trigger a change, even if something else on that line changes
-def _run_test_global_ignore(client, as_source=False, extra_ignore=""):
+def _run_test_global_ignore(client, datastore_path, as_source=False, extra_ignore=""):
     ignore_text = "XXXXX\r\nYYYYY\r\nZZZZZ\r\n"+extra_ignore
 
-    set_original_ignore_response()
+    set_original_ignore_response(datastore_path=datastore_path)
 
     # Goto the settings page, add our ignore text
     res = client.post(
@@ -222,7 +223,7 @@ def _run_test_global_ignore(client, as_source=False, extra_ignore=""):
     # Make a change which includes the ignore text, it should be ignored and no 'change' triggered
     # It adds text with "ZZZZzzzz" and "ZZZZ" is in the ignore list
     # And tweaks the ver_stamp which should be picked up by global regex ignore
-    set_modified_ignore_response(ver_stamp=time.time())
+    set_modified_ignore_response(ver_stamp=time.time(), datastore_path=datastore_path)
 
     # Trigger a check
     client.get(url_for("ui.form_watch_checknow"), follow_redirects=True)
@@ -236,7 +237,7 @@ def _run_test_global_ignore(client, as_source=False, extra_ignore=""):
     assert b'/test-endpoint' in res.data
 
     # Just to be sure.. set a regular modified change that will trigger it
-    set_modified_original_ignore_response()
+    set_modified_original_ignore_response(datastore_path=datastore_path)
     client.get(url_for("ui.form_watch_checknow"), follow_redirects=True)
     wait_for_all_checks(client)
     res = client.get(url_for("watchlist.index"))
@@ -244,10 +245,10 @@ def _run_test_global_ignore(client, as_source=False, extra_ignore=""):
 
     delete_all_watches(client)
 
-def test_check_global_ignore_text_functionality(client, live_server, measure_memory_usage):
+def test_check_global_ignore_text_functionality(client, live_server, measure_memory_usage, datastore_path):
     
-    _run_test_global_ignore(client, as_source=False)
+    _run_test_global_ignore(client, as_source=False, datastore_path=datastore_path)
 
-def test_check_global_ignore_text_functionality_as_source(client, live_server, measure_memory_usage):
+def test_check_global_ignore_text_functionality_as_source(client, live_server, measure_memory_usage, datastore_path):
     
-    _run_test_global_ignore(client, as_source=True, extra_ignore='/\?v=\d/')
+    _run_test_global_ignore(client, as_source=True, extra_ignore='/\?v=\d/', datastore_path=datastore_path)

changedetectionio/tests/test_ignorehyperlinks.py

@@ -4,9 +4,10 @@
 import time
 from flask import url_for
 from .util import live_server_setup, wait_for_all_checks, delete_all_watches
+import os
 
 
-def set_original_ignore_response():
+def set_original_ignore_response(datastore_path):
     test_return_data = """<html>
        <body>
      Some initial text<br>
@@ -17,13 +18,13 @@ def set_original_ignore_response():
      </html>
     """
 
-    with open("test-datastore/endpoint-content.txt", "w") as f:
+    with open(os.path.join(datastore_path, "endpoint-content.txt"), "w") as f:
         f.write(test_return_data)
 
 
-# Should be the same as set_original_ignore_response() but with a different
+# Should be the same as set_original_ignore_response(datastore_path=datastore_path) but with a different
 # link
-def set_modified_ignore_response():
+def set_modified_ignore_response(datastore_path):
     test_return_data = """<html>
        <body>
      Some initial text<br>
@@ -34,10 +35,10 @@ def set_modified_ignore_response():
      </html>
     """
 
-    with open("test-datastore/endpoint-content.txt", "w") as f:
+    with open(os.path.join(datastore_path, "endpoint-content.txt"), "w") as f:
         f.write(test_return_data)
 
-def test_render_anchor_tag_content_true(client, live_server, measure_memory_usage):
+def test_render_anchor_tag_content_true(client, live_server, measure_memory_usage, datastore_path):
     """Testing that the link changes are detected when
     render_anchor_tag_content setting is set to true"""
     sleep_time_for_fetch_thread = 3
@@ -46,7 +47,7 @@ def test_render_anchor_tag_content_true(client, live_server, measure_memory_usag
     time.sleep(1)
 
     # set original html text
-    set_original_ignore_response()
+    set_original_ignore_response(datastore_path=datastore_path)
 
     # Goto the settings page, choose to ignore links (dont select/send "application-render_anchor_tag_content")
     res = client.post(
@@ -72,7 +73,7 @@ def test_render_anchor_tag_content_true(client, live_server, measure_memory_usag
     client.get(url_for("ui.form_watch_checknow"), follow_redirects=True)
 
     # set a new html text with a modified link
-    set_modified_ignore_response()
+    set_modified_ignore_response(datastore_path=datastore_path)
     wait_for_all_checks(client)
 
     # Trigger a check

changedetectionio/tests/test_ignorestatuscode.py

@@ -3,12 +3,13 @@
 import time
 from flask import url_for
 from .util import live_server_setup, wait_for_all_checks
+import os
 
 
 
 
 
-def set_original_response():
+def set_original_response(datastore_path):
     test_return_data = """<html>
        <body>
      Some initial text<br>
@@ -19,11 +20,11 @@ def set_original_response():
      </html>
     """
 
-    with open("test-datastore/endpoint-content.txt", "w") as f:
+    with open(os.path.join(datastore_path, "endpoint-content.txt"), "w") as f:
         f.write(test_return_data)
 
 
-def set_some_changed_response():
+def set_some_changed_response(datastore_path):
     test_return_data = """<html>
        <body>
      Some initial text<br>
@@ -34,17 +35,17 @@ def set_some_changed_response():
      </html>
     """
 
-    with open("test-datastore/endpoint-content.txt", "w") as f:
+    with open(os.path.join(datastore_path, "endpoint-content.txt"), "w") as f:
         f.write(test_return_data)
 
 
-def test_normal_page_check_works_with_ignore_status_code(client, live_server, measure_memory_usage):
+def test_normal_page_check_works_with_ignore_status_code(client, live_server, measure_memory_usage, datastore_path):
 
 
     # Give the endpoint time to spin up
     time.sleep(1)
 
-    set_original_response()
+    set_original_response(datastore_path=datastore_path)
 
     # Goto the settings page, add our ignore text
     res = client.post(
@@ -65,7 +66,7 @@ def test_normal_page_check_works_with_ignore_status_code(client, live_server, me
 
     wait_for_all_checks(client)
 
-    set_some_changed_response()
+    set_some_changed_response(datastore_path=datastore_path)
     wait_for_all_checks(client)
     # Trigger a check
     client.get(url_for("ui.form_watch_checknow"), follow_redirects=True)
@@ -80,10 +81,10 @@ def test_normal_page_check_works_with_ignore_status_code(client, live_server, me
 
 
 # Tests the whole stack works with staus codes ignored
-def test_403_page_check_works_with_ignore_status_code(client, live_server, measure_memory_usage):
+def test_403_page_check_works_with_ignore_status_code(client, live_server, measure_memory_usage, datastore_path):
     sleep_time_for_fetch_thread = 3
 
-    set_original_response()
+    set_original_response(datastore_path=datastore_path)
 
     # Give the endpoint time to spin up
     time.sleep(1)
@@ -109,7 +110,7 @@ def test_403_page_check_works_with_ignore_status_code(client, live_server, measu
     wait_for_all_checks(client)
 
     #  Make a change
-    set_some_changed_response()
+    set_some_changed_response(datastore_path=datastore_path)
 
     # Trigger a check
     client.get(url_for("ui.form_watch_checknow"), follow_redirects=True)

changedetectionio/tests/test_ignorewhitespace.py

@@ -3,12 +3,13 @@
 import time
 from flask import url_for
 from . util import live_server_setup
+import os
 
 
 
 
-# Should be the same as set_original_ignore_response() but with a little more whitespacing
-def set_original_ignore_response_but_with_whitespace():
+# Should be the same as set_original_ignore_response(datastore_path=datastore_path) but with a little more whitespacing
+def set_original_ignore_response_but_with_whitespace(datastore_path):
     test_return_data = """<html>
        <body>
      Some initial text<br>
@@ -26,11 +27,11 @@ def set_original_ignore_response_but_with_whitespace():
      </html>
 
     """
-    with open("test-datastore/endpoint-content.txt", "w") as f:
+    with open(os.path.join(datastore_path, "endpoint-content.txt"), "w") as f:
         f.write(test_return_data)
 
 
-def set_original_ignore_response():
+def set_original_ignore_response(datastore_path):
     test_return_data = """<html>
        <body>
      Some initial text<br>
@@ -42,19 +43,19 @@ def set_original_ignore_response():
 
     """
 
-    with open("test-datastore/endpoint-content.txt", "w") as f:
+    with open(os.path.join(datastore_path, "endpoint-content.txt"), "w") as f:
         f.write(test_return_data)
 
 
 
 # If there was only a change in the whitespacing, then we shouldnt have a change detected
-def test_check_ignore_whitespace(client, live_server, measure_memory_usage):
+def test_check_ignore_whitespace(client, live_server, measure_memory_usage, datastore_path):
     sleep_time_for_fetch_thread = 3
 
     # Give the endpoint time to spin up
     time.sleep(1)
 
-    set_original_ignore_response()
+    set_original_ignore_response(datastore_path=datastore_path)
 
     # Goto the settings page, add our ignore text
     res = client.post(
@@ -77,7 +78,7 @@ def test_check_ignore_whitespace(client, live_server, measure_memory_usage):
     # Trigger a check
     client.get(url_for("ui.form_watch_checknow"), follow_redirects=True)
 
-    set_original_ignore_response_but_with_whitespace()
+    set_original_ignore_response_but_with_whitespace(datastore_path)
     time.sleep(sleep_time_for_fetch_thread)
     # Trigger a check
     client.get(url_for("ui.form_watch_checknow"), follow_redirects=True)

changedetectionio/tests/test_import.py

@@ -8,10 +8,10 @@ from flask import url_for
 from .util import live_server_setup, wait_for_all_checks, delete_all_watches
 
 
-# def test_setup(client, live_server, measure_memory_usage):
+# def test_setup(client, live_server, measure_memory_usage, datastore_path):
    #  live_server_setup(live_server) # Setup on conftest per function
 
-def test_import(client, live_server, measure_memory_usage):
+def test_import(client, live_server, measure_memory_usage, datastore_path):
     # Give the endpoint time to spin up
     wait_for_all_checks(client)
 
@@ -34,7 +34,7 @@ https://example.com tag1, other tag"""
     res = client.get( url_for("watchlist.index"))
     res = client.get( url_for("watchlist.index"))
 
-def xtest_import_skip_url(client, live_server, measure_memory_usage):
+def xtest_import_skip_url(client, live_server, measure_memory_usage, datastore_path):
 
 
     # Give the endpoint time to spin up
@@ -57,7 +57,7 @@ def xtest_import_skip_url(client, live_server, measure_memory_usage):
     # Clear flask alerts
     res = client.get( url_for("watchlist.index"))
 
-def test_import_distillio(client, live_server, measure_memory_usage):
+def test_import_distillio(client, live_server, measure_memory_usage, datastore_path):
 
     distill_data='''
 {
@@ -123,7 +123,7 @@ def test_import_distillio(client, live_server, measure_memory_usage):
     # Clear flask alerts
     res = client.get(url_for("watchlist.index"))
 
-def test_import_custom_xlsx(client, live_server, measure_memory_usage):
+def test_import_custom_xlsx(client, live_server, measure_memory_usage, datastore_path):
     """Test can upload a excel spreadsheet and the watches are created correctly"""
 
     
@@ -171,7 +171,7 @@ def test_import_custom_xlsx(client, live_server, measure_memory_usage):
 
     delete_all_watches(client)
 
-def test_import_watchete_xlsx(client, live_server, measure_memory_usage):
+def test_import_watchete_xlsx(client, live_server, measure_memory_usage, datastore_path):
     """Test can upload a excel spreadsheet and the watches are created correctly"""
 
     

changedetectionio/tests/test_jinja2.py

@@ -7,11 +7,11 @@ from .util import live_server_setup, wait_for_all_checks
 from ..jinja2_custom import render
 
 
-# def test_setup(client, live_server, measure_memory_usage):
+# def test_setup(client, live_server, measure_memory_usage, datastore_path):
    # #  live_server_setup(live_server) # Setup on conftest per function
 
 # If there was only a change in the whitespacing, then we shouldnt have a change detected
-def test_jinja2_in_url_query(client, live_server, measure_memory_usage):
+def test_jinja2_in_url_query(client, live_server, measure_memory_usage, datastore_path):
     
 
     # Add our URL to the import page
@@ -36,7 +36,7 @@ def test_jinja2_in_url_query(client, live_server, measure_memory_usage):
     assert b'date=2' in res.data
 
 # Test for issue #1493 - jinja2-time offset functionality
-def test_jinja2_time_offset_in_url_query(client, live_server, measure_memory_usage):
+def test_jinja2_time_offset_in_url_query(client, live_server, measure_memory_usage, datastore_path):
     """Test that jinja2 time offset expressions work in watch URLs (issue #1493)."""
 
     # Add our URL to the import page with time offset expression
@@ -64,29 +64,21 @@ def test_jinja2_time_offset_in_url_query(client, live_server, measure_memory_usa
     # Should not have template error
     assert b'Invalid template' not in res.data
 
-# https://techtonics.medium.com/secure-templating-with-jinja2-understanding-ssti-and-jinja2-sandbox-environment-b956edd60456
-def test_jinja2_security_url_query(client, live_server, measure_memory_usage):
-    
 
+# https://techtonics.medium.com/secure-templating-with-jinja2-understanding-ssti-and-jinja2-sandbox-environment-b956edd60456
+def test_jinja2_security_url_query(client, live_server, measure_memory_usage, datastore_path):
     # Add our URL to the import page
     test_url = url_for('test_return_query', _external=True)
 
-    # because url_for() will URL-encode the var, but we dont here
-    full_url = "{}?{}".format(test_url,
-                              "date={{ ''.__class__.__mro__[1].__subclasses__()}}", )
+    full_url = test_url + "?date={{ ''.__class__.__mro__[1].__subclasses__()}}"
+
     res = client.post(
         url_for("ui.ui_views.form_quick_watch_add"),
         data={"url": full_url, "tags": "test"},
         follow_redirects=True
     )
-    assert b"Watch added" in res.data
-    wait_for_all_checks(client)
+    assert b"Watch added" not in res.data
 
-    # It should report nothing found (no new 'has-unread-changes' class)
-    res = client.get(url_for("watchlist.index"))
-    assert b'is invalid and cannot be used' in res.data
-    # Some of the spewed output from the subclasses
-    assert b'dict_values' not in res.data
 
 def test_timezone(mocker):
     """Verify that timezone is parsed."""
@@ -169,4 +161,161 @@ def test_default_timezone_subtraction(environment):
 
     finalRender = render("{% now '' - 'minutes=11' %}")
 
-    assert finalRender == "Wed, 09 Dec 2015 23:22:01"
+    assert finalRender == "Wed, 09 Dec 2015 23:22:01"
+
+def test_regex_replace_basic():
+    """Test basic regex_replace functionality."""
+
+    # Simple word replacement
+    finalRender = render("{{ 'hello world' | regex_replace('world', 'universe') }}")
+    assert finalRender == "hello universe"
+
+def test_regex_replace_with_groups():
+    """Test regex_replace with capture groups (issue #3501 use case)."""
+
+    # Transform HTML table data as described in the issue
+    template = "{{ '<td>thing</td><td>other</td>' | regex_replace('<td>([^<]+)</td><td>([^<]+)</td>', 'ThingLabel: \\\\1\\nOtherLabel: \\\\2') }}"
+    finalRender = render(template)
+    assert "ThingLabel: thing" in finalRender
+    assert "OtherLabel: other" in finalRender
+
+def test_regex_replace_multiple_matches():
+    """Test regex_replace replacing multiple occurrences."""
+
+    finalRender = render("{{ 'foo bar foo baz' | regex_replace('foo', 'qux') }}")
+    assert finalRender == "qux bar qux baz"
+
+def test_regex_replace_count_parameter():
+    """Test regex_replace with count parameter to limit replacements."""
+
+    finalRender = render("{{ 'foo bar foo baz' | regex_replace('foo', 'qux', 1) }}")
+    assert finalRender == "qux bar foo baz"
+
+def test_regex_replace_empty_replacement():
+    """Test regex_replace with empty replacement (removal)."""
+
+    finalRender = render("{{ 'hello world 123' | regex_replace('[0-9]+', '') }}")
+    assert finalRender == "hello world "
+
+def test_regex_replace_no_match():
+    """Test regex_replace when pattern doesn't match."""
+
+    finalRender = render("{{ 'hello world' | regex_replace('xyz', 'abc') }}")
+    assert finalRender == "hello world"
+
+def test_regex_replace_invalid_regex():
+    """Test regex_replace with invalid regex pattern returns original value."""
+
+    # Invalid regex (unmatched parenthesis)
+    finalRender = render("{{ 'hello world' | regex_replace('(invalid', 'replacement') }}")
+    assert finalRender == "hello world"
+
+def test_regex_replace_special_characters():
+    """Test regex_replace with special regex characters."""
+
+    finalRender = render("{{ 'Price: $50.00' | regex_replace('\\\\$([0-9.]+)', 'USD \\\\1') }}")
+    assert finalRender == "Price: USD 50.00"
+
+def test_regex_replace_multiline():
+    """Test regex_replace on multiline text."""
+
+    template = "{{ 'line1\\nline2\\nline3' | regex_replace('^line', 'row') }}"
+    finalRender = render(template)
+    # By default re.sub doesn't use MULTILINE flag, so only first line matches with ^
+    assert finalRender == "row1\nline2\nline3"
+
+def test_regex_replace_with_notification_context():
+    """Test regex_replace with notification diff variable."""
+
+    # Simulate how it would be used in notifications with diff variable
+    from changedetectionio.notification_service import NotificationContextData
+
+    context = NotificationContextData()
+    context['diff'] = '<td>value1</td><td>value2</td>'
+
+    template = "{{ diff | regex_replace('<td>([^<]+)</td>', '\\\\1 ') }}"
+
+    from changedetectionio.jinja2_custom import create_jinja_env
+    from jinja2 import BaseLoader
+
+    jinja2_env = create_jinja_env(loader=BaseLoader)
+    jinja2_env.globals.update(context)
+    finalRender = jinja2_env.from_string(template).render()
+
+    assert "value1 value2 " in finalRender
+
+def test_regex_replace_security_large_input():
+    """Test regex_replace handles large input safely."""
+
+    # Create a large input string (over 10MB)
+    large_input = "x" * (1024 * 1024 * 10 + 1000)
+    template = "{{ large_input | regex_replace('x', 'y') }}"
+
+    from changedetectionio.jinja2_custom import create_jinja_env
+    from jinja2 import BaseLoader
+
+    jinja2_env = create_jinja_env(loader=BaseLoader)
+    jinja2_env.globals['large_input'] = large_input
+    finalRender = jinja2_env.from_string(template).render()
+
+    # Should be truncated to 10MB
+    assert len(finalRender) == 1024 * 1024 * 10
+
+def test_regex_replace_security_long_pattern():
+    """Test regex_replace rejects very long patterns."""
+
+    # Pattern longer than 500 chars should be rejected
+    long_pattern = "a" * 501
+    finalRender = render("{{ 'test' | regex_replace('" + long_pattern + "', 'replacement') }}")
+
+    # Should return original value when pattern is too long
+    assert finalRender == "test"
+
+def test_regex_replace_security_dangerous_pattern():
+    """Test regex_replace detects and rejects dangerous nested quantifiers."""
+
+    # Patterns that could cause catastrophic backtracking
+    dangerous_patterns = [
+        "(a+)+",
+        "(a*)+",
+        "(a+)*",
+        "(a*)*",
+    ]
+
+    for dangerous in dangerous_patterns:
+        # Create a template with the dangerous pattern
+        # Using single quotes to avoid escaping issues
+        from changedetectionio.jinja2_custom import create_jinja_env
+        from jinja2 import BaseLoader
+
+        jinja2_env = create_jinja_env(loader=BaseLoader)
+        jinja2_env.globals['pattern'] = dangerous
+        template = "{{ 'aaaaaaaaaa' | regex_replace(pattern, 'x') }}"
+        finalRender = jinja2_env.from_string(template).render()
+
+        # Should return original value when dangerous pattern is detected
+        assert finalRender == "aaaaaaaaaa"
+
+def test_regex_replace_security_timeout_protection():
+    """Test that regex_replace has timeout protection (if SIGALRM available)."""
+    import signal
+
+    # Only test on systems that support SIGALRM
+    if not hasattr(signal, 'SIGALRM'):
+        # Skip test on Windows and other systems without SIGALRM
+        return
+
+    # This pattern is known to cause exponential backtracking on certain inputs
+    # but should be caught by our dangerous pattern detector
+    # We're mainly testing that the timeout mechanism works
+
+    from changedetectionio.jinja2_custom import regex_replace
+
+    # Create input that could trigger slow regex
+    test_input = "a" * 50 + "b"
+
+    # This shouldn't take long due to our protections
+    result = regex_replace(test_input, "a+b", "x")
+
+    # Should complete and return a result
+    assert result is not None

changedetectionio/tests/test_jsonpath_jq_selector.py

@@ -2,9 +2,11 @@
 # coding=utf-8
 
 import time
-from flask import url_for, escape
+from flask import url_for
+from markupsafe import escape
 from . util import live_server_setup, wait_for_all_checks, delete_all_watches
 import pytest
+import os
 jq_support = True
 
 try:
@@ -91,7 +93,7 @@ def test_unittest_inline_extract_body():
     text = html_tools.extract_json_as_string(content, "json:$.testKey")
     assert text == '42'
 
-def set_original_ext_response():
+def set_original_ext_response(datastore_path):
     data = """
         [
         {
@@ -108,11 +110,11 @@ def set_original_ext_response():
     ]
         """
 
-    with open("test-datastore/endpoint-content.txt", "w") as f:
+    with open(os.path.join(datastore_path, "endpoint-content.txt"), "w") as f:
         f.write(data)
     return None
 
-def set_modified_ext_response():
+def set_modified_ext_response(datastore_path):
     # This should get reformatted
     data = """ [ { "isPriceLowered": false,  "status": "Sold",  "statusOrig": "sold" }, {
         "_id": "5e7b3e1fb3262d306323ff1e",
@@ -123,11 +125,11 @@ def set_modified_ext_response():
 ]
     """
 
-    with open("test-datastore/endpoint-content.txt", "w") as f:
+    with open(os.path.join(datastore_path, "endpoint-content.txt"), "w") as f:
         f.write(data)
     return None
 
-def set_original_response():
+def set_original_response(datastore_path):
     test_return_data = """
     {
       "employees": [
@@ -148,12 +150,12 @@ def set_original_response():
       "available": true
     }
     """
-    with open("test-datastore/endpoint-content.txt", "w") as f:
+    with open(os.path.join(datastore_path, "endpoint-content.txt"), "w") as f:
         f.write(test_return_data)
     return None
 
 
-def set_json_response_with_html():
+def set_json_response_with_html(datastore_path):
     test_return_data = """
     {
       "test": [
@@ -163,11 +165,11 @@ def set_json_response_with_html():
       ]
     }
     """
-    with open("test-datastore/endpoint-content.txt", "w") as f:
+    with open(os.path.join(datastore_path, "endpoint-content.txt"), "w") as f:
         f.write(test_return_data)
     return None
 
-def set_modified_response():
+def set_modified_response(datastore_path):
     test_return_data = """
     {
       "employees": [
@@ -189,15 +191,15 @@ def set_modified_response():
     }
         """
 
-    with open("test-datastore/endpoint-content.txt", "w") as f:
+    with open(os.path.join(datastore_path, "endpoint-content.txt"), "w") as f:
         f.write(test_return_data)
 
     return None
 
-def test_check_json_without_filter(client, live_server, measure_memory_usage):
+def test_check_json_without_filter(client, live_server, measure_memory_usage, datastore_path):
     # Request a JSON document from a application/json source containing HTML
     # and be sure it doesn't get chewed up by instriptis
-    set_json_response_with_html()
+    set_json_response_with_html(datastore_path=datastore_path)
 
     # Add our URL to the import page
     test_url = url_for('test_endpoint', content_type="application/json", _external=True)
@@ -218,8 +220,8 @@ def test_check_json_without_filter(client, live_server, measure_memory_usage):
 
     delete_all_watches(client)
 
-def check_json_filter(json_filter, client, live_server):
-    set_original_response()
+def check_json_filter(json_filter, client, live_server, datastore_path):
+    set_original_response(datastore_path=datastore_path)
 
 
     # Add our URL to the import page
@@ -239,7 +241,7 @@ def check_json_filter(json_filter, client, live_server):
     # Give the thread time to pick it up
     wait_for_all_checks(client)
     #  Make a change
-    set_modified_response()
+    set_modified_response(datastore_path=datastore_path)
 
     # Trigger a check
     client.get(url_for("ui.form_watch_checknow"), follow_redirects=True)
@@ -259,19 +261,19 @@ def check_json_filter(json_filter, client, live_server):
 
     delete_all_watches(client)
 
-def test_check_jsonpath_filter(client, live_server, measure_memory_usage):
-    check_json_filter('json:boss.name', client, live_server)
+def test_check_jsonpath_filter(client, live_server, measure_memory_usage, datastore_path):
+    check_json_filter('json:boss.name', client, live_server, datastore_path=datastore_path)
 
-def test_check_jq_filter(client, live_server, measure_memory_usage):
+def test_check_jq_filter(client, live_server, measure_memory_usage, datastore_path):
     if jq_support:
-        check_json_filter('jq:.boss.name', client, live_server)
+        check_json_filter('jq:.boss.name', client, live_server, datastore_path=datastore_path)
 
-def test_check_jqraw_filter(client, live_server, measure_memory_usage):
+def test_check_jqraw_filter(client, live_server, measure_memory_usage, datastore_path):
     if jq_support:
-        check_json_filter('jqraw:.boss.name', client, live_server)
+        check_json_filter('jqraw:.boss.name', client, live_server, datastore_path=datastore_path)
 
-def check_json_filter_bool_val(json_filter, client, live_server):
-    set_original_response()
+def check_json_filter_bool_val(json_filter, client, live_server, datastore_path):
+    set_original_response(datastore_path=datastore_path)
 
     test_url = url_for('test_endpoint', content_type="application/json", _external=True)
 
@@ -280,7 +282,7 @@ def check_json_filter_bool_val(json_filter, client, live_server):
     wait_for_all_checks(client)
 
     #  Make a change
-    set_modified_response()
+    set_modified_response(datastore_path=datastore_path)
 
     # Trigger a check
     client.get(url_for("ui.form_watch_checknow"), follow_redirects=True)
@@ -293,24 +295,24 @@ def check_json_filter_bool_val(json_filter, client, live_server):
 
     delete_all_watches(client)
 
-def test_check_jsonpath_filter_bool_val(client, live_server, measure_memory_usage):
-    check_json_filter_bool_val("json:$['available']", client, live_server)
+def test_check_jsonpath_filter_bool_val(client, live_server, measure_memory_usage, datastore_path):
+    check_json_filter_bool_val("json:$['available']", client, live_server, datastore_path=datastore_path)
 
-def test_check_jq_filter_bool_val(client, live_server, measure_memory_usage):
+def test_check_jq_filter_bool_val(client, live_server, measure_memory_usage, datastore_path):
     if jq_support:
-        check_json_filter_bool_val("jq:.available", client, live_server)
+        check_json_filter_bool_val("jq:.available", client, live_server, datastore_path=datastore_path)
 
-def test_check_jqraw_filter_bool_val(client, live_server, measure_memory_usage):
+def test_check_jqraw_filter_bool_val(client, live_server, measure_memory_usage, datastore_path):
     if jq_support:
-        check_json_filter_bool_val("jq:.available", client, live_server)
+        check_json_filter_bool_val("jq:.available", client, live_server, datastore_path=datastore_path)
 
 # Re #265 - Extended JSON selector test
 # Stuff to consider here
 # - Selector should be allowed to return empty when it doesnt match (people might wait for some condition)
 # - The 'diff' tab could show the old and new content
 # - Form should let us enter a selector that doesnt (yet) match anything
-def check_json_ext_filter(json_filter, client, live_server):
-    set_original_ext_response()
+def check_json_ext_filter(json_filter, client, live_server, datastore_path):
+    set_original_ext_response(datastore_path=datastore_path)
 
     # Add our URL to the import page
     test_url = url_for('test_endpoint', content_type="application/json", _external=True)
@@ -342,7 +344,7 @@ def check_json_ext_filter(json_filter, client, live_server):
     # Give the thread time to pick it up
     wait_for_all_checks(client)
     #  Make a change
-    set_modified_ext_response()
+    set_modified_ext_response(datastore_path=datastore_path)
 
     # Trigger a check
     client.get(url_for("ui.form_watch_checknow"), follow_redirects=True)
@@ -375,10 +377,10 @@ def check_json_ext_filter(json_filter, client, live_server):
 
     delete_all_watches(client)
 
-def test_ignore_json_order(client, live_server, measure_memory_usage):
+def test_ignore_json_order(client, live_server, measure_memory_usage, datastore_path):
     # A change in order shouldn't trigger a notification
 
-    with open("test-datastore/endpoint-content.txt", "w") as f:
+    with open(os.path.join(datastore_path, "endpoint-content.txt"), "w") as f:
         f.write('{"hello" : 123, "world": 123}')
 
 
@@ -389,7 +391,7 @@ def test_ignore_json_order(client, live_server, measure_memory_usage):
 
     wait_for_all_checks(client)
 
-    with open("test-datastore/endpoint-content.txt", "w") as f:
+    with open(os.path.join(datastore_path, "endpoint-content.txt"), "w") as f:
         f.write('{"world" : 123, "hello": 123}')
 
     # Trigger a check
@@ -400,7 +402,7 @@ def test_ignore_json_order(client, live_server, measure_memory_usage):
     assert b'has-unread-changes' not in res.data
 
     # Just to be sure it still works
-    with open("test-datastore/endpoint-content.txt", "w") as f:
+    with open(os.path.join(datastore_path, "endpoint-content.txt"), "w") as f:
         f.write('{"world" : 123, "hello": 124}')
 
     # Trigger a check
@@ -412,10 +414,10 @@ def test_ignore_json_order(client, live_server, measure_memory_usage):
 
     delete_all_watches(client)
 
-def test_correct_header_detect(client, live_server, measure_memory_usage):
+def test_correct_header_detect(client, live_server, measure_memory_usage, datastore_path):
     # Like in https://github.com/dgtlmoon/changedetection.io/pull/1593
     # Specify extra html that JSON is sometimes wrapped in - when using SockpuppetBrowser / Puppeteer / Playwrightetc
-    with open("test-datastore/endpoint-content.txt", "w") as f:
+    with open(os.path.join(datastore_path, "endpoint-content.txt"), "w") as f:
         f.write('<html><body>{ "world": 123, "hello" : 123}')
 
     # Add our URL to the import page
@@ -449,18 +451,18 @@ def test_correct_header_detect(client, live_server, measure_memory_usage):
 
     delete_all_watches(client)
 
-def test_check_jsonpath_ext_filter(client, live_server, measure_memory_usage):
-    check_json_ext_filter('json:$[?(@.status==Sold)]', client, live_server)
+def test_check_jsonpath_ext_filter(client, live_server, measure_memory_usage, datastore_path):
+    check_json_ext_filter('json:$[?(@.status==Sold)]', client, live_server, datastore_path=datastore_path)
 
-def test_check_jq_ext_filter(client, live_server, measure_memory_usage):
+def test_check_jq_ext_filter(client, live_server, measure_memory_usage, datastore_path):
     if jq_support:
-        check_json_ext_filter('jq:.[] | select(.status | contains("Sold"))', client, live_server)
+        check_json_ext_filter('jq:.[] | select(.status | contains("Sold"))', client, live_server, datastore_path=datastore_path)
 
-def test_check_jqraw_ext_filter(client, live_server, measure_memory_usage):
+def test_check_jqraw_ext_filter(client, live_server, measure_memory_usage, datastore_path):
     if jq_support:
-        check_json_ext_filter('jq:.[] | select(.status | contains("Sold"))', client, live_server)
+        check_json_ext_filter('jq:.[] | select(.status | contains("Sold"))', client, live_server, datastore_path=datastore_path)
 
-def test_jsonpath_BOM_utf8(client, live_server, measure_memory_usage):
+def test_jsonpath_BOM_utf8(client, live_server, measure_memory_usage, datastore_path):
     from .. import html_tools
 
     # JSON string with BOM and correct double-quoted keys

changedetectionio/tests/test_live_preview.py

@@ -2,9 +2,10 @@
 
 from flask import url_for
 from changedetectionio.tests.util import live_server_setup, wait_for_all_checks, extract_UUID_from_client, delete_all_watches
+import os
 
 
-def set_response():
+def set_response(datastore_path):
 
     data = """<html>
        <body>Awesome, you made it<br>
@@ -15,12 +16,12 @@ something to trigger<br>
      </html>
     """
 
-    with open("test-datastore/endpoint-content.txt", "w") as f:
+    with open(os.path.join(datastore_path, "endpoint-content.txt"), "w") as f:
         f.write(data)
 
-def test_content_filter_live_preview(client, live_server, measure_memory_usage):
+def test_content_filter_live_preview(client, live_server, measure_memory_usage, datastore_path):
    #  live_server_setup(live_server) # Setup on conftest per function
-    set_response()
+    set_response(datastore_path=datastore_path)
 
     test_url = url_for('test_endpoint', _external=True)
 

changedetectionio/tests/test_nonrenderable_pages.py

@@ -3,9 +3,10 @@
 from flask import url_for
 from .util import set_original_response, set_modified_response, live_server_setup, wait_for_all_checks, delete_all_watches
 import time
+import os
 
 
-def set_nonrenderable_response():
+def set_nonrenderable_response(datastore_path):
     test_return_data = """<html>
     <head><title>modified head title</title></head>
     <!-- like when some angular app was broken and doesnt render or whatever -->
@@ -13,20 +14,20 @@ def set_nonrenderable_response():
      </body>
      </html>
     """
-    with open("test-datastore/endpoint-content.txt", "w") as f:
+    with open(os.path.join(datastore_path, "endpoint-content.txt"), "w") as f:
         f.write(test_return_data)
     time.sleep(1)
 
     return None
 
-def set_zero_byte_response():
-    with open("test-datastore/endpoint-content.txt", "w") as f:
+def set_zero_byte_response(datastore_path):
+    with open(os.path.join(datastore_path, "endpoint-content.txt"), "w") as f:
         f.write("")
     time.sleep(1)
     return None
 
-def test_check_basic_change_detection_functionality(client, live_server, measure_memory_usage):
-    set_original_response()
+def test_check_basic_change_detection_functionality(client, live_server, measure_memory_usage, datastore_path):
+    set_original_response(datastore_path=datastore_path)
    #  live_server_setup(live_server) # Setup on conftest per function
 
     # Add our URL to the import page
@@ -55,7 +56,7 @@ def test_check_basic_change_detection_functionality(client, live_server, measure
     )
 
     # this should not trigger a change, because no good text could be converted from the HTML
-    set_nonrenderable_response()
+    set_nonrenderable_response(datastore_path)
 
     client.get(url_for("ui.form_watch_checknow"), follow_redirects=True)
 
@@ -84,7 +85,7 @@ def test_check_basic_change_detection_functionality(client, live_server, measure
               'application-fetch_backend': "html_requests"},
         follow_redirects=True
     )
-    set_modified_response()
+    set_modified_response(datastore_path=datastore_path)
 
 
     client.get(url_for("ui.form_watch_checknow"), follow_redirects=True)
@@ -100,7 +101,7 @@ def test_check_basic_change_detection_functionality(client, live_server, measure
 
 
     # A totally zero byte (#2528) response should also not trigger an error
-    set_zero_byte_response()
+    set_zero_byte_response(datastore_path=datastore_path)
     client.get(url_for("ui.form_watch_checknow"), follow_redirects=True)
     wait_for_all_checks(client)
     # 2877

changedetectionio/tests/test_notification.py

@@ -13,17 +13,17 @@ import base64
 from changedetectionio.notification import (
     default_notification_body,
     default_notification_format,
-    default_notification_title,
-    valid_notification_formats,
+    default_notification_title, valid_notification_formats
 )
-
+from ..diff import HTML_CHANGED_STYLE
+from ..model import USE_SYSTEM_DEFAULT_NOTIFICATION_FORMAT_FOR_WATCH
 
 
 # Hard to just add more live server URLs when one test is already running (I think)
 # So we add our test here (was in a different file)
-def test_check_notification(client, live_server, measure_memory_usage):
+def test_check_notification(client, live_server, measure_memory_usage, datastore_path):
     
-    set_original_response()
+    set_original_response(datastore_path=datastore_path)
 
     # Re 360 - new install should have defaults set
     res = client.get(url_for("settings.settings_page"))
@@ -47,6 +47,14 @@ def test_check_notification(client, live_server, measure_memory_usage):
 
     assert b"Settings updated." in res.data
 
+    res = client.get(url_for("settings.settings_page"))
+    for k,v in valid_notification_formats.items():
+        if k == USE_SYSTEM_DEFAULT_NOTIFICATION_FORMAT_FOR_WATCH:
+            continue
+        assert f'value="{k}"'.encode() in res.data # Should be by key NOT value
+        assert f'value="{v}"'.encode() not in res.data # Should be by key NOT value
+
+
     # When test mode is in BASE_URL env mode, we should see this already configured
     env_base_url = os.getenv('BASE_URL', '').strip()
     if len(env_base_url):
@@ -75,8 +83,7 @@ def test_check_notification(client, live_server, measure_memory_usage):
 
 
     uuid = next(iter(live_server.app.config['DATASTORE'].data['watching']))
-    datastore = 'test-datastore'
-    with open(os.path.join(datastore, str(uuid), 'last-screenshot.png'), 'wb') as f:
+    with open(os.path.join(datastore_path, str(uuid), 'last-screenshot.png'), 'wb') as f:
         f.write(base64.b64decode(testimage_png))
 
     # Goto the edit page, add our ignore text
@@ -101,7 +108,7 @@ def test_check_notification(client, live_server, measure_memory_usage):
                                                    "Diff as Patch: {{diff_patch}}\n"
                                                    ":-)",
                               "notification_screenshot": True,
-                              "notification_format": "Text"}
+                              "notification_format": 'text'}
 
     notification_form_data.update({
         "url": test_url,
@@ -130,7 +137,7 @@ def test_check_notification(client, live_server, measure_memory_usage):
 
     ## Now recheck, and it should have sent the notification
     wait_for_all_checks(client)
-    set_modified_response()
+    set_modified_response(datastore_path=datastore_path)
 
     # Trigger a check
     client.get(url_for("ui.form_watch_checknow"), follow_redirects=True)
@@ -143,9 +150,9 @@ def test_check_notification(client, live_server, measure_memory_usage):
 
 
     # Verify what was sent as a notification, this file should exist
-    with open("test-datastore/notification.txt", "r") as f:
+    with open(os.path.join(datastore_path, "notification.txt"), "r") as f:
         notification_submission = f.read()
-    os.unlink("test-datastore/notification.txt")
+    os.unlink(os.path.join(datastore_path, "notification.txt"))
 
     # Did we see the URL that had a change, in the notification?
     # Diff was correctly executed
@@ -189,18 +196,18 @@ def test_check_notification(client, live_server, measure_memory_usage):
 
 
     # This should insert the {current_snapshot}
-    set_more_modified_response()
+    set_more_modified_response(datastore_path=datastore_path)
     client.get(url_for("ui.form_watch_checknow"), follow_redirects=True)
     time.sleep(3)
     # Verify what was sent as a notification, this file should exist
-    with open("test-datastore/notification.txt", "r") as f:
+    with open(os.path.join(datastore_path, "notification.txt"), "r") as f:
         notification_submission = f.read()
     assert "Ohh yeah awesome" in notification_submission
 
 
     # Prove that "content constantly being marked as Changed with no Updating causes notification" is not a thing
     # https://github.com/dgtlmoon/changedetection.io/discussions/192
-    os.unlink("test-datastore/notification.txt")
+    os.unlink(os.path.join(datastore_path, "notification.txt"))
 
     # Trigger a check
     client.get(url_for("ui.form_watch_checknow"), follow_redirects=True)
@@ -209,13 +216,13 @@ def test_check_notification(client, live_server, measure_memory_usage):
     wait_for_all_checks(client)
     client.get(url_for("ui.form_watch_checknow"), follow_redirects=True)
     wait_for_all_checks(client)
-    assert os.path.exists("test-datastore/notification.txt") == False
+    assert os.path.exists(os.path.join(datastore_path, "notification.txt")) == False
 
     res = client.get(url_for("settings.notification_logs"))
     # be sure we see it in the output log
     assert b'New ChangeDetection.io Notification - ' + test_url.encode('utf-8') in res.data
 
-    set_original_response()
+    set_original_response(datastore_path=datastore_path)
     res = client.post(
         url_for("ui.ui_edit.edit_page", uuid="first"),
         data={
@@ -235,7 +242,7 @@ def test_check_notification(client, live_server, measure_memory_usage):
     time.sleep(2)
 
     # Verify what was sent as a notification, this file should exist
-    with open("test-datastore/notification.txt", "r") as f:
+    with open(os.path.join(datastore_path, "notification.txt"), "r") as f:
         notification_submission = f.read()
     assert "fallback-title" in notification_submission
     assert "fallback-body" in notification_submission
@@ -246,7 +253,7 @@ def test_check_notification(client, live_server, measure_memory_usage):
         follow_redirects=True
     )
 
-def test_notification_validation(client, live_server, measure_memory_usage):
+def test_notification_validation(client, live_server, measure_memory_usage, datastore_path):
 
     time.sleep(1)
 
@@ -267,7 +274,7 @@ def test_notification_validation(client, live_server, measure_memory_usage):
 #        data={"notification_urls": 'json://localhost/foobar',
 #              "notification_title": "",
 #              "notification_body": "",
-#              "notification_format": "Text",
+#              "notification_format": 'text',
 #              "url": test_url,
 #              "tag": "my tag",
 #              "title": "my title",
@@ -284,7 +291,7 @@ def test_notification_validation(client, live_server, measure_memory_usage):
     )
 
 
-def test_notification_urls_jinja2_apprise_integration(client, live_server, measure_memory_usage):
+def test_notification_urls_jinja2_apprise_integration(client, live_server, measure_memory_usage, datastore_path):
 
     #
     # https://github.com/caronc/apprise/wiki/Notify_Custom_JSON#header-manipulation
@@ -306,14 +313,14 @@ def test_notification_urls_jinja2_apprise_integration(client, live_server, measu
     assert b'Settings updated' in res.data
 
 
-def test_notification_custom_endpoint_and_jinja2(client, live_server, measure_memory_usage):
+def test_notification_custom_endpoint_and_jinja2(client, live_server, measure_memory_usage, datastore_path):
     
 
     # test_endpoint - that sends the contents of a file
     # test_notification_endpoint - that takes a POST and writes it to file (test-datastore/notification.txt)
 
     # CUSTOM JSON BODY CHECK for POST://
-    set_original_response()
+    set_original_response(datastore_path=datastore_path)
     # https://github.com/caronc/apprise/wiki/Notify_Custom_JSON#header-manipulation
     test_notification_url = url_for('test_notification_endpoint', _external=True).replace('http://', 'post://')+"?status_code=204&watch_uuid={{ watch_uuid }}&xxx={{ watch_url }}&now={% now 'Europe/London', '%Y-%m-%d' %}&+custom-header=123&+second=hello+world%20%22space%22"
 
@@ -344,7 +351,7 @@ def test_notification_custom_endpoint_and_jinja2(client, live_server, measure_me
     watch_uuid = next(iter(live_server.app.config['DATASTORE'].data['watching']))
 
     wait_for_all_checks(client)
-    set_modified_response()
+    set_modified_response(datastore_path=datastore_path)
 
     client.get(url_for("ui.form_watch_checknow"), follow_redirects=True)
     wait_for_all_checks(client)
@@ -356,7 +363,7 @@ def test_notification_custom_endpoint_and_jinja2(client, live_server, measure_me
     res = client.get(url_for("watchlist.index"))
     assert b'notification-error' not in res.data
 
-    with open("test-datastore/notification.txt", 'r') as f:
+    with open(os.path.join(datastore_path, "notification.txt"), 'r') as f:
         x = f.read()
         j = json.loads(x)
         assert j['url'].startswith('http://localhost')
@@ -365,8 +372,8 @@ def test_notification_custom_endpoint_and_jinja2(client, live_server, measure_me
 
 
     # URL check, this will always be converted to lowercase
-    assert os.path.isfile("test-datastore/notification-url.txt")
-    with open("test-datastore/notification-url.txt", 'r') as f:
+    assert os.path.isfile(os.path.join(datastore_path, "notification-url.txt"))
+    with open(os.path.join(datastore_path, "notification-url.txt"), 'r') as f:
         notification_url = f.read()
         assert 'xxx=http' in notification_url
         # apprise style headers should be stripped
@@ -377,18 +384,18 @@ def test_notification_custom_endpoint_and_jinja2(client, live_server, measure_me
         assert f'watch_uuid={watch_uuid}' in notification_url
 
 
-    with open("test-datastore/notification-headers.txt", 'r') as f:
+    with open(os.path.join(datastore_path, "notification-headers.txt"), 'r') as f:
         notification_headers = f.read()
         assert 'custom-header: 123' in notification_headers.lower()
         assert 'second: hello world "space"' in notification_headers.lower()
 
 
-    # Should always be automatically detected as JSON content type even when we set it as 'Text' (default)
-    assert os.path.isfile("test-datastore/notification-content-type.txt")
-    with open("test-datastore/notification-content-type.txt", 'r') as f:
+    # Should always be automatically detected as JSON content type even when we set it as 'Plain Text' (default)
+    assert os.path.isfile(os.path.join(datastore_path, "notification-content-type.txt"))
+    with open(os.path.join(datastore_path, "notification-content-type.txt"), 'r') as f:
         assert 'application/json' in f.read()
 
-    os.unlink("test-datastore/notification-url.txt")
+    os.unlink(os.path.join(datastore_path, "notification-url.txt"))
 
     client.get(
         url_for("ui.form_delete", uuid="all"),
@@ -397,12 +404,12 @@ def test_notification_custom_endpoint_and_jinja2(client, live_server, measure_me
 
 
 #2510
-def test_global_send_test_notification(client, live_server, measure_memory_usage):
+def test_global_send_test_notification(client, live_server, measure_memory_usage, datastore_path):
 
     
-    set_original_response()
-    if os.path.isfile("test-datastore/notification.txt"):
-        os.unlink("test-datastore/notification.txt") \
+    set_original_response(datastore_path=datastore_path)
+    if os.path.isfile(os.path.join(datastore_path, "notification.txt")):
+        os.unlink(os.path.join(datastore_path, "notification.txt")) \
 
     # 1995 UTF-8 content should be encoded
     test_body = 'change detection is cool 网站监测 内容更新了'
@@ -443,11 +450,11 @@ def test_global_send_test_notification(client, live_server, measure_memory_usage
     assert res.status_code != 400
     assert res.status_code != 500
 
-    with open("test-datastore/notification.txt", 'r') as f:
+    with open(os.path.join(datastore_path, "notification.txt"), 'r') as f:
         x = f.read()
         assert test_body in x
 
-    os.unlink("test-datastore/notification.txt")
+    os.unlink(os.path.join(datastore_path, "notification.txt"))
 
     ######### Test group/tag settings
     res = client.post(
@@ -462,11 +469,30 @@ def test_global_send_test_notification(client, live_server, measure_memory_usage
     # Give apprise time to fire
     time.sleep(4)
 
-    with open("test-datastore/notification.txt", 'r') as f:
+    with open(os.path.join(datastore_path, "notification.txt"), 'r') as f:
         x = f.read()
         # Should come from notification.py default handler when there is no notification body to pull from
         assert 'change detection is cool 网站监测 内容更新了' in x
 
+    ## Check that 'test' catches errors
+    test_notification_url = 'post://akjsdfkjasdkfjasdkfjasdkjfas232323/should-error'
+
+    ######### Test global/system settings
+    res = client.post(
+        url_for("ui.ui_notification.ajax_callback_send_notification_test")+"?mode=global-settings",
+        data={"notification_urls": test_notification_url},
+        follow_redirects=True
+    )
+    assert res.status_code == 400
+    assert (
+        b"No address found" in res.data or
+        b"Name or service not known" in res.data or
+        b"nodename nor servname provided" in res.data or
+        b"Temporary failure in name resolution" in res.data or
+        b"Failed to establish a new connection" in res.data or
+        b"Connection error occurred" in res.data
+    )
+    
     client.get(
         url_for("ui.form_delete", uuid="all"),
         follow_redirects=True
@@ -483,14 +509,13 @@ def test_global_send_test_notification(client, live_server, measure_memory_usage
     assert b"Error: You must have atleast one watch configured for 'test notification' to work" in res.data
 
 
-def _test_color_notifications(client, notification_body_token):
 
-    from changedetectionio.diff import ADDED_STYLE, REMOVED_STYLE
+def _test_color_notifications(client, notification_body_token, datastore_path):
 
-    set_original_response()
+    set_original_response(datastore_path=datastore_path)
 
-    if os.path.isfile("test-datastore/notification.txt"):
-        os.unlink("test-datastore/notification.txt")
+    if os.path.isfile(os.path.join(datastore_path, "notification.txt")):
+        os.unlink(os.path.join(datastore_path, "notification.txt"))
 
 
     test_notification_url = url_for('test_notification_endpoint', _external=True).replace('http://', 'post://')+"?xxx={{ watch_url }}&+custom-header=123"
@@ -503,7 +528,7 @@ def _test_color_notifications(client, notification_body_token):
             "application-fetch_backend": "html_requests",
             "application-minutes_between_check": 180,
             "application-notification_body": notification_body_token,
-            "application-notification_format": "HTML Color",
+            "application-notification_format": "htmlcolor",
             "application-notification_urls": test_notification_url,
             "application-notification_title": "New ChangeDetection.io Notification - {{ watch_url }}",
         },
@@ -522,7 +547,7 @@ def _test_color_notifications(client, notification_body_token):
 
     wait_for_all_checks(client)
 
-    set_modified_response()
+    set_modified_response(datastore_path=datastore_path)
 
 
     res = client.get(url_for("ui.form_watch_checknow"), follow_redirects=True)
@@ -531,9 +556,10 @@ def _test_color_notifications(client, notification_body_token):
     wait_for_all_checks(client)
     time.sleep(3)
 
-    with open("test-datastore/notification.txt", 'r') as f:
+    with open(os.path.join(datastore_path, "notification.txt"), 'r') as f:
         x = f.read()
-        assert f'<span style="{REMOVED_STYLE}">Which is across multiple lines' in x
+        s =  f'<span style="{HTML_CHANGED_STYLE}" role="note" aria-label="Changed text" title="Changed text">Which is across multiple lines'
+        assert s in x
 
 
     client.get(
@@ -542,6 +568,7 @@ def _test_color_notifications(client, notification_body_token):
     )
 
 # Just checks the format of the colour notifications was correct
-def test_html_color_notifications(client, live_server, measure_memory_usage):
-    _test_color_notifications(client, '{{diff}}')
-    _test_color_notifications(client, '{{diff_full}}')
+def test_html_color_notifications(client, live_server, measure_memory_usage, datastore_path):
+    _test_color_notifications(client, '{{diff}}',datastore_path=datastore_path)
+    _test_color_notifications(client, '{{diff_full}}',datastore_path=datastore_path)
+

changedetectionio/tests/test_notification_errors.py

@@ -4,10 +4,10 @@ from flask import url_for
 from .util import set_original_response, set_modified_response, live_server_setup, wait_for_all_checks
 import logging
 
-def test_check_notification_error_handling(client, live_server, measure_memory_usage):
+def test_check_notification_error_handling(client, live_server, measure_memory_usage, datastore_path):
 
    #  live_server_setup(live_server) # Setup on conftest per function
-    set_original_response()
+    set_original_response(datastore_path=datastore_path)
 
     # Set a URL and fetch it, then set a notification URL which is going to give errors
     test_url = url_for('test_endpoint', _external=True)
@@ -19,7 +19,7 @@ def test_check_notification_error_handling(client, live_server, measure_memory_u
     assert b"Watch added" in res.data
 
     wait_for_all_checks(client)
-    set_modified_response()
+    set_modified_response(datastore_path=datastore_path)
 
     working_notification_url = url_for('test_notification_endpoint', _external=True).replace('http', 'json')
     broken_notification_url = "jsons://broken-url-xxxxxxxx123/test"
@@ -30,7 +30,7 @@ def test_check_notification_error_handling(client, live_server, measure_memory_u
         data={"notification_urls": f"{broken_notification_url}\r\n{working_notification_url}",
               "notification_title": "xxx",
               "notification_body": "xxxxx",
-              "notification_format": "Text",
+              "notification_format": 'text',
               "url": test_url,
               "tags": "",
               "title": "",
@@ -73,9 +73,9 @@ def test_check_notification_error_handling(client, live_server, measure_memory_u
     assert found_name_resolution_error
 
     # And the working one, which is after the 'broken' one should still have fired
-    with open("test-datastore/notification.txt", "r") as f:
+    with open(os.path.join(datastore_path, "notification.txt"), "r") as f:
         notification_submission = f.read()
-    os.unlink("test-datastore/notification.txt")
+    os.unlink(os.path.join(datastore_path, "notification.txt"))
     assert 'xxxxx' in notification_submission
 
     client.get(url_for("ui.form_delete", uuid="all"), follow_redirects=True)

changedetectionio/tests/test_obfuscations.py

@@ -3,9 +3,10 @@
 import time
 from flask import url_for
 from .util import live_server_setup
+import os
 
 
-def set_original_ignore_response():
+def set_original_ignore_response(datastore_path):
     test_return_data = """<html>
        <body>
      <span>The price is</span><span>$<!-- -->90<!-- -->.<!-- -->74</span>
@@ -14,12 +15,12 @@ def set_original_ignore_response():
 
     """
 
-    with open("test-datastore/endpoint-content.txt", "w") as f:
+    with open(os.path.join(datastore_path, "endpoint-content.txt"), "w") as f:
         f.write(test_return_data)
 
 
-def test_obfuscations(client, live_server, measure_memory_usage):
-    set_original_ignore_response()
+def test_obfuscations(client, live_server, measure_memory_usage, datastore_path):
+    set_original_ignore_response(datastore_path)
    #  live_server_setup(live_server) # Setup on conftest per function
     time.sleep(1)
     # Add our URL to the import page

changedetectionio/tests/test_pdf.py

@@ -3,15 +3,16 @@
 import time
 from flask import url_for
 from .util import set_original_response, set_modified_response, live_server_setup, wait_for_all_checks
+import os
 
 
 # `subtractive_selectors` should still work in `source:` type requests
-def test_fetch_pdf(client, live_server, measure_memory_usage):
+def test_fetch_pdf(client, live_server, measure_memory_usage, datastore_path):
     import shutil
     import os
 
-    shutil.copy("tests/test.pdf", "test-datastore/endpoint-test.pdf")
-    first_version_size = os.path.getsize("test-datastore/endpoint-test.pdf")
+    shutil.copy("tests/test.pdf", os.path.join(datastore_path, "endpoint-test.pdf"))
+    first_version_size = os.path.getsize(os.path.join(datastore_path, "endpoint-test.pdf"))
 
     test_url = url_for('test_pdf_endpoint', _external=True)
     uuid = client.application.config.get('DATASTORE').add_watch(url=test_url)
@@ -35,14 +36,14 @@ def test_fetch_pdf(client, live_server, measure_memory_usage):
 
     # So we know if the file changes in other ways
     import hashlib
-    original_md5 = hashlib.md5(open("test-datastore/endpoint-test.pdf", 'rb').read()).hexdigest().upper()
+    original_md5 = hashlib.md5(open(os.path.join(datastore_path, "endpoint-test.pdf"), 'rb').read()).hexdigest().upper()
     # We should have one
     assert len(original_md5) >0
     # And it's going to be in the document
     assert f'Document checksum - {original_md5}' in snapshot_contents
 
-    shutil.copy("tests/test2.pdf", "test-datastore/endpoint-test.pdf")
-    changed_md5 = hashlib.md5(open("test-datastore/endpoint-test.pdf", 'rb').read()).hexdigest().upper()
+    shutil.copy("tests/test2.pdf", os.path.join(datastore_path, "endpoint-test.pdf"))
+    changed_md5 = hashlib.md5(open(os.path.join(datastore_path, "endpoint-test.pdf"), 'rb').read()).hexdigest().upper()
     res = client.get(url_for("ui.form_watch_checknow"), follow_redirects=True)
     assert b'Queued 1 watch for rechecking.' in res.data
 
@@ -76,9 +77,9 @@ def test_fetch_pdf(client, live_server, measure_memory_usage):
     # new snapshot was also OK, no HTML
     snapshot_contents = watch.get_history_snapshot(dates[1])
     assert 'html' not in snapshot_contents.lower()
-    assert f'Original file size - {os.path.getsize("test-datastore/endpoint-test.pdf")}' in snapshot_contents
+    assert f'Original file size - {os.path.getsize(os.path.join(datastore_path, "endpoint-test.pdf"))}' in snapshot_contents
     assert f'here is a change' in snapshot_contents
-    assert os.path.getsize("test-datastore/endpoint-test.pdf") != first_version_size # And the disk change worked
+    assert os.path.getsize(os.path.join(datastore_path, "endpoint-test.pdf")) != first_version_size # And the disk change worked
 
 
     

changedetectionio/tests/test_preview_endpoints.py

@@ -3,12 +3,13 @@
 import time
 from flask import url_for
 from .util import set_original_response, set_modified_response, live_server_setup, wait_for_all_checks
+import os
 
 
 # `subtractive_selectors` should still work in `source:` type requests
-def test_fetch_pdf(client, live_server, measure_memory_usage):
+def test_fetch_pdf(client, live_server, measure_memory_usage, datastore_path):
     import shutil
-    shutil.copy("tests/test.pdf", "test-datastore/endpoint-test.pdf")
+    shutil.copy("tests/test.pdf", os.path.join(datastore_path, "endpoint-test.pdf"))
 
    #  live_server_setup(live_server) # Setup on conftest per function
     test_url = url_for('test_pdf_endpoint', _external=True)
@@ -29,14 +30,14 @@ def test_fetch_pdf(client, live_server, measure_memory_usage):
 
     # So we know if the file changes in other ways
     import hashlib
-    original_md5 = hashlib.md5(open("test-datastore/endpoint-test.pdf", 'rb').read()).hexdigest().upper()
+    original_md5 = hashlib.md5(open(os.path.join(datastore_path, "endpoint-test.pdf"), 'rb').read()).hexdigest().upper()
     # We should have one
     assert len(original_md5) > 0
     # And it's going to be in the document
     assert b'Document checksum - ' + bytes(str(original_md5).encode('utf-8')) in res.data
 
-    shutil.copy("tests/test2.pdf", "test-datastore/endpoint-test.pdf")
-    changed_md5 = hashlib.md5(open("test-datastore/endpoint-test.pdf", 'rb').read()).hexdigest().upper()
+    shutil.copy("tests/test2.pdf", os.path.join(datastore_path, "endpoint-test.pdf"))
+    changed_md5 = hashlib.md5(open(os.path.join(datastore_path, "endpoint-test.pdf"), 'rb').read()).hexdigest().upper()
     res = client.get(url_for("ui.form_watch_checknow"), follow_redirects=True)
     assert b'Queued 1 watch for rechecking.' in res.data
 

changedetectionio/tests/test_request.py

@@ -8,7 +8,7 @@ from . util import set_original_response, set_modified_response, live_server_set
 
 # Hard to just add more live server URLs when one test is already running (I think)
 # So we add our test here (was in a different file)
-def test_headers_in_request(client, live_server, measure_memory_usage):
+def test_headers_in_request(client, live_server, measure_memory_usage, datastore_path):
     #ve_server_setup(live_server)
     # Add our URL to the import page
     test_url = url_for('test_headers', _external=True)
@@ -76,7 +76,8 @@ def test_headers_in_request(client, live_server, measure_memory_usage):
 
     delete_all_watches(client)
 
-def test_body_in_request(client, live_server, measure_memory_usage):
+def test_body_in_request(client, live_server, measure_memory_usage, datastore_path):
+    import os
 
     # Add our URL to the import page
     test_url = url_for('test_body', _external=True)
@@ -141,7 +142,7 @@ def test_body_in_request(client, live_server, measure_memory_usage):
     client.get(url_for("ui.form_watch_checknow"), follow_redirects=True)
     wait_for_all_checks(client)
     watches_with_body = 0
-    with open('test-datastore/url-watches.json') as f:
+    with open(os.path.join(datastore_path, 'url-watches.json')) as f:
         app_struct = json.load(f)
         for uuid in app_struct['watching']:
             if app_struct['watching'][uuid]['body']==body_value:
@@ -165,7 +166,8 @@ def test_body_in_request(client, live_server, measure_memory_usage):
     assert b"Body must be empty when Request Method is set to GET" in res.data
     delete_all_watches(client)
 
-def test_method_in_request(client, live_server, measure_memory_usage):
+def test_method_in_request(client, live_server, measure_memory_usage, datastore_path):
+    import os
     # Add our URL to the import page
     test_url = url_for('test_method', _external=True)
     if os.getenv('PLAYWRIGHT_DRIVER_URL'):
@@ -223,7 +225,7 @@ def test_method_in_request(client, live_server, measure_memory_usage):
     wait_for_all_checks(client)
 
     watches_with_method = 0
-    with open('test-datastore/url-watches.json') as f:
+    with open(os.path.join(datastore_path, 'url-watches.json')) as f:
         app_struct = json.load(f)
         for uuid in app_struct['watching']:
             if app_struct['watching'][uuid]['method'] == 'PATCH':
@@ -235,7 +237,7 @@ def test_method_in_request(client, live_server, measure_memory_usage):
     delete_all_watches(client)
 
 # Re #2408 - user-agent override test, also should handle case-insensitive header deduplication
-def test_ua_global_override(client, live_server, measure_memory_usage):
+def test_ua_global_override(client, live_server, measure_memory_usage, datastore_path):
     ##  live_server_setup(live_server) # Setup on conftest per function
     test_url = url_for('test_headers', _external=True)
 
@@ -286,8 +288,9 @@ def test_ua_global_override(client, live_server, measure_memory_usage):
     assert b"html-requests-user-agent" not in res.data
     delete_all_watches(client)
 
-def test_headers_textfile_in_request(client, live_server, measure_memory_usage):
-    
+def test_headers_textfile_in_request(client, live_server, measure_memory_usage, datastore_path):
+    import os
+
     # Add our URL to the import page
 
     webdriver_ua = "Hello fancy webdriver UA 1.0"
@@ -343,14 +346,14 @@ def test_headers_textfile_in_request(client, live_server, measure_memory_usage):
     assert b"Updated watch." in res.data
     wait_for_all_checks(client)
 
-    with open('test-datastore/headers-testtag.txt', 'w') as f:
+    with open(os.path.join(datastore_path, 'headers-testtag.txt'), 'w') as f:
         f.write("tag-header: test\r\nurl-header: http://example.com")
 
-    with open('test-datastore/headers.txt', 'w') as f:
+    with open(os.path.join(datastore_path, 'headers.txt'), 'w') as f:
         f.write("global-header: nice\r\nnext-global-header: nice\r\nurl-header-global: http://example.com/global")
 
     uuid = next(iter(live_server.app.config['DATASTORE'].data['watching']))
-    with open(f'test-datastore/{uuid}/headers.txt', 'w') as f:
+    with open(os.path.join(datastore_path, uuid, 'headers.txt'), 'w') as f:
         f.write("watch-header: nice\r\nurl-header-watch: http://example.com/watch")
 
     wait_for_all_checks(client)
@@ -368,8 +371,8 @@ def test_headers_textfile_in_request(client, live_server, measure_memory_usage):
     assert b"Extra headers file found and will be added to this watch" in res.data
 
     # Not needed anymore
-    os.unlink('test-datastore/headers.txt')
-    os.unlink('test-datastore/headers-testtag.txt')
+    os.unlink(os.path.join(datastore_path, 'headers.txt'))
+    os.unlink(os.path.join(datastore_path, 'headers-testtag.txt'))
 
     # The service should echo back the request verb
     res = client.get(
@@ -395,7 +398,7 @@ def test_headers_textfile_in_request(client, live_server, measure_memory_usage):
     # unlink headers.txt on start/stop
     delete_all_watches(client)
 
-def test_headers_validation(client, live_server, measure_memory_usage):
+def test_headers_validation(client, live_server, measure_memory_usage, datastore_path):
     
 
     test_url = url_for('test_headers', _external=True)