More translatable strings

.github/nginx-reverse-proxy-test.conf

@@ -1,33 +0,0 @@
-server {
-    listen 80;
-    server_name localhost;
-
-    # Test basic reverse proxy to changedetection.io
-    location / {
-        proxy_pass http://changedet-app:5000;
-        proxy_set_header Host $host;
-        proxy_set_header X-Real-IP $remote_addr;
-        proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
-        proxy_set_header X-Forwarded-Proto $scheme;
-
-        # WebSocket support
-        proxy_http_version 1.1;
-        proxy_set_header Upgrade $http_upgrade;
-        proxy_set_header Connection "upgrade";
-    }
-
-    # Test subpath deployment with X-Forwarded-Prefix
-    location /changedet-sub/ {
-        proxy_pass http://changedet-app:5000/;
-        proxy_set_header X-Forwarded-Prefix /changedet-sub;
-        proxy_set_header Host $host;
-        proxy_set_header X-Real-IP $remote_addr;
-        proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
-        proxy_set_header X-Forwarded-Proto $scheme;
-
-        # WebSocket support
-        proxy_http_version 1.1;
-        proxy_set_header Upgrade $http_upgrade;
-        proxy_set_header Connection "upgrade";
-    }
-}

.github/workflows/containers.yml

@@ -15,6 +15,7 @@ on:
   push:
     branches:
       - master
+      - dev
 
 jobs:
   metadata:
@@ -92,28 +93,17 @@ jobs:
           version: latest
           driver-opts: image=moby/buildkit:master
 
-      # master branch -> :dev container tag
-      - name: Docker meta :dev
-        if: ${{ github.ref == 'refs/heads/master' && github.event_name != 'release' }}
-        uses: docker/metadata-action@v5
-        id: meta_dev
-        with:
-          images: |
-            ${{ secrets.DOCKER_HUB_USERNAME }}/changedetection.io
-            ghcr.io/${{ github.repository }}
-          tags: |
-            type=raw,value=dev
-
+      # dev branch -> :dev container tag
       - name: Build and push :dev
         id: docker_build
-        if: ${{ github.ref == 'refs/heads/master' && github.event_name != 'release' }}
+        if: ${{ github.ref == 'refs/heads/dev' }}
         uses: docker/build-push-action@v6
         with:
           context: ./
           file: ./Dockerfile
           push: true
-          tags: ${{ steps.meta_dev.outputs.tags }}
-          labels: ${{ steps.meta_dev.outputs.labels }}
+          tags: |
+            ${{ secrets.DOCKER_HUB_USERNAME }}/changedetection.io:dev,ghcr.io/${{ github.repository }}:dev
           platforms: linux/amd64,linux/arm64,linux/arm/v7,linux/arm/v8
           cache-from: type=gha
           cache-to: type=gha,mode=max
@@ -152,7 +142,6 @@ jobs:
           file: ./Dockerfile
           push: true
           tags: ${{ steps.meta.outputs.tags }}
-          labels: ${{ steps.meta.outputs.labels }}
           platforms: linux/amd64,linux/arm64,linux/arm/v7,linux/arm/v8
           cache-from: type=gha
           cache-to: type=gha,mode=max

.github/workflows/pypi-release.yml

@@ -61,8 +61,8 @@ jobs:
         
         # --- API test ---
         # This also means that the docs/api-spec.yml was shipped and could be read
-        test -f /tmp/changedetection.json
-        API_KEY=$(jq -r '.. | .api_access_token? // empty' /tmp/changedetection.json)
+        test -f /tmp/url-watches.json
+        API_KEY=$(jq -r '.. | .api_access_token? // empty' /tmp/url-watches.json)
         echo Test API KEY is $API_KEY
         curl -X POST "http://127.0.0.1:10000/api/v1/watch" \
           -H "x-api-key: ${API_KEY}" \

.github/workflows/test-stack-reusable-workflow.yml

@@ -37,29 +37,10 @@ jobs:
             ${{ runner.os }}-pip-py${{ env.PYTHON_VERSION }}-
             ${{ runner.os }}-pip-
 
-      - name: Get current date for cache key
-        id: date
-        run: echo "date=$(date +'%Y-%m-%d')" >> $GITHUB_OUTPUT
-
-      - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v3
-
       - name: Build changedetection.io container for testing under Python ${{ env.PYTHON_VERSION }}
-        uses: docker/build-push-action@v6
-        with:
-          context: ./
-          file: ./Dockerfile
-          build-args: |
-            PYTHON_VERSION=${{ env.PYTHON_VERSION }}
-            LOGGER_LEVEL=TRACE
-          tags: test-changedetectionio
-          load: true
-          cache-from: type=gha,scope=build-${{ github.ref_name }}-py${{ env.PYTHON_VERSION }}-${{ hashFiles('requirements.txt', 'Dockerfile') }}-${{ steps.date.outputs.date }}
-          cache-to: type=gha,mode=max,scope=build-${{ github.ref_name }}-py${{ env.PYTHON_VERSION }}-${{ hashFiles('requirements.txt', 'Dockerfile') }}-${{ steps.date.outputs.date }}
-
-      - name: Verify build
         run: |
-          echo "---- Built for Python ${{ env.PYTHON_VERSION }} -----"
+          echo "---- Building for Python ${{ env.PYTHON_VERSION }} -----"
+          docker build --build-arg PYTHON_VERSION=${{ env.PYTHON_VERSION }} --build-arg LOGGER_LEVEL=TRACE -t test-changedetectionio .
           docker run test-changedetectionio bash -c 'pip list'
 
       - name: We should be Python ${{ env.PYTHON_VERSION }} ...
@@ -103,7 +84,6 @@ jobs:
           docker run test-changedetectionio bash -c 'python3 -m unittest changedetectionio.tests.unit.test_watch_model'
           docker run test-changedetectionio bash -c 'python3 -m unittest changedetectionio.tests.unit.test_jinja2_security'
           docker run test-changedetectionio bash -c 'python3 -m unittest changedetectionio.tests.unit.test_semver'
-          docker run test-changedetectionio bash -c 'python3 -m unittest changedetectionio.tests.unit.test_html_to_text'         
 
   # Basic pytest tests with ancillary services
   basic-tests:
@@ -130,32 +110,6 @@ jobs:
           docker network inspect changedet-network >/dev/null 2>&1 || docker network create changedet-network
           docker run --name test-cdio-basic-tests --network changedet-network test-changedetectionio bash -c 'cd changedetectionio && ./run_basic_tests.sh'
 
-      - name: Test CLI options
-        run: |
-          docker network inspect changedet-network >/dev/null 2>&1 || docker network create changedet-network
-          docker run --name test-cdio-cli-opts --network changedet-network test-changedetectionio bash -c 'changedetectionio/test_cli_opts.sh' &> cli-opts-output.txt
-          echo "=== CLI Options Test Output ==="
-          cat cli-opts-output.txt
-
-      - name: CLI Memory Test
-        run: |
-          echo "=== Checking CLI batch mode memory usage ==="
-          # Extract RSS memory value from output
-          RSS_MB=$(grep -oP "Memory consumption before worker shutdown: RSS=\K[\d.]+" cli-opts-output.txt | head -1 || echo "0")
-          echo "RSS Memory: ${RSS_MB} MB"
-
-          # Check if RSS is less than 100MB
-          if [ -n "$RSS_MB" ]; then
-            if (( $(echo "$RSS_MB < 100" | bc -l) )); then
-              echo "✓ Memory usage is acceptable: ${RSS_MB} MB < 100 MB"
-            else
-              echo "✗ Memory usage too high: ${RSS_MB} MB >= 100 MB"
-              exit 1
-            fi
-          else
-            echo "⚠ Could not extract memory usage, skipping check"
-          fi
-
       - name: Extract memory report and logs
         if: always()
         uses: ./.github/actions/extract-memory-report
@@ -170,13 +124,6 @@ jobs:
           name: test-cdio-basic-tests-output-py${{ env.PYTHON_VERSION }}
           path: output-logs
 
-      - name: Store CLI test output
-        if: always()
-        uses: actions/upload-artifact@v6
-        with:
-          name: test-cdio-cli-opts-output-py${{ env.PYTHON_VERSION }}
-          path: cli-opts-output.txt
-
   # Playwright tests
   playwright-tests:
     runs-on: ubuntu-latest
@@ -324,175 +271,6 @@ jobs:
         run: |
           docker run --rm --network changedet-network test-changedetectionio bash -c 'cd changedetectionio;pytest tests/smtp/test_notification_smtp.py'
 
-  nginx-reverse-proxy:
-    runs-on: ubuntu-latest
-    needs: build
-    timeout-minutes: 10
-    env:
-      PYTHON_VERSION: ${{ inputs.python-version }}
-    steps:
-      - uses: actions/checkout@v6
-
-      - name: Download Docker image artifact
-        uses: actions/download-artifact@v7
-        with:
-          name: test-changedetectionio-${{ env.PYTHON_VERSION }}
-          path: /tmp
-
-      - name: Load Docker image
-        run: |
-          docker load -i /tmp/test-changedetectionio.tar
-
-      - name: Spin up services
-        run: |
-          docker network create changedet-network
-
-          # Start changedetection.io container with X-Forwarded headers support
-          docker run --name changedet-app --hostname changedet-app --network changedet-network \
-            -e USE_X_SETTINGS=true \
-            -d test-changedetectionio
-          sleep 3
-
-      - name: Start nginx reverse proxy
-        run: |
-          # Start nginx with our test configuration
-          docker run --name nginx-proxy --network changedet-network -d -p 8080:80 --rm \
-            -v ${{ github.workspace }}/.github/nginx-reverse-proxy-test.conf:/etc/nginx/conf.d/default.conf:ro \
-            nginx:alpine
-          sleep 2
-
-      - name: Test reverse proxy - root path
-        run: |
-          echo "=== Testing nginx reverse proxy at root path ==="
-          curl --retry-connrefused --retry 6 -s http://localhost:8080/ > /tmp/nginx-test-root.html
-
-          # Check for changedetection.io UI elements
-          if grep -q "checkbox-uuid" /tmp/nginx-test-root.html; then
-            echo "✓ Found checkbox-uuid in response"
-          else
-            echo "ERROR: checkbox-uuid not found in response"
-            cat /tmp/nginx-test-root.html
-            exit 1
-          fi
-
-          # Check for watchlist content
-          if grep -q -i "watch" /tmp/nginx-test-root.html; then
-            echo "✓ Found watch/watchlist content in response"
-          else
-            echo "ERROR: watchlist content not found"
-            cat /tmp/nginx-test-root.html
-            exit 1
-          fi
-
-          echo "✓ Root path reverse proxy working correctly"
-
-      - name: Test reverse proxy - subpath with X-Forwarded-Prefix
-        run: |
-          echo "=== Testing nginx reverse proxy at subpath /changedet-sub/ ==="
-          curl --retry-connrefused --retry 6 -s http://localhost:8080/changedet-sub/ > /tmp/nginx-test-subpath.html
-
-          # Check for changedetection.io UI elements
-          if grep -q "checkbox-uuid" /tmp/nginx-test-subpath.html; then
-            echo "✓ Found checkbox-uuid in subpath response"
-          else
-            echo "ERROR: checkbox-uuid not found in subpath response"
-            cat /tmp/nginx-test-subpath.html
-            exit 1
-          fi
-
-          echo "✓ Subpath reverse proxy working correctly"
-
-      - name: Test API through reverse proxy subpath
-        run: |
-          echo "=== Testing API endpoints through nginx subpath /changedet-sub/ ==="
-
-          # Extract API key from the changedetection.io datastore
-          API_KEY=$(docker exec changedet-app cat /datastore/changedetection.json | grep -o '"api_access_token": *"[^"]*"' | cut -d'"' -f4)
-
-          if [ -z "$API_KEY" ]; then
-            echo "ERROR: Could not extract API key from datastore"
-            docker exec changedet-app cat /datastore/changedetection.json
-            exit 1
-          fi
-
-          echo "✓ Extracted API key: ${API_KEY:0:8}..."
-
-          # Create a watch via API through nginx proxy subpath
-          echo "Creating watch via POST to /changedet-sub/api/v1/watch"
-          RESPONSE=$(curl -s -w "\n%{http_code}" -X POST "http://localhost:8080/changedet-sub/api/v1/watch" \
-            -H "x-api-key: ${API_KEY}" \
-            -H "Content-Type: application/json" \
-            -d '{
-              "url": "https://example.com/test-nginx-proxy",
-              "tag": "nginx-test"
-            }')
-
-          HTTP_CODE=$(echo "$RESPONSE" | tail -n1)
-          BODY=$(echo "$RESPONSE" | head -n-1)
-
-          if [ "$HTTP_CODE" != "201" ]; then
-            echo "ERROR: Expected HTTP 201, got $HTTP_CODE"
-            echo "Response: $BODY"
-            exit 1
-          fi
-
-          echo "✓ Watch created successfully (HTTP 201)"
-
-          # Extract the watch UUID from response
-          WATCH_UUID=$(echo "$BODY" | grep -o '"uuid": *"[^"]*"' | cut -d'"' -f4)
-          echo "✓ Watch UUID: $WATCH_UUID"
-
-          # Update the watch via PUT through nginx proxy subpath
-          echo "Updating watch via PUT to /changedet-sub/api/v1/watch/${WATCH_UUID}"
-          RESPONSE=$(curl -s -w "\n%{http_code}" -X PUT "http://localhost:8080/changedet-sub/api/v1/watch/${WATCH_UUID}" \
-            -H "x-api-key: ${API_KEY}" \
-            -H "Content-Type: application/json" \
-            -d '{
-              "paused": true
-            }')
-
-          HTTP_CODE=$(echo "$RESPONSE" | tail -n1)
-          BODY=$(echo "$RESPONSE" | head -n-1)
-
-          if [ "$HTTP_CODE" != "200" ]; then
-            echo "ERROR: Expected HTTP 200, got $HTTP_CODE"
-            echo "Response: $BODY"
-            exit 1
-          fi
-
-          if echo "$BODY" | grep -q 'OK'; then
-            echo "✓ Watch updated successfully (HTTP 200, response: OK)"
-          else
-            echo "ERROR: Expected response 'OK', got: $BODY"
-            echo "Response: $BODY"
-            exit 1
-          fi
-
-          # Verify the watch is paused via GET
-          echo "Verifying watch is paused via GET"
-          RESPONSE=$(curl -s "http://localhost:8080/changedet-sub/api/v1/watch/${WATCH_UUID}" \
-            -H "x-api-key: ${API_KEY}")
-
-          if echo "$RESPONSE" | grep -q '"paused": *true'; then
-            echo "✓ Watch is paused as expected"
-          else
-            echo "ERROR: Watch paused state not confirmed"
-            echo "Response: $RESPONSE"
-            exit 1
-          fi
-
-          echo "✓ API tests through nginx subpath completed successfully"
-
-      - name: Cleanup nginx test
-        if: always()
-        run: |
-          docker logs nginx-proxy || true
-          docker logs changedet-app || true
-          docker stop nginx-proxy changedet-app || true
-          docker rm nginx-proxy changedet-app || true
-
-
-
   # Proxy tests
   proxy-tests:
     runs-on: ubuntu-latest
@@ -564,29 +342,6 @@ jobs:
           cd changedetectionio
           ./run_custom_browser_url_tests.sh
 
-  processor-plugin-tests:
-    runs-on: ubuntu-latest
-    needs: build
-    timeout-minutes: 20
-    env:
-      PYTHON_VERSION: ${{ inputs.python-version }}
-    steps:
-      - uses: actions/checkout@v6
-
-      - name: Download Docker image artifact
-        uses: actions/download-artifact@v7
-        with:
-          name: test-changedetectionio-${{ env.PYTHON_VERSION }}
-          path: /tmp
-
-      - name: Load Docker image
-        run: |
-          docker load -i /tmp/test-changedetectionio.tar
-
-      - name: Basic processor plugin registration and checks
-        run: |
-          docker run  -e EXTRA_PACKAGES=changedetection.io-osint-processor test-changedetectionio bash -c 'cd changedetectionio;pytest -vvv -s  tests/plugins/test_processor.py::test_check_plugin_processor'
-
   # Container startup tests
   container-tests:
     runs-on: ubuntu-latest
@@ -685,142 +440,3 @@ jobs:
             exit 1
           fi
           docker rm sig-test
-
-  # Upgrade path test
-  upgrade-path-test:
-    runs-on: ubuntu-latest
-    needs: build
-    timeout-minutes: 25
-    env:
-      PYTHON_VERSION: ${{ inputs.python-version }}
-    steps:
-      - uses: actions/checkout@v6
-        with:
-          fetch-depth: 0  # Fetch all history and tags for upgrade testing
-
-      - name: Set up Python ${{ env.PYTHON_VERSION }}
-        uses: actions/setup-python@v6
-        with:
-          python-version: ${{ env.PYTHON_VERSION }}
-
-      - name: Check upgrade works without error
-        run: |
-          echo "=== Testing upgrade path from 0.49.1 to ${{ github.ref_name }} (${{ github.sha }}) ==="
-
-          # Checkout old version and create datastore
-          git checkout 0.49.1
-          python3 -m venv .venv
-          source .venv/bin/activate
-          pip install -r requirements.txt
-          pip install 'pyOpenSSL>=23.2.0'
-
-          echo "=== Running version 0.49.1 to create datastore ==="
-          python3 ./changedetection.py -C -d /tmp/data &
-          APP_PID=$!
-
-          # Wait for app to be ready
-          echo "Waiting for 0.49.1 to be ready..."
-          sleep 6
-
-          # Extract API key from datastore (0.49.1 uses url-watches.json)
-          API_KEY=$(jq -r '.settings.application.api_access_token // empty' /tmp/data/url-watches.json)
-          echo "API Key: ${API_KEY:0:8}..."
-
-          # Create a watch with tag "github-group-test" via API
-          echo "Creating test watch with tag via API..."
-          curl -X POST "http://127.0.0.1:5000/api/v1/watch" \
-            -H "x-api-key: ${API_KEY}" \
-            -H "Content-Type: application/json" \
-            --show-error --fail \
-            --retry 6 --retry-delay 1 --retry-connrefused \
-            -d '{
-              "url": "https://example.com/upgrade-test",
-              "tag": "github-group-test"
-            }'
-
-          echo "✓ Created watch with tag 'github-group-test'"
-
-          # Create a specific test URL watch
-          echo "Creating test URL watch via API..."
-          curl -X POST "http://127.0.0.1:5000/api/v1/watch" \
-            -H "x-api-key: ${API_KEY}" \
-            -H "Content-Type: application/json" \
-            --show-error --fail \
-            -d '{
-              "url": "http://localhost/test.txt"
-            }'
-
-          echo "✓ Created watch for 'http://localhost/test.txt' in version 0.49.1"
-
-          # Stop the old version gracefully
-          kill $APP_PID
-          wait $APP_PID || true
-          echo "✓ Version 0.49.1 stopped"
-
-          # Upgrade to current version (use commit SHA since we're in detached HEAD)
-          echo "Upgrading to commit ${{ github.sha }}"
-          git checkout ${{ github.sha }}
-          pip install -r requirements.txt
-
-          echo "=== Running current version (commit ${{ github.sha }}) with old datastore (testing mode) ==="
-          TESTING_SHUTDOWN_AFTER_DATASTORE_LOAD=1 python3 ./changedetection.py -d /tmp/data > /tmp/upgrade-test.log 2>&1
-
-          echo "=== Upgrade test output ==="
-          cat /tmp/upgrade-test.log
-          echo "✓ Datastore upgraded successfully"
-
-          # Now start the current version normally to verify the tag survived
-          echo "=== Starting current version to verify tag exists after upgrade ==="
-          timeout 20 python3 ./changedetection.py -d /tmp/data > /tmp/ui-test.log 2>&1 &
-          APP_PID=$!
-
-          # Wait for app to be ready and fetch UI
-          echo "Waiting for current version to be ready..."
-          sleep 5
-          curl --retry 6 --retry-delay 1 --retry-connrefused --silent http://127.0.0.1:5000 > /tmp/ui-output.html
-
-          # Verify tag exists in UI
-          if grep -q "github-group-test" /tmp/ui-output.html; then
-            echo "✓ Tag 'github-group-test' found in UI after upgrade"
-          else
-            echo "ERROR: Tag 'github-group-test' not found in UI after upgrade"
-            echo "=== UI Output ==="
-            cat /tmp/ui-output.html
-            echo "=== App Log ==="
-            cat /tmp/ui-test.log
-            kill $APP_PID || true
-            exit 1
-          fi
-
-          # Verify test URL exists in UI
-          if grep -q "http://localhost/test.txt" /tmp/ui-output.html; then
-            echo "✓ Watch URL 'http://localhost/test.txt' found in UI after upgrade"
-          else
-            echo "ERROR: Watch URL 'http://localhost/test.txt' not found in UI after upgrade"
-            echo "=== UI Output ==="
-            cat /tmp/ui-output.html
-            echo "=== App Log ==="
-            cat /tmp/ui-test.log
-            kill $APP_PID || true
-            exit 1
-          fi
-
-          # Cleanup
-          kill $APP_PID || true
-          wait $APP_PID || true
-
-          echo ""
-          echo "✓✓✓ Upgrade test passed: 0.49.1 → ${{ github.ref_name }} ✓✓✓"
-          echo "    - Commit: ${{ github.sha }}"
-          echo "    - Datastore migrated successfully"
-          echo "    - Tag 'github-group-test' survived upgrade"
-          echo "    - Watch URL 'http://localhost/test.txt' survived upgrade"
-
-          echo "✓ Upgrade test passed: 0.49.1 → ${{ github.ref_name }}"
-
-      - name: Upload upgrade test logs
-        if: always()
-        uses: actions/upload-artifact@v6
-        with:
-          name: upgrade-test-logs-py${{ env.PYTHON_VERSION }}
-          path: /tmp/upgrade-test.log

.gitignore

@@ -29,4 +29,3 @@ test-datastore/
 
 # Memory consumption log
 test-memory.log
-tests/logs/

Dockerfile

@@ -78,12 +78,6 @@ RUN --mount=type=cache,id=pip,sharing=locked,target=/tmp/pip-cache \
 # Final image stage
 FROM python:${PYTHON_VERSION}-slim-bookworm
 LABEL org.opencontainers.image.source="https://github.com/dgtlmoon/changedetection.io"
-LABEL org.opencontainers.image.url="https://changedetection.io"
-LABEL org.opencontainers.image.documentation="https://changedetection.io/tutorials"
-LABEL org.opencontainers.image.title="changedetection.io"
-LABEL org.opencontainers.image.description="Self-hosted web page change monitoring and notification service"
-LABEL org.opencontainers.image.licenses="Apache-2.0"
-LABEL org.opencontainers.image.vendor="changedetection.io"
 
 RUN apt-get update && apt-get install -y --no-install-recommends \
     libxslt1.1 \
@@ -138,15 +132,6 @@ ENV LOGGER_LEVEL="$LOGGER_LEVEL"
 ENV LC_ALL=en_US.UTF-8
 
 WORKDIR /app
-
-# Copy and set up entrypoint script for installing extra packages
-COPY docker-entrypoint.sh /docker-entrypoint.sh
-RUN chmod +x /docker-entrypoint.sh
-
-# Set entrypoint to handle EXTRA_PACKAGES env var
-ENTRYPOINT ["/docker-entrypoint.sh"]
-
-# Default command (can be overridden in docker-compose.yml)
 CMD ["python", "./changedetection.py", "-d", "/datastore"]
 
 

MANIFEST.in

@@ -9,7 +9,6 @@ recursive-include changedetectionio/notification *
 recursive-include changedetectionio/processors *
 recursive-include changedetectionio/realtime *
 recursive-include changedetectionio/static *
-recursive-include changedetectionio/store *
 recursive-include changedetectionio/templates *
 recursive-include changedetectionio/tests *
 recursive-include changedetectionio/translations *
@@ -17,7 +16,6 @@ recursive-include changedetectionio/widgets *
 prune changedetectionio/static/package-lock.json
 prune changedetectionio/static/styles/node_modules
 prune changedetectionio/static/styles/package-lock.json
-include changedetectionio/favicon_utils.py
 include changedetection.py
 include requirements.txt
 include README-pip.md

babel.cfg

@@ -1,5 +0,0 @@
-[python: **.py]
-keywords = _:1,_l:1,gettext:1
-
-[jinja2: **/templates/**.html]
-encoding = utf-8

changedetectionio/__init__.py

@@ -2,24 +2,23 @@
 
 # Read more https://github.com/dgtlmoon/changedetection.io/wiki
 # Semver means never use .01, or 00. Should be .1.
-__version__ = '0.53.6'
+__version__ = '0.52.6'
 
 from changedetectionio.strtobool import strtobool
 from json.decoder import JSONDecodeError
-
-from loguru import logger
-import getopt
 import logging
 import os
+import getopt
 import platform
 import signal
-import threading
-import time
+
+import sys
 
 # Eventlet completely removed - using threading mode for SocketIO
 # This provides better Python 3.12+ compatibility and eliminates eventlet/asyncio conflicts
-# Note: store and changedetection_app are imported inside main() to avoid
-# initialization before argument parsing (allows --help to work without loading everything)
+from changedetectionio import store
+from changedetectionio.flask_app import changedetection_app
+from loguru import logger
 
 # ==============================================================================
 # Multiprocessing Configuration - CRITICAL for Thread Safety
@@ -84,26 +83,15 @@ def get_version():
 def sigshutdown_handler(_signo, _stack_frame):
     name = signal.Signals(_signo).name
     logger.critical(f'Shutdown: Got Signal - {name} ({_signo}), Fast shutdown initiated')
-
+    
     # Set exit flag immediately to stop all loops
     app.config.exit.set()
     datastore.stop_thread = True
-
-    # Log memory consumption before shutting down workers (cross-platform)
-    try:
-        import psutil
-        process = psutil.Process()
-        mem_info = process.memory_info()
-        rss_mb = mem_info.rss / 1024 / 1024
-        vms_mb = mem_info.vms / 1024 / 1024
-        logger.info(f"Memory consumption before worker shutdown: RSS={rss_mb:,.2f} MB, VMS={vms_mb:,.2f} MB")
-    except Exception as e:
-        logger.warning(f"Could not retrieve memory stats: {str(e)}")
-
+    
     # Shutdown workers and queues immediately
     try:
-        from changedetectionio import worker_pool
-        worker_pool.shutdown_workers()
+        from changedetectionio import worker_handler
+        worker_handler.shutdown_workers()
     except Exception as e:
         logger.error(f"Error shutting down workers: {str(e)}")
     
@@ -112,9 +100,9 @@ def sigshutdown_handler(_signo, _stack_frame):
         from changedetectionio.flask_app import update_q, notification_q
         update_q.close()
         notification_q.close()
-        logger.debug("Queues closed successfully")
+        logger.debug("Janus queues closed successfully")
     except Exception as e:
-        logger.critical(f"CRITICAL: Failed to close queues: {e}")
+        logger.critical(f"CRITICAL: Failed to close janus queues: {e}")
     
     # Shutdown socketio server fast
     from changedetectionio.flask_app import socketio_server
@@ -124,80 +112,31 @@ def sigshutdown_handler(_signo, _stack_frame):
         except Exception as e:
             logger.error(f"Error shutting down Socket.IO server: {str(e)}")
     
-    # With immediate persistence, all data is already saved
-    logger.success('All data already persisted (immediate commits enabled).')
-
+    # Save data quickly
+    try:
+        datastore.sync_to_json()
+        logger.success('Fast sync to disk complete.')
+    except Exception as e:
+        logger.error(f"Error syncing to disk: {str(e)}")
+    
     sys.exit()
 
-def print_help():
-    """Print help text for command line options"""
-    print('Usage: changedetection.py [options]')
-    print('')
-    print('Standard options:')
-    print('  -s                SSL enable')
-    print('  -h HOST           Listen host (default: 0.0.0.0)')
-    print('  -p PORT           Listen port (default: 5000)')
-    print('  -d PATH           Datastore path')
-    print('  -l LEVEL          Log level (TRACE, DEBUG, INFO, SUCCESS, WARNING, ERROR, CRITICAL)')
-    print('  -c                Cleanup unused snapshots')
-    print('  -C                Create datastore directory if it doesn\'t exist')
-    print('  -P true/false     Set all watches paused (true) or active (false)')
-    print('')
-    print('Add URLs on startup:')
-    print('  -u URL            Add URL to watch (can be used multiple times)')
-    print('  -u0 \'JSON\'        Set options for first -u URL (e.g. \'{"processor":"text_json_diff"}\')')
-    print('  -u1 \'JSON\'        Set options for second -u URL (0-indexed)')
-    print('  -u2 \'JSON\'        Set options for third -u URL, etc.')
-    print('                    Available options: processor, fetch_backend, headers, method, etc.')
-    print('                    See model/Watch.py for all available options')
-    print('')
-    print('Recheck on startup:')
-    print('  -r all            Queue all watches for recheck on startup')
-    print('  -r UUID,...       Queue specific watches (comma-separated UUIDs)')
-    print('  -r all N          Queue all watches, wait for completion, repeat N times')
-    print('  -r UUID,... N     Queue specific watches, wait for completion, repeat N times')
-    print('')
-    print('Batch mode:')
-    print('  -b                Run in batch mode (process queue then exit)')
-    print('                    Useful for CI/CD, cron jobs, or one-time checks')
-    print('                    NOTE: Batch mode checks if Flask is running and aborts if port is in use')
-    print('                    Use -p PORT to specify a different port if needed')
-    print('')
-
 def main():
     global datastore
     global app
 
-    # Early help/version check before any initialization
-    if '--help' in sys.argv or '-help' in sys.argv:
-        print_help()
-        sys.exit(0)
-
-    if '--version' in sys.argv or '-v' in sys.argv:
-        print(f'changedetection.io {__version__}')
-        sys.exit(0)
-
-    # Import heavy modules after help/version checks to keep startup fast for those flags
-    from changedetectionio import store
-    from changedetectionio.flask_app import changedetection_app
-
     datastore_path = None
+    do_cleanup = False
+    # Optional URL to watch since start
+    default_url = None
     # Set a default logger level
     logger_level = 'DEBUG'
     include_default_watches = True
-    all_paused = None  # None means don't change, True/False to set
 
     host = os.environ.get("LISTEN_HOST", "0.0.0.0").strip()
     port = int(os.environ.get('PORT', 5000))
     ssl_mode = False
 
-    # Lists for multiple URLs and their options
-    urls_to_add = []
-    url_options = {}  # Key: index (0-based), Value: dict of options
-    recheck_watches = None  # None, 'all', or list of UUIDs
-    recheck_repeat_count = 1  # Number of times to repeat recheck cycle
-    batch_mode = False  # Run once then exit when queue is empty
-
     # On Windows, create and use a default path.
     if os.name == 'nt':
         datastore_path = os.path.expandvars(r'%APPDATA%\changedetection.io')
@@ -206,68 +145,10 @@ def main():
         # Must be absolute so that send_from_directory doesnt try to make it relative to backend/
         datastore_path = os.path.join(os.getcwd(), "../datastore")
 
-    # Pre-process arguments to extract -u, -u<N>, and -r options before getopt
-    # This allows unlimited -u0, -u1, -u2, ... options without predefining them
-    cleaned_argv = ['changedetection.py']  # Start with program name
-    i = 1
-    while i < len(sys.argv):
-        arg = sys.argv[i]
-
-        # Handle -u (add URL)
-        if arg == '-u' and i + 1 < len(sys.argv):
-            urls_to_add.append(sys.argv[i + 1])
-            i += 2
-            continue
-
-        # Handle -u<N> (set options for URL at index N)
-        if arg.startswith('-u') and len(arg) > 2 and arg[2:].isdigit():
-            idx = int(arg[2:])
-            if i + 1 < len(sys.argv):
-                try:
-                    import json
-                    url_options[idx] = json.loads(sys.argv[i + 1])
-                except json.JSONDecodeError as e:
-                    print(f'Error: Invalid JSON for {arg}: {sys.argv[i + 1]}')
-                    print(f'JSON decode error: {e}')
-                    sys.exit(2)
-                i += 2
-                continue
-
-        # Handle -r (recheck watches)
-        if arg == '-r' and i + 1 < len(sys.argv):
-            recheck_arg = sys.argv[i + 1]
-            if recheck_arg.lower() == 'all':
-                recheck_watches = 'all'
-            else:
-                # Parse comma-separated list of UUIDs
-                recheck_watches = [uuid.strip() for uuid in recheck_arg.split(',') if uuid.strip()]
-
-            # Check for optional repeat count as third argument
-            if i + 2 < len(sys.argv) and sys.argv[i + 2].isdigit():
-                recheck_repeat_count = int(sys.argv[i + 2])
-                if recheck_repeat_count < 1:
-                    print(f'Error: Repeat count must be at least 1, got {recheck_repeat_count}')
-                    sys.exit(2)
-                i += 3
-            else:
-                i += 2
-            continue
-
-        # Handle -b (batch mode - run once and exit)
-        if arg == '-b':
-            batch_mode = True
-            i += 1
-            continue
-
-        # Keep other arguments for getopt
-        cleaned_argv.append(arg)
-        i += 1
-
     try:
-        opts, args = getopt.getopt(cleaned_argv[1:], "6Csd:h:p:l:P:", "port")
-    except getopt.GetoptError as e:
-        print_help()
-        print(f'Error: {e}')
+        opts, args = getopt.getopt(sys.argv[1:], "6Ccsd:h:p:l:u:", "port")
+    except getopt.GetoptError:
+        print('backend.py -s SSL enable -h [host] -p [port] -d [datastore path] -u [default URL to watch] -l [debug level - TRACE, DEBUG(default), INFO, SUCCESS, WARNING, ERROR, CRITICAL]')
         sys.exit(2)
 
     create_datastore_dir = False
@@ -292,6 +173,14 @@ def main():
         if opt == '-d':
             datastore_path = arg
 
+        if opt == '-u':
+            default_url = arg
+            include_default_watches = False
+
+        # Cleanup (remove text files that arent in the index)
+        if opt == '-c':
+            do_cleanup = True
+
         # Create the datadir if it doesnt exist
         if opt == '-C':
             create_datastore_dir = True
@@ -299,18 +188,6 @@ def main():
         if opt == '-l':
             logger_level = int(arg) if arg.isdigit() else arg.upper()
 
-        if opt == '-P':
-            try:
-                all_paused = bool(strtobool(arg))
-            except ValueError:
-                print(f'Error: Invalid value for -P option: {arg}')
-                print('Expected: true, false, yes, no, 1, or 0')
-                sys.exit(2)
-
-    # If URLs are provided, don't include default watches
-    if urls_to_add:
-        include_default_watches = False
-
 
     logger.success(f"changedetection.io version {get_version()} starting.")
     # Launch using SocketIO run method for proper integration (if enabled)
@@ -347,16 +224,11 @@ def main():
     logging.getLogger('pyppeteer.connection.Connection').setLevel(logging.WARNING)
 
     # isnt there some @thingy to attach to each route to tell it, that this route needs a datastore
-    app_config = {
-        'datastore_path': datastore_path,
-        'batch_mode': batch_mode,
-        'recheck_watches': recheck_watches,
-        'recheck_repeat_count': recheck_repeat_count
-    }
+    app_config = {'datastore_path': datastore_path}
 
     if not os.path.isdir(app_config['datastore_path']):
         if create_datastore_dir:
-            os.makedirs(app_config['datastore_path'], exist_ok=True)
+            os.mkdir(app_config['datastore_path'])
         else:
             logger.critical(
                 f"ERROR: Directory path for the datastore '{app_config['datastore_path']}'"
@@ -371,219 +243,17 @@ def main():
         # Dont' start if the JSON DB looks corrupt
         logger.critical(f"ERROR: JSON DB or Proxy List JSON at '{app_config['datastore_path']}' appears to be corrupt, aborting.")
         logger.critical(str(e))
-        sys.exit(1)
-
-    # Testing mode: Exit cleanly after datastore initialization (for CI/CD upgrade tests)
-    if os.environ.get('TESTING_SHUTDOWN_AFTER_DATASTORE_LOAD'):
-        logger.success(f"TESTING MODE: Datastore loaded successfully from {app_config['datastore_path']}")
-        logger.success(f"TESTING MODE: Schema version: {datastore.data['settings']['application'].get('schema_version', 'unknown')}")
-        logger.success(f"TESTING MODE: Loaded {len(datastore.data['watching'])} watches")
-        logger.success("TESTING MODE: Exiting cleanly (TESTING_SHUTDOWN_AFTER_DATASTORE_LOAD is set)")
-        sys.exit(0)
-
-    # Apply all_paused setting if specified via CLI
-    if all_paused is not None:
-        datastore.data['settings']['application']['all_paused'] = all_paused
-        logger.info(f"Setting all watches paused: {all_paused}")
+        return
 
     # Inject datastore into plugins that need access to settings
     from changedetectionio.pluggy_interface import inject_datastore_into_plugins
     inject_datastore_into_plugins(datastore)
 
-    # Step 1: Add URLs with their options (if provided via -u flags)
-    added_watch_uuids = []
-    if urls_to_add:
-        logger.info(f"Adding {len(urls_to_add)} URL(s) from command line")
-        for idx, url in enumerate(urls_to_add):
-            extras = url_options.get(idx, {})
-            if extras:
-                logger.debug(f"Adding watch {idx}: {url} with options: {extras}")
-            else:
-                logger.debug(f"Adding watch {idx}: {url}")
-
-            new_uuid = datastore.add_watch(url=url, extras=extras)
-            if new_uuid:
-                added_watch_uuids.append(new_uuid)
-                logger.success(f"Added watch: {url} (UUID: {new_uuid})")
-            else:
-                logger.error(f"Failed to add watch: {url}")
+    if default_url:
+        datastore.add_watch(url = default_url)
 
     app = changedetection_app(app_config, datastore)
 
-    # Step 2: Queue newly added watches (if -u was provided in batch mode)
-    # This must happen AFTER app initialization so update_q is available
-    if batch_mode and added_watch_uuids:
-        from changedetectionio.flask_app import update_q
-        from changedetectionio import queuedWatchMetaData, worker_pool
-
-        logger.info(f"Batch mode: Queuing {len(added_watch_uuids)} newly added watches")
-        for watch_uuid in added_watch_uuids:
-            try:
-                worker_pool.queue_item_async_safe(
-                    update_q,
-                    queuedWatchMetaData.PrioritizedItem(priority=1, item={'uuid': watch_uuid})
-                )
-                logger.debug(f"Queued newly added watch: {watch_uuid}")
-            except Exception as e:
-                logger.error(f"Failed to queue watch {watch_uuid}: {e}")
-
-    # Step 3: Queue watches for recheck (if -r was provided)
-    # This must happen AFTER app initialization so update_q is available
-    if recheck_watches is not None:
-        from changedetectionio.flask_app import update_q
-        from changedetectionio import queuedWatchMetaData, worker_pool
-
-        watches_to_queue = []
-        if recheck_watches == 'all':
-            # Queue all watches, excluding those already queued in batch mode
-            all_watches = list(datastore.data['watching'].keys())
-            if batch_mode and added_watch_uuids:
-                # Exclude newly added watches that were already queued in batch mode
-                watches_to_queue = [uuid for uuid in all_watches if uuid not in added_watch_uuids]
-                logger.info(f"Queuing {len(watches_to_queue)} existing watches for recheck ({len(added_watch_uuids)} newly added watches already queued)")
-            else:
-                watches_to_queue = all_watches
-                logger.info(f"Queuing all {len(watches_to_queue)} watches for recheck")
-        else:
-            # Queue specific UUIDs
-            watches_to_queue = recheck_watches
-            logger.info(f"Queuing {len(watches_to_queue)} specific watches for recheck")
-
-        queued_count = 0
-        for watch_uuid in watches_to_queue:
-            if watch_uuid in datastore.data['watching']:
-                try:
-                    worker_pool.queue_item_async_safe(
-                        update_q,
-                        queuedWatchMetaData.PrioritizedItem(priority=1, item={'uuid': watch_uuid})
-                    )
-                    queued_count += 1
-                    logger.debug(f"Queued watch for recheck: {watch_uuid}")
-                except Exception as e:
-                    logger.error(f"Failed to queue watch {watch_uuid}: {e}")
-            else:
-                logger.warning(f"Watch UUID not found in datastore: {watch_uuid}")
-
-        logger.success(f"Successfully queued {queued_count} watches for recheck")
-
-    # Step 4: Setup batch mode monitor (if -b was provided)
-    if batch_mode:
-        from changedetectionio.flask_app import update_q
-
-        # Safety check: Ensure Flask app is not already running on this port
-        # Batch mode should never run alongside the web server
-        import socket
-        test_socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
-
-        try:
-            # Try to bind to the configured host:port (no SO_REUSEADDR - strict check)
-            test_socket.bind((host, port))
-            test_socket.close()
-            logger.debug(f"Batch mode: Port {port} is available (Flask app not running)")
-        except OSError as e:
-            test_socket.close()
-            # errno 98 = EADDRINUSE (Linux)
-            # errno 48 = EADDRINUSE (macOS)
-            # errno 10048 = WSAEADDRINUSE (Windows)
-            if e.errno in (48, 98, 10048) or "Address already in use" in str(e) or "already in use" in str(e).lower():
-                logger.critical(f"ERROR: Batch mode cannot run - port {port} is already in use")
-                logger.critical(f"The Flask web server appears to be running on {host}:{port}")
-                logger.critical(f"Batch mode is designed for standalone operation (CI/CD, cron jobs, etc.)")
-                logger.critical(f"Please either stop the Flask web server, or use a different port with -p PORT")
-                sys.exit(1)
-            else:
-                # Some other socket error - log but continue (might be network configuration issue)
-                logger.warning(f"Port availability check failed with unexpected error: {e}")
-                logger.warning(f"Continuing with batch mode anyway - be aware of potential conflicts")
-
-        def queue_watches_for_recheck(datastore, iteration):
-            """Helper function to queue watches for recheck"""
-            watches_to_queue = []
-            if recheck_watches == 'all':
-                all_watches = list(datastore.data['watching'].keys())
-                if batch_mode and added_watch_uuids and iteration == 1:
-                    # Only exclude newly added watches on first iteration
-                    watches_to_queue = [uuid for uuid in all_watches if uuid not in added_watch_uuids]
-                else:
-                    watches_to_queue = all_watches
-                logger.info(f"Batch mode (iteration {iteration}): Queuing all {len(watches_to_queue)} watches")
-            elif recheck_watches:
-                watches_to_queue = recheck_watches
-                logger.info(f"Batch mode (iteration {iteration}): Queuing {len(watches_to_queue)} specific watches")
-
-            queued_count = 0
-            for watch_uuid in watches_to_queue:
-                if watch_uuid in datastore.data['watching']:
-                    try:
-                        worker_pool.queue_item_async_safe(
-                            update_q,
-                            queuedWatchMetaData.PrioritizedItem(priority=1, item={'uuid': watch_uuid})
-                        )
-                        queued_count += 1
-                    except Exception as e:
-                        logger.error(f"Failed to queue watch {watch_uuid}: {e}")
-                else:
-                    logger.warning(f"Watch UUID not found in datastore: {watch_uuid}")
-            logger.success(f"Batch mode (iteration {iteration}): Successfully queued {queued_count} watches")
-            return queued_count
-
-        def batch_mode_monitor():
-            """Monitor queue and workers, shutdown or repeat when work is complete"""
-            import time
-
-            # Track iterations if repeat mode is enabled
-            current_iteration = 1
-            total_iterations = recheck_repeat_count if recheck_watches and recheck_repeat_count > 1 else 1
-
-            if total_iterations > 1:
-                logger.info(f"Batch mode: Will repeat recheck {total_iterations} times")
-            else:
-                logger.info("Batch mode: Waiting for all queued items to complete...")
-
-            # Wait a bit for workers to start processing
-            time.sleep(3)
-
-            try:
-                while current_iteration <= total_iterations:
-                    logger.info(f"Batch mode: Waiting for iteration {current_iteration}/{total_iterations} to complete...")
-
-                    # Use the shared wait_for_all_checks function
-                    completed = worker_pool.wait_for_all_checks(update_q, timeout=300)
-
-                    if not completed:
-                        logger.warning(f"Batch mode: Iteration {current_iteration} timed out after 300 seconds")
-
-                    logger.success(f"Batch mode: Iteration {current_iteration}/{total_iterations} completed")
-
-                    # Check if we need to repeat
-                    if current_iteration < total_iterations:
-                        logger.info(f"Batch mode: Starting iteration {current_iteration + 1}...")
-                        current_iteration += 1
-
-                        # Re-queue watches for next iteration
-                        queue_watches_for_recheck(datastore, current_iteration)
-
-                        # Brief pause before continuing
-                        time.sleep(2)
-                    else:
-                        # All iterations complete
-                        logger.success(f"Batch mode: All {total_iterations} iterations completed, initiating shutdown")
-                        # Trigger shutdown
-                        import os, signal
-                        os.kill(os.getpid(), signal.SIGTERM)
-                        return
-
-            except Exception as e:
-                logger.error(f"Batch mode monitor error: {e}")
-                logger.error(f"Initiating emergency shutdown")
-                import os, signal
-                os.kill(os.getpid(), signal.SIGTERM)
-
-        # Start monitor in background thread
-        monitor_thread = threading.Thread(target=batch_mode_monitor, daemon=True, name="BatchModeMonitor")
-        monitor_thread.start()
-        logger.info("Batch mode enabled: Will exit after all queued items are processed")
-
     # Get the SocketIO instance from the Flask app (created in flask_app.py)
     from changedetectionio.flask_app import socketio_server
     global socketio
@@ -605,17 +275,19 @@ def main():
     else:
         logger.info("SIGUSR1 handler only registered on Linux, skipped.")
 
+    # Go into cleanup mode
+    if do_cleanup:
+        datastore.remove_unused_snapshots()
+
     app.config['datastore_path'] = datastore_path
 
 
     @app.context_processor
     def inject_template_globals():
-        return dict(right_sticky="v"+__version__,
+        return dict(right_sticky="v{}".format(datastore.data['version_tag']),
                     new_version_available=app.config['NEW_VERSION_AVAILABLE'],
                     has_password=datastore.data['settings']['application']['password'] != False,
-                    socket_io_enabled=datastore.data['settings']['application'].get('ui', {}).get('socket_io_enabled', True),
-                    all_paused=datastore.data['settings']['application'].get('all_paused', False),
-                    all_muted=datastore.data['settings']['application'].get('all_muted', False)
+                    socket_io_enabled=datastore.data['settings']['application']['ui'].get('socket_io_enabled', True)
                     )
 
     # Monitored websites will not receive a Referer header when a user clicks on an outgoing link.
@@ -637,43 +309,23 @@ def main():
     if os.getenv('USE_X_SETTINGS'):
         logger.info("USE_X_SETTINGS is ENABLED")
         from werkzeug.middleware.proxy_fix import ProxyFix
-        app.wsgi_app = ProxyFix(
-            app.wsgi_app,
-            x_for=1,      # X-Forwarded-For (client IP)
-            x_proto=1,    # X-Forwarded-Proto (http/https)
-            x_host=1,     # X-Forwarded-Host (original host)
-            x_port=1,     # X-Forwarded-Port (original port)
-            x_prefix=1    # X-Forwarded-Prefix (URL prefix)
-        )
+        app.wsgi_app = ProxyFix(app.wsgi_app, x_prefix=1, x_host=1)
 
 
-    # In batch mode, skip starting the HTTP server - just keep workers running
-    if batch_mode:
-        logger.info("Batch mode: Skipping HTTP server startup, workers will process queue")
-        logger.info("Batch mode: Main thread will wait for shutdown signal")
-        # Keep main thread alive until batch monitor triggers shutdown
-        try:
-            while True:
-                time.sleep(1)
-        except KeyboardInterrupt:
-            logger.info("Batch mode: Keyboard interrupt received")
-            pass
-    else:
-        # Normal mode: Start HTTP server
-        # SocketIO instance is already initialized in flask_app.py
-        if socketio_server:
-            if ssl_mode:
-                logger.success(f"SSL mode enabled, attempting to start with '{ssl_cert_file}' and '{ssl_privkey_file}' in {os.getcwd()}")
-                socketio.run(app, host=host, port=int(port), debug=False,
-                             ssl_context=(ssl_cert_file, ssl_privkey_file), allow_unsafe_werkzeug=True)
-            else:
-                socketio.run(app, host=host, port=int(port), debug=False, allow_unsafe_werkzeug=True)
+    # SocketIO instance is already initialized in flask_app.py
+    if socketio_server:
+        if ssl_mode:
+            logger.success(f"SSL mode enabled, attempting to start with '{ssl_cert_file}' and '{ssl_privkey_file}' in {os.getcwd()}")
+            socketio.run(app, host=host, port=int(port), debug=False,
+                         ssl_context=(ssl_cert_file, ssl_privkey_file), allow_unsafe_werkzeug=True)
         else:
-            # Run Flask app without Socket.IO if disabled
-            logger.info("Starting Flask app without Socket.IO server")
-            if ssl_mode:
-                logger.success(f"SSL mode enabled, attempting to start with '{ssl_cert_file}' and '{ssl_privkey_file}' in {os.getcwd()}")
-                app.run(host=host, port=int(port), debug=False,
-                        ssl_context=(ssl_cert_file, ssl_privkey_file))
-            else:
-                app.run(host=host, port=int(port), debug=False)
+            socketio.run(app, host=host, port=int(port), debug=False, allow_unsafe_werkzeug=True)
+    else:
+        # Run Flask app without Socket.IO if disabled
+        logger.info("Starting Flask app without Socket.IO server")
+        if ssl_mode:
+            logger.success(f"SSL mode enabled, attempting to start with '{ssl_cert_file}' and '{ssl_privkey_file}' in {os.getcwd()}")
+            app.run(host=host, port=int(port), debug=False,
+                    ssl_context=(ssl_cert_file, ssl_privkey_file))
+        else:
+            app.run(host=host, port=int(port), debug=False)

changedetectionio/api/Import.py

@@ -4,10 +4,6 @@ from flask import request
 from functools import wraps
 from . import auth, validate_openapi_request
 from ..validate_url import is_safe_valid_url
-import json
-
-# Number of URLs above which import switches to background processing
-IMPORT_SWITCH_TO_BACKGROUND_THRESHOLD = 20
 
 
 def default_content_type(content_type='text/plain'):
@@ -23,76 +19,6 @@ def default_content_type(content_type='text/plain'):
     return decorator
 
 
-def convert_query_param_to_type(value, schema_property):
-    """
-    Convert a query parameter string to the appropriate type based on schema definition.
-
-    Args:
-        value: String value from query parameter
-        schema_property: Schema property definition with 'type' or 'anyOf' field
-
-    Returns:
-        Converted value in the appropriate type
-
-    Supports both OpenAPI 3.1 formats:
-    - type: [string, 'null']  (array format)
-    - anyOf: [{type: string}, {type: null}]  (anyOf format)
-    """
-    prop_type = schema_property.get('type')
-
-    # Handle OpenAPI 3.1 type arrays: type: [string, 'null']
-    if isinstance(prop_type, list):
-        # Use the first non-null type from the array
-        for t in prop_type:
-            if t != 'null':
-                prop_type = t
-                break
-        else:
-            prop_type = None
-
-    # Handle anyOf schemas (older format)
-    elif 'anyOf' in schema_property:
-        # Use the first non-null type from anyOf
-        for option in schema_property['anyOf']:
-            if option.get('type') and option.get('type') != 'null':
-                prop_type = option.get('type')
-                break
-        else:
-            prop_type = None
-
-    # Handle array type (e.g., notification_urls)
-    if prop_type == 'array':
-        # Support both comma-separated and JSON array format
-        if value.startswith('['):
-            try:
-                return json.loads(value)
-            except json.JSONDecodeError:
-                return [v.strip() for v in value.split(',')]
-        return [v.strip() for v in value.split(',')]
-
-    # Handle object type (e.g., time_between_check, headers)
-    elif prop_type == 'object':
-        try:
-            return json.loads(value)
-        except json.JSONDecodeError:
-            raise ValueError(f"Invalid JSON object for field: {value}")
-
-    # Handle boolean type
-    elif prop_type == 'boolean':
-        return strtobool(value)
-
-    # Handle integer type
-    elif prop_type == 'integer':
-        return int(value)
-
-    # Handle number type (float)
-    elif prop_type == 'number':
-        return float(value)
-
-    # Default: return as string
-    return value
-
-
 class Import(Resource):
     def __init__(self, **kwargs):
         # datastore is a black box dependency
@@ -102,128 +28,40 @@ class Import(Resource):
     @default_content_type('text/plain') #3547 #3542
     @validate_openapi_request('importWatches')
     def post(self):
-        """Import a list of watched URLs with optional watch configuration."""
-        from . import get_watch_schema_properties
-        # Special parameters that are NOT watch configuration
-        special_params = {'tag', 'tag_uuids', 'dedupe', 'proxy'}
+        """Import a list of watched URLs."""
 
         extras = {}
 
-        # Handle special 'proxy' parameter
         if request.args.get('proxy'):
             plist = self.datastore.proxy_list
             if not request.args.get('proxy') in plist:
-                proxy_list_str = ', '.join(plist) if plist else 'none configured'
-                return f"Invalid proxy choice, currently supported proxies are '{proxy_list_str}'", 400
+                return "Invalid proxy choice, currently supported proxies are '{}'".format(', '.join(plist)), 400
             else:
                 extras['proxy'] = request.args.get('proxy')
 
-        # Handle special 'dedupe' parameter
         dedupe = strtobool(request.args.get('dedupe', 'true'))
 
-        # Handle special 'tag' and 'tag_uuids' parameters
         tags = request.args.get('tag')
         tag_uuids = request.args.get('tag_uuids')
 
         if tag_uuids:
             tag_uuids = tag_uuids.split(',')
 
-        # Extract ALL other query parameters as watch configuration
-        # Get schema from OpenAPI spec (replaces old schema_create_watch)
-        schema_properties = get_watch_schema_properties()
-        for param_name, param_value in request.args.items():
-            # Skip special parameters
-            if param_name in special_params:
-                continue
-
-            # Skip if not in schema (unknown parameter)
-            if param_name not in schema_properties:
-                return f"Unknown watch configuration parameter: {param_name}", 400
-
-            # Convert to appropriate type based on schema
-            try:
-                converted_value = convert_query_param_to_type(param_value, schema_properties[param_name])
-                extras[param_name] = converted_value
-            except (ValueError, json.JSONDecodeError) as e:
-                return f"Invalid value for parameter '{param_name}': {str(e)}", 400
-
-        # Validate processor if provided
-        if 'processor' in extras:
-            from changedetectionio.processors import available_processors
-            available = [p[0] for p in available_processors()]
-            if extras['processor'] not in available:
-                return f"Invalid processor '{extras['processor']}'. Available processors: {', '.join(available)}", 400
-
-        # Validate fetch_backend if provided
-        if 'fetch_backend' in extras:
-            from changedetectionio.content_fetchers import available_fetchers
-            available = [f[0] for f in available_fetchers()]
-            # Also allow 'system' and extra_browser_* patterns
-            is_valid = (
-                extras['fetch_backend'] == 'system' or
-                extras['fetch_backend'] in available or
-                extras['fetch_backend'].startswith('extra_browser_')
-            )
-            if not is_valid:
-                return f"Invalid fetch_backend '{extras['fetch_backend']}'. Available: system, {', '.join(available)}", 400
-
-        # Validate notification_urls if provided
-        if 'notification_urls' in extras:
-            from wtforms import ValidationError
-            from changedetectionio.api.Notifications import validate_notification_urls
-            try:
-                validate_notification_urls(extras['notification_urls'])
-            except ValidationError as e:
-                return f"Invalid notification_urls: {str(e)}", 400
-
         urls = request.get_data().decode('utf8').splitlines()
-        # Clean and validate URLs upfront
-        urls_to_import = []
+        added = []
         for url in urls:
             url = url.strip()
             if not len(url):
                 continue
 
-            # Validate URL
+            # If hosts that only contain alphanumerics are allowed ("localhost" for example)
             if not is_safe_valid_url(url):
                 return f"Invalid or unsupported URL - {url}", 400
 
-            # Check for duplicates if dedupe is enabled
             if dedupe and self.datastore.url_exists(url):
                 continue
 
-            urls_to_import.append(url)
+            new_uuid = self.datastore.add_watch(url=url, extras=extras, tag=tags, tag_uuids=tag_uuids)
+            added.append(new_uuid)
 
-        # For small imports, process synchronously for immediate feedback
-        if len(urls_to_import) < IMPORT_SWITCH_TO_BACKGROUND_THRESHOLD:
-            added = []
-            for url in urls_to_import:
-                new_uuid = self.datastore.add_watch(url=url, extras=extras, tag=tags, tag_uuids=tag_uuids)
-                added.append(new_uuid)
-            return added, 200
-
-        # For large imports (>= 20), process in background thread
-        else:
-            import threading
-            from loguru import logger
-
-            def import_watches_background():
-                """Background thread to import watches - discarded after completion."""
-                try:
-                    added_count = 0
-                    for url in urls_to_import:
-                        try:
-                            self.datastore.add_watch(url=url, extras=extras, tag=tags, tag_uuids=tag_uuids)
-                            added_count += 1
-                        except Exception as e:
-                            logger.error(f"Error importing URL {url}: {e}")
-
-                    logger.info(f"Background import complete: {added_count} watches created")
-                except Exception as e:
-                    logger.error(f"Error in background import: {e}")
-
-            # Start background thread and return immediately
-            thread = threading.Thread(target=import_watches_background, daemon=True, name="ImportWatches-Background")
-            thread.start()
-
-            return {'status': f'Importing {len(urls_to_import)} URLs in background', 'count': len(urls_to_import)}, 202
+        return added

changedetectionio/api/Notifications.py

@@ -1,6 +1,8 @@
+from flask_expects_json import expects_json
 from flask_restful import Resource, abort
 from flask import request
 from . import auth, validate_openapi_request
+from . import schema_create_notification_urls, schema_delete_notification_urls
 
 class Notifications(Resource):
     def __init__(self, **kwargs):
@@ -20,6 +22,7 @@ class Notifications(Resource):
     
     @auth.check_token
     @validate_openapi_request('addNotifications')
+    @expects_json(schema_create_notification_urls)
     def post(self):
         """Create Notification URLs."""
 
@@ -47,6 +50,7 @@ class Notifications(Resource):
     
     @auth.check_token
     @validate_openapi_request('replaceNotifications')
+    @expects_json(schema_create_notification_urls)
     def put(self):
         """Replace Notification URLs."""
         json_data = request.get_json()
@@ -63,12 +67,13 @@ class Notifications(Resource):
 
         clean_urls = [url.strip() for url in notification_urls if isinstance(url, str)]
         self.datastore.data['settings']['application']['notification_urls'] = clean_urls
-        self.datastore.commit()
+        self.datastore.needs_write = True
 
         return {'notification_urls': clean_urls}, 200
         
     @auth.check_token
     @validate_openapi_request('deleteNotifications')
+    @expects_json(schema_delete_notification_urls)
     def delete(self):
         """Delete Notification URLs."""
 
@@ -90,7 +95,7 @@ class Notifications(Resource):
             abort(400, message="No matching notification URLs found.")
 
         self.datastore.data['settings']['application']['notification_urls'] = notification_urls
-        self.datastore.commit()
+        self.datastore.needs_write = True
 
         return 'OK', 204
     

changedetectionio/api/Tags.py

@@ -1,13 +1,13 @@
 from changedetectionio import queuedWatchMetaData
-from changedetectionio import worker_pool
+from changedetectionio import worker_handler
+from flask_expects_json import expects_json
 from flask_restful import abort, Resource
-from loguru import logger
 
-import threading
 from flask import request
 from . import auth
 
-from . import validate_openapi_request
+# Import schemas from __init__.py
+from . import schema_tag, schema_create_tag, schema_update_tag, validate_openapi_request
 
 
 class Tag(Resource):
@@ -22,70 +22,33 @@ class Tag(Resource):
     @validate_openapi_request('getTag')
     def get(self, uuid):
         """Get data for a single tag/group, toggle notification muting, or recheck all."""
-        tag = self.datastore.data['settings']['application']['tags'].get(uuid)
+        from copy import deepcopy
+        tag = deepcopy(self.datastore.data['settings']['application']['tags'].get(uuid))
         if not tag:
             abort(404, message=f'No tag exists with the UUID of {uuid}')
 
         if request.args.get('recheck'):
-            # Recheck all watches with this tag, including muted
-            # First collect watches to queue
-            watches_to_queue = []
+            # Recheck all, including muted
+            # Get most overdue first
+            i=0
             for k in sorted(self.datastore.data['watching'].items(), key=lambda item: item[1].get('last_checked', 0)):
                 watch_uuid = k[0]
                 watch = k[1]
-                if not watch['paused'] and tag['uuid'] in watch['tags']:
-                    watches_to_queue.append(watch_uuid)
+                if not watch['paused'] and tag['uuid'] not in watch['tags']:
+                    continue
+                worker_handler.queue_item_async_safe(self.update_q, queuedWatchMetaData.PrioritizedItem(priority=1, item={'uuid': watch_uuid}))
+                i+=1
 
-            # If less than 20 watches, queue synchronously for immediate feedback
-            if len(watches_to_queue) < 20:
-                for watch_uuid in watches_to_queue:
-                    worker_pool.queue_item_async_safe(self.update_q, queuedWatchMetaData.PrioritizedItem(priority=1, item={'uuid': watch_uuid}))
-                return {'status': f'OK, queued {len(watches_to_queue)} watches for rechecking'}, 200
-            else:
-                # 20+ watches - queue in background thread to avoid blocking API response
-                def queue_watches_background():
-                    """Background thread to queue watches - discarded after completion."""
-                    try:
-                        for watch_uuid in watches_to_queue:
-                            worker_pool.queue_item_async_safe(self.update_q, queuedWatchMetaData.PrioritizedItem(priority=1, item={'uuid': watch_uuid}))
-                        logger.info(f"Background queueing complete for tag {tag['uuid']}: {len(watches_to_queue)} watches queued")
-                    except Exception as e:
-                        logger.error(f"Error in background queueing for tag {tag['uuid']}: {e}")
-
-                # Start background thread and return immediately
-                thread = threading.Thread(target=queue_watches_background, daemon=True, name=f"QueueTag-{tag['uuid'][:8]}")
-                thread.start()
-
-                return {'status': f'OK, queueing {len(watches_to_queue)} watches in background'}, 202
+            return f"OK, {i} watches queued", 200
 
         if request.args.get('muted', '') == 'muted':
-            tag['notification_muted'] = True
-            tag.commit()
+            self.datastore.data['settings']['application']['tags'][uuid]['notification_muted'] = True
             return "OK", 200
         elif request.args.get('muted', '') == 'unmuted':
-            tag['notification_muted'] = False
-            tag.commit()
+            self.datastore.data['settings']['application']['tags'][uuid]['notification_muted'] = False
             return "OK", 200
 
-        # Filter out Watch-specific runtime fields that don't apply to Tags (yet)
-        # TODO: Future enhancement - aggregate these values from all Watches that have this tag:
-        #   - check_count: sum of all watches' check_count
-        #   - last_checked: most recent last_checked from all watches
-        #   - last_changed: most recent last_changed from all watches
-        #   - consecutive_filter_failures: count of watches with failures
-        #   - etc.
-        # These come from watch_base inheritance but currently have no meaningful value for Tags
-        watch_only_fields = {
-            'browser_steps_last_error_step', 'check_count', 'consecutive_filter_failures',
-            'content-type', 'fetch_time', 'last_changed', 'last_checked', 'last_error',
-            'last_notification_error', 'last_viewed', 'notification_alert_count',
-            'page_title', 'previous_md5', 'remote_server_reply'
-        }
-
-        # Create clean tag dict without Watch-specific fields
-        clean_tag = {k: v for k, v in tag.items() if k not in watch_only_fields}
-
-        return clean_tag
+        return tag
 
     @auth.check_token
     @validate_openapi_request('deleteTag')
@@ -96,84 +59,38 @@ class Tag(Resource):
 
         # Delete the tag, and any tag reference
         del self.datastore.data['settings']['application']['tags'][uuid]
-
+        
         # Remove tag from all watches
         for watch_uuid, watch in self.datastore.data['watching'].items():
             if watch.get('tags') and uuid in watch['tags']:
                 watch['tags'].remove(uuid)
-                watch.commit()
 
         return 'OK', 204
 
     @auth.check_token
     @validate_openapi_request('updateTag')
+    @expects_json(schema_update_tag)
     def put(self, uuid):
         """Update tag information."""
         tag = self.datastore.data['settings']['application']['tags'].get(uuid)
         if not tag:
             abort(404, message='No tag exists with the UUID of {}'.format(uuid))
 
-        # Make a mutable copy of request.json for modification
-        json_data = dict(request.json)
-
-        # Validate notification_urls if provided
-        if 'notification_urls' in json_data:
-            from wtforms import ValidationError
-            from changedetectionio.api.Notifications import validate_notification_urls
-            try:
-                notification_urls = json_data.get('notification_urls', [])
-                validate_notification_urls(notification_urls)
-            except ValidationError as e:
-                return str(e), 400
-
-        # Filter out readOnly fields (extracted from OpenAPI spec Tag schema)
-        # These are system-managed fields that should never be user-settable
-        from . import get_readonly_tag_fields
-        readonly_fields = get_readonly_tag_fields()
-
-        # Tag model inherits from watch_base but has no @property attributes of its own
-        # So we only need to filter readOnly fields
-        for field in readonly_fields:
-            json_data.pop(field, None)
-
-        # Validate remaining fields - reject truly unknown fields
-        # Get valid fields from Tag schema
-        from . import get_tag_schema_properties
-        valid_fields = set(get_tag_schema_properties().keys())
-
-        # Check for unknown fields
-        unknown_fields = set(json_data.keys()) - valid_fields
-        if unknown_fields:
-            return f"Unknown field(s): {', '.join(sorted(unknown_fields))}", 400
-
-        tag.update(json_data)
-        tag.commit()
-
-        # Clear checksums for all watches using this tag to force reprocessing
-        # Tag changes affect inherited configuration
-        cleared_count = self.datastore.clear_checksums_for_tag(uuid)
-        logger.info(f"Tag {uuid} updated via API, cleared {cleared_count} watch checksums")
+        tag.update(request.json)
+        self.datastore.needs_write_urgent = True
 
         return "OK", 200
 
 
     @auth.check_token
     @validate_openapi_request('createTag')
+    # Only cares for {'title': 'xxxx'}
     def post(self):
         """Create a single tag/group."""
 
         json_data = request.get_json()
         title = json_data.get("title",'').strip()
 
-        # Validate that only valid fields are provided
-        # Get valid fields from Tag schema
-        from . import get_tag_schema_properties
-        valid_fields = set(get_tag_schema_properties().keys())
-
-        # Check for unknown fields
-        unknown_fields = set(json_data.keys()) - valid_fields
-        if unknown_fields:
-            return f"Unknown field(s): {', '.join(sorted(unknown_fields))}", 400
 
         new_uuid = self.datastore.add_tag(title=title)
         if new_uuid:

changedetectionio/api/Watch.py

@@ -1,18 +1,18 @@
 import os
-import threading
 
 from changedetectionio.validate_url import is_safe_valid_url
-from changedetectionio.favicon_utils import get_favicon_mime_type
 
 from . import auth
 from changedetectionio import queuedWatchMetaData, strtobool
-from changedetectionio import worker_pool
+from changedetectionio import worker_handler
 from flask import request, make_response, send_from_directory
+from flask_expects_json import expects_json
 from flask_restful import abort, Resource
 from loguru import logger
 import copy
 
-from . import validate_openapi_request, get_readonly_watch_fields
+# Import schemas from __init__.py
+from . import schema, schema_create_watch, schema_update_watch, validate_openapi_request
 from ..notification import valid_notification_formats
 from ..notification.handler import newline_re
 
@@ -64,46 +64,43 @@ class Watch(Resource):
     @validate_openapi_request('getWatch')
     def get(self, uuid):
         """Get information about a single watch, recheck, pause, or mute."""
-        # Get watch reference first (for pause/mute operations)
-        watch_obj = self.datastore.data['watching'].get(uuid)
-        if not watch_obj:
+        import time
+        from copy import deepcopy
+        watch = None
+        for _ in range(20):
+            try:
+                watch = deepcopy(self.datastore.data['watching'].get(uuid))
+                break
+            except RuntimeError:
+                # Incase dict changed, try again
+                time.sleep(0.01)
+
+        if not watch:
             abort(404, message='No watch exists with the UUID of {}'.format(uuid))
 
-        # Create a dict copy for JSON response (with lock for thread safety)
-        # This is much faster than deepcopy and doesn't copy the datastore reference
-        # WARNING: dict() is a SHALLOW copy - nested dicts are shared with original!
-        # Only safe because we only ADD scalar properties (line 97-101), never modify nested dicts
-        # If you need to modify nested dicts, use: from copy import deepcopy; watch = deepcopy(dict(watch_obj))
-        with self.datastore.lock:
-            watch = dict(watch_obj)
-
         if request.args.get('recheck'):
-            worker_pool.queue_item_async_safe(self.update_q, queuedWatchMetaData.PrioritizedItem(priority=1, item={'uuid': uuid}))
+            worker_handler.queue_item_async_safe(self.update_q, queuedWatchMetaData.PrioritizedItem(priority=1, item={'uuid': uuid}))
             return "OK", 200
         if request.args.get('paused', '') == 'paused':
-            watch_obj.pause()
-            watch_obj.commit()
+            self.datastore.data['watching'].get(uuid).pause()
             return "OK", 200
         elif request.args.get('paused', '') == 'unpaused':
-            watch_obj.unpause()
-            watch_obj.commit()
+            self.datastore.data['watching'].get(uuid).unpause()
             return "OK", 200
         if request.args.get('muted', '') == 'muted':
-            watch_obj.mute()
-            watch_obj.commit()
+            self.datastore.data['watching'].get(uuid).mute()
             return "OK", 200
         elif request.args.get('muted', '') == 'unmuted':
-            watch_obj.unmute()
-            watch_obj.commit()
+            self.datastore.data['watching'].get(uuid).unmute()
             return "OK", 200
 
         # Return without history, get that via another API call
         # Properties are not returned as a JSON, so add the required props manually
-        watch['history_n'] = watch_obj.history_n
+        watch['history_n'] = watch.history_n
         # attr .last_changed will check for the last written text snapshot on change
-        watch['last_changed'] = watch_obj.last_changed
-        watch['viewed'] = watch_obj.viewed
-        watch['link'] = watch_obj.link,
+        watch['last_changed'] = watch.last_changed
+        watch['viewed'] = watch.viewed
+        watch['link'] = watch.link,
 
         return watch
 
@@ -119,6 +116,7 @@ class Watch(Resource):
 
     @auth.check_token
     @validate_openapi_request('updateWatch')
+    @expects_json(schema_update_watch)
     def put(self, uuid):
         """Update watch information."""
         watch = self.datastore.data['watching'].get(uuid)
@@ -127,86 +125,72 @@ class Watch(Resource):
 
         if request.json.get('proxy'):
             plist = self.datastore.proxy_list
-            if not plist or request.json.get('proxy') not in plist:
-                proxy_list_str = ', '.join(plist) if plist else 'none configured'
-                return f"Invalid proxy choice, currently supported proxies are '{proxy_list_str}'", 400
+            if not request.json.get('proxy') in plist:
+                return "Invalid proxy choice, currently supported proxies are '{}'".format(', '.join(plist)), 400
 
         # Validate time_between_check when not using defaults
         validation_error = validate_time_between_check_required(request.json)
         if validation_error:
             return validation_error, 400
 
-        # Validate notification_urls if provided
-        if 'notification_urls' in request.json:
-            from wtforms import ValidationError
-            from changedetectionio.api.Notifications import validate_notification_urls
-            try:
-                notification_urls = request.json.get('notification_urls', [])
-                validate_notification_urls(notification_urls)
-            except ValidationError as e:
-                return str(e), 400
-
-        # XSS etc protection - validate URL if it's being updated
-        if 'url' in request.json:
-            new_url = request.json.get('url')
-
-            # URL must be a non-empty string
-            if new_url is None:
-                return "URL cannot be null", 400
-
-            if not isinstance(new_url, str):
-                return "URL must be a string", 400
-
-            if not new_url.strip():
-                return "URL cannot be empty or whitespace only", 400
-
-            if not is_safe_valid_url(new_url.strip()):
-                return "Invalid or unsupported URL format. URL must use http://, https://, or ftp:// protocol", 400
+        # XSS etc protection
+        if request.json.get('url') and not is_safe_valid_url(request.json.get('url')):
+            return "Invalid URL", 400
 
         # Handle processor-config-* fields separately (save to JSON, not datastore)
         from changedetectionio import processors
+        processor_config_data = {}
+        regular_data = {}
 
-        # Make a mutable copy of request.json for modification
-        json_data = dict(request.json)
-
-        # Extract and remove processor config fields from json_data
-        processor_config_data = processors.extract_processor_config_from_form_data(json_data)
-
-        # Filter out readOnly fields (extracted from OpenAPI spec Watch schema)
-        # These are system-managed fields that should never be user-settable
-        readonly_fields = get_readonly_watch_fields()
-
-        # Also filter out @property attributes (computed/derived values from the model)
-        # These are not stored and should be ignored in PUT requests
-        from changedetectionio.model.Watch import model as WatchModel
-        property_fields = WatchModel.get_property_names()
-
-        # Combine both sets of fields to ignore
-        fields_to_ignore = readonly_fields | property_fields
-
-        # Remove all ignored fields from update data
-        for field in fields_to_ignore:
-            json_data.pop(field, None)
-
-        # Validate remaining fields - reject truly unknown fields
-        # Get valid fields from WatchBase schema
-        from . import get_watch_schema_properties
-        valid_fields = set(get_watch_schema_properties().keys())
-
-        # Also allow last_viewed (explicitly defined in UpdateWatch schema)
-        valid_fields.add('last_viewed')
-
-        # Check for unknown fields
-        unknown_fields = set(json_data.keys()) - valid_fields
-        if unknown_fields:
-            return f"Unknown field(s): {', '.join(sorted(unknown_fields))}", 400
+        for key, value in request.json.items():
+            if key.startswith('processor_config_'):
+                config_key = key.replace('processor_config_', '')
+                if value:  # Only save non-empty values
+                    processor_config_data[config_key] = value
+            else:
+                regular_data[key] = value
 
         # Update watch with regular (non-processor-config) fields
-        watch.update(json_data)
-        watch.commit()
+        watch.update(regular_data)
 
-        # Save processor config to JSON file
-        processors.save_processor_config(self.datastore, uuid, processor_config_data)
+        # Save processor config to JSON file if any config data exists
+        if processor_config_data:
+            try:
+                processor_name = request.json.get('processor', watch.get('processor'))
+                if processor_name:
+                    # Create a processor instance to access config methods
+                    from changedetectionio.processors import difference_detection_processor
+                    processor_instance = difference_detection_processor(self.datastore, uuid)
+                    # Use processor name as filename so each processor keeps its own config
+                    config_filename = f'{processor_name}.json'
+                    processor_instance.update_extra_watch_config(config_filename, processor_config_data)
+                    logger.debug(f"API: Saved processor config to {config_filename}: {processor_config_data}")
+
+                    # Call optional edit_hook if processor has one
+                    try:
+                        import importlib
+                        edit_hook_module_name = f'changedetectionio.processors.{processor_name}.edit_hook'
+
+                        try:
+                            edit_hook = importlib.import_module(edit_hook_module_name)
+                            logger.debug(f"API: Found edit_hook module for {processor_name}")
+
+                            if hasattr(edit_hook, 'on_config_save'):
+                                logger.info(f"API: Calling edit_hook.on_config_save for {processor_name}")
+                                # Call hook and get updated config
+                                updated_config = edit_hook.on_config_save(watch, processor_config_data, self.datastore)
+                                # Save updated config back to file
+                                processor_instance.update_extra_watch_config(config_filename, updated_config)
+                                logger.info(f"API: Edit hook updated config: {updated_config}")
+                            else:
+                                logger.debug(f"API: Edit hook module found but no on_config_save function")
+                        except ModuleNotFoundError:
+                            logger.debug(f"API: No edit_hook module for processor {processor_name} (this is normal)")
+                    except Exception as hook_error:
+                        logger.error(f"API: Edit hook error (non-fatal): {hook_error}", exc_info=True)
+
+            except Exception as e:
+                logger.error(f"API: Failed to save processor config: {e}")
 
         return "OK", 200
 
@@ -247,10 +231,6 @@ class WatchSingleHistory(Resource):
         if timestamp == 'latest':
             timestamp = list(watch.history.keys())[-1]
 
-        # Validate that the timestamp exists in history
-        if timestamp not in watch.history:
-            abort(404, message=f"No history snapshot found for timestamp '{timestamp}'")
-
         if request.args.get('html'):
             content = watch.get_fetched_html(timestamp)
             if content:
@@ -399,11 +379,18 @@ class WatchFavicon(Resource):
 
         favicon_filename = watch.get_favicon_filename()
         if favicon_filename:
-            # Use cached MIME type detection
-            filepath = os.path.join(watch.data_dir, favicon_filename)
-            mime = get_favicon_mime_type(filepath)
+            try:
+                import magic
+                mime = magic.from_file(
+                    os.path.join(watch.watch_data_dir, favicon_filename),
+                    mime=True
+                )
+            except ImportError:
+                # Fallback, no python-magic
+                import mimetypes
+                mime, encoding = mimetypes.guess_type(favicon_filename)
 
-            response = make_response(send_from_directory(watch.data_dir, favicon_filename))
+            response = make_response(send_from_directory(watch.watch_data_dir, favicon_filename))
             response.headers['Content-type'] = mime
             response.headers['Cache-Control'] = 'max-age=300, must-revalidate'  # Cache for 5 minutes, then revalidate
             return response
@@ -419,6 +406,7 @@ class CreateWatch(Resource):
 
     @auth.check_token
     @validate_openapi_request('createWatch')
+    @expects_json(schema_create_watch)
     def post(self):
         """Create a single watch."""
 
@@ -430,33 +418,16 @@ class CreateWatch(Resource):
 
         if json_data.get('proxy'):
             plist = self.datastore.proxy_list
-            if not plist or json_data.get('proxy') not in plist:
-                proxy_list_str = ', '.join(plist) if plist else 'none configured'
-                return f"Invalid proxy choice, currently supported proxies are '{proxy_list_str}'", 400
+            if not json_data.get('proxy') in plist:
+                return "Invalid proxy choice, currently supported proxies are '{}'".format(', '.join(plist)), 400
 
         # Validate time_between_check when not using defaults
         validation_error = validate_time_between_check_required(json_data)
         if validation_error:
             return validation_error, 400
 
-        # Validate notification_urls if provided
-        if 'notification_urls' in json_data:
-            from wtforms import ValidationError
-            from changedetectionio.api.Notifications import validate_notification_urls
-            try:
-                notification_urls = json_data.get('notification_urls', [])
-                validate_notification_urls(notification_urls)
-            except ValidationError as e:
-                return str(e), 400
-
-        # Handle processor-config-* fields separately (save to JSON, not watch)
-        from changedetectionio import processors
-
         extras = copy.deepcopy(json_data)
 
-        # Extract and remove processor config fields from extras
-        processor_config_data = processors.extract_processor_config_from_form_data(extras)
-
         # Because we renamed 'tag' to 'tags' but don't want to change the API (can do this in v2 of the API)
         tags = None
         if extras.get('tag'):
@@ -466,25 +437,11 @@ class CreateWatch(Resource):
         del extras['url']
 
         new_uuid = self.datastore.add_watch(url=url, extras=extras, tag=tags)
-
-        # Save processor config to separate JSON file
-        if new_uuid and processor_config_data:
-            processors.save_processor_config(self.datastore, new_uuid, processor_config_data)
         if new_uuid:
 # Dont queue because the scheduler will check that it hasnt been checked before anyway
-#            worker_pool.queue_item_async_safe(self.update_q, queuedWatchMetaData.PrioritizedItem(priority=1, item={'uuid': new_uuid}))
+#            worker_handler.queue_item_async_safe(self.update_q, queuedWatchMetaData.PrioritizedItem(priority=1, item={'uuid': new_uuid}))
             return {'uuid': new_uuid}, 201
         else:
-            # Check if it was a limit issue
-            page_watch_limit = os.getenv('PAGE_WATCH_LIMIT')
-            if page_watch_limit:
-                try:
-                    page_watch_limit = int(page_watch_limit)
-                    current_watch_count = len(self.datastore.data['watching'])
-                    if current_watch_count >= page_watch_limit:
-                        return f"Watch limit reached ({current_watch_count}/{page_watch_limit} watches). Cannot add more watches.", 429
-                except ValueError:
-                    pass
             return "Invalid or unsupported URL", 400
 
     @auth.check_token
@@ -506,65 +463,14 @@ class CreateWatch(Resource):
                 'last_error': watch['last_error'],
                 'link': watch.link,
                 'page_title': watch['page_title'],
-                'tags': [*tags],  # Unpack dict keys to list (can't use list() since variable named 'list')
                 'title': watch['title'],
                 'url': watch['url'],
                 'viewed': watch.viewed
             }
 
         if request.args.get('recheck_all'):
-            # Collect all watches to queue
-            watches_to_queue = self.datastore.data['watching'].keys()
-
-            # If less than 20 watches, queue synchronously for immediate feedback
-            if len(watches_to_queue) < 20:
-                # Get already queued/running UUIDs once (efficient)
-                queued_uuids = set(self.update_q.get_queued_uuids())
-                running_uuids = set(worker_pool.get_running_uuids())
-
-                # Filter out watches that are already queued or running
-                watches_to_queue_filtered = [
-                    uuid for uuid in watches_to_queue
-                    if uuid not in queued_uuids and uuid not in running_uuids
-                ]
-
-                # Queue only the filtered watches
-                for uuid in watches_to_queue_filtered:
-                    worker_pool.queue_item_async_safe(self.update_q, queuedWatchMetaData.PrioritizedItem(priority=1, item={'uuid': uuid}))
-
-                # Provide feedback about skipped watches
-                skipped_count = len(watches_to_queue) - len(watches_to_queue_filtered)
-                if skipped_count > 0:
-                    return {'status': f'OK, queued {len(watches_to_queue_filtered)} watches for rechecking ({skipped_count} already queued or running)'}, 200
-                else:
-                    return {'status': f'OK, queued {len(watches_to_queue_filtered)} watches for rechecking'}, 200
-            else:
-                # 20+ watches - queue in background thread to avoid blocking API response
-                # Capture queued/running state before background thread
-                queued_uuids = set(self.update_q.get_queued_uuids())
-                running_uuids = set(worker_pool.get_running_uuids())
-
-                def queue_all_watches_background():
-                    """Background thread to queue all watches - discarded after completion."""
-                    try:
-                        queued_count = 0
-                        skipped_count = 0
-                        for uuid in watches_to_queue:
-                            # Check if already queued or running (state captured at start)
-                            if uuid not in queued_uuids and uuid not in running_uuids:
-                                worker_pool.queue_item_async_safe(self.update_q, queuedWatchMetaData.PrioritizedItem(priority=1, item={'uuid': uuid}))
-                                queued_count += 1
-                            else:
-                                skipped_count += 1
-
-                        logger.info(f"Background queueing complete: {queued_count} watches queued, {skipped_count} skipped (already queued/running)")
-                    except Exception as e:
-                        logger.error(f"Error in background queueing all watches: {e}")
-
-                # Start background thread and return immediately
-                thread = threading.Thread(target=queue_all_watches_background, daemon=True, name="QueueAllWatches-Background")
-                thread.start()
-
-                return {'status': f'OK, queueing {len(watches_to_queue)} watches in background'}, 202
+            for uuid in self.datastore.data['watching'].keys():
+                worker_handler.queue_item_async_safe(self.update_q, queuedWatchMetaData.PrioritizedItem(priority=1, item={'uuid': uuid}))
+            return {'status': "OK"}, 200
 
         return list, 200

changedetectionio/api/__init__.py

@@ -1,6 +1,33 @@
+import copy
 import functools
 from flask import request, abort
 from loguru import logger
+from . import api_schema
+from ..model import watch_base
+
+# Build a JSON Schema atleast partially based on our Watch model
+watch_base_config = watch_base()
+schema = api_schema.build_watch_json_schema(watch_base_config)
+
+schema_create_watch = copy.deepcopy(schema)
+schema_create_watch['required'] = ['url']
+del schema_create_watch['properties']['last_viewed']
+
+schema_update_watch = copy.deepcopy(schema)
+schema_update_watch['additionalProperties'] = False
+
+# Tag schema is also based on watch_base since Tag inherits from it
+schema_tag = copy.deepcopy(schema)
+schema_create_tag = copy.deepcopy(schema_tag)
+schema_create_tag['required'] = ['title']
+schema_update_tag = copy.deepcopy(schema_tag)
+schema_update_tag['additionalProperties'] = False
+
+schema_notification_urls = copy.deepcopy(schema)
+schema_create_notification_urls = copy.deepcopy(schema_notification_urls)
+schema_create_notification_urls['required'] = ['notification_urls']
+schema_delete_notification_urls = copy.deepcopy(schema_notification_urls)
+schema_delete_notification_urls['required'] = ['notification_urls']
 
 @functools.cache
 def get_openapi_spec():
@@ -19,79 +46,6 @@ def get_openapi_spec():
     _openapi_spec = OpenAPI.from_dict(spec_dict)
     return _openapi_spec
 
-@functools.cache
-def get_openapi_schema_dict():
-    """
-    Get the raw OpenAPI spec dictionary for schema access.
-
-    Used by Import endpoint to validate and convert query parameters.
-    Returns the YAML dict directly (not the OpenAPI object).
-    """
-    import os
-    import yaml
-
-    spec_path = os.path.join(os.path.dirname(__file__), '../../docs/api-spec.yaml')
-    if not os.path.exists(spec_path):
-        spec_path = os.path.join(os.path.dirname(__file__), '../docs/api-spec.yaml')
-
-    with open(spec_path, 'r', encoding='utf-8') as f:
-        return yaml.safe_load(f)
-
-@functools.cache
-def _resolve_schema_properties(schema_name):
-    """
-    Generic helper to resolve schema properties, including allOf inheritance.
-
-    Args:
-        schema_name: Name of the schema (e.g., 'WatchBase', 'Watch', 'Tag')
-
-    Returns:
-        dict: All properties including inherited ones from $ref schemas
-    """
-    spec_dict = get_openapi_schema_dict()
-    schema = spec_dict['components']['schemas'].get(schema_name, {})
-
-    properties = {}
-
-    # Handle allOf (schema inheritance)
-    if 'allOf' in schema:
-        for item in schema['allOf']:
-            # Resolve $ref to parent schema
-            if '$ref' in item:
-                ref_path = item['$ref'].split('/')[-1]
-                ref_schema = spec_dict['components']['schemas'].get(ref_path, {})
-                properties.update(ref_schema.get('properties', {}))
-            # Add schema-specific properties
-            if 'properties' in item:
-                properties.update(item['properties'])
-    else:
-        # Direct properties (no inheritance)
-        properties = schema.get('properties', {})
-
-    return properties
-
-
-@functools.cache
-def get_watch_schema_properties():
-    """
-    Extract watch schema properties from OpenAPI spec for Import endpoint.
-
-    Returns WatchBase properties (all writable Watch fields).
-    """
-    return _resolve_schema_properties('WatchBase')
-
-# Import readonly field utilities from shared module (avoids circular dependencies with model layer)
-from changedetectionio.model.schema_utils import get_readonly_watch_fields, get_readonly_tag_fields
-
-@functools.cache
-def get_tag_schema_properties():
-    """
-    Extract Tag schema properties from OpenAPI spec.
-
-    Returns WatchBase properties + Tag-specific properties (overrides_watch).
-    """
-    return _resolve_schema_properties('Tag')
-
 def validate_openapi_request(operation_id):
     """Decorator to validate incoming requests against OpenAPI spec."""
     def decorator(f):
@@ -103,7 +57,6 @@ def validate_openapi_request(operation_id):
                 if request.method.upper() != 'GET':
                     # Lazy import - only loaded when actually validating a request
                     from openapi_core.contrib.flask import FlaskOpenAPIRequest
-                    from openapi_core.templating.paths.exceptions import ServerNotFound, PathNotFound, PathError
 
                     spec = get_openapi_spec()
                     openapi_request = FlaskOpenAPIRequest(request)
@@ -111,29 +64,8 @@ def validate_openapi_request(operation_id):
                     if result.errors:
                         error_details = []
                         for error in result.errors:
-                            # Skip path/server validation errors for reverse proxy compatibility
-                            # Flask routing already validates that endpoints exist (returns 404 if not).
-                            # OpenAPI validation here is primarily for request body schema validation.
-                            # When behind nginx/reverse proxy, URLs may have path prefixes that don't
-                            # match the OpenAPI server definitions, causing false positives.
-                            if isinstance(error, PathError):
-                                logger.debug(f"API Call - Skipping path/server validation (delegated to Flask): {error}")
-                                continue
-
-                            error_str = str(error)
-                            # Extract detailed schema errors from __cause__
-                            if hasattr(error, '__cause__') and hasattr(error.__cause__, 'schema_errors'):
-                                for schema_error in error.__cause__.schema_errors:
-                                    field = '.'.join(str(p) for p in schema_error.path) if schema_error.path else 'body'
-                                    msg = schema_error.message if hasattr(schema_error, 'message') else str(schema_error)
-                                    error_details.append(f"{field}: {msg}")
-                            else:
-                                error_details.append(error_str)
-
-                        # Only raise if we have actual validation errors (not path/server issues)
-                        if error_details:
-                            logger.error(f"API Call - Validation failed: {'; '.join(error_details)}")
-                            raise BadRequest(f"Validation failed: {'; '.join(error_details)}")
+                            error_details.append(str(error))
+                        raise BadRequest(f"OpenAPI validation failed: {error_details}")
             except BadRequest:
                 # Re-raise BadRequest exceptions (validation failures)
                 raise

changedetectionio/api/api_schema.py

@@ -0,0 +1,162 @@
+# Responsible for building the storage dict into a set of rules ("JSON Schema") acceptable via the API
+# Probably other ways to solve this when the backend switches to some ORM
+from changedetectionio.notification import valid_notification_formats
+
+
+def build_time_between_check_json_schema():
+    # Setup time between check schema
+    schema_properties_time_between_check = {
+        "type": "object",
+        "additionalProperties": False,
+        "properties": {}
+    }
+    for p in ['weeks', 'days', 'hours', 'minutes', 'seconds']:
+        schema_properties_time_between_check['properties'][p] = {
+            "anyOf": [
+                {
+                    "type": "integer"
+                },
+                {
+                    "type": "null"
+                }
+            ]
+        }
+
+    return schema_properties_time_between_check
+
+def build_watch_json_schema(d):
+    # Base JSON schema
+    schema = {
+        'type': 'object',
+        'properties': {},
+    }
+
+    for k, v in d.items():
+        # @todo 'integer' is not covered here because its almost always for internal usage
+
+        if isinstance(v, type(None)):
+            schema['properties'][k] = {
+                "anyOf": [
+                    {"type": "null"},
+                ]
+            }
+        elif isinstance(v, list):
+            schema['properties'][k] = {
+                "anyOf": [
+                    {"type": "array",
+                     # Always is an array of strings, like text or regex or something
+                     "items": {
+                         "type": "string",
+                         "maxLength": 5000
+                     }
+                     },
+                ]
+            }
+        elif isinstance(v, bool):
+            schema['properties'][k] = {
+                "anyOf": [
+                    {"type": "boolean"},
+                ]
+            }
+        elif isinstance(v, str):
+            schema['properties'][k] = {
+                "anyOf": [
+                    {"type": "string",
+                     "maxLength": 5000},
+                ]
+            }
+
+    # Can also be a string (or None by default above)
+    for v in ['body',
+              'notification_body',
+              'notification_format',
+              'notification_title',
+              'proxy',
+              'tag',
+              'title',
+              'webdriver_js_execute_code'
+              ]:
+        schema['properties'][v]['anyOf'].append({'type': 'string', "maxLength": 5000})
+
+    for v in ['last_viewed']:
+        schema['properties'][v] = {
+            "type": "integer",
+            "description": "Unix timestamp in seconds of the last time the watch was viewed.",
+            "minimum": 0
+        }
+
+    # None or Boolean
+    schema['properties']['track_ldjson_price_data']['anyOf'].append({'type': 'boolean'})
+
+    schema['properties']['method'] = {"type": "string",
+                                      "enum": ["GET", "POST", "DELETE", "PUT"]
+                                      }
+
+    schema['properties']['fetch_backend']['anyOf'].append({"type": "string",
+                                                           "enum": ["html_requests", "html_webdriver"]
+                                                           })
+
+    schema['properties']['processor'] = {"anyOf": [
+        {"type": "string", "enum": ["restock_diff", "text_json_diff"]},
+        {"type": "null"}
+    ]}
+
+    # All headers must be key/value type dict
+    schema['properties']['headers'] = {
+        "type": "object",
+        "patternProperties": {
+            # Should always be a string:string type value
+            ".*": {"type": "string"},
+        }
+    }
+
+    schema['properties']['notification_format'] = {'type': 'string',
+                                                   'enum': list(valid_notification_formats.keys())
+                                                   }
+
+    # Stuff that shouldn't be available but is just state-storage
+    for v in ['previous_md5', 'last_error', 'has_ldjson_price_data', 'previous_md5_before_filters', 'uuid']:
+        del schema['properties'][v]
+
+    schema['properties']['webdriver_delay']['anyOf'].append({'type': 'integer'})
+
+    schema['properties']['time_between_check'] = build_time_between_check_json_schema()
+
+    schema['properties']['time_between_check_use_default'] = {
+        "type": "boolean",
+        "default": True,
+        "description": "Whether to use global settings for time between checks - defaults to true if not set"
+    }
+
+    schema['properties']['browser_steps'] = {
+        "anyOf": [
+            {
+                "type": "array",
+                "items": {
+                    "type": "object",
+                    "properties": {
+                        "operation": {
+                            "type": ["string", "null"],
+                            "maxLength": 5000  # Allows null and any string up to 5000 chars (including "")
+                        },
+                        "selector": {
+                            "type": ["string", "null"],
+                            "maxLength": 5000
+                        },
+                        "optional_value": {
+                            "type": ["string", "null"],
+                            "maxLength": 5000
+                        }
+                    },
+                    "required": ["operation", "selector", "optional_value"],
+                    "additionalProperties": False  # No extra keys allowed
+                }
+            },
+            {"type": "null"},  # Allows null for `browser_steps`
+            {"type": "array", "maxItems": 0}  # Allows empty array []
+        ]
+    }
+
+    # headers ?
+    return schema
+

changedetectionio/async_update_worker.py

@@ -3,11 +3,10 @@ from .processors.exceptions import ProcessorException
 import changedetectionio.content_fetchers.exceptions as content_fetchers_exceptions
 from changedetectionio.processors.text_json_diff.processor import FilterNotFoundInResponse
 from changedetectionio import html_tools
-from changedetectionio import worker_pool
-from changedetectionio.queuedWatchMetaData import PrioritizedItem
-from changedetectionio.pluggy_interface import apply_update_handler_alter, apply_update_finalize
+from changedetectionio.flask_app import watch_check_update
 
 import asyncio
+import importlib
 import os
 import sys
 import time
@@ -31,68 +30,28 @@ async def async_update_worker(worker_id, q, notification_q, app, datastore, exec
         app: Flask application instance
         datastore: Application datastore
         executor: ThreadPoolExecutor for queue operations (optional)
-
-    Returns:
-        "restart" if worker should restart, "shutdown" for clean exit
     """
     # Set a descriptive name for this task
     task = asyncio.current_task()
     if task:
         task.set_name(f"async-worker-{worker_id}")
 
-    # Read restart policy from environment
-    max_jobs = int(os.getenv("WORKER_MAX_JOBS", "10"))
-    max_runtime_seconds = int(os.getenv("WORKER_MAX_RUNTIME", "3600"))  # 1 hour default
-
-    jobs_processed = 0
-    start_time = time.time()
-
-    # Log thread name for debugging
-    import threading
-    thread_name = threading.current_thread().name
-    logger.info(f"Starting async worker {worker_id} on thread '{thread_name}' (max_jobs={max_jobs}, max_runtime={max_runtime_seconds}s)")
+    logger.info(f"Starting async worker {worker_id}")
 
     while not app.config.exit.is_set():
         update_handler = None
         watch = None
-        processing_exception = None  # Reset at start of each iteration to prevent state bleeding
 
         try:
-            # Efficient blocking via run_in_executor (no polling overhead!)
-            # Worker blocks in threading.Queue.get() which uses Condition.wait()
-            # Executor must be sized to match worker count (see worker_pool.py: 50 threads default)
-            # Single timeout (no double-timeout wrapper) = no race condition
-            queued_item_data = await q.async_get(executor=executor, timeout=1.0)
-
-            # CRITICAL: Claim UUID immediately after getting from queue to prevent race condition
-            # in wait_for_all_checks() which checks qsize() and running_uuids separately
-            uuid = queued_item_data.item.get('uuid')
-            if not worker_pool.claim_uuid_for_processing(uuid, worker_id):
-                # Already being processed - re-queue and continue
-                logger.trace(f"Worker {worker_id} detected UUID {uuid} already processing during claim - deferring")
-                await asyncio.sleep(DEFER_SLEEP_TIME_ALREADY_QUEUED)
-                deferred_priority = max(1000, queued_item_data.priority * 10)
-                deferred_item = PrioritizedItem(priority=deferred_priority, item=queued_item_data.item)
-                worker_pool.queue_item_async_safe(q, deferred_item, silent=True)
-                continue
+            # Use sync interface via run_in_executor since each worker has its own event loop
+            loop = asyncio.get_event_loop()
+            queued_item_data = await asyncio.wait_for(
+                loop.run_in_executor(executor, q.get, True, 1.0),  # block=True, timeout=1.0
+                timeout=1.5
+            )
 
         except asyncio.TimeoutError:
-            # No jobs available - check if we should restart based on time while idle
-            runtime = time.time() - start_time
-            if runtime >= max_runtime_seconds:
-                logger.info(f"Worker {worker_id} idle and reached max runtime ({runtime:.0f}s), restarting")
-                return "restart"
-            continue
-        except RuntimeError as e:
-            # Handle executor shutdown gracefully - this is expected during shutdown
-            if "cannot schedule new futures after shutdown" in str(e):
-                # Executor shut down - exit gracefully without logging in pytest
-                if not IN_PYTEST:
-                    logger.debug(f"Worker {worker_id} detected executor shutdown, exiting")
-                break
-            # Other RuntimeError - log and continue
-            logger.error(f"Worker {worker_id} runtime error: {e}")
-            await asyncio.sleep(0.1)
+            # No jobs available, continue loop
             continue
         except Exception as e:
             # Handle expected Empty exception from queue timeout
@@ -115,11 +74,28 @@ async def async_update_worker(worker_id, q, notification_q, app, datastore, exec
             await asyncio.sleep(0.1)
             continue
 
-        # UUID already claimed above immediately after getting from queue
-        # to prevent race condition with wait_for_all_checks()
+        uuid = queued_item_data.item.get('uuid')
+
+        # RACE CONDITION FIX: Check if this UUID is already being processed by another worker
+        from changedetectionio import worker_handler
+        from changedetectionio.queuedWatchMetaData import PrioritizedItem
+        if worker_handler.is_watch_running_by_another_worker(uuid, worker_id):
+            logger.trace(f"Worker {worker_id} detected UUID {uuid} already being processed by another worker - deferring")
+            # Sleep to avoid tight loop and give the other worker time to finish
+            await asyncio.sleep(DEFER_SLEEP_TIME_ALREADY_QUEUED)
+
+            # Re-queue with lower priority so it gets checked again after current processing finishes
+            deferred_priority = max(1000, queued_item_data.priority * 10)
+            deferred_item = PrioritizedItem(priority=deferred_priority, item=queued_item_data.item)
+            worker_handler.queue_item_async_safe(q, deferred_item, silent=True)
+            logger.debug(f"Worker {worker_id} re-queued UUID {uuid} for subsequent check")
+            continue
 
         fetch_start_time = round(time.time())
 
+        # Mark this UUID as being processed by this worker
+        worker_handler.set_uuid_processing(uuid, worker_id=worker_id, processing=True)
+        
         try:
             if uuid in list(datastore.data['watching'].keys()) and datastore.data['watching'][uuid].get('url'):
                 changed_detected = False
@@ -136,43 +112,29 @@ async def async_update_worker(worker_id, q, notification_q, app, datastore, exec
                 logger.info(f"Worker {worker_id} processing watch UUID {uuid} Priority {queued_item_data.priority} URL {watch['url']}")
 
                 try:
-                    # Retrieve signal by name to ensure thread-safe access across worker threads
-                    watch_check_update = signal('watch_check_update')
                     watch_check_update.send(watch_uuid=uuid)
 
                     # Processor is what we are using for detecting the "Change"
                     processor = watch.get('processor', 'text_json_diff')
 
                     # Init a new 'difference_detection_processor'
-                    # Use get_processor_module() to support both built-in and plugin processors
-                    from changedetectionio.processors import get_processor_module
-                    processor_module = get_processor_module(processor)
-
-                    if not processor_module:
-                        error_msg = f"Processor module '{processor}' not found."
-                        logger.error(error_msg)
-                        raise ModuleNotFoundError(error_msg)
+                    try:
+                        processor_module = importlib.import_module(f"changedetectionio.processors.{processor}.processor")
+                    except ModuleNotFoundError as e:
+                        print(f"Processor module '{processor}' not found.")
+                        raise e
 
                     update_handler = processor_module.perform_site_check(datastore=datastore,
                                                                          watch_uuid=uuid)
 
-                    # Allow plugins to modify/wrap the update_handler
-                    update_handler = apply_update_handler_alter(update_handler, watch, datastore)
-
                     update_signal = signal('watch_small_status_comment')
                     update_signal.send(watch_uuid=uuid, status="Fetching page..")
 
                     # All fetchers are now async, so call directly
                     await update_handler.call_browser()
 
-                    # Run change detection in executor to avoid blocking event loop
-                    # This includes CPU-intensive operations like HTML parsing (lxml/inscriptis)
-                    # which can take 2-10ms and cause GIL contention across workers
-                    loop = asyncio.get_event_loop()
-                    changed_detected, update_obj, contents = await loop.run_in_executor(
-                        executor,
-                        lambda: update_handler.run_changedetection(watch=watch)
-                    )
+                    # Run change detection (this is synchronous)
+                    changed_detected, update_obj, contents = update_handler.run_changedetection(watch=watch)
 
                 except PermissionError as e:
                     logger.critical(f"File permission error updating file, watch: {uuid}")
@@ -182,10 +144,8 @@ async def async_update_worker(worker_id, q, notification_q, app, datastore, exec
                 except ProcessorException as e:
                     if e.screenshot:
                         watch.save_screenshot(screenshot=e.screenshot)
-                        e.screenshot = None  # Free memory immediately
                     if e.xpath_data:
                         watch.save_xpath_data(data=e.xpath_data)
-                        e.xpath_data = None  # Free memory immediately
                     datastore.update_watch(uuid=uuid, update_obj={'last_error': e.message})
                     process_changedetection_results = False
 
@@ -205,11 +165,9 @@ async def async_update_worker(worker_id, q, notification_q, app, datastore, exec
 
                     if e.screenshot:
                         watch.save_screenshot(screenshot=e.screenshot, as_error=True)
-                        e.screenshot = None  # Free memory immediately
 
                     if e.xpath_data:
                         watch.save_xpath_data(data=e.xpath_data)
-                        e.xpath_data = None  # Free memory immediately
                         
                     process_changedetection_results = False
 
@@ -228,10 +186,8 @@ async def async_update_worker(worker_id, q, notification_q, app, datastore, exec
 
                     if e.screenshot:
                         watch.save_screenshot(screenshot=e.screenshot, as_error=True)
-                        e.screenshot = None  # Free memory immediately
                     if e.xpath_data:
                         watch.save_xpath_data(data=e.xpath_data, as_error=True)
-                        e.xpath_data = None  # Free memory immediately
                     if e.page_text:
                         watch.save_error_text(contents=e.page_text)
 
@@ -241,7 +197,6 @@ async def async_update_worker(worker_id, q, notification_q, app, datastore, exec
                 except FilterNotFoundInResponse as e:
                     if not datastore.data['watching'].get(uuid):
                         continue
-                    logger.debug(f"Received FilterNotFoundInResponse exception for {uuid}")
 
                     err_text = "Warning, no filters were found, no change detection ran - Did the page change layout? update your Visual Filter if necessary."
                     datastore.update_watch(uuid=uuid, update_obj={'last_error': err_text})
@@ -249,11 +204,9 @@ async def async_update_worker(worker_id, q, notification_q, app, datastore, exec
                     # Filter wasnt found, but we should still update the visual selector so that they can have a chance to set it up again
                     if e.screenshot:
                         watch.save_screenshot(screenshot=e.screenshot)
-                        e.screenshot = None  # Free memory immediately
 
                     if e.xpath_data:
                         watch.save_xpath_data(data=e.xpath_data)
-                        e.xpath_data = None  # Free memory immediately
 
                     # Only when enabled, send the notification
                     if watch.get('filter_failure_notification_send', False):
@@ -261,19 +214,17 @@ async def async_update_worker(worker_id, q, notification_q, app, datastore, exec
                         c += 1
                         # Send notification if we reached the threshold?
                         threshold = datastore.data['settings']['application'].get('filter_failure_notification_threshold_attempts', 0)
-                        logger.debug(f"FilterNotFoundInResponse - Filter for {uuid} not found, consecutive_filter_failures: {c} of threshold {threshold}")
+                        logger.debug(f"Filter for {uuid} not found, consecutive_filter_failures: {c} of threshold {threshold}")
                         if c >= threshold:
                             if not watch.get('notification_muted'):
-                                logger.debug(f"FilterNotFoundInResponse - Sending filter failed notification for {uuid}")
+                                logger.debug(f"Sending filter failed notification for {uuid}")
                                 await send_filter_failure_notification(uuid, notification_q, datastore)
                             c = 0
-                            logger.debug(f"FilterNotFoundInResponse - Reset filter failure count back to zero")
-                        else:
-                            logger.debug(f"FilterNotFoundInResponse - {c} of threshold {threshold}..")
+                            logger.debug(f"Reset filter failure count back to zero")
 
                         datastore.update_watch(uuid=uuid, update_obj={'consecutive_filter_failures': c})
                     else:
-                        logger.trace(f"FilterNotFoundInResponse - {uuid} - filter_failure_notification_send not enabled, skipping")
+                        logger.trace(f"{uuid} - filter_failure_notification_send not enabled, skipping")
 
                     process_changedetection_results = False
 
@@ -281,9 +232,6 @@ async def async_update_worker(worker_id, q, notification_q, app, datastore, exec
                     # Yes fine, so nothing todo, don't continue to process.
                     process_changedetection_results = False
                     changed_detected = False
-                    logger.debug(f'[{uuid}] - checksumFromPreviousCheckWasTheSame - Checksum from previous check was the same, nothing todo here.')
-                    # Reset the edited flag since we successfully completed the check
-                    watch.reset_watch_edited_flag()
                     
                 except content_fetchers_exceptions.BrowserConnectError as e:
                     datastore.update_watch(uuid=uuid,
@@ -350,7 +298,6 @@ async def async_update_worker(worker_id, q, notification_q, app, datastore, exec
                     err_text = "Error running JS Actions - Page request - "+e.message
                     if e.screenshot:
                         watch.save_screenshot(screenshot=e.screenshot, as_error=True)
-                        e.screenshot = None  # Free memory immediately
                     datastore.update_watch(uuid=uuid, update_obj={'last_error': err_text,
                                                                 'last_check_status': e.status_code})
                     process_changedetection_results = False
@@ -362,7 +309,6 @@ async def async_update_worker(worker_id, q, notification_q, app, datastore, exec
 
                     if e.screenshot:
                         watch.save_screenshot(screenshot=e.screenshot, as_error=True)
-                        e.screenshot = None  # Free memory immediately
 
                     datastore.update_watch(uuid=uuid, update_obj={'last_error': err_text,
                                                                 'last_check_status': e.status_code,
@@ -376,9 +322,8 @@ async def async_update_worker(worker_id, q, notification_q, app, datastore, exec
                     logger.error(f"Exception (BrowserStepsInUnsupportedFetcher) reached processing watch UUID: {uuid}")
 
                 except Exception as e:
-                    import traceback
                     logger.error(f"Worker {worker_id} exception processing watch UUID: {uuid}")
-                    logger.exception(f"Worker {worker_id} full exception details:")
+                    logger.error(str(e))
                     datastore.update_watch(uuid=uuid, update_obj={'last_error': "Exception: " + str(e)})
                     process_changedetection_results = False
 
@@ -386,7 +331,7 @@ async def async_update_worker(worker_id, q, notification_q, app, datastore, exec
                     if not datastore.data['watching'].get(uuid):
                         continue
 
-                    update_obj['content-type'] = str(update_handler.fetcher.get_all_headers().get('content-type', '') or "").lower()
+                    update_obj['content-type'] = update_handler.fetcher.get_all_headers().get('content-type', '').lower()
 
                     if not watch.get('ignore_status_codes'):
                         update_obj['consecutive_filter_failures'] = 0
@@ -397,27 +342,17 @@ async def async_update_worker(worker_id, q, notification_q, app, datastore, exec
                 if not datastore.data['watching'].get(uuid):
                     continue
 
-                logger.debug(f"Processing watch UUID: {uuid} - xpath_data length returned {len(update_handler.xpath_data) if update_handler and update_handler.xpath_data else 'empty.'}")
-                if update_handler and process_changedetection_results:
+                logger.debug(f"Processing watch UUID: {uuid} - xpath_data length returned {len(update_handler.xpath_data) if update_handler.xpath_data else 'empty.'}")
+                if process_changedetection_results:
                     try:
-                        # Reset the edited flag BEFORE update_watch (which calls watch.update() and would set it again)
-                        watch.reset_watch_edited_flag()
                         datastore.update_watch(uuid=uuid, update_obj=update_obj)
 
                         if changed_detected or not watch.history_n:
                             if update_handler.screenshot:
                                 watch.save_screenshot(screenshot=update_handler.screenshot)
-                                # Free screenshot memory immediately after saving
-                                update_handler.screenshot = None
-                                if hasattr(update_handler, 'fetcher') and hasattr(update_handler.fetcher, 'screenshot'):
-                                    update_handler.fetcher.screenshot = None
 
                             if update_handler.xpath_data:
                                 watch.save_xpath_data(data=update_handler.xpath_data)
-                                # Free xpath data memory
-                                update_handler.xpath_data = None
-                                if hasattr(update_handler, 'fetcher') and hasattr(update_handler.fetcher, 'xpath_data'):
-                                    update_handler.fetcher.xpath_data = None
 
                             # Ensure unique timestamp for history
                             if watch.newest_history_key and int(fetch_start_time) == int(watch.newest_history_key):
@@ -444,69 +379,50 @@ async def async_update_worker(worker_id, q, notification_q, app, datastore, exec
                                     await send_content_changed_notification(uuid, notification_q, datastore)
 
                     except Exception as e:
-
                         logger.critical(f"Worker {worker_id} exception in process_changedetection_results")
-                        logger.exception(f"Worker {worker_id} full exception details:")
+                        logger.critical(str(e))
                         datastore.update_watch(uuid=uuid, update_obj={'last_error': str(e)})
 
-
                 # Always record attempt count
                 count = watch.get('check_count', 0) + 1
 
-                final_updates = {'fetch_time': round(time.time() - fetch_start_time, 3),
-                                                                  'check_count': count,
-                                                                  }
+                # Always record page title (used in notifications, and can change even when the content is the same)
+                if update_obj.get('content-type') and 'html' in update_obj.get('content-type'):
+                    try:
+                        page_title = html_tools.extract_title(data=update_handler.fetcher.content)
+                        if page_title:
+                            page_title = page_title.strip()[:2000]
+                            logger.debug(f"UUID: {uuid} Page <title> is '{page_title}'")
+                            datastore.update_watch(uuid=uuid, update_obj={'page_title': page_title})
+                    except Exception as e:
+                        logger.warning(f"UUID: {uuid} Exception when extracting <title> - {str(e)}")
+
                 # Record server header
                 try:
-                    server_header = str(update_handler.fetcher.get_all_headers().get('server', '') or "").strip().lower()[:255]
-                    if server_header:
-                        final_updates['remote_server_reply'] = server_header
+                    server_header = update_handler.fetcher.headers.get('server', '').strip().lower()[:255]
+                    datastore.update_watch(uuid=uuid, update_obj={'remote_server_reply': server_header})
                 except Exception as e:
-                    server_header = None
                     pass
 
-                if update_handler: # Could be none or empty if the processor was not found
-                    # Always record page title (used in notifications, and can change even when the content is the same)
-                    if update_obj.get('content-type') and 'html' in update_obj.get('content-type'):
-                        try:
-                            page_title = html_tools.extract_title(data=update_handler.fetcher.content)
-                            if page_title:
-                                page_title = page_title.strip()[:2000]
-                                logger.debug(f"UUID: {uuid} Page <title> is '{page_title}'")
-                                final_updates['page_title'] = page_title
-                        except Exception as e:
-                            logger.exception(f"Worker {worker_id} full exception details:")
-                            logger.warning(f"UUID: {uuid} Exception when extracting <title> - {str(e)}")
+                # Store favicon if necessary
+                if update_handler.fetcher.favicon_blob and update_handler.fetcher.favicon_blob.get('base64'):
+                    watch.bump_favicon(url=update_handler.fetcher.favicon_blob.get('url'),
+                                       favicon_base_64=update_handler.fetcher.favicon_blob.get('base64')
+                                       )
 
-                    # Store favicon if necessary
-                    if update_handler.fetcher.favicon_blob and update_handler.fetcher.favicon_blob.get('base64'):
-                        watch.bump_favicon(url=update_handler.fetcher.favicon_blob.get('url'),
-                                           favicon_base_64=update_handler.fetcher.favicon_blob.get('base64')
-                                           )
+                datastore.update_watch(uuid=uuid, update_obj={'fetch_time': round(time.time() - fetch_start_time, 3),
+                                                               'check_count': count})
 
-                    datastore.update_watch(uuid=uuid, update_obj=final_updates)
-
-                    # NOW clear fetcher content - after all processing is complete
-                    # This is the last point where we need the fetcher data
-                    if update_handler and hasattr(update_handler, 'fetcher') and update_handler.fetcher:
-                        update_handler.fetcher.clear_content()
-
-                    # Explicitly delete update_handler to free all references
-                    if update_handler:
-                        del update_handler
-                        update_handler = None
-
-                # Force garbage collection
-                import gc
-                gc.collect()
+                # NOW clear fetcher content - after all processing is complete
+                # This is the last point where we need the fetcher data
+                if update_handler and hasattr(update_handler, 'fetcher') and update_handler.fetcher:
+                    update_handler.fetcher.clear_content()
+                    logger.debug(f"Cleared fetcher content for UUID {uuid}")
 
         except Exception as e:
-            # Store the processing exception for plugin finalization hook
-            processing_exception = e
-
             logger.error(f"Worker {worker_id} unexpected error processing {uuid}: {e}")
-            logger.exception(f"Worker {worker_id} full exception details:")
-
+            logger.error(f"Worker {worker_id} traceback:", exc_info=True)
+            
             # Also update the watch with error information
             if datastore and uuid in datastore.data['watching']:
                 datastore.update_watch(uuid=uuid, update_obj={'last_error': f"Worker error: {str(e)}"})
@@ -514,74 +430,51 @@ async def async_update_worker(worker_id, q, notification_q, app, datastore, exec
         finally:
             # Always cleanup - this runs whether there was an exception or not
             if uuid:
-                # Capture references for plugin finalize hook BEFORE cleanup
-                # (cleanup may delete these variables, but plugins need the original references)
-                finalize_handler = update_handler  # Capture now, before cleanup deletes it
-                finalize_watch = watch              # Capture now, before any modifications
-
-                # Call quit() as backup (Puppeteer/Playwright have internal cleanup, but this acts as safety net)
                 try:
                     if update_handler and hasattr(update_handler, 'fetcher') and update_handler.fetcher:
                         await update_handler.fetcher.quit(watch=watch)
                 except Exception as e:
                     logger.error(f"Exception while cleaning/quit after calling browser: {e}")
-                    logger.exception(f"Worker {worker_id} full exception details:")
-
                 try:
+                    # Mark UUID as no longer being processed by this worker
+                    worker_handler.set_uuid_processing(uuid, worker_id=worker_id, processing=False)
+                    
+                    # Send completion signal
+                    if watch:
+                        #logger.info(f"Worker {worker_id} sending completion signal for UUID {watch['uuid']}")
+                        watch_check_update.send(watch_uuid=watch['uuid'])
 
-                    # Clean up all memory references BEFORE garbage collection
+                    # Explicitly clean up update_handler and all its references
                     if update_handler:
+                        # Clear fetcher content using the proper method
                         if hasattr(update_handler, 'fetcher') and update_handler.fetcher:
                             update_handler.fetcher.clear_content()
+
+                        # Clear processor references
                         if hasattr(update_handler, 'content_processor'):
                             update_handler.content_processor = None
-                        del update_handler
+
                         update_handler = None
 
-                    # Clear large content variables
+                    # Clear local contents variable if it still exists
                     if 'contents' in locals():
                         del contents
 
-                    # Force garbage collection after all references are cleared
+                    # Note: We don't set watch = None here because:
+                    # 1. watch is just a local reference to datastore.data['watching'][uuid]
+                    # 2. Setting it to None doesn't affect the datastore
+                    # 3. GC can't collect the object anyway (still referenced by datastore)
+                    # 4. It would just cause confusion
+
+                    # Force garbage collection after cleanup
                     import gc
                     gc.collect()
 
                     logger.debug(f"Worker {worker_id} completed watch {uuid} in {time.time()-fetch_start_time:.2f}s")
                 except Exception as cleanup_error:
                     logger.error(f"Worker {worker_id} error during cleanup: {cleanup_error}")
-                    logger.exception(f"Worker {worker_id} full exception details:")
 
-                # Call plugin finalization hook after all cleanup is done
-                # Use captured references from before cleanup
-                try:
-                    apply_update_finalize(
-                        update_handler=finalize_handler,
-                        watch=finalize_watch,
-                        datastore=datastore,
-                        processing_exception=processing_exception
-                    )
-                except Exception as finalize_error:
-                    logger.error(f"Worker {worker_id} error in finalize hook: {finalize_error}")
-                    logger.exception(f"Worker {worker_id} full exception details:")
-                finally:
-                    # Clean up captured references to allow immediate garbage collection
-                    del finalize_handler
-                    del finalize_watch
-
-                # Release UUID from processing AFTER all cleanup and hooks complete (thread-safe)
-                # This ensures wait_for_all_checks() waits for finalize hooks to complete
-                try:
-                    worker_pool.release_uuid_from_processing(uuid, worker_id=worker_id)
-                except Exception as release_error:
-                    logger.error(f"Worker {worker_id} error releasing UUID: {release_error}")
-                    logger.exception(f"Worker {worker_id} full exception details:")
-                finally:
-                    # Send completion signal - retrieve by name to ensure thread-safe access
-                    if watch:
-                        watch_check_update = signal('watch_check_update')
-                        watch_check_update.send(watch_uuid=watch['uuid'])
-
-            del (uuid)
+            del(uuid)
 
             # Brief pause before continuing to avoid tight error loops (only on error)
             if 'e' in locals():
@@ -590,19 +483,6 @@ async def async_update_worker(worker_id, q, notification_q, app, datastore, exec
                 # Small yield for normal completion
                 await asyncio.sleep(0.01)
 
-            # Job completed - increment counter and check restart conditions
-            jobs_processed += 1
-            runtime = time.time() - start_time
-
-            # Check if we should restart (only when idle, between jobs)
-            should_restart_jobs = jobs_processed >= max_jobs
-            should_restart_time = runtime >= max_runtime_seconds
-
-            if should_restart_jobs or should_restart_time:
-                reason = f"{jobs_processed} jobs" if should_restart_jobs else f"{runtime:.0f}s runtime"
-                logger.info(f"Worker {worker_id} restarting after {reason} ({jobs_processed} jobs, {runtime:.0f}s runtime)")
-                return "restart"
-
         # Check if we should exit
         if app.config.exit.is_set():
             break
@@ -610,12 +490,10 @@ async def async_update_worker(worker_id, q, notification_q, app, datastore, exec
     # Check if we're in pytest environment - if so, be more gentle with logging
     import sys
     in_pytest = "pytest" in sys.modules or "PYTEST_CURRENT_TEST" in os.environ
-
+    
     if not in_pytest:
         logger.info(f"Worker {worker_id} shutting down")
 
-    return "shutdown"
-
 
 def cleanup_error_artifacts(uuid, datastore):
     """Helper function to clean up error artifacts"""

changedetectionio/blueprint/backups/__init__.py

@@ -13,7 +13,7 @@ from loguru import logger
 BACKUP_FILENAME_FORMAT = "changedetection-backup-{}.zip"
 
 
-def create_backup(datastore_path, watches: dict, tags: dict = None):
+def create_backup(datastore_path, watches: dict):
     logger.debug("Creating backup...")
     import zipfile
     from pathlib import Path
@@ -27,36 +27,15 @@ def create_backup(datastore_path, watches: dict, tags: dict = None):
                          compression=zipfile.ZIP_DEFLATED,
                          compresslevel=8) as zipObj:
 
-        # Add the settings file (supports both formats)
-        # New format: changedetection.json
-        changedetection_json = os.path.join(datastore_path, "changedetection.json")
-        if os.path.isfile(changedetection_json):
-            zipObj.write(changedetection_json, arcname="changedetection.json")
-            logger.debug("Added changedetection.json to backup")
+        # Add the index
+        zipObj.write(os.path.join(datastore_path, "url-watches.json"), arcname="url-watches.json")
 
-        # Legacy format: url-watches.json (for backward compatibility)
-        url_watches_json = os.path.join(datastore_path, "url-watches.json")
-        if os.path.isfile(url_watches_json):
-            zipObj.write(url_watches_json, arcname="url-watches.json")
-            logger.debug("Added url-watches.json to backup")
-
-        # Add the flask app secret (if it exists)
-        secret_file = os.path.join(datastore_path, "secret.txt")
-        if os.path.isfile(secret_file):
-            zipObj.write(secret_file, arcname="secret.txt")
-
-        # Add tag data directories (each tag has its own {uuid}/tag.json)
-        for uuid, tag in (tags or {}).items():
-            for f in Path(tag.data_dir).glob('*'):
-                zipObj.write(f,
-                             arcname=os.path.join(f.parts[-2], f.parts[-1]),
-                             compress_type=zipfile.ZIP_DEFLATED,
-                             compresslevel=8)
-            logger.debug(f"Added tag '{tag.get('title')}' ({uuid}) to backup")
+        # Add the flask app secret
+        zipObj.write(os.path.join(datastore_path, "secret.txt"), arcname="secret.txt")
 
         # Add any data in the watch data directory.
         for uuid, w in watches.items():
-            for f in Path(w.data_dir).glob('*'):
+            for f in Path(w.watch_data_dir).glob('*'):
                 zipObj.write(f,
                              # Use the full path to access the file, but make the file 'relative' in the Zip.
                              arcname=os.path.join(f.parts[-2], f.parts[-1]),
@@ -97,10 +76,7 @@ def create_backup(datastore_path, watches: dict, tags: dict = None):
 
 
 def construct_blueprint(datastore: ChangeDetectionStore):
-    from .restore import construct_restore_blueprint
-
     backups_blueprint = Blueprint('backups', __name__, template_folder="templates")
-    backups_blueprint.register_blueprint(construct_restore_blueprint(datastore))
     backup_threads = []
 
     @login_optionally_required
@@ -108,17 +84,17 @@ def construct_blueprint(datastore: ChangeDetectionStore):
     def request_backup():
         if any(thread.is_alive() for thread in backup_threads):
             flash(gettext("A backup is already running, check back in a few minutes"), "error")
-            return redirect(url_for('backups.create'))
+            return redirect(url_for('backups.index'))
 
         if len(find_backups()) > int(os.getenv("MAX_NUMBER_BACKUPS", 100)):
             flash(gettext("Maximum number of backups reached, please remove some"), "error")
-            return redirect(url_for('backups.create'))
+            return redirect(url_for('backups.index'))
 
-        # With immediate persistence, all data is already saved
+        # Be sure we're written fresh
+        datastore.sync_to_json()
         zip_thread = threading.Thread(
             target=create_backup,
             args=(datastore.datastore_path, datastore.data.get("watching")),
-            kwargs={'tags': datastore.data['settings']['application'].get('tags', {})},
             daemon=True,
             name="BackupCreator"
         )
@@ -126,7 +102,7 @@ def construct_blueprint(datastore: ChangeDetectionStore):
         backup_threads.append(zip_thread)
         flash(gettext("Backup building in background, check back in a few minutes."))
 
-        return redirect(url_for('backups.create'))
+        return redirect(url_for('backups.index'))
 
     def find_backups():
         backup_filepath = os.path.join(datastore.datastore_path, BACKUP_FILENAME_FORMAT.format("*"))
@@ -168,14 +144,14 @@ def construct_blueprint(datastore: ChangeDetectionStore):
         return send_from_directory(os.path.abspath(datastore.datastore_path), filename, as_attachment=True)
 
     @login_optionally_required
-    @backups_blueprint.route("/", methods=['GET'])
-    @backups_blueprint.route("/create", methods=['GET'])
-    def create():
+    @backups_blueprint.route("", methods=['GET'])
+    def index():
         backups = find_backups()
-        output = render_template("backup_create.html",
+        output = render_template("overview.html",
                                  available_backups=backups,
                                  backup_running=any(thread.is_alive() for thread in backup_threads)
                                  )
+
         return output
 
     @login_optionally_required
@@ -189,6 +165,6 @@ def construct_blueprint(datastore: ChangeDetectionStore):
 
         flash(gettext("Backups were deleted."))
 
-        return redirect(url_for('backups.create'))
+        return redirect(url_for('backups.index'))
 
     return backups_blueprint

changedetectionio/blueprint/backups/restore.py

@@ -1,208 +0,0 @@
-import io
-import json
-import os
-import shutil
-import tempfile
-import threading
-import zipfile
-
-from flask import Blueprint, render_template, flash, url_for, redirect, request
-from flask_babel import gettext, lazy_gettext as _l
-from wtforms import Form, BooleanField, SubmitField
-from flask_wtf.file import FileField, FileAllowed
-from loguru import logger
-
-from changedetectionio.flask_app import login_optionally_required
-
-
-class RestoreForm(Form):
-    zip_file = FileField(_l('Backup zip file'), validators=[
-        FileAllowed(['zip'], _l('Must be a .zip backup file!'))
-    ])
-    include_groups = BooleanField(_l('Include groups'), default=True)
-    include_groups_replace_existing = BooleanField(_l('Replace existing groups of the same UUID'), default=True)
-    include_watches = BooleanField(_l('Include watches'), default=True)
-    include_watches_replace_existing = BooleanField(_l('Replace existing watches of the same UUID'), default=True)
-    submit = SubmitField(_l('Restore backup'))
-
-
-def import_from_zip(zip_stream, datastore, include_groups, include_groups_replace, include_watches, include_watches_replace):
-    """
-    Extract and import watches and groups from a backup zip stream.
-
-    Mirrors the store's _load_watches / _load_tags loading pattern:
-      - UUID dirs with tag.json  → Tag.model + tag_obj.commit()
-      - UUID dirs with watch.json → rehydrate_entity + watch_obj.commit()
-
-    Returns a dict with counts: restored_groups, skipped_groups, restored_watches, skipped_watches.
-    Raises zipfile.BadZipFile if the stream is not a valid zip.
-    """
-    from changedetectionio.model import Tag
-
-    restored_groups = 0
-    skipped_groups = 0
-    restored_watches = 0
-    skipped_watches = 0
-
-    current_tags = datastore.data['settings']['application'].get('tags', {})
-    current_watches = datastore.data['watching']
-
-    with tempfile.TemporaryDirectory() as tmpdir:
-        logger.debug(f"Restore: extracting zip to {tmpdir}")
-        with zipfile.ZipFile(zip_stream, 'r') as zf:
-            zf.extractall(tmpdir)
-        logger.debug("Restore: zip extracted, scanning UUID directories")
-
-        for entry in os.scandir(tmpdir):
-            if not entry.is_dir():
-                continue
-
-            uuid = entry.name
-            tag_json_path = os.path.join(entry.path, 'tag.json')
-            watch_json_path = os.path.join(entry.path, 'watch.json')
-
-            # --- Tags (groups) ---
-            if include_groups and os.path.exists(tag_json_path):
-                if uuid in current_tags and not include_groups_replace:
-                    logger.debug(f"Restore: skipping existing group {uuid} (replace not requested)")
-                    skipped_groups += 1
-                    continue
-
-                try:
-                    with open(tag_json_path, 'r', encoding='utf-8') as f:
-                        tag_data = json.load(f)
-                except (json.JSONDecodeError, IOError) as e:
-                    logger.error(f"Restore: failed to read tag.json for {uuid}: {e}")
-                    continue
-
-                title = tag_data.get('title', uuid)
-                logger.debug(f"Restore: importing group '{title}' ({uuid})")
-
-                # Mirror _load_tags: set uuid and force processor
-                tag_data['uuid'] = uuid
-                tag_data['processor'] = 'restock_diff'
-
-                # Copy the UUID directory so data_dir exists for commit()
-                dst_dir = os.path.join(datastore.datastore_path, uuid)
-                if os.path.exists(dst_dir):
-                    shutil.rmtree(dst_dir)
-                shutil.copytree(entry.path, dst_dir)
-
-                tag_obj = Tag.model(
-                    datastore_path=datastore.datastore_path,
-                    __datastore=datastore.data,
-                    default=tag_data
-                )
-                current_tags[uuid] = tag_obj
-                tag_obj.commit()
-                restored_groups += 1
-                logger.success(f"Restore: group '{title}' ({uuid}) restored")
-
-            # --- Watches ---
-            elif include_watches and os.path.exists(watch_json_path):
-                if uuid in current_watches and not include_watches_replace:
-                    logger.debug(f"Restore: skipping existing watch {uuid} (replace not requested)")
-                    skipped_watches += 1
-                    continue
-
-                try:
-                    with open(watch_json_path, 'r', encoding='utf-8') as f:
-                        watch_data = json.load(f)
-                except (json.JSONDecodeError, IOError) as e:
-                    logger.error(f"Restore: failed to read watch.json for {uuid}: {e}")
-                    continue
-
-                url = watch_data.get('url', uuid)
-                logger.debug(f"Restore: importing watch '{url}' ({uuid})")
-
-                # Copy UUID directory first so data_dir and history files exist
-                dst_dir = os.path.join(datastore.datastore_path, uuid)
-                if os.path.exists(dst_dir):
-                    shutil.rmtree(dst_dir)
-                shutil.copytree(entry.path, dst_dir)
-
-                # Mirror _load_watches / rehydrate_entity
-                watch_data['uuid'] = uuid
-                watch_obj = datastore.rehydrate_entity(uuid, watch_data)
-                current_watches[uuid] = watch_obj
-                watch_obj.commit()
-                restored_watches += 1
-                logger.success(f"Restore: watch '{url}' ({uuid}) restored")
-
-        logger.debug(f"Restore: scan complete - groups {restored_groups} restored / {skipped_groups} skipped, "
-                     f"watches {restored_watches} restored / {skipped_watches} skipped")
-
-    # Persist changedetection.json (includes the updated tags dict)
-    logger.debug("Restore: committing datastore settings")
-    datastore.commit()
-
-    return {
-        'restored_groups': restored_groups,
-        'skipped_groups': skipped_groups,
-        'restored_watches': restored_watches,
-        'skipped_watches': skipped_watches,
-    }
-
-
-
-def construct_restore_blueprint(datastore):
-    restore_blueprint = Blueprint('restore', __name__, template_folder="templates")
-    restore_threads = []
-
-    @login_optionally_required
-    @restore_blueprint.route("/restore", methods=['GET'])
-    def restore():
-        form = RestoreForm()
-        return render_template("backup_restore.html",
-                               form=form,
-                               restore_running=any(t.is_alive() for t in restore_threads))
-
-    @login_optionally_required
-    @restore_blueprint.route("/restore/start", methods=['POST'])
-    def backups_restore_start():
-        if any(t.is_alive() for t in restore_threads):
-            flash(gettext("A restore is already running, check back in a few minutes"), "error")
-            return redirect(url_for('backups.restore.restore'))
-
-        zip_file = request.files.get('zip_file')
-        if not zip_file or not zip_file.filename:
-            flash(gettext("No file uploaded"), "error")
-            return redirect(url_for('backups.restore.restore'))
-
-        if not zip_file.filename.lower().endswith('.zip'):
-            flash(gettext("File must be a .zip backup file"), "error")
-            return redirect(url_for('backups.restore.restore'))
-
-        # Read into memory now — the request stream is gone once we return
-        try:
-            zip_bytes = io.BytesIO(zip_file.read())
-            zipfile.ZipFile(zip_bytes)  # quick validity check before spawning
-            zip_bytes.seek(0)
-        except zipfile.BadZipFile:
-            flash(gettext("Invalid or corrupted zip file"), "error")
-            return redirect(url_for('backups.restore.restore'))
-
-        include_groups = request.form.get('include_groups') == 'y'
-        include_groups_replace = request.form.get('include_groups_replace_existing') == 'y'
-        include_watches = request.form.get('include_watches') == 'y'
-        include_watches_replace = request.form.get('include_watches_replace_existing') == 'y'
-
-        restore_thread = threading.Thread(
-            target=import_from_zip,
-            kwargs={
-                'zip_stream': zip_bytes,
-                'datastore': datastore,
-                'include_groups': include_groups,
-                'include_groups_replace': include_groups_replace,
-                'include_watches': include_watches,
-                'include_watches_replace': include_watches_replace,
-            },
-            daemon=True,
-            name="BackupRestore"
-        )
-        restore_thread.start()
-        restore_threads.append(restore_thread)
-        flash(gettext("Restore started in background, check back in a few minutes."))
-        return redirect(url_for('backups.restore.restore'))
-
-    return restore_blueprint

changedetectionio/blueprint/backups/templates/backup_create.html

@@ -1,49 +0,0 @@
-{% extends 'base.html' %}
-{% block content %}
-    {% from '_helpers.html' import render_simple_field, render_field %}
-
-    <div class="edit-form">
-        <div class="tabs collapsable">
-            <ul>
-                <li class="tab active" id=""><a href="{{ url_for('backups.create') }}">{{ _('Create') }}</a></li>
-                <li class="tab"><a href="{{ url_for('backups.restore.restore') }}">{{ _('Restore') }}</a></li>
-            </ul>
-        </div>
-        <div class="box-wrap inner">
-            <div id="general">
-                {% if backup_running %}
-                    <p>
-                        <span class="spinner"></span>&nbsp;<strong>{{ _('A backup is running!') }}</strong>
-                    </p>
-                {% endif %}
-
-                <p>
-                    {{ _('Here you can download and request a new backup, when a backup is completed you will see it listed below.') }}
-                </p>
-                <br>
-                {% if available_backups %}
-                    <ul>
-                        {% for backup in available_backups %}
-                            <li>
-                                <a href="{{ url_for('backups.download_backup', filename=backup["filename"]) }}">{{ backup["filename"] }}</a> {{ backup["filesize"] }} {{ _('Mb') }}
-                            </li>
-                        {% endfor %}
-                    </ul>
-                {% else %}
-                    <p>
-                        <strong>{{ _('No backups found.') }}</strong>
-                    </p>
-                {% endif %}
-
-                <a class="pure-button pure-button-primary"
-                   href="{{ url_for('backups.request_backup') }}">{{ _('Create backup') }}</a>
-                {% if available_backups %}
-                    <a class="pure-button button-small button-error "
-                       href="{{ url_for('backups.remove_backups') }}">{{ _('Remove backups') }}</a>
-                {% endif %}
-
-            </div>
-
-        </div>
-    </div>
-{% endblock %}

changedetectionio/blueprint/backups/templates/backup_restore.html

@@ -1,58 +0,0 @@
-{% extends 'base.html' %}
-{% block content %}
-    {% from '_helpers.html' import render_field, render_checkbox_field %}
-
-    <div class="edit-form">
-        <div class="tabs collapsable">
-            <ul>
-                <li class="tab"><a href="{{ url_for('backups.create') }}">{{ _('Create') }}</a></li>
-                <li class="tab active"><a href="{{ url_for('backups.restore.restore') }}">{{ _('Restore') }}</a></li>
-            </ul>
-        </div>
-        <div class="box-wrap inner">
-            <div id="general">
-                {% if restore_running %}
-                    <p>
-                        <span class="spinner"></span>&nbsp;<strong>{{ _('A restore is running!') }}</strong>
-                    </p>
-                {% endif %}
-
-                <p>{{ _('Restore a backup. Must be a .zip backup file created on/after v0.53.1 (new database layout).') }}</p>
-                <p>{{ _('Note: This does not override the main application settings, only watches and groups.') }}</p>
-
-                <form class="pure-form pure-form-stacked settings"
-                      action="{{ url_for('backups.restore.backups_restore_start') }}"
-                      method="POST"
-                      enctype="multipart/form-data">
-                    <input type="hidden" name="csrf_token" value="{{ csrf_token() }}">
-
-                    <div class="pure-control-group">
-                        {{ render_checkbox_field(form.include_groups) }}
-                        <span class="pure-form-message-inline">{{ _('Include all groups found in backup?') }}</span>
-                    </div>
-                    <div class="pure-control-group">
-                        {{ render_checkbox_field(form.include_groups_replace_existing) }}
-                        <span class="pure-form-message-inline">{{ _('Replace any existing groups of the same UUID?') }}</span>
-                    </div>
-
-                    <div class="pure-control-group">
-                        {{ render_checkbox_field(form.include_watches) }}
-                        <span class="pure-form-message-inline">{{ _('Include all watches found in backup?') }}</span>
-                    </div>
-                    <div class="pure-control-group">
-                        {{ render_checkbox_field(form.include_watches_replace_existing) }}
-                        <span class="pure-form-message-inline">{{ _('Replace any existing watches of the same UUID?') }}</span>
-                    </div>
-
-                    <div class="pure-control-group">
-                        {{ render_field(form.zip_file) }}
-                    </div>
-
-                    <div class="pure-controls">
-                        <button type="submit" class="pure-button pure-button-primary">{{ _('Restore backup') }}</button>
-                    </div>
-                </form>
-            </div>
-        </div>
-    </div>
-{% endblock %}

changedetectionio/blueprint/backups/templates/overview.html

@@ -0,0 +1,36 @@
+{% extends 'base.html' %}
+{% block content %}
+    {% from '_helpers.html' import render_simple_field, render_field %}
+    <div class="edit-form">
+        <div class="box-wrap inner">
+            <h4>{{ _('Backups') }}</h4>
+            {% if backup_running %}
+                <p>
+                    <strong>{{ _('A backup is running!') }}</strong>
+                </p>
+            {% endif %}
+            <p>
+                {{ _('Here you can download and request a new backup, when a backup is completed you will see it listed below.') }}
+            </p>
+            <br>
+                {% if available_backups %}
+                    <ul>
+                    {% for backup in available_backups %}
+                        <li><a href="{{ url_for('backups.download_backup', filename=backup["filename"]) }}">{{ backup["filename"] }}</a> {{  backup["filesize"] }} {{ _('Mb') }}</li>
+                    {% endfor %}
+                    </ul>
+                {% else %}
+                    <p>
+                    <strong>{{ _('No backups found.') }}</strong>
+                    </p>
+                {% endif %}
+
+            <a class="pure-button pure-button-primary" href="{{ url_for('backups.request_backup') }}">{{ _('Create backup') }}</a>
+            {% if available_backups %}
+                <a class="pure-button button-small button-error " href="{{ url_for('backups.remove_backups') }}">{{ _('Remove backups') }}</a>
+            {% endif %}
+        </div>
+    </div>
+
+
+{% endblock %}

changedetectionio/blueprint/browser_steps/__init__.py

@@ -174,7 +174,7 @@ def construct_blueprint(datastore: ChangeDetectionStore):
     browser_steps_blueprint = Blueprint('browser_steps', __name__, template_folder="templates")
 
     async def start_browsersteps_session(watch_uuid):
-        from changedetectionio.browser_steps import browser_steps
+        from . import browser_steps
         import time
         from playwright.async_api import async_playwright
 
@@ -238,6 +238,7 @@ def construct_blueprint(datastore: ChangeDetectionStore):
     @browser_steps_blueprint.route("/browsersteps_start_session", methods=['GET'])
     def browsersteps_start_session():
         # A new session was requested, return sessionID
+        import asyncio
         import uuid
         browsersteps_session_id = str(uuid.uuid4())
         watch_uuid = request.args.get('uuid')
@@ -284,8 +285,8 @@ def construct_blueprint(datastore: ChangeDetectionStore):
         watch = datastore.data['watching'].get(uuid)
         filename = f"step_before-{step_n}.jpeg" if request.args.get('type', '') == 'before' else f"step_{step_n}.jpeg"
 
-        if step_n and watch and os.path.isfile(os.path.join(watch.data_dir, filename)):
-            response = make_response(send_from_directory(directory=watch.data_dir, path=filename))
+        if step_n and watch and os.path.isfile(os.path.join(watch.watch_data_dir, filename)):
+            response = make_response(send_from_directory(directory=watch.watch_data_dir, path=filename))
             response.headers['Content-type'] = 'image/jpeg'
             response.headers['Cache-Control'] = 'no-cache, no-store, must-revalidate'
             response.headers['Pragma'] = 'no-cache'
@@ -300,10 +301,11 @@ def construct_blueprint(datastore: ChangeDetectionStore):
     @browser_steps_blueprint.route("/browsersteps_update", methods=['POST'])
     def browsersteps_ui_update():
         import base64
+        import playwright._impl._errors
+        from changedetectionio.blueprint.browser_steps import browser_steps
 
-        remaining = 0
+        remaining =0
         uuid = request.args.get('uuid')
-        goto_website_url_first_step = request.args.get('goto_website_url_first_step')
 
         browsersteps_session_id = request.args.get('browsersteps_session_id')
 
@@ -314,33 +316,33 @@ def construct_blueprint(datastore: ChangeDetectionStore):
             return make_response('No session exists under that ID', 500)
 
         is_last_step = False
-
-        # @todo - should always be an existing session
-        if goto_website_url_first_step:
-            logger.debug("Going to site (requested automatically before stepping)..")
-            step_operation = "Goto site"
-            step_selector = None
-            step_optional_value = None
-        else:
+        # Actions - step/apply/etc, do the thing and return state
+        if request.method == 'POST':
+            # @todo - should always be an existing session
             step_operation = request.form.get('operation')
             step_selector = request.form.get('selector')
             step_optional_value = request.form.get('optional_value')
             is_last_step = strtobool(request.form.get('is_last_step'))
 
-        try:
-            # Run the async call_action method in the dedicated browser steps event loop
-            run_async_in_browser_loop(
-                browsersteps_sessions[browsersteps_session_id]['browserstepper'].call_action(
-                    action_name=step_operation,
-                    selector=step_selector,
-                    optional_value=step_optional_value
+            try:
+                # Run the async call_action method in the dedicated browser steps event loop
+                run_async_in_browser_loop(
+                    browsersteps_sessions[browsersteps_session_id]['browserstepper'].call_action(
+                        action_name=step_operation,
+                        selector=step_selector,
+                        optional_value=step_optional_value
+                    )
                 )
-            )
 
-        except Exception as e:
-            logger.error(f"Exception when calling step operation {step_operation} {str(e)}")
-            # Try to find something of value to give back to the user
-            return make_response(str(e).splitlines()[0], 401)
+            except Exception as e:
+                logger.error(f"Exception when calling step operation {step_operation} {str(e)}")
+                # Try to find something of value to give back to the user
+                return make_response(str(e).splitlines()[0], 401)
+
+
+#        if not this_session.page:
+#            cleanup_playwright_session()
+#            return make_response('Browser session ran out of time :( Please reload this page.', 401)
 
         # Screenshots and other info only needed on requesting a step (POST)
         try:
@@ -348,7 +350,7 @@ def construct_blueprint(datastore: ChangeDetectionStore):
             (screenshot, xpath_data) = run_async_in_browser_loop(
                 browsersteps_sessions[browsersteps_session_id]['browserstepper'].get_current_state()
             )
-
+                
             if is_last_step:
                 watch = datastore.data['watching'].get(uuid)
                 u = browsersteps_sessions[browsersteps_session_id]['browserstepper'].page.url

changedetectionio/blueprint/browser_steps/browser_steps.py

@@ -8,17 +8,6 @@ from changedetectionio.content_fetchers import SCREENSHOT_MAX_HEIGHT_DEFAULT
 from changedetectionio.content_fetchers.base import manage_user_agent
 from changedetectionio.jinja2_custom import render as jinja_render
 
-def browser_steps_get_valid_steps(browser_steps: list):
-    if browser_steps is not None and len(browser_steps):
-        valid_steps = list(filter(
-            lambda s: (s['operation'] and len(s['operation']) and s['operation'] != 'Choose one'),browser_steps))
-
-        # Just incase they selected Goto site by accident with older JS
-        if valid_steps and valid_steps[0]['operation'] == 'Goto site':
-            del(valid_steps[0])
-
-        return valid_steps
-    return []
 
 
 # Two flags, tell the JS which of the "Selector" or "Value" field should be enabled in the front end

changedetectionio/blueprint/imports/__init__.py

@@ -14,7 +14,7 @@ def construct_blueprint(datastore: ChangeDetectionStore, update_q, queuedWatchMe
         from changedetectionio import forms
 #
         if request.method == 'POST':
-#            from changedetectionio import worker_pool
+#            from changedetectionio import worker_handler
 
             from changedetectionio.blueprint.imports.importer import (
                 import_url_list,
@@ -26,13 +26,12 @@ def construct_blueprint(datastore: ChangeDetectionStore, update_q, queuedWatchMe
             # URL List import
             if request.values.get('urls') and len(request.values.get('urls').strip()):
                 # Import and push into the queue for immediate update check
-                from changedetectionio import processors
                 importer_handler = import_url_list()
-                importer_handler.run(data=request.values.get('urls'), flash=flash, datastore=datastore, processor=request.values.get('processor', processors.get_default_processor()))
+                importer_handler.run(data=request.values.get('urls'), flash=flash, datastore=datastore, processor=request.values.get('processor', 'text_json_diff'))
                 logger.debug(f"Imported {len(importer_handler.new_uuids)} new UUIDs")
                 # Dont' add to queue because scheduler can see that they haven't been checked and will add them to the queue
 #                for uuid in importer_handler.new_uuids:
-#                    worker_pool.queue_item_async_safe(update_q, queuedWatchMetaData.PrioritizedItem(priority=1, item={'uuid': uuid}))
+#                    worker_handler.queue_item_async_safe(update_q, queuedWatchMetaData.PrioritizedItem(priority=1, item={'uuid': uuid}))
 
                 if len(importer_handler.remaining_data) == 0:
                     return redirect(url_for('watchlist.index'))
@@ -46,7 +45,7 @@ def construct_blueprint(datastore: ChangeDetectionStore, update_q, queuedWatchMe
                 d_importer.run(data=request.values.get('distill-io'), flash=flash, datastore=datastore)
                 # Dont' add to queue because scheduler can see that they haven't been checked and will add them to the queue
 #                for uuid in importer_handler.new_uuids:
-#                    worker_pool.queue_item_async_safe(update_q, queuedWatchMetaData.PrioritizedItem(priority=1, item={'uuid': uuid}))
+#                    worker_handler.queue_item_async_safe(update_q, queuedWatchMetaData.PrioritizedItem(priority=1, item={'uuid': uuid}))
 
 
             # XLSX importer
@@ -71,7 +70,7 @@ def construct_blueprint(datastore: ChangeDetectionStore, update_q, queuedWatchMe
 
                 # Dont' add to queue because scheduler can see that they haven't been checked and will add them to the queue
 #                for uuid in importer_handler.new_uuids:
-#                    worker_pool.queue_item_async_safe(update_q, queuedWatchMetaData.PrioritizedItem(priority=1, item={'uuid': uuid}))
+#                    worker_handler.queue_item_async_safe(update_q, queuedWatchMetaData.PrioritizedItem(priority=1, item={'uuid': uuid}))
 
 
         # Could be some remaining, or we could be on GET

changedetectionio/blueprint/imports/importer.py

@@ -62,7 +62,7 @@ class import_url_list(Importer):
                 extras = None
                 if processor:
                     extras = {'processor': processor}
-                new_uuid = datastore.add_watch(url=url.strip(), tag=tags, save_immediately=False, extras=extras)
+                new_uuid = datastore.add_watch(url=url.strip(), tag=tags, write_to_disk_now=False, extras=extras)
 
                 if new_uuid:
                     # Straight into the queue.
@@ -129,7 +129,7 @@ class import_distill_io_json(Importer):
                 new_uuid = datastore.add_watch(url=d['uri'].strip(),
                                                tag=",".join(d.get('tags', [])),
                                                extras=extras,
-                                               save_immediately=False)
+                                               write_to_disk_now=False)
 
                 if new_uuid:
                     # Straight into the queue.
@@ -204,7 +204,7 @@ class import_xlsx_wachete(Importer):
                     new_uuid = datastore.add_watch(url=data['url'].strip(),
                                                    extras=extras,
                                                    tag=data.get('folder'),
-                                                   save_immediately=False)
+                                                   write_to_disk_now=False)
                     if new_uuid:
                         # Straight into the queue.
                         self.new_uuids.append(new_uuid)
@@ -287,7 +287,7 @@ class import_xlsx_custom(Importer):
                     new_uuid = datastore.add_watch(url=url,
                                                    extras=extras,
                                                    tag=tags,
-                                                   save_immediately=False)
+                                                   write_to_disk_now=False)
                     if new_uuid:
                         # Straight into the queue.
                         self.new_uuids.append(new_uuid)

changedetectionio/blueprint/imports/templates/import.html

@@ -16,11 +16,6 @@
         <form class="pure-form" action="{{url_for('imports.import_page')}}" method="POST" enctype="multipart/form-data">
             <input type="hidden" name="csrf_token" value="{{ csrf_token() }}">
             <div class="tab-pane-inner" id="url-list">
-
-                <p>
-                {{ _('Restoring changedetection.io backups is in the') }}<a href="{{ url_for('backups.restore.restore') }}"> {{ _('backups section') }}</a>.
-                <br>
-                </p>
                 <div class="pure-control-group">
                         {{ _('Enter one URL per line, and optionally add tags for each URL after a space, delineated by comma (,):') }}
                         <br>
@@ -42,6 +37,9 @@
             </div>
 
             <div class="tab-pane-inner" id="distill-io">
+
+
+
                     <div class="pure-control-group">
                         {{ _('Copy and Paste your Distill.io watch \'export\' file, this should be a JSON file.') }}<br>
                         {{ _('This is') }} <i>{{ _('experimental') }}</i>, {{ _('supported fields are') }} <code>name</code>, <code>uri</code>, <code>tags</code>, <code>config:selections</code>, {{ _('the rest (including') }} <code>schedule</code>) {{ _('are ignored.') }}
@@ -51,6 +49,8 @@
                         {{ _('Be sure to set your default fetcher to Chrome if required.') }}<br>
                         </p>
                     </div>
+
+
                     <textarea name="distill-io" class="pure-input-1-2" style="width: 100%;
                                 font-family:monospace;
                                 white-space: pre;
@@ -114,7 +114,6 @@
                 </div>
             </div>
             <button type="submit" class="pure-button pure-input-1-2 pure-button-primary">{{ _('Import') }}</button>
-
         </form>
 
     </div>

changedetectionio/blueprint/price_data_follower/__init__.py

@@ -4,7 +4,7 @@ from flask import Blueprint, flash, redirect, url_for
 from flask_login import login_required
 from changedetectionio.store import ChangeDetectionStore
 from changedetectionio import queuedWatchMetaData
-from changedetectionio import worker_pool
+from changedetectionio import worker_handler
 from queue import PriorityQueue
 
 PRICE_DATA_TRACK_ACCEPT = 'accepted'
@@ -20,15 +20,13 @@ def construct_blueprint(datastore: ChangeDetectionStore, update_q: PriorityQueue
         datastore.data['watching'][uuid]['track_ldjson_price_data'] = PRICE_DATA_TRACK_ACCEPT
         datastore.data['watching'][uuid]['processor'] = 'restock_diff'
         datastore.data['watching'][uuid].clear_watch()
-        datastore.data['watching'][uuid].commit()
-        worker_pool.queue_item_async_safe(update_q, queuedWatchMetaData.PrioritizedItem(priority=1, item={'uuid': uuid}))
+        worker_handler.queue_item_async_safe(update_q, queuedWatchMetaData.PrioritizedItem(priority=1, item={'uuid': uuid}))
         return redirect(url_for("watchlist.index"))
 
     @login_required
     @price_data_follower_blueprint.route("/<string:uuid>/reject", methods=['GET'])
     def reject(uuid):
         datastore.data['watching'][uuid]['track_ldjson_price_data'] = PRICE_DATA_TRACK_REJECT
-        datastore.data['watching'][uuid].commit()
         return redirect(url_for("watchlist.index"))
 
 

changedetectionio/blueprint/rss/single_watch.py

@@ -37,8 +37,6 @@ def construct_single_watch_routes(rss_blueprint, datastore):
 
         rss_content_format = datastore.data['settings']['application'].get('rss_content_format')
 
-        if uuid == 'first':
-            uuid = list(datastore.data['watching'].keys()).pop()
         # Get the watch by UUID
         watch = datastore.data['watching'].get(uuid)
         if not watch:

changedetectionio/blueprint/settings/__init__.py

@@ -1,9 +1,8 @@
 import os
 from copy import deepcopy
-from datetime import datetime, timedelta
+from datetime import datetime
 from zoneinfo import ZoneInfo, available_timezones
 import secrets
-import time
 import flask_login
 from flask import Blueprint, render_template, request, redirect, url_for, flash
 from flask_babel import gettext
@@ -75,37 +74,26 @@ def construct_blueprint(datastore: ChangeDetectionStore):
                     del (app_update['password'])
 
                 datastore.data['settings']['application'].update(app_update)
-
+                
                 # Handle dynamic worker count adjustment
                 old_worker_count = datastore.data['settings']['requests'].get('workers', 1)
                 new_worker_count = form.data['requests'].get('workers', 1)
-
+                
                 datastore.data['settings']['requests'].update(form.data['requests'])
-                datastore.commit()
-
-                # Clear all checksums to force reprocessing with new settings
-                # Global settings can affect watch behavior (filters, rendering, etc.)
-                datastore.clear_all_last_checksums()
-
+                
                 # Adjust worker count if it changed
                 if new_worker_count != old_worker_count:
-                    from changedetectionio import worker_pool
+                    from changedetectionio import worker_handler
                     from changedetectionio.flask_app import update_q, notification_q, app, datastore as ds
-
-                    # Check CPU core availability and warn if worker count is high
-                    cpu_count = os.cpu_count()
-                    if cpu_count and new_worker_count >= (cpu_count * 0.9):
-                        flash(gettext("Warning: Worker count ({}) is close to or exceeds available CPU cores ({})").format(
-                            new_worker_count, cpu_count), 'warning')
-
-                    result = worker_pool.adjust_async_worker_count(
+                    
+                    result = worker_handler.adjust_async_worker_count(
                         new_count=new_worker_count,
                         update_q=update_q,
                         notification_q=notification_q,
                         app=app,
                         datastore=ds
                     )
-
+                    
                     if result['status'] == 'success':
                         flash(gettext("Worker count adjusted: {}").format(result['message']), 'notice')
                     elif result['status'] == 'not_supported':
@@ -115,11 +103,13 @@ def construct_blueprint(datastore: ChangeDetectionStore):
 
                 if not os.getenv("SALTED_PASS", False) and len(form.application.form.password.encrypted_password):
                     datastore.data['settings']['application']['password'] = form.application.form.password.encrypted_password
-                    datastore.commit()
+                    datastore.needs_write_urgent = True
                     flash(gettext("Password protection enabled."), 'notice')
                     flask_login.logout_user()
                     return redirect(url_for('watchlist.index'))
 
+                datastore.needs_write_urgent = True
+
                 # Also save plugin settings from the same form submission
                 plugin_tabs_list = get_plugin_settings_tabs()
                 for tab in plugin_tabs_list:
@@ -147,9 +137,6 @@ def construct_blueprint(datastore: ChangeDetectionStore):
         active_plugins = get_active_plugins()
         python_version = f"{sys.version_info.major}.{sys.version_info.minor}.{sys.version_info.micro}"
 
-        # Calculate uptime in seconds
-        uptime_seconds = time.time() - datastore.start_time
-
         # Get plugin settings tabs and instantiate forms
         plugin_tabs = get_plugin_settings_tabs()
         plugin_forms = {}
@@ -168,7 +155,6 @@ def construct_blueprint(datastore: ChangeDetectionStore):
                                 active_plugins=active_plugins,
                                 api_key=datastore.data['settings']['application'].get('api_access_token'),
                                 python_version=python_version,
-                                uptime_seconds=uptime_seconds,
                                 available_timezones=sorted(available_timezones()),
                                 emailprefix=os.getenv('NOTIFICATION_MAIL_BUTTON_PREFIX', False),
                                 extra_notification_token_placeholder_info=datastore.get_unique_notification_token_placeholders_available(),
@@ -189,7 +175,7 @@ def construct_blueprint(datastore: ChangeDetectionStore):
     def settings_reset_api_key():
         secret = secrets.token_hex(16)
         datastore.data['settings']['application']['api_access_token'] = secret
-        datastore.commit()
+        datastore.needs_write_urgent = True
         flash(gettext("API Key was regenerated."))
         return redirect(url_for('settings.settings_page')+'#api')
         
@@ -201,32 +187,4 @@ def construct_blueprint(datastore: ChangeDetectionStore):
                                logs=notification_debug_log if len(notification_debug_log) else ["Notification logs are empty - no notifications sent yet."])
         return output
 
-    @settings_blueprint.route("/toggle-all-paused", methods=['GET'])
-    @login_optionally_required
-    def toggle_all_paused():
-        current_state = datastore.data['settings']['application'].get('all_paused', False)
-        datastore.data['settings']['application']['all_paused'] = not current_state
-        datastore.commit()
-
-        if datastore.data['settings']['application']['all_paused']:
-            flash(gettext("Automatic scheduling paused - checks will not be queued."), 'notice')
-        else:
-            flash(gettext("Automatic scheduling resumed - checks will be queued normally."), 'notice')
-
-        return redirect(url_for('watchlist.index'))
-
-    @settings_blueprint.route("/toggle-all-muted", methods=['GET'])
-    @login_optionally_required
-    def toggle_all_muted():
-        current_state = datastore.data['settings']['application'].get('all_muted', False)
-        datastore.data['settings']['application']['all_muted'] = not current_state
-        datastore.commit()
-
-        if datastore.data['settings']['application']['all_muted']:
-            flash(gettext("All notifications muted."), 'notice')
-        else:
-            flash(gettext("All notifications unmuted."), 'notice')
-
-        return redirect(url_for('watchlist.index'))
-
     return settings_blueprint

changedetectionio/blueprint/settings/templates/settings.html

@@ -25,7 +25,6 @@
             <li class="tab"><a href="#ui-options">{{ _('UI Options') }}</a></li>
             <li class="tab"><a href="#api">{{ _('API') }}</a></li>
             <li class="tab"><a href="#rss">{{ _('RSS') }}</a></li>
-            <li class="tab"><a href="{{ url_for('backups.create') }}">{{ _('Backups') }}</a></li>
             <li class="tab"><a href="#timedate">{{ _('Time & Date') }}</a></li>
             <li class="tab"><a href="#proxies">{{ _('CAPTCHA & Proxies') }}</a></li>
             {% if plugin_tabs %}
@@ -54,50 +53,33 @@
                     </div>
                     <div class="pure-control-group">
                         {{ render_field(form.application.form.filter_failure_notification_threshold_attempts, class="filter_failure_notification_threshold_attempts") }}
-                        <span class="pure-form-message-inline">{{ _('After this many consecutive times that the CSS/xPath filter is missing, send a notification') }}
+                        <span class="pure-form-message-inline">After this many consecutive times that the CSS/xPath filter is missing, send a notification
                             <br>
-                        {{ _('Set to') }} <strong>0</strong> {{ _('to disable') }}
+                        Set to <strong>0</strong> to disable
                         </span>
                     </div>
-                    <div class="pure-control-group">
-                        {{ render_field(form.application.form.history_snapshot_max_length, class="history_snapshot_max_length") }}
-                        <span class="pure-form-message-inline">{{ _('Limit collection of history snapshots for each watch to this number of history items.') }}
-                            <br>
-                        {{ _('Set to empty to disable / no limit') }}
-                        </span>
-                    </div>
-
                     <div class="pure-control-group">
                         {% if not hide_remove_pass %}
                             {% if current_user.is_authenticated %}
                                 {{ render_button(form.application.form.removepassword_button) }}
                             {% else %}
                             {{ render_field(form.application.form.password) }}
-                            <span class="pure-form-message-inline">{{ _('Password protection for your changedetection.io application.') }}</span>
+                            <span class="pure-form-message-inline">Password protection for your changedetection.io application.</span>
                             {% endif %}
                         {% else %}
-                            <span class="pure-form-message-inline">{{ _('Password is locked.') }}</span>
+                            <span class="pure-form-message-inline">Password is locked.</span>
                         {% endif %}
                     </div>
 
                     <div class="pure-control-group">
                         {{ render_checkbox_field(form.application.form.shared_diff_access, class="shared_diff_access") }}
-                        <span class="pure-form-message-inline">{{ _('Allow access to the watch change history page when password is enabled (Good for sharing the diff page)') }}</span>
+                        <span class="pure-form-message-inline">Allow access to the watch change history page when password is enabled (Good for sharing the diff page)
+                        </span>
                     </div>
                     <div class="pure-control-group">
                         {{ render_checkbox_field(form.application.form.empty_pages_are_a_change) }}
-                        <span class="pure-form-message-inline">{{ _('When a request returns no content, or the HTML does not contain any text, is this considered a change?') }}</span>
+                        <span class="pure-form-message-inline">When a request returns no content, or the HTML does not contain any text, is this considered a change?</span>
                     </div>
-                {% if form.requests.proxy %}
-                    <div>
-                    <br>
-                        <div class="inline-radio">
-                            {{ render_field(form.requests.form.proxy, class="fetch-backend-proxy") }}
-                            <span class="pure-form-message-inline">{{ _('Choose a default proxy for all watches') }}</span>
-                        </div>
-                    </div>
-                {% endif %}
-
                 </fieldset>
             </div>
 
@@ -108,8 +90,8 @@
                 <div class="pure-control-group" id="notification-base-url">
                     {{ render_field(form.application.form.base_url, class="m-d") }}
                     <span class="pure-form-message-inline">
-                        {{ _('Base URL used for the') }} <code>{{ '{{ base_url }}' }}</code> {{ _('token in notification links.') }}<br>
-                        {{ _('Default value is the system environment variable') }} '<code>BASE_URL</code>' - <a href="https://github.com/dgtlmoon/changedetection.io/wiki/Configurable-BASE_URL-setting">{{ _('read more here') }}</a>.
+                        Base URL used for the <code>{{ '{{ base_url }}' }}</code> token in notification links.<br>
+                        Default value is the system environment variable '<code>BASE_URL</code>' - <a href="https://github.com/dgtlmoon/changedetection.io/wiki/Configurable-BASE_URL-setting">read more here</a>.
                     </span>
                 </div>
             </div>
@@ -118,15 +100,15 @@
                 <div class="pure-control-group inline-radio">
                     {{ render_field(form.application.form.fetch_backend, class="fetch-backend") }}
                     <span class="pure-form-message-inline">
-                        <p>{{ _('Use the') }} <strong>{{ _('Basic') }}</strong> {{ _('method (default) where your watched sites don\'t need Javascript to render.') }}</p>
-                        <p>{{ _('The') }} <strong>{{ _('Chrome/Javascript') }}</strong> {{ _('method requires a network connection to a running WebDriver+Chrome server, set by the ENV var') }} 'WEBDRIVER_URL'. </p>
+                        <p>Use the <strong>Basic</strong> method (default) where your watched sites don't need Javascript to render.</p>
+                        <p>The <strong>Chrome/Javascript</strong> method requires a network connection to a running WebDriver+Chrome server, set by the ENV var 'WEBDRIVER_URL'. </p>
                     </span>
                 </div>
                 <fieldset class="pure-group" id="webdriver-override-options" data-visible-for="application-fetch_backend=html_webdriver">
                     <div class="pure-form-message-inline">
-                        <strong>{{ _('If you\'re having trouble waiting for the page to be fully rendered (text missing etc), try increasing the \'wait\' time here.') }}</strong>
+                        <strong>If you're having trouble waiting for the page to be fully rendered (text missing etc), try increasing the 'wait' time here.</strong>
                         <br>
-                        {{ _('This will wait') }} <i>n</i> {{ _('seconds before extracting the text.') }}
+                        This will wait <i>n</i> seconds before extracting the text.
                     </div>
                     <div class="pure-control-group">
                         {{ render_field(form.application.form.webdriver_delay) }}
@@ -135,27 +117,27 @@
                 <div class="pure-control-group">
                     {{ render_field(form.requests.form.workers) }}
                     {% set worker_info = get_worker_status_info() %}
-                    <span class="pure-form-message-inline">{{ _('Number of concurrent workers to process watches. More workers = faster processing but higher memory usage.') }}<br>
-                    {{ _('Currently running:') }} <strong>{{ worker_info.count }}</strong> {{ _('operational') }} {{ worker_info.type }} {{ _('workers') }}{% if worker_info.active_workers > 0 %} ({{ worker_info.active_workers }} {{ _('actively processing') }}){% endif %}.</span>
+                    <span class="pure-form-message-inline">Number of concurrent workers to process watches. More workers = faster processing but higher memory usage.<br>
+                    Currently running: <strong>{{ worker_info.count }}</strong> operational {{ worker_info.type }} workers{% if worker_info.active_workers > 0 %} ({{ worker_info.active_workers }} actively processing){% endif %}.</span>
                 </div>
                 <div class="pure-control-group">
                     {{ render_field(form.requests.form.jitter_seconds, class="jitter_seconds") }}
-                    <span class="pure-form-message-inline">{{ _('Example - 3 seconds random jitter could trigger up to 3 seconds earlier or up to 3 seconds later') }}</span>
+                    <span class="pure-form-message-inline">Example - 3 seconds random jitter could trigger up to 3 seconds earlier or up to 3 seconds later</span>
                 </div>
                 <div class="pure-control-group">
                     {{ render_field(form.requests.form.timeout) }}
-                    <span class="pure-form-message-inline">{{ _('For regular plain requests (not chrome based), maximum number of seconds until timeout, 1-999.') }}</span><br>
+                    <span class="pure-form-message-inline">For regular plain requests (not chrome based), maximum number of seconds until timeout, 1-999.</span><br>
                 </div>
                 <div class="pure-control-group inline-radio">
                     {{ render_field(form.requests.form.default_ua) }}
                     <span class="pure-form-message-inline">
-                        {{ _('Applied to all requests.') }}<br><br>
-                        {{ _('Note: Simply changing the User-Agent often does not defeat anti-robot technologies, it\'s important to consider') }} <a href="https://changedetection.io/tutorial/what-are-main-types-anti-robot-mechanisms">{{ _('all of the ways that the browser is detected') }}</a>.
+                        Applied to all requests.<br><br>
+                        Note: Simply changing the User-Agent often does not defeat anti-robot technologies, it's important to consider <a href="https://changedetection.io/tutorial/what-are-main-types-anti-robot-mechanisms">all of the ways that the browser is detected</a>.
                     </span>
                 </div>
                 <div class="pure-control-group">
                                         <br>
-                    {{ _('Tip:') }} <a href="https://github.com/dgtlmoon/changedetection.io/wiki/Proxy-configuration#brightdata-proxy-support">{{ _('Connect using Bright Data and Oxylabs Proxies, find out more here.') }}</a>
+                    Tip: <a href="https://github.com/dgtlmoon/changedetection.io/wiki/Proxy-configuration#brightdata-proxy-support">Connect using Bright Data and Oxylabs Proxies, find out more here.</a>
 
                 </div>
             </div>
@@ -164,15 +146,15 @@
 
                     <fieldset class="pure-group">
                     {{ render_checkbox_field(form.application.form.ignore_whitespace) }}
-                    <span class="pure-form-message-inline">{{ _('Ignore whitespace, tabs and new-lines/line-feeds when considering if a change was detected.') }}<br>
-                    <i>{{ _('Note:') }}</i> {{ _('Changing this will change the status of your existing watches, possibly trigger alerts etc.') }}
+                    <span class="pure-form-message-inline">Ignore whitespace, tabs and new-lines/line-feeds when considering if a change was detected.<br>
+                    <i>Note:</i> Changing this will change the status of your existing watches, possibly trigger alerts etc.
                     </span>
                     </fieldset>
                 <fieldset class="pure-group">
                     {{ render_checkbox_field(form.application.form.render_anchor_tag_content) }}
-                    <span class="pure-form-message-inline">{{ _('Render anchor tag content, default disabled, when enabled renders links as') }} <code>(link text)[https://somesite.com]</code>
+                    <span class="pure-form-message-inline">Render anchor tag content, default disabled, when enabled renders links as <code>(link text)[https://somesite.com]</code>
                         <br>
-                    <i>{{ _('Note:') }}</i> {{ _('Changing this could affect the content of your existing watches, possibly trigger alerts etc.') }}
+                    <i>Note:</i> Changing this could affect the content of your existing watches, possibly trigger alerts etc.
                     </span>
                     </fieldset>
                     <fieldset class="pure-group">
@@ -183,9 +165,9 @@ nav
 //*[contains(text(), 'Advertisement')]") }}
                       <span class="pure-form-message-inline">
                         <ul>
-                          <li> {{ _('Remove HTML element(s) by CSS and XPath selectors before text conversion.') }} </li>
-                          <li> {{ _('Don\'t paste HTML here, use only CSS and XPath selectors') }} </li>
-                          <li> {{ _('Add multiple elements, CSS or XPath selectors per line to ignore multiple parts of the HTML.') }} </li>
+                          <li> Remove HTML element(s) by CSS and XPath selectors before text conversion. </li>
+                          <li> Don't paste HTML here, use only CSS and XPath selectors </li>
+                          <li> Add multiple elements, CSS or XPath selectors per line to ignore multiple parts of the HTML. </li>
                         </ul>
                       </span>
                     </fieldset>
@@ -193,50 +175,50 @@ nav
                     {{ render_field(form.application.form.global_ignore_text, rows=5, placeholder="Some text to ignore in a line
 /some.regex\d{2}/ for case-INsensitive regex
                     ") }}
-                    <span class="pure-form-message-inline">{{ _('Note: This is applied globally in addition to the per-watch rules.') }}</span><br>
+                    <span class="pure-form-message-inline">Note: This is applied globally in addition to the per-watch rules.</span><br>
                     <span class="pure-form-message-inline">
                         <ul>
-                            <li>{{ _('Matching text will be') }} <strong>{{ _('ignored') }}</strong> {{ _('in the text snapshot (you can still see it but it wont trigger a change)') }}</li>
-                            <li>{{ _('Note: This is applied globally in addition to the per-watch rules.') }}</li>
-                            <li>{{ _('Each line processed separately, any line matching will be ignored (removed before creating the checksum)') }}</li>
-                            <li>{{ _('Regular Expression support, wrap the entire line in forward slash') }} <code>/regex/</code></li>
-                            <li>{{ _('Changing this will affect the comparison checksum which may trigger an alert') }}</li>
+                            <li>Matching text will be <strong>ignored</strong> in the text snapshot (you can still see it but it wont trigger a change)</li>
+                            <li>Note: This is applied globally in addition to the per-watch rules.</li>
+                            <li>Each line processed separately, any line matching will be ignored (removed before creating the checksum)</li>
+                            <li>Regular Expression support, wrap the entire line in forward slash <code>/regex/</code></li>
+                            <li>Changing this will affect the comparison checksum which may trigger an alert</li>
                         </ul>
                      </span>
                     </fieldset>
                     <fieldset class="pure-group">
                         {{ render_checkbox_field(form.application.form.strip_ignored_lines) }}
-                        <span class="pure-form-message-inline">{{ _('Remove any text that appears in the "Ignore text" from the output (otherwise its just ignored for change-detection)') }}<br>
-                        <i>{{ _('Note:') }}</i> {{ _('Changing this will change the status of your existing watches, possibly trigger alerts etc.') }}
+                        <span class="pure-form-message-inline">Remove any text that appears in the "Ignore text" from the output (otherwise its just ignored for change-detection)<br>
+                        <i>Note:</i> Changing this will change the status of your existing watches, possibly trigger alerts etc.
                         </span>
                     </fieldset>
            </div>
 
             <div class="tab-pane-inner" id="api">
-                <h4>{{ _('API Access') }}</h4>
-                <p>{{ _('Drive your changedetection.io via API, More about') }} <a href="https://changedetection.io/docs/api_v1/index.html">{{ _('API access and examples here') }}</a>.</p>
+                <h4>API Access</h4>
+                <p>Drive your changedetection.io via API, More about <a href="https://changedetection.io/docs/api_v1/index.html">API access and examples here</a>.</p>
 
                 <div class="pure-control-group">
                     {{ render_checkbox_field(form.application.form.api_access_token_enabled) }}
-                    <div class="pure-form-message-inline">{{ _('Restrict API access limit by using') }} <code>x-api-key</code> {{ _('header - required for the Chrome Extension to work') }}</div><br>
-                    <div class="pure-form-message-inline"><br>{{ _('API Key') }} <span id="api-key">{{api_key}}</span>
-                        <span style="display:none;" id="api-key-copy" >{{ _('copy') }}</span>
+                    <div class="pure-form-message-inline">Restrict API access limit by using <code>x-api-key</code> header - required for the Chrome Extension to work</div><br>
+                    <div class="pure-form-message-inline"><br>API Key <span id="api-key">{{api_key}}</span>
+                        <span style="display:none;" id="api-key-copy" >copy</span>
                     </div>
                 </div>
                 <div class="pure-control-group">
-                    <a href="{{url_for('settings.settings_reset_api_key')}}" class="pure-button button-small button-cancel">{{ _('Regenerate API key') }}</a>
+                    <a href="{{url_for('settings.settings_reset_api_key')}}" class="pure-button button-small button-cancel">Regenerate API key</a>
                 </div>
                 <div class="pure-control-group">
-                    <h4>{{ _('Chrome Extension') }}</h4>
-                    <p>{{ _('Easily add any web-page to your changedetection.io installation from within Chrome.') }}</p>
-                    <strong>{{ _('Step 1') }}</strong> {{ _('Install the extension,') }} <strong>{{ _('Step 2') }}</strong> {{ _('Navigate to this page,') }}
-                    <strong>{{ _('Step 3') }}</strong> {{ _('Open the extension from the toolbar and click') }} "<i>{{ _('Sync API Access') }}</i>"
+                    <h4>Chrome Extension</h4>
+                    <p>Easily add any web-page to your changedetection.io installation from within Chrome.</p>
+                    <strong>Step 1</strong> Install the extension, <strong>Step 2</strong> Navigate to this page,
+                    <strong>Step 3</strong> Open the extension from the toolbar and click "<i>Sync API Access</i>"
                     <p>
                         <a id="chrome-extension-link"
-                           title="{{ _('Try our new Chrome Extension!') }}"
+                           title="Try our new Chrome Extension!"
                            href="https://chromewebstore.google.com/detail/changedetectionio-website/kefcfmgmlhmankjmnbijimhofdjekbop">
-                            <img alt="{{ _('Chrome store icon') }}" src="{{ url_for('static_content', group='images', filename='google-chrome-icon.png') }}" >
-                            {{ _('Chrome Webstore') }}
+                            <img alt="Chrome store icon" src="{{ url_for('static_content', group='images', filename='google-chrome-icon.png') }}" >
+                            Chrome Webstore
                         </a>
                     </p>
                 </div>
@@ -247,20 +229,20 @@ nav
                 </div>
                 <div class="pure-control-group">
                     {{ render_field(form.application.form.rss_diff_length) }}
-                    <span class="pure-form-message-inline">{{ _('Maximum number of history snapshots to include in the watch specific RSS feed.') }}</span>
+                    <span class="pure-form-message-inline">Maximum number of history snapshots to include in the watch specific RSS feed.</span>
                 </div>
                 <div class="pure-control-group">
                     {{ render_checkbox_field(form.application.form.rss_reader_mode) }}
-                    <span class="pure-form-message-inline">{{ _('For watching other RSS feeds - When watching RSS/Atom feeds, convert them into clean text for better change detection.') }}</span>
+                    <span class="pure-form-message-inline">For watching other RSS feeds - When watching RSS/Atom feeds, convert them into clean text for better change detection.</span>
                 </div>
                 <div class="pure-control-group grey-form-border">
                     <div class="pure-control-group">
                         {{ render_field(form.application.form.rss_content_format) }}
-                        <span class="pure-form-message-inline">{{ _('Does your reader support HTML? Set it here') }}</span>
+                        <span class="pure-form-message-inline">Does your reader support HTML? Set it here</span>
                     </div>
                     <div class="pure-control-group">
                         {{ render_field(form.application.form.rss_template_type) }}
-                        <span class="pure-form-message-inline">{{ _('\'System default\' for the same template for all items, or re-use your "Notification Body" as the template.') }}</span>
+                        <span class="pure-form-message-inline">'System default' for the same template for all items, or re-use your "Notification Body" as the template.</span>
                     </div>
                     <div>
                        {{ render_field(form.application.form.rss_template_override) }}
@@ -273,11 +255,11 @@ nav
             </div>
                 <div class="tab-pane-inner" id="timedate">
                 <div class="pure-control-group">
-                    {{ _('Ensure the settings below are correct, they are used to manage the time schedule for checking your web page watches.') }}
+                    Ensure the settings below are correct, they are used to manage the time schedule for checking your web page watches.
                 </div>
                 <div class="pure-control-group">
-                    <p><strong>{{ _('UTC Time & Date from Server:') }}</strong> <span id="utc-time" >{{ utc_time }}</span></p>
-                    <p><strong>{{ _('Local Time & Date in Browser:') }}</strong> <span class="local-time" data-utc="{{ utc_time }}"></span></p>
+                    <p><strong>UTC Time &amp; Date from Server:</strong> <span id="utc-time" >{{ utc_time }}</span></p>
+                    <p><strong>Local Time &amp; Date in Browser:</strong> <span class="local-time" data-utc="{{ utc_time }}"></span></p>
                     <div>
                        {{ render_field(form.application.form.scheduler_timezone_default) }}
                         <datalist id="timezones" style="display: none;">
@@ -289,22 +271,22 @@ nav
             <div class="tab-pane-inner" id="ui-options">
                 <div class="pure-control-group">
                     {{ render_checkbox_field(form.application.form.ui.form.open_diff_in_new_tab, class="open_diff_in_new_tab") }}
-                    <span class="pure-form-message-inline">{{ _('Enable this setting to open the diff page in a new tab. If disabled, the diff page will open in the current tab.') }}</span>
+                    <span class="pure-form-message-inline">Enable this setting to open the diff page in a new tab. If disabled, the diff page will open in the current tab.</span>
                 </div>
                 <div class="pure-control-group">
                     {{ render_checkbox_field(form.application.form.ui.form.socket_io_enabled, class="socket_io_enabled") }}
-                    <span class="pure-form-message-inline">{{ _('Realtime UI Updates Enabled - (Restart required if this is changed)') }}</span>
+                    <span class="pure-form-message-inline">Realtime UI Updates Enabled - (Restart required if this is changed)</span>
                 </div>
                 <div class="pure-control-group">
                     {{ render_checkbox_field(form.application.form.ui.form.favicons_enabled, class="") }}
-                    <span class="pure-form-message-inline">{{ _('Enable or Disable Favicons next to the watch list') }}</span>
+                    <span class="pure-form-message-inline">Enable or Disable Favicons next to the watch list</span>
                 </div>
                 <div class="pure-control-group">
                     {{ render_checkbox_field(form.application.form.ui.use_page_title_in_list) }}
                 </div>
                 <div class="pure-control-group">
                     {{ render_field(form.application.form.pager_size) }}
-                    <span class="pure-form-message-inline">{{ _('Number of items per page in the watch overview list, 0 to disable.') }}</span>
+                    <span class="pure-form-message-inline">Number of items per page in the watch overview list, 0 to disable.</span>
                 </div>
 
             </div>
@@ -352,12 +334,21 @@ nav
                     </div>
                 </div>
 
-                <p><strong>{{ _('Tip') }}</strong>: {{ _('"Residential" and "Mobile" proxy type can be more successfull than "Data Center" for blocked websites.') }}</p>
+                <p><strong>Tip</strong>: "Residential" and "Mobile" proxy type can be more successfull than "Data Center" for blocked websites.</p>
 
                 <div class="pure-control-group" id="extra-proxies-setting">
                 {{ render_fieldlist_with_inline_errors(form.requests.form.extra_proxies) }}
-                <span class="pure-form-message-inline">{{ _('"Name" will be used for selecting the proxy in the Watch Edit settings') }}</span><br>
-                <span class="pure-form-message-inline">{{ _('SOCKS5 proxies with authentication are only supported with \'plain requests\' fetcher, for other fetchers you should whitelist the IP access instead') }}</span>
+                <span class="pure-form-message-inline">"Name" will be used for selecting the proxy in the Watch Edit settings</span><br>
+                <span class="pure-form-message-inline">SOCKS5 proxies with authentication are only supported with 'plain requests' fetcher, for other fetchers you should whitelist the IP access instead</span>
+                {% if form.requests.proxy %}
+                <div>
+                <br>
+                    <div class="inline-radio">
+                        {{ render_field(form.requests.form.proxy, class="fetch-backend-proxy") }}
+                        <span class="pure-form-message-inline">Choose a default proxy for all watches</span>
+                    </div>
+                </div>
+                {% endif %}
                 </div>
                 <div class="pure-control-group" id="extra-browsers-setting">
                     <p>
@@ -394,7 +385,6 @@ nav
                 {% endfor %}
             {% endif %}
             <div class="tab-pane-inner" id="info">
-                <p><strong>{{ _('Uptime:') }}</strong> {{ uptime_seconds|format_duration }}</p>
                 <p><strong>{{ _('Python version:') }}</strong> {{ python_version }}</p>
                 <p><strong>{{ _('Plugins active:') }}</strong></p>
                 {% if active_plugins %}

changedetectionio/blueprint/tags/__init__.py

@@ -1,7 +1,5 @@
-import threading
 from flask import Blueprint, request, render_template, flash, url_for, redirect
 from flask_babel import gettext
-from loguru import logger
 
 from changedetectionio.store import ChangeDetectionStore
 from changedetectionio.flask_app import login_optionally_required
@@ -57,88 +55,46 @@ def construct_blueprint(datastore: ChangeDetectionStore):
     @tags_blueprint.route("/mute/<string:uuid>", methods=['GET'])
     @login_optionally_required
     def mute(uuid):
-        tag = datastore.data['settings']['application']['tags'].get(uuid)
-        if tag:
-            tag['notification_muted'] = not tag['notification_muted']
-            tag.commit()
+        if datastore.data['settings']['application']['tags'].get(uuid):
+            datastore.data['settings']['application']['tags'][uuid]['notification_muted'] = not datastore.data['settings']['application']['tags'][uuid]['notification_muted']
         return redirect(url_for('tags.tags_overview_page'))
 
     @tags_blueprint.route("/delete/<string:uuid>", methods=['GET'])
     @login_optionally_required
     def delete(uuid):
-        # Delete the tag from settings immediately
+        removed = 0
+        # Delete the tag, and any tag reference
         if datastore.data['settings']['application']['tags'].get(uuid):
             del datastore.data['settings']['application']['tags'][uuid]
 
-        # Remove tag from all watches in background thread to avoid blocking
-        def remove_tag_background(tag_uuid):
-            """Background thread to remove tag from watches - discarded after completion."""
-            removed_count = 0
-            try:
-                for watch_uuid, watch in datastore.data['watching'].items():
-                    if watch.get('tags') and tag_uuid in watch['tags']:
-                        watch['tags'].remove(tag_uuid)
-                        watch.commit()
-                        removed_count += 1
-                logger.info(f"Background: Tag {tag_uuid} removed from {removed_count} watches")
-            except Exception as e:
-                logger.error(f"Error removing tag from watches: {e}")
+        for watch_uuid, watch in datastore.data['watching'].items():
+            if watch.get('tags') and uuid in watch['tags']:
+                removed += 1
+                watch['tags'].remove(uuid)
 
-        # Start daemon thread
-        threading.Thread(target=remove_tag_background, args=(uuid,), daemon=True).start()
-
-        flash(gettext("Tag deleted, removing from watches in background"))
+        flash(gettext("Tag deleted and removed from {} watches").format(removed))
         return redirect(url_for('tags.tags_overview_page'))
 
     @tags_blueprint.route("/unlink/<string:uuid>", methods=['GET'])
     @login_optionally_required
     def unlink(uuid):
-        # Unlink tag from all watches in background thread to avoid blocking
-        def unlink_tag_background(tag_uuid):
-            """Background thread to unlink tag from watches - discarded after completion."""
-            unlinked_count = 0
-            try:
-                for watch_uuid, watch in datastore.data['watching'].items():
-                    if watch.get('tags') and tag_uuid in watch['tags']:
-                        watch['tags'].remove(tag_uuid)
-                        watch.commit()
-                        unlinked_count += 1
-                logger.info(f"Background: Tag {tag_uuid} unlinked from {unlinked_count} watches")
-            except Exception as e:
-                logger.error(f"Error unlinking tag from watches: {e}")
+        unlinked = 0
+        for watch_uuid, watch in datastore.data['watching'].items():
+            if watch.get('tags') and uuid in watch['tags']:
+                unlinked += 1
+                watch['tags'].remove(uuid)
 
-        # Start daemon thread
-        threading.Thread(target=unlink_tag_background, args=(uuid,), daemon=True).start()
-
-        flash(gettext("Unlinking tag from watches in background"))
+        flash(gettext("Tag unlinked removed from {} watches").format(unlinked))
         return redirect(url_for('tags.tags_overview_page'))
 
     @tags_blueprint.route("/delete_all", methods=['GET'])
     @login_optionally_required
     def delete_all():
+        for watch_uuid, watch in datastore.data['watching'].items():
+            watch['tags'] = []
+        datastore.data['settings']['application']['tags'] = {}
 
-        for tag_uuid in list(datastore.data['settings']['application']['tags'].keys()):
-# TagsDict 'del' handler will remove the dir
-            del datastore.data['settings']['application']['tags'][tag_uuid]
-
-
-        # Clear tags from all watches in background thread to avoid blocking
-        def clear_all_tags_background():
-            """Background thread to clear tags from all watches - discarded after completion."""
-            cleared_count = 0
-            try:
-                for watch_uuid, watch in datastore.data['watching'].items():
-                    watch['tags'] = []
-                    watch.commit()
-                    cleared_count += 1
-                logger.info(f"Background: Cleared tags from {cleared_count} watches")
-            except Exception as e:
-                logger.error(f"Error clearing tags from watches: {e}")
-
-        # Start daemon thread
-        threading.Thread(target=clear_all_tags_background, daemon=True).start()
-
-        flash(gettext("All tags deleted, clearing from watches in background"))
+        flash(gettext("All tags deleted"))
         return redirect(url_for('tags.tags_overview_page'))
 
     @tags_blueprint.route("/edit/<string:uuid>", methods=['GET'])
@@ -210,10 +166,10 @@ def construct_blueprint(datastore: ChangeDetectionStore):
         if uuid == 'first':
             uuid = list(datastore.data['settings']['application']['tags'].keys()).pop()
 
-        tag = datastore.data['settings']['application']['tags'].get(uuid)
+        default = datastore.data['settings']['application']['tags'].get(uuid)
 
         form = group_restock_settings_form(formdata=request.form if request.method == 'POST' else None,
-                               data=tag,
+                               data=default,
                                extra_notification_tokens=datastore.get_unique_notification_tokens_available()
                                )
         # @todo subclass form so validation works
@@ -222,18 +178,15 @@ def construct_blueprint(datastore: ChangeDetectionStore):
 #                flash(','.join(l), 'error')
 #           return redirect(url_for('tags.form_tag_edit_submit', uuid=uuid))
 
-        tag.update(form.data)
-        tag['processor'] = 'restock_diff'
-        tag.commit()
-
-        # Clear checksums for all watches using this tag to force reprocessing
-        # Tag changes affect inherited configuration
-        cleared_count = datastore.clear_checksums_for_tag(uuid)
-        logger.info(f"Tag {uuid} updated, cleared {cleared_count} watch checksums")
-
+        datastore.data['settings']['application']['tags'][uuid].update(form.data)
+        datastore.data['settings']['application']['tags'][uuid]['processor'] = 'restock_diff'
+        datastore.needs_write_urgent = True
         flash(gettext("Updated"))
 
         return redirect(url_for('tags.tags_overview_page'))
 
 
+    @tags_blueprint.route("/delete/<string:uuid>", methods=['GET'])
+    def form_tag_delete(uuid):
+        return redirect(url_for('tags.tags_overview_page'))
     return tags_blueprint

changedetectionio/blueprint/ui/__init__.py

@@ -1,6 +1,5 @@
 import time
-import threading
-from flask import Blueprint, request, redirect, url_for, flash, render_template, session, current_app
+from flask import Blueprint, request, redirect, url_for, flash, render_template, session
 from flask_babel import gettext
 from loguru import logger
 
@@ -10,7 +9,7 @@ from changedetectionio.blueprint.ui.notification import construct_blueprint as c
 from changedetectionio.blueprint.ui.views import construct_blueprint as construct_views_blueprint
 from changedetectionio.blueprint.ui import diff, preview
 
-def _handle_operations(op, uuids, datastore, worker_pool, update_q, queuedWatchMetaData, watch_check_update, extra_data=None, emit_flash=True):
+def _handle_operations(op, uuids, datastore, worker_handler, update_q, queuedWatchMetaData, watch_check_update, extra_data=None, emit_flash=True):
     from flask import request, flash
 
     if op == 'delete':
@@ -24,7 +23,6 @@ def _handle_operations(op, uuids, datastore, worker_pool, update_q, queuedWatchM
         for uuid in uuids:
             if datastore.data['watching'].get(uuid):
                 datastore.data['watching'][uuid]['paused'] = True
-                datastore.data['watching'][uuid].commit()
         if emit_flash:
             flash(gettext("{} watches paused").format(len(uuids)))
 
@@ -32,7 +30,6 @@ def _handle_operations(op, uuids, datastore, worker_pool, update_q, queuedWatchM
         for uuid in uuids:
             if datastore.data['watching'].get(uuid):
                 datastore.data['watching'][uuid.strip()]['paused'] = False
-                datastore.data['watching'][uuid].commit()
         if emit_flash:
             flash(gettext("{} watches unpaused").format(len(uuids)))
 
@@ -47,7 +44,6 @@ def _handle_operations(op, uuids, datastore, worker_pool, update_q, queuedWatchM
         for uuid in uuids:
             if datastore.data['watching'].get(uuid):
                 datastore.data['watching'][uuid]['notification_muted'] = True
-                datastore.data['watching'][uuid].commit()
         if emit_flash:
             flash(gettext("{} watches muted").format(len(uuids)))
 
@@ -55,7 +51,6 @@ def _handle_operations(op, uuids, datastore, worker_pool, update_q, queuedWatchM
         for uuid in uuids:
             if datastore.data['watching'].get(uuid):
                 datastore.data['watching'][uuid]['notification_muted'] = False
-                datastore.data['watching'][uuid].commit()
         if emit_flash:
             flash(gettext("{} watches un-muted").format(len(uuids)))
 
@@ -63,7 +58,7 @@ def _handle_operations(op, uuids, datastore, worker_pool, update_q, queuedWatchM
         for uuid in uuids:
             if datastore.data['watching'].get(uuid):
                 # Recheck and require a full reprocessing
-                worker_pool.queue_item_async_safe(update_q, queuedWatchMetaData.PrioritizedItem(priority=1, item={'uuid': uuid}))
+                worker_handler.queue_item_async_safe(update_q, queuedWatchMetaData.PrioritizedItem(priority=1, item={'uuid': uuid}))
         if emit_flash:
             flash(gettext("{} watches queued for rechecking").format(len(uuids)))
 
@@ -71,7 +66,6 @@ def _handle_operations(op, uuids, datastore, worker_pool, update_q, queuedWatchM
         for uuid in uuids:
             if datastore.data['watching'].get(uuid):
                 datastore.data['watching'][uuid]["last_error"] = False
-                datastore.data['watching'][uuid].commit()
         if emit_flash:
             flash(gettext("{} watches errors cleared").format(len(uuids)))
 
@@ -92,7 +86,6 @@ def _handle_operations(op, uuids, datastore, worker_pool, update_q, queuedWatchM
                 datastore.data['watching'][uuid]['notification_body'] = None
                 datastore.data['watching'][uuid]['notification_urls'] = []
                 datastore.data['watching'][uuid]['notification_format'] = USE_SYSTEM_DEFAULT_NOTIFICATION_FORMAT_FOR_WATCH
-                datastore.data['watching'][uuid].commit()
         if emit_flash:
             flash(gettext("{} watches set to use default notification settings").format(len(uuids)))
 
@@ -108,7 +101,6 @@ def _handle_operations(op, uuids, datastore, worker_pool, update_q, queuedWatchM
                             datastore.data['watching'][uuid]['tags'] = []
 
                         datastore.data['watching'][uuid]['tags'].append(tag_uuid)
-                        datastore.data['watching'][uuid].commit()
         if emit_flash:
             flash(gettext("{} watches were tagged").format(len(uuids)))
 
@@ -116,7 +108,7 @@ def _handle_operations(op, uuids, datastore, worker_pool, update_q, queuedWatchM
         for uuid in uuids:
             watch_check_update.send(watch_uuid=uuid)
 
-def construct_blueprint(datastore: ChangeDetectionStore, update_q, worker_pool, queuedWatchMetaData, watch_check_update):
+def construct_blueprint(datastore: ChangeDetectionStore, update_q, worker_handler, queuedWatchMetaData, watch_check_update):
     ui_blueprint = Blueprint('ui', __name__, template_folder="templates")
     
     # Register the edit blueprint
@@ -159,24 +151,9 @@ def construct_blueprint(datastore: ChangeDetectionStore, update_q, worker_pool,
             confirmtext = request.form.get('confirmtext')
 
             if confirmtext == 'clear':
-                # Run in background thread to avoid blocking
-                def clear_history_background():
-                    # Capture UUIDs first to avoid race conditions
-                    watch_uuids = list(datastore.data['watching'].keys())
-                    logger.info(f"Background: Clearing history for {len(watch_uuids)} watches")
-
-                    for uuid in watch_uuids:
-                        try:
-                            datastore.clear_watch_history(uuid)
-                        except Exception as e:
-                            logger.error(f"Error clearing history for watch {uuid}: {e}")
-
-                    logger.info("Background: Completed clearing history")
-
-                # Start daemon thread
-                threading.Thread(target=clear_history_background, daemon=True).start()
-
-                flash(gettext("History clearing started in background"))
+                for uuid in datastore.data['watching'].keys():
+                    datastore.clear_watch_history(uuid)
+                flash(gettext("Cleared snapshot history for all watches"))
             else:
                 flash(gettext('Incorrect confirmation text.'), 'error')
 
@@ -192,37 +169,17 @@ def construct_blueprint(datastore: ChangeDetectionStore, update_q, worker_pool,
         # Save the current newest history as the most recently viewed
         with_errors = request.args.get('with_errors') == "1"
         tag_limit = request.args.get('tag')
+        logger.debug(f"Limiting to tag {tag_limit}")
         now = int(time.time())
+        for watch_uuid, watch in datastore.data['watching'].items():
+            if with_errors and not watch.get('last_error'):
+                continue
 
-        # Mark watches as viewed - use background thread only for large watch counts
-        def mark_viewed_impl():
-            """Mark watches as viewed - can run synchronously or in background thread."""
-            marked_count = 0
-            try:
-                for watch_uuid, watch in datastore.data['watching'].items():
-                    if with_errors and not watch.get('last_error'):
-                        continue
+            if tag_limit and ( not watch.get('tags') or tag_limit not in watch['tags'] ):
+                logger.debug(f"Skipping watch {watch_uuid}")
+                continue
 
-                    if tag_limit and (not watch.get('tags') or tag_limit not in watch['tags']):
-                        continue
-
-                    datastore.set_last_viewed(watch_uuid, now)
-                    marked_count += 1
-
-                logger.info(f"Marking complete: {marked_count} watches marked as viewed")
-            except Exception as e:
-                logger.error(f"Error marking as viewed: {e}")
-
-        # For small watch counts (< 10), run synchronously to avoid race conditions in tests
-        # For larger counts, use background thread to avoid blocking the UI
-        watch_count = len(datastore.data['watching'])
-        if watch_count < 10:
-            # Run synchronously for small watch counts
-            mark_viewed_impl()
-        else:
-            # Start background thread for large watch counts
-            thread = threading.Thread(target=mark_viewed_impl, daemon=True)
-            thread.start()
+            datastore.set_last_viewed(watch_uuid, now)
 
         return redirect(url_for('watchlist.index', tag=tag_limit))
 
@@ -230,14 +187,14 @@ def construct_blueprint(datastore: ChangeDetectionStore, update_q, worker_pool,
     @login_optionally_required
     def form_delete():
         uuid = request.args.get('uuid')
-        # More for testing, possible to return the first/only
-        if uuid == 'first':
-            uuid = list(datastore.data['watching'].keys()).pop()
 
         if uuid != 'all' and not uuid in datastore.data['watching'].keys():
             flash(gettext('The watch by UUID {} does not exist.').format(uuid), 'error')
             return redirect(url_for('watchlist.index'))
 
+        # More for testing, possible to return the first/only
+        if uuid == 'first':
+            uuid = list(datastore.data['watching'].keys()).pop()
         datastore.delete(uuid)
         flash(gettext('Deleted.'))
 
@@ -247,14 +204,14 @@ def construct_blueprint(datastore: ChangeDetectionStore, update_q, worker_pool,
     @login_optionally_required
     def form_clone():
         uuid = request.args.get('uuid')
-
+        # More for testing, possible to return the first/only
         if uuid == 'first':
             uuid = list(datastore.data['watching'].keys()).pop()
 
         new_uuid = datastore.clone(uuid)
 
         if not datastore.data['watching'].get(uuid).get('paused'):
-            worker_pool.queue_item_async_safe(update_q, queuedWatchMetaData.PrioritizedItem(priority=5, item={'uuid': new_uuid}))
+            worker_handler.queue_item_async_safe(update_q, queuedWatchMetaData.PrioritizedItem(priority=5, item={'uuid': new_uuid}))
 
         flash(gettext('Cloned, you are editing the new watch.'))
 
@@ -268,81 +225,35 @@ def construct_blueprint(datastore: ChangeDetectionStore, update_q, worker_pool,
         uuid = request.args.get('uuid')
         with_errors = request.args.get('with_errors') == "1"
 
+        i = 0
+
         if uuid:
-            # Single watch - check if already queued or running
-            if worker_pool.is_watch_running(uuid) or uuid in update_q.get_queued_uuids():
-                flash(gettext("Watch is already queued or being checked."))
-            else:
-                worker_pool.queue_item_async_safe(update_q, queuedWatchMetaData.PrioritizedItem(priority=1, item={'uuid': uuid}))
-                flash(gettext("Queued 1 watch for rechecking."))
+            worker_handler.queue_item_async_safe(update_q, queuedWatchMetaData.PrioritizedItem(priority=1, item={'uuid': uuid}))
+            i += 1
+
         else:
-            # Multiple watches - first count how many need to be queued
-            watches_to_queue = []
+            # Recheck all, including muted
+            # Get most overdue first
             for k in sorted(datastore.data['watching'].items(), key=lambda item: item[1].get('last_checked', 0)):
                 watch_uuid = k[0]
                 watch = k[1]
-                if not watch['paused'] and watch_uuid:
-                    if with_errors and not watch.get('last_error'):
-                        continue
-                    if tag != None and tag not in watch['tags']:
-                        continue
-                    watches_to_queue.append(watch_uuid)
+                if not watch['paused']:
+                    if watch_uuid:
+                        if with_errors and not watch.get('last_error'):
+                            continue
 
-            # If less than 20 watches, queue synchronously for immediate feedback
-            if len(watches_to_queue) < 20:
-                # Get already queued/running UUIDs once (efficient)
-                queued_uuids = set(update_q.get_queued_uuids())
-                running_uuids = set(worker_pool.get_running_uuids())
+                        if tag != None and tag not in watch['tags']:
+                            continue
 
-                # Filter out watches that are already queued or running
-                watches_to_queue_filtered = []
-                for watch_uuid in watches_to_queue:
-                    if watch_uuid not in queued_uuids and watch_uuid not in running_uuids:
-                        watches_to_queue_filtered.append(watch_uuid)
+                        worker_handler.queue_item_async_safe(update_q, queuedWatchMetaData.PrioritizedItem(priority=1, item={'uuid': watch_uuid}))
+                        i += 1
 
-                # Queue only the filtered watches
-                for watch_uuid in watches_to_queue_filtered:
-                    worker_pool.queue_item_async_safe(update_q, queuedWatchMetaData.PrioritizedItem(priority=1, item={'uuid': watch_uuid}))
-
-                # Provide feedback about skipped watches
-                skipped_count = len(watches_to_queue) - len(watches_to_queue_filtered)
-                if skipped_count > 0:
-                    flash(gettext("Queued {} watches for rechecking ({} already queued or running).").format(
-                        len(watches_to_queue_filtered), skipped_count))
-                else:
-                    if len(watches_to_queue_filtered) == 1:
-                        flash(gettext("Queued 1 watch for rechecking."))
-                    else:
-                        flash(gettext("Queued {} watches for rechecking.").format(len(watches_to_queue_filtered)))
-            else:
-                # 20+ watches - queue in background thread to avoid blocking HTTP response
-                # Capture queued/running state before background thread
-                queued_uuids = set(update_q.get_queued_uuids())
-                running_uuids = set(worker_pool.get_running_uuids())
-
-                def queue_watches_background():
-                    """Background thread to queue watches - discarded after completion."""
-                    try:
-                        queued_count = 0
-                        skipped_count = 0
-                        for watch_uuid in watches_to_queue:
-                            # Check if already queued or running (state captured at start)
-                            if watch_uuid not in queued_uuids and watch_uuid not in running_uuids:
-                                worker_pool.queue_item_async_safe(update_q, queuedWatchMetaData.PrioritizedItem(priority=1, item={'uuid': watch_uuid}))
-                                queued_count += 1
-                            else:
-                                skipped_count += 1
-
-                        logger.info(f"Background queueing complete: {queued_count} watches queued, {skipped_count} skipped (already queued/running)")
-                    except Exception as e:
-                        logger.error(f"Error in background queueing: {e}")
-
-                # Start background thread and return immediately
-                thread = threading.Thread(target=queue_watches_background, daemon=True, name="QueueWatches-Background")
-                thread.start()
-
-                # Return immediately with approximate message
-                flash(gettext("Queueing watches for rechecking in background..."))
+        if i == 1:
+            flash(gettext("Queued 1 watch for rechecking."))
+        if i > 1:
+            flash(gettext("Queued {} watches for rechecking.").format(i))
+        if i == 0:
+            flash(gettext("No watches available to recheck."))
 
         return redirect(url_for('watchlist.index', **({'tag': tag} if tag else {})))
 
@@ -357,7 +268,7 @@ def construct_blueprint(datastore: ChangeDetectionStore, update_q, worker_pool,
             extra_data=extra_data,
             queuedWatchMetaData=queuedWatchMetaData,
             uuids=uuids,
-            worker_pool=worker_pool,
+            worker_handler=worker_handler,
             update_q=update_q,
             watch_check_update=watch_check_update,
             op=op,
@@ -375,6 +286,9 @@ def construct_blueprint(datastore: ChangeDetectionStore, update_q, worker_pool,
         import json
         from copy import deepcopy
 
+        # more for testing
+        if uuid == 'first':
+            uuid = list(datastore.data['watching'].keys()).pop()
 
         # copy it to memory as trim off what we dont need (history)
         watch = deepcopy(datastore.data['watching'].get(uuid))
@@ -414,25 +328,4 @@ def construct_blueprint(datastore: ChangeDetectionStore, update_q, worker_pool,
 
         return redirect(url_for('watchlist.index'))
 
-    @ui_blueprint.route("/language/auto-detect", methods=['GET'])
-    def delete_locale_language_session_var_if_it_exists():
-        """Clear the session locale preference to auto-detect from browser Accept-Language header"""
-        if 'locale' in session:
-            session.pop('locale', None)
-            # Refresh Flask-Babel to clear cached locale
-            from flask_babel import refresh
-            refresh()
-            flash(gettext("Language set to auto-detect from browser"))
-
-        # Check if there's a redirect parameter to return to the same page
-        redirect_url = request.args.get('redirect')
-
-        # If redirect is provided and safe, use it
-        from changedetectionio.is_safe_url import is_safe_url
-        if redirect_url and is_safe_url(redirect_url, current_app):
-            return redirect(redirect_url)
-
-        # Otherwise redirect to watchlist
-        return redirect(url_for('watchlist.index'))
-
     return ui_blueprint

changedetectionio/blueprint/ui/diff.py

@@ -83,6 +83,7 @@ def construct_blueprint(datastore: ChangeDetectionStore):
         If a processor doesn't have a difference module, falls back to text_json_diff.
         """
 
+        # More for testing, possible to return the first/only
         if uuid == 'first':
             uuid = list(datastore.data['watching'].keys()).pop()
 
@@ -100,21 +101,23 @@ def construct_blueprint(datastore: ChangeDetectionStore):
         # Get the processor type for this watch
         processor_name = watch.get('processor', 'text_json_diff')
 
-        # Try to get the processor's difference module (works for both built-in and plugin processors)
-        from changedetectionio.processors import get_processor_submodule
-        processor_module = get_processor_submodule(processor_name, 'difference')
+        try:
+            # Try to import the processor's difference module
+            processor_module = importlib.import_module(f'changedetectionio.processors.{processor_name}.difference')
 
-        # Call the processor's render() function
-        if processor_module and hasattr(processor_module, 'render'):
-            return processor_module.render(
-                watch=watch,
-                datastore=datastore,
-                request=request,
-                url_for=url_for,
-                render_template=render_template,
-                flash=flash,
-                redirect=redirect
-            )
+            # Call the processor's render() function
+            if hasattr(processor_module, 'render'):
+                return processor_module.render(
+                    watch=watch,
+                    datastore=datastore,
+                    request=request,
+                    url_for=url_for,
+                    render_template=render_template,
+                    flash=flash,
+                    redirect=redirect
+                )
+        except (ImportError, ModuleNotFoundError) as e:
+            logger.warning(f"Processor {processor_name} does not have a difference module, falling back to text_json_diff: {e}")
 
         # Fallback: if processor doesn't have difference module, use text_json_diff as default
         from changedetectionio.processors.text_json_diff.difference import render as default_render
@@ -141,10 +144,10 @@ def construct_blueprint(datastore: ChangeDetectionStore):
         Each processor implements processors/{type}/extract.py::render_form()
         If a processor doesn't have an extract module, falls back to text_json_diff.
         """
-
-
+        # More for testing, possible to return the first/only
         if uuid == 'first':
             uuid = list(datastore.data['watching'].keys()).pop()
+
         try:
             watch = datastore.data['watching'][uuid]
         except KeyError:
@@ -154,21 +157,23 @@ def construct_blueprint(datastore: ChangeDetectionStore):
         # Get the processor type for this watch
         processor_name = watch.get('processor', 'text_json_diff')
 
-        # Try to get the processor's extract module (works for both built-in and plugin processors)
-        from changedetectionio.processors import get_processor_submodule
-        processor_module = get_processor_submodule(processor_name, 'extract')
+        try:
+            # Try to import the processor's extract module
+            processor_module = importlib.import_module(f'changedetectionio.processors.{processor_name}.extract')
 
-        # Call the processor's render_form() function
-        if processor_module and hasattr(processor_module, 'render_form'):
-            return processor_module.render_form(
-                watch=watch,
-                datastore=datastore,
-                request=request,
-                url_for=url_for,
-                render_template=render_template,
-                flash=flash,
-                redirect=redirect
-            )
+            # Call the processor's render_form() function
+            if hasattr(processor_module, 'render_form'):
+                return processor_module.render_form(
+                    watch=watch,
+                    datastore=datastore,
+                    request=request,
+                    url_for=url_for,
+                    render_template=render_template,
+                    flash=flash,
+                    redirect=redirect
+                )
+        except (ImportError, ModuleNotFoundError) as e:
+            logger.warning(f"Processor {processor_name} does not have an extract module, falling back to base extractor: {e}")
 
         # Fallback: if processor doesn't have extract module, use base processors.extract as default
         from changedetectionio.processors.extract import render_form as default_render_form
@@ -195,7 +200,7 @@ def construct_blueprint(datastore: ChangeDetectionStore):
         Each processor implements processors/{type}/extract.py::process_extraction()
         If a processor doesn't have an extract module, falls back to text_json_diff.
         """
-
+        # More for testing, possible to return the first/only
         if uuid == 'first':
             uuid = list(datastore.data['watching'].keys()).pop()
 
@@ -208,22 +213,24 @@ def construct_blueprint(datastore: ChangeDetectionStore):
         # Get the processor type for this watch
         processor_name = watch.get('processor', 'text_json_diff')
 
-        # Try to get the processor's extract module (works for both built-in and plugin processors)
-        from changedetectionio.processors import get_processor_submodule
-        processor_module = get_processor_submodule(processor_name, 'extract')
+        try:
+            # Try to import the processor's extract module
+            processor_module = importlib.import_module(f'changedetectionio.processors.{processor_name}.extract')
 
-        # Call the processor's process_extraction() function
-        if processor_module and hasattr(processor_module, 'process_extraction'):
-            return processor_module.process_extraction(
-                watch=watch,
-                datastore=datastore,
-                request=request,
-                url_for=url_for,
-                make_response=make_response,
-                send_from_directory=send_from_directory,
-                flash=flash,
-                redirect=redirect
-            )
+            # Call the processor's process_extraction() function
+            if hasattr(processor_module, 'process_extraction'):
+                return processor_module.process_extraction(
+                    watch=watch,
+                    datastore=datastore,
+                    request=request,
+                    url_for=url_for,
+                    make_response=make_response,
+                    send_from_directory=send_from_directory,
+                    flash=flash,
+                    redirect=redirect
+                )
+        except (ImportError, ModuleNotFoundError) as e:
+            logger.warning(f"Processor {processor_name} does not have an extract module, falling back to base extractor: {e}")
 
         # Fallback: if processor doesn't have extract module, use base processors.extract as default
         from changedetectionio.processors.extract import process_extraction as default_process_extraction
@@ -260,7 +267,7 @@ def construct_blueprint(datastore: ChangeDetectionStore):
         - /diff/{uuid}/processor-asset/after
         - /diff/{uuid}/processor-asset/rendered_diff
         """
-
+        # More for testing, possible to return the first/only
         if uuid == 'first':
             uuid = list(datastore.data['watching'].keys()).pop()
 
@@ -273,33 +280,38 @@ def construct_blueprint(datastore: ChangeDetectionStore):
         # Get the processor type for this watch
         processor_name = watch.get('processor', 'text_json_diff')
 
-        # Try to get the processor's difference module (works for both built-in and plugin processors)
-        from changedetectionio.processors import get_processor_submodule
-        processor_module = get_processor_submodule(processor_name, 'difference')
+        try:
+            # Try to import the processor's difference module
+            processor_module = importlib.import_module(f'changedetectionio.processors.{processor_name}.difference')
 
-        # Call the processor's get_asset() function
-        if processor_module and hasattr(processor_module, 'get_asset'):
-            result = processor_module.get_asset(
-                asset_name=asset_name,
-                watch=watch,
-                datastore=datastore,
-                request=request
-            )
+            # Call the processor's get_asset() function
+            if hasattr(processor_module, 'get_asset'):
+                result = processor_module.get_asset(
+                    asset_name=asset_name,
+                    watch=watch,
+                    datastore=datastore,
+                    request=request
+                )
 
-            if result is None:
+                if result is None:
+                    from flask import abort
+                    abort(404, description=f"Asset '{asset_name}' not found")
+
+                binary_data, content_type, cache_control = result
+
+                response = make_response(binary_data)
+                response.headers['Content-Type'] = content_type
+                if cache_control:
+                    response.headers['Cache-Control'] = cache_control
+                return response
+            else:
+                logger.warning(f"Processor {processor_name} does not implement get_asset()")
                 from flask import abort
-                abort(404, description=f"Asset '{asset_name}' not found")
+                abort(404, description=f"Processor '{processor_name}' does not support assets")
 
-            binary_data, content_type, cache_control = result
-
-            response = make_response(binary_data)
-            response.headers['Content-Type'] = content_type
-            if cache_control:
-                response.headers['Cache-Control'] = cache_control
-            return response
-        else:
-            logger.warning(f"Processor {processor_name} does not implement get_asset()")
+        except (ImportError, ModuleNotFoundError) as e:
+            logger.warning(f"Processor {processor_name} does not have a difference module: {e}")
             from flask import abort
-            abort(404, description=f"Processor '{processor_name}' does not support assets")
+            abort(404, description=f"Processor '{processor_name}' not found")
 
     return diff_blueprint

changedetectionio/blueprint/ui/edit.py

@@ -9,7 +9,7 @@ from jinja2 import Environment, FileSystemLoader
 from changedetectionio.store import ChangeDetectionStore
 from changedetectionio.auth_decorator import login_optionally_required
 from changedetectionio.time_handler import is_within_schedule
-from changedetectionio import worker_pool
+from changedetectionio import worker_handler
 
 def construct_blueprint(datastore: ChangeDetectionStore, update_q, queuedWatchMetaData):
     edit_blueprint = Blueprint('ui_edit', __name__, template_folder="../ui/templates")
@@ -26,17 +26,18 @@ def construct_blueprint(datastore: ChangeDetectionStore, update_q, queuedWatchMe
     # https://wtforms.readthedocs.io/en/3.0.x/forms/#wtforms.form.Form.populate_obj ?
     def edit_page(uuid):
         from changedetectionio import forms
-        from changedetectionio.browser_steps.browser_steps import browser_step_ui_config
+        from changedetectionio.blueprint.browser_steps.browser_steps import browser_step_ui_config
         from changedetectionio import processors
         import importlib
 
-        if uuid == 'first':
-            uuid = list(datastore.data['watching'].keys()).pop()
         # More for testing, possible to return the first/only
         if not datastore.data['watching'].keys():
             flash(gettext("No watches to edit"), "error")
             return redirect(url_for('watchlist.index'))
 
+        if uuid == 'first':
+            uuid = list(datastore.data['watching'].keys()).pop()
+
         if not uuid in datastore.data['watching']:
             flash(gettext("No watch with the UUID {} found.").format(uuid), "error")
             return redirect(url_for('watchlist.index'))
@@ -71,13 +72,8 @@ def construct_blueprint(datastore: ChangeDetectionStore, update_q, queuedWatchMe
         processor_name = datastore.data['watching'][uuid].get('processor', '')
         processor_classes = next((tpl for tpl in processors.find_processors() if tpl[1] == processor_name), None)
         if not processor_classes:
-            flash(gettext("Could not load '{}' processor, processor plugin might be missing. Please select a different processor.").format(processor_name), 'error')
-            # Fall back to default processor so user can still edit and change processor
-            processor_classes = next((tpl for tpl in processors.find_processors() if tpl[1] == 'text_json_diff'), None)
-            if not processor_classes:
-                # If even text_json_diff is missing, something is very wrong
-                flash(gettext("Could not load '{}' processor, processor plugin might be missing.").format(processor_name), 'error')
-                return redirect(url_for('watchlist.index'))
+            flash(gettext("Cannot load the edit form for processor/plugin '{}', plugin missing?").format(processor_classes[1]), 'error')
+            return redirect(url_for('watchlist.index'))
 
         parent_module = processors.get_parent_module(processor_classes[0])
 
@@ -154,10 +150,58 @@ def construct_blueprint(datastore: ChangeDetectionStore, update_q, queuedWatchMe
 
             extra_update_obj['time_between_check'] = form.time_between_check.data
 
-            # Handle processor-config-* fields separately (save to JSON, not datastore)
-            # IMPORTANT: These must NOT be saved to url-watches.json, only to the processor-specific JSON file
-            processor_config_data = processors.extract_processor_config_from_form_data(form.data)
-            processors.save_processor_config(datastore, uuid, processor_config_data)
+             # Handle processor-config-* fields separately (save to JSON, not datastore)
+            processor_config_data = {}
+            fields_to_remove = []
+            for field_name, field_value in form.data.items():
+                if field_name.startswith('processor_config_'):
+                    config_key = field_name.replace('processor_config_', '')
+                    if field_value:  # Only save non-empty values
+                        processor_config_data[config_key] = field_value
+                    fields_to_remove.append(field_name)
+
+            # Save processor config to JSON file if any config data exists
+            if processor_config_data:
+                try:
+                    processor_name = form.data.get('processor')
+                    # Create a processor instance to access config methods
+                    processor_instance = processors.difference_detection_processor(datastore, uuid)
+                    # Use processor name as filename so each processor keeps its own config
+                    config_filename = f'{processor_name}.json'
+                    processor_instance.update_extra_watch_config(config_filename, processor_config_data)
+                    logger.debug(f"Saved processor config to {config_filename}: {processor_config_data}")
+
+                    # Call optional edit_hook if processor has one
+                    try:
+                        # Try to import the edit_hook module from the processor package
+                        import importlib
+                        edit_hook_module_name = f'changedetectionio.processors.{processor_name}.edit_hook'
+
+                        try:
+                            edit_hook = importlib.import_module(edit_hook_module_name)
+                            logger.debug(f"Found edit_hook module for {processor_name}")
+
+                            if hasattr(edit_hook, 'on_config_save'):
+                                logger.info(f"Calling edit_hook.on_config_save for {processor_name}")
+                                watch_obj = datastore.data['watching'][uuid]
+                                # Call hook and get updated config
+                                updated_config = edit_hook.on_config_save(watch_obj, processor_config_data, datastore)
+                                # Save updated config back to file
+                                processor_instance.update_extra_watch_config(config_filename, updated_config)
+                                logger.info(f"Edit hook updated config: {updated_config}")
+                            else:
+                                logger.debug(f"Edit hook module found but no on_config_save function")
+                        except ModuleNotFoundError:
+                            logger.debug(f"No edit_hook module for processor {processor_name} (this is normal)")
+                    except Exception as hook_error:
+                        logger.error(f"Edit hook error (non-fatal): {hook_error}", exc_info=True)
+
+                except Exception as e:
+                    logger.error(f"Failed to save processor config: {e}")
+
+            # Remove processor-config-* fields from form.data before updating datastore
+            for field_name in fields_to_remove:
+                form.data.pop(field_name, None)
 
             # Ignore text
             form_ignore_text = form.ignore_text.data
@@ -197,11 +241,7 @@ def construct_blueprint(datastore: ChangeDetectionStore, update_q, queuedWatchMe
 
             # Recast it if need be to right data Watch handler
             watch_class = processors.get_custom_watch_obj_for_processor(form.data.get('processor'))
-            datastore.data['watching'][uuid] = watch_class(datastore_path=datastore.datastore_path, __datastore=datastore.data, default=datastore.data['watching'][uuid])
-
-            # Save the watch immediately
-            datastore.data['watching'][uuid].commit()
-
+            datastore.data['watching'][uuid] = watch_class(datastore_path=datastore.datastore_path, default=datastore.data['watching'][uuid])
             flash(gettext("Updated watch - unpaused!") if request.args.get('unpause_on_save') else gettext("Updated watch."))
 
             # Cleanup any browsersteps session for this watch
@@ -211,6 +251,10 @@ def construct_blueprint(datastore: ChangeDetectionStore, update_q, queuedWatchMe
             except Exception as e:
                 logger.debug(f"Error cleaning up browsersteps session: {e}")
 
+            # Re #286 - We wait for syncing new data to disk in another thread every 60 seconds
+            # But in the case something is added we should save straight away
+            datastore.needs_write_urgent = True
+
             # Do not queue on edit if its not within the time range
 
             # @todo maybe it should never queue anyway on edit...
@@ -239,7 +283,7 @@ def construct_blueprint(datastore: ChangeDetectionStore, update_q, queuedWatchMe
             #############################
             if not datastore.data['watching'][uuid].get('paused') and is_in_schedule:
                 # Queue the watch for immediate recheck, with a higher priority
-                worker_pool.queue_item_async_safe(update_q, queuedWatchMetaData.PrioritizedItem(priority=1, item={'uuid': uuid}))
+                worker_handler.queue_item_async_safe(update_q, queuedWatchMetaData.PrioritizedItem(priority=1, item={'uuid': uuid}))
 
             # Diff page [edit] link should go back to diff page
             if request.args.get("next") and request.args.get("next") == 'diff':
@@ -267,17 +311,10 @@ def construct_blueprint(datastore: ChangeDetectionStore, update_q, queuedWatchMe
 
             # Get fetcher capabilities instead of hardcoded logic
             capabilities = get_fetcher_capabilities(watch, datastore)
-
-            # Add processor capabilities from module
-            capabilities['supports_visual_selector'] = getattr(parent_module, 'supports_visual_selector', False)
-            capabilities['supports_text_filters_and_triggers'] = getattr(parent_module, 'supports_text_filters_and_triggers', False)
-            capabilities['supports_text_filters_and_triggers_elements'] = getattr(parent_module, 'supports_text_filters_and_triggers_elements', False)
-            capabilities['supports_request_type'] = getattr(parent_module, 'supports_request_type', False)
-
             app_rss_token = datastore.data['settings']['application'].get('rss_access_token'),
 
             c = [f"processor-{watch.get('processor')}"]
-            if worker_pool.is_watch_running(uuid):
+            if worker_handler.is_watch_running(uuid):
                 c.append('checking-now')
 
             template_args = {
@@ -334,12 +371,10 @@ def construct_blueprint(datastore: ChangeDetectionStore, update_q, queuedWatchMe
         from flask import send_file
         import brotli
 
-        if uuid == 'first':
-            uuid = list(datastore.data['watching'].keys()).pop()
         watch = datastore.data['watching'].get(uuid)
-        if watch and watch.history.keys() and os.path.isdir(watch.data_dir):
+        if watch and watch.history.keys() and os.path.isdir(watch.watch_data_dir):
             latest_filename = list(watch.history.keys())[-1]
-            html_fname = os.path.join(watch.data_dir, f"{latest_filename}.html.br")
+            html_fname = os.path.join(watch.watch_data_dir, f"{latest_filename}.html.br")
             with open(html_fname, 'rb') as f:
                 if html_fname.endswith('.br'):
                     # Read and decompress the Brotli file
@@ -354,65 +389,12 @@ def construct_blueprint(datastore: ChangeDetectionStore, update_q, queuedWatchMe
         # Return a 500 error
         abort(500)
 
-    @edit_blueprint.route("/edit/<string:uuid>/get-data-package", methods=['GET'])
-    @login_optionally_required
-    def watch_get_data_package(uuid):
-        """Download all data for a single watch as a zip file"""
-        from io import BytesIO
-        from flask import send_file
-        import zipfile
-        from pathlib import Path
-        import datetime
-
-        watch = datastore.data['watching'].get(uuid)
-        if not watch:
-            abort(404)
-
-        # Create zip in memory
-        memory_file = BytesIO()
-
-        with zipfile.ZipFile(memory_file, 'w',
-                           compression=zipfile.ZIP_DEFLATED,
-                           compresslevel=8) as zipObj:
-
-            # Add the watch's JSON file if it exists
-            watch_json_path = os.path.join(watch.data_dir, 'watch.json')
-            if os.path.isfile(watch_json_path):
-                zipObj.write(watch_json_path,
-                           arcname=os.path.join(uuid, 'watch.json'),
-                           compress_type=zipfile.ZIP_DEFLATED,
-                           compresslevel=8)
-
-            # Add all files in the watch data directory
-            if os.path.isdir(watch.data_dir):
-                for f in Path(watch.data_dir).glob('*'):
-                    if f.is_file() and f.name != 'watch.json':  # Skip watch.json since we already added it
-                        zipObj.write(f,
-                                   arcname=os.path.join(uuid, f.name),
-                                   compress_type=zipfile.ZIP_DEFLATED,
-                                   compresslevel=8)
-
-        # Seek to beginning of file
-        memory_file.seek(0)
-
-        # Generate filename with timestamp
-        timestamp = datetime.datetime.now().strftime("%Y%m%d%H%M%S")
-        filename = f"watch-data-{uuid[:8]}-{timestamp}.zip"
-
-        return send_file(memory_file,
-                        as_attachment=True,
-                        download_name=filename,
-                        mimetype='application/zip')
-
     # Ajax callback
     @edit_blueprint.route("/edit/<string:uuid>/preview-rendered", methods=['POST'])
     @login_optionally_required
     def watch_get_preview_rendered(uuid):
         '''For when viewing the "preview" of the rendered text from inside of Edit'''
         from flask import jsonify
-
-        if uuid == 'first':
-            uuid = list(datastore.data['watching'].keys()).pop()
         from changedetectionio.processors.text_json_diff import prepare_filter_prevew
         result = prepare_filter_prevew(watch_uuid=uuid, form_data=request.form, datastore=datastore)
         return jsonify(result)
@@ -436,9 +418,6 @@ def construct_blueprint(datastore: ChangeDetectionStore, update_q, queuedWatchMe
                     s = re.sub(r'[0-9]+', r'\\d+', s)
                     datastore.data["watching"][uuid]['ignore_text'].append('/' + s + '/')
 
-            # Save the updated ignore_text
-            datastore.data["watching"][uuid].commit()
-
         return f"<a href={url_for('ui.ui_preview.preview_page', uuid=uuid)}>Click to preview</a>"
     
     return edit_blueprint

changedetectionio/blueprint/ui/preview.py

@@ -26,9 +26,10 @@ def construct_blueprint(datastore: ChangeDetectionStore):
         Each processor implements processors/{type}/preview.py::render()
         If a processor doesn't have a preview module, falls back to default text preview.
         """
-
+        # More for testing, possible to return the first/only
         if uuid == 'first':
             uuid = list(datastore.data['watching'].keys()).pop()
+
         try:
             watch = datastore.data['watching'][uuid]
         except KeyError:
@@ -38,21 +39,24 @@ def construct_blueprint(datastore: ChangeDetectionStore):
         # Get the processor type for this watch
         processor_name = watch.get('processor', 'text_json_diff')
 
-        # Try to get the processor's preview module (works for both built-in and plugin processors)
-        from changedetectionio.processors import get_processor_submodule
-        processor_module = get_processor_submodule(processor_name, 'preview')
+        try:
+            # Try to import the processor's preview module
+            import importlib
+            processor_module = importlib.import_module(f'changedetectionio.processors.{processor_name}.preview')
 
-        # Call the processor's render() function
-        if processor_module and hasattr(processor_module, 'render'):
-            return processor_module.render(
-                watch=watch,
-                datastore=datastore,
-                request=request,
-                url_for=url_for,
-                render_template=render_template,
-                flash=flash,
-                redirect=redirect
-            )
+            # Call the processor's render() function
+            if hasattr(processor_module, 'render'):
+                return processor_module.render(
+                    watch=watch,
+                    datastore=datastore,
+                    request=request,
+                    url_for=url_for,
+                    render_template=render_template,
+                    flash=flash,
+                    redirect=redirect
+                )
+        except (ImportError, ModuleNotFoundError) as e:
+            logger.debug(f"Processor {processor_name} does not have a preview module, using default preview: {e}")
 
         # Fallback: if processor doesn't have preview module, use default text preview
         content = []
@@ -146,8 +150,10 @@ def construct_blueprint(datastore: ChangeDetectionStore):
         """
         from flask import make_response
 
+        # More for testing, possible to return the first/only
         if uuid == 'first':
             uuid = list(datastore.data['watching'].keys()).pop()
+
         try:
             watch = datastore.data['watching'][uuid]
         except KeyError:
@@ -157,33 +163,39 @@ def construct_blueprint(datastore: ChangeDetectionStore):
         # Get the processor type for this watch
         processor_name = watch.get('processor', 'text_json_diff')
 
-        # Try to get the processor's preview module (works for both built-in and plugin processors)
-        from changedetectionio.processors import get_processor_submodule
-        processor_module = get_processor_submodule(processor_name, 'preview')
+        try:
+            # Try to import the processor's preview module
+            import importlib
+            processor_module = importlib.import_module(f'changedetectionio.processors.{processor_name}.preview')
 
-        # Call the processor's get_asset() function
-        if processor_module and hasattr(processor_module, 'get_asset'):
-            result = processor_module.get_asset(
-                asset_name=asset_name,
-                watch=watch,
-                datastore=datastore,
-                request=request
-            )
+            # Call the processor's get_asset() function
+            if hasattr(processor_module, 'get_asset'):
+                result = processor_module.get_asset(
+                    asset_name=asset_name,
+                    watch=watch,
+                    datastore=datastore,
+                    request=request
+                )
 
-            if result is None:
+                if result is None:
+                    from flask import abort
+                    abort(404, description=f"Asset '{asset_name}' not found")
+
+                binary_data, content_type, cache_control = result
+
+                response = make_response(binary_data)
+                response.headers['Content-Type'] = content_type
+                if cache_control:
+                    response.headers['Cache-Control'] = cache_control
+                return response
+            else:
+                logger.warning(f"Processor {processor_name} does not implement get_asset()")
                 from flask import abort
-                abort(404, description=f"Asset '{asset_name}' not found")
+                abort(404, description=f"Processor '{processor_name}' does not support assets")
 
-            binary_data, content_type, cache_control = result
-
-            response = make_response(binary_data)
-            response.headers['Content-Type'] = content_type
-            if cache_control:
-                response.headers['Cache-Control'] = cache_control
-            return response
-        else:
-            logger.warning(f"Processor {processor_name} does not implement get_asset()")
+        except (ImportError, ModuleNotFoundError) as e:
+            logger.warning(f"Processor {processor_name} does not have a preview module: {e}")
             from flask import abort
-            abort(404, description=f"Processor '{processor_name}' does not support assets")
+            abort(404, description=f"Processor '{processor_name}' not found")
 
     return preview_blueprint

changedetectionio/blueprint/ui/templates/edit.html

@@ -45,19 +45,14 @@
     <div class="tabs collapsable">
         <ul>
             <li class="tab"><a href="#general">{{ _('General') }}</a></li>
-            {% if capabilities.supports_request_type %}
             <li class="tab"><a href="#request">{{ _('Request') }}</a></li>
-            {% endif %}
             {% if extra_tab_content %}
             <li class="tab"><a href="#extras_tab">{{ extra_tab_content }}</a></li>
             {% endif %}
-            {% if capabilities.supports_browser_steps %}
             <li class="tab"><a id="browsersteps-tab" href="#browser-steps">{{ _('Browser Steps') }}</a></li>
-            {% endif %}
-            {% if capabilities.supports_visual_selector %}
+        <!-- should goto extra forms? -->
+            {% if watch['processor'] == 'text_json_diff' or watch['processor'] == 'image_ssim_diff' %}
             <li class="tab"><a id="visualselector-tab" href="#visualselector">{{ _('Visual Filter Selector') }}</a></li>
-            {% endif %}
-            {% if capabilities.supports_text_filters_and_triggers %}
             <li class="tab" id="filters-and-triggers-tab"><a href="#filters-and-triggers">{{ _('Filters & Triggers') }}</a></li>
             <li class="tab" id="conditions-tab"><a href="#conditions">{{ _('Conditions') }}</a></li>
             {% endif %}
@@ -115,20 +110,12 @@
                          {{ _('Sends a notification when the filter can no longer be seen on the page, good for knowing when the page changed and your filter will not work anymore.') }}
                         </span>
                     </div>
-                    <div class="pure-control-group">
-                        {{ render_field(form.history_snapshot_max_length, class="history_snapshot_max_length") }}
-                        <span class="pure-form-message-inline">{{ _('Limit collection of history snapshots for each watch to this number of history items.') }}
-                        <br>
-                        {{ _('Set to empty to use system settings default') }}
-                        </span>
-                    </div>
                     <div class="pure-control-group">
                         {{ render_ternary_field(form.use_page_title_in_list) }}
                     </div>
                 </fieldset>
             </div>
 
-            {% if capabilities.supports_request_type %}
             <div class="tab-pane-inner" id="request">
                     <div class="pure-control-group inline-radio">
                         {{ render_field(form.fetch_backend, class="fetch-backend") }}
@@ -216,7 +203,6 @@ Math: {{ 1 + 1 }}") }}
                     </div>
             </fieldset>
             </div>
-            {% endif %}
 
             <div class="tab-pane-inner" id="browser-steps">
             {% if capabilities.supports_browser_steps %}
@@ -297,7 +283,8 @@ Math: {{ 1 + 1 }}") }}
                 </fieldset>
             </div>
 
-            {% if capabilities.supports_text_filters_and_triggers %}
+            {% if watch['processor'] == 'text_json_diff' or watch['processor'] == 'image_ssim_diff' %}
+
             <div class="tab-pane-inner" id="conditions">
                     <script>
                         const verify_condition_rule_url="{{url_for('conditions.verify_condition_single_rule', watch_uuid=uuid)}}";
@@ -316,9 +303,7 @@ Math: {{ 1 + 1 }}") }}
                 <span id="activate-text-preview" class="pure-button pure-button-primary button-xsmall">{{ _('Activate preview') }}</span>
               <div>
               <div id="edit-text-filter">
-
-{% if capabilities.supports_text_filters_and_triggers_elements %}
-                        <div class="pure-control-group" id="pro-tips">
+                    <div class="pure-control-group" id="pro-tips">
                             <strong>{{ _('Pro-tips:') }}</strong><br>
                             <ul>
                                 <li>
@@ -329,8 +314,8 @@ Math: {{ 1 + 1 }}") }}
                                 </li>
                             </ul>
                     </div>
+
 {% include "edit/include_subtract.html" %}
-{% endif %}
                 <div class="text-filtering border-fieldset">
                 <fieldset class="pure-group" id="text-filtering-type-options">
                     <h3>{{ _('Text filtering') }}</h3>
@@ -389,7 +374,7 @@ Math: {{ 1 + 1 }}") }}
             {{ extra_form_content|safe }}
             </div>
         {% endif %}
-            {% if capabilities.supports_visual_selector %}
+            {% if watch['processor'] == 'text_json_diff' or watch['processor'] == 'image_ssim_diff' %}
             <div class="tab-pane-inner visual-selector-ui" id="visualselector">
                 <img class="beta-logo" src="{{url_for('static_content', group='images', filename='beta-logo.png')}}" alt="New beta functionality">
 
@@ -401,7 +386,7 @@ Math: {{ 1 + 1 }}") }}
                                     {{ _('The Visual Selector tool lets you select the') }} <i>{{ _('text') }}</i> {{ _('elements that will be used for the change detection. It automatically fills-in the filters in the "CSS/JSONPath/JQ/XPath Filters" box of the') }} <a href="#filters-and-triggers">{{ _('Filters & Triggers') }}</a> {{ _('tab. Use') }} <strong>{{ _('Shift+Click') }}</strong> {{ _('to select multiple items.') }}
                                 </span>
 
-                                {% if watch['processor'] == 'image_ssim_diff' %} {# @todo, integrate with image_ssim_diff selector better, use some extra form ? #}
+                                {% if watch['processor'] == 'image_ssim_diff' %}
                                 <div id="selection-mode-controls" style="margin: 10px 0; padding: 10px; background: var(--color-background-tab); border-radius: 5px;">
                                     <label style="font-weight: 600; margin-right: 15px;">{{ _('Selection Mode:') }}</label>
                                     <label style="margin-right: 15px;">
@@ -488,7 +473,6 @@ Math: {{ 1 + 1 }}") }}
                     {% if watch.history_n %}
                         <p>
                              <a href="{{url_for('ui.ui_edit.watch_get_latest_html', uuid=uuid)}}" class="pure-button button-small">{{ _('Download latest HTML snapshot') }}</a>
-                             <a href="{{url_for('ui.ui_edit.watch_get_data_package', uuid=uuid)}}" class="pure-button button-small">{{ _('Download watch data package') }}</a>
                         </p>
                     {% endif %}
 

changedetectionio/blueprint/ui/views.py

@@ -2,7 +2,7 @@ from flask import Blueprint, request, redirect, url_for, flash
 from flask_babel import gettext
 from changedetectionio.store import ChangeDetectionStore
 from changedetectionio.auth_decorator import login_optionally_required
-from changedetectionio import worker_pool
+from changedetectionio import worker_handler
 
 
 def construct_blueprint(datastore: ChangeDetectionStore, update_q, queuedWatchMetaData, watch_check_update):
@@ -24,9 +24,8 @@ def construct_blueprint(datastore: ChangeDetectionStore, update_q, queuedWatchMe
             flash(gettext('Warning, URL {} already exists').format(url), "notice")
 
         add_paused = request.form.get('edit_and_watch_submit_button') != None
-        from changedetectionio import processors
-        processor = request.form.get('processor', processors.get_default_processor())
-        new_uuid = datastore.add_watch(url=url, tag=request.form.get('tags','').strip(), extras={'paused': add_paused, 'processor': processor})
+        processor = request.form.get('processor', 'text_json_diff')
+        new_uuid = datastore.add_watch(url=url, tag=request.form.get('tags').strip(), extras={'paused': add_paused, 'processor': processor})
 
         if new_uuid:
             if add_paused:
@@ -34,9 +33,9 @@ def construct_blueprint(datastore: ChangeDetectionStore, update_q, queuedWatchMe
                 return redirect(url_for('ui.ui_edit.edit_page', uuid=new_uuid, unpause_on_save=1, tag=request.args.get('tag')))
             else:
                 # Straight into the queue.
-                worker_pool.queue_item_async_safe(update_q, queuedWatchMetaData.PrioritizedItem(priority=1, item={'uuid': new_uuid}))
+                worker_handler.queue_item_async_safe(update_q, queuedWatchMetaData.PrioritizedItem(priority=1, item={'uuid': new_uuid}))
                 flash(gettext("Watch added."))
 
         return redirect(url_for('watchlist.index', tag=request.args.get('tag','')))
 
-    return views_blueprint
+    return views_blueprint

changedetectionio/blueprint/watchlist/__init__.py

@@ -3,7 +3,6 @@ import time
 
 from flask import Blueprint, request, make_response, render_template, redirect, url_for, flash, session
 from flask_paginate import Pagination, get_page_parameter
-from flask_babel import gettext as _
 
 from changedetectionio import forms
 from changedetectionio import processors
@@ -39,7 +38,7 @@ def construct_blueprint(datastore: ChangeDetectionStore, update_q, queuedWatchMe
             elif op == 'mute':
                 datastore.data['watching'][uuid].toggle_mute()
 
-            datastore.data['watching'][uuid].commit()
+            datastore.needs_write = True
             return redirect(url_for('watchlist.index', tag = active_tag_uuid))
 
         # Sort by last_changed and add the uuid which is usually the key..
@@ -74,10 +73,7 @@ def construct_blueprint(datastore: ChangeDetectionStore, update_q, queuedWatchMe
 
         pagination = Pagination(page=page,
                                 total=total_count,
-                                per_page=datastore.data['settings']['application'].get('pager_size', 50),
-                                css_framework="semantic",
-                                display_msg=_('displaying <b>{start} - {end}</b> {record_name} in total <b>{total}</b>'),
-                                record_name=_('records'))
+                                per_page=datastore.data['settings']['application'].get('pager_size', 50), css_framework="semantic")
 
         sorted_tags = sorted(datastore.data['settings']['application'].get('tags').items(), key=lambda x: x[1]['title'])
 

changedetectionio/blueprint/watchlist/templates/watch-overview.html

@@ -1,9 +1,5 @@
 {%- extends 'base.html' -%}
 {%- block content -%}
-{%- set tips = [
-    _("Changedetection.io can monitor more than just web-pages! See our plugins!") ~ ' <a href="https://changedetection.io/plugins">' ~ _('More info') ~ '</a>',
-    _("You can also add 'shared' watches.") ~ ' <a href="https://github.com/dgtlmoon/changedetection.io/wiki/Sharing-a-Watch">' ~ _('More info') ~ '</a>'
-] -%}
 {%- from '_helpers.html' import render_simple_field, render_field, render_nolabel_field, sort_by_title -%}
 <script src="{{url_for('static_content', group='js', filename='jquery-3.6.0.min.js')}}"></script>
 <script src="{{url_for('static_content', group='js', filename='watch-overview.js')}}" defer></script>
@@ -14,46 +10,6 @@
 // Initialize Feather icons after the page loads
 document.addEventListener('DOMContentLoaded', function() {
     feather.replace();
-
-    // Intersection Observer for lazy loading favicons
-    // Only load favicon images when they enter the viewport
-    if ('IntersectionObserver' in window) {
-        const faviconObserver = new IntersectionObserver((entries, observer) => {
-            entries.forEach(entry => {
-                if (entry.isIntersecting) {
-                    const img = entry.target;
-                    const src = img.getAttribute('data-src');
-
-                    if (src) {
-                        // Load the actual favicon
-                        img.src = src;
-                        img.removeAttribute('data-src');
-                    }
-
-                    // Stop observing this image
-                    observer.unobserve(img);
-                }
-            });
-        }, {
-            // Start loading slightly before the image enters viewport
-            rootMargin: '50px',
-            threshold: 0.01
-        });
-
-        // Observe all lazy favicon images
-        document.querySelectorAll('.lazy-favicon').forEach(img => {
-            faviconObserver.observe(img);
-        });
-    } else {
-        // Fallback for older browsers: load all favicons immediately
-        document.querySelectorAll('.lazy-favicon').forEach(img => {
-            const src = img.getAttribute('data-src');
-            if (src) {
-                img.src = src;
-                img.removeAttribute('data-src');
-            }
-        });
-    }
 });
 </script>
 <style>
@@ -106,16 +62,14 @@ html[data-darkmode="true"] .watch-tag-list.tag-{{ class_name }} {
                     {{ render_nolabel_field(form.edit_and_watch_submit_button, title=_("Edit first then Watch") ) }}
             </div>
             <div id="watch-group-tag">
-               {{ render_field(form.tags, value=active_tag.title if active_tag_uuid else '', placeholder=_("Watch group / tag"), class="transparent-field") }}
+               {{ render_field(form.tags, value=active_tag.title if active_tag_uuid else '', placeholder="Watch group / tag", class="transparent-field") }}
             </div>
             <div id="quick-watch-processor-type">
                 {{ render_simple_field(form.processor) }}
             </div>
 
         </fieldset>
-        <span style="color:#eee; font-size: 80%;">
-            <strong>Tip: </strong> {{ tips | random | safe }}
-        </span>
+        <span style="color:#eee; font-size: 80%;"><img alt="{{ _('Create a shareable link') }}" style="height: 1em;display:inline-block;" src="{{url_for('static_content', group='images', filename='spread-white.svg')}}" > {{ _("Tip: You can also add 'shared' watches.") }} <a href="https://github.com/dgtlmoon/changedetection.io/wiki/Sharing-a-Watch">{{ _('More info') }}</a></span>
     </form>
 </div>
 <div class="box">
@@ -246,40 +200,30 @@ html[data-darkmode="true"] .watch-tag-list.tag-{{ class_name }} {
                 <td class="title-col inline">
                     <div class="flex-wrapper">
                     {% if 'favicons_enabled' not in ui_settings or ui_settings['favicons_enabled'] %}
-                        <div>
-                            {# Intersection Observer lazy loading: store real URL in data-src, load only when visible in viewport #}
-                            <img alt="Favicon thumbnail"
-                                 class="favicon lazy-favicon"
-                                 loading="lazy"
-                                 decoding="async"
-                                 fetchpriority="low"
-                                 {% if favicon %}
-                                 data-src="{{url_for('static_content', group='favicon', filename=watch.uuid)}}"
-                                 {% endif %}
-                                 src='data:image/svg+xml;utf8,%3Csvg xmlns="http://www.w3.org/2000/svg" width="7.087" height="7.087" viewBox="0 0 7.087 7.087"%3E%3Ccircle cx="3.543" cy="3.543" r="3.279" stroke="%23e1e1e1" stroke-width="0.45" fill="none" opacity="0.74"/%3E%3C/svg%3E'>
+                        <div>{# A page might have hundreds of these images, set IMG options for lazy loading, don't set SRC if we dont have it so it doesnt fetch the placeholder'  #}
+                            <img alt="Favicon thumbnail" class="favicon" loading="lazy" decoding="async" fetchpriority="low" {% if favicon %} src="{{url_for('static_content', group='favicon', filename=watch.uuid)}}" {% else %} src='data:image/svg+xml;utf8,%3Csvg xmlns="http://www.w3.org/2000/svg" width="7.087" height="7.087" viewBox="0 0 7.087 7.087"%3E%3Ccircle cx="3.543" cy="3.543" r="3.279" stroke="%23e1e1e1" stroke-width="0.45" fill="none" opacity="0.74"/%3E%3C/svg%3E' {%  endif %} >
                         </div>
                     {%  endif %}
                         <div>
-                            {%- if watch['processor'] and watch['processor'] in processor_badge_texts -%}
-                                <span class="processor-badge processor-badge-{{ watch['processor'] }}" title="{{ processor_descriptions.get(watch['processor'], watch['processor']) }}">{{ processor_badge_texts[watch['processor']] }}</span>
-                            {%- endif -%}
-                            <span class="watch-title">
-                                {% if system_use_url_watchlist or watch.get('use_page_title_in_list') %}
-                                    {{ watch.label }}
-                                {% else %}
-                                    {{ watch.get('title') or watch.link }}
-                                {% endif %}
-                               <a class="external" target="_blank" rel="noopener" href="{{ watch.link.replace('source:','') }}">&nbsp;</a>
-                            </span>
+                        <span class="watch-title">
+                            {% if system_use_url_watchlist or watch.get('use_page_title_in_list') %}
+                                {{ watch.label }}
+                            {% else %}
+                                {{ watch.get('title') or watch.link }}
+                            {% endif %}
+                           <a class="external" target="_blank" rel="noopener" href="{{ watch.link.replace('source:','') }}">&nbsp;</a>
+                        </span>
                             <div class="error-text" style="display:none;">{{ watch.compile_error_texts(has_proxies=datastore.proxy_list)|safe }}</div>
                             {%- if watch['processor'] == 'text_json_diff'  -%}
                                 {%- if watch['has_ldjson_price_data'] and not watch['track_ldjson_price_data']  -%}
                                 <div class="ldjson-price-track-offer">Switch to Restock & Price watch mode? <a href="{{url_for('price_data_follower.accept', uuid=watch.uuid)}}" class="pure-button button-xsmall">Yes</a> <a href="{{url_for('price_data_follower.reject', uuid=watch.uuid)}}" class="">No</a></div>
                                 {%- endif -%}
                             {%- endif -%}
-
+                            {%- if watch['processor'] and watch['processor'] in processor_badge_texts -%}
+                                <span class="processor-badge processor-badge-{{ watch['processor'] }}" title="{{ processor_descriptions.get(watch['processor'], watch['processor']) }}">{{ processor_badge_texts[watch['processor']] }}</span>
+                            {%- endif -%}
                             {%- for watch_tag_uuid, watch_tag in datastore.get_all_tags_for_watch(watch['uuid']).items() -%}
-                              <a href="{{url_for('watchlist.index', tag=watch_tag_uuid) }}" class="watch-tag-list tag-{{ watch_tag.title|sanitize_tag_class }}">{{ watch_tag.title }}</a>
+                              <span class="watch-tag-list tag-{{ watch_tag.title|sanitize_tag_class }}">{{ watch_tag.title }}</span>
                             {%- endfor -%}
                         </div>
                     <div class="status-icons">
@@ -304,13 +248,12 @@ html[data-darkmode="true"] .watch-tag-list.tag-{{ class_name }} {
                             </span>
                         {%- endif -%}
 
-                        {%- if watch.get('restock') and watch['restock'].get('price') -%}
-                                {%- if watch['restock']['price'] is number -%}
+                        {%- if watch.get('restock') and watch['restock']['price'] != None -%}
+                            {%- if watch['restock']['price'] != None -%}
                                 <span class="restock-label price" title="{{ _('Price') }}">
                                 {{ watch['restock']['price']|format_number_locale if watch['restock'].get('price') else '' }} {{ watch['restock'].get('currency','') }}
                                 </span>
-                                {%- else -%} <!-- watch['restock']['price']' is not a number, cant output it -->
-                                {%- endif -%}
+                            {%- endif -%}
                         {%- elif not watch.has_restock_info -%}
                             <span class="restock-label error">{{ _('No information') }}</span>
                         {%- endif -%}

changedetectionio/content_fetchers/base.py

@@ -38,6 +38,7 @@ def manage_user_agent(headers, current_ua=''):
 
     return None
 
+
 class Fetcher():
     browser_connection_is_custom = None
     browser_connection_url = None
@@ -70,19 +71,10 @@ class Fetcher():
     supports_screenshots = False        # Can capture page screenshots
     supports_xpath_element_data = False # Can extract xpath element positions/data for visual selector
 
-    # Screenshot element locking - prevents layout shifts during screenshot capture
-    # Only needed for visual comparison (image_ssim_diff processor)
-    # Locks element dimensions in the first viewport to prevent headers/ads from resizing
-    lock_viewport_elements = False      # Default: disabled for performance
-
     def __init__(self, **kwargs):
         if kwargs and 'screenshot_format' in kwargs:
             self.screenshot_format = kwargs.get('screenshot_format')
 
-        # Allow lock_viewport_elements to be set via kwargs
-        if kwargs and 'lock_viewport_elements' in kwargs:
-            self.lock_viewport_elements = kwargs.get('lock_viewport_elements')
-
 
     @classmethod
     def get_status_icon_data(cls):
@@ -162,16 +154,30 @@ class Fetcher():
         """
         return {k.lower(): v for k, v in self.headers.items()}
 
+    def browser_steps_get_valid_steps(self):
+        if self.browser_steps is not None and len(self.browser_steps):
+            valid_steps = list(filter(
+                lambda s: (s['operation'] and len(s['operation']) and s['operation'] != 'Choose one'),
+                self.browser_steps))
+
+            # Just incase they selected Goto site by accident with older JS
+            if valid_steps and valid_steps[0]['operation'] == 'Goto site':
+                del(valid_steps[0])
+
+            return valid_steps
+
+        return None
+
     async def iterate_browser_steps(self, start_url=None):
-        from changedetectionio.browser_steps.browser_steps import steppable_browser_interface, browser_steps_get_valid_steps
+        from changedetectionio.blueprint.browser_steps.browser_steps import steppable_browser_interface
         from playwright._impl._errors import TimeoutError, Error
         from changedetectionio.jinja2_custom import render as jinja_render
         step_n = 0
 
-        if self.browser_steps:
+        if self.browser_steps is not None and len(self.browser_steps):
             interface = steppable_browser_interface(start_url=start_url)
             interface.page = self.page
-            valid_steps = browser_steps_get_valid_steps(self.browser_steps)
+            valid_steps = self.browser_steps_get_valid_steps()
 
             for step in valid_steps:
                 step_n += 1

changedetectionio/content_fetchers/playwright.py

@@ -1,4 +1,3 @@
-import asyncio
 import gc
 import json
 import os
@@ -9,24 +8,20 @@ from loguru import logger
 from changedetectionio.content_fetchers import SCREENSHOT_MAX_HEIGHT_DEFAULT, visualselector_xpath_selectors, \
     SCREENSHOT_SIZE_STITCH_THRESHOLD, SCREENSHOT_MAX_TOTAL_HEIGHT, XPATH_ELEMENT_JS, INSTOCK_DATA_JS, FAVICON_FETCHER_JS
 from changedetectionio.content_fetchers.base import Fetcher, manage_user_agent
-from changedetectionio.content_fetchers.exceptions import PageUnloadable, Non200ErrorCodeReceived, EmptyReply, ScreenshotUnavailable, \
-    BrowserStepsStepException
+from changedetectionio.content_fetchers.exceptions import PageUnloadable, Non200ErrorCodeReceived, EmptyReply, ScreenshotUnavailable
 
-
-async def capture_full_page_async(page, screenshot_format='JPEG', watch_uuid=None, lock_viewport_elements=False):
+async def capture_full_page_async(page, screenshot_format='JPEG'):
     import os
     import time
+    import multiprocessing
 
     start = time.time()
-    watch_info = f"[{watch_uuid}] " if watch_uuid else ""
 
-    setup_start = time.time()
     page_height = await page.evaluate("document.documentElement.scrollHeight")
     page_width = await page.evaluate("document.documentElement.scrollWidth")
     original_viewport = page.viewport_size
-    dimensions_time = time.time() - setup_start
 
-    logger.debug(f"{watch_info}Playwright viewport size {page.viewport_size} page height {page_height} page width {page_width} (got dimensions in {dimensions_time:.2f}s)")
+    logger.debug(f"Playwright viewport size {page.viewport_size} page height {page_height} page width {page_width}")
 
     # Use an approach similar to puppeteer: set a larger viewport and take screenshots in chunks
     step_size = SCREENSHOT_SIZE_STITCH_THRESHOLD # Size that won't cause GPU to overflow
@@ -34,31 +29,25 @@ async def capture_full_page_async(page, screenshot_format='JPEG', watch_uuid=Non
     y = 0
     elements_locked = False
 
-    # Only lock viewport elements if explicitly enabled (for image_ssim_diff processor)
-    # This prevents headers/ads from resizing when viewport changes
-    if lock_viewport_elements and page_height > page.viewport_size['height']:
-        lock_start = time.time()
+    if page_height > page.viewport_size['height']:
+
+        # Lock all element dimensions BEFORE screenshot to prevent CSS media queries from resizing
+        # capture_full_page_async() changes viewport height which triggers @media (min-height) rules
         lock_elements_js_path = os.path.join(os.path.dirname(__file__), 'res', 'lock-elements-sizing.js')
         with open(lock_elements_js_path, 'r') as f:
             lock_elements_js = f.read()
         await page.evaluate(lock_elements_js)
         elements_locked = True
-        lock_time = time.time() - lock_start
-        logger.debug(f"{watch_info}Viewport element locking enabled (took {lock_time:.2f}s)")
 
-    if page_height > page.viewport_size['height']:
+        logger.debug("Element dimensions locked before screenshot capture")
+
         if page_height < step_size:
             step_size = page_height # Incase page is bigger than default viewport but smaller than proposed step size
-        viewport_start = time.time()
-        logger.debug(f"{watch_info}Setting bigger viewport to step through large page width W{page.viewport_size['width']}xH{step_size} because page_height > viewport_size")
+        logger.debug(f"Setting bigger viewport to step through large page width W{page.viewport_size['width']}xH{step_size} because page_height > viewport_size")
         # Set viewport to a larger size to capture more content at once
         await page.set_viewport_size({'width': page.viewport_size['width'], 'height': step_size})
-        viewport_time = time.time() - viewport_start
-        logger.debug(f"{watch_info}Viewport changed to {page.viewport_size['width']}x{step_size} (took {viewport_time:.2f}s)")
 
     # Capture screenshots in chunks up to the max total height
-    capture_start = time.time()
-    chunk_times = []
     # Use PNG for better quality (no compression artifacts), JPEG for smaller size
     screenshot_type = screenshot_format.lower() if screenshot_format else 'jpeg'
     # PNG should use quality 100, JPEG uses configurable quality
@@ -80,11 +69,7 @@ async def capture_full_page_async(page, screenshot_format='JPEG', watch_uuid=Non
         if screenshot_type == 'jpeg':
             screenshot_kwargs['quality'] = screenshot_quality
 
-        chunk_start = time.time()
         screenshot_chunks.append(await page.screenshot(**screenshot_kwargs))
-        chunk_time = time.time() - chunk_start
-        chunk_times.append(chunk_time)
-        logger.debug(f"{watch_info}Chunk {len(screenshot_chunks)} captured in {chunk_time:.2f}s")
         y += step_size
 
     # Restore original viewport size
@@ -96,54 +81,40 @@ async def capture_full_page_async(page, screenshot_format='JPEG', watch_uuid=Non
         with open(unlock_elements_js_path, 'r') as f:
             unlock_elements_js = f.read()
         await page.evaluate(unlock_elements_js)
-        logger.debug(f"{watch_info}Element dimensions unlocked after screenshot capture")
-
-    capture_time = time.time() - capture_start
-    total_capture_time = sum(chunk_times)
-    logger.debug(f"{watch_info}All {len(screenshot_chunks)} chunks captured in {capture_time:.2f}s (total chunk time: {total_capture_time:.2f}s)")
+        logger.debug("Element dimensions unlocked after screenshot capture")
 
     # If we have multiple chunks, stitch them together
     if len(screenshot_chunks) > 1:
-        stitch_start = time.time()
-        logger.debug(f"{watch_info}Starting stitching of {len(screenshot_chunks)} chunks")
+        logger.debug(f"Screenshot stitching {len(screenshot_chunks)} chunks together")
 
-        # Always use spawn subprocess for ANY stitching (2+ chunks)
-        # PIL allocates at C level and Python GC never releases it - subprocess exit forces OS to reclaim
-        # Trade-off: 35MB resource_tracker vs 500MB+ PIL leak in main process
-        from changedetectionio.content_fetchers.screenshot_handler import stitch_images_worker_raw_bytes
-        import multiprocessing
-        import struct
+        # For small number of chunks (2-3), stitch inline to avoid multiprocessing overhead
+        # Only use separate process for many chunks (4+) to avoid blocking the event loop
+        if len(screenshot_chunks) <= 3:
+            from changedetectionio.content_fetchers.screenshot_handler import stitch_images_inline
+            screenshot = stitch_images_inline(screenshot_chunks, page_height, SCREENSHOT_MAX_TOTAL_HEIGHT)
+        else:
+            # Use separate process for many chunks to avoid blocking
+            # Always use spawn for thread safety - consistent behavior in tests and production
+            from changedetectionio.content_fetchers.screenshot_handler import stitch_images_worker
+            ctx = multiprocessing.get_context('spawn')
+            parent_conn, child_conn = ctx.Pipe()
+            p = ctx.Process(target=stitch_images_worker, args=(child_conn, screenshot_chunks, page_height, SCREENSHOT_MAX_TOTAL_HEIGHT))
+            p.start()
+            screenshot = parent_conn.recv_bytes()
+            p.join()
+            # Explicit cleanup
+            del p
+            del parent_conn, child_conn
 
-        ctx = multiprocessing.get_context('spawn')
-        parent_conn, child_conn = ctx.Pipe()
-        p = ctx.Process(target=stitch_images_worker_raw_bytes, args=(child_conn, page_height, SCREENSHOT_MAX_TOTAL_HEIGHT))
-        p.start()
-
-        # Send via raw bytes (no pickle)
-        parent_conn.send_bytes(struct.pack('I', len(screenshot_chunks)))
-        for chunk in screenshot_chunks:
-            parent_conn.send_bytes(chunk)
-
-        screenshot = parent_conn.recv_bytes()
-        p.join()
-
-        parent_conn.close()
-        child_conn.close()
-        del p, parent_conn, child_conn
-
-        stitch_time = time.time() - stitch_start
-        total_time = time.time() - start
-        setup_time = total_time - capture_time - stitch_time
         logger.debug(
-            f"{watch_info}Screenshot complete - Page height: {page_height}px, Capture height: {SCREENSHOT_MAX_TOTAL_HEIGHT}px | "
-            f"Setup: {setup_time:.2f}s, Capture: {capture_time:.2f}s, Stitching: {stitch_time:.2f}s, Total: {total_time:.2f}s")
+            f"Screenshot (chunked/stitched) - Page height: {page_height} Capture height: {SCREENSHOT_MAX_TOTAL_HEIGHT} - Stitched together in {time.time() - start:.2f}s")
+        # Explicit cleanup
+        del screenshot_chunks
+        screenshot_chunks = None
         return screenshot
 
-    total_time = time.time() - start
-    setup_time = total_time - capture_time
     logger.debug(
-        f"{watch_info}Screenshot complete - Page height: {page_height}px, Capture height: {SCREENSHOT_MAX_TOTAL_HEIGHT}px | "
-        f"Setup: {setup_time:.2f}s, Single chunk: {capture_time:.2f}s, Total: {total_time:.2f}s")
+        f"Screenshot Page height: {page_height} Capture height: {SCREENSHOT_MAX_TOTAL_HEIGHT} - Stitched together in {time.time() - start:.2f}s")
 
     return screenshot_chunks[0]
 
@@ -213,8 +184,7 @@ class fetcher(Fetcher):
 
     async def screenshot_step(self, step_n=''):
         super().screenshot_step(step_n=step_n)
-        watch_uuid = getattr(self, 'watch_uuid', None)
-        screenshot = await capture_full_page_async(page=self.page, screenshot_format=self.screenshot_format, watch_uuid=watch_uuid, lock_viewport_elements=self.lock_viewport_elements)
+        screenshot = await capture_full_page_async(page=self.page, screenshot_format=self.screenshot_format)
 
         # Request GC immediately after screenshot to free memory
         # Screenshots can be large and browser steps take many of them
@@ -263,7 +233,6 @@ class fetcher(Fetcher):
         import playwright._impl._errors
         import time
         self.delete_browser_steps_screenshots()
-        self.watch_uuid = watch_uuid  # Store for use in screenshot_step
         response = None
 
         async with async_playwright() as p:
@@ -295,7 +264,7 @@ class fetcher(Fetcher):
             self.page.on("console", lambda msg: logger.debug(f"Playwright console: Watch URL: {url} {msg.type}: {msg.text} {msg.args}"))
 
             # Re-use as much code from browser steps as possible so its the same
-            from changedetectionio.browser_steps.browser_steps import steppable_browser_interface
+            from changedetectionio.blueprint.browser_steps.browser_steps import steppable_browser_interface
             browsersteps_interface = steppable_browser_interface(start_url=url)
             browsersteps_interface.page = self.page
 
@@ -349,8 +318,13 @@ class fetcher(Fetcher):
                     logger.error(f"Error fetching FavIcon info {str(e)}, continuing.")
 
             if self.status_code != 200 and not ignore_status_codes:
-                screenshot = await capture_full_page_async(self.page, screenshot_format=self.screenshot_format, watch_uuid=watch_uuid, lock_viewport_elements=self.lock_viewport_elements)
-                # Finally block will handle cleanup
+                screenshot = await capture_full_page_async(self.page, screenshot_format=self.screenshot_format)
+                # Cleanup before raising to prevent memory leak
+                await self.page.close()
+                await context.close()
+                await browser.close()
+                # Force garbage collection to release Playwright resources immediately
+                gc.collect()
                 raise Non200ErrorCodeReceived(url=url, status_code=self.status_code, screenshot=screenshot)
 
             if not empty_pages_are_a_change and len((await self.page.content()).strip()) == 0:
@@ -362,12 +336,8 @@ class fetcher(Fetcher):
             # Wrap remaining operations in try/finally to ensure cleanup
             try:
                 # Run Browser Steps here
-                if self.browser_steps:
-                    try:
-                        await self.iterate_browser_steps(start_url=url)
-                    except BrowserStepsStepException:
-                        # Finally block will handle cleanup
-                        raise
+                if self.browser_steps_get_valid_steps():
+                    await self.iterate_browser_steps(start_url=url)
 
                     await self.page.wait_for_timeout(extra_wait * 1000)
 
@@ -404,51 +374,44 @@ class fetcher(Fetcher):
                 # which will significantly increase the IO size between the server and client, it's recommended to use the lowest
                 # acceptable screenshot quality here
                 # The actual screenshot - this always base64 and needs decoding! horrible! huge CPU usage
-                self.screenshot = await capture_full_page_async(page=self.page, screenshot_format=self.screenshot_format, watch_uuid=watch_uuid, lock_viewport_elements=self.lock_viewport_elements)
-
-                # Force aggressive memory cleanup - screenshots are large and base64 decode creates temporary buffers
-                await self.page.request_gc()
-                gc.collect()
+                self.screenshot = await capture_full_page_async(page=self.page, screenshot_format=self.screenshot_format)
 
             except ScreenshotUnavailable:
                 # Re-raise screenshot unavailable exceptions
+                raise
+            except Exception as e:
+                # It's likely the screenshot was too long/big and something crashed
                 raise ScreenshotUnavailable(url=url, status_code=self.status_code)
-
             finally:
-                # Clean up resources properly with timeouts to prevent hanging
+                # Request garbage collection one more time before closing
                 try:
-                    if hasattr(self, 'page') and self.page:
-                        await self.page.request_gc()
-                        await asyncio.wait_for(self.page.close(), timeout=5.0)
-                        logger.debug(f"Successfully closed page for {url}")
-                except asyncio.TimeoutError:
-                    logger.warning(f"Timed out closing page for {url} (5s)")
-                except Exception as e:
-                    logger.warning(f"Error closing page for {url}: {e}")
-                finally:
-                    self.page = None
+                    await self.page.request_gc()
+                except:
+                    pass
+                
+                # Clean up resources properly
+                try:
+                    await self.page.request_gc()
+                except:
+                    pass
 
                 try:
-                    if context:
-                        await asyncio.wait_for(context.close(), timeout=5.0)
-                        logger.debug(f"Successfully closed context for {url}")
-                except asyncio.TimeoutError:
-                    logger.warning(f"Timed out closing context for {url} (5s)")
-                except Exception as e:
-                    logger.warning(f"Error closing context for {url}: {e}")
-                finally:
-                    context = None
+                    await self.page.close()
+                except:
+                    pass
+                self.page = None
 
                 try:
-                    if browser:
-                        await asyncio.wait_for(browser.close(), timeout=5.0)
-                        logger.debug(f"Successfully closed browser connection for {url}")
-                except asyncio.TimeoutError:
-                    logger.warning(f"Timed out closing browser connection for {url} (5s)")
-                except Exception as e:
-                    logger.warning(f"Error closing browser for {url}: {e}")
-                finally:
-                    browser = None
+                    await context.close()
+                except:
+                    pass
+                context = None
+
+                try:
+                    await browser.close()
+                except:
+                    pass
+                browser = None
 
                 # Force Python GC to release Playwright resources immediately
                 # Playwright objects can have circular references that delay cleanup

changedetectionio/content_fetchers/puppeteer.py

@@ -1,5 +1,4 @@
 import asyncio
-import gc
 import json
 import os
 import websockets.exceptions
@@ -21,20 +20,18 @@ from changedetectionio.content_fetchers.exceptions import PageUnloadable, Non200
 # Screenshots also travel via the ws:// (websocket) meaning that the binary data is base64 encoded
 # which will significantly increase the IO size between the server and client, it's recommended to use the lowest
 # acceptable screenshot quality here
-async def capture_full_page(page, screenshot_format='JPEG', watch_uuid=None, lock_viewport_elements=False):
+async def capture_full_page(page, screenshot_format='JPEG'):
     import os
     import time
+    import multiprocessing
 
     start = time.time()
-    watch_info = f"[{watch_uuid}] " if watch_uuid else ""
 
-    setup_start = time.time()
     page_height = await page.evaluate("document.documentElement.scrollHeight")
     page_width = await page.evaluate("document.documentElement.scrollWidth")
     original_viewport = page.viewport
-    dimensions_time = time.time() - setup_start
 
-    logger.debug(f"{watch_info}Puppeteer viewport size {page.viewport} page height {page_height} page width {page_width} (got dimensions in {dimensions_time:.2f}s)")
+    logger.debug(f"Puppeteer viewport size {page.viewport} page height {page_height} page width {page_width}")
 
     # Bug 3 in Playwright screenshot handling
     # Some bug where it gives the wrong screenshot size, but making a request with the clip set first seems to solve it
@@ -53,41 +50,26 @@ async def capture_full_page(page, screenshot_format='JPEG', watch_uuid=None, loc
     screenshot_chunks = []
     y = 0
     elements_locked = False
-
-    # Only lock viewport elements if explicitly enabled (for image_ssim_diff processor)
-    # This prevents headers/ads from resizing when viewport changes
-    if lock_viewport_elements and page_height > page.viewport['height']:
-        lock_start = time.time()
+    if page_height > page.viewport['height']:
+        # Lock all element dimensions BEFORE screenshot to prevent CSS media queries from resizing
+        # capture_full_page() changes viewport height which triggers @media (min-height) rules
         lock_elements_js_path = os.path.join(os.path.dirname(__file__), 'res', 'lock-elements-sizing.js')
-        file_read_start = time.time()
         with open(lock_elements_js_path, 'r') as f:
             lock_elements_js = f.read()
-        file_read_time = time.time() - file_read_start
-
-        evaluate_start = time.time()
         await page.evaluate(lock_elements_js)
-        evaluate_time = time.time() - evaluate_start
-
         elements_locked = True
-        lock_time = time.time() - lock_start
-        logger.debug(f"{watch_info}Viewport element locking enabled - File read: {file_read_time:.3f}s, Browser evaluate: {evaluate_time:.2f}s, Total: {lock_time:.2f}s")
+        logger.debug("Element dimensions locked before screenshot capture")
 
-    if page_height > page.viewport['height']:
         if page_height < step_size:
             step_size = page_height # Incase page is bigger than default viewport but smaller than proposed step size
-        viewport_start = time.time()
         await page.setViewport({'width': page.viewport['width'], 'height': step_size})
-        viewport_time = time.time() - viewport_start
-        logger.debug(f"{watch_info}Viewport changed to {page.viewport['width']}x{step_size} (took {viewport_time:.2f}s)")
 
-    capture_start = time.time()
-    chunk_times = []
     while y < min(page_height, SCREENSHOT_MAX_TOTAL_HEIGHT):
         # better than scrollTo incase they override it in the page
         await page.evaluate(
             """(y) => {
-                const el = document.scrollingElement;
-                if (el) el.scrollTop = y;
+                document.documentElement.scrollTop = y;
+                document.body.scrollTop = y;
             }""",
             y
         )
@@ -100,11 +82,7 @@ async def capture_full_page(page, screenshot_format='JPEG', watch_uuid=None, loc
         if screenshot_type == 'jpeg':
             screenshot_kwargs['quality'] = screenshot_quality
 
-        chunk_start = time.time()
         screenshot_chunks.append(await page.screenshot(**screenshot_kwargs))
-        chunk_time = time.time() - chunk_start
-        chunk_times.append(chunk_time)
-        logger.debug(f"{watch_info}Chunk {len(screenshot_chunks)} captured in {chunk_time:.2f}s")
         y += step_size
 
     await page.setViewport({'width': original_viewport['width'], 'height': original_viewport['height']})
@@ -115,53 +93,26 @@ async def capture_full_page(page, screenshot_format='JPEG', watch_uuid=None, loc
         with open(unlock_elements_js_path, 'r') as f:
             unlock_elements_js = f.read()
         await page.evaluate(unlock_elements_js)
-        logger.debug(f"{watch_info}Element dimensions unlocked after screenshot capture")
-
-    capture_time = time.time() - capture_start
-    total_capture_time = sum(chunk_times)
-    logger.debug(f"{watch_info}All {len(screenshot_chunks)} chunks captured in {capture_time:.2f}s (total chunk time: {total_capture_time:.2f}s)")
+        logger.debug("Element dimensions unlocked after screenshot capture")
 
     if len(screenshot_chunks) > 1:
-        stitch_start = time.time()
-        logger.debug(f"{watch_info}Starting stitching of {len(screenshot_chunks)} chunks")
-
-        # Always use spawn subprocess for ANY stitching (2+ chunks)
-        # PIL allocates at C level and Python GC never releases it - subprocess exit forces OS to reclaim
-        # Trade-off: 35MB resource_tracker vs 500MB+ PIL leak in main process
-        from changedetectionio.content_fetchers.screenshot_handler import stitch_images_worker_raw_bytes
-        import multiprocessing
-        import struct
-
+        # Always use spawn for thread safety - consistent behavior in tests and production
+        from changedetectionio.content_fetchers.screenshot_handler import stitch_images_worker
+        logger.debug(f"Screenshot stitching {len(screenshot_chunks)} chunks together")
         ctx = multiprocessing.get_context('spawn')
         parent_conn, child_conn = ctx.Pipe()
-        p = ctx.Process(target=stitch_images_worker_raw_bytes, args=(child_conn, page_height, SCREENSHOT_MAX_TOTAL_HEIGHT))
+        p = ctx.Process(target=stitch_images_worker, args=(child_conn, screenshot_chunks, page_height, SCREENSHOT_MAX_TOTAL_HEIGHT))
         p.start()
-
-        # Send via raw bytes (no pickle)
-        parent_conn.send_bytes(struct.pack('I', len(screenshot_chunks)))
-        for chunk in screenshot_chunks:
-            parent_conn.send_bytes(chunk)
-
         screenshot = parent_conn.recv_bytes()
         p.join()
-
-        parent_conn.close()
-        child_conn.close()
-        del p, parent_conn, child_conn
-
-        stitch_time = time.time() - stitch_start
-        total_time = time.time() - start
-        setup_time = total_time - capture_time - stitch_time
         logger.debug(
-            f"{watch_info}Screenshot complete - Page height: {page_height}px, Capture height: {SCREENSHOT_MAX_TOTAL_HEIGHT}px | "
-            f"Setup: {setup_time:.2f}s, Capture: {capture_time:.2f}s, Stitching: {stitch_time:.2f}s, Total: {total_time:.2f}s")
+            f"Screenshot (chunked/stitched) - Page height: {page_height} Capture height: {SCREENSHOT_MAX_TOTAL_HEIGHT} - Stitched together in {time.time() - start:.2f}s")
+
+        screenshot_chunks = None
         return screenshot
 
-    total_time = time.time() - start
-    setup_time = total_time - capture_time
     logger.debug(
-        f"{watch_info}Screenshot complete - Page height: {page_height}px, Capture height: {SCREENSHOT_MAX_TOTAL_HEIGHT}px | "
-        f"Setup: {setup_time:.2f}s, Single chunk: {capture_time:.2f}s, Total: {total_time:.2f}s")
+        f"Screenshot Page height: {page_height} Capture height: {SCREENSHOT_MAX_TOTAL_HEIGHT} - Stitched together in {time.time() - start:.2f}s")
     return screenshot_chunks[0]
 
 
@@ -222,36 +173,19 @@ class fetcher(Fetcher):
                 self.browser_connection_url += f"{r}--proxy-server={proxy_url}"
 
     async def quit(self, watch=None):
-        watch_uuid = watch.get('uuid') if watch else 'unknown'
-
-        # Close page
         try:
-            if hasattr(self, 'page') and self.page:
-                await asyncio.wait_for(self.page.close(), timeout=5.0)
-                logger.debug(f"[{watch_uuid}] Page closed successfully")
-        except asyncio.TimeoutError:
-            logger.warning(f"[{watch_uuid}] Timed out closing page (5s)")
+            await self.page.close()
+            del self.page
         except Exception as e:
-            logger.warning(f"[{watch_uuid}] Error closing page: {e}")
-        finally:
-            self.page = None
+            pass
 
-        # Close browser connection
         try:
-            if hasattr(self, 'browser') and self.browser:
-                await asyncio.wait_for(self.browser.close(), timeout=5.0)
-                logger.debug(f"[{watch_uuid}] Browser closed successfully")
-        except asyncio.TimeoutError:
-            logger.warning(f"[{watch_uuid}] Timed out closing browser (5s)")
+            await self.browser.close()
+            del self.browser
         except Exception as e:
-            logger.warning(f"[{watch_uuid}] Error closing browser: {e}")
-        finally:
-            self.browser = None
+            pass
 
-        logger.info(f"[{watch_uuid}] Cleanup puppeteer complete")
-
-        # Force garbage collection to release resources
-        gc.collect()
+        logger.info("Cleanup puppeteer complete.")
 
     async def fetch_page(self,
                          current_include_filters,
@@ -281,11 +215,9 @@ class fetcher(Fetcher):
         # Connect directly using the specified browser_ws_endpoint
         # @todo timeout
         try:
-            logger.debug(f"[{watch_uuid}] Connecting to browser at {self.browser_connection_url}")
             self.browser = await pyppeteer_instance.connect(browserWSEndpoint=self.browser_connection_url,
                                                             ignoreHTTPSErrors=True
                                                             )
-            logger.debug(f"[{watch_uuid}] Browser connected successfully")
         except websockets.exceptions.InvalidStatusCode as e:
             raise BrowserConnectError(msg=f"Error while trying to connect the browser, Code {e.status_code} (check your access, whitelist IP, password etc)")
         except websockets.exceptions.InvalidURI:
@@ -294,20 +226,7 @@ class fetcher(Fetcher):
             raise BrowserConnectError(msg=f"Error connecting to the browser - Exception '{str(e)}'")
 
         # more reliable is to just request a new page
-        try:
-            logger.debug(f"[{watch_uuid}] Creating new page")
-            self.page = await self.browser.newPage()
-            logger.debug(f"[{watch_uuid}] Page created successfully")
-        except Exception as e:
-            logger.error(f"[{watch_uuid}] Failed to create new page: {e}")
-            # Browser is connected but page creation failed - must cleanup browser
-            try:
-                await asyncio.wait_for(self.browser.close(), timeout=3.0)
-            except Exception as cleanup_error:
-                logger.error(f"[{watch_uuid}] Failed to cleanup browser after page creation failure: {cleanup_error}")
-            finally:
-                self.browser = None
-            raise
+        self.page = await self.browser.newPage()
         
         # Add console handler to capture console.log from favicon fetcher
         #self.page.on('console', lambda msg: logger.debug(f"Browser console [{msg.type}]: {msg.text}"))
@@ -376,12 +295,6 @@ class fetcher(Fetcher):
             w = extra_wait - 2 if extra_wait > 4 else 2
             logger.debug(f"Waiting {w} seconds before calling Page.stopLoading...")
             await asyncio.sleep(w)
-
-            # Check if page still exists (might have been closed due to error during sleep)
-            if not self.page or not hasattr(self.page, '_client'):
-                logger.debug("Page already closed, skipping stopLoading")
-                return
-
             logger.debug("Issuing stopLoading command...")
             await self.page._client.send('Page.stopLoading')
             logger.debug("stopLoading command sent!")
@@ -407,9 +320,7 @@ class fetcher(Fetcher):
             asyncio.create_task(handle_frame_navigation())
             response = await self.page.goto(url, timeout=0)
             await asyncio.sleep(1 + extra_wait)
-            # Check if page still exists before sending command
-            if self.page and hasattr(self.page, '_client'):
-                await self.page._client.send('Page.stopLoading')
+            await self.page._client.send('Page.stopLoading')
 
             if response:
                 break
@@ -446,7 +357,7 @@ class fetcher(Fetcher):
                 logger.error(f"Error fetching FavIcon info {str(e)}, continuing.")
 
         if self.status_code != 200 and not ignore_status_codes:
-            screenshot = await capture_full_page(page=self.page, screenshot_format=self.screenshot_format, watch_uuid=watch_uuid, lock_viewport_elements=self.lock_viewport_elements)
+            screenshot = await capture_full_page(page=self.page, screenshot_format=self.screenshot_format)
 
             raise Non200ErrorCodeReceived(url=url, status_code=self.status_code, screenshot=screenshot)
 
@@ -458,7 +369,7 @@ class fetcher(Fetcher):
 
         # Run Browser Steps here
         # @todo not yet supported, we switch to playwright in this case
-        #            if self.browser_steps:
+        #            if self.browser_steps_get_valid_steps():
         #                self.iterate_browser_steps()
 
 
@@ -476,11 +387,7 @@ class fetcher(Fetcher):
 
         # Now take screenshot (scrolling may trigger layout changes, but measurements are already captured)
         logger.debug(f"Screenshot format {self.screenshot_format}")
-        self.screenshot = await capture_full_page(page=self.page, screenshot_format=self.screenshot_format, watch_uuid=watch_uuid, lock_viewport_elements=self.lock_viewport_elements)
-
-        # Force garbage collection - pyppeteer base64 decode creates temporary buffers
-        import gc
-        gc.collect()
+        self.screenshot = await capture_full_page(page=self.page, screenshot_format=self.screenshot_format)
         self.xpath_data = await self.page.evaluate(XPATH_ELEMENT_JS, {
             "visualselector_xpath_selectors": visualselector_xpath_selectors,
             "max_height": MAX_TOTAL_HEIGHT
@@ -534,14 +441,6 @@ class fetcher(Fetcher):
             )
         except asyncio.TimeoutError:
             raise (BrowserFetchTimedOut(msg=f"Browser connected but was unable to process the page in {max_time} seconds."))
-        finally:
-            # Internal cleanup on any exception/timeout - call quit() immediately
-            # This prevents connection leaks during exception bursts
-            # Worker.py's quit() call becomes a redundant safety net (idempotent)
-            try:
-                await self.quit(watch={'uuid': watch_uuid} if watch_uuid else None)
-            except Exception as cleanup_error:
-                logger.error(f"[{watch_uuid}] Error during internal quit() cleanup: {cleanup_error}")
 
 
 # Plugin registration for built-in fetcher

changedetectionio/content_fetchers/requests.py

@@ -3,7 +3,7 @@ import hashlib
 import os
 import re
 import asyncio
-
+from functools import partial
 from changedetectionio import strtobool
 from changedetectionio.content_fetchers.exceptions import BrowserStepsInUnsupportedFetcher, EmptyReply, Non200ErrorCodeReceived
 from changedetectionio.content_fetchers.base import Fetcher
@@ -36,7 +36,7 @@ class fetcher(Fetcher):
         import requests
         from requests.exceptions import ProxyError, ConnectionError, RequestException
 
-        if self.browser_steps:
+        if self.browser_steps_get_valid_steps():
             raise BrowserStepsInUnsupportedFetcher(url=url)
 
         proxies = {}
@@ -55,26 +55,6 @@ class fetcher(Fetcher):
 
         session = requests.Session()
 
-        # Configure retry adapter for low-level network errors only
-        # Retries connection timeouts, read timeouts, connection resets - not HTTP status codes
-        # Especially helpful in parallel test execution when servers are slow/overloaded
-        # Configurable via REQUESTS_RETRY_MAX_COUNT (default: 3 attempts)
-        from requests.adapters import HTTPAdapter
-        from urllib3.util.retry import Retry
-
-        max_retries = int(os.getenv("REQUESTS_RETRY_MAX_COUNT", "6"))
-        retry_strategy = Retry(
-            total=max_retries,
-            connect=max_retries,  # Retry connection timeouts
-            read=max_retries,     # Retry read timeouts
-            status=0,             # Don't retry on HTTP status codes
-            backoff_factor=0.5,   # Wait 0.3s, 0.6s, 1.2s between retries
-            allowed_methods=["HEAD", "GET", "OPTIONS", "POST"],
-            raise_on_status=False
-        )
-        adapter = HTTPAdapter(max_retries=retry_strategy)
-        session.mount("http://", adapter)
-        session.mount("https://", adapter)
 
         if strtobool(os.getenv('ALLOW_FILE_URI', 'false')) and url.startswith('file://'):
             from requests_file import FileAdapter
@@ -162,11 +142,10 @@ class fetcher(Fetcher):
                   watch_uuid=None,
                   ):
         """Async wrapper that runs the synchronous requests code in a thread pool"""
-
+        
         loop = asyncio.get_event_loop()
-
+        
         # Run the synchronous _run_sync in a thread pool to avoid blocking the event loop
-        # Retry logic is handled by requests' HTTPAdapter (see _run_sync for configuration)
         await loop.run_in_executor(
             None,  # Use default ThreadPoolExecutor
             lambda: self._run_sync(
@@ -184,6 +163,7 @@ class fetcher(Fetcher):
         )
 
     async def quit(self, watch=None):
+
         # In case they switched to `requests` fetcher from something else
         # Then the screenshot could be old, in any case, it's not used here.
         # REMOVE_REQUESTS_OLD_SCREENSHOTS - Mainly used for testing

changedetectionio/content_fetchers/res/lock-elements-sizing.js

@@ -1,5 +1,5 @@
 /**
- * Lock Element Dimensions for Screenshot Capture (First Viewport Only)
+ * Lock Element Dimensions for Screenshot Capture
  *
  * THE PROBLEM:
  * When taking full-page screenshots of tall pages, Chrome/Puppeteer/Playwright need to:
@@ -10,31 +10,40 @@
  * However, changing the viewport height triggers CSS media queries like:
  *   @media (min-height: 860px) { .ad { height: 250px; } }
  *
- * This causes elements (especially ads/headers) to resize during screenshot capture.
+ * This causes elements (especially ads) to resize during screenshot capture, creating a mismatch:
+ * - Screenshot shows element at NEW size (after media query triggered)
+ * - xpath element coordinates measured at OLD size (before viewport change)
+ * - Visual selector overlays don't align with screenshot
+ *
+ * EXAMPLE BUG:
+ * - Initial viewport: 1280x800, ad height: 138px, article position: 279px ✓
+ * - Viewport changes to 1280x3809 for screenshot
+ * - Media query triggers: ad expands to 250px
+ * - All content below shifts down by 112px (250-138)
+ * - Article now at position: 391px (279+112)
+ * - But xpath data says 279px → 112px mismatch! ✗
  *
  * THE SOLUTION:
- * Lock element dimensions in the FIRST VIEWPORT ONLY with !important inline styles.
- * This prevents headers, navigation, and top ads from resizing when viewport changes.
- * We only lock the visible portion because:
- * - Most layout shifts happen in headers/navbars/top ads
- * - Locking only visible elements is 100x+ faster (100-200 elements vs 10,000+)
- * - Below-fold content shifts don't affect visual comparison accuracy
+ * Before changing viewport, lock ALL element dimensions with !important inline styles.
+ * Inline styles with !important override media query CSS, preventing layout changes.
  *
  * WHAT THIS SCRIPT DOES:
- * 1. Gets current viewport height
- * 2. Finds elements within first viewport (top of page to bottom of screen)
- * 3. Locks their dimensions with !important inline styles
+ * 1. Iterates through every element on the page
+ * 2. Captures current computed dimensions (width, height)
+ * 3. Sets inline styles with !important to freeze those dimensions
  * 4. Disables ResizeObserver API (for JS-based resizing)
+ * 5. When viewport changes for screenshot, media queries can't resize anything
+ * 6. Layout remains consistent → xpath coordinates match screenshot ✓
  *
  * USAGE:
  * Execute this script BEFORE calling capture_full_page() / screenshot functions.
- * Only enabled for image_ssim_diff processor (visual comparison).
- * Default: OFF for performance.
+ * The page must be fully loaded and settled at its initial viewport size.
+ * No need to restore state afterward - page is closed after screenshot.
  *
  * PERFORMANCE:
- * - Only processes 100-300 elements (first viewport) vs 10,000+ (entire page)
- * - Typically completes in 10-50ms
- * - 100x+ faster than locking entire page
+ * - Iterates all DOM elements (can be 1000s on complex pages)
+ * - Typically completes in 50-200ms
+ * - One-time cost before screenshot, well worth it for coordinate accuracy
  *
  * @see https://github.com/dgtlmoon/changedetection.io/issues/XXXX
  */
@@ -43,34 +52,11 @@
     // Store original styles in a global WeakMap for later restoration
     window.__elementSizingRestore = new WeakMap();
 
-    const start = performance.now();
-
-    // Get current viewport height (visible portion of page)
-    const viewportHeight = window.innerHeight;
-
-    // Get all elements and filter to FIRST VIEWPORT ONLY
-    // This dramatically reduces elements to process (100-300 vs 10,000+)
-    const allElements = Array.from(document.querySelectorAll('*'));
-
-    // BATCH READ PHASE: Get bounding rects and filter to viewport
-    const measurements = allElements.map(el => {
-        const rect = el.getBoundingClientRect();
+    // Lock ALL element dimensions to prevent media query layout changes
+    document.querySelectorAll('*').forEach(el => {
         const computed = window.getComputedStyle(el);
+        const rect = el.getBoundingClientRect();
 
-        // Only lock elements in the first viewport (visible on initial page load)
-        // rect.top < viewportHeight means element starts within visible area
-        const inViewport = rect.top < viewportHeight && rect.top >= 0;
-        const hasSize = rect.height > 0 && rect.width > 0;
-
-        return inViewport && hasSize ? { el, computed, rect } : null;
-    }).filter(Boolean);  // Remove null entries
-
-    const elapsed = performance.now() - start;
-    console.log(`Locked first viewport elements: ${measurements.length} of ${allElements.length} total elements (viewport height: ${viewportHeight}px, took ${elapsed.toFixed(0)}ms)`);
-
-    // BATCH WRITE PHASE: Apply all inline styles without triggering layout
-    // No interleaved reads means browser can optimize style application
-    measurements.forEach(({el, computed, rect}) => {
         // Save original inline style values BEFORE locking
         const properties = ['height', 'min-height', 'max-height', 'width', 'min-width', 'max-width'];
         const originalStyles = {};
@@ -103,5 +89,5 @@
         disconnect() {}
     };
 
-    console.log(`✓ Element dimensions locked (${measurements.length} elements) to prevent media query changes during screenshot`);
+    console.log('✓ Element dimensions locked to prevent media query changes during screenshot');
 })();

changedetectionio/content_fetchers/screenshot_handler.py

@@ -8,42 +8,92 @@ from loguru import logger
 
 from changedetectionio.content_fetchers import SCREENSHOT_MAX_HEIGHT_DEFAULT, SCREENSHOT_DEFAULT_QUALITY
 
-def stitch_images_worker_raw_bytes(pipe_conn, original_page_height, capture_height):
+# Cache font to avoid loading on every stitch
+_cached_font = None
+
+def _get_caption_font():
+    """Get or create cached font for caption text."""
+    global _cached_font
+    if _cached_font is None:
+        from PIL import ImageFont
+        try:
+            _cached_font = ImageFont.truetype("arial.ttf", 35)
+        except IOError:
+            _cached_font = ImageFont.load_default()
+    return _cached_font
+
+
+def stitch_images_inline(chunks_bytes, original_page_height, capture_height):
     """
-    Stitch image chunks together in a separate process.
-
-    Uses spawn multiprocessing to isolate PIL's C-level memory allocation.
-    When the subprocess exits, the OS reclaims ALL memory including C-level allocations
-    that Python's GC cannot release. This prevents the ~50MB per stitch from accumulating
-    in the main process.
-
-    Trade-off: Adds 35MB resource_tracker subprocess, but prevents 500MB+ memory leak
-    in main process (much better at scale: 35GB vs 500GB for 1000 instances).
+    Stitch image chunks together inline (no multiprocessing).
+    Optimized for small number of chunks (2-3) to avoid process creation overhead.
 
     Args:
-        pipe_conn: Pipe connection to receive data and send result
+        chunks_bytes: List of JPEG image bytes
         original_page_height: Original page height in pixels
         capture_height: Maximum capture height
+
+    Returns:
+        bytes: Stitched JPEG image
+    """
+    import os
+    import io
+    from PIL import Image, ImageDraw
+
+    # Load images from byte chunks
+    images = [Image.open(io.BytesIO(b)) for b in chunks_bytes]
+    total_height = sum(im.height for im in images)
+    max_width = max(im.width for im in images)
+
+    # Create stitched image
+    stitched = Image.new('RGB', (max_width, total_height))
+    y_offset = 0
+    for im in images:
+        stitched.paste(im, (0, y_offset))
+        y_offset += im.height
+        im.close()  # Close immediately after pasting
+
+    # Draw caption only if page was trimmed
+    if original_page_height > capture_height:
+        draw = ImageDraw.Draw(stitched)
+        caption_text = f"WARNING: Screenshot was {original_page_height}px but trimmed to {capture_height}px because it was too long"
+        padding = 10
+        font = _get_caption_font()
+
+        bbox = draw.textbbox((0, 0), caption_text, font=font)
+        text_width = bbox[2] - bbox[0]
+        text_height = bbox[3] - bbox[1]
+
+        # Draw white background rectangle
+        draw.rectangle([(0, 0), (max_width, text_height + 2 * padding)], fill=(255, 255, 255))
+
+        # Draw text centered
+        text_x = (max_width - text_width) // 2
+        draw.text((text_x, padding), caption_text, font=font, fill=(255, 0, 0))
+
+    # Encode to JPEG
+    output = io.BytesIO()
+    stitched.save(output, format="JPEG", quality=int(os.getenv("SCREENSHOT_QUALITY", SCREENSHOT_DEFAULT_QUALITY)), optimize=True)
+    result = output.getvalue()
+
+    # Cleanup
+    stitched.close()
+
+    return result
+
+
+def stitch_images_worker(pipe_conn, chunks_bytes, original_page_height, capture_height):
+    """
+    Stitch image chunks together in a separate process.
+    Used for large number of chunks (4+) to avoid blocking the main event loop.
     """
     import os
     import io
-    import struct
     from PIL import Image, ImageDraw, ImageFont
 
     try:
-        # Receive chunk count as 4-byte integer (no pickle!)
-        count_bytes = pipe_conn.recv_bytes()
-        chunk_count = struct.unpack('I', count_bytes)[0]
-
-        # Receive each chunk as raw bytes (no pickle!)
-        chunks_bytes = []
-        for _ in range(chunk_count):
-            chunks_bytes.append(pipe_conn.recv_bytes())
-
         # Load images from byte chunks
         images = [Image.open(io.BytesIO(b)) for b in chunks_bytes]
-        del chunks_bytes
-
         total_height = sum(im.height for im in images)
         max_width = max(im.width for im in images)
 
@@ -53,14 +103,15 @@ def stitch_images_worker_raw_bytes(pipe_conn, original_page_height, capture_heig
         for im in images:
             stitched.paste(im, (0, y_offset))
             y_offset += im.height
-            im.close()
-        del images
+            im.close()  # Close immediately after pasting
 
         # Draw caption only if page was trimmed
         if original_page_height > capture_height:
             draw = ImageDraw.Draw(stitched)
             caption_text = f"WARNING: Screenshot was {original_page_height}px but trimmed to {capture_height}px because it was too long"
             padding = 10
+
+            # Try to load font
             try:
                 font = ImageFont.truetype("arial.ttf", 35)
             except IOError:
@@ -69,26 +120,23 @@ def stitch_images_worker_raw_bytes(pipe_conn, original_page_height, capture_heig
             bbox = draw.textbbox((0, 0), caption_text, font=font)
             text_width = bbox[2] - bbox[0]
             text_height = bbox[3] - bbox[1]
+
+            # Draw white background rectangle
             draw.rectangle([(0, 0), (max_width, text_height + 2 * padding)], fill=(255, 255, 255))
+
+            # Draw text centered
             text_x = (max_width - text_width) // 2
             draw.text((text_x, padding), caption_text, font=font, fill=(255, 0, 0))
 
-        # Encode and send
+        # Encode and send image with optimization
         output = io.BytesIO()
         stitched.save(output, format="JPEG", quality=int(os.getenv("SCREENSHOT_QUALITY", SCREENSHOT_DEFAULT_QUALITY)), optimize=True)
-        result_bytes = output.getvalue()
+        pipe_conn.send_bytes(output.getvalue())
 
         stitched.close()
-        del stitched
-        output.close()
-        del output
-
-        pipe_conn.send_bytes(result_bytes)
-        del result_bytes
-
     except Exception as e:
-        logger.error(f"Error in stitch_images_worker_raw_bytes: {e}")
-        error_msg = f"error:{e}".encode('utf-8')
-        pipe_conn.send_bytes(error_msg)
+        pipe_conn.send(f"error:{e}")
     finally:
         pipe_conn.close()
+
+

changedetectionio/favicon_utils.py

@@ -1,43 +0,0 @@
-"""
-Favicon utilities for changedetection.io
-Handles favicon MIME type detection with caching
-"""
-
-from functools import lru_cache
-
-
-@lru_cache(maxsize=1000)
-def get_favicon_mime_type(filepath):
-    """
-    Detect MIME type of favicon by reading file content using puremagic.
-    Results are cached to avoid repeatedly reading the same files.
-
-    Args:
-        filepath: Full path to the favicon file
-
-    Returns:
-        MIME type string (e.g., 'image/png')
-    """
-    mime = None
-
-    try:
-        import puremagic
-        with open(filepath, 'rb') as f:
-            content_bytes = f.read(200)  # Read first 200 bytes
-
-        detections = puremagic.magic_string(content_bytes)
-        if detections:
-            mime = detections[0].mime_type
-    except Exception:
-        pass
-
-    # Fallback to mimetypes if puremagic fails
-    if not mime:
-        import mimetypes
-        mime, _ = mimetypes.guess_type(filepath)
-
-    # Final fallback based on extension
-    if not mime:
-        mime = 'image/x-icon' if filepath.endswith('.ico') else 'image/png'
-
-    return mime

changedetectionio/flask_app.py

@@ -14,7 +14,7 @@ from pathlib import Path
 from changedetectionio.strtobool import strtobool
 from threading import Event
 from changedetectionio.queue_handlers import RecheckPriorityQueue, NotificationQueue
-from changedetectionio import worker_pool
+from changedetectionio import worker_handler
 
 from flask import (
     Flask,
@@ -27,6 +27,7 @@ from flask import (
     session,
     url_for,
 )
+from flask_compress import Compress as FlaskCompress
 from flask_restful import abort, Api
 from flask_cors import CORS
 
@@ -43,8 +44,6 @@ from changedetectionio.api import Watch, WatchHistory, WatchSingleHistory, Watch
 from changedetectionio.api.Search import Search
 from .time_handler import is_within_schedule
 from changedetectionio.languages import get_available_languages, get_language_codes, get_flag_for_locale, get_timeago_locale
-from changedetectionio.favicon_utils import get_favicon_mime_type
-
 IN_PYTEST = "pytest" in sys.modules or "PYTEST_CURRENT_TEST" in os.environ
 
 datastore = None
@@ -69,18 +68,10 @@ socketio_server = None
 # Enable CORS, especially useful for the Chrome extension to operate from anywhere
 CORS(app)
 
-# Flask-Compress handles HTTP compression, Socket.IO compression disabled to prevent memory leak.
-# There's also a bug between flask compress and socketio that causes some kind of slow memory leak
-# It's better to use compression on your reverse proxy (nginx etc) instead.
-if strtobool(os.getenv("FLASK_ENABLE_COMPRESSION")):
-    from flask_compress import Compress as FlaskCompress
-    app.config['COMPRESS_MIN_SIZE'] = 2096
-    app.config['COMPRESS_MIMETYPES'] = ['text/html', 'text/css', 'text/javascript', 'application/json', 'application/javascript', 'image/svg+xml']
-    # Use gzip only - smaller memory footprint than zstd/brotli (4-8KB vs 200-500KB contexts)
-    app.config['COMPRESS_ALGORITHM'] = ['gzip']
-    compress = FlaskCompress()
-    compress.init_app(app)
-
+# Super handy for compressing large BrowserSteps responses and others
+FlaskCompress(app)
+app.config['COMPRESS_MIN_SIZE'] = 4096
+app.config['COMPRESS_MIMETYPES'] = ['text/html', 'text/css', 'text/javascript', 'application/json', 'application/javascript', 'image/svg+xml']
 app.config['TEMPLATES_AUTO_RELOAD'] = False
 
 
@@ -97,14 +88,6 @@ if os.getenv('FLASK_SERVER_NAME'):
 app.config['BABEL_TRANSLATION_DIRECTORIES'] = str(Path(__file__).parent / 'translations')
 app.config['BABEL_DEFAULT_LOCALE'] = 'en_GB'
 
-# Session configuration
-# NOTE: Flask session (for locale, etc.) is separate from Flask-Login's remember-me cookie
-# - Flask session stores data like session['locale'] in a signed cookie
-# - Flask-Login's remember=True creates a separate authentication cookie
-# - Setting PERMANENT_SESSION_LIFETIME controls how long the Flask session cookie lasts
-from datetime import timedelta
-app.config['PERMANENT_SESSION_LIFETIME'] = timedelta(days=3650)  # ~10 years (effectively unlimited)
-
 #app.config["EXPLAIN_TEMPLATE_LOADING"] = True
 
 
@@ -198,7 +181,7 @@ def _jinja2_filter_format_number_locale(value: float) -> str:
 
 @app.template_global('is_checking_now')
 def _watch_is_checking_now(watch_obj, format="%Y-%m-%d %H:%M:%S"):
-    return worker_pool.is_watch_running(watch_obj['uuid'])
+    return worker_handler.is_watch_running(watch_obj['uuid'])
 
 @app.template_global('get_watch_queue_position')
 def _get_watch_queue_position(watch_obj):
@@ -209,13 +192,13 @@ def _get_watch_queue_position(watch_obj):
 @app.template_global('get_current_worker_count')
 def _get_current_worker_count():
     """Get the current number of operational workers"""
-    return worker_pool.get_worker_count()
+    return worker_handler.get_worker_count()
 
 @app.template_global('get_worker_status_info')
 def _get_worker_status_info():
     """Get detailed worker status information for display"""
-    status = worker_pool.get_worker_status()
-    running_uuids = worker_pool.get_running_uuids()
+    status = worker_handler.get_worker_status()
+    running_uuids = worker_handler.get_running_uuids()
     
     return {
         'count': status['worker_count'],
@@ -269,47 +252,6 @@ def _jinja2_filter_seconds_precise(timestamp):
 
     return format(int(time.time()-timestamp), ',d')
 
-@app.template_filter('format_duration')
-def _jinja2_filter_format_duration(seconds):
-    """Format a duration in seconds into human readable string like '5 days, 3 hours, 30 minutes'"""
-    from datetime import timedelta
-
-    if not seconds or seconds < 0:
-        return gettext('0 seconds')
-
-    td = timedelta(seconds=int(seconds))
-
-    # Calculate components
-    years = td.days // 365
-    remaining_days = td.days % 365
-    months = remaining_days // 30
-    remaining_days = remaining_days % 30
-    weeks = remaining_days // 7
-    days = remaining_days % 7
-
-    hours = td.seconds // 3600
-    minutes = (td.seconds % 3600) // 60
-    secs = td.seconds % 60
-
-    # Build parts list
-    parts = []
-    if years > 0:
-        parts.append(f"{years} {gettext('year') if years == 1 else gettext('years')}")
-    if months > 0:
-        parts.append(f"{months} {gettext('month') if months == 1 else gettext('months')}")
-    if weeks > 0:
-        parts.append(f"{weeks} {gettext('week') if weeks == 1 else gettext('weeks')}")
-    if days > 0:
-        parts.append(f"{days} {gettext('day') if days == 1 else gettext('days')}")
-    if hours > 0:
-        parts.append(f"{hours} {gettext('hour') if hours == 1 else gettext('hours')}")
-    if minutes > 0:
-        parts.append(f"{minutes} {gettext('minute') if minutes == 1 else gettext('minutes')}")
-    if secs > 0 or not parts:
-        parts.append(f"{secs} {gettext('second') if secs == 1 else gettext('seconds')}")
-
-    return ", ".join(parts)
-
 @app.template_filter('fetcher_status_icons')
 def _jinja2_filter_fetcher_status_icons(fetcher_name):
     """Get status icon HTML for a given fetcher.
@@ -432,9 +374,6 @@ def changedetection_app(config=None, datastore_o=None):
     global datastore, socketio_server
     datastore = datastore_o
 
-    # Set datastore reference in notification queue for all_muted checking
-    notification_q.set_datastore(datastore)
-
     # Import and create a wrapper for is_safe_url that has access to app
     from changedetectionio.is_safe_url import is_safe_url as _is_safe_url
 
@@ -445,10 +384,7 @@ def changedetection_app(config=None, datastore_o=None):
     # so far just for read-only via tests, but this will be moved eventually to be the main source
     # (instead of the global var)
     app.config['DATASTORE'] = datastore_o
-
-    # Store batch mode flag to skip background threads when running in batch mode
-    app.config['batch_mode'] = config.get('batch_mode', False) if config else False
-
+    
     # Store the signal in the app config to ensure it's accessible everywhere
     app.config['watch_check_update_SIGNAL'] = watch_check_update
 
@@ -461,27 +397,11 @@ def changedetection_app(config=None, datastore_o=None):
     language_codes = get_language_codes()
 
     def get_locale():
-        # Locale aliases: map browser language codes to translation directory names
-        # This handles cases where browsers send standard codes (e.g., zh-TW)
-        # but our translations use more specific codes (e.g., zh_Hant_TW)
-        locale_aliases = {
-            'zh-TW': 'zh_Hant_TW',  # Traditional Chinese: browser sends zh-TW, we use zh_Hant_TW
-            'zh_TW': 'zh_Hant_TW',  # Also handle underscore variant
-        }
-
         # 1. Try to get locale from session (user explicitly selected)
         if 'locale' in session:
             return session['locale']
-
         # 2. Fall back to Accept-Language header
-        # Get the best match from browser's Accept-Language header
-        browser_locale = request.accept_languages.best_match(language_codes + list(locale_aliases.keys()))
-
-        # 3. Check if we need to map the browser locale to our internal locale
-        if browser_locale in locale_aliases:
-            return locale_aliases[browser_locale]
-
-        return browser_locale
+        return request.accept_languages.best_match(language_codes)
 
     # Initialize Babel with locale selector
     babel = Babel(app, locale_selector=get_locale)
@@ -605,9 +525,6 @@ def changedetection_app(config=None, datastore_o=None):
 
         # Validate the locale against available languages
         if locale in language_codes:
-            # Make session permanent so language preference persists across browser sessions
-            # NOTE: This is the Flask session cookie (separate from Flask-Login's remember-me auth cookie)
-            session.permanent = True
             session['locale'] = locale
 
             # CRITICAL: Flask-Babel caches the locale in the request context (ctx.babel_locale)
@@ -711,14 +628,8 @@ def changedetection_app(config=None, datastore_o=None):
     def static_content(group, filename):
         from flask import make_response
         import re
-
-        # Strict sanitization: only allow a-z, 0-9, and underscore (blocks .. and other traversal)
-        group = re.sub(r'[^a-z0-9_-]+', '', group.lower())
-        filename = filename
-
-        # Additional safety: reject if sanitization resulted in empty strings
-        if not group or not filename:
-            abort(404)
+        group = re.sub(r'[^\w.-]+', '', group.lower())
+        filename = re.sub(r'[^\w.-]+', '', filename.lower())
 
         if group == 'screenshot':
             # Could be sensitive, follow password requirements
@@ -752,11 +663,18 @@ def changedetection_app(config=None, datastore_o=None):
 
             favicon_filename = watch.get_favicon_filename()
             if favicon_filename:
-                # Use cached MIME type detection
-                filepath = os.path.join(watch.data_dir, favicon_filename)
-                mime = get_favicon_mime_type(filepath)
+                try:
+                    import magic
+                    mime = magic.from_file(
+                        os.path.join(watch.watch_data_dir, favicon_filename),
+                        mime=True
+                    )
+                except ImportError:
+                    # Fallback, no python-magic
+                    import mimetypes
+                    mime, encoding = mimetypes.guess_type(favicon_filename)
 
-                response = make_response(send_from_directory(watch.data_dir, favicon_filename))
+                response = make_response(send_from_directory(watch.watch_data_dir, favicon_filename))
                 response.headers['Content-type'] = mime
                 response.headers['Cache-Control'] = 'max-age=300, must-revalidate'  # Cache for 5 minutes, then revalidate
                 return response
@@ -851,15 +769,13 @@ def changedetection_app(config=None, datastore_o=None):
 
     # watchlist UI buttons etc
     import changedetectionio.blueprint.ui as ui
-    app.register_blueprint(ui.construct_blueprint(datastore, update_q, worker_pool, queuedWatchMetaData, watch_check_update))
+    app.register_blueprint(ui.construct_blueprint(datastore, update_q, worker_handler, queuedWatchMetaData, watch_check_update))
 
     import changedetectionio.blueprint.watchlist as watchlist
     app.register_blueprint(watchlist.construct_blueprint(datastore=datastore, update_q=update_q, queuedWatchMetaData=queuedWatchMetaData), url_prefix='')
 
     # Initialize Socket.IO server conditionally based on settings
-    socket_io_enabled = datastore.data['settings']['application'].get('ui', {}).get('socket_io_enabled', True)
-    if socket_io_enabled and app.config.get('batch_mode'):
-        socket_io_enabled = False
+    socket_io_enabled = datastore.data['settings']['application']['ui'].get('socket_io_enabled', True)
     if socket_io_enabled:
         from changedetectionio.realtime.socket_server import init_socketio
         global socketio_server
@@ -888,10 +804,10 @@ def changedetection_app(config=None, datastore_o=None):
         expected_workers = int(os.getenv("FETCH_WORKERS", datastore.data['settings']['requests']['workers']))
         
         # Get basic status
-        status = worker_pool.get_worker_status()
+        status = worker_handler.get_worker_status()
         
         # Perform health check
-        health_result = worker_pool.check_worker_health(
+        health_result = worker_handler.check_worker_health(
             expected_count=expected_workers,
             update_q=update_q,
             notification_q=notification_q,
@@ -955,31 +871,16 @@ def changedetection_app(config=None, datastore_o=None):
     # Can be overridden by ENV or use the default settings
     n_workers = int(os.getenv("FETCH_WORKERS", datastore.data['settings']['requests']['workers']))
     logger.info(f"Starting {n_workers} workers during app initialization")
-    worker_pool.start_workers(n_workers, update_q, notification_q, app, datastore)
+    worker_handler.start_workers(n_workers, update_q, notification_q, app, datastore)
 
-    # Skip background threads in batch mode (just process queue and exit)
-    batch_mode = app.config.get('batch_mode', False)
-    if not batch_mode:
-        # @todo handle ctrl break
-        ticker_thread = threading.Thread(target=ticker_thread_check_time_launch_checks, daemon=True, name="TickerThread-ScheduleChecker").start()
+    # @todo handle ctrl break
+    ticker_thread = threading.Thread(target=ticker_thread_check_time_launch_checks, daemon=True, name="TickerThread-ScheduleChecker").start()
+    threading.Thread(target=notification_runner, daemon=True, name="NotificationRunner").start()
 
-        # Start configurable number of notification workers (default 1)
-        notification_workers = int(os.getenv("NOTIFICATION_WORKERS", "1"))
-        for i in range(notification_workers):
-            threading.Thread(
-                target=notification_runner,
-                args=(i,),
-                daemon=True,
-                name=f"NotificationRunner-{i}"
-            ).start()
-        logger.info(f"Started {notification_workers} notification worker(s)")
-
-        in_pytest = "pytest" in sys.modules or "PYTEST_CURRENT_TEST" in os.environ
-        # Check for new release version, but not when running in test/build or pytest
-        if not os.getenv("GITHUB_REF", False) and not strtobool(os.getenv('DISABLE_VERSION_CHECK', 'no')) and not in_pytest:
-            threading.Thread(target=check_for_new_version, daemon=True, name="VersionChecker").start()
-    else:
-        logger.info("Batch mode: Skipping ticker thread, notification runner, and version checker")
+    in_pytest = "pytest" in sys.modules or "PYTEST_CURRENT_TEST" in os.environ
+    # Check for new release version, but not when running in test/build or pytest
+    if not os.getenv("GITHUB_REF", False) and not strtobool(os.getenv('DISABLE_VERSION_CHECK', 'no')) and not in_pytest:
+        threading.Thread(target=check_for_new_version, daemon=True, name="VersionChecker").start()
 
     # Return the Flask app - the Socket.IO will be attached to it but initialized separately
     # This avoids circular dependencies
@@ -1014,14 +915,14 @@ def check_for_new_version():
         app.config.exit.wait(86400)
 
 
-def notification_runner(worker_id=0):
+def notification_runner():
     global notification_debug_log
     from datetime import datetime
     import json
     with app.app_context():
         while not app.config.exit.is_set():
             try:
-                # Multiple workers can run concurrently (configurable via NOTIFICATION_WORKERS)
+                # At the moment only one thread runs (single runner)
                 n_object = notification_q.get(block=False)
             except queue.Empty:
                 app.config.exit.wait(1)
@@ -1047,7 +948,7 @@ def notification_runner(worker_id=0):
                         sent_obj = process_notification(n_object, datastore)
 
                 except Exception as e:
-                    logger.error(f"Notification worker {worker_id} - Watch URL: {n_object['watch_url']}  Error {str(e)}")
+                    logger.error(f"Watch URL: {n_object['watch_url']}  Error {str(e)}")
 
                     # UUID wont be present when we submit a 'test' from the global settings
                     if 'uuid' in n_object:
@@ -1088,7 +989,7 @@ def ticker_thread_check_time_launch_checks():
         now = time.time()
         if now - last_health_check > 60:
             expected_workers = int(os.getenv("FETCH_WORKERS", datastore.data['settings']['requests']['workers']))
-            health_result = worker_pool.check_worker_health(
+            health_result = worker_handler.check_worker_health(
                 expected_count=expected_workers,
                 update_q=update_q,
                 notification_q=notification_q,
@@ -1098,16 +999,11 @@ def ticker_thread_check_time_launch_checks():
             
             if health_result['status'] != 'healthy':
                 logger.warning(f"Worker health check: {health_result['message']}")
-
+                
             last_health_check = now
 
-        # Check if all checks are paused
-        if datastore.data['settings']['application'].get('all_paused', False):
-            app.config.exit.wait(1)
-            continue
-
         # Get a list of watches by UUID that are currently fetching data
-        running_uuids = worker_pool.get_running_uuids()
+        running_uuids = worker_handler.get_running_uuids()
 
         # Build set of queued UUIDs once for O(1) lookup instead of O(n) per watch
         queued_uuids = {q_item.item['uuid'] for q_item in update_q.queue}
@@ -1213,7 +1109,7 @@ def ticker_thread_check_time_launch_checks():
                     priority = int(time.time())
 
                     # Into the queue with you
-                    queued_successfully = worker_pool.queue_item_async_safe(update_q,
+                    queued_successfully = worker_handler.queue_item_async_safe(update_q,
                                                                                queuedWatchMetaData.PrioritizedItem(priority=priority,
                                                                                                                    item={'uuid': uuid})
                                                                                )

changedetectionio/forms.py

@@ -7,6 +7,8 @@ from flask_babel import lazy_gettext as _l, gettext
 from changedetectionio.blueprint.rss import RSS_FORMAT_TYPES, RSS_TEMPLATE_TYPE_OPTIONS, RSS_TEMPLATE_HTML_DEFAULT
 from changedetectionio.conditions.form import ConditionFormRow
 from changedetectionio.notification_service import NotificationContextData
+from changedetectionio.processors.image_ssim_diff import SCREENSHOT_COMPARISON_THRESHOLD_OPTIONS, \
+    SCREENSHOT_COMPARISON_THRESHOLD_OPTIONS_DEFAULT
 from changedetectionio.strtobool import strtobool
 from changedetectionio import processors
 
@@ -35,7 +37,7 @@ from changedetectionio.widgets import TernaryNoneBooleanField
 
 # default
 # each select <option data-enabled="enabled-0-0"
-from changedetectionio.browser_steps.browser_steps import browser_step_ui_config
+from changedetectionio.blueprint.browser_steps.browser_steps import browser_step_ui_config
 
 from changedetectionio import html_tools, content_fetchers
 
@@ -492,6 +494,7 @@ class ValidateJinja2Template(object):
     Validates that a {token} is from a valid set
     """
     def __call__(self, form, field):
+        from changedetectionio import notification
         from changedetectionio.jinja2_custom import create_jinja_env
         from jinja2 import BaseLoader, TemplateSyntaxError, UndefinedError
         from jinja2.meta import find_undeclared_variables
@@ -724,10 +727,10 @@ class ValidateStartsWithRegex(object):
                 raise ValidationError(self.message or _l("Invalid value."))
 
 class quickWatchForm(Form):
-    url = fields.URLField(_l('URL'), validators=[validateURL()])
-    tags = StringTagUUID(_l('Group tag'), validators=[validators.Optional()])
+    url = fields.URLField('URL', validators=[validateURL()])
+    tags = StringTagUUID('Group tag', [validators.Optional()])
     watch_submit_button = SubmitField(_l('Watch'), render_kw={"class": "pure-button pure-button-primary"})
-    processor = RadioField(_l('Processor'), choices=lambda: processors.available_processors(), default=processors.get_default_processor)
+    processor = RadioField(_l('Processor'), choices=lambda: processors.available_processors(), default="text_json_diff")
     edit_and_watch_submit_button = SubmitField(_l('Edit > Watch'), render_kw={"class": "pure-button pure-button-primary"})
 
 
@@ -746,7 +749,7 @@ class commonSettingsForm(Form):
     notification_format = SelectField(_l('Notification format'), choices=list(valid_notification_formats.items()))
     notification_title = StringField(_l('Notification Title'), default='ChangeDetection.io Notification - {{ watch_url }}', validators=[validators.Optional(), ValidateJinja2Template()])
     notification_urls = StringListField(_l('Notification URL List'), validators=[validators.Optional(), ValidateAppRiseServers(), ValidateJinja2Template()])
-    processor = RadioField( label=_l("Processor - What do you want to achieve?"), choices=lambda: processors.available_processors(), default=processors.get_default_processor)
+    processor = RadioField( label=_l("Processor - What do you want to achieve?"), choices=lambda: processors.available_processors(), default="text_json_diff")
     scheduler_timezone_default = StringField(_l("Default timezone for watch check scheduler"), render_kw={"list": "timezones"}, validators=[validateTimeZoneName()])
     webdriver_delay = IntegerField(_l('Wait seconds before extracting text'), validators=[validators.Optional(), validators.NumberRange(min=1, message=_l("Should contain one or more seconds"))])
 
@@ -760,7 +763,7 @@ class commonSettingsForm(Form):
 
 
 class importForm(Form):
-    processor = RadioField(_l('Processor'), choices=lambda: processors.available_processors(), default=processors.get_default_processor)
+    processor = RadioField(_l('Processor'), choices=lambda: processors.available_processors(), default="text_json_diff")
     urls = TextAreaField(_l('URLs'))
     xlsx_file = FileField(_l('Upload .xlsx file'), validators=[FileAllowed(['xlsx'], _l('Must be .xlsx file!'))])
     file_mapping = SelectField(_l('File mapping'), [validators.DataRequired()], choices={('wachete', 'Wachete mapping'), ('custom','Custom mapping')})
@@ -783,7 +786,6 @@ class processor_text_json_diff_form(commonSettingsForm):
 
     time_between_check = EnhancedFormField(
         TimeBetweenCheckForm,
-        label=_l('Time Between Check'),
         conditional_field='time_between_check_use_default',
         conditional_message=REQUIRE_ATLEAST_ONE_TIME_PART_WHEN_NOT_GLOBAL_DEFAULT,
         conditional_test_function=validate_time_between_check_has_values
@@ -817,7 +819,8 @@ class processor_text_json_diff_form(commonSettingsForm):
     filter_text_removed = BooleanField(_l('Removed lines'), default=True)
 
     trigger_text = StringListField(_l('Keyword triggers - Trigger/wait for text'), [validators.Optional(), ValidateListRegex()])
-    browser_steps = FieldList(FormField(SingleBrowserStep), min_entries=10)
+    if os.getenv("PLAYWRIGHT_DRIVER_URL"):
+        browser_steps = FieldList(FormField(SingleBrowserStep), min_entries=10)
     text_should_not_be_present = StringListField(_l('Block change-detection while text matches'), [validators.Optional(), ValidateListRegex()])
     webdriver_js_execute_code = TextAreaField(_l('Execute JavaScript before change detection'), render_kw={"rows": "5"}, validators=[validators.Optional()])
 
@@ -833,8 +836,6 @@ class processor_text_json_diff_form(commonSettingsForm):
     conditions = FieldList(FormField(ConditionFormRow), min_entries=1)  # Add rule logic here
     use_page_title_in_list = TernaryNoneBooleanField(_l('Use page <title> in list'), default=None)
 
-    history_snapshot_max_length = IntegerField(_l('Number of history items per watch to keep'), render_kw={"style": "width: 5em;"}, validators=[validators.Optional(), validators.NumberRange(min=2)])
-
     def extra_tab_content(self):
         return None
 
@@ -946,7 +947,7 @@ class DefaultUAInputForm(Form):
 
 # datastore.data['settings']['requests']..
 class globalSettingsRequestForm(Form):
-    time_between_check = RequiredFormField(TimeBetweenCheckForm, label=_l('Time Between Check'))
+    time_between_check = RequiredFormField(TimeBetweenCheckForm)
     time_schedule_limit = FormField(ScheduleLimitForm)
     proxy = RadioField(_l('Default proxy'))
     jitter_seconds = IntegerField(_l('Random jitter seconds ± check'),
@@ -1006,7 +1007,7 @@ class globalSettingsApplicationForm(commonSettingsForm):
         render_kw={"placeholder": "0.1", "style": "width: 8em;"}
     )
 
-    password = SaltyPasswordField(_l('Password'))
+    password = SaltyPasswordField()
     pager_size = IntegerField(_l('Pager size'),
                               render_kw={"style": "width: 5em;"},
                               validators=[validators.NumberRange(min=0,
@@ -1032,8 +1033,6 @@ class globalSettingsApplicationForm(commonSettingsForm):
                                                                   render_kw={"style": "width: 5em;"},
                                                                   validators=[validators.NumberRange(min=0,
                                                                                                      message=_l("Should contain zero or more attempts"))])
-
-    history_snapshot_max_length = IntegerField(_l('Number of history items per watch to keep'), render_kw={"style": "width: 5em;"}, validators=[validators.Optional(), validators.NumberRange(min=2)])
     ui = FormField(globalSettingsApplicationUIForm)
 
 

changedetectionio/html_tools.py

@@ -539,18 +539,6 @@ def cdata_in_document_to_text(html_content: str, render_anchor_tag_content=False
 
 
 def html_to_text(html_content: str, render_anchor_tag_content=False, is_rss=False, timeout=10) -> str:
-    """
-    Convert HTML content to plain text using inscriptis.
-
-    Thread-Safety: This function uses inscriptis.get_text() which internally calls
-    lxml.html.fromstring() with the default parser. Testing with 50 concurrent threads
-    confirms this approach is thread-safe and produces deterministic output.
-
-    Alternative Approach Rejected: An explicit HTMLParser instance (thread-local or fresh)
-    would also be thread-safe, but was found to break change detection logic in subtle ways
-    (test_check_basic_change_detection_functionality). The default parser provides correct
-    and reliable behavior.
-    """
     from inscriptis import get_text
     from inscriptis.model.config import ParserConfig
 
@@ -561,33 +549,10 @@ def html_to_text(html_content: str, render_anchor_tag_content=False, is_rss=Fals
         )
     else:
         parser_config = None
+
     if is_rss:
         html_content = re.sub(r'<title([\s>])', r'<h1\1', html_content)
         html_content = re.sub(r'</title>', r'</h1>', html_content)
-    else:
-        # Use BS4 html.parser to strip bloat — SPA's often dump 10MB+ of CSS/JS into <head>,
-        # causing inscriptis to silently give up. Regex-based stripping is unsafe because tags
-        # can appear inside JSON data attributes with JS-escaped closing tags (e.g. <\/script>),
-        # causing the regex to scan past the intended close and eat real page content.
-        from bs4 import BeautifulSoup
-        soup = BeautifulSoup(html_content, 'html.parser')
-        # Strip tags that inscriptis cannot render as meaningful text and which can be very large.
-        # svg/math: produce path-data/MathML garbage; canvas/iframe/template: no inscriptis handlers.
-        # video/audio/picture are kept — they may contain meaningful fallback text or captions.
-        for tag in soup.find_all(['head', 'script', 'style', 'noscript', 'svg',
-                                  'math', 'canvas', 'iframe', 'template']):
-            tag.decompose()
-
-        # SPAs often use <body style="display:none"> to hide content until JS loads.
-        # inscriptis respects CSS display rules, so strip hiding styles from the body tag.
-        body_tag = soup.find('body')
-        if body_tag and body_tag.get('style'):
-            style = body_tag['style']
-            if re.search(r'\b(?:display\s*:\s*none|visibility\s*:\s*hidden)\b', style, re.IGNORECASE):
-                logger.debug(f"html_to_text: Removing hiding styles from body tag (found: '{style}')")
-                del body_tag['style']
-
-        html_content = str(soup)
 
     text_content = get_text(html_content, config=parser_config)
     return text_content

changedetectionio/jinja2_custom/safe_jinja.py

@@ -52,13 +52,7 @@ def render(template_str, **args: t.Any) -> str:
     return output[:JINJA2_MAX_RETURN_PAYLOAD_SIZE]
 
 def render_fully_escaped(content):
-    """
-    Escape HTML content safely.
-
-    MEMORY LEAK FIX: Use markupsafe.escape() directly instead of creating
-    Jinja2 environments (was causing 1M+ compilations per page load).
-    Simpler, faster, and no concerns about environment state.
-    """
-    from markupsafe import escape
-    return str(escape(content))
+    env = jinja2.sandbox.ImmutableSandboxedEnvironment(autoescape=True)
+    template = env.from_string("{{ some_html|e }}")
+    return template.render(some_html=content)
 

changedetectionio/model/App.py

@@ -2,7 +2,6 @@ from os import getenv
 from copy import deepcopy
 
 from changedetectionio.blueprint.rss import RSS_FORMAT_TYPES, RSS_CONTENT_FORMAT_DEFAULT
-from changedetectionio.model.Tags import TagsDict
 
 from changedetectionio.notification import (
     default_notification_body,
@@ -30,7 +29,7 @@ class model(dict):
                     'proxy': None, # Preferred proxy connection
                     'time_between_check': {'weeks': None, 'days': None, 'hours': 3, 'minutes': None, 'seconds': None},
                     'timeout': int(getenv("DEFAULT_SETTINGS_REQUESTS_TIMEOUT", "45")),  # Default 45 seconds
-                    'workers': int(getenv("DEFAULT_SETTINGS_REQUESTS_WORKERS", "5")),  # Number of threads, lower is better for slow connections
+                    'workers': int(getenv("DEFAULT_SETTINGS_REQUESTS_WORKERS", "10")),  # Number of threads, lower is better for slow connections
                     'default_ua': {
                         'html_requests': getenv("DEFAULT_SETTINGS_HEADERS_USERAGENT", DEFAULT_SETTINGS_HEADERS_USERAGENT),
                         'html_webdriver': None,
@@ -38,8 +37,6 @@ class model(dict):
                 },
                 'application': {
                     # Custom notification content
-                    'all_paused': False,
-                    'all_muted': False,
                     'api_access_token_enabled': True,
                     'base_url' : None,
                     'empty_pages_are_a_change': False,
@@ -47,7 +44,6 @@ class model(dict):
                     'filter_failure_notification_threshold_attempts': _FILTER_FAILURE_THRESHOLD_ATTEMPTS_DEFAULT,
                     'global_ignore_text': [], # List of text to ignore when calculating the comparison checksum
                     'global_subtractive_selectors': [],
-                    'history_snapshot_max_length': None,
                     'ignore_whitespace': True,
                     'ignore_status_codes': False, #@todo implement, as ternary.
                     'ssim_threshold': '0.96',  # Default SSIM threshold for screenshot comparison
@@ -69,7 +65,7 @@ class model(dict):
                     'schema_version' : 0,
                     'shared_diff_access': False,
                     'strip_ignored_lines': False,
-                    'tags': None,  # Initialized in __init__ with real datastore_path
+                    'tags': {}, #@todo use Tag.model initialisers
                     'webdriver_delay': None , # Extra delay in seconds before extracting text
                     'ui': {
                         'use_page_title_in_list': True,
@@ -81,16 +77,10 @@ class model(dict):
             }
         }
 
-    def __init__(self, *arg, datastore_path=None, **kw):
+    def __init__(self, *arg, **kw):
         super(model, self).__init__(*arg, **kw)
-        # Capture any tags data passed in before base_config overwrites the structure
-        existing_tags = self.get('settings', {}).get('application', {}).get('tags') or {}
         # CRITICAL: deepcopy to avoid sharing mutable objects between instances
         self.update(deepcopy(self.base_config))
-        # TagsDict requires the real datastore_path at runtime (cannot be set at class-definition time)
-        if datastore_path is None:
-            raise ValueError("App.model() requires 'datastore_path' keyword argument")
-        self['settings']['application']['tags'] = TagsDict(existing_tags, datastore_path=datastore_path)
 
 
 def parse_headers_from_text_file(filepath):

changedetectionio/model/Tag.py

@@ -1,48 +1,10 @@
-"""
-Tag/Group domain model for organizing and overriding watch settings.
-
-ARCHITECTURE NOTE: Configuration Override Hierarchy
-===================================================
-
-Tags can override Watch settings when overrides_watch=True.
-Current implementation requires manual checking in processors:
-
-    for tag_uuid in watch.get('tags'):
-        tag = datastore['settings']['application']['tags'][tag_uuid]
-        if tag.get('overrides_watch'):
-            restock_settings = tag.get('restock_settings', {})
-            break
-
-With Pydantic, this would be automatic via chain resolution:
-    Watch → Tag (first with overrides_watch) → Global
-
-See: Watch.py model docstring for full Pydantic architecture explanation
-See: processors/restock_diff/processor.py:184-192 for current manual implementation
-"""
 
 from changedetectionio.model import watch_base
-from changedetectionio.model.persistence import EntityPersistenceMixin
 
-class model(EntityPersistenceMixin, watch_base):
-    """
-    Tag domain model - groups watches and can override their settings.
 
-    Tags inherit from watch_base to reuse all the same fields as Watch.
-    When overrides_watch=True, tag settings take precedence over watch settings
-    for all watches in this tag/group.
-
-    Fields:
-        overrides_watch (bool): If True, this tag's settings override watch settings
-        title (str): Display name for this tag/group
-        uuid (str): Unique identifier
-        ... (all fields from watch_base can be set as tag-level overrides)
-
-    Resolution order when overrides_watch=True:
-        Watch.field → Tag.field (if overrides_watch) → Global.field
-    """
+class model(watch_base):
 
     def __init__(self, *arg, **kw):
-        # Parent class (watch_base) handles __datastore and __datastore_path
         super(model, self).__init__(*arg, **kw)
 
         self['overrides_watch'] = kw.get('default', {}).get('overrides_watch')
@@ -50,7 +12,3 @@ class model(EntityPersistenceMixin, watch_base):
         if kw.get('default'):
             self.update(kw['default'])
             del kw['default']
-
-    # _save_to_disk() method provided by EntityPersistenceMixin
-    # commit() and _get_commit_data() methods inherited from watch_base
-    # Tag uses default _get_commit_data() (includes all keys)

changedetectionio/model/Tags.py

@@ -1,39 +0,0 @@
-import os
-import shutil
-from pathlib import Path
-from loguru import logger
-
-_SENTINEL = object()
-
-
-class TagsDict(dict):
-    """Dict subclass that removes the corresponding tag.json file when a tag is deleted."""
-
-    def __init__(self, *args, datastore_path: str | os.PathLike, **kwargs) -> None:
-        self._datastore_path = Path(datastore_path)
-        super().__init__(*args, **kwargs)
-
-    def __delitem__(self, key: str) -> None:
-        super().__delitem__(key)
-        tag_dir = self._datastore_path / key
-        tag_json_file = tag_dir / "tag.json"
-        if not os.path.exists(tag_json_file):
-            logger.critical(f"Aborting deletion of directory '{tag_dir}' because '{tag_json_file}' does not exist.")
-            return
-        try:
-            shutil.rmtree(tag_dir)
-            logger.info(f"Deleted tag directory for tag {key!r}")
-        except FileNotFoundError:
-            pass
-        except OSError as e:
-            logger.error(f"Failed to delete tag directory for tag {key!r}: {e}")
-
-    def pop(self, key: str, default=_SENTINEL):
-        """Remove and return tag, deleting its tag.json file. Raises KeyError if missing and no default given."""
-        if key in self:
-            value = self[key]
-            del self[key]
-            return value
-        if default is _SENTINEL:
-            raise KeyError(key)
-        return default

changedetectionio/model/Watch.py

@@ -1,37 +1,9 @@
-"""
-Watch domain model for change detection monitoring.
-
-ARCHITECTURE NOTE: Configuration Override Hierarchy
-===================================================
-
-This module implements Watch objects that inherit from dict (technical debt).
-The dream architecture would use Pydantic for:
-
-1. CHAIN RESOLUTION (Watch → Tag → Global Settings)
-   - Current: Manual resolution scattered across codebase
-   - Future: @computed_field properties with automatic resolution
-   - Examples: resolved_fetch_backend, resolved_restock_settings, etc.
-
-2. DATABASE BACKEND ABSTRACTION
-   - Current: Domain model tightly coupled to file-based JSON storage
-   - Future: Domain model (Pydantic) separate from persistence layer
-   - Enables: Easy migration to PostgreSQL, MongoDB, etc.
-
-3. TYPE SAFETY & VALIDATION
-   - Current: Dict access with no compile-time checks
-   - Future: Type hints, IDE autocomplete, validation at boundaries
-
-See class model docstring for detailed explanation and examples.
-See: processors/restock_diff/processor.py:184-192 for manual resolution example
-"""
-
 from blinker import signal
 from changedetectionio.validate_url import is_safe_valid_url
 
 from changedetectionio.strtobool import strtobool
 from changedetectionio.jinja2_custom import render as jinja_render
 from . import watch_base
-from .persistence import EntityPersistenceMixin
 import os
 import re
 from pathlib import Path
@@ -41,16 +13,15 @@ from .. import jinja2_custom as safe_jinja
 from ..html_tools import TRANSLATE_WHITESPACE_TABLE
 
 FAVICON_RESAVE_THRESHOLD_SECONDS=86400
-BROTLI_COMPRESS_SIZE_THRESHOLD = int(os.getenv('SNAPSHOT_BROTLI_COMPRESSION_THRESHOLD', 1024*20))
+BROTLI_COMPRESS_SIZE_THRESHOLD = int(os.getenv('SNAPSHOT_BROTLI_COMPRESSION_THRESHOLD', 1024))
 
 minimum_seconds_recheck_time = int(os.getenv('MINIMUM_SECONDS_RECHECK_TIME', 3))
 mtable = {'seconds': 1, 'minutes': 60, 'hours': 3600, 'days': 86400, 'weeks': 86400 * 7}
 
 def _brotli_save(contents, filepath, mode=None, fallback_uncompressed=False):
     """
-    Save compressed data using native brotli with streaming compression.
-    Uses chunked compression to minimize peak memory usage and malloc_trim()
-    to force release of C-level memory back to the OS.
+    Save compressed data using native brotli.
+    Testing shows no memory leak when using gc.collect() after compression.
 
     Args:
         contents: data to compress (str or bytes)
@@ -66,52 +37,27 @@ def _brotli_save(contents, filepath, mode=None, fallback_uncompressed=False):
     """
     import brotli
     import gc
-    import ctypes
 
     # Ensure contents are bytes
     if isinstance(contents, str):
         contents = contents.encode('utf-8')
 
     try:
-        original_size = len(contents)
-        logger.debug(f"Starting brotli streaming compression of {original_size} bytes.")
+        logger.debug(f"Starting brotli compression of {len(contents)} bytes.")
 
-        # Create streaming compressor
-        compressor = brotli.Compressor(quality=6, mode=mode if mode is not None else brotli.MODE_GENERIC)
-
-        # Stream compress in chunks to minimize memory usage
-        chunk_size = 65536  # 64KB chunks
-        total_compressed_size = 0
+        if mode is not None:
+            compressed_data = brotli.compress(contents, mode=mode)
+        else:
+            compressed_data = brotli.compress(contents)
 
         with open(filepath, 'wb') as f:
-            # Process data in chunks
-            offset = 0
-            while offset < len(contents):
-                chunk = contents[offset:offset + chunk_size]
-                compressed_chunk = compressor.process(chunk)
-                if compressed_chunk:
-                    f.write(compressed_chunk)
-                    total_compressed_size += len(compressed_chunk)
-                offset += chunk_size
+            f.write(compressed_data)
 
-            # Finalize compression - critical for proper cleanup
-            final_chunk = compressor.finish()
-            if final_chunk:
-                f.write(final_chunk)
-                total_compressed_size += len(final_chunk)
+        logger.debug(f"Finished brotli compression - From {len(contents)} to {len(compressed_data)} bytes.")
 
-        logger.debug(f"Finished brotli compression - From {original_size} to {total_compressed_size} bytes.")
-
-        # Cleanup: Delete compressor, force Python GC, then force C-level memory release
-        del compressor
+        # Force garbage collection to prevent memory buildup
         gc.collect()
 
-        # Force release of C-level memory back to OS (since brotli is a C library)
-        try:
-            ctypes.CDLL('libc.so.6').malloc_trim(0)
-        except Exception:
-            pass  # malloc_trim not available on all systems (e.g., macOS)
-
         return filepath
 
     except Exception as e:
@@ -128,112 +74,17 @@ def _brotli_save(contents, filepath, mode=None, fallback_uncompressed=False):
             raise Exception(f"Brotli compression failed for {filepath}: {e}")
 
 
-class model(EntityPersistenceMixin, watch_base):
-    """
-    Watch domain model for monitoring URL changes.
-
-    Inherits from watch_base (which inherits dict) - see watch_base docstring for field documentation.
-
-    ## Configuration Override Hierarchy (Chain Resolution)
-
-    The dream architecture uses a 3-level resolution chain:
-        Watch settings → Tag/Group settings → Global settings
-
-    Current implementation is MANUAL (see processor.py:184-192 for example):
-        - Processors manually check watch.get('field')
-        - Then loop through watch.tags to find first tag with overrides_watch=True
-        - Finally fall back to datastore['settings']['application']['field']
-
-    FUTURE: Pydantic-based chain resolution would enable:
-
-        ```python
-        # Instead of manual resolution in every processor:
-        restock_settings = watch.get('restock_settings', {})
-        for tag_uuid in watch.get('tags'):
-            tag = datastore['settings']['application']['tags'][tag_uuid]
-            if tag.get('overrides_watch'):
-                restock_settings = tag.get('restock_settings', {})
-                break
-
-        # Clean computed properties with automatic resolution:
-        @computed_field
-        def resolved_restock_settings(self) -> dict:
-            if self.restock_settings:
-                return self.restock_settings
-            for tag_uuid in self.tags:
-                tag = self._datastore.get_tag(tag_uuid)
-                if tag.overrides_watch and tag.restock_settings:
-                    return tag.restock_settings
-            return self._datastore.settings.restock_settings or {}
-
-        # Usage: watch.resolved_restock_settings (automatic, type-safe, tested once)
-        ```
-
-    Benefits of Pydantic migration:
-        1. Single source of truth for resolution logic (not scattered across processors)
-        2. Type safety + IDE autocomplete (watch.resolved_fetch_backend vs dict navigation)
-        3. Database backend abstraction (domain model separate from persistence)
-        4. Automatic validation at boundaries
-        5. Self-documenting via type hints
-        6. Easy to test resolution independently
-
-    Resolution chain examples that would benefit:
-        - fetch_backend: watch → tag → global (see get_fetch_backend property)
-        - notification_urls: watch → tag → global
-        - time_between_check: watch → global (see threshold_seconds)
-        - restock_settings: watch → tag (see processors/restock_diff/processor.py:184-192)
-        - history_snapshot_max_length: watch → global (see save_history_blob:550-556)
-        - All processor_config_* settings could use tag overrides
-
-    ## Database Backend Abstraction with Pydantic
-
-    Current: Watch inherits dict, tightly coupled to file-based JSON storage
-    Future: Domain model (Watch) separate from persistence layer
-
-        ```python
-        # Domain model (database-agnostic)
-        class Watch(BaseModel):
-            uuid: str
-            url: str
-            # ... validation, business logic
-
-        # Pluggable backends
-        class DataStoreBackend(ABC):
-            def save_watch(self, watch: Watch): ...
-            def load_watch(self, uuid: str) -> Watch: ...
-
-        # Implementations: FileBackend, MongoBackend, PostgresBackend, etc.
-        ```
-
-    This would enable:
-        - Easy migration between storage backends (file → postgres → mongodb)
-        - Pydantic handles serialization/deserialization automatically
-        - Domain logic stays clean (no storage concerns in Watch methods)
-
-    ## Migration Path
-
-    Given existing codebase, incremental migration recommended:
-        1. Create Pydantic models alongside existing dict-based models
-        2. Add .to_pydantic() / .from_pydantic() bridge methods
-        3. Gradually migrate code to use Pydantic models
-        4. Remove dict inheritance once migration complete
-
-    See: watch_base docstring for technical debt discussion
-    See: processors/restock_diff/processor.py:184-192 for manual resolution example
-    See: Watch.py:550-556 for nested dict navigation that would become watch.resolved_*
-    """
+class model(watch_base):
     __newest_history_key = None
     __history_n = 0
     jitter_seconds = 0
 
     def __init__(self, *arg, **kw):
-        # Validate __datastore before calling parent (Watch requires it)
-        if not kw.get('__datastore'):
-            raise ValueError("Watch object requires '__datastore' reference - cannot access global settings without it")
-
-        # Parent class (watch_base) handles __datastore and __datastore_path
+        self.__datastore_path = kw.get('datastore_path')
+        if kw.get('datastore_path'):
+            del kw['datastore_path']
+            
         super(model, self).__init__(*arg, **kw)
-
         if kw.get('default'):
             self.update(kw['default'])
             del kw['default']
@@ -244,9 +95,6 @@ class model(EntityPersistenceMixin, watch_base):
         # Be sure the cached timestamp is ready
         bump = self.history
 
-    # Note: __deepcopy__, __getstate__, and __setstate__ are inherited from watch_base
-    # This prevents memory leaks by sharing __datastore reference instead of copying it
-
     @property
     def viewed(self):
         # Don't return viewed when last_viewed is 0 and newest_key is 0
@@ -259,6 +107,11 @@ class model(EntityPersistenceMixin, watch_base):
     def has_unviewed(self):
         return int(self.newest_history_key) > int(self['last_viewed']) and self.__history_n >= 2
 
+    def ensure_data_dir_exists(self):
+        if not os.path.isdir(self.watch_data_dir):
+            logger.debug(f"> Creating data dir {self.watch_data_dir}")
+            os.mkdir(self.watch_data_dir)
+
     @property
     def link(self):
 
@@ -314,8 +167,7 @@ class model(EntityPersistenceMixin, watch_base):
 
         # JSON Data, Screenshots, Textfiles (history index and snapshots), HTML in the future etc
         # But preserve processor config files (they're configuration, not history data)
-        # Use glob not rglob here for safety.
-        for item in pathlib.Path(str(self.data_dir)).glob("*.*"):
+        for item in pathlib.Path(str(self.watch_data_dir)).rglob("*.*"):
             # Skip processor config files
             if item.name in processor_config_files:
                 continue
@@ -335,6 +187,7 @@ class model(EntityPersistenceMixin, watch_base):
             'last_notification_error': False,
             'last_viewed': 0,
             'previous_md5': False,
+            'previous_md5_before_filters': False,
             'remote_server_reply': None,
             'track_ldjson_price_data': None
         })
@@ -351,30 +204,8 @@ class model(EntityPersistenceMixin, watch_base):
     @property
     def get_fetch_backend(self):
         """
-        Get the fetch backend for this watch with special case handling.
-
-        CHAIN RESOLUTION OPPORTUNITY:
-        Currently returns watch.fetch_backend directly, but doesn't implement
-        Watch → Tag → Global resolution chain. With Pydantic:
-
-        @computed_field
-        def resolved_fetch_backend(self) -> str:
-            # Special case: PDFs always use html_requests
-            if self.is_pdf:
-                return 'html_requests'
-
-            # Watch override
-            if self.fetch_backend and self.fetch_backend != 'system':
-                return self.fetch_backend
-
-            # Tag override (first tag with overrides_watch=True wins)
-            for tag_uuid in self.tags:
-                tag = self._datastore.get_tag(tag_uuid)
-                if tag.overrides_watch and tag.fetch_backend:
-                    return tag.fetch_backend
-
-            # Global default
-            return self._datastore.settings.fetch_backend
+        Like just using the `fetch_backend` key but there could be some logic
+        :return:
         """
         # Maybe also if is_image etc?
         # This is because chrome/playwright wont render the PDF in the browser and we will just fetch it and use pdf2html to see the text.
@@ -385,16 +216,10 @@ class model(EntityPersistenceMixin, watch_base):
 
     @property
     def is_pdf(self):
-        url = str(self.get("url") or "").lower()
-        content_type = str(self.get("content-type") or "").lower()
-
-        if content_type in ("none", "null", ""):
-            content_type = ""
-
-        return (
-                url.endswith(".pdf")
-                or content_type.split(";")[0].strip() == "application/pdf"
-        )
+        # content_type field is set in the future
+        # https://github.com/dgtlmoon/changedetection.io/issues/1392
+        # Not sure the best logic here
+        return self.get('url', '').lower().endswith('.pdf') or 'pdf' in self.get('content_type', '').lower()
 
     @property
     def label(self):
@@ -429,11 +254,11 @@ class model(EntityPersistenceMixin, watch_base):
         tmp_history = {}
 
         # In the case we are only using the watch for processing without history
-        if not self.data_dir:
+        if not self.watch_data_dir:
             return []
 
         # Read the history file as a dict
-        fname = os.path.join(self.data_dir, self.history_index_filename)
+        fname = os.path.join(self.watch_data_dir, self.history_index_filename)
         if os.path.isfile(fname):
             logger.debug(f"Reading watch history index for {self.get('uuid')}")
             with open(fname, "r", encoding='utf-8') as f:
@@ -446,13 +271,13 @@ class model(EntityPersistenceMixin, watch_base):
                         # Cross-platform: check for any path separator (works on Windows and Unix)
                         if os.sep not in v and '/' not in v and '\\' not in v:
                             # Relative filename only, no path separators
-                            v = os.path.join(self.data_dir, v)
+                            v = os.path.join(self.watch_data_dir, v)
                         else:
                             # It's possible that they moved the datadir on older versions
                             # So the snapshot exists but is in a different path
                             # Cross-platform: use os.path.basename instead of split('/')
                             snapshot_fname = os.path.basename(v)
-                            proposed_new_path = os.path.join(self.data_dir, snapshot_fname)
+                            proposed_new_path = os.path.join(self.watch_data_dir, snapshot_fname)
                             if not os.path.exists(v) and os.path.exists(proposed_new_path):
                                 v = proposed_new_path
 
@@ -469,7 +294,7 @@ class model(EntityPersistenceMixin, watch_base):
 
     @property
     def has_history(self):
-        fname = os.path.join(self.data_dir, self.history_index_filename)
+        fname = os.path.join(self.watch_data_dir, self.history_index_filename)
         return os.path.isfile(fname)
 
     @property
@@ -572,49 +397,16 @@ class model(EntityPersistenceMixin, watch_base):
         with open(filepath, 'r', encoding='utf-8', errors='ignore') as f:
             return f.read()
 
-    def _write_atomic(self, dest, data, mode='wb'):
+    def _write_atomic(self, dest, data):
         """Write data atomically to dest using a temp file"""
-        import tempfile
-        with tempfile.NamedTemporaryFile(mode, delete=False, dir=self.data_dir) as tmp:
-            tmp.write(data)
-            tmp.flush()
-            os.fsync(tmp.fileno())
-            tmp_path = tmp.name
-        os.replace(tmp_path, dest)
-
-    def history_trim(self, newest_n_items):
-        from pathlib import Path
-        import gc
-        # Sort by timestamp (key)
-        sorted_items = sorted(self.history.items(), key=lambda x: int(x[0]))
-
-        keep_part = dict(sorted_items[-newest_n_items:])
-        delete_part = dict(sorted_items[:-newest_n_items])
-        logger.info( f"[{self.get('uuid')}] Trimming history to most recent {newest_n_items} items, keeping {len(keep_part)} items deleting {len(delete_part)} items.")
-
-        if delete_part:
-            for item in delete_part.items():
-                try:
-                    Path(item[1]).unlink(missing_ok=True)
-                except Exception as e:
-                    logger.critical(f"{str(e)}")
-                finally:
-                    logger.debug(f"[{self.get('uuid')}] Deleted {item[1]} history snapshot")
-        try:
-            dest = os.path.join(self.data_dir, self.history_index_filename)
-            output = "\r\n".join(
-                f"{k},{Path(v).name}"
-                for k, v in keep_part.items()
-            )+"\r\n"
-            self._write_atomic(dest=dest, data=output, mode='w')
-        except Exception as e:
-            logger.critical(f"{str(e)}")
-        finally:
-            logger.debug(f"[{self.get('uuid')}] Updated history index {dest}")
-
-        # reimport
-        bump = self.history
-        gc.collect()
+        if not os.path.exists(dest):
+            import tempfile
+            with tempfile.NamedTemporaryFile('wb', delete=False, dir=self.watch_data_dir) as tmp:
+                tmp.write(data)
+                tmp.flush()
+                os.fsync(tmp.fileno())
+                tmp_path = tmp.name
+            os.replace(tmp_path, dest)
 
     # Save some text file to the appropriate path and bump the history
     # result_obj from fetch_site_status.run()
@@ -623,6 +415,7 @@ class model(EntityPersistenceMixin, watch_base):
         logger.trace(f"{self.get('uuid')} - Updating {self.history_index_filename} with timestamp {timestamp}")
 
         self.ensure_data_dir_exists()
+
         skip_brotli = strtobool(os.getenv('DISABLE_BROTLI_TEXT_SNAPSHOT', 'False'))
 
         # Binary data - detect file type and save without compression
@@ -640,7 +433,7 @@ class model(EntityPersistenceMixin, watch_base):
                 ext = 'bin'
 
             snapshot_fname = f"{snapshot_id}.{ext}"
-            dest = os.path.join(self.data_dir, snapshot_fname)
+            dest = os.path.join(self.watch_data_dir, snapshot_fname)
             self._write_atomic(dest, contents)
             logger.trace(f"Saved binary snapshot as {snapshot_fname} ({len(contents)} bytes)")
 
@@ -650,7 +443,7 @@ class model(EntityPersistenceMixin, watch_base):
                 # Compressed text
                 import brotli
                 snapshot_fname = f"{snapshot_id}.txt.br"
-                dest = os.path.join(self.data_dir, snapshot_fname)
+                dest = os.path.join(self.watch_data_dir, snapshot_fname)
 
                 if not os.path.exists(dest):
                     try:
@@ -661,16 +454,16 @@ class model(EntityPersistenceMixin, watch_base):
                         logger.error(f"{self.get('uuid')} - Brotli compression failed: {e}")
                         # Fallback to uncompressed
                         snapshot_fname = f"{snapshot_id}.txt"
-                        dest = os.path.join(self.data_dir, snapshot_fname)
+                        dest = os.path.join(self.watch_data_dir, snapshot_fname)
                         self._write_atomic(dest, contents.encode('utf-8'))
             else:
                 # Plain text
                 snapshot_fname = f"{snapshot_id}.txt"
-                dest = os.path.join(self.data_dir, snapshot_fname)
+                dest = os.path.join(self.watch_data_dir, snapshot_fname)
                 self._write_atomic(dest, contents.encode('utf-8'))
 
         # Append to history.txt atomically
-        index_fname = os.path.join(self.data_dir, self.history_index_filename)
+        index_fname = os.path.join(self.watch_data_dir, self.history_index_filename)
         index_line = f"{timestamp},{snapshot_fname}\n"
 
         with open(index_fname, 'a', encoding='utf-8') as f:
@@ -682,17 +475,6 @@ class model(EntityPersistenceMixin, watch_base):
         self.__newest_history_key = timestamp
         self.__history_n += 1
 
-        # MANUAL CHAIN RESOLUTION: Watch → Global
-        # With Pydantic, this would become: maxlen = watch.resolved_history_snapshot_max_length
-        # @computed_field def resolved_history_snapshot_max_length(self) -> Optional[int]:
-        #     if self.history_snapshot_max_length: return self.history_snapshot_max_length
-        #     if tag := self._get_override_tag(): return tag.history_snapshot_max_length
-        #     return self._datastore.settings.history_snapshot_max_length
-        maxlen = self.get('history_snapshot_max_length') or self.get_global_setting('application', 'history_snapshot_max_length')
-
-        if maxlen and self.__history_n and self.__history_n > maxlen:
-            self.history_trim(newest_n_items=maxlen)
-
         # @todo bump static cache of the last timestamp so we dont need to examine the file to set a proper ''viewed'' status
         return snapshot_fname
 
@@ -745,7 +527,7 @@ class model(EntityPersistenceMixin, watch_base):
         return not local_lines.issubset(existing_history)
 
     def get_screenshot(self):
-        fname = os.path.join(self.data_dir, "last-screenshot.png")
+        fname = os.path.join(self.watch_data_dir, "last-screenshot.png")
         if os.path.isfile(fname):
             return fname
 
@@ -760,7 +542,7 @@ class model(EntityPersistenceMixin, watch_base):
         if not favicon_fname:
             return True
         try:
-            fname = next(iter(glob.glob(os.path.join(self.data_dir, "favicon.*"))), None)
+            fname = next(iter(glob.glob(os.path.join(self.watch_data_dir, "favicon.*"))), None)
             logger.trace(f"Favicon file maybe found at {fname}")
             if os.path.isfile(fname):
                 file_age = int(time.time() - os.path.getmtime(fname))
@@ -793,7 +575,7 @@ class model(EntityPersistenceMixin, watch_base):
             base = "favicon"
             extension = "ico"
 
-        fname = os.path.join(self.data_dir, f"favicon.{extension}")
+        fname = os.path.join(self.watch_data_dir, f"favicon.{extension}")
 
         try:
             # validate=True makes sure the string only contains valid base64 chars
@@ -805,11 +587,6 @@ class model(EntityPersistenceMixin, watch_base):
                 try:
                     with open(fname, 'wb') as f:
                         f.write(decoded)
-
-                    # Invalidate favicon filename cache
-                    if hasattr(self, '_favicon_filename_cache'):
-                        delattr(self, '_favicon_filename_cache')
-
                     # A signal that could trigger the socket server to update the browser also
                     watch_check_update = signal('watch_favicon_bump')
                     if watch_check_update:
@@ -826,32 +603,20 @@ class model(EntityPersistenceMixin, watch_base):
         Find any favicon.* file in the current working directory
         and return the contents of the newest one.
 
-        MEMORY LEAK FIX: Cache the result to avoid repeated glob.glob() operations.
-        glob.glob() causes millions of fnmatch allocations when called for every watch on page load.
-
         Returns:
-            str: Basename of the newest favicon file, or None if not found.
+            bytes: Contents of the newest favicon file, or None if not found.
         """
-        # Check cache first (prevents 26M+ allocations from repeated glob operations)
-        cache_key = '_favicon_filename_cache'
-        if hasattr(self, cache_key):
-            return getattr(self, cache_key)
-
         import glob
 
         # Search for all favicon.* files
-        files = glob.glob(os.path.join(self.data_dir, "favicon.*"))
+        files = glob.glob(os.path.join(self.watch_data_dir, "favicon.*"))
 
         if not files:
-            result = None
-        else:
-            # Find the newest by modification time
-            newest_file = max(files, key=os.path.getmtime)
-            result = os.path.basename(newest_file)
+            return None
 
-        # Cache the result
-        setattr(self, cache_key, result)
-        return result
+        # Find the newest by modification time
+        newest_file = max(files, key=os.path.getmtime)
+        return os.path.basename(newest_file)
 
     def get_screenshot_as_thumbnail(self, max_age=3200):
         """Return path to a square thumbnail of the most recent screenshot.
@@ -867,7 +632,7 @@ class model(EntityPersistenceMixin, watch_base):
         import os
         import time
 
-        thumbnail_path = os.path.join(self.data_dir, "thumbnail.jpeg")
+        thumbnail_path = os.path.join(self.watch_data_dir, "thumbnail.jpeg")
         top_trim = 500  # Pixels from top of screenshot to use
 
         screenshot_path = self.get_screenshot()
@@ -918,7 +683,7 @@ class model(EntityPersistenceMixin, watch_base):
             return None
 
     def __get_file_ctime(self, filename):
-        fname = os.path.join(self.data_dir, filename)
+        fname = os.path.join(self.watch_data_dir, filename)
         if os.path.isfile(fname):
             return int(os.path.getmtime(fname))
         return False
@@ -943,9 +708,14 @@ class model(EntityPersistenceMixin, watch_base):
     def snapshot_error_screenshot_ctime(self):
         return self.__get_file_ctime('last-error-screenshot.png')
 
+    @property
+    def watch_data_dir(self):
+        # The base dir of the watch data
+        return os.path.join(self.__datastore_path, self['uuid']) if self.__datastore_path else None
+
     def get_error_text(self):
         """Return the text saved from a previous request that resulted in a non-200 error"""
-        fname = os.path.join(self.data_dir, "last-error.txt")
+        fname = os.path.join(self.watch_data_dir, "last-error.txt")
         if os.path.isfile(fname):
             with open(fname, 'r', encoding='utf-8') as f:
                 return f.read()
@@ -953,7 +723,7 @@ class model(EntityPersistenceMixin, watch_base):
 
     def get_error_snapshot(self):
         """Return path to the screenshot that resulted in a non-200 error"""
-        fname = os.path.join(self.data_dir, "last-error-screenshot.png")
+        fname = os.path.join(self.watch_data_dir, "last-error-screenshot.png")
         if os.path.isfile(fname):
             return fname
         return False
@@ -977,37 +747,6 @@ class model(EntityPersistenceMixin, watch_base):
     def toggle_mute(self):
         self['notification_muted'] ^= True
 
-    def _get_commit_data(self):
-        """
-        Prepare watch data for commit.
-
-        Excludes processor_config_* keys (stored in separate files).
-        Normalizes browser_steps to empty list if no meaningful steps.
-        """
-        import copy
-
-        # Get base snapshot with lock
-        lock = self._datastore.lock if self._datastore and hasattr(self._datastore, 'lock') else None
-
-        if lock:
-            with lock:
-                snapshot = dict(self)
-        else:
-            snapshot = dict(self)
-
-        # Exclude processor config keys (stored separately)
-        watch_dict = {k: copy.deepcopy(v) for k, v in snapshot.items() if not k.startswith('processor_config_')}
-
-        # Normalize browser_steps: if no meaningful steps, save as empty list
-        if not self.has_browser_steps:
-            watch_dict['browser_steps'] = []
-
-        return watch_dict
-
-    # _save_to_disk() method provided by EntityPersistenceMixin
-    # commit() method inherited from watch_base
-
-
     def extra_notification_token_values(self):
         # Used for providing extra tokens
         # return {'widget': 555}
@@ -1037,7 +776,7 @@ class model(EntityPersistenceMixin, watch_base):
                         if not csv_writer:
                             # A file on the disk can be transferred much faster via flask than a string reply
                             csv_output_filename = f"report-{self.get('uuid')}.csv"
-                            f = open(os.path.join(self.data_dir, csv_output_filename), 'w')
+                            f = open(os.path.join(self.watch_data_dir, csv_output_filename), 'w')
                             # @todo some headers in the future
                             #fieldnames = ['Epoch seconds', 'Date']
                             csv_writer = csv.writer(f,
@@ -1079,7 +818,7 @@ class model(EntityPersistenceMixin, watch_base):
 
     def save_error_text(self, contents):
         self.ensure_data_dir_exists()
-        target_path = os.path.join(self.data_dir, "last-error.txt")
+        target_path = os.path.join(self.watch_data_dir, "last-error.txt")
         with open(target_path, 'w', encoding='utf-8') as f:
             f.write(contents)
 
@@ -1088,9 +827,9 @@ class model(EntityPersistenceMixin, watch_base):
         import zlib
 
         if as_error:
-            target_path = os.path.join(str(self.data_dir), "elements-error.deflate")
+            target_path = os.path.join(str(self.watch_data_dir), "elements-error.deflate")
         else:
-            target_path = os.path.join(str(self.data_dir), "elements.deflate")
+            target_path = os.path.join(str(self.watch_data_dir), "elements.deflate")
 
         self.ensure_data_dir_exists()
 
@@ -1105,9 +844,9 @@ class model(EntityPersistenceMixin, watch_base):
     def save_screenshot(self, screenshot: bytes, as_error=False):
 
         if as_error:
-            target_path = os.path.join(self.data_dir, "last-error-screenshot.png")
+            target_path = os.path.join(self.watch_data_dir, "last-error-screenshot.png")
         else:
-            target_path = os.path.join(self.data_dir, "last-screenshot.png")
+            target_path = os.path.join(self.watch_data_dir, "last-screenshot.png")
 
         self.ensure_data_dir_exists()
 
@@ -1118,7 +857,7 @@ class model(EntityPersistenceMixin, watch_base):
 
     def get_last_fetched_text_before_filters(self):
         import brotli
-        filepath = os.path.join(self.data_dir, 'last-fetched.br')
+        filepath = os.path.join(self.watch_data_dir, 'last-fetched.br')
 
         if not os.path.isfile(filepath) or os.path.getsize(filepath) == 0:
             # If a previous attempt doesnt yet exist, just snarf the previous snapshot instead
@@ -1133,13 +872,13 @@ class model(EntityPersistenceMixin, watch_base):
 
     def save_last_text_fetched_before_filters(self, contents):
         import brotli
-        filepath = os.path.join(self.data_dir, 'last-fetched.br')
+        filepath = os.path.join(self.watch_data_dir, 'last-fetched.br')
         _brotli_save(contents, filepath, mode=brotli.MODE_TEXT, fallback_uncompressed=False)
 
     def save_last_fetched_html(self, timestamp, contents):
         self.ensure_data_dir_exists()
         snapshot_fname = f"{timestamp}.html.br"
-        filepath = os.path.join(self.data_dir, snapshot_fname)
+        filepath = os.path.join(self.watch_data_dir, snapshot_fname)
         _brotli_save(contents, filepath, mode=None, fallback_uncompressed=True)
         self._prune_last_fetched_html_snapshots()
 
@@ -1147,7 +886,7 @@ class model(EntityPersistenceMixin, watch_base):
         import brotli
 
         snapshot_fname = f"{timestamp}.html.br"
-        filepath = os.path.join(self.data_dir, snapshot_fname)
+        filepath = os.path.join(self.watch_data_dir, snapshot_fname)
         if os.path.isfile(filepath):
             with open(filepath, 'rb') as f:
                 return (brotli.decompress(f.read()).decode('utf-8'))
@@ -1162,7 +901,7 @@ class model(EntityPersistenceMixin, watch_base):
 
         for index, timestamp in enumerate(dates):
             snapshot_fname = f"{timestamp}.html.br"
-            filepath = os.path.join(self.data_dir, snapshot_fname)
+            filepath = os.path.join(self.watch_data_dir, snapshot_fname)
 
             # Keep only the first 2
             if index > 1 and os.path.isfile(filepath):
@@ -1173,7 +912,7 @@ class model(EntityPersistenceMixin, watch_base):
     def get_browsersteps_available_screenshots(self):
         "For knowing which screenshots are available to show the user in BrowserSteps UI"
         available = []
-        for f in Path(self.data_dir).glob('step_before-*.jpeg'):
+        for f in Path(self.watch_data_dir).glob('step_before-*.jpeg'):
             step_n=re.search(r'step_before-(\d+)', f.name)
             if step_n:
                 available.append(step_n.group(1))

changedetectionio/model/__init__.py

@@ -2,175 +2,12 @@ import os
 import uuid
 
 from changedetectionio import strtobool
-from .persistence import EntityPersistenceMixin, _determine_entity_type
-
-__all__ = ['EntityPersistenceMixin', 'watch_base']
-
-from ..browser_steps.browser_steps import browser_steps_get_valid_steps
-
 USE_SYSTEM_DEFAULT_NOTIFICATION_FORMAT_FOR_WATCH = 'System default'
 CONDITIONS_MATCH_LOGIC_DEFAULT = 'ALL'
 
-
 class watch_base(dict):
-    """
-    Base watch domain model (inherits from dict for backward compatibility).
-
-    WARNING: This class inherits from dict, which violates proper encapsulation.
-    Dict inheritance is legacy technical debt that should be refactored to a proper
-    domain model (e.g., Pydantic BaseModel) for better type safety and validation.
-
-    TODO: Migrate to Pydantic BaseModel for:
-          - Type safety and IDE autocomplete
-          - Automatic validation
-          - Clear separation between domain model and serialization
-          - Database backend abstraction (file → postgres → mongodb)
-          - Configuration override chain resolution (Watch → Tag → Global)
-          - Immutability options
-          - Better testing
-          - USE https://docs.pydantic.dev/latest/integrations/datamodel_code_generator TO BUILD THE MODEL FROM THE API-SPEC!!!
-
-    CHAIN RESOLUTION ARCHITECTURE:
-        The dream is a 3-level override hierarchy:
-            Watch settings → Tag/Group settings → Global settings
-
-        Current implementation: MANUAL resolution scattered across codebase
-        - Processors manually check watch.get('field')
-        - Loop through tags to find overrides_watch=True
-        - Fall back to datastore['settings']['application']['field']
-
-        Pydantic implementation: AUTOMATIC resolution via @computed_field
-        - Single source of truth for each setting's resolution logic
-        - Type-safe, testable, self-documenting
-        - Example: watch.resolved_fetch_backend (instead of nested dict navigation)
-
-        See: Watch.py model docstring for detailed Pydantic architecture plan
-        See: Tag.py model docstring for tag override explanation
-        See: processors/restock_diff/processor.py:184-192 for current manual example
-
-    Core Fields:
-        uuid (str): Unique identifier for this watch (auto-generated)
-        url (str): Target URL to monitor for changes
-        title (str|None): Custom display name (overrides page_title if set)
-        page_title (str|None): Title extracted from <title> tag of monitored page
-        tags (List[str]): List of tag UUIDs for categorization
-        tag (str): DEPRECATED - Old single-tag system, use tags instead
-
-    Check Configuration:
-        processor (str): Processor type ('text_json_diff', 'restock_diff', etc.)
-        fetch_backend (str): Fetcher to use ('system', 'html_requests', 'playwright', etc.)
-        method (str): HTTP method ('GET', 'POST', etc.)
-        headers (dict): Custom HTTP headers to send
-        proxy (str|None): Preferred proxy server
-        paused (bool): Whether change detection is paused
-
-    Scheduling:
-        time_between_check (dict): Check interval {'weeks': int, 'days': int, 'hours': int, 'minutes': int, 'seconds': int}
-        time_between_check_use_default (bool): Use global default interval if True
-        time_schedule_limit (dict): Weekly schedule limiting when checks can run
-            Structure: {
-                'enabled': bool,
-                'monday/tuesday/.../sunday': {
-                    'enabled': bool,
-                    'start_time': str ('HH:MM'),
-                    'duration': {'hours': str, 'minutes': str}
-                }
-            }
-
-    Content Filtering:
-        include_filters (List[str]): CSS/XPath selectors to extract content
-        subtractive_selectors (List[str]): Selectors to remove from content
-        ignore_text (List[str]): Text patterns to ignore in change detection
-        trigger_text (List[str]): Text/regex that must be present to trigger change
-        text_should_not_be_present (List[str]): Text that should NOT be present
-        extract_text (List[str]): Regex patterns to extract specific text after filtering
-
-    Text Processing:
-        trim_text_whitespace (bool): Strip leading/trailing whitespace
-        sort_text_alphabetically (bool): Sort lines alphabetically before comparison
-        remove_duplicate_lines (bool): Remove duplicate lines
-        check_unique_lines (bool): Compare against all history for unique lines
-        strip_ignored_lines (bool|None): Remove lines matching ignore patterns
-
-    Change Detection Filters:
-        filter_text_added (bool): Include added text in change detection
-        filter_text_removed (bool): Include removed text in change detection
-        filter_text_replaced (bool): Include replaced text in change detection
-
-    Browser Automation:
-        browser_steps (List[dict]): Browser automation steps for JS-heavy sites
-        browser_steps_last_error_step (int|None): Last step that caused error
-        webdriver_delay (int|None): Seconds to wait after page load
-        webdriver_js_execute_code (str|None): JavaScript to execute before extraction
-
-    Restock Detection:
-        in_stock_only (bool): Only trigger on in-stock transitions
-        follow_price_changes (bool): Monitor price changes
-        has_ldjson_price_data (bool|None): Whether page has LD-JSON price data
-        track_ldjson_price_data (str|None): Track LD-JSON price data ('ACCEPT', 'REJECT', None)
-        price_change_threshold_percent (float|None): Minimum price change % to trigger
-
-    Notifications:
-        notification_urls (List[str]): Apprise URLs for notifications
-        notification_title (str|None): Custom notification title template
-        notification_body (str|None): Custom notification body template
-        notification_format (str): Notification format (e.g., 'System default', 'Text', 'HTML')
-        notification_muted (bool): Disable notifications for this watch
-        notification_screenshot (bool): Include screenshot in notifications
-        notification_alert_count (int): Number of notifications sent
-        last_notification_error (str|None): Last notification error message
-        body (str|None): DEPRECATED? Legacy notification body field
-        filter_failure_notification_send (bool): Send notification on filter failures
-
-    History & State:
-        date_created (int|None): Unix timestamp of watch creation
-        last_checked (int): Unix timestamp of last check
-        last_viewed (int): History snapshot key of last user view
-        last_error (str|bool): Last error message or False if no error
-        check_count (int): Total number of checks performed
-        fetch_time (float): Duration of last fetch in seconds
-        consecutive_filter_failures (int): Counter for consecutive filter match failures
-        previous_md5 (str|bool): MD5 hash of previous content
-        history_snapshot_max_length (int|None): Max history snapshots to keep (None = use global)
-
-    Conditions:
-        conditions (dict): Custom conditions for change detection logic
-        conditions_match_logic (str): Logic operator ('ALL', 'ANY') for conditions
-
-    Metadata:
-        content-type (str|None): Content-Type from last fetch
-        remote_server_reply (str|None): Server header from last response
-        ignore_status_codes (List[int]|None): HTTP status codes to ignore
-        use_page_title_in_list (bool|None): Display page title in watch list (None = use system default)
-
-    Instance Attributes (not serialized):
-        __datastore: Reference to parent DataStore (set externally after creation)
-        data_dir: Filesystem path for this watch's data directory
-
-    Notes:
-        - Many fields default to None to distinguish "not set" from "set to default"
-        - When field is None, system-level defaults are used
-        - Processor-specific configs (e.g., processor_config_*) are NOT stored in watch.json
-          They are stored in separate {processor_name}.json files
-        - This class is used for both Watch and Tag objects (tags reuse the structure)
-    """
 
     def __init__(self, *arg, **kw):
-        # Store datastore reference (common to Watch and Tag)
-        # Use single underscore to avoid name mangling issues in subclasses
-        self._datastore = kw.get('__datastore')
-        if kw.get('__datastore'):
-            del kw['__datastore']
-
-        # Store datastore_path (common to Watch and Tag)
-        self._datastore_path = kw.get('datastore_path')
-        if kw.get('datastore_path'):
-            del kw['datastore_path']
-
-        # IMPORTANT: Don't initialize __watch_was_edited yet!
-        # We'll initialize it AFTER the initial update() call below
-        # This prevents marking the watch as edited during initialization
-
         self.update({
             # Custom notification content
             # Re #110, so then if this is set to None, we know to use the default value instead
@@ -179,7 +16,7 @@ class watch_base(dict):
             'body': None,
             'browser_steps': [],
             'browser_steps_last_error_step': None,
-            'conditions' : [],
+            'conditions' : {},
             'conditions_match_logic': CONDITIONS_MATCH_LOGIC_DEFAULT,
             'check_count': 0,
             'check_unique_lines': False,  # On change-detected, compare against all history if its something new
@@ -195,7 +32,6 @@ class watch_base(dict):
             'filter_text_replaced': True,
             'follow_price_changes': True,
             'has_ldjson_price_data': None,
-            'history_snapshot_max_length': None,
             'headers': {},  # Extra headers to send
             'ignore_text': [],  # List of text to ignore when calculating the comparison checksum
             'ignore_status_codes': None,
@@ -216,6 +52,7 @@ class watch_base(dict):
             'page_title': None, # <title> from the page
             'paused': False,
             'previous_md5': False,
+            'previous_md5_before_filters': False,  # Used for skipping changedetection entirely
             'processor': 'text_json_diff',  # could be restock_diff or others from .processors
             'price_change_threshold_percent': None,
             'proxy': None,  # Preferred proxy connection
@@ -301,372 +138,5 @@ class watch_base(dict):
 
         super(watch_base, self).__init__(*arg, **kw)
 
-        # Check if we're being initialized from an existing watch object
-        # that has was_edited=True, so we can preserve the flag
-        preserve_edited_flag = False
         if self.get('default'):
-            # When creating a new watch object from an existing one (e.g., changing processor),
-            # preserve the was_edited flag if it was True
-            default_watch = self.get('default')
-            if hasattr(default_watch, 'was_edited') and default_watch.was_edited:
-                preserve_edited_flag = True
-            del self['default']
-
-        # NOW initialize the edited flag after all initial setup is complete
-        # This ensures initialization doesn't trigger the edited flag
-        # But preserve it if the source watch had it set to True
-        self.__watch_was_edited = preserve_edited_flag
-
-    def _mark_field_as_edited(self, key):
-        """
-        Helper to mark a field as edited if it's writable.
-
-        Internal method used by __setitem__, update(), pop(), etc.
-        """
-        # Don't track edits during initial load or if already edited
-        if not hasattr(self, '_watch_base__watch_was_edited'):
-            return
-        if self.__watch_was_edited:
-            return  # Already marked as edited
-
-        # Import from shared schema utilities (no circular dependency)
-        from .schema_utils import get_readonly_watch_fields
-        readonly_fields = get_readonly_watch_fields()
-
-        # Additional system-managed fields not in OpenAPI spec (yet)
-        # These are set by processors/workers and should not trigger edited flag
-        additional_system_fields = {
-            'last_check_status',  # Set by processors
-            'restock',  # Set by restock processor
-            'last_viewed',  # Set by mark_all_viewed endpoint
-        }
-
-        # Only mark as edited if this is a user-writable field
-        if key not in readonly_fields and key not in additional_system_fields:
-            self.__watch_was_edited = True
-
-    def __setitem__(self, key, value):
-        """
-        Override dict.__setitem__ to track when writable watch fields are modified.
-
-        This enables skipping reprocessing when:
-        1. HTML content is unchanged (checksumFromPreviousCheckWasTheSame)
-        2. AND watch configuration was not edited
-
-        Only sets the edited flag when field is NOT in readonly_fields (from OpenAPI spec).
-        """
-        # Set the value first (always)
-        super().__setitem__(key, value)
-        # Mark as edited if writable field
-        self._mark_field_as_edited(key)
-
-    def __delitem__(self, key):
-        """Override dict.__delitem__ to track deletions of writable fields."""
-        super().__delitem__(key)
-        self._mark_field_as_edited(key)
-
-    def update(self, *args, **kwargs):
-
-        if args and args[0].get('browser_steps'):
-            args[0]['browser_steps'] = browser_steps_get_valid_steps(args[0].get('browser_steps'))
-
-        """Override dict.update() to track modifications to writable fields."""
-        # Call parent update first
-        super().update(*args, **kwargs)
-
-        # Mark as edited for any writable fields that were updated
-        # Handle both update(dict) and update(key=value) forms
-        if args:
-            for key in args[0].keys():
-                self._mark_field_as_edited(key)
-        for key in kwargs.keys():
-            self._mark_field_as_edited(key)
-
-
-    def pop(self, key, *args):
-        """Override dict.pop() to track removal of writable fields."""
-        result = super().pop(key, *args)
-        self._mark_field_as_edited(key)
-        return result
-
-    def setdefault(self, key, default=None):
-        """Override dict.setdefault() to track modifications to writable fields."""
-        # Only marks as edited if key didn't exist (i.e., a new value was set)
-        existed = key in self
-        result = super().setdefault(key, default)
-        if not existed:
-            self._mark_field_as_edited(key)
-        return result
-
-    @property
-    def was_edited(self):
-        """
-        Check if watch configuration was edited since last processing.
-
-        Returns:
-            bool: True if writable fields were modified, False otherwise
-        """
-        return getattr(self, '_watch_base__watch_was_edited', False)
-
-    def reset_watch_edited_flag(self):
-        """
-        Reset the watch edited flag after successful processing.
-
-        Call this after processing completes to allow future content-only change detection.
-        """
-        self.__watch_was_edited = False
-
-    @classmethod
-    def get_property_names(cls):
-        """
-        Get all @property attribute names from this model class using introspection.
-
-        This discovers computed/derived properties that are not stored in the datastore.
-        These properties should be filtered out during PUT/POST requests.
-
-        Returns:
-            frozenset: Immutable set of @property attribute names from the model class
-        """
-        import functools
-
-        # Create a cached version if it doesn't exist
-        if not hasattr(cls, '_cached_get_property_names'):
-            @functools.cache
-            def _get_props():
-                properties = set()
-                # Use introspection to find all @property attributes
-                for name in dir(cls):
-                    # Skip private/magic attributes
-                    if name.startswith('_'):
-                        continue
-                    try:
-                        attr = getattr(cls, name)
-                        # Check if it's a property descriptor
-                        if isinstance(attr, property):
-                            properties.add(name)
-                    except (AttributeError, TypeError):
-                        continue
-                return frozenset(properties)
-
-            cls._cached_get_property_names = _get_props
-
-        return cls._cached_get_property_names()
-
-    def __deepcopy__(self, memo):
-        """
-        Custom deepcopy for all watch_base subclasses (Watch, Tag, etc.).
-
-        CRITICAL FIX: Prevents copying large reference objects like __datastore
-        which would cause exponential memory growth when Watch objects are deepcopied.
-
-        This is called by:
-        - api/Watch.py:76 (API endpoint)
-        - api/Tags.py:28 (Tags API)
-        - processors/base.py:26 (EVERY processor run)
-        - store/__init__.py:544 (clone watch)
-        - And other locations
-        """
-        from copy import deepcopy
-
-        # Create new instance without calling __init__
-        cls = self.__class__
-        new_obj = cls.__new__(cls)
-        memo[id(self)] = new_obj
-
-        # Copy the dict data (all the settings)
-        for key, value in self.items():
-            new_obj[key] = deepcopy(value, memo)
-
-        # Copy instance attributes dynamically
-        # This handles Watch-specific attrs (like __datastore) and any future subclass attrs
-        for attr_name in dir(self):
-            # Skip methods, special attrs, and dict keys
-            if attr_name.startswith('_') and not attr_name.startswith('__'):
-                # This catches _model__datastore, _model__history_n, etc.
-                try:
-                    attr_value = getattr(self, attr_name)
-
-                    # Special handling: Share references to large objects instead of copying
-                    # Examples: _datastore, __datastore, __app_reference, __global_settings, etc.
-                    if (attr_name == '_datastore' or
-                        attr_name.endswith('__datastore') or
-                        attr_name.endswith('__app')):
-                        # Share the reference (don't copy!) to prevent memory leaks
-                        setattr(new_obj, attr_name, attr_value)
-                    # Skip cache attributes - let them regenerate on demand
-                    elif 'cache' in attr_name.lower():
-                        pass  # Don't copy caches
-                    # Copy regular instance attributes
-                    elif not callable(attr_value):
-                        setattr(new_obj, attr_name, attr_value)
-                except AttributeError:
-                    pass  # Attribute doesn't exist in this instance
-
-        return new_obj
-
-    def __getstate__(self):
-        """
-        Custom pickle serialization for all watch_base subclasses.
-
-        Excludes large reference objects (like __datastore) from serialization.
-        """
-        # Get the dict data
-        state = dict(self)
-
-        # Collect instance attributes (excluding methods and large references)
-        instance_attrs = {}
-        for attr_name in dir(self):
-            if attr_name.startswith('_') and not attr_name.startswith('__'):
-                try:
-                    attr_value = getattr(self, attr_name)
-                    # Exclude large reference objects and caches from serialization
-                    if not (attr_name == '_datastore' or
-                           attr_name.endswith('__datastore') or
-                           attr_name.endswith('__app') or
-                           'cache' in attr_name.lower() or
-                           callable(attr_value)):
-                        instance_attrs[attr_name] = attr_value
-                except AttributeError:
-                    pass
-
-        if instance_attrs:
-            state['__instance_metadata__'] = instance_attrs
-
-        return state
-
-    def __setstate__(self, state):
-        """
-        Custom pickle deserialization for all watch_base subclasses.
-
-        WARNING: Large reference objects (like __datastore) are NOT restored!
-        Caller must restore these references after unpickling if needed.
-        """
-        # Extract metadata
-        metadata = state.pop('__instance_metadata__', {})
-
-        # Restore dict data
-        self.update(state)
-
-        # Restore instance attributes
-        for attr_name, attr_value in metadata.items():
-            setattr(self, attr_name, attr_value)
-
-    @property
-    def data_dir(self):
-        """
-        The base directory for this watch/tag data (property, computed from UUID).
-
-        Common property for both Watch and Tag objects.
-        Returns path like: /datastore/{uuid}/
-        """
-        return os.path.join(self._datastore_path, self['uuid']) if self._datastore_path else None
-
-    def ensure_data_dir_exists(self):
-        """
-        Create the data directory if it doesn't exist.
-
-        Common method for both Watch and Tag objects.
-        """
-        from loguru import logger
-        if not os.path.isdir(self.data_dir):
-            logger.debug(f"> Creating data dir {self.data_dir}")
-            os.mkdir(self.data_dir)
-
-    def get_global_setting(self, *path):
-        """
-        Get a setting from the global datastore configuration.
-
-        Args:
-            *path: Path to the setting (e.g., 'application', 'history_snapshot_max_length')
-
-        Returns:
-            The setting value, or None if not found
-
-        Example:
-            maxlen = self.get_global_setting('application', 'history_snapshot_max_length')
-        """
-        if not self._datastore:
-            return None
-
-        try:
-            value = self._datastore['settings']
-            for key in path:
-                value = value[key]
-            return value
-        except (KeyError, TypeError):
-            return None
-
-    def _get_commit_data(self):
-        """
-        Prepare data for commit (can be overridden by subclasses).
-
-        Returns:
-            dict: Data to serialize (filtered as needed by subclass)
-        """
-        import copy
-
-        # Acquire datastore lock to prevent concurrent modifications during copy
-        lock = self._datastore.lock if self._datastore and hasattr(self._datastore, 'lock') else None
-
-        if lock:
-            with lock:
-                snapshot = dict(self)
-        else:
-            snapshot = dict(self)
-
-        # Deep copy snapshot (slower, but done outside lock to minimize contention)
-        # Subclasses can override to filter keys (e.g., Watch excludes processor_config_*)
-        return {k: copy.deepcopy(v) for k, v in snapshot.items()}
-
-    def _save_to_disk(self, data_dict, uuid):
-        """
-        Save data to disk (must be implemented by subclasses).
-
-        Args:
-            data_dict: Dictionary to save
-            uuid: UUID for logging
-
-        Raises:
-            NotImplementedError: If subclass doesn't implement
-        """
-        raise NotImplementedError("Subclass must implement _save_to_disk()")
-
-    def commit(self):
-        """
-        Save this watch/tag immediately to disk using atomic write.
-
-        Common commit logic for Watch and Tag objects.
-        Subclasses override _get_commit_data() and _save_to_disk() for specifics.
-
-        Fire-and-forget: Logs errors but does not raise exceptions.
-        Data remains in memory even if save fails, so next commit will retry.
-        """
-        from loguru import logger
-
-        if not self.data_dir:
-            entity_type = self.__class__.__name__
-            logger.error(f"Cannot commit {entity_type} {self.get('uuid')} without datastore_path")
-            return
-
-        uuid = self.get('uuid')
-        if not uuid:
-            entity_type = self.__class__.__name__
-            logger.error(f"Cannot commit {entity_type} without UUID")
-            return
-
-        # Get data from subclass (may filter keys)
-        try:
-            data_dict = self._get_commit_data()
-        except Exception as e:
-            logger.error(f"Failed to prepare commit data for {uuid}: {e}")
-            return
-
-        # Save to disk via subclass implementation
-        try:
-            # Determine entity type from module name (Watch.py -> watch, Tag.py -> tag)
-            entity_type = _determine_entity_type(self.__class__)
-            filename = f"{entity_type}.json"
-            self._save_to_disk(data_dict, uuid)
-            logger.debug(f"Committed {entity_type} {uuid} to {uuid}/{filename}")
-        except Exception as e:
-            logger.error(f"Failed to commit {uuid}: {e}")
+            del self['default']

changedetectionio/model/persistence.py

@@ -1,84 +0,0 @@
-"""
-Entity persistence mixin for Watch and Tag models.
-
-Provides file-based persistence using atomic writes.
-"""
-
-import functools
-import inspect
-
-
-@functools.lru_cache(maxsize=None)
-def _determine_entity_type(cls):
-    """
-    Determine entity type from class hierarchy (cached at class level).
-
-    Args:
-        cls: The class to inspect
-
-    Returns:
-        str: Entity type ('watch', 'tag', etc.)
-
-    Raises:
-        ValueError: If entity type cannot be determined
-    """
-    for base_class in inspect.getmro(cls):
-        module_name = base_class.__module__
-        if module_name.startswith('changedetectionio.model.'):
-            # Get last part after dot: "changedetectionio.model.Watch" -> "watch"
-            return module_name.split('.')[-1].lower()
-
-    raise ValueError(
-        f"Cannot determine entity type for {cls.__module__}.{cls.__name__}. "
-        f"Entity must inherit from a class in changedetectionio.model (Watch or Tag)."
-    )
-
-
-class EntityPersistenceMixin:
-    """
-    Mixin providing file persistence for watch_base subclasses (Watch, Tag, etc.).
-
-    This mixin provides the _save_to_disk() method required by watch_base.commit().
-    It automatically determines the correct filename and size limits based on class hierarchy.
-
-    Usage:
-        class model(EntityPersistenceMixin, watch_base):  # in Watch.py
-            pass
-
-        class model(EntityPersistenceMixin, watch_base):  # in Tag.py
-            pass
-    """
-
-    def _save_to_disk(self, data_dict, uuid):
-        """
-        Save entity to disk using atomic write.
-
-        Implements the abstract method required by watch_base.commit().
-        Automatically determines filename and size limits from class hierarchy.
-
-        Args:
-            data_dict: Dictionary to save
-            uuid: UUID for logging
-
-        Raises:
-            ValueError: If entity type cannot be determined from class hierarchy
-        """
-        # Import here to avoid circular dependency
-        from changedetectionio.store.file_saving_datastore import save_entity_atomic
-
-        # Determine entity type (cached at class level, not instance level)
-        entity_type = _determine_entity_type(self.__class__)
-
-        # Set filename and size limits based on entity type
-        filename = f'{entity_type}.json'
-        max_size_mb = 10 if entity_type == 'watch' else 1
-
-        # Save using generic function
-        save_entity_atomic(
-            self.data_dir,
-            uuid,
-            data_dict,
-            filename=filename,
-            entity_type=entity_type,
-            max_size_mb=max_size_mb
-        )

changedetectionio/model/schema_utils.py

@@ -1,92 +0,0 @@
-"""
-Schema utilities for Watch and Tag models.
-
-Provides functions to extract readonly fields and properties from OpenAPI spec.
-Shared by both the model layer and API layer to avoid circular dependencies.
-"""
-
-import functools
-
-
-@functools.cache
-def get_openapi_schema_dict():
-    """
-    Get the raw OpenAPI spec dictionary for schema access.
-
-    Returns the YAML dict directly (not the OpenAPI object).
-    """
-    import os
-    import yaml
-
-    spec_path = os.path.join(os.path.dirname(__file__), '../../docs/api-spec.yaml')
-    if not os.path.exists(spec_path):
-        spec_path = os.path.join(os.path.dirname(__file__), '../docs/api-spec.yaml')
-
-    with open(spec_path, 'r', encoding='utf-8') as f:
-        return yaml.safe_load(f)
-
-
-@functools.cache
-def _resolve_readonly_fields(schema_name):
-    """
-    Generic helper to resolve readOnly fields, including allOf inheritance.
-
-    Args:
-        schema_name: Name of the schema (e.g., 'Watch', 'Tag')
-
-    Returns:
-        frozenset: All readOnly field names including inherited ones
-    """
-    spec_dict = get_openapi_schema_dict()
-    schema = spec_dict['components']['schemas'].get(schema_name, {})
-
-    readonly_fields = set()
-
-    # Handle allOf (schema inheritance)
-    if 'allOf' in schema:
-        for item in schema['allOf']:
-            # Resolve $ref to parent schema
-            if '$ref' in item:
-                ref_path = item['$ref'].split('/')[-1]
-                ref_schema = spec_dict['components']['schemas'].get(ref_path, {})
-                if 'properties' in ref_schema:
-                    for field_name, field_def in ref_schema['properties'].items():
-                        if field_def.get('readOnly') is True:
-                            readonly_fields.add(field_name)
-            # Check schema-specific properties
-            if 'properties' in item:
-                for field_name, field_def in item['properties'].items():
-                    if field_def.get('readOnly') is True:
-                        readonly_fields.add(field_name)
-    else:
-        # Direct properties (no inheritance)
-        if 'properties' in schema:
-            for field_name, field_def in schema['properties'].items():
-                if field_def.get('readOnly') is True:
-                    readonly_fields.add(field_name)
-
-    return frozenset(readonly_fields)
-
-
-@functools.cache
-def get_readonly_watch_fields():
-    """
-    Extract readOnly field names from Watch schema in OpenAPI spec.
-
-    Returns readOnly fields from WatchBase (uuid, date_created) + Watch-specific readOnly fields.
-
-    Used by:
-    - model/watch_base.py: Track when writable fields are edited
-    - api/Watch.py: Filter readonly fields from PUT requests
-    """
-    return _resolve_readonly_fields('Watch')
-
-
-@functools.cache
-def get_readonly_tag_fields():
-    """
-    Extract readOnly field names from Tag schema in OpenAPI spec.
-
-    Returns readOnly fields from WatchBase (uuid, date_created) + Tag-specific readOnly fields.
-    """
-    return _resolve_readonly_fields('Tag')

changedetectionio/pluggy_interface.py

@@ -105,75 +105,6 @@ class ChangeDetectionSpec:
         """
         pass
 
-    @hookspec
-    def register_processor(self):
-        """Register an external processor plugin.
-
-        External packages can implement this hook to register custom processors
-        that will be discovered alongside built-in processors.
-
-        Returns:
-            dict or None: Dictionary with processor information:
-                {
-                    'processor_name': str,      # Machine name (e.g., 'osint_recon')
-                    'processor_module': module, # Module containing processor.py
-                    'processor_class': class,   # The perform_site_check class
-                    'metadata': {               # Optional metadata
-                        'name': str,            # Display name
-                        'description': str,     # Description
-                        'processor_weight': int,# Sort weight (lower = higher priority)
-                        'list_badge_text': str, # Badge text for UI
-                    }
-                }
-                Return None if this plugin doesn't provide a processor
-        """
-        pass
-
-    @hookspec
-    def update_handler_alter(update_handler, watch, datastore):
-        """Modify or wrap the update_handler before it processes a watch.
-
-        This hook is called after the update_handler (perform_site_check instance) is created
-        but before it calls call_browser() and run_changedetection(). Plugins can use this to:
-        - Wrap the handler to add logging/metrics
-        - Modify handler configuration
-        - Add custom preprocessing logic
-
-        Args:
-            update_handler: The perform_site_check instance that will process the watch
-            watch: The watch dict being processed
-            datastore: The application datastore
-
-        Returns:
-            object or None: Return a modified/wrapped handler, or None to keep the original.
-                           If multiple plugins return handlers, they are chained in registration order.
-        """
-        pass
-
-    @hookspec
-    def update_finalize(update_handler, watch, datastore, processing_exception):
-        """Called after watch processing completes (success or failure).
-
-        This hook is called in the finally block after all processing is complete,
-        allowing plugins to perform cleanup, update metrics, or log final status.
-
-        The plugin can access update_handler.last_logging_insert_id if it was stored
-        during update_handler_alter, and use processing_exception to determine if
-        the processing succeeded or failed.
-
-        Args:
-            update_handler: The perform_site_check instance (may be None if creation failed)
-            watch: The watch dict that was processed (may be None if not loaded)
-            datastore: The application datastore
-            processing_exception: The exception from the main processing block, or None if successful.
-                                 This does NOT include cleanup exceptions - only exceptions from
-                                 the actual watch processing (fetch, diff, etc).
-
-        Returns:
-            None: This hook doesn't return a value
-        """
-        pass
-
 
 # Set up Plugin Manager
 plugin_manager = pluggy.PluginManager(PLUGIN_NAMESPACE)
@@ -544,66 +475,4 @@ def get_plugin_template_paths():
                 template_paths.append(templates_dir)
                 logger.debug(f"Added plugin template path: {templates_dir}")
 
-    return template_paths
-
-
-def apply_update_handler_alter(update_handler, watch, datastore):
-    """Apply update_handler_alter hooks from all plugins.
-
-    Allows plugins to wrap or modify the update_handler before it processes a watch.
-    Multiple plugins can chain modifications - each plugin receives the result from
-    the previous plugin.
-
-    Args:
-        update_handler: The perform_site_check instance to potentially modify
-        watch: The watch dict being processed
-        datastore: The application datastore
-
-    Returns:
-        object: The (potentially modified/wrapped) update_handler
-    """
-    # Get all plugins that implement the update_handler_alter hook
-    results = plugin_manager.hook.update_handler_alter(
-        update_handler=update_handler,
-        watch=watch,
-        datastore=datastore
-    )
-
-    # Chain results - each plugin gets the result from the previous one
-    current_handler = update_handler
-    if results:
-        for result in results:
-            if result is not None:
-                logger.debug(f"Plugin modified update_handler for watch {watch.get('uuid')}")
-                current_handler = result
-
-    return current_handler
-
-
-def apply_update_finalize(update_handler, watch, datastore, processing_exception):
-    """Apply update_finalize hooks from all plugins.
-
-    Called in the finally block after watch processing completes, allowing plugins
-    to perform cleanup, update metrics, or log final status.
-
-    Args:
-        update_handler: The perform_site_check instance (may be None)
-        watch: The watch dict that was processed (may be None)
-        datastore: The application datastore
-        processing_exception: The exception from processing, or None if successful
-
-    Returns:
-        None
-    """
-    try:
-        # Call all plugins that implement the update_finalize hook
-        plugin_manager.hook.update_finalize(
-            update_handler=update_handler,
-            watch=watch,
-            datastore=datastore,
-            processing_exception=processing_exception
-        )
-    except Exception as e:
-        # Don't let plugin errors crash the worker
-        logger.error(f"Error in update_finalize hook: {e}")
-        logger.exception(f"update_finalize hook exception details:")
+    return template_paths

changedetectionio/processors/__init__.py

@@ -1,6 +1,6 @@
 from functools import lru_cache
 from loguru import logger
-from flask_babel import gettext, get_locale
+from flask_babel import gettext
 import importlib
 import inspect
 import os
@@ -17,11 +17,9 @@ def find_sub_packages(package_name):
     return [name for _, name, is_pkg in pkgutil.iter_modules(package.__path__) if is_pkg]
 
 
-@lru_cache(maxsize=1)
 def find_processors():
     """
     Find all subclasses of DifferenceDetectionProcessor in the specified package.
-    Results are cached to avoid repeated discovery.
 
     :param package_name: The name of the package to scan for processor modules.
     :return: A list of (module, class) tuples.
@@ -48,23 +46,6 @@ def find_processors():
         except (ModuleNotFoundError, ImportError) as e:
             logger.warning(f"Failed to import module {module_name}: {e} (find_processors())")
 
-    # Discover plugin processors via pluggy
-    try:
-        from changedetectionio.pluggy_interface import plugin_manager
-        plugin_results = plugin_manager.hook.register_processor()
-
-        for result in plugin_results:
-            if result and isinstance(result, dict):
-                processor_module = result.get('processor_module')
-                processor_name = result.get('processor_name')
-
-                if processor_module and processor_name:
-                    processors.append((processor_module, processor_name))
-                    plugin_path = getattr(processor_module, '__file__', 'unknown location')
-                    logger.info(f"Registered plugin processor: {processor_name} from {plugin_path}")
-    except Exception as e:
-        logger.warning(f"Error loading plugin processors: {e}")
-
     return processors
 
 
@@ -116,138 +97,54 @@ def find_processor_module(processor_name):
     return None
 
 
-def get_processor_module(processor_name):
+def available_processors():
     """
-    Get the actual processor module (with perform_site_check class) by name.
-    Works for both built-in and plugin processors.
-
-    Args:
-        processor_name: Processor machine name (e.g., 'text_json_diff', 'osint_recon')
-
-    Returns:
-        module: The processor module containing perform_site_check, or None if not found
+    Get a list of processors by name and description for the UI elements.
+    Can be filtered via ALLOWED_PROCESSORS environment variable (comma-separated list).
+    :return: A list :)
     """
-    processor_classes = find_processors()
-    processor_tuple = next((tpl for tpl in processor_classes if tpl[1] == processor_name), None)
 
-    if processor_tuple:
-        # Return the actual processor module (first element of tuple)
-        return processor_tuple[0]
-
-    return None
-
-
-def get_processor_submodule(processor_name, submodule_name):
-    """
-    Get an optional submodule from a processor (e.g., 'difference', 'extract', 'preview').
-    Works for both built-in and plugin processors.
-
-    Args:
-        processor_name: Processor machine name (e.g., 'text_json_diff', 'osint_recon')
-        submodule_name: Name of the submodule (e.g., 'difference', 'extract', 'preview')
-
-    Returns:
-        module: The submodule if it exists, or None if not found
-    """
-    processor_classes = find_processors()
-    processor_tuple = next((tpl for tpl in processor_classes if tpl[1] == processor_name), None)
-
-    if not processor_tuple:
-        return None
-
-    processor_module = processor_tuple[0]
-    parent_module = get_parent_module(processor_module)
-
-    if not parent_module:
-        return None
-
-    # Try to import the submodule
-    try:
-        # For built-in processors: changedetectionio.processors.text_json_diff.difference
-        # For plugin processors: changedetectionio_osint.difference
-        parent_module_name = parent_module.__name__
-        submodule_full_name = f"{parent_module_name}.{submodule_name}"
-        return importlib.import_module(submodule_full_name)
-    except (ModuleNotFoundError, ImportError):
-        return None
-
-
-@lru_cache(maxsize=1)
-def get_plugin_processor_metadata():
-    """Get metadata from plugin processors."""
-    metadata = {}
-    try:
-        from changedetectionio.pluggy_interface import plugin_manager
-        plugin_results = plugin_manager.hook.register_processor()
-
-        for result in plugin_results:
-            if result and isinstance(result, dict):
-                processor_name = result.get('processor_name')
-                meta = result.get('metadata', {})
-                if processor_name:
-                    metadata[processor_name] = meta
-    except Exception as e:
-        logger.warning(f"Error getting plugin processor metadata: {e}")
-    return metadata
-
-@lru_cache(maxsize=32)
-def _available_processors_cached(locale_str):
-    """
-    Internal cached function that includes locale in cache key.
-    This ensures translations are cached per-language instead of globally.
-
-    :param locale_str: The locale string (e.g., 'en', 'it', 'zh')
-    :return: A list of tuples (processor_name, translated_description, weight)
-    """
     processor_classes = find_processors()
 
-    # Check if DISABLED_PROCESSORS env var is set
-    disabled_processors_env = os.getenv('DISABLED_PROCESSORS', 'image_ssim_diff').strip()
-    disabled_processors = []
-    if disabled_processors_env:
+    # Check if ALLOWED_PROCESSORS env var is set
+    # For now we disable it, need to make a deploy with lots of new code and this will be an overload
+    allowed_processors_env = os.getenv('ALLOWED_PROCESSORS', 'text_json_diff, restock_diff').strip()
+    allowed_processors = None
+    if allowed_processors_env:
         # Parse comma-separated list and strip whitespace
-        disabled_processors = [p.strip() for p in disabled_processors_env.split(',') if p.strip()]
-        logger.info(f"DISABLED_PROCESSORS set, disabling: {disabled_processors}")
+        allowed_processors = [p.strip() for p in allowed_processors_env.split(',') if p.strip()]
+        logger.info(f"ALLOWED_PROCESSORS set, filtering to: {allowed_processors}")
 
     available = []
-    plugin_metadata = get_plugin_processor_metadata()
-
     for module, sub_package_name in processor_classes:
-        # Skip disabled processors
-        if sub_package_name in disabled_processors:
-            logger.debug(f"Skipping processor '{sub_package_name}' (in DISABLED_PROCESSORS)")
+        # Filter by allowed processors if set
+        if allowed_processors and sub_package_name not in allowed_processors:
+            logger.debug(f"Skipping processor '{sub_package_name}' (not in ALLOWED_PROCESSORS)")
             continue
 
-        # Check if this is a plugin processor
-        if sub_package_name in plugin_metadata:
-            meta = plugin_metadata[sub_package_name]
-            description = gettext(meta.get('name', sub_package_name))
-            # Plugin processors start from weight 10 to separate them from built-in processors
-            weight = 100 + meta.get('processor_weight', 0)
+        # Try to get the 'name' attribute from the processor module first
+        if hasattr(module, 'name'):
+            description = gettext(module.name)
         else:
-            # Try to get the 'name' attribute from the processor module first
-            if hasattr(module, 'name'):
-                description = gettext(module.name)
+            # Fall back to processor_description from parent module's __init__.py
+            parent_module = get_parent_module(module)
+            if parent_module and hasattr(parent_module, 'processor_description'):
+                description = gettext(parent_module.processor_description)
             else:
-                # Fall back to processor_description from parent module's __init__.py
-                parent_module = get_parent_module(module)
-                if parent_module and hasattr(parent_module, 'processor_description'):
-                    description = gettext(parent_module.processor_description)
-                else:
-                    # Final fallback to a readable name
-                    description = sub_package_name.replace('_', ' ').title()
+                # Final fallback to a readable name
+                description = sub_package_name.replace('_', ' ').title()
 
-            # Get weight for sorting (lower weight = higher in list)
-            weight = 0  # Default weight for processors without explicit weight
+        # Get weight for sorting (lower weight = higher in list)
+        weight = 0  # Default weight for processors without explicit weight
 
-            # Check processor module itself first
-            if hasattr(module, 'processor_weight'):
-                weight = module.processor_weight
-            else:
-                # Fall back to parent module (package __init__.py)
-                parent_module = get_parent_module(module)
-                if parent_module and hasattr(parent_module, 'processor_weight'):
-                    weight = parent_module.processor_weight
+        # Check processor module itself first
+        if hasattr(module, 'processor_weight'):
+            weight = module.processor_weight
+        else:
+            # Fall back to parent module (package __init__.py)
+            parent_module = get_parent_module(module)
+            if parent_module and hasattr(parent_module, 'processor_weight'):
+                weight = parent_module.processor_weight
 
         available.append((sub_package_name, description, weight))
 
@@ -257,36 +154,6 @@ def _available_processors_cached(locale_str):
     # Return as tuples without weight (for backwards compatibility)
     return [(name, desc) for name, desc, weight in available]
 
-def available_processors():
-    """
-    Get a list of processors by name and description for the UI elements.
-    Can be filtered via DISABLED_PROCESSORS environment variable (comma-separated list).
-
-    This function delegates to a locale-aware cached version to ensure translations
-    are cached per-language instead of globally.
-
-    :return: A list of tuples (processor_name, translated_description)
-    """
-    # Get current locale and use it as cache key
-    # Convert Babel Locale object to string for use as cache key
-    locale = get_locale()
-    locale_str = str(locale) if locale else 'en'
-    return _available_processors_cached(locale_str)
-
-
-def get_default_processor():
-    """
-    Get the default processor to use when none is specified.
-    Returns the first available processor based on weight (lowest weight = highest priority).
-    This ensures forms auto-select a valid processor even when DISABLED_PROCESSORS filters the list.
-
-    :return: The processor name string (e.g., 'text_json_diff')
-    """
-    available = available_processors()
-    if available:
-        return available[0][0]  # Return the processor name from first tuple
-    return 'text_json_diff'  # Fallback if somehow no processors are available
-
 
 def get_processor_badge_texts():
     """
@@ -412,76 +279,3 @@ def get_processor_badge_css():
 
     return '\n\n'.join(css_rules)
 
-
-def save_processor_config(datastore, watch_uuid, config_data):
-    """
-    Save processor-specific configuration to JSON file.
-
-    This is a shared helper function used by both the UI edit form and API endpoints
-    to consistently handle processor configuration storage.
-
-    Args:
-        datastore: The application datastore instance
-        watch_uuid: UUID of the watch
-        config_data: Dictionary of configuration data to save (with processor_config_* prefix removed)
-
-    Returns:
-        bool: True if saved successfully, False otherwise
-    """
-    if not config_data:
-        return True
-
-    try:
-        from changedetectionio.processors.base import difference_detection_processor
-
-        # Get processor name from watch
-        watch = datastore.data['watching'].get(watch_uuid)
-        if not watch:
-            logger.error(f"Cannot save processor config: watch {watch_uuid} not found")
-            return False
-
-        processor_name = watch.get('processor', 'text_json_diff')
-
-        # Create a processor instance to access config methods
-        processor_instance = difference_detection_processor(datastore, watch_uuid)
-
-        # Use processor name as filename so each processor keeps its own config
-        config_filename = f'{processor_name}.json'
-        processor_instance.update_extra_watch_config(config_filename, config_data)
-
-        logger.debug(f"Saved processor config to {config_filename}: {config_data}")
-        return True
-
-    except Exception as e:
-        logger.error(f"Failed to save processor config: {e}")
-        return False
-
-
-def extract_processor_config_from_form_data(form_data):
-    """
-    Extract processor_config_* fields from form data and return separate dicts.
-
-    This is a shared helper function used by both the UI edit form and API endpoints
-    to consistently handle processor configuration extraction.
-
-    IMPORTANT: This function modifies form_data in-place by removing processor_config_* fields.
-
-    Args:
-        form_data: Dictionary of form data (will be modified in-place)
-
-    Returns:
-        dict: Dictionary of processor config data (with processor_config_* prefix removed)
-    """
-    processor_config_data = {}
-
-    # Use list() to create a copy of keys since we're modifying the dict
-    for field_name in list(form_data.keys()):
-        if field_name.startswith('processor_config_'):
-            config_key = field_name.replace('processor_config_', '')
-            # Save all values (including empty strings) to allow explicit clearing of settings
-            processor_config_data[config_key] = form_data[field_name]
-            # Remove from form_data to prevent it from reaching datastore
-            del form_data[field_name]
-
-    return processor_config_data
-

changedetectionio/processors/base.py

@@ -1,7 +1,5 @@
 import re
 import hashlib
-
-from changedetectionio.browser_steps.browser_steps import browser_steps_get_valid_steps
 from changedetectionio.content_fetchers.base import Fetcher
 from changedetectionio.strtobool import strtobool
 from copy import deepcopy
@@ -21,80 +19,14 @@ class difference_detection_processor():
     xpath_data = None
     preferred_proxy = None
     screenshot_format = SCREENSHOT_FORMAT_JPEG
-    last_raw_content_checksum = None
 
     def __init__(self, datastore, watch_uuid):
         self.datastore = datastore
         self.watch_uuid = watch_uuid
-
-        # Create a stable snapshot of the watch for processing
-        # Why deepcopy?
-        # 1. Prevents "dict changed during iteration" errors if watch is modified during processing
-        # 2. Preserves Watch object with properties (.link, .is_pdf, etc.) - can't use dict()
-        # 3. Safe now: Watch.__deepcopy__() shares datastore ref (no memory leak) but copies dict data
         self.watch = deepcopy(self.datastore.data['watching'].get(watch_uuid))
-
         # Generic fetcher that should be extended (requests, playwright etc)
         self.fetcher = Fetcher()
 
-        # Load the last raw content checksum from file
-        self.read_last_raw_content_checksum()
-
-    def update_last_raw_content_checksum(self, checksum):
-        """
-        Save the raw content MD5 checksum to file.
-        This is used for skip logic - avoid reprocessing if raw HTML unchanged.
-        """
-        if not checksum:
-            return
-
-        watch = self.datastore.data['watching'].get(self.watch_uuid)
-        if not watch:
-            return
-
-        data_dir = watch.data_dir
-        if not data_dir:
-            return
-
-        watch.ensure_data_dir_exists()
-        checksum_file = os.path.join(data_dir, 'last-checksum.txt')
-
-        try:
-            with open(checksum_file, 'w', encoding='utf-8') as f:
-                f.write(checksum)
-            self.last_raw_content_checksum = checksum
-        except IOError as e:
-            logger.warning(f"Failed to write checksum file for {self.watch_uuid}: {e}")
-
-    def read_last_raw_content_checksum(self):
-        """
-        Read the last raw content MD5 checksum from file.
-        Returns None if file doesn't exist (first run) or can't be read.
-        """
-        watch = self.datastore.data['watching'].get(self.watch_uuid)
-        if not watch:
-            self.last_raw_content_checksum = None
-            return
-
-        data_dir = watch.data_dir
-        if not data_dir:
-            self.last_raw_content_checksum = None
-            return
-
-        checksum_file = os.path.join(data_dir, 'last-checksum.txt')
-
-        if not os.path.isfile(checksum_file):
-            self.last_raw_content_checksum = None
-            return
-
-        try:
-            with open(checksum_file, 'r', encoding='utf-8') as f:
-                self.last_raw_content_checksum = f.read().strip()
-        except IOError as e:
-            logger.warning(f"Failed to read checksum file for {self.watch_uuid}: {e}")
-            self.last_raw_content_checksum = None
-
-
     async def call_browser(self, preferred_proxy_id=None):
 
         from requests.structures import CaseInsensitiveDict
@@ -171,7 +103,7 @@ class difference_detection_processor():
                                    )
 
         if self.watch.has_browser_steps:
-            self.fetcher.browser_steps = browser_steps_get_valid_steps(self.watch.get('browser_steps', []))
+            self.fetcher.browser_steps = self.watch.get('browser_steps', [])
             self.fetcher.browser_steps_screenshot_path = os.path.join(self.datastore.datastore_path, self.watch.get('uuid'))
 
         # Tweak the base config with the per-watch ones
@@ -254,12 +186,12 @@ class difference_detection_processor():
         import os
 
         watch = self.datastore.data['watching'].get(self.watch_uuid)
-        data_dir = watch.data_dir
+        watch_data_dir = watch.watch_data_dir
 
-        if not data_dir:
+        if not watch_data_dir:
             return {}
 
-        filepath = os.path.join(data_dir, filename)
+        filepath = os.path.join(watch_data_dir, filename)
 
         if not os.path.isfile(filepath):
             return {}
@@ -284,16 +216,16 @@ class difference_detection_processor():
         import os
 
         watch = self.datastore.data['watching'].get(self.watch_uuid)
-        data_dir = watch.data_dir
+        watch_data_dir = watch.watch_data_dir
 
-        if not data_dir:
-            logger.warning(f"Cannot save extra watch config {filename}: no data_dir")
+        if not watch_data_dir:
+            logger.warning(f"Cannot save extra watch config {filename}: no watch_data_dir")
             return
 
         # Ensure directory exists
         watch.ensure_data_dir_exists()
 
-        filepath = os.path.join(data_dir, filename)
+        filepath = os.path.join(watch_data_dir, filename)
 
         try:
             # If merge is enabled, read existing data first
@@ -318,16 +250,8 @@ class difference_detection_processor():
         except IOError as e:
             logger.error(f"Failed to write extra watch config {filename}: {e}")
 
-    def get_raw_document_checksum(self):
-        checksum = None
-
-        if self.fetcher.content:
-            checksum = hashlib.md5(self.fetcher.content.encode('utf-8')).hexdigest()
-
-        return checksum
-
     @abstractmethod
-    def run_changedetection(self, watch, force_reprocess=False):
+    def run_changedetection(self, watch):
         update_obj = {'last_notification_error': False, 'last_error': False}
         some_data = 'xxxxx'
         update_obj["previous_md5"] = hashlib.md5(some_data.encode('utf-8')).hexdigest()

changedetectionio/processors/image_ssim_diff/__init__.py

@@ -12,13 +12,6 @@ processor_description = "Visual/Screenshot change detection (Fast)"
 processor_name = "image_ssim_diff"
 processor_weight = 2  # Lower weight = appears at top, heavier weight = appears lower (bottom)
 
-# Processor capabilities
-supports_visual_selector = True
-supports_browser_steps = True
-supports_text_filters_and_triggers = False
-supports_text_filters_and_triggers_elements = False
-supports_request_type = True
-
 PROCESSOR_CONFIG_NAME = f"{Path(__file__).parent.name}.json"
 
 # Subprocess timeout settings

changedetectionio/processors/image_ssim_diff/difference.py

@@ -130,7 +130,7 @@ def get_asset(asset_name, watch, datastore, request):
                 except Exception as e:
                     exception_container[0] = e
 
-            thread = threading.Thread(target=thread_target, daemon=True, name="ImageDiff-Asset")
+            thread = threading.Thread(target=thread_target)
             thread.start()
             thread.join(timeout=60)
 
@@ -284,7 +284,7 @@ def _draw_bounding_box_if_configured(img_bytes, watch, datastore):
             except Exception as e:
                 exception_container[0] = e
 
-        thread = threading.Thread(target=thread_target, daemon=True, name="ImageDiff-BoundingBox")
+        thread = threading.Thread(target=thread_target)
         thread.start()
         thread.join(timeout=15)
 
@@ -393,7 +393,7 @@ def render(watch, datastore, request, url_for, render_template, flash, redirect)
             except Exception as e:
                 exception_container[0] = e
 
-        thread = threading.Thread(target=thread_target, daemon=True, name="ImageDiff-ChangePercentage")
+        thread = threading.Thread(target=thread_target)
         thread.start()
         thread.join(timeout=60)
 
@@ -414,7 +414,7 @@ def render(watch, datastore, request, url_for, render_template, flash, redirect)
 
     # Load historical data if available (for charts/visualization)
     comparison_data = {}
-    comparison_config_path = os.path.join(watch.data_dir, "visual_comparison_data.json")
+    comparison_config_path = os.path.join(watch.watch_data_dir, "visual_comparison_data.json")
     if os.path.isfile(comparison_config_path):
         try:
             with open(comparison_config_path, 'r') as f:

changedetectionio/processors/image_ssim_diff/edit_hook.py

@@ -90,7 +90,7 @@ def on_config_save(watch, processor_config, datastore):
             processor_config['auto_track_region'] = False
 
             # Remove old template file if exists
-            template_path = os.path.join(watch.data_dir, CROPPED_IMAGE_TEMPLATE_FILENAME)
+            template_path = os.path.join(watch.watch_data_dir, CROPPED_IMAGE_TEMPLATE_FILENAME)
             if os.path.exists(template_path):
                 os.remove(template_path)
                 logger.debug(f"Removed old template file: {template_path}")

changedetectionio/processors/image_ssim_diff/processor.py

@@ -30,7 +30,7 @@ class perform_site_check(difference_detection_processor):
     # Override to use PNG format for better image comparison (JPEG compression creates noise)
     screenshot_format = SCREENSHOT_FORMAT_PNG
 
-    def run_changedetection(self, watch, force_reprocess=False):
+    def run_changedetection(self, watch):
         """
         Perform screenshot comparison using OpenCV subprocess handler.
 
@@ -204,7 +204,7 @@ class perform_site_check(difference_detection_processor):
                 except Exception as e:
                     exception_container[0] = e
 
-            thread = threading.Thread(target=thread_target, daemon=True, name="ImageDiff-Processor")
+            thread = threading.Thread(target=thread_target)
             thread.start()
             thread.join(timeout=60)
 

changedetectionio/processors/restock_diff/__init__.py

@@ -4,13 +4,6 @@ from changedetectionio.model.Watch import model as BaseWatch
 from typing import Union
 import re
 
-# Processor capabilities
-supports_visual_selector = True
-supports_browser_steps = True
-supports_text_filters_and_triggers = True
-supports_text_filters_and_triggers_elements = True
-supports_request_type = True
-
 class Restock(dict):
 
     def parse_currency(self, raw_value: str) -> Union[float, None]:

changedetectionio/processors/restock_diff/processor.py

@@ -2,7 +2,6 @@ from ..base import difference_detection_processor
 from ..exceptions import ProcessorException
 from . import Restock
 from loguru import logger
-from changedetectionio.content_fetchers.exceptions import checksumFromPreviousCheckWasTheSame
 
 import urllib3
 import time
@@ -57,259 +56,6 @@ def _deduplicate_prices(data):
     return list(unique_data)
 
 
-# =============================================================================
-# MEMORY MANAGEMENT: Why We Use Multiprocessing (Linux Only)
-# =============================================================================
-#
-# The get_itemprop_availability() function uses 'extruct' to parse HTML metadata
-# (JSON-LD, microdata, OpenGraph, etc). Extruct internally uses lxml, which wraps
-# libxml2 - a C library that allocates memory at the C level.
-#
-# Memory Leak Problem:
-# --------------------
-# 1. lxml's document_fromstring() creates thousands of Python objects backed by
-#    C-level allocations (nodes, attributes, text content)
-# 2. Python's garbage collector can mark these objects as collectible, but
-#    cannot force the OS to reclaim the actual C-level memory
-# 3. malloc/free typically doesn't return memory to OS - it just marks it as
-#    "free in the process address space"
-# 4. With repeated parsing of large HTML (5MB+ pages), memory accumulates even
-#    after Python GC runs
-#
-# Why Multiprocessing Fixes This:
-# --------------------------------
-# When a subprocess exits, the OS forcibly reclaims ALL memory including C-level
-# allocations that Python GC couldn't release. This ensures clean memory state
-# after each extraction.
-#
-# Performance Impact:
-# -------------------
-# - Memray analysis showed 1.2M document_fromstring allocations per page
-# - Without subprocess: memory grows by ~50-500MB per parse and lingers
-# - With subprocess: ~35MB overhead but forces full cleanup after each run
-# - Trade-off: 35MB resource_tracker vs 500MB+ accumulated leak = much better at scale
-#
-# References:
-# -----------
-# - lxml memory issues: https://medium.com/devopss-hole/python-lxml-memory-leak-b8d0b1000dc7
-# - libxml2 caching behavior: https://www.mail-archive.com/lxml@python.org/msg00026.html
-# - GC limitations with C extensions: https://benbernardblog.com/tracking-down-a-freaky-python-memory-leak-part-2/
-#
-# Additional Context:
-# -------------------
-# - jsonpath_ng (used to query the parsed data) is pure Python and doesn't leak
-# - The leak is specifically from lxml's document parsing, not the JSONPath queries
-# - Linux-only because multiprocessing spawn is well-tested there; other platforms
-#   use direct call as fallback
-#
-# Alternative Solution (Future Optimization):
-# -------------------------------------------
-# This entire problem could be avoided by using regex to extract just the machine
-# data blocks (JSON-LD, microdata, OpenGraph tags) BEFORE parsing with lxml:
-#
-#   1. Use regex to extract <script type="application/ld+json">...</script> blocks
-#   2. Use regex to extract <meta property="og:*"> tags
-#   3. Use regex to find itemprop/itemtype attributes and their containing elements
-#   4. Parse ONLY those extracted snippets instead of the entire HTML document
-#
-# Benefits:
-#   - Avoids parsing 5MB of HTML when we only need a few KB of metadata
-#   - Eliminates the lxml memory leak entirely
-#   - Faster extraction (regex is much faster than DOM parsing)
-#   - No subprocess overhead needed
-#
-# Trade-offs:
-#   - Regex for HTML is brittle (comments, CDATA, edge cases)
-#   - Microdata extraction would be complex (need to track element boundaries)
-#   - Would need extensive testing to ensure we don't miss valid data
-#   - extruct is battle-tested; regex solution would need similar maturity
-#
-# For now, the subprocess approach is safer and leverages existing extruct code.
-# =============================================================================
-
-
-def _extract_itemprop_availability_worker(pipe_conn):
-    """
-    Subprocess worker for itemprop extraction (Linux memory management).
-
-    Uses spawn multiprocessing to isolate extruct/lxml memory allocations.
-    When the subprocess exits, the OS reclaims ALL memory including lxml's
-    C-level allocations that Python's GC cannot release.
-
-    Args:
-        pipe_conn: Pipe connection to receive HTML and send result
-    """
-    import json
-    import gc
-
-    html_content = None
-    result_data = None
-
-    try:
-        # Receive HTML as raw bytes (no pickle)
-        html_bytes = pipe_conn.recv_bytes()
-        html_content = html_bytes.decode('utf-8')
-
-        # Explicitly delete html_bytes to free memory
-        del html_bytes
-        gc.collect()
-
-        # Perform extraction in subprocess (uses extruct/lxml)
-        result_data = get_itemprop_availability(html_content)
-
-        # Convert Restock object to dict for JSON serialization
-        result = {
-            'success': True,
-            'data': dict(result_data) if result_data else {}
-        }
-        pipe_conn.send_bytes(json.dumps(result).encode('utf-8'))
-
-        # Clean up before exit
-        del result_data, html_content, result
-        gc.collect()
-
-    except MoreThanOnePriceFound:
-        # Serialize the specific exception type
-        result = {
-            'success': False,
-            'exception_type': 'MoreThanOnePriceFound'
-        }
-        pipe_conn.send_bytes(json.dumps(result).encode('utf-8'))
-
-    except Exception as e:
-        # Serialize other exceptions
-        result = {
-            'success': False,
-            'exception_type': type(e).__name__,
-            'exception_message': str(e)
-        }
-        pipe_conn.send_bytes(json.dumps(result).encode('utf-8'))
-
-    finally:
-        # Final cleanup before subprocess exits
-        # Variables may already be deleted in try block, so use try/except
-        try:
-            del html_content
-        except (NameError, UnboundLocalError):
-            pass
-        try:
-            del result_data
-        except (NameError, UnboundLocalError):
-            pass
-        gc.collect()
-        pipe_conn.close()
-
-
-def extract_itemprop_availability_safe(html_content) -> Restock:
-    """
-    Extract itemprop availability with hybrid approach for memory efficiency.
-
-    Strategy (fastest to slowest, least to most memory):
-    1. Try pure Python extraction (JSON-LD, OpenGraph, microdata) - covers 80%+ of cases
-    2. Fall back to extruct with subprocess isolation on Linux for complex cases
-
-    Args:
-        html_content: HTML string to parse
-
-    Returns:
-        Restock: Extracted availability data
-
-    Raises:
-        MoreThanOnePriceFound: When multiple prices detected
-        Other exceptions: From extruct/parsing
-    """
-    import platform
-
-    # Step 1: Try pure Python extraction first (fast, no lxml, no memory leak)
-    try:
-        from .pure_python_extractor import extract_metadata_pure_python, query_price_availability
-
-        logger.trace("Attempting pure Python metadata extraction (no lxml)")
-        extracted_data = extract_metadata_pure_python(html_content)
-        price_data = query_price_availability(extracted_data)
-
-        # If we got price AND availability, we're done!
-        if price_data.get('price') and price_data.get('availability'):
-            result = Restock(price_data)
-            logger.debug(f"Pure Python extraction successful: {dict(result)}")
-            return result
-
-        # If we got some data but not everything, still try extruct for completeness
-        if price_data.get('price') or price_data.get('availability'):
-            logger.debug(f"Pure Python extraction partial: {price_data}, will try extruct for completeness")
-
-    except Exception as e:
-        logger.debug(f"Pure Python extraction failed: {e}, falling back to extruct")
-
-    # Step 2: Fall back to extruct (uses lxml, needs subprocess on Linux)
-    logger.trace("Falling back to extruct (lxml-based) with subprocess isolation")
-
-    # Only use subprocess isolation on Linux
-    # Other platforms may have issues with spawn or don't need the aggressive memory management
-    if platform.system() == 'Linux':
-        import multiprocessing
-        import json
-        import gc
-
-        try:
-            ctx = multiprocessing.get_context('spawn')
-            parent_conn, child_conn = ctx.Pipe()
-            p = ctx.Process(target=_extract_itemprop_availability_worker, args=(child_conn,))
-            p.start()
-
-            # Send HTML as raw bytes (no pickle)
-            html_bytes = html_content.encode('utf-8')
-            parent_conn.send_bytes(html_bytes)
-
-            # Explicitly delete html_bytes copy immediately after sending
-            del html_bytes
-            gc.collect()
-
-            # Receive result as JSON
-            result_bytes = parent_conn.recv_bytes()
-            result = json.loads(result_bytes.decode('utf-8'))
-
-            # Wait for subprocess to complete
-            p.join()
-
-            # Close pipes
-            parent_conn.close()
-            child_conn.close()
-
-            # Clean up all subprocess-related objects
-            del p, parent_conn, child_conn, result_bytes
-            gc.collect()
-
-            # Handle result or re-raise exception
-            if result['success']:
-                # Reconstruct Restock object from dict
-                restock_obj = Restock(result['data'])
-                # Clean up result dict
-                del result
-                gc.collect()
-                return restock_obj
-            else:
-                # Re-raise the exception that occurred in subprocess
-                exception_type = result['exception_type']
-                exception_msg = result.get('exception_message', '')
-                del result
-                gc.collect()
-
-                if exception_type == 'MoreThanOnePriceFound':
-                    raise MoreThanOnePriceFound()
-                else:
-                    raise Exception(f"{exception_type}: {exception_msg}")
-
-        except Exception as e:
-            # If multiprocessing itself fails, log and fall back to direct call
-            logger.warning(f"Subprocess extraction failed: {e}, falling back to direct call")
-            gc.collect()
-            return get_itemprop_availability(html_content)
-    else:
-        # Non-Linux: direct call (no subprocess overhead needed)
-        return get_itemprop_availability(html_content)
-
-
 # should return Restock()
 # add casting?
 def get_itemprop_availability(html_content) -> Restock:
@@ -404,37 +150,22 @@ class perform_site_check(difference_detection_processor):
     screenshot = None
     xpath_data = None
 
-    def run_changedetection(self, watch, force_reprocess=False):
+    def run_changedetection(self, watch):
         import hashlib
 
         if not watch:
             raise Exception("Watch no longer exists.")
 
-        current_raw_document_checksum = self.get_raw_document_checksum()
-        # Skip processing only if BOTH conditions are true:
-        # 1. HTML content unchanged (checksum matches last saved checksum)
-        # 2. Watch configuration was not edited (including trigger_text, filters, etc.)
-        # The was_edited flag handles all watch configuration changes, so we don't need
-        # separate checks for trigger_text or other processing rules.
-        if (not force_reprocess and
-            not watch.was_edited and
-            self.last_raw_content_checksum and
-            self.last_raw_content_checksum == current_raw_document_checksum):
-            raise checksumFromPreviousCheckWasTheSame()
-
         # Unset any existing notification error
         update_obj = {'last_notification_error': False, 'last_error': False, 'restock':  Restock()}
 
         self.screenshot = self.fetcher.screenshot
         self.xpath_data = self.fetcher.xpath_data
 
-        # Track the content type (readonly field, doesn't trigger was_edited)
-        update_obj['content-type'] = self.fetcher.headers.get('Content-Type', '')  # Use hyphen (matches OpenAPI spec)
+        # Track the content type
+        update_obj['content_type'] = self.fetcher.headers.get('Content-Type', '')
         update_obj["last_check_status"] = self.fetcher.get_last_status_code()
 
-        # Save the raw content checksum to file (processor implementation detail, not watch config)
-        self.update_last_raw_content_checksum(current_raw_document_checksum)
-
         # Only try to process restock information (like scraping for keywords) if the page was actually rendered correctly.
         # Otherwise it will assume "in stock" because nothing suggesting the opposite was found
         from ...html_tools import html_to_text
@@ -462,18 +193,18 @@ class perform_site_check(difference_detection_processor):
 
 
         itemprop_availability = {}
-        multiple_prices_found = False
 
         # Try built-in extraction first, this will scan metadata in the HTML
-        # On Linux, this runs in a subprocess to prevent lxml/extruct memory leaks
         try:
-            itemprop_availability = extract_itemprop_availability_safe(self.fetcher.content)
+            itemprop_availability = get_itemprop_availability(self.fetcher.content)
         except MoreThanOnePriceFound as e:
-            # Don't raise immediately - let plugins try to handle this case
-            # Plugins might be able to determine which price is correct
-            logger.warning(f"Built-in detection found multiple prices on {watch.get('url')}, will try plugin override")
-            multiple_prices_found = True
-            itemprop_availability = {}
+            # Add the real data
+            raise ProcessorException(message="Cannot run, more than one price detected, this plugin is only for product pages with ONE product, try the content-change detection mode.",
+                                     url=watch.get('url'),
+                                     status_code=self.fetcher.get_last_status_code(),
+                                     screenshot=self.fetcher.screenshot,
+                                     xpath_data=self.fetcher.xpath_data
+                                     )
 
         # If built-in extraction didn't get both price AND availability, try plugin override
         # Only check plugin if this watch is using a fetcher that might provide better data
@@ -485,21 +216,9 @@ class perform_site_check(difference_detection_processor):
             from changedetectionio.pluggy_interface import get_itemprop_availability_from_plugin
             fetcher_name = watch.get('fetch_backend', 'html_requests')
 
-            # Resolve 'system' to the actual fetcher being used
-            # This allows plugins to work even when watch uses "system settings default"
-            if fetcher_name == 'system':
-                # Get the actual fetcher that was used (from self.fetcher)
-                # Fetcher class name gives us the actual backend (e.g., 'html_requests', 'html_webdriver')
-                actual_fetcher = type(self.fetcher).__name__
-                if 'html_requests' in actual_fetcher.lower():
-                    fetcher_name = 'html_requests'
-                elif 'webdriver' in actual_fetcher.lower() or 'playwright' in actual_fetcher.lower():
-                    fetcher_name = 'html_webdriver'
-                logger.debug(f"Resolved 'system' fetcher to actual fetcher: {fetcher_name}")
-
-            # Try plugin override - plugins can decide if they support this fetcher
-            if fetcher_name:
-                logger.debug(f"Calling extra plugins for getting item price/availability (fetcher: {fetcher_name})")
+            # Only try plugin override if not using system default (which might be anything)
+            if fetcher_name and fetcher_name != 'system':
+                logger.debug("Calling extra plugins for getting item price/availability")
                 plugin_availability = get_itemprop_availability_from_plugin(self.fetcher.content, fetcher_name, self.fetcher, watch.link)
 
                 if plugin_availability:
@@ -514,16 +233,6 @@ class perform_site_check(difference_detection_processor):
                 if not plugin_availability:
                     logger.debug("No item price/availability from plugins")
 
-        # If we had multiple prices and plugins also failed, NOW raise the exception
-        if multiple_prices_found and not itemprop_availability.get('price'):
-            raise ProcessorException(
-                message="Cannot run, more than one price detected, this plugin is only for product pages with ONE product, try the content-change detection mode.",
-                url=watch.get('url'),
-                status_code=self.fetcher.get_last_status_code(),
-                screenshot=self.fetcher.screenshot,
-                xpath_data=self.fetcher.xpath_data
-            )
-
         # Something valid in get_itemprop_availability() by scraping metadata ?
         if itemprop_availability.get('price') or itemprop_availability.get('availability'):
             # Store for other usage

changedetectionio/processors/restock_diff/pure_python_extractor.py

@@ -1,286 +0,0 @@
-"""
-Pure Python metadata extractor - no lxml, no memory leaks.
-
-This module provides a fast, memory-efficient alternative to extruct for common
-e-commerce metadata extraction. It handles:
-- JSON-LD (covers 80%+ of modern sites)
-- OpenGraph meta tags
-- Basic microdata attributes
-
-Uses Python's built-in html.parser instead of lxml/libxml2, avoiding C-level
-memory allocation issues. For edge cases, the main processor can fall back to
-extruct (with subprocess isolation on Linux).
-"""
-
-from html.parser import HTMLParser
-import json
-import re
-from loguru import logger
-
-
-class JSONLDExtractor(HTMLParser):
-    """
-    Extract JSON-LD structured data from HTML.
-
-    Finds all <script type="application/ld+json"> tags and parses their content.
-    Handles multiple JSON-LD blocks on the same page.
-    """
-
-    def __init__(self):
-        super().__init__()
-        self.in_jsonld = False
-        self.data = []  # List of all parsed JSON-LD objects
-        self.current_script = []
-
-    def handle_starttag(self, tag, attrs):
-        if tag == 'script':
-            # Check if this is a JSON-LD script tag
-            for attr, value in attrs:
-                if attr == 'type' and value == 'application/ld+json':
-                    self.in_jsonld = True
-                    self.current_script = []
-                    break
-
-    def handle_data(self, data):
-        if self.in_jsonld:
-            self.current_script.append(data)
-
-    def handle_endtag(self, tag):
-        if tag == 'script' and self.in_jsonld:
-            # Parse the accumulated script content
-            script_content = ''.join(self.current_script)
-            if script_content.strip():
-                try:
-                    # Parse JSON (handles both objects and arrays)
-                    parsed = json.loads(script_content)
-                    if isinstance(parsed, list):
-                        self.data.extend(parsed)
-                    else:
-                        self.data.append(parsed)
-                except json.JSONDecodeError as e:
-                    logger.debug(f"Failed to parse JSON-LD: {e}")
-                    pass
-
-            self.in_jsonld = False
-            self.current_script = []
-
-
-class OpenGraphExtractor(HTMLParser):
-    """
-    Extract OpenGraph meta tags from HTML.
-
-    Finds <meta property="og:*"> tags commonly used for social media sharing.
-    """
-
-    def __init__(self):
-        super().__init__()
-        self.og_data = {}
-
-    def handle_starttag(self, tag, attrs):
-        if tag == 'meta':
-            attrs_dict = dict(attrs)
-            prop = attrs_dict.get('property', '')
-
-            # Extract OpenGraph properties
-            if prop.startswith('og:'):
-                content = attrs_dict.get('content', '')
-                if content:
-                    self.og_data[prop] = content
-
-
-class MicrodataExtractor(HTMLParser):
-    """
-    Extract basic microdata attributes from HTML.
-
-    Finds elements with itemprop attributes. This is a simplified extractor
-    that doesn't handle nested itemscope/itemtype hierarchies - for complex
-    cases, use extruct as fallback.
-    """
-
-    def __init__(self):
-        super().__init__()
-        self.microdata = {}
-        self.current_itemprop = None
-
-    def handle_starttag(self, tag, attrs):
-        attrs_dict = dict(attrs)
-
-        if 'itemprop' in attrs_dict:
-            itemprop = attrs_dict['itemprop']
-
-            # Price/currency/availability can be in content/href attributes
-            if itemprop == 'price':
-                if 'content' in attrs_dict:
-                    self.microdata['price'] = attrs_dict['content']
-                else:
-                    self.current_itemprop = 'price'
-
-            elif itemprop == 'priceCurrency':
-                if 'content' in attrs_dict:
-                    self.microdata['currency'] = attrs_dict['content']
-                else:
-                    self.current_itemprop = 'priceCurrency'
-
-            elif itemprop == 'availability':
-                # Can be in href (link) or content (meta)
-                if 'href' in attrs_dict:
-                    self.microdata['availability'] = attrs_dict['href']
-                elif 'content' in attrs_dict:
-                    self.microdata['availability'] = attrs_dict['content']
-                else:
-                    self.current_itemprop = 'availability'
-
-    def handle_data(self, data):
-        # Capture text content for itemprop elements
-        if self.current_itemprop == 'price':
-            # Try to extract numeric price from text
-            try:
-                price_text = re.sub(r'[^\d.]', '', data.strip())
-                if price_text:
-                    self.microdata['price'] = float(price_text)
-            except ValueError:
-                pass
-        elif self.current_itemprop == 'priceCurrency':
-            currency = data.strip()
-            if currency:
-                self.microdata['currency'] = currency
-        elif self.current_itemprop == 'availability':
-            availability = data.strip()
-            if availability:
-                self.microdata['availability'] = availability
-
-    def handle_endtag(self, tag):
-        # Reset current itemprop after closing tag
-        self.current_itemprop = None
-
-
-def extract_metadata_pure_python(html_content):
-    """
-    Extract structured metadata from HTML using pure Python parsers.
-
-    Returns a dict with three keys:
-    - 'json-ld': List of parsed JSON-LD objects
-    - 'opengraph': Dict of OpenGraph properties
-    - 'microdata': Dict of microdata properties
-
-    Args:
-        html_content: HTML string to parse
-
-    Returns:
-        dict: Extracted metadata in three formats
-    """
-    result = {
-        'json-ld': [],
-        'opengraph': {},
-        'microdata': {}
-    }
-
-    # Extract JSON-LD
-    try:
-        jsonld_extractor = JSONLDExtractor()
-        jsonld_extractor.feed(html_content)
-        result['json-ld'] = jsonld_extractor.data
-        logger.trace(f"Pure Python: Found {len(jsonld_extractor.data)} JSON-LD blocks")
-    except Exception as e:
-        logger.debug(f"JSON-LD extraction failed: {e}")
-
-    # Extract OpenGraph
-    try:
-        og_extractor = OpenGraphExtractor()
-        og_extractor.feed(html_content)
-        result['opengraph'] = og_extractor.og_data
-        if result['opengraph']:
-            logger.trace(f"Pure Python: Found {len(og_extractor.og_data)} OpenGraph tags")
-    except Exception as e:
-        logger.debug(f"OpenGraph extraction failed: {e}")
-
-    # Extract Microdata
-    try:
-        microdata_extractor = MicrodataExtractor()
-        microdata_extractor.feed(html_content)
-        result['microdata'] = microdata_extractor.microdata
-        if result['microdata']:
-            logger.trace(f"Pure Python: Found microdata: {result['microdata']}")
-    except Exception as e:
-        logger.debug(f"Microdata extraction failed: {e}")
-
-    return result
-
-
-def query_price_availability(extracted_data):
-    """
-    Query extracted metadata for price and availability information.
-
-    Uses jsonpath_ng to query JSON-LD data (same approach as extruct).
-    Falls back to OpenGraph and microdata if JSON-LD doesn't have the data.
-
-    Args:
-        extracted_data: Dict from extract_metadata_pure_python()
-
-    Returns:
-        dict: {'price': float, 'currency': str, 'availability': str}
-    """
-    from jsonpath_ng import parse
-
-    result = {}
-
-    # 1. Try JSON-LD first (most reliable and common)
-    for data in extracted_data.get('json-ld', []):
-        try:
-            # Use jsonpath to find price/availability anywhere in the structure
-            price_parse = parse('$..(price|Price)')
-            availability_parse = parse('$..(availability|Availability)')
-            currency_parse = parse('$..(priceCurrency|currency|priceCurrency)')
-
-            price_results = [m.value for m in price_parse.find(data)]
-            if price_results and not result.get('price'):
-                # Handle various price formats
-                price_val = price_results[0]
-                if isinstance(price_val, (int, float)):
-                    result['price'] = float(price_val)
-                elif isinstance(price_val, str):
-                    # Extract numeric value from string
-                    try:
-                        result['price'] = float(re.sub(r'[^\d.]', '', price_val))
-                    except ValueError:
-                        pass
-
-            avail_results = [m.value for m in availability_parse.find(data)]
-            if avail_results and not result.get('availability'):
-                result['availability'] = str(avail_results[0])
-
-            curr_results = [m.value for m in currency_parse.find(data)]
-            if curr_results and not result.get('currency'):
-                result['currency'] = str(curr_results[0])
-
-            # If we found price, this JSON-LD block is good
-            if result.get('price'):
-                logger.debug(f"Pure Python: Found price data in JSON-LD: {result}")
-                break
-
-        except Exception as e:
-            logger.debug(f"Error querying JSON-LD: {e}")
-            continue
-
-    # 2. Try OpenGraph if JSON-LD didn't provide everything
-    og_data = extracted_data.get('opengraph', {})
-    if not result.get('price') and 'og:price:amount' in og_data:
-        try:
-            result['price'] = float(og_data['og:price:amount'])
-        except ValueError:
-            pass
-    if not result.get('currency') and 'og:price:currency' in og_data:
-        result['currency'] = og_data['og:price:currency']
-    if not result.get('availability') and 'og:availability' in og_data:
-        result['availability'] = og_data['og:availability']
-
-    # 3. Use microdata as last resort
-    microdata = extracted_data.get('microdata', {})
-    if not result.get('price') and 'price' in microdata:
-        result['price'] = microdata['price']
-    if not result.get('currency') and 'currency' in microdata:
-        result['currency'] = microdata['currency']
-    if not result.get('availability') and 'availability' in microdata:
-        result['availability'] = microdata['availability']
-
-    return result

changedetectionio/processors/text_json_diff/__init__.py

@@ -1,11 +1,5 @@
-from loguru import logger
 
-# Processor capabilities
-supports_visual_selector = True
-supports_browser_steps = True
-supports_text_filters_and_triggers = True
-supports_text_filters_and_triggers_elements = True
-supports_request_type = True
+from loguru import logger
 
 
 
@@ -17,8 +11,7 @@ def _task(watch, update_handler):
 
     try:
         # The slow process (we run 2 of these in parallel)
-        # Always force reprocess for preview - we want to show the filtered content regardless of checksums
-        changed_detected, update_obj, text_after_filter = update_handler.run_changedetection(watch=watch, force_reprocess=True)
+        changed_detected, update_obj, text_after_filter = update_handler.run_changedetection(watch=watch)
     except FilterNotFoundInResponse as e:
         text_after_filter = f"Filter not found in HTML: {str(e)}"
     except ReplyWithContentButNoText as e:
@@ -56,7 +49,7 @@ def prepare_filter_prevew(datastore, watch_uuid, form_data):
 
     tmp_watch = deepcopy(datastore.data['watching'].get(watch_uuid))
 
-    if tmp_watch and tmp_watch.history and os.path.isdir(tmp_watch.data_dir):
+    if tmp_watch and tmp_watch.history and os.path.isdir(tmp_watch.watch_data_dir):
         # Splice in the temporary stuff from the form
         form = forms.processor_text_json_diff_form(formdata=form_data if request.method == 'POST' else None,
                                                    data=form_data
@@ -65,11 +58,11 @@ def prepare_filter_prevew(datastore, watch_uuid, form_data):
         # Only update vars that came in via the AJAX post
         p = {k: v for k, v in form.data.items() if k in form_data.keys()}
         tmp_watch.update(p)
-        blank_watch_no_filters = watch_model(datastore_path=datastore.datastore_path, __datastore=datastore.data)
+        blank_watch_no_filters = watch_model()
         blank_watch_no_filters['url'] = tmp_watch.get('url')
 
         latest_filename = next(reversed(tmp_watch.history))
-        html_fname = os.path.join(tmp_watch.data_dir, f"{latest_filename}.html.br")
+        html_fname = os.path.join(tmp_watch.watch_data_dir, f"{latest_filename}.html.br")
         with open(html_fname, 'rb') as f:
             decompressed_data = brotli.decompress(f.read()).decode('utf-8') if html_fname.endswith('.br') else f.read().decode('utf-8')
 

changedetectionio/processors/text_json_diff/processor.py

@@ -7,7 +7,6 @@ import re
 import urllib3
 
 from changedetectionio.conditions import execute_ruleset_against_all_plugins
-from changedetectionio.content_fetchers.exceptions import checksumFromPreviousCheckWasTheSame
 from ..base import difference_detection_processor
 from changedetectionio.html_tools import PERL_STYLE_REGEX, cdata_in_document_to_text, TRANSLATE_WHITESPACE_TABLE
 from changedetectionio import html_tools, content_fetchers
@@ -347,7 +346,6 @@ class ContentProcessor:
     def extract_text_from_html(self, html_content, stream_content_type):
         """Convert HTML to plain text."""
         do_anchor = self.datastore.data["settings"]["application"].get("render_anchor_tag_content", False)
-
         return html_tools.html_to_text(
             html_content=html_content,
             render_anchor_tag_content=do_anchor,
@@ -370,24 +368,12 @@ class ChecksumCalculator:
 # (set_proxy_from_list)
 class perform_site_check(difference_detection_processor):
 
-    def run_changedetection(self, watch, force_reprocess=False):
+    def run_changedetection(self, watch):
         changed_detected = False
 
         if not watch:
             raise Exception("Watch no longer exists.")
 
-        current_raw_document_checksum = self.get_raw_document_checksum()
-        # Skip processing only if BOTH conditions are true:
-        # 1. HTML content unchanged (checksum matches last saved checksum)
-        # 2. Watch configuration was not edited (including trigger_text, filters, etc.)
-        # The was_edited flag handles all watch configuration changes, so we don't need
-        # separate checks for trigger_text or other processing rules.
-        if (not force_reprocess and
-            not watch.was_edited and
-            self.last_raw_content_checksum and
-            self.last_raw_content_checksum == current_raw_document_checksum):
-            raise checksumFromPreviousCheckWasTheSame()
-
         # Initialize components
         filter_config = FilterConfig(watch, self.datastore)
         content_processor = ContentProcessor(self.fetcher, watch, filter_config, self.datastore)
@@ -405,11 +391,9 @@ class perform_site_check(difference_detection_processor):
         self.screenshot = self.fetcher.screenshot
         self.xpath_data = self.fetcher.xpath_data
 
-        # Track the content type (readonly field, doesn't trigger was_edited)
-        update_obj['content-type'] = ctype_header  # Use hyphen (matches OpenAPI spec and watch_base default)
-
-        # Save the raw content checksum to file (processor implementation detail, not watch config)
-        self.update_last_raw_content_checksum(current_raw_document_checksum)
+        # Track the content type and checksum before filters
+        update_obj['content_type'] = ctype_header
+        update_obj['previous_md5_before_filters'] = hashlib.md5(self.fetcher.content.encode('utf-8')).hexdigest()
 
         # === CONTENT PREPROCESSING ===
         # Avoid creating unnecessary intermediate string copies by reassigning only when needed

changedetectionio/queue_handlers.py

@@ -5,57 +5,51 @@ import heapq
 import queue
 import threading
 
-# Janus is no longer required - we use pure threading.Queue for multi-loop support
-# try:
-#     import janus
-# except ImportError:
-#     pass  # Not needed anymore
+try:
+    import janus
+except ImportError:
+    logger.critical(f"CRITICAL: janus library is required. Install with: pip install janus")
+    raise
 
 
 class RecheckPriorityQueue:
     """
-    Thread-safe priority queue supporting multiple async event loops.
-
-    ARCHITECTURE:
-    - Multiple async workers, each with its own event loop in its own thread
-    - Hybrid sync/async design for maximum scalability
-    - Sync interface for ticker thread (threading.Queue)
-    - Async interface for workers (asyncio.Event - NO executor threads!)
-
-    SCALABILITY:
-    - Scales to 100-200+ workers without executor thread exhaustion
-    - Async workers wait on asyncio.Event (pure coroutines, no threads)
-    - Sync callers use threading.Queue (backward compatible)
-
-    WHY NOT JANUS:
-    - Janus binds to ONE event loop at creation time
-    - Our architecture has 15+ workers, each with separate event loops
-    - Workers in different threads/loops cannot share janus async interface
-
-    WHY NOT RUN_IN_EXECUTOR:
-    - With 200 workers, run_in_executor() would block 200 threads
-    - Exhausts ThreadPoolExecutor, starves Flask HTTP handlers
-    - Pure async approach uses 0 threads while waiting
+    Ultra-reliable priority queue using janus for async/sync bridging.
+    
+    CRITICAL DESIGN NOTE: Both sync_q and async_q are required because:
+    - sync_q: Used by Flask routes, ticker threads, and other synchronous code
+    - async_q: Used by async workers (the actual fetchers/processors) and coroutines
+    
+    DO NOT REMOVE EITHER INTERFACE - they bridge different execution contexts:
+    - Synchronous code (Flask, threads) cannot use async methods without blocking
+    - Async code cannot use sync methods without blocking the event loop
+    - janus provides the only safe bridge between these two worlds
+    
+    Attempting to unify to async-only would require:
+    - Converting all Flask routes to async (major breaking change)
+    - Using asyncio.run() in sync contexts (causes deadlocks)
+    - Thread-pool wrapping (adds complexity and overhead)
+    
+    Minimal implementation focused on reliability:
+    - Pure janus for sync/async bridge
+    - Thread-safe priority ordering  
+    - Bulletproof error handling with critical logging
     """
-
+    
     def __init__(self, maxsize: int = 0):
         try:
-            import asyncio
-
-            # Sync interface: threading.Queue for ticker thread and Flask routes
-            self._notification_queue = queue.Queue(maxsize=maxsize if maxsize > 0 else 0)
-
+            self._janus_queue = janus.Queue(maxsize=maxsize)
+            # BOTH interfaces required - see class docstring for why
+            self.sync_q = self._janus_queue.sync_q   # Flask routes, ticker thread
+            self.async_q = self._janus_queue.async_q # Async workers
+            
             # Priority storage - thread-safe
             self._priority_items = []
             self._lock = threading.RLock()
-
-            # No event signaling needed - pure polling approach
-            # Workers check queue every 50ms (latency acceptable: 0-500ms)
-            # Scales to 1000+ workers: each sleeping worker = ~4KB coroutine, not thread
-
+            
             # Signals for UI updates
             self.queue_length_signal = signal('queue_length')
-
+            
             logger.debug("RecheckPriorityQueue initialized successfully")
         except Exception as e:
             logger.critical(f"CRITICAL: Failed to initialize RecheckPriorityQueue: {str(e)}")
@@ -64,48 +58,38 @@ class RecheckPriorityQueue:
     # SYNC INTERFACE (for ticker thread)
     def put(self, item, block: bool = True, timeout: Optional[float] = None):
         """Thread-safe sync put with priority ordering"""
-        logger.trace(f"RecheckQueue.put() called for item: {self._get_item_uuid(item)}, block={block}, timeout={timeout}")
         try:
-            # CRITICAL: Add to both priority storage AND notification queue atomically
-            # to prevent desynchronization where item exists but no notification
+            # Add to priority storage
             with self._lock:
                 heapq.heappush(self._priority_items, item)
-
-                # Add notification - use blocking with timeout for safety
-                # Notification queue is unlimited size, so should never block in practice
-                # but timeout ensures we detect any unexpected issues (deadlock, etc)
-                try:
-                    self._notification_queue.put(True, block=True, timeout=5.0)
-                except Exception as notif_e:
-                    # Notification failed - MUST remove from priority_items to keep in sync
-                    # This prevents "Priority queue inconsistency" errors in get()
-                    logger.critical(f"CRITICAL: Notification queue put failed, removing from priority_items: {notif_e}")
-                    self._priority_items.remove(item)
-                    heapq.heapify(self._priority_items)
-                    raise  # Re-raise to be caught by outer exception handler
-
-            # Signal emission after successful queue - log but don't fail the operation
-            # Item is already safely queued, so signal failure shouldn't affect queue state
-            try:
-                self._emit_put_signals(item)
-            except Exception as signal_e:
-                logger.error(f"Failed to emit put signals but item queued successfully: {signal_e}")
-
-            logger.trace(f"Successfully queued item: {self._get_item_uuid(item)}")
+            
+            # Notify via janus sync queue
+            self.sync_q.put(True, block=block, timeout=timeout)
+            
+            # Emit signals
+            self._emit_put_signals(item)
+            
+            logger.debug(f"Successfully queued item: {self._get_item_uuid(item)}")
             return True
-
+            
         except Exception as e:
-            logger.critical(f"CRITICAL: Failed to put item {self._get_item_uuid(item)}: {type(e).__name__}: {str(e)}")
-            # Item should have been cleaned up in the inner try/except if notification failed
+            logger.critical(f"CRITICAL: Failed to put item {self._get_item_uuid(item)}: {str(e)}")
+            # Remove from priority storage if janus put failed
+            try:
+                with self._lock:
+                    if item in self._priority_items:
+                        self._priority_items.remove(item)
+                        heapq.heapify(self._priority_items)
+            except Exception as cleanup_e:
+                logger.critical(f"CRITICAL: Failed to cleanup after put failure: {str(e)}")
             return False
     
     def get(self, block: bool = True, timeout: Optional[float] = None):
         """Thread-safe sync get with priority ordering"""
-        logger.trace(f"RecheckQueue.get() called, block={block}, timeout={timeout}")
-        import queue as queue_module
+        import queue
         try:
-            # Wait for notification (this doesn't return the actual item, just signals availability)
-            self._notification_queue.get(block=block, timeout=timeout)
+            # Wait for notification
+            self.sync_q.get(block=block, timeout=timeout)
 
             # Get highest priority item
             with self._lock:
@@ -114,91 +98,69 @@ class RecheckPriorityQueue:
                     raise Exception("Priority queue inconsistency")
                 item = heapq.heappop(self._priority_items)
 
-            # Signal emission after successful retrieval - log but don't lose the item
-            # Item is already retrieved, so signal failure shouldn't affect queue state
-            try:
-                self._emit_get_signals()
-            except Exception as signal_e:
-                logger.error(f"Failed to emit get signals but item retrieved successfully: {signal_e}")
+            # Emit signals
+            self._emit_get_signals()
 
-            logger.trace(f"RecheckQueue.get() successfully retrieved item: {self._get_item_uuid(item)}")
-            return item
-
-        except queue_module.Empty:
-            # Queue is empty with timeout - expected behavior
-            logger.trace(f"RecheckQueue.get() timed out - queue is empty (timeout={timeout})")
-            raise  # noqa
-        except Exception as e:
-            # Re-raise without logging - caller (worker) will handle and log appropriately
-            logger.trace(f"RecheckQueue.get() failed with exception: {type(e).__name__}: {str(e)}")
-            raise
-    
-    # ASYNC INTERFACE (for workers)
-    async def async_put(self, item, executor=None):
-        """Async put with priority ordering - uses thread pool to avoid blocking
-
-        Args:
-            item: Item to add to queue
-            executor: Optional ThreadPoolExecutor. If None, uses default pool.
-        """
-        logger.trace(f"RecheckQueue.async_put() called for item: {self._get_item_uuid(item)}, executor={executor}")
-        import asyncio
-        try:
-            # Use run_in_executor to call sync put without blocking event loop
-            loop = asyncio.get_event_loop()
-            result = await loop.run_in_executor(
-                executor,  # Use provided executor or default
-                lambda: self.put(item, block=True, timeout=5.0)
-            )
-
-            logger.trace(f"RecheckQueue.async_put() successfully queued item: {self._get_item_uuid(item)}")
-            return result
-
-        except Exception as e:
-            logger.critical(f"CRITICAL: Failed to async put item {self._get_item_uuid(item)}: {str(e)}")
-            return False
-
-    async def async_get(self, executor=None, timeout=1.0):
-        """
-        Efficient async get using executor for blocking call.
-
-        HYBRID APPROACH: Best of both worlds
-        - Uses run_in_executor for efficient blocking (no polling overhead)
-        - Single timeout (no double-timeout race condition)
-        - Scales well: executor sized to match worker count
-
-        With FETCH_WORKERS=10: 10 threads blocked max (acceptable)
-        With FETCH_WORKERS=200: Need executor with 200+ threads (see worker_pool.py)
-
-        Args:
-            executor: ThreadPoolExecutor (sized to match worker count)
-            timeout: Maximum time to wait in seconds
-
-        Returns:
-            Item from queue
-
-        Raises:
-            queue.Empty: If timeout expires with no item available
-        """
-        logger.trace(f"RecheckQueue.async_get() called, timeout={timeout}")
-        import asyncio
-        try:
-            # Use run_in_executor to call sync get efficiently
-            # No outer asyncio.wait_for wrapper = no double timeout issue!
-            loop = asyncio.get_event_loop()
-            item = await loop.run_in_executor(
-                executor,
-                lambda: self.get(block=True, timeout=timeout)
-            )
-
-            logger.trace(f"RecheckQueue.async_get() successfully retrieved item: {self._get_item_uuid(item)}")
+            logger.debug(f"Successfully retrieved item: {self._get_item_uuid(item)}")
             return item
 
         except queue.Empty:
-            logger.trace(f"RecheckQueue.async_get() timed out - queue is empty")
+            # Queue is empty with timeout - expected behavior, re-raise without logging
             raise
         except Exception as e:
-            logger.critical(f"CRITICAL: Failed to async get item from queue: {type(e).__name__}: {str(e)}")
+            # Re-raise without logging - caller (worker) will handle and log appropriately
+            raise
+    
+    # ASYNC INTERFACE (for workers)
+    async def async_put(self, item):
+        """Pure async put with priority ordering"""
+        try:
+            # Add to priority storage
+            with self._lock:
+                heapq.heappush(self._priority_items, item)
+            
+            # Notify via janus async queue
+            await self.async_q.put(True)
+            
+            # Emit signals
+            self._emit_put_signals(item)
+            
+            logger.debug(f"Successfully async queued item: {self._get_item_uuid(item)}")
+            return True
+            
+        except Exception as e:
+            logger.critical(f"CRITICAL: Failed to async put item {self._get_item_uuid(item)}: {str(e)}")
+            # Remove from priority storage if janus put failed
+            try:
+                with self._lock:
+                    if item in self._priority_items:
+                        self._priority_items.remove(item)
+                        heapq.heapify(self._priority_items)
+            except Exception as cleanup_e:
+                logger.critical(f"CRITICAL: Failed to cleanup after async put failure: {str(e)}")
+            return False
+    
+    async def async_get(self):
+        """Pure async get with priority ordering"""
+        try:
+            # Wait for notification
+            await self.async_q.get()
+
+            # Get highest priority item
+            with self._lock:
+                if not self._priority_items:
+                    logger.critical(f"CRITICAL: Async queue notification received but no priority items available")
+                    raise Exception("Priority queue inconsistency")
+                item = heapq.heappop(self._priority_items)
+
+            # Emit signals
+            self._emit_get_signals()
+
+            logger.debug(f"Successfully async retrieved item: {self._get_item_uuid(item)}")
+            return item
+
+        except Exception as e:
+            logger.critical(f"CRITICAL: Failed to async get item from queue: {str(e)}")
             raise
     
     # UTILITY METHODS
@@ -224,35 +186,10 @@ class RecheckPriorityQueue:
             logger.critical(f"CRITICAL: Failed to get queued UUIDs: {str(e)}")
             return []
 
-    def clear(self):
-        """Clear all items from both priority storage and notification queue"""
-        try:
-            with self._lock:
-                # Clear priority items
-                self._priority_items.clear()
-
-                # Drain all notifications to prevent stale notifications
-                # This is critical for test cleanup to prevent queue desynchronization
-                drained = 0
-                while not self._notification_queue.empty():
-                    try:
-                        self._notification_queue.get_nowait()
-                        drained += 1
-                    except queue.Empty:
-                        break
-
-                if drained > 0:
-                    logger.debug(f"Cleared queue: removed {drained} notifications")
-
-            return True
-        except Exception as e:
-            logger.critical(f"CRITICAL: Failed to clear queue: {str(e)}")
-            return False
-
     def close(self):
-        """Close the queue"""
+        """Close the janus queue"""
         try:
-            # Nothing to close for threading.Queue
+            self._janus_queue.close()
             logger.debug("RecheckPriorityQueue closed successfully")
         except Exception as e:
             logger.critical(f"CRITICAL: Failed to close RecheckPriorityQueue: {str(e)}")
@@ -384,7 +321,7 @@ class RecheckPriorityQueue:
         except Exception:
             pass
         return 'unknown'
-
+    
     def _emit_put_signals(self, item):
         """Emit signals when item is added"""
         try:
@@ -393,14 +330,14 @@ class RecheckPriorityQueue:
                 watch_check_update = signal('watch_check_update')
                 if watch_check_update:
                     watch_check_update.send(watch_uuid=item.item['uuid'])
-
+            
             # Queue length signal
             if self.queue_length_signal:
                 self.queue_length_signal.send(length=self.qsize())
-
+                
         except Exception as e:
             logger.critical(f"CRITICAL: Failed to emit put signals: {str(e)}")
-
+    
     def _emit_get_signals(self):
         """Emit signals when item is removed"""
         try:
@@ -424,115 +361,76 @@ class NotificationQueue:
     Simple wrapper around janus with bulletproof error handling.
     """
     
-    def __init__(self, maxsize: int = 0, datastore=None):
+    def __init__(self, maxsize: int = 0):
         try:
-            # Use pure threading.Queue to avoid event loop binding issues
-            self._notification_queue = queue.Queue(maxsize=maxsize if maxsize > 0 else 0)
+            self._janus_queue = janus.Queue(maxsize=maxsize)
+            # BOTH interfaces required - see class docstring for why
+            self.sync_q = self._janus_queue.sync_q   # Flask routes, threads
+            self.async_q = self._janus_queue.async_q # Async workers
             self.notification_event_signal = signal('notification_event')
-            self.datastore = datastore  # For checking all_muted setting
-            self._lock = threading.RLock()
             logger.debug("NotificationQueue initialized successfully")
         except Exception as e:
             logger.critical(f"CRITICAL: Failed to initialize NotificationQueue: {str(e)}")
             raise
-
-    def set_datastore(self, datastore):
-        """Set datastore reference after initialization (for circular dependency handling)"""
-        self.datastore = datastore
     
     def put(self, item: Dict[str, Any], block: bool = True, timeout: Optional[float] = None):
         """Thread-safe sync put with signal emission"""
-        logger.trace(f"NotificationQueue.put() called for item: {item.get('uuid', 'unknown')}, block={block}, timeout={timeout}")
         try:
-            # Check if all notifications are muted
-            if self.datastore and self.datastore.data['settings']['application'].get('all_muted', False):
-                logger.debug(f"Notification blocked - all notifications are muted: {item.get('uuid', 'unknown')}")
-                return False
-
-            with self._lock:
-                self._notification_queue.put(item, block=block, timeout=timeout)
+            self.sync_q.put(item, block=block, timeout=timeout)
             self._emit_notification_signal(item)
-            logger.trace(f"NotificationQueue.put() successfully queued notification: {item.get('uuid', 'unknown')}")
+            logger.debug(f"Successfully queued notification: {item.get('uuid', 'unknown')}")
             return True
         except Exception as e:
             logger.critical(f"CRITICAL: Failed to put notification {item.get('uuid', 'unknown')}: {str(e)}")
             return False
     
-    async def async_put(self, item: Dict[str, Any], executor=None):
-        """Async put with signal emission - uses thread pool
-
-        Args:
-            item: Notification item to queue
-            executor: Optional ThreadPoolExecutor
-        """
-        logger.trace(f"NotificationQueue.async_put() called for item: {item.get('uuid', 'unknown')}, executor={executor}")
-        import asyncio
+    async def async_put(self, item: Dict[str, Any]):
+        """Pure async put with signal emission"""
         try:
-            # Check if all notifications are muted
-            if self.datastore and self.datastore.data['settings']['application'].get('all_muted', False):
-                logger.debug(f"Notification blocked - all notifications are muted: {item.get('uuid', 'unknown')}")
-                return False
-
-            loop = asyncio.get_event_loop()
-            await loop.run_in_executor(executor, lambda: self.put(item, block=True, timeout=5.0))
-            logger.trace(f"NotificationQueue.async_put() successfully queued notification: {item.get('uuid', 'unknown')}")
+            await self.async_q.put(item)
+            self._emit_notification_signal(item)
+            logger.debug(f"Successfully async queued notification: {item.get('uuid', 'unknown')}")
             return True
         except Exception as e:
             logger.critical(f"CRITICAL: Failed to async put notification {item.get('uuid', 'unknown')}: {str(e)}")
             return False
-
+    
     def get(self, block: bool = True, timeout: Optional[float] = None):
         """Thread-safe sync get"""
-        logger.trace(f"NotificationQueue.get() called, block={block}, timeout={timeout}")
         try:
-            with self._lock:
-                item = self._notification_queue.get(block=block, timeout=timeout)
-            logger.trace(f"NotificationQueue.get() retrieved item: {item.get('uuid', 'unknown') if isinstance(item, dict) else 'unknown'}")
-            return item
+            return self.sync_q.get(block=block, timeout=timeout)
         except queue.Empty as e:
-            logger.trace(f"NotificationQueue.get() timed out - queue is empty (timeout={timeout})")
             raise e
         except Exception as e:
-            logger.critical(f"CRITICAL: Failed to get notification: {type(e).__name__}: {str(e)}")
+            logger.critical(f"CRITICAL: Failed to get notification: {str(e)}")
             raise e
-
-    async def async_get(self, executor=None):
-        """Async get - uses thread pool
-
-        Args:
-            executor: Optional ThreadPoolExecutor
-        """
-        logger.trace(f"NotificationQueue.async_get() called, executor={executor}")
-        import asyncio
+    
+    async def async_get(self):
+        """Pure async get"""
         try:
-            loop = asyncio.get_event_loop()
-            item = await loop.run_in_executor(executor, lambda: self.get(block=True, timeout=1.0))
-            logger.trace(f"NotificationQueue.async_get() retrieved item: {item.get('uuid', 'unknown') if isinstance(item, dict) else 'unknown'}")
-            return item
+            return await self.async_q.get()
         except queue.Empty as e:
-            logger.trace(f"NotificationQueue.async_get() timed out - queue is empty")
             raise e
         except Exception as e:
-            logger.critical(f"CRITICAL: Failed to async get notification: {type(e).__name__}: {str(e)}")
+            logger.critical(f"CRITICAL: Failed to async get notification: {str(e)}")
             raise e
     
     def qsize(self) -> int:
         """Get current queue size"""
         try:
-            with self._lock:
-                return self._notification_queue.qsize()
+            return self.sync_q.qsize()
         except Exception as e:
             logger.critical(f"CRITICAL: Failed to get notification queue size: {str(e)}")
             return 0
-
+    
     def empty(self) -> bool:
         """Check if queue is empty"""
         return self.qsize() == 0
-
+    
     def close(self):
-        """Close the queue"""
+        """Close the janus queue"""
         try:
-            # Nothing to close for threading.Queue
+            self._janus_queue.close()
             logger.debug("NotificationQueue closed successfully")
         except Exception as e:
             logger.critical(f"CRITICAL: Failed to close NotificationQueue: {str(e)}")

changedetectionio/realtime/events.py

@@ -37,9 +37,9 @@ def register_watch_operation_handlers(socketio, datastore):
                 # Import here to avoid circular imports
                 from changedetectionio.flask_app import update_q
                 from changedetectionio import queuedWatchMetaData
-                from changedetectionio import worker_pool
+                from changedetectionio import worker_handler
                 
-                worker_pool.queue_item_async_safe(update_q, queuedWatchMetaData.PrioritizedItem(priority=1, item={'uuid': uuid}))
+                worker_handler.queue_item_async_safe(update_q, queuedWatchMetaData.PrioritizedItem(priority=1, item={'uuid': uuid}))
                 logger.info(f"Socket.IO: Queued recheck for watch {uuid}")
             else:
                 emit('operation_result', {'success': False, 'error': f'Unknown operation: {op}'})

changedetectionio/realtime/socket_server.py

@@ -145,10 +145,10 @@ def handle_watch_update(socketio, **kwargs):
         # Emit the watch update to all connected clients
         from changedetectionio.flask_app import update_q
         from changedetectionio.flask_app import _jinja2_filter_datetime
-        from changedetectionio import worker_pool
+        from changedetectionio import worker_handler
 
         # Get list of watches that are currently running
-        running_uuids = worker_pool.get_running_uuids()
+        running_uuids = worker_handler.get_running_uuids()
 
         # Get list of watches in the queue (efficient single-lock method)
         queue_list = update_q.get_queued_uuids()
@@ -198,6 +198,7 @@ def handle_watch_update(socketio, **kwargs):
     except Exception as e:
         logger.error(f"Socket.IO error in handle_watch_update: {str(e)}")
 
+
 def init_socketio(app, datastore):
     """Initialize SocketIO with the main Flask app"""
     import platform
@@ -239,10 +240,7 @@ def init_socketio(app, datastore):
                         async_mode=async_mode,
                         cors_allowed_origins=cors_origins,  # None means same-origin only
                         logger=strtobool(os.getenv('SOCKETIO_LOGGING', 'False')),
-                        engineio_logger=strtobool(os.getenv('SOCKETIO_LOGGING', 'False')),
-                        # Disable WebSocket compression to prevent memory accumulation
-                        # Flask-Compress already handles HTTP response compression
-                        engineio_options={'http_compression': False, 'compression_threshold': 0})
+                        engineio_logger=strtobool(os.getenv('SOCKETIO_LOGGING', 'False')))
 
     # Set up event handlers
     logger.info("Socket.IO: Registering connect event handler")
@@ -251,34 +249,23 @@ def init_socketio(app, datastore):
     def event_checkbox_operations(data):
         from changedetectionio.blueprint.ui import _handle_operations
         from changedetectionio import queuedWatchMetaData
-        from changedetectionio import worker_pool
+        from changedetectionio import worker_handler
         from changedetectionio.flask_app import update_q, watch_check_update
-        import threading
-
         logger.trace(f"Got checkbox operations event: {data}")
 
         datastore = socketio.datastore
 
-        def run_operation():
-            """Run the operation in a background thread to avoid blocking the socket.io event loop"""
-            try:
-                _handle_operations(
-                    op=data.get('op'),
-                    uuids=data.get('uuids'),
-                    datastore=datastore,
-                    extra_data=data.get('extra_data'),
-                    worker_pool=worker_pool,
-                    update_q=update_q,
-                    queuedWatchMetaData=queuedWatchMetaData,
-                    watch_check_update=watch_check_update,
-                    emit_flash=False
-                )
-            except Exception as e:
-                logger.error(f"Error in checkbox operation thread: {e}")
-
-        # Start operation in a disposable daemon thread
-        thread = threading.Thread(target=run_operation, daemon=True, name=f"checkbox-op-{data.get('op')}")
-        thread.start()
+        _handle_operations(
+            op=data.get('op'),
+            uuids=data.get('uuids'),
+            datastore=datastore,
+            extra_data=data.get('extra_data'),
+            worker_handler=worker_handler,
+            update_q=update_q,
+            queuedWatchMetaData=queuedWatchMetaData,
+            watch_check_update=watch_check_update,
+            emit_flash=False
+        )
 
     @socketio.on('connect')
     def handle_connect():

changedetectionio/run_basic_tests.sh

@@ -10,7 +10,6 @@
 set -e
 
 SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )
-rm tests/logs/* -f
 
 # Since theres no curl installed lets roll with python3
 check_sanity() {
@@ -65,21 +64,18 @@ data_sanity_test
 echo "-------------------- Running rest of tests in parallel -------------------------------"
 
 # REMOVE_REQUESTS_OLD_SCREENSHOTS disabled so that we can write a screenshot and send it in test_notifications.py without a real browser
-FETCH_WORKERS=2 REMOVE_REQUESTS_OLD_SCREENSHOTS=false \
+REMOVE_REQUESTS_OLD_SCREENSHOTS=false \
 pytest tests/test_*.py \
-  -n 8 \
+  -n 30 \
   --dist=load \
   -vvv \
   -s \
   --capture=no \
-  -k "not test_queue_system" \
   --log-cli-level=DEBUG \
   --log-cli-format="%(asctime)s [%(process)d] [%(levelname)s] %(name)s: %(message)s"
 
 echo "---------------------------- DONE parallel test ---------------------------------------"
 
-FETCH_WORKERS=20 pytest -vvv -s tests/test_queue_handler.py
-
 echo "RUNNING WITH BASE_URL SET"
 
 # Now re-run some tests with BASE_URL enabled
@@ -95,8 +91,8 @@ REMOVE_REQUESTS_OLD_SCREENSHOTS=false pytest -vv -s --maxfail=1 tests/test_notif
 # And again with brotli+screenshot attachment
 SNAPSHOT_BROTLI_COMPRESSION_THRESHOLD=5 REMOVE_REQUESTS_OLD_SCREENSHOTS=false pytest -vv -s --maxfail=1 --dist=load tests/test_backend.py tests/test_rss.py tests/test_unique_lines.py tests/test_notification.py  tests/test_access_control.py
 
-# Try high concurrency with aggressive worker restarts
-FETCH_WORKERS=50 WORKER_MAX_RUNTIME=2 WORKER_MAX_JOBS=1 pytest  tests/test_history_consistency.py -vv -l -s
+# Try high concurrency
+FETCH_WORKERS=50 pytest  tests/test_history_consistency.py -vv -l -s
 
 # Check file:// will pickup a file when enabled
 echo "Hello world" > /tmp/test-file.txt

changedetectionio/static/favicons/site.webmanifest

@@ -1,25 +1,19 @@
 {
-    "name": "ChangeDetection.io",
-    "short_name": "ChangeDetect",
-    "description": "Self-hosted website change detection and monitoring",
+    "name": "",
+    "short_name": "",
     "icons": [
         {
             "src": "android-chrome-192x192.png",
             "sizes": "192x192",
-            "type": "image/png",
-            "purpose": "any maskable"
+            "type": "image/png"
         },
         {
             "src": "android-chrome-256x256.png",
             "sizes": "256x256",
-            "type": "image/png",
-            "purpose": "any maskable"
+            "type": "image/png"
         }
     ],
-    "start_url": "/",
-    "theme_color": "#5bbad5",
+    "theme_color": "#ffffff",
     "background_color": "#ffffff",
-    "display": "standalone",
-    "categories": ["utilities", "productivity"],
-    "orientation": "any"
+    "display": "standalone"
 }

changedetectionio/static/js/browser-steps.js

@@ -17,6 +17,8 @@ $(document).ready(function () {
         set_scale();
     });
     // Should always be disabled
+    $('#browser_steps-0-operation option[value="Goto site"]').prop("selected", "selected");
+    $('#browser_steps-0-operation').attr('disabled', 'disabled');
 
     $('#browsersteps-click-start').click(function () {
         $("#browsersteps-click-start").fadeOut();
@@ -43,6 +45,12 @@ $(document).ready(function () {
         browsersteps_session_id = false;
         apply_buttons_disabled = false;
         ctx.clearRect(0, 0, c.width, c.height);
+        set_first_gotosite_disabled();
+    }
+
+    function set_first_gotosite_disabled() {
+        $('#browser_steps >li:first-child select').val('Goto site').attr('disabled', 'disabled');
+        $('#browser_steps >li:first-child').css('opacity', '0.5');
     }
 
     // Show seconds remaining until the browser interface needs to restart the session
@@ -235,54 +243,14 @@ $(document).ready(function () {
         ctx.fill();
     }
 
-    // Reusable AJAX function for browser step operations
-    function executeBrowserStep(url, data = {}) {
-        $('#browser-steps-ui .loader .spinner').fadeIn();
-        apply_buttons_disabled = true;
-        $('ul#browser_steps li .control .apply').css('opacity', 0.5);
-        $("#browsersteps-img").css('opacity', 0.65);
-
-        return $.ajax({
-            method: "POST",
-            url: url,
-            data: data,
-            statusCode: {
-                400: function () {
-                    alert("There was a problem processing the request, please reload the page.");
-                    $("#loading-status-text").hide();
-                    $('#browser-steps-ui .loader .spinner').fadeOut();
-                },
-                401: function (data) {
-                    alert(data.responseText);
-                    $("#loading-status-text").hide();
-                    $('#browser-steps-ui .loader .spinner').fadeOut();
-                }
-            }
-        }).done(function (data) {
-            xpath_data = data.xpath_data;
-            $('#browsersteps-img').attr('src', data.screenshot);
-            $('#browser-steps-ui .loader .spinner').fadeOut();
-            apply_buttons_disabled = false;
-            $("#browsersteps-img").css('opacity', 1);
-            $('ul#browser_steps li .control .apply').css('opacity', 1);
-            $("#loading-status-text").hide();
-        }).fail(function (data) {
-            console.log(data);
-            if (data.responseText && data.responseText.includes("Browser session expired")) {
-                disable_browsersteps_ui();
-            }
-            apply_buttons_disabled = false;
-            $("#loading-status-text").hide();
-            $('ul#browser_steps li .control .apply').css('opacity', 1);
-            $("#browsersteps-img").css('opacity', 1);
-        });
-    }
-
     function start() {
         console.log("Starting browser-steps UI");
         browsersteps_session_id = false;
+        // @todo This setting of the first one should be done at the datalayer but wtforms doesnt wanna play nice
+        $('#browser_steps >li:first-child').removeClass('empty');
+        set_first_gotosite_disabled();
         $('#browser-steps-ui .loader .spinner').show();
-        // Request a new session
+        $('.clear,.remove', $('#browser_steps >li:first-child')).hide();
         $.ajax({
             type: "GET",
             url: browser_steps_start_url,
@@ -299,12 +267,11 @@ $(document).ready(function () {
         }).done(function (data) {
             $("#loading-status-text").fadeIn();
             browsersteps_session_id = data.browsersteps_session_id;
+            // This should trigger 'Goto site'
+            console.log("Got startup response, requesting Goto-Site (first) step fake click");
+            $('#browser_steps >li:first-child .apply').click();
             browser_interface_seconds_remaining = 500;
-            // Request goto_site operation
-            executeBrowserStep(
-                browser_steps_sync_url + "&browsersteps_session_id=" + browsersteps_session_id + "&goto_website_url_first_step=true"
-            );
-
+            set_first_gotosite_disabled();
         }).fail(function (data) {
             console.log(data);
             alert('There was an error communicating with the server.');
@@ -313,6 +280,7 @@ $(document).ready(function () {
     }
 
     function disable_browsersteps_ui() {
+        set_first_gotosite_disabled();
         $("#browser-steps-ui").css('opacity', '0.3');
         $('#browsersteps-selector-canvas').off("mousemove mousedown click");
     }
@@ -360,13 +328,16 @@ $(document).ready(function () {
     // Add the extra buttons to the steps
     $('ul#browser_steps li').each(function (i) {
             var s = '<div class="control">' + '<a data-step-index=' + i + ' class="pure-button button-secondary button-green button-xsmall apply" >Apply</a>&nbsp;';
-            s += `<a data-step-index="${i}" class="pure-button button-secondary button-xsmall clear" >Clear</a>&nbsp;` +
-                `<a data-step-index="${i}" class="pure-button button-secondary button-red button-xsmall remove" >Remove</a>`;
+            if (i > 0) {
+                // The first step never gets these (Goto-site)
+                s += `<a data-step-index="${i}" class="pure-button button-secondary button-xsmall clear" >Clear</a>&nbsp;` +
+                    `<a data-step-index="${i}" class="pure-button button-secondary button-red button-xsmall remove" >Remove</a>`;
 
-            // if a screenshot is available
-            if (browser_steps_available_screenshots.includes(i.toString())) {
-                var d = (browser_steps_last_error_step === i+1) ? 'before' : 'after';
-                s += `&nbsp;<a data-step-index="${i}" class="pure-button button-secondary button-xsmall show-screenshot" title="Show screenshot from last run" data-type="${d}">Pic</a>&nbsp;`;
+                // if a screenshot is available
+                if (browser_steps_available_screenshots.includes(i.toString())) {
+                    var d = (browser_steps_last_error_step === i+1) ? 'before' : 'after';
+                    s += `&nbsp;<a data-step-index="${i}" class="pure-button button-secondary button-xsmall show-screenshot" title="Show screenshot from last run" data-type="${d}">Pic</a>&nbsp;`;
+                }
             }
             s += '</div>';
             $(this).append(s)
@@ -405,35 +376,80 @@ $(document).ready(function () {
     });
 
     $('ul#browser_steps li .control .apply').click(function (event) {
+        // sequential requests @todo refactor
         if (apply_buttons_disabled) {
             return;
         }
 
         var current_data = $(event.currentTarget).closest('li');
+        $('#browser-steps-ui .loader .spinner').fadeIn();
+        apply_buttons_disabled = true;
+        $('ul#browser_steps li .control .apply').css('opacity', 0.5);
+        $("#browsersteps-img").css('opacity', 0.65);
+
+        var is_last_step = 0;
         var step_n = $(event.currentTarget).data('step-index');
 
-        // Determine if this is the last configured step
-        var is_last_step = 0;
+        // On the last step, we should also be getting data ready for the visual selector
         $('ul#browser_steps li select').each(function (i) {
             if ($(this).val() !== 'Choose one') {
                 is_last_step += 1;
             }
         });
-        is_last_step = (is_last_step == (step_n + 1));
+
+        if (is_last_step == (step_n + 1)) {
+            is_last_step = true;
+        } else {
+            is_last_step = false;
+        }
 
         console.log("Requesting step via POST " + $("select[id$='operation']", current_data).first().val());
-
-        // Execute the browser step
-        executeBrowserStep(
-            browser_steps_sync_url + "&browsersteps_session_id=" + browsersteps_session_id,
-            {
+        // POST the currently clicked step form widget back and await response, redraw
+        $.ajax({
+            method: "POST",
+            url: browser_steps_sync_url + "&browsersteps_session_id=" + browsersteps_session_id,
+            data: {
                 'operation': $("select[id$='operation']", current_data).first().val(),
                 'selector': $("input[id$='selector']", current_data).first().val(),
                 'optional_value': $("input[id$='optional_value']", current_data).first().val(),
                 'step_n': step_n,
                 'is_last_step': is_last_step
+            },
+            statusCode: {
+                400: function () {
+                    // More than likely the CSRF token was lost when the server restarted
+                    alert("There was a problem processing the request, please reload the page.");
+                    $("#loading-status-text").hide();
+                    $('#browser-steps-ui .loader .spinner').fadeOut();
+                },
+                401: function (data) {
+                    // More than likely the CSRF token was lost when the server restarted
+                    alert(data.responseText);
+                    $("#loading-status-text").hide();
+                    $('#browser-steps-ui .loader .spinner').fadeOut();
+                }
             }
-        );
+        }).done(function (data) {
+            // it should return the new state (selectors available and screenshot)
+            xpath_data = data.xpath_data;
+            $('#browsersteps-img').attr('src', data.screenshot);
+            $('#browser-steps-ui .loader .spinner').fadeOut();
+            apply_buttons_disabled = false;
+            $("#browsersteps-img").css('opacity', 1);
+            $('ul#browser_steps li .control .apply').css('opacity', 1);
+            $("#loading-status-text").hide();
+            set_first_gotosite_disabled();
+        }).fail(function (data) {
+            console.log(data);
+            if (data.responseText.includes("Browser session expired")) {
+                disable_browsersteps_ui();
+            }
+            apply_buttons_disabled = false;
+            $("#loading-status-text").hide();
+            $('ul#browser_steps li .control .apply').css('opacity', 1);
+            $("#browsersteps-img").css('opacity', 1);
+        });
+
     });
 
     $('ul#browser_steps li .control .show-screenshot').click(function (element) {

changedetectionio/static/js/modal.js

@@ -184,8 +184,7 @@ $(document).ready(function() {
         }
         // If it's a button in a form, submit the form
         else if ($element.is('button')) {
-          // Use requestSubmit() to include the button's name/value in the form data
-          $element.closest('form')[0].requestSubmit($element[0]);
+          $element.closest('form').submit();
         }
       }
     };

changedetectionio/static/js/search-modal.js

@@ -69,19 +69,6 @@
       }
     });
 
-    // Handle Enter key in search input
-    if (searchInput) {
-      searchInput.addEventListener('keydown', function(e) {
-        if (e.key === 'Enter') {
-          e.preventDefault();
-          if (searchForm) {
-            // Trigger form submission programmatically
-            searchForm.dispatchEvent(new Event('submit'));
-          }
-        }
-      });
-    }
-
     // Handle form submission
     if (searchForm) {
       searchForm.addEventListener('submit', function(e) {
@@ -101,10 +88,8 @@
           params.append('tags', tags);
         }
 
-        // Navigate to search results (always redirect to watchlist home)
-        // Use base_path if available (for sub-path deployments like /enlighten-richerx)
-        const basePath = typeof base_path !== 'undefined' ? base_path : '';
-        window.location.href = basePath + '/?' + params.toString();
+        // Navigate to search results
+        window.location.href = '?' + params.toString();
       });
     }
   });

changedetectionio/static/styles/diff-image.css

@@ -1 +1 @@
-.comparison-score{padding:1em;background:var(--color-table-stripe);border-radius:4px;margin:1em 0;border:1px solid var(--color-border-table-cell);color:var(--color-text)}.change-detected{color:#d32f2f;font-weight:bold}.no-change{color:#388e3c;font-weight:bold}.comparison-grid{display:grid;grid-template-columns:1fr 1fr;gap:1em;margin:1em 1em}@media(max-width: 1200px){.comparison-grid{grid-template-columns:1fr}}.image-comparison{position:relative;width:100%;overflow:hidden;border:1px solid var(--color-border-table-cell);box-shadow:0 2px 4px rgba(0,0,0,.1);user-select:none}.image-comparison img{display:block;width:100%;height:auto;max-width:100%;border:none;box-shadow:none}.comparison-image-wrapper{position:relative;width:100%;display:flex;align-items:flex-start;justify-content:center;background-color:var(--color-background);background-image:linear-gradient(45deg, var(--color-table-stripe) 25%, transparent 25%),linear-gradient(-45deg, var(--color-table-stripe) 25%, transparent 25%),linear-gradient(45deg, transparent 75%, var(--color-table-stripe) 75%),linear-gradient(-45deg, transparent 75%, var(--color-table-stripe) 75%);background-size:20px 20px;background-position:0 0,0 10px,10px -10px,-10px 0px}.comparison-after{position:absolute;top:0;left:0;width:100%;height:100%;clip-path:inset(0 0 0 50%)}.comparison-slider{position:absolute;top:0;left:50%;width:4px;height:100%;background:#0078e7;cursor:ew-resize;transform:translateX(-2px);z-index:10}.comparison-handle{position:absolute;top:50%;left:50%;width:48px;height:48px;background:#0078e7;border:3px solid #fff;border-radius:50%;transform:translate(-50%, -50%);box-shadow:0 2px 8px rgba(0,0,0,.3);display:flex;align-items:center;justify-content:center;cursor:ew-resize;transition:top .1s ease-out}.comparison-handle::after{content:"⇄";color:#fff;font-size:24px;font-weight:bold;pointer-events:none}.comparison-labels{position:absolute;top:10px;width:100%;display:flex;justify-content:space-between;padding:0 0px;z-index:5;pointer-events:none}.comparison-label{background:rgba(0,0,0,.7);color:#fff;padding:.5em 1em;border-radius:4px;font-size:.9em;font-weight:bold}.screenshot-panel{text-align:center;background:var(--color-background);border:1px solid var(--color-border-table-cell);border-radius:4px;padding:1em;box-shadow:0 2px 4px rgba(0,0,0,.05)}.screenshot-panel h3{margin:0 0 1em 0;font-size:1.1em;color:var(--color-text);border-bottom:2px solid var(--color-background-button-primary);padding-bottom:.5em}.screenshot-panel.diff h3{border-bottom-color:#d32f2f}.screenshot-panel img{max-width:100%;height:auto;border:1px solid var(--color-border-table-cell);box-shadow:0 2px 4px rgba(0,0,0,.1)}.version-selector{display:inline-block;margin:0 .5em}.version-selector label{font-weight:bold;margin-right:.5em;color:var(--color-text)}#settings{background:var(--color-background);padding:1.5em;border-radius:4px;box-shadow:0 2px 4px rgba(0,0,0,.05);margin-bottom:2em;border:1px solid var(--color-border-table-cell)}#settings h2{margin-top:0;color:var(--color-text)}.diff-fieldset{border:none;padding:0;margin:0}.edit-link{float:right;margin-top:-0.5em}.comparison-description{color:var(--color-text-input-description);font-size:.9em;margin-bottom:1em}.download-link{color:var(--color-link);text-decoration:none;display:inline-flex;align-items:center;gap:.3em;font-size:.85em}.download-link:hover{text-decoration:underline}.diff-section-header{color:#d32f2f;font-size:.9em;margin-bottom:1em;font-weight:bold;display:flex;align-items:center;justify-content:center;gap:1em}.comparison-history-section{margin-top:3em;padding:1em;background:var(--color-background);border:1px solid var(--color-border-table-cell);border-radius:4px;box-shadow:0 2px 4px rgba(0,0,0,.05)}.comparison-history-section h3{color:var(--color-text)}.comparison-history-section p{color:var(--color-text-input-description);font-size:.9em}.history-changed-yes{color:#d32f2f;font-weight:bold}.history-changed-no{color:#388e3c}
+.comparison-score{padding:1em;background:var(--color-table-stripe);border-radius:4px;margin:1em 0;border:1px solid var(--color-border-table-cell);color:var(--color-text)}.change-detected{color:#d32f2f;font-weight:bold}.no-change{color:#388e3c;font-weight:bold}.comparison-grid{display:grid;grid-template-columns:1fr 1fr;gap:1em;margin:1em 1em}@media(max-width: 1200px){.comparison-grid{grid-template-columns:1fr}}.image-comparison{position:relative;width:100%;overflow:hidden;border:1px solid var(--color-border-table-cell);box-shadow:0 2px 4px rgba(0, 0, 0, 0.1);user-select:none}.image-comparison img{display:block;width:100%;height:auto;max-width:100%;border:none;box-shadow:none}.comparison-image-wrapper{position:relative;width:100%;display:flex;align-items:flex-start;justify-content:center;background-color:var(--color-background);background-image:linear-gradient(45deg, var(--color-table-stripe) 25%, transparent 25%),linear-gradient(-45deg, var(--color-table-stripe) 25%, transparent 25%),linear-gradient(45deg, transparent 75%, var(--color-table-stripe) 75%),linear-gradient(-45deg, transparent 75%, var(--color-table-stripe) 75%);background-size:20px 20px;background-position:0 0,0 10px,10px -10px,-10px 0px}.comparison-after{position:absolute;top:0;left:0;width:100%;height:100%;clip-path:inset(0 0 0 50%)}.comparison-slider{position:absolute;top:0;left:50%;width:4px;height:100%;background:#0078e7;cursor:ew-resize;transform:translateX(-2px);z-index:10}.comparison-handle{position:absolute;top:50%;left:50%;width:48px;height:48px;background:#0078e7;border:3px solid #fff;border-radius:50%;transform:translate(-50%, -50%);box-shadow:0 2px 8px rgba(0, 0, 0, 0.3);display:flex;align-items:center;justify-content:center;cursor:ew-resize;transition:top .1s ease-out}.comparison-handle::after{content:"⇄";color:#fff;font-size:24px;font-weight:bold;pointer-events:none}.comparison-labels{position:absolute;top:10px;width:100%;display:flex;justify-content:space-between;padding:0 0px;z-index:5;pointer-events:none}.comparison-label{background:rgba(0, 0, 0, 0.7);color:#fff;padding:.5em 1em;border-radius:4px;font-size:.9em;font-weight:bold}.screenshot-panel{text-align:center;background:var(--color-background);border:1px solid var(--color-border-table-cell);border-radius:4px;padding:1em;box-shadow:0 2px 4px rgba(0, 0, 0, 0.05)}.screenshot-panel h3{margin:0 0 1em 0;font-size:1.1em;color:var(--color-text);border-bottom:2px solid var(--color-background-button-primary);padding-bottom:.5em}.screenshot-panel.diff h3{border-bottom-color:#d32f2f}.screenshot-panel img{max-width:100%;height:auto;border:1px solid var(--color-border-table-cell);box-shadow:0 2px 4px rgba(0, 0, 0, 0.1)}.version-selector{display:inline-block;margin:0 .5em}.version-selector label{font-weight:bold;margin-right:.5em;color:var(--color-text)}#settings{background:var(--color-background);padding:1.5em;border-radius:4px;box-shadow:0 2px 4px rgba(0, 0, 0, 0.05);margin-bottom:2em;border:1px solid var(--color-border-table-cell)}#settings h2{margin-top:0;color:var(--color-text)}.diff-fieldset{border:none;padding:0;margin:0}.edit-link{float:right;margin-top:-0.5em}.comparison-description{color:var(--color-text-input-description);font-size:.9em;margin-bottom:1em}.download-link{color:var(--color-link);text-decoration:none;display:inline-flex;align-items:center;gap:.3em;font-size:.85em}.download-link:hover{text-decoration:underline}.diff-section-header{color:#d32f2f;font-size:.9em;margin-bottom:1em;font-weight:bold;display:flex;align-items:center;justify-content:center;gap:1em}.comparison-history-section{margin-top:3em;padding:1em;background:var(--color-background);border:1px solid var(--color-border-table-cell);border-radius:4px;box-shadow:0 2px 4px rgba(0, 0, 0, 0.05)}.comparison-history-section h3{color:var(--color-text)}.comparison-history-section p{color:var(--color-text-input-description);font-size:.9em}.history-changed-yes{color:#d32f2f;font-weight:bold}.history-changed-no{color:#388e3c}

changedetectionio/static/styles/diff.css

@@ -1 +1 @@
-#diff-form{background:rgba(0,0,0,.05);padding:1em;border-radius:10px;margin-bottom:1em;color:#fff;font-size:.9rem;text-align:center}#diff-form label.from-to-label{width:4rem;text-decoration:none;padding:.5rem}#diff-form label.from-to-label#change-from{color:#b30000;background:#fadad7}#diff-form label.from-to-label#change-to{background:#eaf2c2;color:#406619}#diff-form #diff-style>span{display:inline-block;padding:.3em}#diff-form #diff-style>span label{font-weight:normal}#diff-form *{vertical-align:middle}body.difference-page section.content{padding-top:40px}#diff-ui{background:var(--color-background);padding:1rem;border-radius:5px}@media(min-width: 767px){#diff-ui{min-width:50%}}#diff-ui #text{font-size:11px}#diff-ui pre{white-space:break-spaces;overflow-wrap:anywhere}#diff-ui h1{display:inline;font-size:100%}#diff-ui #result{white-space:pre-wrap;word-break:break-word;overflow-wrap:break-word}#diff-ui .source{position:absolute;right:1%;top:.2em}@-moz-document url-prefix(){#diff-ui body{height:99%}}#diff-ui td#diff-col div{text-align:justify;white-space:pre-wrap}#diff-ui .ignored{background-color:#ccc;opacity:.7}#diff-ui .triggered{background-color:#1b98f8}#diff-ui .ignored.triggered{background-color:red}#diff-ui .tab-pane-inner#screenshot{text-align:center}#diff-ui .tab-pane-inner#screenshot img{max-width:99%}#diff-ui .pure-form button.reset-margin{margin:0px}#diff-ui .diff-fieldset{display:flex;align-items:center;gap:4px;flex-wrap:wrap}#diff-ui ul#highlightSnippetActions{list-style-type:none;display:flex;align-items:center;justify-content:center;gap:1.5rem;flex-wrap:wrap;padding:0;margin:0}#diff-ui ul#highlightSnippetActions li{display:flex;flex-direction:column;align-items:center;text-align:center;padding:.5rem;gap:.3rem}#diff-ui ul#highlightSnippetActions li button,#diff-ui ul#highlightSnippetActions li a{white-space:nowrap}#diff-ui ul#highlightSnippetActions span{font-size:.8rem;color:var(--color-text-input-description)}#diff-ui #cell-diff-jump-visualiser{display:flex;flex-direction:row;gap:1px;background:var(--color-background);border-radius:3px;overflow-x:hidden;position:sticky;top:0;z-index:10;padding-top:1rem;padding-bottom:1rem;justify-content:center}#diff-ui #cell-diff-jump-visualiser>div{flex:1;min-width:1px;max-width:10px;height:10px;background:var(--color-background-button-cancel);opacity:.3;border-radius:1px;transition:opacity .2s;position:relative}#diff-ui #cell-diff-jump-visualiser>div.deletion{background:#b30000;opacity:1}#diff-ui #cell-diff-jump-visualiser>div.insertion{background:#406619;opacity:1}#diff-ui #cell-diff-jump-visualiser>div.note{background:#406619;opacity:1}#diff-ui #cell-diff-jump-visualiser>div.mixed{background:linear-gradient(to right, #b30000 50%, #406619 50%);opacity:1}#diff-ui #cell-diff-jump-visualiser>div.current-position::after{content:"";position:absolute;bottom:-6px;left:50%;transform:translateX(-50%);width:0;height:0;border-left:4px solid rgba(0,0,0,0);border-right:4px solid rgba(0,0,0,0);border-bottom:4px solid var(--color-text)}#diff-ui #cell-diff-jump-visualiser>div:hover{opacity:.8;cursor:pointer}#text-diff-heading-area .snapshot-age{padding:4px;margin:.5rem 0;background-color:var(--color-background-snapshot-age);border-radius:3px;font-weight:bold;margin-bottom:4px}#text-diff-heading-area .snapshot-age.error{background-color:var(--color-error-background-snapshot-age);color:var(--color-error-text-snapshot-age)}#text-diff-heading-area .snapshot-age>*{padding-right:1rem}
+#diff-form{background:rgba(0, 0, 0, 0.05);padding:1em;border-radius:10px;margin-bottom:1em;color:#fff;font-size:.9rem;text-align:center}#diff-form label.from-to-label{width:4rem;text-decoration:none;padding:.5rem}#diff-form label.from-to-label#change-from{color:#b30000;background:#fadad7}#diff-form label.from-to-label#change-to{background:#eaf2c2;color:#406619}#diff-form #diff-style>span{display:inline-block;padding:.3em}#diff-form #diff-style>span label{font-weight:normal}#diff-form *{vertical-align:middle}body.difference-page section.content{padding-top:40px}#diff-ui{background:var(--color-background);padding:1rem;border-radius:5px}@media(min-width: 767px){#diff-ui{min-width:50%}}#diff-ui #text{font-size:11px}#diff-ui pre{white-space:break-spaces}#diff-ui h1{display:inline;font-size:100%}#diff-ui #result{white-space:pre-wrap;word-break:break-word;overflow-wrap:break-word}#diff-ui .source{position:absolute;right:1%;top:.2em}@-moz-document url-prefix(){#diff-ui body{height:99%}}#diff-ui td#diff-col div{text-align:justify;white-space:pre-wrap}#diff-ui .ignored{background-color:#ccc;opacity:.7}#diff-ui .triggered{background-color:#1b98f8}#diff-ui .ignored.triggered{background-color:red}#diff-ui .tab-pane-inner#screenshot{text-align:center}#diff-ui .tab-pane-inner#screenshot img{max-width:99%}#diff-ui .pure-form button.reset-margin{margin:0px}#diff-ui .diff-fieldset{display:flex;align-items:center;gap:4px;flex-wrap:wrap}#diff-ui ul#highlightSnippetActions{list-style-type:none;display:flex;align-items:center;justify-content:center;gap:1.5rem;flex-wrap:wrap;padding:0;margin:0}#diff-ui ul#highlightSnippetActions li{display:flex;flex-direction:column;align-items:center;text-align:center;padding:.5rem;gap:.3rem}#diff-ui ul#highlightSnippetActions li button,#diff-ui ul#highlightSnippetActions li a{white-space:nowrap}#diff-ui ul#highlightSnippetActions span{font-size:.8rem;color:var(--color-text-input-description)}#diff-ui #cell-diff-jump-visualiser{display:flex;flex-direction:row;gap:1px;background:var(--color-background);border-radius:3px;overflow-x:hidden;position:sticky;top:0;z-index:10;padding-top:1rem;padding-bottom:1rem;justify-content:center}#diff-ui #cell-diff-jump-visualiser>div{flex:1;min-width:1px;max-width:10px;height:10px;background:var(--color-background-button-cancel);opacity:.3;border-radius:1px;transition:opacity .2s;position:relative}#diff-ui #cell-diff-jump-visualiser>div.deletion{background:#b30000;opacity:1}#diff-ui #cell-diff-jump-visualiser>div.insertion{background:#406619;opacity:1}#diff-ui #cell-diff-jump-visualiser>div.note{background:#406619;opacity:1}#diff-ui #cell-diff-jump-visualiser>div.mixed{background:linear-gradient(to right, #b30000 50%, #406619 50%);opacity:1}#diff-ui #cell-diff-jump-visualiser>div.current-position::after{content:"";position:absolute;bottom:-6px;left:50%;transform:translateX(-50%);width:0;height:0;border-left:4px solid rgba(0, 0, 0, 0);border-right:4px solid rgba(0, 0, 0, 0);border-bottom:4px solid var(--color-text)}#diff-ui #cell-diff-jump-visualiser>div:hover{opacity:.8;cursor:pointer}#text-diff-heading-area .snapshot-age{padding:4px;margin:.5rem 0;background-color:var(--color-background-snapshot-age);border-radius:3px;font-weight:bold;margin-bottom:4px}#text-diff-heading-area .snapshot-age.error{background-color:var(--color-error-background-snapshot-age);color:var(--color-error-text-snapshot-age)}#text-diff-heading-area .snapshot-age>*{padding-right:1rem}

changedetectionio/static/styles/scss/diff.scss

@@ -62,7 +62,6 @@ body.difference-page {
 
   pre {
     white-space: break-spaces;
-    overflow-wrap: anywhere;
   }
 
 

changedetectionio/static/styles/scss/parts/_hamburger_menu.scss

@@ -110,9 +110,6 @@
           background: var(--color-background-menu-link-hover);
         }
       }
-      &#menu-pause, &#menu-mute {
-        display: none;
-      }
     }
   }
 }

changedetectionio/static/styles/scss/parts/_menu.scss

@@ -2,13 +2,6 @@
   padding: 0.5rem 1em;
   line-height: 1.2rem;
 }
-#menu-mute, #menu-pause {
-  padding-left: 0.3rem;
-  padding-right: 0.3rem;
-  img {
-    height: 1.2rem;
-  }
-}
 
 .pure-menu-item {
   svg {

changedetectionio/static/styles/scss/parts/_toast.scss

@@ -184,24 +184,24 @@ html[data-darkmode=true] {
 // Mobile adjustments
 @media only screen and (max-width: 768px) {
   .toast-container {
-    left: 50% !important;
-    right: auto !important;
-    top: 80px !important;
-    transform: translateX(-50%) !important;
-    align-items: center;
+    left: 10px !important;
+    right: 10px !important;
+    top: 10px !important;
+    transform: none !important;
+    align-items: stretch;
 
     &.toast-bottom-right,
     &.toast-bottom-center,
     &.toast-bottom-left {
       top: auto !important;
-      bottom: 80px !important;
+      bottom: 10px !important;
     }
   }
 
   .toast {
     min-width: auto;
     max-width: none;
-    width: 80vw;
+    width: 100%;
     transform: translateY(-100px);
 
     &.toast-show {

changedetectionio/static/styles/scss/styles.scss

@@ -222,19 +222,6 @@ code {
   color: var(--color-white);
   background: var(--color-text-watch-tag-list);
   @extend .inline-tag;
-  
-  /* Remove default anchor styling when used as links */
-  text-decoration: none;
-  
-  &:hover {
-    text-decoration: none;
-    opacity: 0.8;
-    cursor: pointer;
-  }
-  
-  &:visited {
-    color: var(--color-white);
-  }
 }
 
 @media (min-width: 768px) {

changedetectionio/store/base.py

@@ -1,83 +0,0 @@
-"""
-Base classes for the datastore.
-
-This module defines the abstract interfaces that all datastore implementations must follow.
-"""
-
-from abc import ABC, abstractmethod
-from threading import Lock
-from loguru import logger
-
-
-class DataStore(ABC):
-    """
-    Abstract base class for all datastore implementations.
-
-    Defines the core interface that all datastores must implement for:
-    - Loading and saving data
-    - Managing watches
-    - Handling settings
-    - Providing data access
-    """
-
-    lock = Lock()
-    datastore_path = None
-
-    @abstractmethod
-    def reload_state(self, datastore_path, include_default_watches, version_tag):
-        """
-        Load data from persistent storage.
-
-        Args:
-            datastore_path: Path to the datastore directory
-            include_default_watches: Whether to create default watches if none exist
-            version_tag: Application version string
-        """
-        pass
-
-    @abstractmethod
-    def add_watch(self, url, **kwargs):
-        """
-        Add a new watch.
-
-        Args:
-            url: URL to watch
-            **kwargs: Additional watch parameters
-
-        Returns:
-            UUID of the created watch
-        """
-        pass
-
-    @abstractmethod
-    def update_watch(self, uuid, update_obj):
-        """
-        Update an existing watch.
-
-        Args:
-            uuid: Watch UUID
-            update_obj: Dictionary of fields to update
-        """
-        pass
-
-    @abstractmethod
-    def delete(self, uuid):
-        """
-        Delete a watch.
-
-        Args:
-            uuid: Watch UUID to delete
-        """
-        pass
-
-    @property
-    @abstractmethod
-    def data(self):
-        """
-        Access to the underlying data structure.
-
-        Returns:
-            Dictionary containing all datastore data
-        """
-        pass
-

changedetectionio/store/file_saving_datastore.py

@@ -1,509 +0,0 @@
-"""
-File-based datastore with individual watch persistence and immediate commits.
-
-This module provides the FileSavingDataStore abstract class that implements:
-- Individual watch.json file persistence
-- Immediate commit-based persistence (watch.commit(), datastore.commit())
-- Atomic file writes safe for NFS/NAS
-"""
-
-import glob
-import json
-import os
-import tempfile
-import time
-from loguru import logger
-
-from .base import DataStore
-from .. import strtobool
-
-# Try to import orjson for faster JSON serialization
-try:
-    import orjson
-    HAS_ORJSON = True
-except ImportError:
-    HAS_ORJSON = False
-
-# Fsync configuration: Force file data to disk for crash safety
-# Default False to match legacy behavior (write-and-rename without fsync)
-# Set to True for mission-critical deployments requiring crash consistency
-FORCE_FSYNC_DATA_IS_CRITICAL = bool(strtobool(os.getenv('FORCE_FSYNC_DATA_IS_CRITICAL', 'False')))
-
-# ============================================================================
-# Helper Functions for Atomic File Operations
-# ============================================================================
-
-def save_json_atomic(file_path, data_dict, label="file", max_size_mb=10):
-    """
-    Save JSON data to disk using atomic write pattern.
-
-    Generic helper for saving any JSON data (settings, watches, etc.) with:
-    - Atomic write (temp file + rename)
-    - Directory fsync for crash consistency (only for new files)
-    - Size validation
-    - Proper error handling
-
-    Thread safety: Caller must hold datastore.lock to prevent concurrent modifications.
-    Multi-process safety: Not supported - run only one app instance per datastore.
-
-    Args:
-        file_path: Full path to target JSON file
-        data_dict: Dictionary to serialize
-        label: Human-readable label for error messages (e.g., "watch", "settings")
-        max_size_mb: Maximum allowed file size in MB
-
-    Raises:
-        ValueError: If serialized data exceeds max_size_mb
-        OSError: If disk is full (ENOSPC) or other I/O error
-    """
-    # Check if file already exists (before we start writing)
-    # Directory fsync only needed for NEW files to persist the filename
-    file_exists = os.path.exists(file_path)
-
-    # Ensure parent directory exists
-    parent_dir = os.path.dirname(file_path)
-    os.makedirs(parent_dir, exist_ok=True)
-
-    # Create temp file in same directory (required for NFS atomicity)
-    fd, temp_path = tempfile.mkstemp(
-        suffix='.tmp',
-        prefix='json-',
-        dir=parent_dir,
-        text=False
-    )
-
-    fd_closed = False
-    try:
-        # Serialize data
-        t0 = time.time()
-        if HAS_ORJSON:
-            data = orjson.dumps(data_dict, option=orjson.OPT_INDENT_2)
-        else:
-            data = json.dumps(data_dict, indent=2, ensure_ascii=False).encode('utf-8')
-        serialize_ms = (time.time() - t0) * 1000
-
-        # Safety check: validate size
-        MAX_SIZE = max_size_mb * 1024 * 1024
-        data_size = len(data)
-        if data_size > MAX_SIZE:
-            raise ValueError(
-                f"{label.capitalize()} data is unexpectedly large: {data_size / 1024 / 1024:.2f}MB "
-                f"(max: {max_size_mb}MB). This indicates a bug or data corruption."
-            )
-
-        # Write to temp file
-        t1 = time.time()
-        os.write(fd, data)
-        write_ms = (time.time() - t1) * 1000
-
-        # Optional fsync: Force file data to disk for crash safety
-        # Only if FORCE_FSYNC_DATA_IS_CRITICAL=True (default: False, matches legacy behavior)
-        t2 = time.time()
-        if FORCE_FSYNC_DATA_IS_CRITICAL:
-            os.fsync(fd)
-        file_fsync_ms = (time.time() - t2) * 1000
-
-        os.close(fd)
-        fd_closed = True
-
-        # Atomic rename
-        t3 = time.time()
-        os.replace(temp_path, file_path)
-        rename_ms = (time.time() - t3) * 1000
-
-        # Sync directory to ensure filename metadata is durable
-        # OPTIMIZATION: Only needed for NEW files. Existing files already have
-        # directory entry persisted, so we only need file fsync for data durability.
-        dir_fsync_ms = 0
-        if not file_exists:
-            try:
-                dir_fd = os.open(parent_dir, os.O_RDONLY)
-                try:
-                    t4 = time.time()
-                    os.fsync(dir_fd)
-                    dir_fsync_ms = (time.time() - t4) * 1000
-                finally:
-                    os.close(dir_fd)
-            except (OSError, AttributeError):
-                # Windows doesn't support fsync on directories
-                pass
-
-        # Log timing breakdown for slow saves
-#        total_ms = serialize_ms + write_ms + file_fsync_ms + rename_ms + dir_fsync_ms
-#        if total_ms:  # Log if save took more than 10ms
-#            file_status = "new" if not file_exists else "update"
-#            logger.trace(
-#                f"Save timing breakdown ({total_ms:.1f}ms total, {file_status}): "
-#                f"serialize={serialize_ms:.1f}ms, write={write_ms:.1f}ms, "
-#                f"file_fsync={file_fsync_ms:.1f}ms, rename={rename_ms:.1f}ms, "
-#                f"dir_fsync={dir_fsync_ms:.1f}ms, using_orjson={HAS_ORJSON}"
-#            )
-
-    except OSError as e:
-        # Cleanup temp file
-        if not fd_closed:
-            try:
-                os.close(fd)
-            except:
-                pass
-        if os.path.exists(temp_path):
-            try:
-                os.unlink(temp_path)
-            except:
-                pass
-
-        # Provide helpful error messages
-        if e.errno == 28:  # ENOSPC
-            raise OSError(f"Disk full: Cannot save {label}") from e
-        elif e.errno == 122:  # EDQUOT
-            raise OSError(f"Disk quota exceeded: Cannot save {label}") from e
-        else:
-            raise OSError(f"I/O error saving {label}: {e}") from e
-
-    except Exception as e:
-        # Cleanup temp file
-        if not fd_closed:
-            try:
-                os.close(fd)
-            except:
-                pass
-        if os.path.exists(temp_path):
-            try:
-                os.unlink(temp_path)
-            except:
-                pass
-        raise e
-
-
-def save_entity_atomic(entity_dir, uuid, entity_dict, filename, entity_type, max_size_mb):
-    """
-    Save an entity (watch/tag) to disk using atomic write pattern.
-
-    Generic function for saving any watch_base subclass (Watch, Tag, etc.).
-
-    Args:
-        entity_dir: Directory for this entity (e.g., /datastore/{uuid})
-        uuid: Entity UUID (for logging)
-        entity_dict: Dictionary representation of the entity
-        filename: JSON filename (e.g., 'watch.json', 'tag.json')
-        entity_type: Type label for logging (e.g., 'watch', 'tag')
-        max_size_mb: Maximum allowed file size in MB
-
-    Raises:
-        ValueError: If serialized data exceeds max_size_mb
-        OSError: If disk is full (ENOSPC) or other I/O error
-    """
-    entity_json = os.path.join(entity_dir, filename)
-    save_json_atomic(entity_json, entity_dict, label=f"{entity_type} {uuid}", max_size_mb=max_size_mb)
-
-
-def save_watch_atomic(watch_dir, uuid, watch_dict):
-    """
-    Save a watch to disk using atomic write pattern.
-
-    Convenience wrapper around save_entity_atomic for watches.
-    Kept for backwards compatibility.
-    """
-    save_entity_atomic(watch_dir, uuid, watch_dict, "watch.json", "watch", max_size_mb=10)
-
-
-
-def load_watch_from_file(watch_json, uuid, rehydrate_entity_func):
-    """
-    Load a watch from its JSON file.
-
-    Args:
-        watch_json: Path to the watch.json file
-        uuid: Watch UUID
-        rehydrate_entity_func: Function to convert dict to Watch object
-
-    Returns:
-        Watch object or None if failed
-    """
-    try:
-        # Check file size before reading
-        file_size = os.path.getsize(watch_json)
-        MAX_WATCH_SIZE = 10 * 1024 * 1024  # 10MB
-        if file_size > MAX_WATCH_SIZE:
-            logger.critical(
-                f"CORRUPTED WATCH DATA: Watch {uuid} file is unexpectedly large: "
-                f"{file_size / 1024 / 1024:.2f}MB (max: {MAX_WATCH_SIZE / 1024 / 1024}MB). "
-                f"File: {watch_json}. This indicates a bug or data corruption. "
-                f"Watch will be skipped."
-            )
-            return None
-
-        if HAS_ORJSON:
-            with open(watch_json, 'rb') as f:
-                watch_data = orjson.loads(f.read())
-        else:
-            with open(watch_json, 'r', encoding='utf-8') as f:
-                watch_data = json.load(f)
-
-        # Rehydrate and return watch object
-        watch_obj = rehydrate_entity_func(uuid, watch_data)
-        return watch_obj
-
-    except json.JSONDecodeError as e:
-        logger.critical(
-            f"CORRUPTED WATCH DATA: Failed to parse JSON for watch {uuid}. "
-            f"File: {watch_json}. Error: {e}. "
-            f"Watch will be skipped and may need manual recovery from backup."
-        )
-        return None
-    except ValueError as e:
-        # orjson raises ValueError for invalid JSON
-        if "invalid json" in str(e).lower() or HAS_ORJSON:
-            logger.critical(
-                f"CORRUPTED WATCH DATA: Failed to parse JSON for watch {uuid}. "
-                f"File: {watch_json}. Error: {e}. "
-                f"Watch will be skipped and may need manual recovery from backup."
-            )
-            return None
-        # Re-raise if it's not a JSON parsing error
-        raise
-    except FileNotFoundError:
-        logger.error(f"Watch file not found: {watch_json} for watch {uuid}")
-        return None
-    except Exception as e:
-        logger.error(f"Failed to load watch {uuid} from {watch_json}: {e}")
-        return None
-
-
-def load_all_watches(datastore_path, rehydrate_entity_func):
-    """
-    Load all watches from individual watch.json files.
-
-    SYNCHRONOUS loading: Blocks until all watches are loaded.
-    This ensures data consistency - web server won't accept requests
-    until all watches are available. Progress logged every 100 watches.
-
-    Args:
-        datastore_path: Path to the datastore directory
-        rehydrate_entity_func: Function to convert dict to Watch object
-
-    Returns:
-        Dictionary of uuid -> Watch object
-    """
-    start_time = time.time()
-    logger.info("Loading watches from individual watch.json files...")
-
-    watching = {}
-
-    if not os.path.exists(datastore_path):
-        return watching
-
-    # Find all watch.json files using glob (faster than manual directory traversal)
-    glob_start = time.time()
-    watch_files = glob.glob(os.path.join(datastore_path, "*", "watch.json"))
-    glob_time = time.time() - glob_start
-
-    total = len(watch_files)
-    logger.debug(f"Found {total} watch.json files in {glob_time:.3f}s")
-
-    loaded = 0
-    failed = 0
-
-    for watch_json in watch_files:
-        # Extract UUID from path: /datastore/{uuid}/watch.json
-        uuid_dir = os.path.basename(os.path.dirname(watch_json))
-        watch = load_watch_from_file(watch_json, uuid_dir, rehydrate_entity_func)
-        if watch:
-            watching[uuid_dir] = watch
-            loaded += 1
-
-            if loaded % 100 == 0:
-                logger.info(f"Loaded {loaded}/{total} watches...")
-        else:
-            # load_watch_from_file already logged the specific error
-            failed += 1
-
-    elapsed = time.time() - start_time
-
-    if failed > 0:
-        logger.critical(
-            f"LOAD COMPLETE: {loaded} watches loaded successfully, "
-            f"{failed} watches FAILED to load (corrupted or invalid) "
-            f"in {elapsed:.2f}s ({loaded/elapsed:.0f} watches/sec)"
-        )
-    else:
-        logger.info(f"Loaded {loaded} watches from disk in {elapsed:.2f}s ({loaded/elapsed:.0f} watches/sec)")
-
-    return watching
-
-
-def load_tag_from_file(tag_json, uuid, rehydrate_entity_func):
-    """
-    Load a tag from its JSON file.
-
-    Args:
-        tag_json: Path to the tag.json file
-        uuid: Tag UUID
-        rehydrate_entity_func: Function to convert dict to Tag object
-
-    Returns:
-        Tag object or None if failed
-    """
-    try:
-        # Check file size before reading
-        file_size = os.path.getsize(tag_json)
-        MAX_TAG_SIZE = 1 * 1024 * 1024  # 1MB
-        if file_size > MAX_TAG_SIZE:
-            logger.critical(
-                f"CORRUPTED TAG DATA: Tag {uuid} file is unexpectedly large: "
-                f"{file_size / 1024 / 1024:.2f}MB (max: {MAX_TAG_SIZE / 1024 / 1024}MB). "
-                f"File: {tag_json}. This indicates a bug or data corruption. "
-                f"Tag will be skipped."
-            )
-            return None
-
-        if HAS_ORJSON:
-            with open(tag_json, 'rb') as f:
-                tag_data = orjson.loads(f.read())
-        else:
-            with open(tag_json, 'r', encoding='utf-8') as f:
-                tag_data = json.load(f)
-
-        tag_data['processor'] = 'restock_diff'
-        # Rehydrate tag (convert dict to Tag object)
-        # processor_override is set inside the rehydration function
-        tag_obj = rehydrate_entity_func(uuid, tag_data)
-        return tag_obj
-
-    except json.JSONDecodeError as e:
-        logger.critical(
-            f"CORRUPTED TAG DATA: Failed to parse JSON for tag {uuid}. "
-            f"File: {tag_json}. Error: {e}. "
-            f"Tag will be skipped and may need manual recovery from backup."
-        )
-        return None
-    except ValueError as e:
-        # orjson raises ValueError for invalid JSON
-        if "invalid json" in str(e).lower() or HAS_ORJSON:
-            logger.critical(
-                f"CORRUPTED TAG DATA: Failed to parse JSON for tag {uuid}. "
-                f"File: {tag_json}. Error: {e}. "
-                f"Tag will be skipped and may need manual recovery from backup."
-            )
-            return None
-        # Re-raise if it's not a JSON parsing error
-        raise
-    except FileNotFoundError:
-        logger.debug(f"Tag file not found: {tag_json} for tag {uuid}")
-        return None
-    except Exception as e:
-        logger.error(f"Failed to load tag {uuid} from {tag_json}: {e}")
-        return None
-
-
-def load_all_tags(datastore_path, rehydrate_entity_func):
-    """
-    Load all tags from individual tag.json files.
-
-    Tags are stored separately from settings in {uuid}/tag.json files.
-
-    Args:
-        datastore_path: Path to the datastore directory
-        rehydrate_entity_func: Function to convert dict to Tag object
-
-    Returns:
-        Dictionary of uuid -> Tag object
-    """
-    logger.info("Loading tags from individual tag.json files...")
-
-    tags = {}
-
-    if not os.path.exists(datastore_path):
-        return tags
-
-    # Find all tag.json files using glob
-    tag_files = glob.glob(os.path.join(datastore_path, "*", "tag.json"))
-
-    total = len(tag_files)
-    if total == 0:
-        logger.debug("No tag.json files found")
-        return tags
-
-    logger.debug(f"Found {total} tag.json files")
-
-    loaded = 0
-    failed = 0
-
-    for tag_json in tag_files:
-        # Extract UUID from path: /datastore/{uuid}/tag.json
-        uuid_dir = os.path.basename(os.path.dirname(tag_json))
-        tag = load_tag_from_file(tag_json, uuid_dir, rehydrate_entity_func)
-        if tag:
-            tags[uuid_dir] = tag
-            loaded += 1
-        else:
-            # load_tag_from_file already logged the specific error
-            failed += 1
-
-    if failed > 0:
-        logger.warning(f"Loaded {loaded} tags, {failed} tags FAILED to load")
-    else:
-        logger.info(f"Loaded {loaded} tags from disk")
-
-    return tags
-
-
-# ============================================================================
-# FileSavingDataStore Class
-# ============================================================================
-
-class FileSavingDataStore(DataStore):
-    """
-    Abstract datastore that provides file persistence with immediate commits.
-
-    Features:
-    - Individual watch.json files (one per watch)
-    - Immediate persistence via watch.commit() and datastore.commit()
-    - Atomic file writes for crash safety
-
-    Subclasses must implement:
-    - rehydrate_entity(): Convert dict to Watch object
-    - Access to internal __data structure for watch management
-    """
-
-    def __init__(self):
-        super().__init__()
-
-    def _save_settings(self):
-        """
-        Save settings to storage (polymorphic).
-
-        Subclasses must implement for their backend.
-        - File: changedetection.json
-        - Redis: SET settings
-        - SQL: UPDATE settings table
-        """
-        raise NotImplementedError("Subclass must implement _save_settings")
-
-
-    def _load_watches(self):
-        """
-        Load all watches from storage (polymorphic).
-
-        Subclasses must implement for their backend.
-        - File: Read individual watch.json files
-        - Redis: SCAN watch:* keys
-        - SQL: SELECT * FROM watches
-        """
-        raise NotImplementedError("Subclass must implement _load_watches")
-
-    def _delete_watch(self, uuid):
-        """
-        Delete a watch from storage (polymorphic).
-
-        Subclasses must implement for their backend.
-        - File: Delete {uuid}/ directory recursively
-        - Redis: DEL watch:{uuid}
-        - SQL: DELETE FROM watches WHERE uuid=?
-
-        Args:
-            uuid: Watch UUID to delete
-        """
-        raise NotImplementedError("Subclass must implement _delete_watch")
-
-

changedetectionio/store/updates.py

@@ -1,732 +0,0 @@
-"""
-Schema update migrations for the datastore.
-
-This module contains all schema version upgrade methods (update_1 through update_N).
-These are mixed into ChangeDetectionStore to keep the main store file focused.
-
-IMPORTANT: Each update could be run even when they have a new install and the schema is correct.
-Therefore - each `update_n` should be very careful about checking if it needs to actually run.
-"""
-
-import os
-import re
-import shutil
-import tarfile
-import time
-from loguru import logger
-from copy import deepcopy
-
-
-# Try to import orjson for faster JSON serialization
-try:
-    import orjson
-    HAS_ORJSON = True
-except ImportError:
-    HAS_ORJSON = False
-
-from ..html_tools import TRANSLATE_WHITESPACE_TABLE
-from ..processors.restock_diff import Restock
-from ..blueprint.rss import RSS_CONTENT_FORMAT_DEFAULT
-from ..model import USE_SYSTEM_DEFAULT_NOTIFICATION_FORMAT_FOR_WATCH
-
-def create_backup_tarball(datastore_path, update_number):
-    """
-    Create a tarball backup of the entire datastore structure before running an update.
-
-    Includes:
-    - All {uuid}/watch.json files
-    - All {uuid}/tag.json files
-    - changedetection.json (settings, if it exists)
-    - url-watches.json (legacy format, if it exists)
-    - Directory structure preserved
-
-    Args:
-        datastore_path: Path to datastore directory
-        update_number: Update number being applied
-
-    Returns:
-        str: Path to created tarball, or None if backup failed
-
-    Restoration:
-    To restore from a backup:
-        cd /path/to/datastore
-        tar -xzf before-update-N-timestamp.tar.gz
-    This will restore all watch.json and tag.json files and settings to their pre-update state.
-    """
-    timestamp = int(time.time())
-    backup_filename = f"before-update-{update_number}-{timestamp}.tar.gz"
-    backup_path = os.path.join(datastore_path, backup_filename)
-
-    try:
-        logger.info(f"Creating backup tarball: {backup_filename}")
-
-        with tarfile.open(backup_path, "w:gz") as tar:
-            # Backup changedetection.json if it exists (new format)
-            changedetection_json = os.path.join(datastore_path, "changedetection.json")
-            if os.path.isfile(changedetection_json):
-                tar.add(changedetection_json, arcname="changedetection.json")
-                logger.debug("Added changedetection.json to backup")
-
-            # Backup url-watches.json if it exists (legacy format)
-            url_watches_json = os.path.join(datastore_path, "url-watches.json")
-            if os.path.isfile(url_watches_json):
-                tar.add(url_watches_json, arcname="url-watches.json")
-                logger.debug("Added url-watches.json to backup")
-
-            # Backup all watch/tag directories with their JSON files
-            # This preserves the UUID directory structure
-            watch_count = 0
-            tag_count = 0
-            for entry in os.listdir(datastore_path):
-                entry_path = os.path.join(datastore_path, entry)
-
-                # Skip if not a directory
-                if not os.path.isdir(entry_path):
-                    continue
-
-                # Skip hidden directories and backup directories
-                if entry.startswith('.') or entry.startswith('before-update-'):
-                    continue
-
-                # Backup watch.json if exists
-                watch_json = os.path.join(entry_path, "watch.json")
-                if os.path.isfile(watch_json):
-                    tar.add(watch_json, arcname=f"{entry}/watch.json")
-                    watch_count += 1
-
-                    if watch_count % 100 == 0:
-                        logger.debug(f"Backed up {watch_count} watch.json files...")
-
-                # Backup tag.json if exists
-                tag_json = os.path.join(entry_path, "tag.json")
-                if os.path.isfile(tag_json):
-                    tar.add(tag_json, arcname=f"{entry}/tag.json")
-                    tag_count += 1
-
-            logger.success(f"Backup created: {backup_filename} ({watch_count} watches from disk, {tag_count} tags from disk)")
-            return backup_path
-
-    except Exception as e:
-        logger.error(f"Failed to create backup tarball: {e}")
-        # Try to clean up partial backup
-        if os.path.exists(backup_path):
-            try:
-                os.unlink(backup_path)
-            except:
-                pass
-        return None
-
-
-class DatastoreUpdatesMixin:
-    """
-    Mixin class containing all schema update methods.
-
-    This class is inherited by ChangeDetectionStore to provide schema migration functionality.
-    Each update_N method upgrades the schema from version N-1 to version N.
-    """
-
-    def get_updates_available(self):
-        """
-        Discover all available update methods.
-
-        Returns:
-            list: Sorted list of update version numbers (e.g., [1, 2, 3, ..., 26])
-        """
-        import inspect
-        updates_available = []
-        for i, o in inspect.getmembers(self, predicate=inspect.ismethod):
-            m = re.search(r'update_(\d+)$', i)
-            if m:
-                updates_available.append(int(m.group(1)))
-        updates_available.sort()
-
-        return updates_available
-
-    def run_updates(self, current_schema_version=None):
-        import sys
-        """
-        Run all pending schema updates sequentially.
-
-        Args:
-            current_schema_version: Optional current schema version. If provided, only run updates
-                                   greater than this version. If None, uses the schema version from
-                                   the datastore. If no schema version exists in datastore and it appears
-                                   to be a fresh install, sets to latest update number (no updates needed).
-
-        IMPORTANT: Each update could be run even when they have a new install and the schema is correct.
-        Therefore - each `update_n` should be very careful about checking if it needs to actually run.
-
-        Process:
-        1. Get list of available updates
-        2. For each update > current schema version:
-           - Create backup of datastore
-           - Run update method
-           - Update schema version and commit settings
-           - Commit all watches and tags
-        3. If any update fails, stop processing
-        4. All changes saved via individual .commit() calls
-        """
-        updates_available = self.get_updates_available()
-        if self.data.get('watching'):
-            test_watch = self.data['watching'].get(next(iter(self.data.get('watching', {}))))
-            from ..model.Watch import model
-
-            if not isinstance(test_watch, model):
-                import sys
-                logger.critical("Cannot run updates! Watch structure must be re-hydrated back to a Watch model object!")
-                sys.exit(1)
-
-        if self.data['settings']['application'].get('tags',{}):
-            test_tag = self.data['settings']['application'].get('tags',{}).get(next(iter(self.data['settings']['application'].get('tags',{}))))
-            from ..model.Tag import model as tag_model
-
-            if not isinstance(test_tag, tag_model):
-                import sys
-                logger.critical("Cannot run updates! Watch tag/group structure must be re-hydrated back to a Tag model object!")
-                sys.exit(1)
-
-        # Determine current schema version
-        if current_schema_version is None:
-            # Check if schema_version exists in datastore
-            current_schema_version = self.data['settings']['application'].get('schema_version')
-
-            if current_schema_version is None:
-                # No schema version found - could be a fresh install or very old datastore
-                # If this is a fresh/new config with no watches, assume it's up-to-date
-                # and set to latest update number (no updates needed)
-                if len(self.data['watching']) == 0:
-                    # Get the highest update number from available update methods
-                    latest_update = updates_available[-1] if updates_available else 0
-                    logger.info(f"No schema version found and no watches exist - assuming fresh install, setting schema_version to {latest_update}")
-                    self.data['settings']['application']['schema_version'] = latest_update
-                    self.commit()
-                    return  # No updates needed for fresh install
-                else:
-                    # Has watches but no schema version - likely old datastore, run all updates
-                    logger.warning("No schema version found but watches exist - running all updates from version 0")
-                    current_schema_version = 0
-
-        logger.info(f"Current schema version: {current_schema_version}")
-
-        updates_ran = []
-
-        for update_n in updates_available:
-            if update_n > current_schema_version:
-                logger.critical(f"Applying update_{update_n}")
-
-                # Create tarball backup of entire datastore structure
-                # This includes all watch.json files, settings, and preserves directory structure
-                backup_path = create_backup_tarball(self.datastore_path, update_n)
-                if backup_path:
-                    logger.info(f"Backup created at: {backup_path}")
-                else:
-                    logger.warning("Backup creation failed, but continuing with update")
-
-                try:
-                    update_method = getattr(self, f"update_{update_n}")()
-                except Exception as e:
-                    logger.critical(f"Error while trying update_{update_n}")
-                    logger.exception(e)
-                    sys.exit(1)
-                else:
-                    # Bump the version
-                    self.data['settings']['application']['schema_version'] = update_n
-                    self.commit()
-
-                    logger.success(f"Update {update_n} completed")
-
-                    # Track which updates ran
-                    updates_ran.append(update_n)
-
-    # ============================================================================
-    # Individual Update Methods
-    # ============================================================================
-
-    def update_1(self):
-        """Convert minutes to seconds on settings and each watch."""
-        if self.data['settings']['requests'].get('minutes_between_check'):
-            self.data['settings']['requests']['time_between_check']['minutes'] = self.data['settings']['requests']['minutes_between_check']
-            # Remove the default 'hours' that is set from the model
-            self.data['settings']['requests']['time_between_check']['hours'] = None
-
-        for uuid, watch in self.data['watching'].items():
-            if 'minutes_between_check' in watch:
-                # Only upgrade individual watch time if it was set
-                if watch.get('minutes_between_check', False):
-                    self.data['watching'][uuid]['time_between_check']['minutes'] = watch['minutes_between_check']
-
-    def update_2(self):
-        """
-        Move the history list to a flat text file index.
-        Better than SQLite because this list is only appended to, and works across NAS / NFS type setups.
-        """
-        # @todo test running this on a newly updated one (when this already ran)
-        for uuid, watch in self.data['watching'].items():
-            history = []
-
-            if watch.get('history', False):
-                for d, p in watch['history'].items():
-                    d = int(d)  # Used to be keyed as str, we'll fix this now too
-                    history.append("{},{}\n".format(d, p))
-
-                if len(history):
-                    target_path = os.path.join(self.datastore_path, uuid)
-                    if os.path.exists(target_path):
-                        with open(os.path.join(target_path, "history.txt"), "w") as f:
-                            f.writelines(history)
-                    else:
-                        logger.warning(f"Datastore history directory {target_path} does not exist, skipping history import.")
-
-                # No longer needed, dynamically pulled from the disk when needed.
-                # But we should set it back to a empty dict so we don't break if this schema runs on an earlier version.
-                # In the distant future we can remove this entirely
-                self.data['watching'][uuid]['history'] = {}
-
-    def update_3(self):
-        """We incorrectly stored last_changed when there was not a change, and then confused the output list table."""
-        # see https://github.com/dgtlmoon/changedetection.io/pull/835
-        return
-
-    def update_4(self):
-        """`last_changed` not needed, we pull that information from the history.txt index."""
-        for uuid, watch in self.data['watching'].items():
-            try:
-                # Remove it from the struct
-                del(watch['last_changed'])
-            except:
-                continue
-        return
-
-    def update_5(self):
-        """
-        If the watch notification body, title look the same as the global one, unset it, so the watch defaults back to using the main settings.
-        In other words - the watch notification_title and notification_body are not needed if they are the same as the default one.
-        """
-        current_system_body = self.data['settings']['application']['notification_body'].translate(TRANSLATE_WHITESPACE_TABLE)
-        current_system_title = self.data['settings']['application']['notification_body'].translate(TRANSLATE_WHITESPACE_TABLE)
-        for uuid, watch in self.data['watching'].items():
-            try:
-                watch_body = watch.get('notification_body', '')
-                if watch_body and watch_body.translate(TRANSLATE_WHITESPACE_TABLE) == current_system_body:
-                    # Looks the same as the default one, so unset it
-                    watch['notification_body'] = None
-
-                watch_title = watch.get('notification_title', '')
-                if watch_title and watch_title.translate(TRANSLATE_WHITESPACE_TABLE) == current_system_title:
-                    # Looks the same as the default one, so unset it
-                    watch['notification_title'] = None
-            except Exception as e:
-                continue
-        return
-
-    def update_7(self):
-        """
-        We incorrectly used common header overrides that should only apply to Requests.
-        These are now handled in content_fetcher::html_requests and shouldnt be passed to Playwright/Selenium.
-        """
-        # These were hard-coded in early versions
-        for v in ['User-Agent', 'Accept', 'Accept-Encoding', 'Accept-Language']:
-            if self.data['settings']['headers'].get(v):
-                del self.data['settings']['headers'][v]
-
-    def update_8(self):
-        """Convert filters to a list of filters css_filter -> include_filters."""
-        for uuid, watch in self.data['watching'].items():
-            try:
-                existing_filter = watch.get('css_filter', '')
-                if existing_filter:
-                    watch['include_filters'] = [existing_filter]
-            except:
-                continue
-        return
-
-    def update_9(self):
-        """Convert old static notification tokens to jinja2 tokens."""
-        # Each watch
-        # only { } not {{ or }}
-        r = r'(?<!{){(?!{)(\w+)(?<!})}(?!})'
-        for uuid, watch in self.data['watching'].items():
-            try:
-                n_body = watch.get('notification_body', '')
-                if n_body:
-                    watch['notification_body'] = re.sub(r, r'{{\1}}', n_body)
-
-                n_title = watch.get('notification_title')
-                if n_title:
-                    watch['notification_title'] = re.sub(r, r'{{\1}}', n_title)
-
-                n_urls = watch.get('notification_urls')
-                if n_urls:
-                    for i, url in enumerate(n_urls):
-                        watch['notification_urls'][i] = re.sub(r, r'{{\1}}', url)
-
-            except:
-                continue
-
-        # System wide
-        n_body = self.data['settings']['application'].get('notification_body')
-        if n_body:
-            self.data['settings']['application']['notification_body'] = re.sub(r, r'{{\1}}', n_body)
-
-        n_title = self.data['settings']['application'].get('notification_title')
-        if n_body:
-            self.data['settings']['application']['notification_title'] = re.sub(r, r'{{\1}}', n_title)
-
-        n_urls = self.data['settings']['application'].get('notification_urls')
-        if n_urls:
-            for i, url in enumerate(n_urls):
-                self.data['settings']['application']['notification_urls'][i] = re.sub(r, r'{{\1}}', url)
-
-        return
-
-    def update_10(self):
-        """Some setups may have missed the correct default, so it shows the wrong config in the UI, although it will default to system-wide."""
-        for uuid, watch in self.data['watching'].items():
-            try:
-                if not watch.get('fetch_backend', ''):
-                    watch['fetch_backend'] = 'system'
-            except:
-                continue
-        return
-
-    def update_12(self):
-        """Create tag objects and their references from existing tag text."""
-        i = 0
-        for uuid, watch in self.data['watching'].items():
-            # Split out and convert old tag string
-            tag = watch.get('tag')
-            if tag:
-                tag_uuids = []
-                for t in tag.split(','):
-                    tag_uuids.append(self.add_tag(title=t))
-
-                self.data['watching'][uuid]['tags'] = tag_uuids
-
-    def update_13(self):
-        """#1775 - Update 11 did not update the records correctly when adding 'date_created' values for sorting."""
-        i = 0
-        for uuid, watch in self.data['watching'].items():
-            if not watch.get('date_created'):
-                self.data['watching'][uuid]['date_created'] = i
-            i += 1
-        return
-
-    def update_14(self):
-        """#1774 - protect xpath1 against migration."""
-        for awatch in self.data["watching"]:
-            if self.data["watching"][awatch]['include_filters']:
-                for num, selector in enumerate(self.data["watching"][awatch]['include_filters']):
-                    if selector.startswith('/'):
-                        self.data["watching"][awatch]['include_filters'][num] = 'xpath1:' + selector
-                    if selector.startswith('xpath:'):
-                        self.data["watching"][awatch]['include_filters'][num] = selector.replace('xpath:', 'xpath1:', 1)
-
-    def update_15(self):
-        """Use more obvious default time setting."""
-        for uuid in self.data["watching"]:
-            if self.data["watching"][uuid]['time_between_check'] == self.data['settings']['requests']['time_between_check']:
-                # What the old logic was, which was pretty confusing
-                self.data["watching"][uuid]['time_between_check_use_default'] = True
-            elif all(value is None or value == 0 for value in self.data["watching"][uuid]['time_between_check'].values()):
-                self.data["watching"][uuid]['time_between_check_use_default'] = True
-            else:
-                # Something custom here
-                self.data["watching"][uuid]['time_between_check_use_default'] = False
-
-    def update_16(self):
-        """Correctly set datatype for older installs where 'tag' was string and update_12 did not catch it."""
-        for uuid, watch in self.data['watching'].items():
-            if isinstance(watch.get('tags'), str):
-                self.data['watching'][uuid]['tags'] = []
-
-    def update_17(self):
-        """Migrate old 'in_stock' values to the new Restock."""
-        for uuid, watch in self.data['watching'].items():
-            if 'in_stock' in watch:
-                watch['restock'] = Restock({'in_stock': watch.get('in_stock')})
-                del watch['in_stock']
-
-    def update_18(self):
-        """Migrate old restock settings."""
-        for uuid, watch in self.data['watching'].items():
-            if not watch.get('restock_settings'):
-                # So we enable price following by default
-                self.data['watching'][uuid]['restock_settings'] = {'follow_price_changes': True}
-
-            # Migrate and cleanoff old value
-            self.data['watching'][uuid]['restock_settings']['in_stock_processing'] = 'in_stock_only' if watch.get(
-                'in_stock_only') else 'all_changes'
-
-            if self.data['watching'][uuid].get('in_stock_only'):
-                del (self.data['watching'][uuid]['in_stock_only'])
-
-    def update_19(self):
-        """Compress old elements.json to elements.deflate, saving disk, this compression is pretty fast."""
-        import zlib
-
-        for uuid, watch in self.data['watching'].items():
-            json_path = os.path.join(self.datastore_path, uuid, "elements.json")
-            deflate_path = os.path.join(self.datastore_path, uuid, "elements.deflate")
-
-            if os.path.exists(json_path):
-                with open(json_path, "rb") as f_j:
-                    with open(deflate_path, "wb") as f_d:
-                        logger.debug(f"Compressing {str(json_path)} to {str(deflate_path)}..")
-                        f_d.write(zlib.compress(f_j.read()))
-                        os.unlink(json_path)
-
-    def update_20(self):
-        """Migrate extract_title_as_title to use_page_title_in_list."""
-        for uuid, watch in self.data['watching'].items():
-            if self.data['watching'][uuid].get('extract_title_as_title'):
-                self.data['watching'][uuid]['use_page_title_in_list'] = self.data['watching'][uuid].get('extract_title_as_title')
-                del self.data['watching'][uuid]['extract_title_as_title']
-
-        if self.data['settings']['application'].get('extract_title_as_title'):
-            # Ensure 'ui' key exists (defensive for edge cases where base_config merge didn't happen)
-            if 'ui' not in self.data['settings']['application']:
-                self.data['settings']['application']['ui'] = {
-                    'use_page_title_in_list': True,
-                    'open_diff_in_new_tab': True,
-                    'socket_io_enabled': True,
-                    'favicons_enabled': True
-                }
-            self.data['settings']['application']['ui']['use_page_title_in_list'] = self.data['settings']['application'].get('extract_title_as_title')
-
-    def update_21(self):
-        """Migrate timezone to scheduler_timezone_default."""
-        if self.data['settings']['application'].get('timezone'):
-            self.data['settings']['application']['scheduler_timezone_default'] = self.data['settings']['application'].get('timezone')
-            del self.data['settings']['application']['timezone']
-
-    def update_23(self):
-        """Some notification formats got the wrong name type."""
-
-        def re_run(formats):
-            sys_n_format = self.data['settings']['application'].get('notification_format')
-            key_exists_as_value = next((k for k, v in formats.items() if v == sys_n_format), None)
-            if key_exists_as_value:  # key of "Plain text"
-                logger.success(f"['settings']['application']['notification_format'] '{sys_n_format}' -> '{key_exists_as_value}'")
-                self.data['settings']['application']['notification_format'] = key_exists_as_value
-
-            for uuid, watch in self.data['watching'].items():
-                n_format = self.data['watching'][uuid].get('notification_format')
-                key_exists_as_value = next((k for k, v in formats.items() if v == n_format), None)
-                if key_exists_as_value and key_exists_as_value != USE_SYSTEM_DEFAULT_NOTIFICATION_FORMAT_FOR_WATCH:  # key of "Plain text"
-                    logger.success(f"['watching'][{uuid}]['notification_format'] '{n_format}' -> '{key_exists_as_value}'")
-                    self.data['watching'][uuid]['notification_format'] = key_exists_as_value  # should be 'text' or whatever
-
-            for uuid, tag in self.data['settings']['application']['tags'].items():
-                n_format = self.data['settings']['application']['tags'][uuid].get('notification_format')
-                key_exists_as_value = next((k for k, v in formats.items() if v == n_format), None)
-                if key_exists_as_value and key_exists_as_value != USE_SYSTEM_DEFAULT_NOTIFICATION_FORMAT_FOR_WATCH:  # key of "Plain text"
-                    logger.success(
-                        f"['settings']['application']['tags'][{uuid}]['notification_format'] '{n_format}' -> '{key_exists_as_value}'")
-                    self.data['settings']['application']['tags'][uuid][
-                        'notification_format'] = key_exists_as_value  # should be 'text' or whatever
-
-        from ..notification import valid_notification_formats
-        formats = deepcopy(valid_notification_formats)
-        re_run(formats)
-        # And in previous versions, it was "text" instead of Plain text, Markdown instead of "Markdown to HTML"
-        formats['text'] = 'Text'
-        formats['markdown'] = 'Markdown'
-        re_run(formats)
-
-    def update_24(self):
-        """RSS types should be inline with the same names as notification types."""
-        rss_format = self.data['settings']['application'].get('rss_content_format')
-        if not rss_format or 'text' in rss_format:
-            # might have been 'plaintext, 'plain text' or something
-            self.data['settings']['application']['rss_content_format'] = RSS_CONTENT_FORMAT_DEFAULT
-        elif 'html' in rss_format:
-            self.data['settings']['application']['rss_content_format'] = 'htmlcolor'
-        else:
-            # safe fallback to text
-            self.data['settings']['application']['rss_content_format'] = RSS_CONTENT_FORMAT_DEFAULT
-
-    def update_25(self):
-        """Different processors now hold their own history.txt."""
-        for uuid, watch in self.data['watching'].items():
-            processor = self.data['watching'][uuid].get('processor')
-            if processor != 'text_json_diff':
-                old_history_txt = os.path.join(self.datastore_path, "history.txt")
-                target_history_name = f"history-{processor}.txt"
-                if os.path.isfile(old_history_txt) and not os.path.isfile(target_history_name):
-                    new_history_txt = os.path.join(self.datastore_path, target_history_name)
-                    logger.debug(f"Renaming history index {old_history_txt} to {new_history_txt}...")
-                    shutil.move(old_history_txt, new_history_txt)
-
-    def migrate_legacy_db_format(self):
-        """
-        Migration: Individual watch persistence (COPY-based, safe rollback).
-
-        Loads legacy url-watches.json format and migrates to:
-        - {uuid}/watch.json (per watch)
-        - changedetection.json (settings only)
-
-        IMPORTANT:
-        - A tarball backup (before-update-26-timestamp.tar.gz) is created before migration
-        - url-watches.json is LEFT INTACT for rollback safety
-        - Users can roll back by simply downgrading to the previous version
-        - Or restore from tarball: tar -xzf before-update-26-*.tar.gz
-
-        This is a dedicated migration release - users upgrade at their own pace.
-        """
-        logger.critical("=" * 80)
-        logger.critical("Running migration: Individual watch persistence (update_26)")
-        logger.critical("COPY-based migration: url-watches.json will remain intact for rollback")
-        logger.critical("=" * 80)
-
-        # Populate settings from legacy data
-        logger.info("Populating settings from legacy data...")
-        watch_count = len(self.data['watching'])
-        logger.success(f"Loaded {watch_count} watches from legacy format")
-
-        # Phase 1: Save all watches to individual files
-        logger.critical(f"Phase 1/4: Saving {watch_count} watches to individual watch.json files...")
-
-        saved_count = 0
-        for uuid, watch in self.data['watching'].items():
-            try:
-                watch.commit()
-                saved_count += 1
-
-                if saved_count % 100 == 0:
-                    logger.info(f"  Progress: {saved_count}/{watch_count} watches migrated...")
-
-            except Exception as e:
-                logger.error(f"Failed to save watch {uuid}: {e}")
-                raise Exception(
-                    f"Migration failed: Could not save watch {uuid}. "
-                    f"url-watches.json remains intact, safe to retry. Error: {e}"
-                )
-
-        logger.critical(f"Phase 1 complete: Saved {saved_count} watches")
-
-        # Phase 2: Verify all files exist
-        logger.critical("Phase 2/4: Verifying all watch.json files were created...")
-
-        missing = []
-        for uuid in self.data['watching'].keys():
-            watch_json = os.path.join(self.datastore_path, uuid, "watch.json")
-            if not os.path.isfile(watch_json):
-                missing.append(uuid)
-
-        if missing:
-            raise Exception(
-                f"Migration failed: {len(missing)} watch files missing: {missing[:5]}... "
-                f"url-watches.json remains intact, safe to retry."
-            )
-
-        logger.critical(f"Phase 2 complete: Verified {watch_count} watch files")
-
-        # Phase 3: Create new settings file
-        logger.critical("Phase 3/4: Creating changedetection.json...")
-
-        try:
-            self._save_settings()
-        except Exception as e:
-            logger.error(f"Failed to create changedetection.json: {e}")
-            raise Exception(
-                f"Migration failed: Could not create changedetection.json. "
-                f"url-watches.json remains intact, safe to retry. Error: {e}"
-            )
-
-        # Phase 4: Verify settings file exists
-        logger.critical("Phase 4/4: Verifying changedetection.json exists...")
-        changedetection_json_new_schema=os.path.join(self.datastore_path, "changedetection.json")
-        if not os.path.isfile(changedetection_json_new_schema):
-            import sys
-            logger.critical("Migration failed, changedetection.json not found after update ran!")
-            sys.exit(1)
-
-
-        logger.critical("Phase 4 complete: Verified changedetection.json exists")
-
-        # Success! Now reload from new format
-        logger.critical("Reloading datastore from new format...")
-        # write it to disk, it will be saved without ['watching'] in the JSON db because we find it from disk glob
-        self._save_settings()
-        logger.success("Datastore reloaded from new format successfully")
-        logger.critical("=" * 80)
-        logger.critical("MIGRATION COMPLETED SUCCESSFULLY!")
-        logger.critical("=" * 80)
-        logger.info("")
-        logger.info("New format:")
-        logger.info(f"  - {watch_count} individual watch.json files created")
-        logger.info(f"  - changedetection.json created (settings only)")
-        logger.info("")
-        logger.info("Rollback safety:")
-        logger.info("  - url-watches.json preserved for rollback")
-        logger.info("  - To rollback: downgrade to previous version and restart")
-        logger.info("  - No manual file operations needed")
-        logger.info("")
-        logger.info("Optional cleanup (after testing new version):")
-        logger.info(f"  - rm {os.path.join(self.datastore_path, 'url-watches.json')}")
-        logger.info("")
-
-    def update_26(self):
-        self.migrate_legacy_db_format()
-
-    # Re-run tag to JSON migration
-    def update_29(self):
-
-        """
-        Migrate tags to individual tag.json files.
-
-        Tags are currently saved only in changedetection.json (settings).
-        This migration ALSO saves them to individual {uuid}/tag.json files,
-        similar to how watches are stored (dual storage).
-
-        Benefits:
-        - Allows atomic tag updates without rewriting entire settings
-        - Enables independent tag versioning/backup
-        - Maintains backwards compatibility (tags stay in settings too)
-        """
-        logger.critical("=" * 80)
-        logger.critical("Running migration: Individual tag persistence (update_28)")
-        logger.critical("Creating individual tag.json files")
-        logger.critical("=" * 80)
-
-        tags = self.data['settings']['application'].get('tags', {})
-        tag_count = len(tags)
-
-        if tag_count == 0:
-            logger.info("No tags found, skipping migration")
-            return
-
-        logger.info(f"Migrating {tag_count} tags to individual tag.json files...")
-
-        saved_count = 0
-        failed_count = 0
-
-        for uuid, tag_data in tags.items():
-            if os.path.isfile(os.path.join(self.datastore_path, uuid, "tag.json")):
-                logger.debug(f"Tag {uuid} tag.json exists, skipping")
-                continue
-            try:
-                tag_data.commit()
-                saved_count += 1
-                if saved_count % 10 == 0:
-                    logger.info(f"  Progress: {saved_count}/{tag_count} tags migrated...")
-
-            except Exception as e:
-                logger.error(f"Failed to save tag {uuid} ({tag_data.get('title', 'unknown')}): {e}")
-                failed_count += 1
-
-        if failed_count > 0:
-            logger.warning(f"Migration complete: {saved_count} tags saved, {failed_count} tags FAILED")
-        else:
-            logger.success(f"Migration complete: {saved_count} tags saved to individual tag.json files")
-
-        # Tags remain in settings for backwards compatibility AND easy access
-        # On next load, _load_tags() will read from tag.json files and merge with settings
-        logger.info("Tags saved to both settings AND individual tag.json files")
-        logger.info("Future tag edits will update both locations (dual storage)")
-        logger.critical("=" * 80)
-
-        # write it to disk, it will be saved without ['tags'] in the JSON db because we find it from disk glob
-        # (left this out by accident in previous update, added tags={} in the changedetection.json save_to_disk)
-        self._save_settings()
-

changedetectionio/templates/_common_fields.html

@@ -6,7 +6,7 @@
 
     <div class="pure-controls">
             <span class="pure-form-message-inline">
-        {{ _('Body for all notifications — You can use') }} <a target="newwindow" href="https://jinja.palletsprojects.com/en/3.0.x/templates/">Jinja2</a> {{ _('templating in the notification title, body and URL, and tokens from below.') }}
+        Body for all notifications &dash; You can use <a target="newwindow" href="https://jinja.palletsprojects.com/en/3.0.x/templates/">Jinja2</a> templating in the notification title, body and URL, and tokens from below.
     </span><br>
         <div data-target="#notification-tokens-info{{ suffix }}" class="toggle-show pure-button button-tag button-xsmall">{{ _('Show token/placeholders') }}
         </div>
@@ -22,77 +22,77 @@
             <tbody>
             <tr>
                 <td><code>{{ '{{base_url}}' }}</code></td>
-                <td>{{ _('The URL of the changedetection.io instance you are running.') }}</td>
+                <td>The URL of the changedetection.io instance you are running.</td>
             </tr>
             <tr>
                 <td><code>{{ '{{watch_url}}' }}</code></td>
-                <td>{{ _('The URL being watched.') }}</td>
+                <td>The URL being watched.</td>
             </tr>
             <tr>
                 <td><code>{{ '{{watch_uuid}}' }}</code></td>
-                <td>{{ _('The UUID of the watch.') }}</td>
+                <td>The UUID of the watch.</td>
             </tr>
             <tr>
                 <td><code>{{ '{{watch_title}}' }}</code></td>
-                <td>{{ _('The page title of the watch, uses <title> if not set, falls back to URL') }}</td>
+                <td>The page title of the watch, uses &lt;title&gt; if not set, falls back to URL</td>
             </tr>
             <tr>
                 <td><code>{{ '{{watch_tag}}' }}</code></td>
-                <td>{{ _('The watch group / tag') }}</td>
+                <td>The watch group / tag</td>
             </tr>
             <tr>
                 <td><code>{{ '{{preview_url}}' }}</code></td>
-                <td>{{ _('The URL of the preview page generated by changedetection.io.') }}</td>
+                <td>The URL of the preview page generated by changedetection.io.</td>
             </tr>
             <tr>
                 <td><code>{{ '{{diff_url}}' }}</code></td>
-                <td>{{ _('The URL of the diff output for the watch.') }}</td>
+                <td>The URL of the diff output for the watch.</td>
             </tr>
             <tr>
                 <td><code>{{ '{{diff}}' }}</code></td>
-                <td>{{ _('The diff output - only changes, additions, and removals') }}</td>
+                <td>The diff output - only changes, additions, and removals</td>
             </tr>
             <tr>
                 <td><code>{{ '{{diff_clean}}' }}</code></td>
-                <td>{{ _('The diff output - only changes, additions, and removals —') }} <i>{{ _('Without (added) prefix or colors') }}</i>
+                <td>The diff output - only changes, additions, and removals &dash; <i>Without (added) prefix or colors</i>
                 </td>
             </tr>
             <tr>
                 <td><code>{{ '{{diff_added}}' }}</code></td>
-                <td>{{ _('The diff output - only changes and additions') }}</td>
+                <td>The diff output - only changes and additions</td>
             </tr>
             <tr>
                 <td><code>{{ '{{diff_added_clean}}' }}</code></td>
-                <td>{{ _('The diff output - only changes and additions —') }} <i>{{ _('Without (added) prefix or colors') }}</i></td>
+                <td>The diff output - only changes and additions &dash; <i>Without (added) prefix or colors</i></td>
             </tr>
             <tr>
                 <td><code>{{ '{{diff_removed}}' }}</code></td>
-                <td>{{ _('The diff output - only changes and removals') }}</td>
+                <td>The diff output - only changes and removals</td>
             </tr>
             <tr>
                 <td><code>{{ '{{diff_removed_clean}}' }}</code></td>
-                <td>{{ _('The diff output - only changes and removals —') }} <i>{{ _('Without (added) prefix or colors') }}</i></td>
+                <td>The diff output - only changes and removals &dash; <i>Without (added) prefix or colors</i></td>
             </tr>
             <tr>
                 <td><code>{{ '{{diff_full}}' }}</code></td>
-                <td>{{ _('The diff output - full difference output') }}</td>
+                <td>The diff output - full difference output</td>
             </tr>
             <tr>
                 <td><code>{{ '{{diff_full_clean}}' }}</code></td>
-                <td>{{ _('The diff output - full difference output —') }} <i>{{ _('Without (added) prefix or colors') }}</i></td>
+                <td>The diff output - full difference output &dash; <i>Without (added) prefix or colors</i></td>
             </tr>
             <tr>
                 <td><code>{{ '{{diff_patch}}' }}</code></td>
-                <td>{{ _('The diff output - patch in unified format') }}</td>
+                <td>The diff output - patch in unified format</td>
             </tr>
             <tr>
                 <td><code>{{ '{{current_snapshot}}' }}</code></td>
-                <td>{{ _('The current snapshot text contents value, useful when combined with JSON or CSS filters') }}
+                <td>The current snapshot text contents value, useful when combined with JSON or CSS filters
                 </td>
             </tr>
             <tr>
                 <td><code>{{ '{{triggered_text}}' }}</code></td>
-                <td>{{ _('Text that tripped the trigger from filters') }}</td>
+                <td>Text that tripped the trigger from filters</td>
 
                 {% if extra_notification_token_placeholder_info %}
                     {% for token in extra_notification_token_placeholder_info %}
@@ -106,8 +106,8 @@
         </table>
 
         <span class="pure-form-message-inline">
-        {{ _('Warning: Contents of') }} <code>{{ '{{diff}}' }}</code>, <code>{{ '{{diff_removed}}' }}</code>, {{ _('and') }} <code>{{ '{{diff_added}}' }}</code> {{ _('depend on how the difference algorithm perceives the change.') }} <br>
-        {{ _('For example, an addition or removal could be perceived as a change in some cases.') }} <a target="newwindow" href="https://github.com/dgtlmoon/changedetection.io/wiki/Using-the-%7B%7Bdiff%7D%7D,-%7B%7Bdiff_added%7D%7D,-and-%7B%7Bdiff_removed%7D%7D-notification-tokens">{{ _('More Here') }}</a> <br>
+        Warning: Contents of <code>{{ '{{diff}}' }}</code>, <code>{{ '{{diff_removed}}' }}</code>, and <code>{{ '{{diff_added}}' }}</code> depend on how the difference algorithm perceives the change. <br>
+        For example, an addition or removal could be perceived as a change in some cases. <a target="newwindow" href="https://github.com/dgtlmoon/changedetection.io/wiki/Using-the-%7B%7Bdiff%7D%7D,-%7B%7Bdiff_added%7D%7D,-and-%7B%7Bdiff_removed%7D%7D-notification-tokens">More Here</a> <br>
         </span>
     </div>
 {%  endmacro %}
@@ -123,15 +123,15 @@
                             }}
                             <div class="pure-form-message-inline">
                                 <p>
-                                <strong>{{ _('Tip:') }}</strong> {{ _('Use') }} <a target="newwindow" href="https://github.com/caronc/apprise">{{ _('AppRise Notification URLs') }}</a> {{ _('for notification to just about any service!') }} <i><a target="newwindow" href="https://github.com/dgtlmoon/changedetection.io/wiki/Notification-configuration-notes">{{ _('Please read the notification services wiki here for important configuration notes') }}</a></i>.<br>
+                                <strong>Tip:</strong> Use <a target="newwindow" href="https://github.com/caronc/apprise">AppRise Notification URLs</a> for notification to just about any service! <i><a target="newwindow" href="https://github.com/dgtlmoon/changedetection.io/wiki/Notification-configuration-notes">Please read the notification services wiki here for important configuration notes</a></i>.<br>
 </p>
                                 <div data-target="#advanced-help-notifications" class="toggle-show pure-button button-tag button-xsmall">{{ _('Show advanced help and tips') }}</div>
                                 <ul style="display: none" id="advanced-help-notifications">
-                                <li><code><a target="newwindow" href="https://github.com/caronc/apprise/wiki/Notify_discord">discord://</a></code> {{ _('(or') }} <code>https://discord.com/api/webhooks...</code>)) {{ _('only supports a maximum') }} <strong>{{ _('2,000 characters') }}</strong> {{ _('of notification text, including the title.') }}</li>
-                                <li><code><a target="newwindow" href="https://github.com/caronc/apprise/wiki/Notify_telegram">tgram://</a></code> {{ _('bots can\'t send messages to other bots, so you should specify chat ID of non-bot user.') }}</li>
-                                <li><code><a target="newwindow" href="https://github.com/caronc/apprise/wiki/Notify_telegram">tgram://</a></code> {{ _('only supports very limited HTML and can fail when extra tags are sent,') }} <a href="https://core.telegram.org/bots/api#html-style">{{ _('read more here') }}</a> {{ _('(or use plaintext/markdown format)') }}</li>
-                                <li><code>gets://</code>, <code>posts://</code>, <code>puts://</code>, <code>deletes://</code> {{ _('for direct API calls (or omit the') }} "<code>s</code>" {{ _('for non-SSL ie') }} <code>get://</code>) <a href="https://github.com/dgtlmoon/changedetection.io/wiki/Notification-configuration-notes#postposts">{{ _('more help here') }}</a></li>
-                                  <li>{{ _('Accepts the') }} <code>{{ '{{token}}' }}</code> {{ _('placeholders listed below') }}</li>
+                                <li><code><a target="newwindow" href="https://github.com/caronc/apprise/wiki/Notify_discord">discord://</a></code> (or <code>https://discord.com/api/webhooks...</code>)) only supports a maximum <strong>2,000 characters</strong> of notification text, including the title.</li>
+                                <li><code><a target="newwindow" href="https://github.com/caronc/apprise/wiki/Notify_telegram">tgram://</a></code> bots can't send messages to other bots, so you should specify chat ID of non-bot user.</li>
+                                <li><code><a target="newwindow" href="https://github.com/caronc/apprise/wiki/Notify_telegram">tgram://</a></code> only supports very limited HTML and can fail when extra tags are sent, <a href="https://core.telegram.org/bots/api#html-style">read more here</a> (or use plaintext/markdown format)</li>
+                                <li><code>gets://</code>, <code>posts://</code>, <code>puts://</code>, <code>deletes://</code> for direct API calls (or omit the "<code>s</code>" for non-SSL ie <code>get://</code>) <a href="https://github.com/dgtlmoon/changedetection.io/wiki/Notification-configuration-notes#postposts">more help here</a></li>
+                                  <li>Accepts the <code>{{ '{{token}}' }}</code> placeholders listed below</li>
                               </ul>
                             </div>
                             <div class="notifications-wrapper">
@@ -156,16 +156,16 @@
                                 <div class="pure-form-message-inline">
                                     <ul>
                                     <li><span class="pure-form-message-inline">
-                                        {{ _('For JSON payloads, use') }} <strong>|tojson</strong> {{ _('without quotes for automatic escaping, for example -') }} <code>{ "name": {{ '{{ watch_title|tojson }}' }} }</code>
+                                        For JSON payloads, use <strong>|tojson</strong> without quotes for automatic escaping, for example - <code>{ "name": {{ '{{ watch_title|tojson }}' }} }</code>
                                     </span></li>
                                     <li><span class="pure-form-message-inline">
-                                        {{ _('URL encoding, use') }} <strong>|urlencode</strong>, {{ _('for example -') }} <code>gets://hook-website.com/test.php?title={{ '{{ watch_title|urlencode }}' }}</code>
+                                        URL encoding, use <strong>|urlencode</strong>, for example - <code>gets://hook-website.com/test.php?title={{ '{{ watch_title|urlencode }}' }}</code>
                                     </span></li>
                                     <li><span class="pure-form-message-inline">
-                                        {{ _('Regular-expression replace, use') }} <strong>|regex_replace</strong>, {{ _('for example -') }}   <code>{{ "{{ \"hello world 123\" | regex_replace('[0-9]+', 'no-more-numbers') }}" }}</code>
+                                        Regular-expression replace, use <strong>|regex_replace</strong>, for example -   <code>{{ "{{ \"hello world 123\" | regex_replace('[0-9]+', 'no-more-numbers') }}" }}</code>
                                     </span></li>
                                     <li><span class="pure-form-message-inline">
-                                        {{ _('For a complete reference of all Jinja2 built-in filters, users can refer to the') }} <a href="https://jinja.palletsprojects.com/en/3.1.x/templates/#builtin-filters">https://jinja.palletsprojects.com/en/3.1.x/templates/#builtin-filters</a>
+                                        For a complete reference of all Jinja2 built-in filters, users can refer to the <a href="https://jinja.palletsprojects.com/en/3.1.x/templates/#builtin-filters">https://jinja.palletsprojects.com/en/3.1.x/templates/#builtin-filters</a>
                                     </span></li>
                                     </ul>
                                     <br>

changedetectionio/templates/_helpers.html

@@ -1,5 +1,5 @@
 {% macro render_field(field) %}
-    <div {% if field.errors or field.top_errors %} class="error" {% endif %}>{{ field.label }}</div>
+    <div {% if field.errors or field.top_errors %} class="error" {% endif %}><label for="{{ field.id }}">{{ field.label.text | string | forceescape }}</label></div>
     <div {% if field.errors or field.top_errors %} class="error" {% endif %}>{{ field(**kwargs)|safe }}
         {% if field.top_errors %}
             top
@@ -59,7 +59,7 @@
 
 {% macro render_ternary_field(field, BooleanField=false) %}
   {% if BooleanField %}
-    {% set dummy = field.__setattr__('boolean_mode', true) %}
+    {% set _ = field.__setattr__('boolean_mode', true) %}
   {% endif %}
   <div class="ternary-field {% if field.errors %} error {% endif %}">
     <div class="ternary-field-label"><label for="{{ field.id }}">{{ field.label.text | string | forceescape }}</label></div>
@@ -113,17 +113,17 @@
 
 {% macro render_fieldlist_with_inline_errors(fieldlist) %}
   {# Specialized macro for FieldList(FormField(...)) that renders errors inline with each field #}
-  <div {% if fieldlist.errors %} class="error" {% endif %}>{{ _(fieldlist.label.text | string) }}</div>
+  <div {% if fieldlist.errors %} class="error" {% endif %}>{{ fieldlist.label }}</div>
   <div {% if fieldlist.errors %} class="error" {% endif %}>
     <ul id="{{ fieldlist.id }}">
       {% for entry in fieldlist %}
         <li {% if entry.errors %} class="error" {% endif %}>
-          <label for="{{ entry.id }}" {% if entry.errors %} class="error" {% endif %}>{{ _(fieldlist.label.text | string) }}-{{ loop.index0 }}</label>
+          <label for="{{ entry.id }}" {% if entry.errors %} class="error" {% endif %}>{{ fieldlist.label.text }}-{{ loop.index0 }}</label>
           <table id="{{ entry.id }}" {% if entry.errors %} class="error" {% endif %}>
             <tbody>
               {% for subfield in entry %}
                 <tr {% if subfield.errors %} class="error" {% endif %}>
-                  <th {% if subfield.errors %} class="error" {% endif %}><label for="{{ subfield.id }}" {% if subfield.errors %} class="error" {% endif %}>{{ subfield.label.text | string }}</label></th>
+                  <th {% if subfield.errors %} class="error" {% endif %}><label for="{{ subfield.id }}" {% if subfield.errors %} class="error" {% endif %}>{{ subfield.label.text }}</label></th>
                   <td {% if subfield.errors %} class="error" {% endif %}>
                     {{ subfield(**kwargs)|safe }}
                     {% if subfield.errors %}
@@ -148,7 +148,7 @@
   <div class="fieldlist_formfields" id="{{ table_id }}">
     <div class="fieldlist-header">
       {% for subfield in fieldlist[0] %}
-        <div class="fieldlist-header-cell">{{ subfield.label.text | string }}</div>
+        <div class="fieldlist-header-cell">{{ subfield.label }}</div>
       {% endfor %}
       <div class="fieldlist-header-cell">{{ _('Actions') }}</div>
     </div>

changedetectionio/templates/base.html

@@ -27,7 +27,7 @@
     <link rel="apple-touch-icon" sizes="180x180" href="{{url_for('static_content', group='favicons', filename='apple-touch-icon.png')}}">
     <link rel="icon" type="image/png" sizes="32x32" href="{{url_for('static_content', group='favicons', filename='favicon-32x32.png')}}">
     <link rel="icon" type="image/png" sizes="16x16" href="{{url_for('static_content', group='favicons', filename='favicon-16x16.png')}}">
-    <link rel="manifest" href="{{url_for('static_content', group='favicons', filename='site.webmanifest')}}" crossorigin="use-credentials">
+    <link rel="manifest" href="{{url_for('static_content', group='favicons', filename='site.webmanifest')}}">
     <link rel="mask-icon" href="{{url_for('static_content', group='favicons', filename='safari-pinned-tab.svg')}}" color="#5bbad5">
     <link rel="shortcut icon" href="{{url_for('static_content', group='favicons', filename='favicon.ico')}}">
     <meta name="msapplication-TileColor" content="#da532c">
@@ -265,9 +265,6 @@
           </a>
           {% endfor %}
         </div>
-        <div>
-            <a href="{{ url_for('ui.delete_locale_language_session_var_if_it_exists', redirect=request.path) }}" >{{ _('Auto-detect from browser') }}</a>
-        </div>
         <div>
             {{ _('Language support is in beta, please help us improve by opening a PR on GitHub with any updates.') }}
         </div>

changedetectionio/templates/edit/text-options.html

@@ -10,7 +10,6 @@
                         <li>{{ _('Trigger text is processed from the result-text that comes out of any CSS/JSON Filters for this monitor') }}</li>
                         <li>{{ _('Each line is processed separately (think of each line as "OR")') }}</li>
                         <li>{{ _('Note: Wrap in forward slash / to use regex example:') }} <code>/foo\d/</code></li>
-                        <li>{{ _('You can also use')}} <a href="#conditions">{{ _('conditions')}}</a> - {{ _('"Page text" - with Contains, Starts With, Not Contains and many more' ) }} <code>/foo\d/</code></li>
                     </ul>
                         </span>
                     </div>

changedetectionio/templates/menu.html

@@ -13,11 +13,8 @@
     <li class="pure-menu-item menu-collapsible {% if request.endpoint.startswith('imports.') %}active{% endif %}">
         <a href="{{ url_for('imports.import_page') }}" class="pure-menu-link">{{ _('IMPORT') }}</a>
     </li>
-    <li class="pure-menu-item" id="menu-pause">
-        <a href="{{ url_for('settings.toggle_all_paused') }}" ><img src="{{url_for('static_content', group='images', filename='pause.svg')}}" alt="{% if all_paused %}{{ _('Resume automatic scheduling') }}{% else %}{{ _('Pause auto-queue scheduling of watches') }}{% endif %}" title="{% if all_paused %}{{ _('Scheduling is paused - click to resume') }}{% else %}{{ _('Pause auto-queue scheduling of watches') }}{% endif %}" class="icon icon-pause"{% if not all_paused %} style="opacity: 0.3"{% endif %}></a>
-    </li>
-    <li class="pure-menu-item " id="menu-mute">
-        <a href="{{ url_for('settings.toggle_all_muted') }}" ><img src="{{url_for('static_content', group='images', filename='bell-off.svg')}}" alt="{% if all_muted %}{{ _('Unmute notifications') }}{% else %}{{ _('Mute notifications') }}{% endif %}" title="{% if all_muted %}{{ _('Notifications are muted - click to unmute') }}{% else %}{{ _('Mute notifications') }}{% endif %}" class="icon icon-mute"{% if not all_muted %} style="opacity: 0.3"{% endif %}></a>
+    <li class="pure-menu-item menu-collapsible {% if request.endpoint.startswith('backups.') %}active{% endif %}">
+        <a href="{{ url_for('backups.index') }}" class="pure-menu-link">{{ _('BACKUPS') }}</a>
     </li>
 {% else %}
     <li class="pure-menu-item menu-collapsible">
@@ -33,7 +30,7 @@
 
 {% else %}
     <li class="pure-menu-item menu-collapsible">
-        <a class="pure-menu-link" href="https://changedetection.io">{{ _('Website Change Detection and Notification.') }}</a>
+        <a class="pure-menu-link" href="https://changedetection.io">Website Change Detection and Notification.</a>
     </li>
 {% endif %}
     <li class="pure-menu-item menu-collapsible" id="inline-menu-extras-group">
@@ -50,9 +47,7 @@
             <span class="visually-hidden">{{ _('Change language') }}</span>
             <span class="{{ get_flag_for_locale(get_locale()) }}" id="language-selector-flag"></span>
         </button>
-        <a class="github-link" href="https://github.com/dgtlmoon/changedetection.io"
-           target="_blank" 
-          rel="noopener" >
+        <a class="github-link" href="https://github.com/dgtlmoon/changedetection.io">
             {% include "svgs/github.svg" %}
         </a>
     </li>

changedetectionio/test_cli_opts.sh

@@ -1,265 +0,0 @@
-#!/bin/bash
-# Test script for CLI options - Parallel execution
-# Tests -u, -uN, -r, -b flags
-
-set -u  # Exit on undefined variables
-
-# Color output
-RED='\033[0;31m'
-GREEN='\033[0;32m'
-YELLOW='\033[1;33m'
-NC='\033[0m' # No Color
-
-# Test results directory (for parallel safety)
-TEST_RESULTS_DIR="/tmp/cli-test-results-$$"
-mkdir -p "$TEST_RESULTS_DIR"
-
-# Cleanup function
-cleanup() {
-    echo ""
-    echo "=== Cleaning up test directories ==="
-    rm -rf /tmp/cli-test-* 2>/dev/null || true
-    rm -rf "$TEST_RESULTS_DIR" 2>/dev/null || true
-    # Kill any hanging processes
-    pkill -f "changedetection.py.*cli-test" 2>/dev/null || true
-}
-trap cleanup EXIT
-
-# Helper to record test result
-record_result() {
-    local test_num=$1
-    local status=$2  # pass or fail
-    local message=$3
-
-    echo "$status|$message" > "$TEST_RESULTS_DIR/test_${test_num}.result"
-}
-
-# Run a test in background
-run_test() {
-    local test_num=$1
-    local test_name=$2
-    local test_func=$3
-
-    (
-        echo -e "${YELLOW}[Test $test_num]${NC} $test_name"
-        if $test_func "$test_num"; then
-            record_result "$test_num" "pass" "$test_name"
-            echo -e "${GREEN}✓ PASS${NC}: $test_name"
-        else
-            record_result "$test_num" "fail" "$test_name"
-            echo -e "${RED}✗ FAIL${NC}: $test_name"
-        fi
-    ) &
-}
-
-# =============================================================================
-# Test Functions (each runs independently)
-# =============================================================================
-
-test_help_flag() {
-    local test_id=$1
-    timeout 3 python3 changedetection.py --help 2>&1 | grep -q "Add URLs on startup"
-}
-
-test_version_flag() {
-    local test_id=$1
-    timeout 3 python3 changedetection.py --version 2>&1 | grep -qE "changedetection.io [0-9]+\.[0-9]+"
-}
-
-test_single_url() {
-    local test_id=$1
-    local dir="/tmp/cli-test-single-${test_id}-$$"
-    timeout 10 python3 changedetection.py -d "$dir" -C -u https://example.com -b &>/dev/null
-    # Count watch directories (UUID directories containing watch.json)
-    [ "$(find "$dir" -mindepth 2 -maxdepth 2 -name 'watch.json' | wc -l)" -eq 1 ]
-}
-
-test_multiple_urls() {
-    local test_id=$1
-    local dir="/tmp/cli-test-multi-${test_id}-$$"
-    timeout 12 python3 changedetection.py -d "$dir" -C \
-        -u https://example.com \
-        -u https://github.com \
-        -u https://httpbin.org \
-        -b &>/dev/null
-    # Count watch directories (UUID directories containing watch.json)
-    [ "$(find "$dir" -mindepth 2 -maxdepth 2 -name 'watch.json' | wc -l)" -eq 3 ]
-}
-
-test_url_with_options() {
-    local test_id=$1
-    local dir="/tmp/cli-test-opts-${test_id}-$$"
-    timeout 10 python3 changedetection.py -d "$dir" -C \
-        -u https://example.com \
-        -u0 '{"title":"Test Site","processor":"text_json_diff"}' \
-        -b &>/dev/null
-    # Check that at least one watch.json contains the title "Test Site"
-    python3 -c "
-import json, glob, sys
-watch_files = glob.glob('$dir/*/watch.json')
-for wf in watch_files:
-    with open(wf) as f:
-        data = json.load(f)
-        if data.get('title') == 'Test Site':
-            sys.exit(0)
-sys.exit(1)
-"
-}
-
-test_multiple_urls_with_options() {
-    local test_id=$1
-    local dir="/tmp/cli-test-multi-opts-${test_id}-$$"
-    timeout 12 python3 changedetection.py -d "$dir" -C \
-        -u https://example.com \
-        -u0 '{"title":"Site One"}' \
-        -u https://github.com \
-        -u1 '{"title":"Site Two"}' \
-        -b &>/dev/null
-    # Check that we have 2 watches and both titles are present
-    python3 -c "
-import json, glob, sys
-watch_files = glob.glob('$dir/*/watch.json')
-if len(watch_files) != 2:
-    sys.exit(1)
-titles = []
-for wf in watch_files:
-    with open(wf) as f:
-        data = json.load(f)
-        titles.append(data.get('title'))
-sys.exit(0 if 'Site One' in titles and 'Site Two' in titles else 1)
-"
-}
-
-test_batch_mode_exit() {
-    local test_id=$1
-    local dir="/tmp/cli-test-batch-${test_id}-$$"
-    local start=$(date +%s)
-    timeout 15 python3 changedetection.py -d "$dir" -C \
-        -u https://example.com \
-        -b &>/dev/null
-    local end=$(date +%s)
-    local elapsed=$((end - start))
-    [ $elapsed -lt 14 ]
-}
-
-test_recheck_all() {
-    local test_id=$1
-    local dir="/tmp/cli-test-recheck-all-${test_id}-$$"
-    # Create a watch using CLI, then recheck it
-    timeout 10 python3 changedetection.py -d "$dir" -C -u https://example.com -b &>/dev/null
-    # Now recheck all watches
-    timeout 10 python3 changedetection.py -d "$dir" -r all -b 2>&1 | grep -q "Queuing"
-}
-
-test_recheck_specific() {
-    local test_id=$1
-    local dir="/tmp/cli-test-recheck-uuid-${test_id}-$$"
-    # Create 2 watches using CLI
-    timeout 12 python3 changedetection.py -d "$dir" -C \
-        -u https://example.com \
-        -u https://github.com \
-        -b &>/dev/null
-    # Get the UUIDs that were created
-    local uuids=$(find "$dir" -mindepth 2 -maxdepth 2 -name 'watch.json' -exec dirname {} \; | xargs -n1 basename | tr '\n' ',' | sed 's/,$//')
-    # Now recheck specific UUIDs
-    timeout 10 python3 changedetection.py -d "$dir" -r "$uuids" -b 2>&1 | grep -q "Queuing"
-}
-
-test_combined_operations() {
-    local test_id=$1
-    local dir="/tmp/cli-test-combined-${test_id}-$$"
-    timeout 12 python3 changedetection.py -d "$dir" -C \
-        -u https://example.com \
-        -u https://github.com \
-        -r all \
-        -b &>/dev/null
-    # Count watch directories (UUID directories containing watch.json)
-    [ "$(find "$dir" -mindepth 2 -maxdepth 2 -name 'watch.json' | wc -l)" -eq 2 ]
-}
-
-test_invalid_json() {
-    local test_id=$1
-    local dir="/tmp/cli-test-invalid-${test_id}-$$"
-    timeout 5 python3 changedetection.py -d "$dir" -C \
-        -u https://example.com \
-        -u0 'invalid json here' \
-        2>&1 | grep -qi "invalid json\|json decode error"
-}
-
-test_create_directory() {
-    local test_id=$1
-    local dir="/tmp/cli-test-create-${test_id}-$$/nested/path"
-    timeout 10 python3 changedetection.py -d "$dir" -C \
-        -u https://example.com \
-        -b &>/dev/null
-    [ -d "$dir" ]
-}
-
-# =============================================================================
-# Main Test Execution
-# =============================================================================
-
-echo "=========================================="
-echo "  CLI Options Test Suite (Parallel)"
-echo "=========================================="
-echo ""
-
-# Launch all tests in parallel
-run_test 1 "Help flag (--help) shows usage without initialization" test_help_flag
-run_test 2 "Version flag (--version) displays version" test_version_flag
-run_test 3 "Add single URL with -u flag" test_single_url
-run_test 4 "Add multiple URLs with multiple -u flags" test_multiple_urls
-run_test 5 "Add URL with JSON options using -u0" test_url_with_options
-run_test 6 "Add multiple URLs with different options (-u0, -u1)" test_multiple_urls_with_options
-run_test 7 "Batch mode (-b) exits automatically after processing" test_batch_mode_exit
-run_test 8 "Recheck all watches with -r all" test_recheck_all
-run_test 9 "Recheck specific watches with -r UUID" test_recheck_specific
-run_test 10 "Combined: Add URLs and recheck all with -u and -r all" test_combined_operations
-run_test 11 "Invalid JSON in -u0 option should show error" test_invalid_json
-run_test 12 "Create datastore directory with -C flag" test_create_directory
-
-# Wait for all tests to complete
-echo ""
-echo "Waiting for all tests to complete..."
-wait
-
-# Collect results
-echo ""
-echo "=========================================="
-echo "  Test Summary"
-echo "=========================================="
-
-TESTS_RUN=0
-TESTS_PASSED=0
-TESTS_FAILED=0
-
-for result_file in "$TEST_RESULTS_DIR"/test_*.result; do
-    if [ -f "$result_file" ]; then
-        TESTS_RUN=$((TESTS_RUN + 1))
-        status=$(cut -d'|' -f1 < "$result_file")
-        if [ "$status" = "pass" ]; then
-            TESTS_PASSED=$((TESTS_PASSED + 1))
-        else
-            TESTS_FAILED=$((TESTS_FAILED + 1))
-        fi
-    fi
-done
-
-echo "Tests run:    $TESTS_RUN"
-echo -e "${GREEN}Tests passed: $TESTS_PASSED${NC}"
-if [ $TESTS_FAILED -gt 0 ]; then
-    echo -e "${RED}Tests failed: $TESTS_FAILED${NC}"
-else
-    echo -e "${GREEN}Tests failed: $TESTS_FAILED${NC}"
-fi
-echo "=========================================="
-echo ""
-
-# Exit with appropriate code
-if [ $TESTS_FAILED -gt 0 ]; then
-    echo -e "${RED}Some tests failed!${NC}"
-    exit 1
-else
-    echo -e "${GREEN}All tests passed!${NC}"
-    exit 0
-fi

changedetectionio/tests/conftest.py

@@ -5,16 +5,13 @@ from threading import Thread
 
 import pytest
 import arrow
+from changedetectionio import changedetection_app
 from changedetectionio import store
 import os
 import sys
+from loguru import logger
 
-# CRITICAL: Set short timeout for tests to prevent 45-second hangs
-# When test server is slow/unresponsive, workers fail fast instead of holding UUIDs for 45s
-# This prevents exponential priority growth from repeated deferrals (priority × 10 each defer)
-os.environ['DEFAULT_SETTINGS_REQUESTS_TIMEOUT'] = '5'
-
-from changedetectionio.flask_app import init_app_secret, changedetection_app
+from changedetectionio.flask_app import init_app_secret
 from changedetectionio.tests.util import live_server_setup, new_live_server_setup
 
 # https://github.com/pallets/flask/blob/1.1.2/examples/tutorial/tests/test_auth.py
@@ -34,93 +31,6 @@ def reportlog(pytestconfig):
     logger.remove(handler_id)
 
 
-@pytest.fixture(autouse=True)
-def per_test_log_file(request):
-    """Create a separate log file for each test function with pytest output."""
-    import re
-
-    # Create logs directory if it doesn't exist
-    log_dir = os.path.join(os.path.dirname(__file__), "logs")
-    os.makedirs(log_dir, exist_ok=True)
-
-    # Generate log filename from test name and worker ID (for parallel runs)
-    test_name = request.node.name
-
-    # Sanitize test name - replace unsafe characters with underscores
-    # Keep only alphanumeric, dash, underscore, and period
-    safe_test_name = re.sub(r'[^\w\-.]', '_', test_name)
-
-    # Limit length to avoid filesystem issues (max 200 chars)
-    if len(safe_test_name) > 200:
-        # Keep first 150 chars + hash of full name + last 30 chars
-        import hashlib
-        name_hash = hashlib.md5(test_name.encode()).hexdigest()[:8]
-        safe_test_name = f"{safe_test_name[:150]}_{name_hash}_{safe_test_name[-30:]}"
-
-    worker_id = os.environ.get('PYTEST_XDIST_WORKER', 'master')
-    log_file = os.path.join(log_dir, f"{safe_test_name}_{worker_id}.log")
-
-    # Add file handler for this test with TRACE level
-    handler_id = logger.add(
-        log_file,
-        format="{time:YYYY-MM-DD HH:mm:ss.SSS} | {level: <8} | {process} | {name}:{function}:{line} - {message}",
-        level="TRACE",
-        mode="w",  # Overwrite if exists
-        enqueue=True  # Thread-safe
-    )
-
-    logger.info(f"=== Starting test: {test_name} (worker: {worker_id}) ===")
-    logger.info(f"Test location: {request.node.nodeid}")
-
-    yield
-
-    # Capture test outcome (PASSED/FAILED/SKIPPED/ERROR)
-    outcome = "UNKNOWN"
-    exc_info = None
-    stdout = None
-    stderr = None
-
-    if hasattr(request.node, 'rep_call'):
-        outcome = request.node.rep_call.outcome.upper()
-        if request.node.rep_call.failed:
-            exc_info = request.node.rep_call.longreprtext
-        # Capture stdout/stderr from call phase
-        if hasattr(request.node.rep_call, 'sections'):
-            for section_name, section_content in request.node.rep_call.sections:
-                if 'stdout' in section_name.lower():
-                    stdout = section_content
-                elif 'stderr' in section_name.lower():
-                    stderr = section_content
-    elif hasattr(request.node, 'rep_setup'):
-        if request.node.rep_setup.failed:
-            outcome = "SETUP_FAILED"
-            exc_info = request.node.rep_setup.longreprtext
-
-    logger.info(f"=== Test Result: {outcome} ===")
-
-    if exc_info:
-        logger.error(f"=== Test Failure Details ===\n{exc_info}")
-
-    if stdout:
-        logger.info(f"=== Captured stdout ===\n{stdout}")
-
-    if stderr:
-        logger.warning(f"=== Captured stderr ===\n{stderr}")
-
-    logger.info(f"=== Finished test: {test_name} ===")
-    logger.remove(handler_id)
-
-
-@pytest.hookimpl(tryfirst=True, hookwrapper=True)
-def pytest_runtest_makereport(item, call):
-    """Hook to capture test results and attach to the test node."""
-    outcome = yield
-    rep = outcome.get_result()
-
-    # Store report on the test node for access in fixtures
-    setattr(item, f"rep_{rep.when}", rep)
-
-
 @pytest.fixture
 def environment(mocker):
     """Mock arrow.now() to return a fixed datetime for testing jinja2 time extension."""
@@ -249,64 +159,9 @@ def prepare_test_function(live_server, datastore_path):
     # CRITICAL: Get datastore and stop it from writing stale data
     datastore = live_server.app.config.get('DATASTORE')
 
-    # Clear the queue before starting the test to prevent state leakage
-    from changedetectionio.flask_app import update_q
-    while not update_q.empty():
-        try:
-            update_q.get_nowait()
-        except:
-            break
-
-    # Add test helper methods to the app for worker management
-    def set_workers(count):
-        """Set the number of workers for testing - brutal shutdown, no delays"""
-        from changedetectionio import worker_pool
-        from changedetectionio.flask_app import update_q, notification_q
-
-        current_count = worker_pool.get_worker_count()
-
-       # Special case: Setting to 0 means shutdown all workers brutally
-        if count == 0:
-            logger.debug(f"Brutally shutting down all {current_count} workers")
-            worker_pool.shutdown_workers()
-            return {
-                'status': 'success',
-                'message': f'Shutdown all {current_count} workers',
-                'previous_count': current_count,
-                'current_count': 0
-            }
-
-        # Adjust worker count (no delays, no verification)
-        result = worker_pool.adjust_async_worker_count(
-            count,
-            update_q=update_q,
-            notification_q=notification_q,
-            app=live_server.app,
-            datastore=datastore
-        )
-
-        return result
-
-    def check_all_workers_alive(expected_count):
-        """Check that all expected workers are alive"""
-        from changedetectionio import worker_pool
-        from changedetectionio.flask_app import update_q, notification_q
-        result = worker_pool.check_worker_health(
-            expected_count,
-            update_q=update_q,
-            notification_q=notification_q,
-            app=live_server.app,
-            datastore=datastore
-        )
-        assert result['status'] == 'healthy', f"Workers not healthy: {result['message']}"
-        return result
-
-    # Attach helper methods to app for easy test access
-    live_server.app.set_workers = set_workers
-    live_server.app.check_all_workers_alive = check_all_workers_alive
-
-
-
+    # Prevent background thread from writing during cleanup/reload
+    datastore.needs_write = False
+    datastore.needs_write_urgent = False
 
     # CRITICAL: Clean up any files from previous tests
     # This ensures a completely clean directory
@@ -328,31 +183,10 @@ def prepare_test_function(live_server, datastore_path):
 
     yield
 
-    # Cleanup: Clear watches and queue after test
+    # Cleanup: Clear watches again after test
     try:
-        from changedetectionio.flask_app import update_q
-        from pathlib import Path
-
-        # Clear the queue to prevent leakage to next test
-        while not update_q.empty():
-            try:
-                update_q.get_nowait()
-            except:
-                break
-
         datastore.data['watching'] = {}
-
-        # Delete any old watch metadata JSON files
-        base_path = Path(datastore.datastore_path).resolve()
-        max_depth = 2
-
-        for file in base_path.rglob("*.json"):
-            # Calculate depth relative to base path
-            depth = len(file.relative_to(base_path).parts) - 1
-
-            if depth <= max_depth and file.is_file():
-                file.unlink()
-
+        datastore.needs_write = True
     except Exception as e:
         logger.warning(f"Error during datastore cleanup: {e}")
 
@@ -404,20 +238,17 @@ def app(request, datastore_path):
     app.config['TEST_DATASTORE_PATH'] = datastore_path
 
     def teardown():
-        import threading
-        import time
-
         # Stop all threads and services
         datastore.stop_thread = True
         app.config.exit.set()
-
+        
         # Shutdown workers gracefully before loguru cleanup
         try:
-            from changedetectionio import worker_pool
-            worker_pool.shutdown_workers()
+            from changedetectionio import worker_handler
+            worker_handler.shutdown_workers()
         except Exception:
             pass
-
+            
         # Stop socket server threads
         try:
             from changedetectionio.flask_app import socketio_server
@@ -425,35 +256,14 @@ def app(request, datastore_path):
                 socketio_server.shutdown()
         except Exception:
             pass
-
-        # Get all active threads before cleanup
-        main_thread = threading.main_thread()
-        active_threads = [t for t in threading.enumerate() if t != main_thread and t.is_alive()]
-
-        # Wait for non-daemon threads to finish (with timeout)
-        timeout = 2.0  # 2 seconds max wait
-        start_time = time.time()
-
-        for thread in active_threads:
-            if not thread.daemon:
-                remaining_time = timeout - (time.time() - start_time)
-                if remaining_time > 0:
-                    logger.debug(f"Waiting for non-daemon thread to finish: {thread.name}")
-                    thread.join(timeout=remaining_time)
-                    if thread.is_alive():
-                        logger.warning(f"Thread {thread.name} did not finish in time")
-
-        # Give daemon threads a moment to finish their current work
-        time.sleep(0.2)
-
-        # Log any threads still running
-        remaining_threads = [t for t in threading.enumerate() if t != main_thread and t.is_alive()]
-        if remaining_threads:
-            logger.debug(f"Threads still running after teardown: {[t.name for t in remaining_threads]}")
-
+        
+        # Give threads a moment to finish their shutdown
+        import time
+        time.sleep(0.1)
+        
         # Remove all loguru handlers to prevent "closed file" errors
         logger.remove()
-
+        
         # Cleanup files
         cleanup(app_config['datastore_path'])
 
@@ -462,3 +272,4 @@ def app(request, datastore_path):
     yield app
 
 
+

changedetectionio/tests/plugins/test_processor.py

@@ -1,41 +0,0 @@
-import time
-
-from flask import url_for
-
-from changedetectionio.tests.util import wait_for_all_checks
-
-
-def test_check_plugin_processor(client, live_server, measure_memory_usage, datastore_path):
-    # requires os-int intelligence plugin installed (first basic one we test with)
-
-    res = client.get(url_for("watchlist.index"))
-    assert b'OSINT Reconnaissance' in res.data, "Must have the OSINT plugin installed at test time"
-    assert b'<input checked id="processor-0" name="processor" type="radio" value="text_json_diff">' in res.data, "But the first text_json_diff processor should always be selected by default in quick watch form"
-
-    res = client.post(
-        url_for("ui.ui_views.form_quick_watch_add"),
-        data={"url": 'http://127.0.0.1', "tags": '', 'processor': 'osint_recon'},
-        follow_redirects=True
-    )
-    assert b"Watch added" in res.data
-    client.get(url_for("ui.form_watch_checknow"), follow_redirects=True)
-
-    wait_for_all_checks(client)
-
-    res = client.get(
-        url_for("ui.ui_preview.preview_page", uuid="first"),
-        follow_redirects=True
-    )
-
-    assert b'Target: http://127.0.0.1' in res.data
-    assert b'DNSKEY Records' in res.data
-    wait_for_all_checks(client)
-
-
-    # Now change it to something that doesnt exist
-    uuid = next(iter(live_server.app.config['DATASTORE'].data['watching']))
-    live_server.app.config['DATASTORE'].data['watching'][uuid]['processor'] = "now_missing"
-    client.get(url_for("ui.form_watch_checknow"), follow_redirects=True)
-    wait_for_all_checks(client)
-    res = client.get(url_for("watchlist.index"))
-    assert b"Exception: Processor module" in res.data and b'now_missing' in res.data, f'Should register that the plugin is missing for {uuid}'

changedetectionio/tests/test_access_control.py

@@ -124,6 +124,7 @@ def test_check_access_control(app, client, live_server, measure_memory_usage, da
 
         # Menu should be available now
         assert b"SETTINGS" in res.data
+        assert b"BACKUP" in res.data
         assert b"IMPORT" in res.data
         assert b"LOG OUT" in res.data
         assert b"time_between_check-minutes" in res.data