Updating pyppeteer-ng (mainly newer pillow release)

.dockerignore

@@ -1,31 +1,18 @@
-# Git
-.git/
-.gitignore
+.git
+.github
+changedetectionio/processors/__pycache__
+changedetectionio/api/__pycache__
+changedetectionio/model/__pycache__
+changedetectionio/blueprint/price_data_follower/__pycache__
+changedetectionio/blueprint/tags/__pycache__
+changedetectionio/blueprint/__pycache__
+changedetectionio/blueprint/browser_steps/__pycache__
+changedetectionio/fetchers/__pycache__
+changedetectionio/tests/visualselector/__pycache__
+changedetectionio/tests/restock/__pycache__
+changedetectionio/tests/__pycache__
+changedetectionio/tests/fetchers/__pycache__
+changedetectionio/tests/unit/__pycache__
+changedetectionio/tests/proxy_list/__pycache__
+changedetectionio/__pycache__
 
-# GitHub
-.github/
-
-# Byte-compiled / optimized / DLL files
-**/__pycache__
-**/*.py[cod]
-
-# Caches
-.mypy_cache/
-.pytest_cache/
-.ruff_cache/
-
-# Distribution / packaging
-build/
-dist/
-*.egg-info*
-
-# Virtual environment
-.env
-.venv/
-venv/
-
-# IntelliJ IDEA
-.idea/
-
-# Visual Studio
-.vscode/

.github/ISSUE_TEMPLATE/bug_report.md

@@ -27,10 +27,6 @@ A clear and concise description of what the bug is.
 **Version**
 *Exact version* in the top right area: 0....
 
-**How did you install?**
-
-Docker, Pip, from source directly etc
-
 **To Reproduce**
 
 Steps to reproduce the behavior:

.github/dependabot.yml

@@ -4,10 +4,6 @@ updates:
     directory: /
     schedule:
       interval: "weekly"
-    "caronc/apprise":
-      versioning-strategy: "increase"
-      schedule:
-        interval: "daily"
     groups:
       all:
         patterns:

.github/test/Dockerfile-alpine

@@ -2,33 +2,32 @@
 # Test that we can still build on Alpine (musl modified libc https://musl.libc.org/)
 # Some packages wont install via pypi because they dont have a wheel available under this architecture.
 
-FROM ghcr.io/linuxserver/baseimage-alpine:3.21
+FROM ghcr.io/linuxserver/baseimage-alpine:3.18
 ENV PYTHONUNBUFFERED=1
 
 COPY requirements.txt /requirements.txt
 
 RUN \
- apk add --update --no-cache --virtual=build-dependencies \
-    build-base \
+  apk add --update --no-cache --virtual=build-dependencies \
     cargo \
-    git \
+    g++ \
+    gcc \
     jpeg-dev \
     libc-dev \
     libffi-dev \
+    libjpeg \
     libxslt-dev \
+    make \
     openssl-dev \
+    py3-wheel \
     python3-dev \
-    zip \
     zlib-dev && \
   apk add --update --no-cache \
-    libjpeg \
     libxslt \
-    nodejs \
-    poppler-utils \
-    python3 && \
+    python3 \
+    py3-pip && \
   echo "**** pip3 install test of changedetection.io ****" && \
-  python3 -m venv /lsiopy  && \
-  pip install -U pip wheel setuptools && \
-  pip install -U --no-cache-dir --find-links https://wheel-index.linuxserver.io/alpine-3.21/ -r /requirements.txt && \
+  pip3 install -U pip wheel setuptools && \
+  pip3 install -U --no-cache-dir --find-links https://wheel-index.linuxserver.io/alpine-3.18/ -r /requirements.txt && \
   apk del --purge \
     build-dependencies

.github/workflows/containers.yml

@@ -88,14 +88,14 @@ jobs:
       - name: Build and push :dev
         id: docker_build
         if: ${{ github.ref }} == "refs/heads/master"
-        uses: docker/build-push-action@v6
+        uses: docker/build-push-action@v5
         with:
           context: ./
           file: ./Dockerfile
           push: true
           tags: |
             ${{ secrets.DOCKER_HUB_USERNAME }}/changedetection.io:dev,ghcr.io/${{ github.repository }}:dev
-          platforms: linux/amd64,linux/arm64,linux/arm/v7,linux/arm/v8,linux/arm64/v8
+          platforms: linux/amd64,linux/arm64,linux/arm/v6,linux/arm/v7,linux/arm/v8
           cache-from: type=gha
           cache-to: type=gha,mode=max
 
@@ -103,29 +103,20 @@ jobs:
 #          provenance: false
 
       # A new tagged release is required, which builds :tag and :latest
-      - name: Docker meta :tag
-        if: github.event_name == 'release' && startsWith(github.event.release.tag_name, '0.')
-        uses: docker/metadata-action@v5
-        id: meta
-        with:
-            images: |
-                ${{ secrets.DOCKER_HUB_USERNAME }}/changedetection.io
-                ghcr.io/dgtlmoon/changedetection.io
-            tags: |
-                type=semver,pattern={{version}}
-                type=semver,pattern={{major}}.{{minor}}
-                type=semver,pattern={{major}}
-
       - name: Build and push :tag
         id: docker_build_tag_release
         if: github.event_name == 'release' && startsWith(github.event.release.tag_name, '0.')
-        uses: docker/build-push-action@v6
+        uses: docker/build-push-action@v5
         with:
           context: ./
           file: ./Dockerfile
           push: true
-          tags: ${{ steps.meta.outputs.tags }}
-          platforms: linux/amd64,linux/arm64,linux/arm/v7,linux/arm/v8,linux/arm64/v8
+          tags: |
+            ${{ secrets.DOCKER_HUB_USERNAME }}/changedetection.io:${{ github.event.release.tag_name }}
+            ghcr.io/dgtlmoon/changedetection.io:${{ github.event.release.tag_name }}
+            ${{ secrets.DOCKER_HUB_USERNAME }}/changedetection.io:latest
+            ghcr.io/dgtlmoon/changedetection.io:latest
+          platforms: linux/amd64,linux/arm64,linux/arm/v6,linux/arm/v7,linux/arm/v8
           cache-from: type=gha
           cache-to: type=gha,mode=max
 # Looks like this was disabled

.github/workflows/pypi-release.yml

@@ -45,12 +45,9 @@ jobs:
     - name: Test that the basic pip built package runs without error
       run: |
         set -ex
-        ls -alR 
-        
-        # Find and install the first .whl file
-        find dist -type f -name "*.whl" -exec pip3 install {} \; -quit
+        sudo pip3 install --upgrade pip 
+        pip3 install dist/changedetection.io*.whl
         changedetection.io -d /tmp -p 10000 &
-        
         sleep 3
         curl --retry-connrefused --retry 6 http://127.0.0.1:10000/static/styles/pure-min.css >/dev/null
         curl --retry-connrefused --retry 6 http://127.0.0.1:10000/ >/dev/null

.github/workflows/test-container-build.yml

@@ -51,7 +51,7 @@ jobs:
         # Check we can still build under alpine/musl
         - name: Test that the docker containers can build (musl via alpine check)
           id: docker_build_musl
-          uses: docker/build-push-action@v6
+          uses: docker/build-push-action@v5
           with:
             context: ./
             file: ./.github/test/Dockerfile-alpine
@@ -59,12 +59,12 @@ jobs:
 
         - name: Test that the docker containers can build
           id: docker_build
-          uses: docker/build-push-action@v6
+          uses: docker/build-push-action@v5
           # https://github.com/docker/build-push-action#customizing
           with:
             context: ./
             file: ./Dockerfile
-            platforms: linux/amd64,linux/arm64,linux/arm/v7,linux/arm/v8,linux/arm64/v8
+            platforms: linux/amd64,linux/arm64,linux/arm/v6,linux/arm/v7,linux/arm/v8
             cache-from: type=local,src=/tmp/.buildx-cache
             cache-to: type=local,dest=/tmp/.buildx-cache
 

.github/workflows/test-only.yml

@@ -4,10 +4,17 @@ name: ChangeDetection.io App Test
 on: [push, pull_request]
 
 jobs:
-  lint-code:
+  test-application:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v4
+
+      # Mainly just for link/flake8
+      - name: Set up Python 3.11
+        uses: actions/setup-python@v5
+        with:
+          python-version: '3.11'
+
       - name: Lint with flake8
         run: |
           pip3 install flake8
@@ -16,31 +23,201 @@ jobs:
           # exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide
           flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics
 
-  test-application-3-10:
-    needs: lint-code
-    uses: ./.github/workflows/test-stack-reusable-workflow.yml
-    with:
-      python-version: '3.10'
+      - name: Spin up ancillary testable services
+        run: |
+          
+          docker network create changedet-network
+          
+          # Selenium
+          docker run --network changedet-network -d --hostname selenium  -p 4444:4444 --rm --shm-size="2g"  selenium/standalone-chrome:4
+          
+          # SocketPuppetBrowser + Extra for custom browser test
+          docker run --network changedet-network -d -e "LOG_LEVEL=TRACE" --cap-add=SYS_ADMIN --name sockpuppetbrowser --hostname sockpuppetbrowser --rm -p 3000:3000 dgtlmoon/sockpuppetbrowser:latest                    
+          docker run --network changedet-network -d -e "LOG_LEVEL=TRACE" --cap-add=SYS_ADMIN --name sockpuppetbrowser-custom-url --hostname sockpuppetbrowser-custom-url  -p 3001:3000 --rm dgtlmoon/sockpuppetbrowser:latest
+
+      - name: Build changedetection.io container for testing
+        run: |         
+          # Build a changedetection.io container and start testing inside
+          docker build --build-arg LOGGER_LEVEL=TRACE -t test-changedetectionio .
+          # Debug info
+          docker run test-changedetectionio  bash -c 'pip list'
+
+      - name: Spin up ancillary SMTP+Echo message test server
+        run: |
+          # Debug SMTP server/echo message back server
+          docker run --network changedet-network -d -p 11025:11025 -p 11080:11080  --hostname mailserver test-changedetectionio  bash -c 'python changedetectionio/tests/smtp/smtp-test-server.py' 
+
+      - name: Show docker container state and other debug info
+        run: |
+          set -x
+          echo "Running processes in docker..."
+          docker ps
+
+      - name: Test built container with Pytest (generally as requests/plaintext fetching)
+        run: |
+          # Unit tests
+          echo "run test with unittest"
+          docker run test-changedetectionio  bash -c 'python3 -m unittest changedetectionio.tests.unit.test_notification_diff'
+          docker run test-changedetectionio  bash -c 'python3 -m unittest changedetectionio.tests.unit.test_watch_model'
+          
+          # All tests
+          echo "run test with pytest"
+          # The default pytest logger_level is TRACE
+          # To change logger_level for pytest(test/conftest.py),
+          # append the docker option. e.g. '-e LOGGER_LEVEL=DEBUG'
+          docker run --name test-cdio-basic-tests --network changedet-network  test-changedetectionio  bash -c 'cd changedetectionio && ./run_basic_tests.sh'
+
+# PLAYWRIGHT/NODE-> CDP
+      - name: Playwright and SocketPuppetBrowser - Specific tests in built container
+        run: |
+          # Playwright via Sockpuppetbrowser fetch
+          # tests/visualselector/test_fetch_data.py will do browser steps  
+          docker run --rm -e "FLASK_SERVER_NAME=cdio" -e "PLAYWRIGHT_DRIVER_URL=ws://sockpuppetbrowser:3000" --network changedet-network --hostname=cdio test-changedetectionio  bash -c 'cd changedetectionio;pytest --live-server-host=0.0.0.0 --live-server-port=5004 tests/fetchers/test_content.py'
+          docker run --rm -e "FLASK_SERVER_NAME=cdio" -e "PLAYWRIGHT_DRIVER_URL=ws://sockpuppetbrowser:3000" --network changedet-network --hostname=cdio test-changedetectionio  bash -c 'cd changedetectionio;pytest --live-server-host=0.0.0.0 --live-server-port=5004 tests/test_errorhandling.py'
+          docker run --rm -e "FLASK_SERVER_NAME=cdio" -e "PLAYWRIGHT_DRIVER_URL=ws://sockpuppetbrowser:3000" --network changedet-network --hostname=cdio test-changedetectionio  bash -c 'cd changedetectionio;pytest --live-server-host=0.0.0.0 --live-server-port=5004 tests/visualselector/test_fetch_data.py'
+          docker run --rm -e "FLASK_SERVER_NAME=cdio" -e "PLAYWRIGHT_DRIVER_URL=ws://sockpuppetbrowser:3000" --network changedet-network --hostname=cdio test-changedetectionio  bash -c 'cd changedetectionio;pytest --live-server-host=0.0.0.0 --live-server-port=5004 tests/fetchers/test_custom_js_before_content.py'
 
 
-  test-application-3-11:
-    needs: lint-code
-    uses: ./.github/workflows/test-stack-reusable-workflow.yml
-    with:
-      python-version: '3.11'
-      skip-pypuppeteer: true
+      - name: Playwright and SocketPuppetBrowser - Headers and requests
+        run: |       
+          # Settings headers playwright tests - Call back in from Sockpuppetbrowser, check headers
+          docker run --name "changedet" --hostname changedet --rm -e "FLASK_SERVER_NAME=changedet" -e "PLAYWRIGHT_DRIVER_URL=ws://sockpuppetbrowser:3000?dumpio=true" --network changedet-network test-changedetectionio  bash -c 'cd changedetectionio; pytest --live-server-host=0.0.0.0  --live-server-port=5004 tests/test_request.py'
 
-  test-application-3-12:
-    needs: lint-code
-    uses: ./.github/workflows/test-stack-reusable-workflow.yml
-    with:
-      python-version: '3.12'
-      skip-pypuppeteer: true
+      - name: Playwright and SocketPuppetBrowser - Restock detection
+        run: |                            
+          # restock detection via playwright - added name=changedet here so that playwright and sockpuppetbrowser can connect to it
+          docker run --rm --name "changedet" -e "FLASK_SERVER_NAME=changedet" -e "PLAYWRIGHT_DRIVER_URL=ws://sockpuppetbrowser:3000" --network changedet-network test-changedetectionio  bash -c 'cd changedetectionio;pytest --live-server-port=5004 --live-server-host=0.0.0.0 tests/restock/test_restock.py'
 
-  test-application-3-13:
-    needs: lint-code
-    uses: ./.github/workflows/test-stack-reusable-workflow.yml
-    with:
-      python-version: '3.13'
-      skip-pypuppeteer: true
-      
+# STRAIGHT TO CDP
+      - name: Pyppeteer and SocketPuppetBrowser - Specific tests in built container
+        run: |
+          # Playwright via Sockpuppetbrowser fetch 
+          docker run --rm -e "FLASK_SERVER_NAME=cdio" -e "FAST_PUPPETEER_CHROME_FETCHER=True" -e "PLAYWRIGHT_DRIVER_URL=ws://sockpuppetbrowser:3000" --network changedet-network --hostname=cdio test-changedetectionio  bash -c 'cd changedetectionio;pytest --live-server-host=0.0.0.0 --live-server-port=5004 tests/fetchers/test_content.py'
+          docker run --rm -e "FLASK_SERVER_NAME=cdio" -e "FAST_PUPPETEER_CHROME_FETCHER=True" -e "PLAYWRIGHT_DRIVER_URL=ws://sockpuppetbrowser:3000" --network changedet-network --hostname=cdio test-changedetectionio  bash -c 'cd changedetectionio;pytest --live-server-host=0.0.0.0 --live-server-port=5004 tests/test_errorhandling.py'
+          docker run --rm -e "FLASK_SERVER_NAME=cdio" -e "FAST_PUPPETEER_CHROME_FETCHER=True" -e "PLAYWRIGHT_DRIVER_URL=ws://sockpuppetbrowser:3000" --network changedet-network --hostname=cdio test-changedetectionio  bash -c 'cd changedetectionio;pytest --live-server-host=0.0.0.0 --live-server-port=5004 tests/visualselector/test_fetch_data.py'
+          docker run --rm -e "FLASK_SERVER_NAME=cdio" -e "FAST_PUPPETEER_CHROME_FETCHER=True" -e "PLAYWRIGHT_DRIVER_URL=ws://sockpuppetbrowser:3000" --network changedet-network --hostname=cdio test-changedetectionio  bash -c 'cd changedetectionio;pytest --live-server-host=0.0.0.0 --live-server-port=5004 tests/fetchers/test_custom_js_before_content.py'
+
+      - name: Pyppeteer and SocketPuppetBrowser - Headers and requests checks
+        run: |       
+          # Settings headers playwright tests - Call back in from Sockpuppetbrowser, check headers
+          docker run --name "changedet" --hostname changedet --rm  -e "FAST_PUPPETEER_CHROME_FETCHER=True" -e "FLASK_SERVER_NAME=changedet" -e "PLAYWRIGHT_DRIVER_URL=ws://sockpuppetbrowser:3000?dumpio=true" --network changedet-network test-changedetectionio  bash -c 'cd changedetectionio; pytest --live-server-host=0.0.0.0  --live-server-port=5004 tests/test_request.py'
+
+      - name: Pyppeteer and SocketPuppetBrowser - Restock detection
+        run: |                            
+          # restock detection via playwright - added name=changedet here so that playwright and sockpuppetbrowser can connect to it
+          docker run --rm --name "changedet" -e "FLASK_SERVER_NAME=changedet"  -e "FAST_PUPPETEER_CHROME_FETCHER=True"  -e "PLAYWRIGHT_DRIVER_URL=ws://sockpuppetbrowser:3000" --network changedet-network test-changedetectionio  bash -c 'cd changedetectionio;pytest --live-server-port=5004 --live-server-host=0.0.0.0 tests/restock/test_restock.py'
+
+# SELENIUM
+      - name: Specific tests in built container for Selenium
+        run: |
+          # Selenium fetch
+          docker run --rm -e "WEBDRIVER_URL=http://selenium:4444/wd/hub" --network changedet-network test-changedetectionio  bash -c 'cd changedetectionio;pytest tests/fetchers/test_content.py && pytest tests/test_errorhandling.py'
+
+      - name: Specific tests in built container for headers and requests checks with Selenium
+        run: |
+          docker run --name "changedet" --hostname changedet --rm -e "FLASK_SERVER_NAME=changedet" -e "WEBDRIVER_URL=http://selenium:4444/wd/hub" --network changedet-network test-changedetectionio  bash -c 'cd changedetectionio; pytest --live-server-host=0.0.0.0  --live-server-port=5004 tests/test_request.py'
+
+# OTHER STUFF
+      - name: Test SMTP notification mime types
+        run: |
+          # SMTP content types - needs the 'Debug SMTP server/echo message back server' container from above
+          docker run --rm  --network changedet-network test-changedetectionio bash -c 'cd changedetectionio;pytest tests/smtp/test_notification_smtp.py'
+
+      # @todo Add a test via playwright/puppeteer
+      # squid with auth is tested in run_proxy_tests.sh -> tests/proxy_list/test_select_custom_proxy.py
+      - name: Test proxy squid style interaction
+        run: |
+          cd changedetectionio
+          ./run_proxy_tests.sh
+          cd ..
+
+      - name: Test proxy SOCKS5 style interaction
+        run: |
+          cd changedetectionio
+          ./run_socks_proxy_tests.sh
+          cd ..
+
+      - name: Test custom browser URL
+        run: |
+          cd changedetectionio
+          ./run_custom_browser_url_tests.sh
+          cd ..
+
+      - name: Test changedetection.io container starts+runs basically without error
+        run: |
+          docker run --name test-changedetectionio -p 5556:5000  -d test-changedetectionio
+          sleep 3
+          # Should return 0 (no error) when grep finds it
+          curl --retry-connrefused --retry 6  -s http://localhost:5556 |grep -q checkbox-uuid
+          
+          # and IPv6
+          curl --retry-connrefused --retry 6  -s -g -6 "http://[::1]:5556"|grep -q checkbox-uuid
+
+          # Check whether TRACE log is enabled.
+          # Also, check whether TRACE is came from STDERR
+          docker logs test-changedetectionio 2>&1 1>/dev/null | grep 'TRACE log is enabled' || exit 1
+          # Check whether DEBUG is came from STDOUT
+          docker logs test-changedetectionio 2>/dev/null | grep 'DEBUG' || exit 1
+
+          docker kill test-changedetectionio
+
+      - name: Test changedetection.io SIGTERM and SIGINT signal shutdown
+        run: |
+          
+          echo SIGINT Shutdown request test
+          docker run --name sig-test -d test-changedetectionio
+          sleep 3
+          echo ">>> Sending SIGINT to sig-test container"
+          docker kill --signal=SIGINT sig-test
+          sleep 3
+          # invert the check (it should be not 0/not running)
+          docker ps
+          # check signal catch(STDERR) log. Because of
+          # changedetectionio/__init__.py: logger.add(sys.stderr, level=logger_level)
+          docker logs sig-test 2>&1 | grep 'Shutdown: Got Signal - SIGINT' || exit 1
+          test -z "`docker ps|grep sig-test`"
+          if [ $? -ne 0 ]
+          then
+            echo "Looks like container was running when it shouldnt be"
+            docker ps
+            exit 1
+          fi
+          
+          # @todo - scan the container log to see the right "graceful shutdown" text exists 
+          docker rm sig-test
+          
+          echo SIGTERM Shutdown request test
+          docker run --name sig-test -d test-changedetectionio
+          sleep 3
+          echo ">>> Sending SIGTERM to sig-test container"
+          docker kill --signal=SIGTERM sig-test
+          sleep 3
+          # invert the check (it should be not 0/not running)
+          docker ps
+          # check signal catch(STDERR) log. Because of
+          # changedetectionio/__init__.py: logger.add(sys.stderr, level=logger_level)
+          docker logs sig-test 2>&1 | grep 'Shutdown: Got Signal - SIGTERM' || exit 1
+          test -z "`docker ps|grep sig-test`"
+          if [ $? -ne 0 ]
+          then
+            echo "Looks like container was running when it shouldnt be"
+            docker ps
+            exit 1
+          fi
+          
+          # @todo - scan the container log to see the right "graceful shutdown" text exists           
+          docker rm sig-test
+
+      - name: Dump container log
+        if: always()
+        run: |
+          mkdir output-logs
+          docker logs test-cdio-basic-tests > output-logs/test-cdio-basic-tests-stdout.txt
+          docker logs test-cdio-basic-tests 2> output-logs/test-cdio-basic-tests-stderr.txt
+
+      - name: Store container log
+        if: always()
+        uses: actions/upload-artifact@v4
+        with:
+          name: test-cdio-basic-tests-output
+          path: output-logs

.github/workflows/test-stack-reusable-workflow.yml

@@ -1,241 +0,0 @@
-name: ChangeDetection.io App Test
-
-on:
-  workflow_call:
-    inputs:
-      python-version:
-        description: 'Python version to use'
-        required: true
-        type: string
-        default: '3.10'
-      skip-pypuppeteer:
-        description: 'Skip PyPuppeteer (not supported in 3.11/3.12)'
-        required: false
-        type: boolean
-        default: false
-
-jobs:
-  test-application:
-    runs-on: ubuntu-latest
-    env:
-      PYTHON_VERSION: ${{ inputs.python-version }}
-    steps:
-      - uses: actions/checkout@v4
-
-      # Mainly just for link/flake8
-      - name: Set up Python ${{ env.PYTHON_VERSION }}
-        uses: actions/setup-python@v5
-        with:
-          python-version: ${{ env.PYTHON_VERSION }}
-
-      - name: Build changedetection.io container for testing under Python ${{ env.PYTHON_VERSION }}
-        run: |
-          echo "---- Building for Python ${{ env.PYTHON_VERSION }} -----"
-          # Build a changedetection.io container and start testing inside
-          docker build --build-arg PYTHON_VERSION=${{ env.PYTHON_VERSION }} --build-arg LOGGER_LEVEL=TRACE -t test-changedetectionio .
-          # Debug info
-          docker run test-changedetectionio  bash -c 'pip list'         
-
-      - name: We should be Python ${{ env.PYTHON_VERSION }} ...
-        run: |         
-          docker run test-changedetectionio  bash -c 'python3 --version'
-
-      - name: Spin up ancillary testable services
-        run: |
-          
-          docker network create changedet-network
-          
-          # Selenium
-          docker run --network changedet-network -d --hostname selenium  -p 4444:4444 --rm --shm-size="2g"  selenium/standalone-chrome:4
-          
-          # SocketPuppetBrowser + Extra for custom browser test
-          docker run --network changedet-network -d -e "LOG_LEVEL=TRACE" --cap-add=SYS_ADMIN --name sockpuppetbrowser --hostname sockpuppetbrowser --rm -p 3000:3000 dgtlmoon/sockpuppetbrowser:latest                    
-          docker run --network changedet-network -d -e "LOG_LEVEL=TRACE" --cap-add=SYS_ADMIN --name sockpuppetbrowser-custom-url --hostname sockpuppetbrowser-custom-url  -p 3001:3000 --rm dgtlmoon/sockpuppetbrowser:latest
-
-      - name: Spin up ancillary SMTP+Echo message test server
-        run: |
-          # Debug SMTP server/echo message back server
-          docker run --network changedet-network -d -p 11025:11025 -p 11080:11080  --hostname mailserver test-changedetectionio  bash -c 'pip3 install aiosmtpd && python changedetectionio/tests/smtp/smtp-test-server.py'
-          docker ps
-
-      - name: Show docker container state and other debug info
-        run: |
-          set -x
-          echo "Running processes in docker..."
-          docker ps
-
-      - name: Run Unit Tests
-        run: |
-          # Unit tests
-          docker run test-changedetectionio  bash -c 'python3 -m unittest changedetectionio.tests.unit.test_notification_diff'
-          docker run test-changedetectionio  bash -c 'python3 -m unittest changedetectionio.tests.unit.test_watch_model'
-          docker run test-changedetectionio  bash -c 'python3 -m unittest changedetectionio.tests.unit.test_jinja2_security'
-          docker run test-changedetectionio  bash -c 'python3 -m unittest changedetectionio.tests.unit.test_semver'
-
-      - name: Test built container with Pytest (generally as requests/plaintext fetching)
-        run: |
-          # All tests
-          echo "run test with pytest"
-          # The default pytest logger_level is TRACE
-          # To change logger_level for pytest(test/conftest.py),
-          # append the docker option. e.g. '-e LOGGER_LEVEL=DEBUG'
-          docker run --name test-cdio-basic-tests --network changedet-network  test-changedetectionio  bash -c 'cd changedetectionio && ./run_basic_tests.sh'
-
-# PLAYWRIGHT/NODE-> CDP
-      - name: Playwright and SocketPuppetBrowser - Specific tests in built container
-        run: |
-          # Playwright via Sockpuppetbrowser fetch
-          # tests/visualselector/test_fetch_data.py will do browser steps  
-          docker run --rm -e "FLASK_SERVER_NAME=cdio" -e "PLAYWRIGHT_DRIVER_URL=ws://sockpuppetbrowser:3000" --network changedet-network --hostname=cdio test-changedetectionio  bash -c 'cd changedetectionio;pytest --live-server-host=0.0.0.0 --live-server-port=5004 tests/fetchers/test_content.py'
-          docker run --rm -e "FLASK_SERVER_NAME=cdio" -e "PLAYWRIGHT_DRIVER_URL=ws://sockpuppetbrowser:3000" --network changedet-network --hostname=cdio test-changedetectionio  bash -c 'cd changedetectionio;pytest --live-server-host=0.0.0.0 --live-server-port=5004 tests/test_errorhandling.py'
-          docker run --rm -e "FLASK_SERVER_NAME=cdio" -e "PLAYWRIGHT_DRIVER_URL=ws://sockpuppetbrowser:3000" --network changedet-network --hostname=cdio test-changedetectionio  bash -c 'cd changedetectionio;pytest --live-server-host=0.0.0.0 --live-server-port=5004 tests/visualselector/test_fetch_data.py'
-          docker run --rm -e "FLASK_SERVER_NAME=cdio" -e "PLAYWRIGHT_DRIVER_URL=ws://sockpuppetbrowser:3000" --network changedet-network --hostname=cdio test-changedetectionio  bash -c 'cd changedetectionio;pytest --live-server-host=0.0.0.0 --live-server-port=5004 tests/fetchers/test_custom_js_before_content.py'
-
-
-      - name: Playwright and SocketPuppetBrowser - Headers and requests
-        run: |       
-          # Settings headers playwright tests - Call back in from Sockpuppetbrowser, check headers
-          docker run --name "changedet" --hostname changedet --rm -e "FLASK_SERVER_NAME=changedet" -e "PLAYWRIGHT_DRIVER_URL=ws://sockpuppetbrowser:3000?dumpio=true" --network changedet-network test-changedetectionio  bash -c 'find .; cd changedetectionio; pytest --live-server-host=0.0.0.0  --live-server-port=5004 tests/test_request.py; pwd;find .'
-
-      - name: Playwright and SocketPuppetBrowser - Restock detection
-        run: |                            
-          # restock detection via playwright - added name=changedet here so that playwright and sockpuppetbrowser can connect to it
-          docker run --rm --name "changedet" -e "FLASK_SERVER_NAME=changedet" -e "PLAYWRIGHT_DRIVER_URL=ws://sockpuppetbrowser:3000" --network changedet-network test-changedetectionio  bash -c 'cd changedetectionio;pytest --live-server-port=5004 --live-server-host=0.0.0.0 tests/restock/test_restock.py'
-
-# STRAIGHT TO CDP
-      - name: Pyppeteer and SocketPuppetBrowser - Specific tests in built container
-        if: ${{ inputs.skip-pypuppeteer == false }}
-        run: |
-          # Playwright via Sockpuppetbrowser fetch 
-          docker run --rm -e "FLASK_SERVER_NAME=cdio" -e "FAST_PUPPETEER_CHROME_FETCHER=True" -e "PLAYWRIGHT_DRIVER_URL=ws://sockpuppetbrowser:3000" --network changedet-network --hostname=cdio test-changedetectionio  bash -c 'cd changedetectionio;pytest --live-server-host=0.0.0.0 --live-server-port=5004 tests/fetchers/test_content.py'
-          docker run --rm -e "FLASK_SERVER_NAME=cdio" -e "FAST_PUPPETEER_CHROME_FETCHER=True" -e "PLAYWRIGHT_DRIVER_URL=ws://sockpuppetbrowser:3000" --network changedet-network --hostname=cdio test-changedetectionio  bash -c 'cd changedetectionio;pytest --live-server-host=0.0.0.0 --live-server-port=5004 tests/test_errorhandling.py'
-          docker run --rm -e "FLASK_SERVER_NAME=cdio" -e "FAST_PUPPETEER_CHROME_FETCHER=True" -e "PLAYWRIGHT_DRIVER_URL=ws://sockpuppetbrowser:3000" --network changedet-network --hostname=cdio test-changedetectionio  bash -c 'cd changedetectionio;pytest --live-server-host=0.0.0.0 --live-server-port=5004 tests/visualselector/test_fetch_data.py'
-          docker run --rm -e "FLASK_SERVER_NAME=cdio" -e "FAST_PUPPETEER_CHROME_FETCHER=True" -e "PLAYWRIGHT_DRIVER_URL=ws://sockpuppetbrowser:3000" --network changedet-network --hostname=cdio test-changedetectionio  bash -c 'cd changedetectionio;pytest --live-server-host=0.0.0.0 --live-server-port=5004 tests/fetchers/test_custom_js_before_content.py'
-
-      - name: Pyppeteer and SocketPuppetBrowser - Headers and requests checks
-        if: ${{ inputs.skip-pypuppeteer == false }}
-        run: |
-          # Settings headers playwright tests - Call back in from Sockpuppetbrowser, check headers
-          docker run --name "changedet" --hostname changedet --rm  -e "FAST_PUPPETEER_CHROME_FETCHER=True" -e "FLASK_SERVER_NAME=changedet" -e "PLAYWRIGHT_DRIVER_URL=ws://sockpuppetbrowser:3000?dumpio=true" --network changedet-network test-changedetectionio  bash -c 'cd changedetectionio; pytest --live-server-host=0.0.0.0  --live-server-port=5004 tests/test_request.py'
-
-      - name: Pyppeteer and SocketPuppetBrowser - Restock detection
-        if: ${{ inputs.skip-pypuppeteer == false }}
-        run: |                            
-          # restock detection via playwright - added name=changedet here so that playwright and sockpuppetbrowser can connect to it
-          docker run --rm --name "changedet" -e "FLASK_SERVER_NAME=changedet"  -e "FAST_PUPPETEER_CHROME_FETCHER=True"  -e "PLAYWRIGHT_DRIVER_URL=ws://sockpuppetbrowser:3000" --network changedet-network test-changedetectionio  bash -c 'cd changedetectionio;pytest --live-server-port=5004 --live-server-host=0.0.0.0 tests/restock/test_restock.py'
-
-# SELENIUM
-      - name: Specific tests in built container for Selenium
-        run: |
-          # Selenium fetch
-          docker run --rm -e "WEBDRIVER_URL=http://selenium:4444/wd/hub" --network changedet-network test-changedetectionio  bash -c 'cd changedetectionio;pytest tests/fetchers/test_content.py && pytest tests/test_errorhandling.py'
-
-      - name: Specific tests in built container for headers and requests checks with Selenium
-        run: |
-          docker run --name "changedet" --hostname changedet --rm -e "FLASK_SERVER_NAME=changedet" -e "WEBDRIVER_URL=http://selenium:4444/wd/hub" --network changedet-network test-changedetectionio  bash -c 'cd changedetectionio; pytest --live-server-host=0.0.0.0  --live-server-port=5004 tests/test_request.py'
-
-# OTHER STUFF
-      - name: Test SMTP notification mime types
-        run: |
-          # SMTP content types - needs the 'Debug SMTP server/echo message back server' container from above
-          # "mailserver" hostname defined above
-          docker run --rm --network changedet-network test-changedetectionio bash -c 'cd changedetectionio;pytest tests/smtp/test_notification_smtp.py'
-
-      # @todo Add a test via playwright/puppeteer
-      # squid with auth is tested in run_proxy_tests.sh -> tests/proxy_list/test_select_custom_proxy.py
-      - name: Test proxy squid style interaction
-        run: |
-          cd changedetectionio
-          ./run_proxy_tests.sh
-          cd ..
-
-      - name: Test proxy SOCKS5 style interaction
-        run: |
-          cd changedetectionio
-          ./run_socks_proxy_tests.sh
-          cd ..
-
-      - name: Test custom browser URL
-        run: |
-          cd changedetectionio
-          ./run_custom_browser_url_tests.sh
-          cd ..
-
-      - name: Test changedetection.io container starts+runs basically without error
-        run: |
-          docker run --name test-changedetectionio -p 5556:5000  -d test-changedetectionio
-          sleep 3
-          # Should return 0 (no error) when grep finds it
-          curl --retry-connrefused --retry 6  -s http://localhost:5556 |grep -q checkbox-uuid
-          
-          # and IPv6
-          curl --retry-connrefused --retry 6  -s -g -6 "http://[::1]:5556"|grep -q checkbox-uuid
-
-          # Check whether TRACE log is enabled.
-          # Also, check whether TRACE is came from STDERR
-          docker logs test-changedetectionio 2>&1 1>/dev/null | grep 'TRACE log is enabled' || exit 1
-          # Check whether DEBUG is came from STDOUT
-          docker logs test-changedetectionio 2>/dev/null | grep 'DEBUG' || exit 1
-
-          docker kill test-changedetectionio
-
-      - name: Test changedetection.io SIGTERM and SIGINT signal shutdown
-        run: |
-          
-          echo SIGINT Shutdown request test
-          docker run --name sig-test -d test-changedetectionio
-          sleep 3
-          echo ">>> Sending SIGINT to sig-test container"
-          docker kill --signal=SIGINT sig-test
-          sleep 3
-          # invert the check (it should be not 0/not running)
-          docker ps
-          # check signal catch(STDERR) log. Because of
-          # changedetectionio/__init__.py: logger.add(sys.stderr, level=logger_level)
-          docker logs sig-test 2>&1 | grep 'Shutdown: Got Signal - SIGINT' || exit 1
-          test -z "`docker ps|grep sig-test`"
-          if [ $? -ne 0 ]
-          then
-            echo "Looks like container was running when it shouldnt be"
-            docker ps
-            exit 1
-          fi
-          
-          # @todo - scan the container log to see the right "graceful shutdown" text exists 
-          docker rm sig-test
-          
-          echo SIGTERM Shutdown request test
-          docker run --name sig-test -d test-changedetectionio
-          sleep 3
-          echo ">>> Sending SIGTERM to sig-test container"
-          docker kill --signal=SIGTERM sig-test
-          sleep 3
-          # invert the check (it should be not 0/not running)
-          docker ps
-          # check signal catch(STDERR) log. Because of
-          # changedetectionio/__init__.py: logger.add(sys.stderr, level=logger_level)
-          docker logs sig-test 2>&1 | grep 'Shutdown: Got Signal - SIGTERM' || exit 1
-          test -z "`docker ps|grep sig-test`"
-          if [ $? -ne 0 ]
-          then
-            echo "Looks like container was running when it shouldnt be"
-            docker ps
-            exit 1
-          fi
-          
-          # @todo - scan the container log to see the right "graceful shutdown" text exists           
-          docker rm sig-test
-
-      - name: Dump container log
-        if: always()
-        run: |
-          mkdir output-logs
-          docker logs test-cdio-basic-tests > output-logs/test-cdio-basic-tests-stdout-${{ env.PYTHON_VERSION }}.txt
-          docker logs test-cdio-basic-tests 2> output-logs/test-cdio-basic-tests-stderr-${{ env.PYTHON_VERSION }}.txt
-
-      - name: Store everything including test-datastore
-        if: always()
-        uses: actions/upload-artifact@v4
-        with:
-          name: test-cdio-basic-tests-output-py${{ env.PYTHON_VERSION }}
-          path: .

.gitignore

@@ -1,29 +1,14 @@
-# Byte-compiled / optimized / DLL files
-**/__pycache__
-**/*.py[cod]
-
-# Caches
-.mypy_cache/
-.pytest_cache/
-.ruff_cache/
-
-# Distribution / packaging
-build/
-dist/
-*.egg-info*
-
-# Virtual environment
-.env
-.venv/
-venv/
-
-# IDEs
+__pycache__
 .idea
+*.pyc
+datastore/url-watches.json
+datastore/*
+__pycache__
+.pytest_cache
+build
+dist
+venv
+test-datastore/*
+test-datastore
+*.egg-info*
 .vscode/settings.json
-
-# Datastore files
-datastore/
-test-datastore/
-
-# Memory consumption log
-test-memory.log

COMMERCIAL_LICENCE.md

@@ -1,54 +0,0 @@
-# Generally
-
-In any commercial activity involving 'Hosting' (as defined herein), whether in part or in full, this license must be executed and adhered to.
-
-# Commercial License Agreement
-
-This Commercial License Agreement ("Agreement") is entered into by and between Web Technologies s.r.o. here-in ("Licensor") and (your company or personal name) _____________ ("Licensee"). This Agreement sets forth the terms and conditions under which Licensor provides its software ("Software") and services to Licensee for the purpose of reselling the software either in part or full, as part of any commercial activity where the activity involves a third party.
-
-### Definition of Hosting
-
-For the purposes of this Agreement, "hosting" means making the functionality of the Program or modified version available to third parties as a service. This includes, without limitation:
-- Enabling third parties to interact with the functionality of the Program or modified version remotely through a computer network.
-- Offering a service the value of which entirely or primarily derives from the value of the Program or modified version.
-- Offering a service that accomplishes for users the primary purpose of the Program or modified version.
-
-## 1. Grant of License
-Subject to the terms and conditions of this Agreement, Licensor grants Licensee a non-exclusive, non-transferable license to install, use, and resell the Software. Licensee may:
-- Resell the Software as part of a service offering or as a standalone product.
-- Host the Software on a server and provide it as a hosted service (e.g., Software as a Service - SaaS).
-- Integrate the Software into a larger product or service that is then sold or provided for commercial purposes, where the software is used either in part or full.
-
-## 2. License Fees
-Licensee agrees to pay Licensor the license fees specified in the ordering document. License fees are due and payable as specified in the ordering document. The fees may include initial licensing costs and recurring fees based on the number of end users, instances of the Software resold, or revenue generated from the resale activities.
-
-## 3. Resale Conditions
-Licensee must comply with the following conditions when reselling the Software, whether the software is resold in part or full:
-- Provide end users with access to the source code under the same open-source license conditions as provided by Licensor.
-- Clearly state in all marketing and sales materials that the Software is provided under a commercial license from Licensor, and provide a link back to https://changedetection.io.
-- Ensure end users are aware of and agree to the terms of the commercial license prior to resale.
-- Do not sublicense or transfer the Software to third parties except as part of an authorized resale activity.
-
-## 4. Hosting and Provision of Services
-Licensee may host the Software (either in part or full) on its servers and provide it as a hosted service to end users. The following conditions apply:
-- Licensee must ensure that all hosted versions of the Software comply with the terms of this Agreement.
-- Licensee must provide Licensor with regular reports detailing the number of end users and instances of the hosted service.
-- Any modifications to the Software made by Licensee for hosting purposes must be made available to end users under the same open-source license conditions, unless agreed otherwise.
-
-## 5. Services
-Licensor will provide support and maintenance services as described in the support policy referenced in the ordering document should such an agreement be signed by all parties. Additional fees may apply for support services provided to end users resold by Licensee.
-
-## 6. Reporting and Audits
-Licensee agrees to provide Licensor with regular reports detailing the number of instances, end users, and revenue generated from the resale of the Software. Licensor reserves the right to audit Licensee’s records to ensure compliance with this Agreement.
-
-## 7. Term and Termination
-This Agreement shall commence on the effective date and continue for the period set forth in the ordering document unless terminated earlier in accordance with this Agreement. Either party may terminate this Agreement if the other party breaches any material term and fails to cure such breach within thirty (30) days after receipt of written notice.
-
-## 8. Limitation of Liability and Disclaimer of Warranty
-Executing this commercial license does not waive the Limitation of Liability or Disclaimer of Warranty as stated in the open-source LICENSE provided with the Software. The Software is provided "as is," without warranty of any kind, express or implied, including but not limited to the warranties of merchantability, fitness for a particular purpose, and noninfringement. In no event shall the authors or copyright holders be liable for any claim, damages, or other liability, whether in an action of contract, tort, or otherwise, arising from, out of, or in connection with the Software or the use or other dealings in the Software.
-
-## 9. Governing Law
-This Agreement shall be governed by and construed in accordance with the laws of the Czech Republic.
-
-## Contact Information
-For commercial licensing inquiries, please contact contact@changedetection.io and dgtlmoon@gmail.com.

CONTRIBUTING.md

@@ -2,7 +2,7 @@ Contributing is always welcome!
 
 I am no professional flask developer, if you know a better way that something can be done, please let me know!
 
-Otherwise, it's always best to PR into the `master` branch.
+Otherwise, it's always best to PR into the `dev` branch.
 
 Please be sure that all new functionality has a matching test!
 

Dockerfile

@@ -2,10 +2,7 @@
 
 # @NOTE! I would love to move to 3.11 but it breaks the async handler in changedetectionio/content_fetchers/puppeteer.py
 #        If you know how to fix it, please do! and test it for both 3.10 and 3.11
-
-ARG PYTHON_VERSION=3.11
-
-FROM python:${PYTHON_VERSION}-slim-bookworm AS builder
+FROM python:3.10-slim-bookworm as builder
 
 # See `cryptography` pin comment in requirements.txt
 ARG CRYPTOGRAPHY_DONT_BUILD_RUST=1
@@ -26,23 +23,19 @@ WORKDIR /install
 
 COPY requirements.txt /requirements.txt
 
-# --extra-index-url https://www.piwheels.org/simple  is for cryptography module to be prebuilt (or rustc etc needs to be installed)
-RUN pip install --extra-index-url https://www.piwheels.org/simple  --target=/dependencies -r /requirements.txt
+RUN pip install --target=/dependencies -r /requirements.txt
 
 # Playwright is an alternative to Selenium
 # Excluded this package from requirements.txt to prevent arm/v6 and arm/v7 builds from failing
 # https://github.com/dgtlmoon/changedetection.io/pull/1067 also musl/alpine (not supported)
-RUN pip install --target=/dependencies playwright~=1.48.0 \
+RUN pip install --target=/dependencies playwright~=1.41.2 \
     || echo "WARN: Failed to install Playwright. The application can still run, but the Playwright option will be disabled."
 
 # Final image stage
-FROM python:${PYTHON_VERSION}-slim-bookworm
-LABEL org.opencontainers.image.source="https://github.com/dgtlmoon/changedetection.io"
+FROM python:3.10-slim-bookworm
 
 RUN apt-get update && apt-get install -y --no-install-recommends \
     libxslt1.1 \
-    # For presenting price amounts correctly in the restock/price detection overview
-    locales \
     # For pdftohtml
     poppler-utils \
     zlib1g \

MANIFEST.in

@@ -1,8 +1,6 @@
 recursive-include changedetectionio/api *
-recursive-include changedetectionio/apprise_plugin *
 recursive-include changedetectionio/blueprint *
 recursive-include changedetectionio/content_fetchers *
-recursive-include changedetectionio/conditions *
 recursive-include changedetectionio/model *
 recursive-include changedetectionio/processors *
 recursive-include changedetectionio/static *

README.md

@@ -41,20 +41,6 @@ Using the **Browser Steps** configuration, add basic steps before performing cha
 After **Browser Steps** have been run, then visit the **Visual Selector** tab to refine the content you're interested in.
 Requires Playwright to be enabled.
 
-### Awesome restock and price change notifications
-
-Enable the _"Re-stock & Price detection for single product pages"_ option to activate the best way to monitor product pricing, this will extract any meta-data in the HTML page and give you many options to follow the pricing of the product.
-
-Easily organise and monitor prices for products from the dashboard, get alerts and notifications when the price of a product changes or comes back in stock again!
-
-[<img src="docs/restock-overview.png" style="max-width:100%;" alt="Easily keep an eye on product price changes directly from the UI"  title="Easily keep an eye on product price changes directly from the UI" />](https://changedetection.io?src=github)
-
-Set price change notification parameters, upper and lower price, price change percentage and more.
-Always know when a product for sale drops in price.
-
-[<img src="docs/restock-settings.png" style="max-width:100%;" alt="Set upper lower and percentage price change notification values"  title="Set upper lower and percentage price change notification values" />](https://changedetection.io?src=github)
-
-
 
 ### Example use cases
 
@@ -105,22 +91,13 @@ We [recommend and use Bright Data](https://brightdata.grsm.io/n0r16zf7eivq) glob
 
 Please :star: star :star: this project and help it grow! https://github.com/dgtlmoon/changedetection.io/
 
-### Schedule web page watches in any timezone, limit by day of week and time.
-
-Easily set a re-check schedule, for example you could limit the web page change detection to only operate during business hours.
-Or perhaps based on a foreign timezone (for example, you want to check for the latest news-headlines in a foreign country at 0900 AM),
-
-<img src="./docs/scheduler.png" style="max-width:80%;" alt="How to monitor web page changes according to a schedule"  title="How to monitor web page changes according to a schedule"  />
-
-Includes quick short-cut buttons to setup a schedule for **business hours only**, or **weekends**.
-
 ### We have a Chrome extension!
 
 Easily add the current web page to your changedetection.io tool, simply install the extension and click "Sync" to connect it to your existing changedetection.io install.
 
 [<img src="./docs/chrome-extension-screenshot.png" style="max-width:80%;" alt="Chrome Extension to easily add the current web-page to detect a change."  title="Chrome Extension to easily add the current web-page to detect a change."  />](https://chromewebstore.google.com/detail/changedetectionio-website/kefcfmgmlhmankjmnbijimhofdjekbop)
 
-[Goto the Chrome Webstore to download the extension.](https://chromewebstore.google.com/detail/changedetectionio-website/kefcfmgmlhmankjmnbijimhofdjekbop) ( Or check out the [GitHub repo](https://github.com/dgtlmoon/changedetection.io-browser-extension) ) 
+[Goto the Chrome Webstore to download the extension.](https://chromewebstore.google.com/detail/changedetectionio-website/kefcfmgmlhmankjmnbijimhofdjekbop)
 
 ## Installation
 
@@ -280,7 +257,13 @@ Supports managing the website watch list [via our API](https://changedetection.i
 Do you use changedetection.io to make money? does it save you time or money? Does it make your life easier? less stressful? Remember, we write this software when we should be doing actual paid work, we have to buy food and pay rent just like you.
 
 
-Consider taking out an officially supported [website change detection subscription](https://changedetection.io?src=github) , even if you don't use it, you still get the warm fuzzy feeling of helping out the project. (And who knows, you might just use it!)
+Firstly, consider taking out an officially supported [website change detection subscription](https://changedetection.io?src=github) , even if you don't use it, you still get the warm fuzzy feeling of helping out the project. (And who knows, you might just use it!)
+
+Or directly donate an amount PayPal [![Donate](https://img.shields.io/badge/Donate-PayPal-green.svg)](https://www.paypal.com/donate/?hosted_button_id=7CP6HR9ZCNDYJ)
+
+Or BTC `1PLFN327GyUarpJd7nVe7Reqg9qHx5frNn`
+
+<img src="https://raw.githubusercontent.com/dgtlmoon/changedetection.io/master/docs/btc-support.png" style="max-width:50%;" alt="Support us!"  />
 
 ## Commercial Support
 
@@ -295,10 +278,6 @@ I offer commercial support, this software is depended on by network security, ae
 [release-link]: https://github.com/dgtlmoon/changedetection.io/releases
 [docker-link]: https://hub.docker.com/r/dgtlmoon/changedetection.io
 
-## Commercial Licencing
-
-If you are reselling this software either in part or full as part of any commercial arrangement, you must abide by our COMMERCIAL_LICENCE.md found in our code repository, please contact dgtlmoon@gmail.com and contact@changedetection.io .
-
 ## Third-party licenses
 
 changedetectionio.html_tools.elementpath_tostring: Copyright (c), 2018-2021, SISSA (Scuola Internazionale Superiore di Studi Avanzati), Licensed under [MIT license](https://github.com/sissaschool/elementpath/blob/master/LICENSE)

changedetection.py

@@ -1,4 +1,4 @@
-#!/usr/bin/env python3
+#!/usr/bin/python3
 
 # Only exists for direct CLI usage
 

changedetectionio/__init__.py

@@ -1,13 +1,13 @@
-#!/usr/bin/env python3
+#!/usr/bin/python3
 
 # Read more https://github.com/dgtlmoon/changedetection.io/wiki
 
-__version__ = '0.49.4'
+__version__ = '0.45.16'
 
-from changedetectionio.strtobool import strtobool
+from distutils.util import strtobool
 from json.decoder import JSONDecodeError
 import os
-os.environ['EVENTLET_NO_GREENDNS'] = 'yes'
+#os.environ['EVENTLET_NO_GREENDNS'] = 'yes'
 import eventlet
 import eventlet.wsgi
 import getopt
@@ -24,9 +24,6 @@ from loguru import logger
 app = None
 datastore = None
 
-def get_version():
-    return __version__
-
 # Parent wrapper or OS sends us a SIGTERM/SIGINT, do everything required for a clean shutdown
 def sigshutdown_handler(_signo, _stack_frame):
     global app
@@ -163,10 +160,11 @@ def main():
                     )
 
     # Monitored websites will not receive a Referer header when a user clicks on an outgoing link.
+    # @Note: Incompatible with password login (and maybe other features) for now, submit a PR!
     @app.after_request
     def hide_referrer(response):
         if strtobool(os.getenv("HIDE_REFERER", 'false')):
-            response.headers["Referrer-Policy"] = "same-origin"
+            response.headers["Referrer-Policy"] = "no-referrer"
 
         return response
 
@@ -177,7 +175,6 @@ def main():
     #         proxy_set_header Host "localhost";
     #         proxy_set_header X-Forwarded-Prefix /app;
 
-
     if os.getenv('USE_X_SETTINGS'):
         logger.info("USE_X_SETTINGS is ENABLED")
         from werkzeug.middleware.proxy_fix import ProxyFix

changedetectionio/api/api_schema.py

@@ -112,35 +112,6 @@ def build_watch_json_schema(d):
 
     schema['properties']['time_between_check'] = build_time_between_check_json_schema()
 
-    schema['properties']['browser_steps'] = {
-        "anyOf": [
-            {
-                "type": "array",
-                "items": {
-                    "type": "object",
-                    "properties": {
-                        "operation": {
-                            "type": ["string", "null"],
-                            "maxLength": 5000  # Allows null and any string up to 5000 chars (including "")
-                        },
-                        "selector": {
-                            "type": ["string", "null"],
-                            "maxLength": 5000
-                        },
-                        "optional_value": {
-                            "type": ["string", "null"],
-                            "maxLength": 5000
-                        }
-                    },
-                    "required": ["operation", "selector", "optional_value"],
-                    "additionalProperties": False  # No extra keys allowed
-                }
-            },
-            {"type": "null"},  # Allows null for `browser_steps`
-            {"type": "array", "maxItems": 0}  # Allows empty array []
-        ]
-    }
-
     # headers ?
     return schema
 

changedetectionio/api/api_v1.py

@@ -1,5 +1,5 @@
 import os
-from changedetectionio.strtobool import strtobool
+from distutils.util import strtobool
 
 from flask_expects_json import expects_json
 from changedetectionio import queuedWatchMetaData
@@ -12,10 +12,9 @@ import copy
 # See docs/README.md for rebuilding the docs/apidoc information
 
 from . import api_schema
-from ..model import watch_base
 
 # Build a JSON Schema atleast partially based on our Watch model
-watch_base_config = watch_base()
+from changedetectionio.model.Watch import base_config as watch_base_config
 schema = api_schema.build_watch_json_schema(watch_base_config)
 
 schema_create_watch = copy.deepcopy(schema)
@@ -58,7 +57,7 @@ class Watch(Resource):
             abort(404, message='No watch exists with the UUID of {}'.format(uuid))
 
         if request.args.get('recheck'):
-            self.update_q.put(queuedWatchMetaData.PrioritizedItem(priority=1, item={'uuid': uuid}))
+            self.update_q.put(queuedWatchMetaData.PrioritizedItem(priority=1, item={'uuid': uuid, 'skip_when_checksum_same': True}))
             return "OK", 200
         if request.args.get('paused', '') == 'paused':
             self.datastore.data['watching'].get(uuid).pause()
@@ -76,7 +75,6 @@ class Watch(Resource):
         # Return without history, get that via another API call
         # Properties are not returned as a JSON, so add the required props manually
         watch['history_n'] = watch.history_n
-        # attr .last_changed will check for the last written text snapshot on change
         watch['last_changed'] = watch.last_changed
         watch['viewed'] = watch.viewed
         return watch
@@ -172,33 +170,23 @@ class WatchSingleHistory(Resource):
             curl http://localhost:5000/api/v1/watch/cc0cfffa-f449-477b-83ea-0caafd1dc091/history/1677092977 -H"x-api-key:813031b16330fe25e3780cf0325daa45" -H "Content-Type: application/json"
         @apiName Get single snapshot content
         @apiGroup Watch History
-        @apiParam {String} [html]       Optional Set to =1 to return the last HTML (only stores last 2 snapshots, use `latest` as timestamp)
         @apiSuccess (200) {String} OK
         @apiSuccess (404) {String} ERR Not found
         """
         watch = self.datastore.data['watching'].get(uuid)
         if not watch:
-            abort(404, message=f"No watch exists with the UUID of {uuid}")
+            abort(404, message='No watch exists with the UUID of {}'.format(uuid))
 
         if not len(watch.history):
-            abort(404, message=f"Watch found but no history exists for the UUID {uuid}")
+            abort(404, message='Watch found but no history exists for the UUID {}'.format(uuid))
 
         if timestamp == 'latest':
             timestamp = list(watch.history.keys())[-1]
 
-        if request.args.get('html'):
-            content = watch.get_fetched_html(timestamp)
-            if content:
-                response = make_response(content, 200)
-                response.mimetype = "text/html"
-            else:
-                response = make_response("No content found", 404)
-                response.mimetype = "text/plain"
-        else:
-            content = watch.get_history_snapshot(timestamp)
-            response = make_response(content, 200)
-            response.mimetype = "text/plain"
+        content = watch.get_history_snapshot(timestamp)
 
+        response = make_response(content, 200)
+        response.mimetype = "text/plain"
         return response
 
 
@@ -247,7 +235,7 @@ class CreateWatch(Resource):
 
         new_uuid = self.datastore.add_watch(url=url, extras=extras, tag=tags)
         if new_uuid:
-            self.update_q.put(queuedWatchMetaData.PrioritizedItem(priority=1, item={'uuid': new_uuid}))
+            self.update_q.put(queuedWatchMetaData.PrioritizedItem(priority=1, item={'uuid': new_uuid, 'skip_when_checksum_same': True}))
             return {'uuid': new_uuid}, 201
         else:
             return "Invalid or unsupported URL", 400
@@ -304,7 +292,7 @@ class CreateWatch(Resource):
 
         if request.args.get('recheck_all'):
             for uuid in self.datastore.data['watching'].keys():
-                self.update_q.put(queuedWatchMetaData.PrioritizedItem(priority=1, item={'uuid': uuid}))
+                self.update_q.put(queuedWatchMetaData.PrioritizedItem(priority=1, item={'uuid': uuid, 'skip_when_checksum_same': True}))
             return {'status': "OK"}, 200
 
         return list, 200

changedetectionio/apprise_asset.py

@@ -1,4 +1,3 @@
-from changedetectionio import apprise_plugin
 import apprise
 
 # Create our AppriseAsset and populate it with some of our new values:

changedetectionio/apprise_plugin/__init__.py

@@ -1,98 +0,0 @@
-# include the decorator
-from apprise.decorators import notify
-from loguru import logger
-from requests.structures import CaseInsensitiveDict
-
-
-@notify(on="delete")
-@notify(on="deletes")
-@notify(on="get")
-@notify(on="gets")
-@notify(on="post")
-@notify(on="posts")
-@notify(on="put")
-@notify(on="puts")
-def apprise_custom_api_call_wrapper(body, title, notify_type, *args, **kwargs):
-    import requests
-    import json
-    import re
-
-    from urllib.parse import unquote_plus
-    from apprise.utils.parse import parse_url as apprise_parse_url
-
-    url = kwargs['meta'].get('url')
-    schema = kwargs['meta'].get('schema').lower().strip()
-
-    # Choose POST, GET etc from requests
-    method =  re.sub(rf's$', '', schema)
-    requests_method = getattr(requests, method)
-
-    params = CaseInsensitiveDict({}) # Added to requests
-    auth = None
-    has_error = False
-
-    # Convert /foobar?+some-header=hello to proper header dictionary
-    results = apprise_parse_url(url)
-
-    # Add our headers that the user can potentially over-ride if they wish
-    # to to our returned result set and tidy entries by unquoting them
-    headers = CaseInsensitiveDict({unquote_plus(x): unquote_plus(y)
-               for x, y in results['qsd+'].items()})
-
-    # https://github.com/caronc/apprise/wiki/Notify_Custom_JSON#get-parameter-manipulation
-    # In Apprise, it relies on prefixing each request arg with "-", because it uses say &method=update as a flag for apprise
-    # but here we are making straight requests, so we need todo convert this against apprise's logic
-    for k, v in results['qsd'].items():
-        if not k.strip('+-') in results['qsd+'].keys():
-            params[unquote_plus(k)] = unquote_plus(v)
-
-    # Determine Authentication
-    auth = ''
-    if results.get('user') and results.get('password'):
-        auth = (unquote_plus(results.get('user')), unquote_plus(results.get('user')))
-    elif results.get('user'):
-        auth = (unquote_plus(results.get('user')))
-
-    # If it smells like it could be JSON and no content-type was already set, offer a default content type.
-    if body and '{' in body[:100] and not headers.get('Content-Type'):
-        json_header = 'application/json; charset=utf-8'
-        try:
-            # Try if it's JSON
-            json.loads(body)
-            headers['Content-Type'] = json_header
-        except ValueError as e:
-            logger.warning(f"Could not automatically add '{json_header}' header to the notification because the document failed to parse as JSON: {e}")
-            pass
-
-    # POSTS -> HTTPS etc
-    if schema.lower().endswith('s'):
-        url = re.sub(rf'^{schema}', 'https', results.get('url'))
-    else:
-        url = re.sub(rf'^{schema}', 'http', results.get('url'))
-
-    status_str = ''
-    try:
-        r = requests_method(url,
-          auth=auth,
-          data=body.encode('utf-8') if type(body) is str else body,
-          headers=headers,
-          params=params
-        )
-
-        if not (200 <= r.status_code < 300):
-            status_str = f"Error sending '{method.upper()}' request to {url} - Status: {r.status_code}: '{r.reason}'"
-            logger.error(status_str)
-            has_error = True
-        else:
-            logger.info(f"Sent '{method.upper()}' request to {url}")
-            has_error = False
-
-    except requests.RequestException as e:
-        status_str = f"Error sending '{method.upper()}' request to {url} - {str(e)}"
-        logger.error(status_str)
-        has_error = True
-
-    if has_error:
-        raise TypeError(status_str)
-
-    return True

changedetectionio/blueprint/backups/__init__.py

@@ -1,164 +0,0 @@
-import datetime
-import glob
-import threading
-
-from flask import Blueprint, render_template, send_from_directory, flash, url_for, redirect, abort
-import os
-
-from changedetectionio.store import ChangeDetectionStore
-from changedetectionio.flask_app import login_optionally_required
-from loguru import logger
-
-BACKUP_FILENAME_FORMAT = "changedetection-backup-{}.zip"
-
-
-def create_backup(datastore_path, watches: dict):
-    logger.debug("Creating backup...")
-    import zipfile
-    from pathlib import Path
-
-    # create a ZipFile object
-    timestamp = datetime.datetime.now().strftime("%Y%m%d%H%M%S")
-    backupname = BACKUP_FILENAME_FORMAT.format(timestamp)
-    backup_filepath = os.path.join(datastore_path, backupname)
-
-    with zipfile.ZipFile(backup_filepath.replace('.zip', '.tmp'), "w",
-                         compression=zipfile.ZIP_DEFLATED,
-                         compresslevel=8) as zipObj:
-
-        # Add the index
-        zipObj.write(os.path.join(datastore_path, "url-watches.json"), arcname="url-watches.json")
-
-        # Add the flask app secret
-        zipObj.write(os.path.join(datastore_path, "secret.txt"), arcname="secret.txt")
-
-        # Add any data in the watch data directory.
-        for uuid, w in watches.items():
-            for f in Path(w.watch_data_dir).glob('*'):
-                zipObj.write(f,
-                             # Use the full path to access the file, but make the file 'relative' in the Zip.
-                             arcname=os.path.join(f.parts[-2], f.parts[-1]),
-                             compress_type=zipfile.ZIP_DEFLATED,
-                             compresslevel=8)
-
-        # Create a list file with just the URLs, so it's easier to port somewhere else in the future
-        list_file = "url-list.txt"
-        with open(os.path.join(datastore_path, list_file), "w") as f:
-            for uuid in watches:
-                url = watches[uuid]["url"]
-                f.write("{}\r\n".format(url))
-        list_with_tags_file = "url-list-with-tags.txt"
-        with open(
-                os.path.join(datastore_path, list_with_tags_file), "w"
-        ) as f:
-            for uuid in watches:
-                url = watches[uuid].get('url')
-                tag = watches[uuid].get('tags', {})
-                f.write("{} {}\r\n".format(url, tag))
-
-        # Add it to the Zip
-        zipObj.write(
-            os.path.join(datastore_path, list_file),
-            arcname=list_file,
-            compress_type=zipfile.ZIP_DEFLATED,
-            compresslevel=8,
-        )
-        zipObj.write(
-            os.path.join(datastore_path, list_with_tags_file),
-            arcname=list_with_tags_file,
-            compress_type=zipfile.ZIP_DEFLATED,
-            compresslevel=8,
-        )
-
-    # Now it's done, rename it so it shows up finally and its completed being written.
-    os.rename(backup_filepath.replace('.zip', '.tmp'), backup_filepath.replace('.tmp', '.zip'))
-
-
-def construct_blueprint(datastore: ChangeDetectionStore):
-    backups_blueprint = Blueprint('backups', __name__, template_folder="templates")
-    backup_threads = []
-
-    @login_optionally_required
-    @backups_blueprint.route("/request-backup", methods=['GET'])
-    def request_backup():
-        if any(thread.is_alive() for thread in backup_threads):
-            flash("A backup is already running, check back in a few minutes", "error")
-            return redirect(url_for('backups.index'))
-
-        if len(find_backups()) > int(os.getenv("MAX_NUMBER_BACKUPS", 100)):
-            flash("Maximum number of backups reached, please remove some", "error")
-            return redirect(url_for('backups.index'))
-
-        # Be sure we're written fresh
-        datastore.sync_to_json()
-        zip_thread = threading.Thread(target=create_backup, args=(datastore.datastore_path, datastore.data.get("watching")))
-        zip_thread.start()
-        backup_threads.append(zip_thread)
-        flash("Backup building in background, check back in a few minutes.")
-
-        return redirect(url_for('backups.index'))
-
-    def find_backups():
-        backup_filepath = os.path.join(datastore.datastore_path, BACKUP_FILENAME_FORMAT.format("*"))
-        backups = glob.glob(backup_filepath)
-        backup_info = []
-
-        for backup in backups:
-            size = os.path.getsize(backup) / (1024 * 1024)
-            creation_time = os.path.getctime(backup)
-            backup_info.append({
-                'filename': os.path.basename(backup),
-                'filesize': f"{size:.2f}",
-                'creation_time': creation_time
-            })
-
-        backup_info.sort(key=lambda x: x['creation_time'], reverse=True)
-
-        return backup_info
-
-    @login_optionally_required
-    @backups_blueprint.route("/download/<string:filename>", methods=['GET'])
-    def download_backup(filename):
-        import re
-        filename = filename.strip()
-        backup_filename_regex = BACKUP_FILENAME_FORMAT.format("\d+")
-
-        full_path = os.path.join(os.path.abspath(datastore.datastore_path), filename)
-        if not full_path.startswith(os.path.abspath(datastore.datastore_path)):
-            abort(404)
-
-        if filename == 'latest':
-            backups = find_backups()
-            filename = backups[0]['filename']
-
-        if not re.match(r"^" + backup_filename_regex + "$", filename):
-            abort(400)  # Bad Request if the filename doesn't match the pattern
-
-        logger.debug(f"Backup download request for '{full_path}'")
-        return send_from_directory(os.path.abspath(datastore.datastore_path), filename, as_attachment=True)
-
-    @login_optionally_required
-    @backups_blueprint.route("/", methods=['GET'])
-    def index():
-        backups = find_backups()
-        output = render_template("overview.html",
-                                 available_backups=backups,
-                                 backup_running=any(thread.is_alive() for thread in backup_threads)
-                                 )
-
-        return output
-
-    @login_optionally_required
-    @backups_blueprint.route("/remove-backups", methods=['GET'])
-    def remove_backups():
-
-        backup_filepath = os.path.join(datastore.datastore_path, BACKUP_FILENAME_FORMAT.format("*"))
-        backups = glob.glob(backup_filepath)
-        for backup in backups:
-            os.unlink(backup)
-
-        flash("Backups were deleted.")
-
-        return redirect(url_for('backups.index'))
-
-    return backups_blueprint

changedetectionio/blueprint/backups/templates/overview.html

@@ -1,36 +0,0 @@
-{% extends 'base.html' %}
-{% block content %}
-    {% from '_helpers.html' import render_simple_field, render_field %}
-    <div class="edit-form">
-        <div class="box-wrap inner">
-            <h4>Backups</h4>
-            {% if backup_running %}
-                <p>
-                    <strong>A backup is running!</strong>
-                </p>
-            {% endif %}
-            <p>
-                Here you can download and request a new backup, when a backup is completed you will see it listed below.
-            </p>
-            <br>
-                {% if available_backups %}
-                    <ul>
-                    {% for backup in available_backups %}
-                        <li><a href="{{ url_for('backups.download_backup', filename=backup["filename"]) }}">{{ backup["filename"] }}</a> {{  backup["filesize"] }} Mb</li>
-                    {% endfor %}
-                    </ul>
-                {% else %}
-                    <p>
-                    <strong>No backups found.</strong>
-                    </p>
-                {% endif %}
-
-            <a class="pure-button pure-button-primary" href="{{ url_for('backups.request_backup') }}">Create backup</a>
-            {% if available_backups %}
-                <a class="pure-button button-small button-error " href="{{ url_for('backups.remove_backups') }}">Remove backups</a>
-            {% endif %}
-        </div>
-    </div>
-
-
-{% endblock %}

changedetectionio/blueprint/browser_steps/__init__.py

@@ -12,7 +12,7 @@
 #
 #
 
-from changedetectionio.strtobool import strtobool
+from distutils.util import strtobool
 from flask import Blueprint, request, make_response
 import os
 
@@ -22,10 +22,7 @@ from loguru import logger
 
 browsersteps_sessions = {}
 io_interface_context = None
-import json
-import base64
-import hashlib
-from flask import Response
+
 
 def construct_blueprint(datastore: ChangeDetectionStore):
     browser_steps_blueprint = Blueprint('browser_steps', __name__, template_folder="templates")
@@ -87,10 +84,7 @@ def construct_blueprint(datastore: ChangeDetectionStore):
         # Tell Playwright to connect to Chrome and setup a new session via our stepper interface
         browsersteps_start_session['browserstepper'] = browser_steps.browsersteps_live_ui(
             playwright_browser=browsersteps_start_session['browser'],
-            proxy=proxy,
-            start_url=datastore.data['watching'][watch_uuid].link,
-            headers=datastore.data['watching'][watch_uuid].get('headers')
-        )
+            proxy=proxy)
 
         # For test
         #browsersteps_start_session['browserstepper'].action_goto_url(value="http://example.com?time="+str(time.time()))
@@ -163,15 +157,21 @@ def construct_blueprint(datastore: ChangeDetectionStore):
         if not browsersteps_sessions.get(browsersteps_session_id):
             return make_response('No session exists under that ID', 500)
 
-        is_last_step = False
+
         # Actions - step/apply/etc, do the thing and return state
         if request.method == 'POST':
             # @todo - should always be an existing session
             step_operation = request.form.get('operation')
             step_selector = request.form.get('selector')
             step_optional_value = request.form.get('optional_value')
+            step_n = int(request.form.get('step_n'))
             is_last_step = strtobool(request.form.get('is_last_step'))
 
+            if step_operation == 'Goto site':
+                step_operation = 'goto_url'
+                step_optional_value = datastore.data['watching'][uuid].get('url')
+                step_selector = None
+
             # @todo try.. accept.. nice errors not popups..
             try:
 
@@ -184,6 +184,14 @@ def construct_blueprint(datastore: ChangeDetectionStore):
                 # Try to find something of value to give back to the user
                 return make_response(str(e).splitlines()[0], 401)
 
+            # Get visual selector ready/update its data (also use the current filter info from the page?)
+            # When the last 'apply' button was pressed
+            # @todo this adds overhead because the xpath selection is happening twice
+            u = browsersteps_sessions[browsersteps_session_id]['browserstepper'].page.url
+            if is_last_step and u:
+                (screenshot, xpath_data) = browsersteps_sessions[browsersteps_session_id]['browserstepper'].request_visualselector_data()
+                datastore.save_screenshot(watch_uuid=uuid, screenshot=screenshot)
+                datastore.save_xpath_data(watch_uuid=uuid, data=xpath_data)
 
 #        if not this_session.page:
 #            cleanup_playwright_session()
@@ -191,35 +199,31 @@ def construct_blueprint(datastore: ChangeDetectionStore):
 
         # Screenshots and other info only needed on requesting a step (POST)
         try:
-            (screenshot, xpath_data) = browsersteps_sessions[browsersteps_session_id]['browserstepper'].get_current_state()
-            if is_last_step:
-                watch = datastore.data['watching'].get(uuid)
-                u = browsersteps_sessions[browsersteps_session_id]['browserstepper'].page.url
-                if watch and u:
-                    watch.save_screenshot(screenshot=screenshot)
-                    watch.save_xpath_data(data=xpath_data)
-
+            state = browsersteps_sessions[browsersteps_session_id]['browserstepper'].get_current_state()
         except playwright._impl._api_types.Error as e:
             return make_response("Browser session ran out of time :( Please reload this page."+str(e), 401)
-        except Exception as e:
-            return make_response("Error fetching screenshot and element data - " + str(e), 401)
 
-        # SEND THIS BACK TO THE BROWSER
+        # Use send_file() which is way faster than read/write loop on bytes
+        import json
+        from tempfile import mkstemp
+        from flask import send_file
+        tmp_fd, tmp_file = mkstemp(text=True, suffix=".json", prefix="changedetectionio-")
 
-        output = {
-            "screenshot": f"data:image/jpeg;base64,{base64.b64encode(screenshot).decode('ascii')}",
-            "xpath_data": xpath_data,
-            "session_age_start": browsersteps_sessions[browsersteps_session_id]['browserstepper'].age_start,
-            "browser_time_remaining": round(remaining)
-        }
-        json_data = json.dumps(output)
+        output = json.dumps({'screenshot': "data:image/jpeg;base64,{}".format(
+            base64.b64encode(state[0]).decode('ascii')),
+            'xpath_data': state[1],
+            'session_age_start': browsersteps_sessions[browsersteps_session_id]['browserstepper'].age_start,
+            'browser_time_remaining': round(remaining)
+        })
 
-        # Generate an ETag (hash of the response body)
-        etag_hash = hashlib.md5(json_data.encode('utf-8')).hexdigest()
+        with os.fdopen(tmp_fd, 'w') as f:
+            f.write(output)
 
-        # Create the response with ETag
-        response = Response(json_data, mimetype="application/json; charset=UTF-8")
-        response.set_etag(etag_hash)
+        response = make_response(send_file(path_or_file=tmp_file,
+                                           mimetype='application/json; charset=UTF-8',
+                                           etag=True))
+        # No longer needed
+        os.unlink(tmp_file)
 
         return response
 

changedetectionio/blueprint/browser_steps/browser_steps.py

@@ -1,14 +1,12 @@
+#!/usr/bin/python3
+
 import os
 import time
 import re
 from random import randint
 from loguru import logger
 
-from changedetectionio.content_fetchers.helpers import capture_stitched_together_full_page, SCREENSHOT_SIZE_STITCH_THRESHOLD
 from changedetectionio.content_fetchers.base import manage_user_agent
-from changedetectionio.safe_jinja import render as jinja_render
-
-
 
 # Two flags, tell the JS which of the "Selector" or "Value" field should be enabled in the front end
 # 0- off, 1- on
@@ -26,13 +24,11 @@ browser_step_ui_config = {'Choose one': '0 0',
                           'Click element if exists': '1 0',
                           'Click element': '1 0',
                           'Click element containing text': '0 1',
-                          'Click element containing text if exists': '0 1',
                           'Enter text in field': '1 1',
                           'Execute JS': '0 1',
 #                          'Extract text and use as filter': '1 0',
                           'Goto site': '0 0',
                           'Goto URL': '0 1',
-                          'Make all child elements visible': '1 0',
                           'Press Enter': '0 0',
                           'Select by label': '1 1',
                           'Scroll down': '0 0',
@@ -40,7 +36,6 @@ browser_step_ui_config = {'Choose one': '0 0',
                           'Wait for seconds': '0 1',
                           'Wait for text': '0 1',
                           'Wait for text in element': '1 1',
-                          'Remove elements': '1 0',
                           #                          'Press Page Down': '0 0',
                           #                          'Press Page Up': '0 0',
                           # weird bug, come back to it later
@@ -53,12 +48,6 @@ browser_step_ui_config = {'Choose one': '0 0',
 # ONLY Works in Playwright because we need the fullscreen screenshot
 class steppable_browser_interface():
     page = None
-    start_url = None
-
-    action_timeout = 10 * 1000
-
-    def __init__(self, start_url):
-        self.start_url = start_url
 
     # Convert and perform "Click Button" for example
     def call_action(self, action_name, selector=None, optional_value=None):
@@ -75,12 +64,14 @@ class steppable_browser_interface():
         action_handler = getattr(self, "action_" + call_action_name)
 
         # Support for Jinja2 variables in the value and selector
+        from jinja2 import Environment
+        jinja2_env = Environment(extensions=['jinja2_time.TimeExtension'])
 
         if selector and ('{%' in selector or '{{' in selector):
-            selector = jinja_render(template_str=selector)
+            selector = str(jinja2_env.from_string(selector).render())
 
         if optional_value and ('{%' in optional_value or '{{' in optional_value):
-            optional_value = jinja_render(template_str=optional_value)
+            optional_value = str(jinja2_env.from_string(optional_value).render())
 
         action_handler(selector, optional_value)
         self.page.wait_for_timeout(1.5 * 1000)
@@ -97,34 +88,18 @@ class steppable_browser_interface():
         logger.debug(f"Time to goto URL {time.time()-now:.2f}s")
         return response
 
-    # Incase they request to go back to the start
-    def action_goto_site(self, selector=None, value=None):
-        return self.action_goto_url(value=self.start_url)
-
     def action_click_element_containing_text(self, selector=None, value=''):
-        logger.debug("Clicking element containing text")
         if not len(value.strip()):
             return
         elem = self.page.get_by_text(value)
         if elem.count():
-            elem.first.click(delay=randint(200, 500), timeout=self.action_timeout)
-
-    def action_click_element_containing_text_if_exists(self, selector=None, value=''):
-        logger.debug("Clicking element containing text if exists")
-        if not len(value.strip()):
-            return
-        elem = self.page.get_by_text(value)
-        logger.debug(f"Clicking element containing text - {elem.count()} elements found")
-        if elem.count():
-            elem.first.click(delay=randint(200, 500), timeout=self.action_timeout)
-        else:
-            return
+            elem.first.click(delay=randint(200, 500), timeout=3000)
 
     def action_enter_text_in_field(self, selector, value):
         if not len(selector.strip()):
             return
 
-        self.page.fill(selector, value, timeout=self.action_timeout)
+        self.page.fill(selector, value, timeout=10 * 1000)
 
     def action_execute_js(self, selector, value):
         response = self.page.evaluate(value)
@@ -135,7 +110,7 @@ class steppable_browser_interface():
         if not len(selector.strip()):
             return
 
-        self.page.click(selector=selector, timeout=self.action_timeout + 20 * 1000, delay=randint(200, 500))
+        self.page.click(selector=selector, timeout=30 * 1000, delay=randint(200, 500))
 
     def action_click_element_if_exists(self, selector, value):
         import playwright._impl._errors as _api_types
@@ -143,7 +118,7 @@ class steppable_browser_interface():
         if not len(selector.strip()):
             return
         try:
-            self.page.click(selector, timeout=self.action_timeout, delay=randint(200, 500))
+            self.page.click(selector, timeout=10 * 1000, delay=randint(200, 500))
         except _api_types.TimeoutError as e:
             return
         except _api_types.Error as e:
@@ -190,29 +165,11 @@ class steppable_browser_interface():
         self.page.keyboard.press("PageDown", delay=randint(200, 500))
 
     def action_check_checkbox(self, selector, value):
-        self.page.locator(selector).check(timeout=self.action_timeout)
+        self.page.locator(selector).check(timeout=1000)
 
     def action_uncheck_checkbox(self, selector, value):
-        self.page.locator(selector).uncheck(timeout=self.action_timeout)
+        self.page.locator(selector, timeout=1000).uncheck(timeout=1000)
 
-    def action_remove_elements(self, selector, value):
-        """Removes all elements matching the given selector from the DOM."""
-        self.page.locator(selector).evaluate_all("els => els.forEach(el => el.remove())")
-
-    def action_make_all_child_elements_visible(self, selector, value):
-        """Recursively makes all child elements inside the given selector fully visible."""
-        self.page.locator(selector).locator("*").evaluate_all("""
-            els => els.forEach(el => {
-                el.style.display = 'block';   // Forces it to be displayed
-                el.style.visibility = 'visible';   // Ensures it's not hidden
-                el.style.opacity = '1';   // Fully opaque
-                el.style.position = 'relative';   // Avoids 'absolute' hiding
-                el.style.height = 'auto';   // Expands collapsed elements
-                el.style.width = 'auto';   // Ensures full visibility
-                el.removeAttribute('hidden');   // Removes hidden attribute
-                el.classList.remove('hidden', 'd-none');  // Removes common CSS hidden classes
-            })
-        """)
 
 # Responsible for maintaining a live 'context' with the chrome CDP
 # @todo - how long do contexts live for anyway?
@@ -238,11 +195,10 @@ class browsersteps_live_ui(steppable_browser_interface):
 
     browser_type = os.getenv("PLAYWRIGHT_BROWSER_TYPE", 'chromium').strip('"')
 
-    def __init__(self, playwright_browser, proxy=None, headers=None, start_url=None):
+    def __init__(self, playwright_browser, proxy=None, headers=None):
         self.headers = headers or {}
         self.age_start = time.time()
         self.playwright_browser = playwright_browser
-        self.start_url = start_url
         if self.context is None:
             self.connect(proxy=proxy)
 
@@ -280,7 +236,6 @@ class browsersteps_live_ui(steppable_browser_interface):
         logger.debug(f"Time to browser setup {time.time()-now:.2f}s")
         self.page.wait_for_timeout(1 * 1000)
 
-
     def mark_as_closed(self):
         logger.debug("Page closed, cleaning up..")
 
@@ -292,36 +247,46 @@ class browsersteps_live_ui(steppable_browser_interface):
 
     def get_current_state(self):
         """Return the screenshot and interactive elements mapping, generally always called after action_()"""
-        import importlib.resources
-        xpath_element_js = importlib.resources.files("changedetectionio.content_fetchers.res").joinpath('xpath_element_scraper.js').read_text()
-
+        from pkg_resources import resource_string
+        xpath_element_js = resource_string(__name__, "../../content_fetchers/res/xpath_element_scraper.js").decode('utf-8')
         now = time.time()
         self.page.wait_for_timeout(1 * 1000)
 
+        # The actual screenshot
+        screenshot = self.page.screenshot(type='jpeg', full_page=True, quality=40)
 
-        full_height = self.page.evaluate("document.documentElement.scrollHeight")
-
-        if full_height >= SCREENSHOT_SIZE_STITCH_THRESHOLD:
-            logger.warning(f"Page full Height: {full_height}px longer than {SCREENSHOT_SIZE_STITCH_THRESHOLD}px, using 'stitched screenshot method'.")
-            screenshot = capture_stitched_together_full_page(self.page)
-        else:
-            screenshot = self.page.screenshot(type='jpeg', full_page=True, quality=40)
-
-        logger.debug(f"Time to get screenshot from browser {time.time() - now:.2f}s")
-
-        now = time.time()
         self.page.evaluate("var include_filters=''")
         # Go find the interactive elements
         # @todo in the future, something smarter that can scan for elements with .click/focus etc event handlers?
         elements = 'a,button,input,select,textarea,i,th,td,p,li,h1,h2,h3,h4,div,span'
         xpath_element_js = xpath_element_js.replace('%ELEMENTS%', elements)
-
         xpath_data = self.page.evaluate("async () => {" + xpath_element_js + "}")
         # So the JS will find the smallest one first
         xpath_data['size_pos'] = sorted(xpath_data['size_pos'], key=lambda k: k['width'] * k['height'], reverse=True)
-        logger.debug(f"Time to scrape xpath element data in browser {time.time()-now:.2f}s")
-
+        logger.debug(f"Time to complete get_current_state of browser {time.time()-now:.2f}s")
+        # except
         # playwright._impl._api_types.Error: Browser closed.
         # @todo show some countdown timer?
         return (screenshot, xpath_data)
 
+    def request_visualselector_data(self):
+        """
+        Does the same that the playwright operation in content_fetcher does
+        This is used to just bump the VisualSelector data so it' ready to go if they click on the tab
+        @todo refactor and remove duplicate code, add include_filters
+        :param xpath_data:
+        :param screenshot:
+        :param current_include_filters:
+        :return:
+        """
+
+        self.page.evaluate("var include_filters=''")
+        from pkg_resources import resource_string
+        # The code that scrapes elements and makes a list of elements/size/position to click on in the VisualSelector
+        xpath_element_js = resource_string(__name__, "../../content_fetchers/res/xpath_element_scraper.js").decode('utf-8')
+        from changedetectionio.content_fetchers import visualselector_xpath_selectors
+        xpath_element_js = xpath_element_js.replace('%ELEMENTS%', visualselector_xpath_selectors)
+        xpath_data = self.page.evaluate("async () => {" + xpath_element_js + "}")
+        screenshot = self.page.screenshot(type='jpeg', full_page=True, quality=int(os.getenv("SCREENSHOT_QUALITY", 72)))
+
+        return (screenshot, xpath_data)

changedetectionio/blueprint/check_proxies/__init__.py

@@ -1,7 +1,4 @@
-import importlib
 from concurrent.futures import ThreadPoolExecutor
-
-from changedetectionio.processors.text_json_diff.processor import FilterNotFoundInResponse
 from changedetectionio.store import ChangeDetectionStore
 
 from functools import wraps
@@ -33,19 +30,16 @@ def construct_blueprint(datastore: ChangeDetectionStore):
     def long_task(uuid, preferred_proxy):
         import time
         from changedetectionio.content_fetchers import exceptions as content_fetcher_exceptions
-        from changedetectionio.safe_jinja import render as jinja_render
+        from changedetectionio.processors import text_json_diff
 
         status = {'status': '', 'length': 0, 'text': ''}
+        from jinja2 import Environment, BaseLoader
 
         contents = ''
         now = time.time()
         try:
-            processor_module = importlib.import_module("changedetectionio.processors.text_json_diff.processor")
-            update_handler = processor_module.perform_site_check(datastore=datastore,
-                                                                 watch_uuid=uuid
-                                                                 )
-
-            update_handler.call_browser(preferred_proxy_id=preferred_proxy)
+            update_handler = text_json_diff.perform_site_check(datastore=datastore, watch_uuid=uuid)
+            update_handler.call_browser()
         # title, size is len contents not len xfer
         except content_fetcher_exceptions.Non200ErrorCodeReceived as e:
             if e.status_code == 404:
@@ -54,7 +48,7 @@ def construct_blueprint(datastore: ChangeDetectionStore):
                 status.update({'status': 'ERROR', 'length': len(contents), 'text': f"{e.status_code} - Access denied"})
             else:
                 status.update({'status': 'ERROR', 'length': len(contents), 'text': f"Status code: {e.status_code}"})
-        except FilterNotFoundInResponse:
+        except text_json_diff.FilterNotFoundInResponse:
             status.update({'status': 'OK', 'length': len(contents), 'text': f"OK but CSS/xPath filter not found (page changed layout?)"})
         except content_fetcher_exceptions.EmptyReply as e:
             if e.status_code == 403 or e.status_code == 401:
@@ -70,9 +64,7 @@ def construct_blueprint(datastore: ChangeDetectionStore):
             status.update({'status': 'OK', 'length': len(contents), 'text': ''})
 
         if status.get('text'):
-            # parse 'text' as text for safety
-            v = {'text': status['text']}
-            status['text'] = jinja_render(template_str='{{text|e}}', **v)
+            status['text'] = Environment(loader=BaseLoader()).from_string('{{text|e}}').render({'text': status['text']})
 
         status['time'] = "{:.2f}s".format(time.time() - now)
 

changedetectionio/blueprint/price_data_follower/__init__.py

@@ -1,5 +1,5 @@
 
-from changedetectionio.strtobool import strtobool
+from distutils.util import strtobool
 from flask import Blueprint, flash, redirect, url_for
 from flask_login import login_required
 from changedetectionio.store import ChangeDetectionStore
@@ -17,9 +17,7 @@ def construct_blueprint(datastore: ChangeDetectionStore, update_q: PriorityQueue
     @price_data_follower_blueprint.route("/<string:uuid>/accept", methods=['GET'])
     def accept(uuid):
         datastore.data['watching'][uuid]['track_ldjson_price_data'] = PRICE_DATA_TRACK_ACCEPT
-        datastore.data['watching'][uuid]['processor'] = 'restock_diff'
-        datastore.data['watching'][uuid].clear_watch()
-        update_q.put(queuedWatchMetaData.PrioritizedItem(priority=1, item={'uuid': uuid}))
+        update_q.put(queuedWatchMetaData.PrioritizedItem(priority=1, item={'uuid': uuid, 'skip_when_checksum_same': False}))
         return redirect(url_for("index"))
 
     @login_required

changedetectionio/blueprint/tags/__init__.py

@@ -1,6 +1,4 @@
-from flask import Blueprint, request, render_template, flash, url_for, redirect
-
-
+from flask import Blueprint, request, make_response, render_template, flash, url_for, redirect
 from changedetectionio.store import ChangeDetectionStore
 from changedetectionio.flask_app import login_optionally_required
 
@@ -13,17 +11,10 @@ def construct_blueprint(datastore: ChangeDetectionStore):
     def tags_overview_page():
         from .form import SingleTag
         add_form = SingleTag(request.form)
-
         sorted_tags = sorted(datastore.data['settings']['application'].get('tags').items(), key=lambda x: x[1]['title'])
-
-        from collections import Counter
-
-        tag_count = Counter(tag for watch in datastore.data['watching'].values() if watch.get('tags') for tag in watch['tags'])
-
         output = render_template("groups-overview.html",
-                                 available_tags=sorted_tags,
                                  form=add_form,
-                                 tag_count=tag_count
+                                 available_tags=sorted_tags,
                                  )
 
         return output
@@ -99,57 +90,22 @@ def construct_blueprint(datastore: ChangeDetectionStore):
     @tags_blueprint.route("/edit/<string:uuid>", methods=['GET'])
     @login_optionally_required
     def form_tag_edit(uuid):
-        from changedetectionio.blueprint.tags.form import group_restock_settings_form
+        from changedetectionio import forms
+
         if uuid == 'first':
             uuid = list(datastore.data['settings']['application']['tags'].keys()).pop()
 
         default = datastore.data['settings']['application']['tags'].get(uuid)
 
-        form = group_restock_settings_form(
-                                       formdata=request.form if request.method == 'POST' else None,
-                                       data=default,
-                                       extra_notification_tokens=datastore.get_unique_notification_tokens_available(),
-                                       default_system_settings = datastore.data['settings'],
-                                       )
-
-        template_args = {
-            'data': default,
-            'form': form,
-            'watch': default,
-            'extra_notification_token_placeholder_info': datastore.get_unique_notification_token_placeholders_available(),
-        }
-
-        included_content = {}
-        if form.extra_form_content():
-            # So that the extra panels can access _helpers.html etc, we set the environment to load from templates/
-            # And then render the code from the module
-            from jinja2 import Environment, FileSystemLoader
-            import importlib.resources
-            templates_dir = str(importlib.resources.files("changedetectionio").joinpath('templates'))
-            env = Environment(loader=FileSystemLoader(templates_dir))
-            template_str = """{% from '_helpers.html' import render_field, render_checkbox_field, render_button %}
-        <script>        
-            $(document).ready(function () {
-                toggleOpacity('#overrides_watch', '#restock-fieldset-price-group', true);
-            });
-        </script>            
-                <fieldset>
-                    <div class="pure-control-group">
-                        <fieldset class="pure-group">
-                        {{ render_checkbox_field(form.overrides_watch) }}
-                        <span class="pure-form-message-inline">Used for watches in "Restock & Price detection" mode</span>
-                        </fieldset>
-                </fieldset>
-                """
-            template_str += form.extra_form_content()
-            template = env.from_string(template_str)
-            included_content = template.render(**template_args)
+        form = forms.watchForm(formdata=request.form if request.method == 'POST' else None,
+                               data=default,
+                               )
+        form.datastore=datastore # needed?
 
         output = render_template("edit-tag.html",
+                                 data=default,
+                                 form=form,
                                  settings_application=datastore.data['settings']['application'],
-                                 extra_tab_content=form.extra_tab_content() if form.extra_tab_content() else None,
-                                 extra_form_content=included_content,
-                                 **template_args
                                  )
 
         return output
@@ -158,15 +114,14 @@ def construct_blueprint(datastore: ChangeDetectionStore):
     @tags_blueprint.route("/edit/<string:uuid>", methods=['POST'])
     @login_optionally_required
     def form_tag_edit_submit(uuid):
-        from changedetectionio.blueprint.tags.form import group_restock_settings_form
+        from changedetectionio import forms
         if uuid == 'first':
             uuid = list(datastore.data['settings']['application']['tags'].keys()).pop()
 
         default = datastore.data['settings']['application']['tags'].get(uuid)
 
-        form = group_restock_settings_form(formdata=request.form if request.method == 'POST' else None,
+        form = forms.watchForm(formdata=request.form if request.method == 'POST' else None,
                                data=default,
-                               extra_notification_tokens=datastore.get_unique_notification_tokens_available()
                                )
         # @todo subclass form so validation works
         #if not form.validate():
@@ -175,7 +130,6 @@ def construct_blueprint(datastore: ChangeDetectionStore):
 #           return redirect(url_for('tags.form_tag_edit_submit', uuid=uuid))
 
         datastore.data['settings']['application']['tags'][uuid].update(form.data)
-        datastore.data['settings']['application']['tags'][uuid]['processor'] = 'restock_diff'
         datastore.needs_write_urgent = True
         flash("Updated")
 

changedetectionio/blueprint/tags/form.py

@@ -1,15 +1,16 @@
 from wtforms import (
+    BooleanField,
     Form,
+    IntegerField,
+    RadioField,
+    SelectField,
     StringField,
     SubmitField,
+    TextAreaField,
     validators,
 )
-from wtforms.fields.simple import BooleanField
 
-from changedetectionio.processors.restock_diff.forms import processor_settings_form as restock_settings_form
 
-class group_restock_settings_form(restock_settings_form):
-    overrides_watch = BooleanField('Activate for individual watches in this tag/group?', default=False)
 
 class SingleTag(Form):
 

changedetectionio/blueprint/tags/templates/edit-tag.html

@@ -1,9 +1,9 @@
 {% extends 'base.html' %}
 {% block content %}
-{% from '_helpers.html' import render_field, render_checkbox_field, render_button %}
-{% from '_common_fields.html' import render_common_settings_form %}
+{% from '_helpers.jinja' import render_field, render_checkbox_field, render_button %}
+{% from '_common_fields.jinja' import render_common_settings_form %}
 <script>
-    const notification_base_url="{{url_for('ajax_callback_send_notification_test', mode="group-settings")}}";
+    const notification_base_url="{{url_for('ajax_callback_send_notification_test', watch_uuid=uuid)}}";
 </script>
 
 <script src="{{url_for('static_content', group='js', filename='tabs.js')}}" defer></script>
@@ -17,6 +17,7 @@
 </script>
 
 <script src="{{url_for('static_content', group='js', filename='watch-settings.js')}}" defer></script>
+<!--<script src="{{url_for('static_content', group='js', filename='limit.js')}}" defer></script>-->
 <script src="{{url_for('static_content', group='js', filename='notifications.js')}}" defer></script>
 
 <div class="edit-form monospaced-textarea">
@@ -25,9 +26,6 @@
         <ul>
             <li class="tab" id=""><a href="#general">General</a></li>
             <li class="tab"><a href="#filters-and-triggers">Filters &amp; Triggers</a></li>
-            {% if extra_tab_content %}
-            <li class="tab"><a href="#extras_tab">{{ extra_tab_content }}</a></li>
-            {% endif %}
             <li class="tab"><a href="#notifications">Notifications</a></li>
         </ul>
     </div>
@@ -57,15 +55,15 @@ xpath://body/div/span[contains(@class, 'example-class')]",
                         {% if '/text()' in  field %}
                           <span class="pure-form-message-inline"><strong>Note!: //text() function does not work where the &lt;element&gt; contains &lt;![CDATA[]]&gt;</strong></span><br>
                         {% endif %}
-                        <span class="pure-form-message-inline">One CSS, xPath, JSON Path/JQ selector per line, <i>any</i> rules that matches will be used.<br>
-                    <div data-target="#advanced-help-selectors" class="toggle-show pure-button button-tag button-xsmall">Show advanced help and tips</div>
-                    <ul id="advanced-help-selectors">
+                        <span class="pure-form-message-inline">One rule per line, <i>any</i> rules that matches will be used.<br>
+
+                    <ul>
                         <li>CSS - Limit text to this CSS rule, only text matching this CSS rule is included.</li>
                         <li>JSON - Limit text to this JSON rule, using either <a href="https://pypi.org/project/jsonpath-ng/" target="new">JSONPath</a> or <a href="https://stedolan.github.io/jq/" target="new">jq</a> (if installed).
                             <ul>
                                 <li>JSONPath: Prefix with <code>json:</code>, use <code>json:$</code> to force re-formatting if required,  <a href="https://jsonpath.com/" target="new">test your JSONPath here</a>.</li>
                                 {% if jq_support %}
-                                <li>jq: Prefix with <code>jq:</code> and <a href="https://jqplay.org/" target="new">test your jq here</a>. Using <a href="https://stedolan.github.io/jq/" target="new">jq</a> allows for complex filtering and processing of JSON data with built-in functions, regex, filtering, and more. See examples and documentation <a href="https://stedolan.github.io/jq/manual/" target="new">here</a>. Prefix <code>jqraw:</code> outputs the results as text instead of a JSON list.</li>
+                                <li>jq: Prefix with <code>jq:</code> and <a href="https://jqplay.org/" target="new">test your jq here</a>. Using <a href="https://stedolan.github.io/jq/" target="new">jq</a> allows for complex filtering and processing of JSON data with built-in functions, regex, filtering, and more. See examples and documentation <a href="https://stedolan.github.io/jq/manual/" target="new">here</a>.</li>
                                 {% else %}
                                 <li>jq support not installed</li>
                                 {% endif %}
@@ -88,25 +86,17 @@ xpath://body/div/span[contains(@class, 'example-class')]",
                     {{ render_field(form.subtractive_selectors, rows=5, placeholder="header
 footer
 nav
-.stockticker
-//*[contains(text(), 'Advertisement')]") }}
+.stockticker") }}
                     <span class="pure-form-message-inline">
                         <ul>
-                          <li> Remove HTML element(s) by CSS and XPath selectors before text conversion. </li>
-                          <li> Don't paste HTML here, use only CSS and XPath selectors </li>
-                          <li> Add multiple elements, CSS or XPath selectors per line to ignore multiple parts of the HTML. </li>
+                          <li> Remove HTML element(s) by CSS selector before text conversion. </li>
+                          <li> Add multiple elements or CSS selectors per line to ignore multiple parts of the HTML. </li>
                         </ul>
                       </span>
                 </fieldset>
 
             </div>
 
-        {# rendered sub Template #}
-        {% if extra_form_content %}
-            <div class="tab-pane-inner" id="extras_tab">
-            {{ extra_form_content|safe }}
-            </div>
-        {% endif %}
             <div class="tab-pane-inner" id="notifications">
                 <fieldset>
                     <div  class="pure-control-group inline-radio">
@@ -129,7 +119,7 @@ nav
                         {% endif %}
                         <a href="#notifications" id="notification-setting-reset-to-default" class="pure-button button-xsmall" style="right: 20px; top: 20px; position: absolute; background-color: #5f42dd; border-radius: 4px; font-size: 70%; color: #fff">Use system defaults</a>
 
-                        {{ render_common_settings_form(form, emailprefix, settings_application, extra_notification_token_placeholder_info) }}
+                        {{ render_common_settings_form(form, emailprefix, settings_application) }}
                     </div>
                 </fieldset>
             </div>

changedetectionio/blueprint/tags/templates/groups-overview.html

@@ -1,6 +1,6 @@
 {% extends 'base.html' %}
 {% block content %}
-{% from '_helpers.html' import render_simple_field, render_field %}
+{% from '_helpers.jinja' import render_simple_field, render_field %}
 <script src="{{url_for('static_content', group='js', filename='jquery-3.6.0.min.js')}}"></script>
 
 <div class="box">
@@ -27,7 +27,6 @@
             <thead>
             <tr>
                 <th></th>
-                <th># Watches</th>
                 <th>Tag / Label name</th>
                 <th></th>
             </tr>
@@ -46,8 +45,7 @@
                 <td class="watch-controls">
                     <a class="link-mute state-{{'on' if tag.notification_muted else 'off'}}" href="{{url_for('tags.mute', uuid=tag.uuid)}}"><img src="{{url_for('static_content', group='images', filename='bell-off.svg')}}" alt="Mute notifications" title="Mute notifications" class="icon icon-mute" ></a>
                 </td>
-                <td>{{ "{:,}".format(tag_count[uuid]) if uuid in tag_count else 0 }}</td>
-                <td class="title-col inline"> <a href="{{url_for('index', tag=uuid) }}">{{ tag.title }}</a></td>
+                <td class="title-col inline">{{tag.title}}</td>
                 <td>
                     <a class="pure-button pure-button-primary" href="{{ url_for('tags.form_tag_edit', uuid=uuid) }}">Edit</a>&nbsp;
                     <a class="pure-button pure-button-primary" href="{{ url_for('tags.delete', uuid=uuid) }}" title="Deletes and removes tag">Delete</a>

changedetectionio/conditions/__init__.py

@@ -1,135 +0,0 @@
-from flask import Blueprint
-
-from json_logic.builtins import BUILTINS
-
-from .exceptions import EmptyConditionRuleRowNotUsable
-from .pluggy_interface import plugin_manager  # Import the pluggy plugin manager
-from . import default_plugin
-
-# List of all supported JSON Logic operators
-operator_choices = [
-    (None, "Choose one"),
-    (">", "Greater Than"),
-    ("<", "Less Than"),
-    (">=", "Greater Than or Equal To"),
-    ("<=", "Less Than or Equal To"),
-    ("==", "Equals"),
-    ("!=", "Not Equals"),
-    ("in", "Contains"),
-    ("!in", "Does Not Contain"),
-]
-
-# Fields available in the rules
-field_choices = [
-    (None, "Choose one"),
-]
-
-# The data we will feed the JSON Rules to see if it passes the test/conditions or not
-EXECUTE_DATA = {}
-
-
-# Define the extended operations dictionary
-CUSTOM_OPERATIONS = {
-    **BUILTINS,  # Include all standard operators
-}
-
-def filter_complete_rules(ruleset):
-    rules = [
-        rule for rule in ruleset
-        if all(value not in ("", False, "None", None) for value in [rule["operator"], rule["field"], rule["value"]])
-    ]
-    return rules
-
-def convert_to_jsonlogic(logic_operator: str, rule_dict: list):
-    """
-    Convert a structured rule dict into a JSON Logic rule.
-
-    :param rule_dict: Dictionary containing conditions.
-    :return: JSON Logic rule as a dictionary.
-    """
-
-
-    json_logic_conditions = []
-
-    for condition in rule_dict:
-        operator = condition["operator"]
-        field = condition["field"]
-        value = condition["value"]
-
-        if not operator or operator == 'None' or not value or not field:
-            raise EmptyConditionRuleRowNotUsable()
-
-        # Convert value to int/float if possible
-        try:
-            if isinstance(value, str) and "." in value and str != "None":
-                value = float(value)
-            else:
-                value = int(value)
-        except (ValueError, TypeError):
-            pass  # Keep as a string if conversion fails
-
-        # Handle different JSON Logic operators properly
-        if operator == "in":
-            json_logic_conditions.append({"in": [value, {"var": field}]})  # value first
-        elif operator in ("!", "!!", "-"):
-            json_logic_conditions.append({operator: [{"var": field}]})  # Unary operators
-        elif operator in ("min", "max", "cat"):
-            json_logic_conditions.append({operator: value})  # Multi-argument operators
-        else:
-            json_logic_conditions.append({operator: [{"var": field}, value]})  # Standard binary operators
-
-    return {logic_operator: json_logic_conditions} if len(json_logic_conditions) > 1 else json_logic_conditions[0]
-
-
-def execute_ruleset_against_all_plugins(current_watch_uuid: str, application_datastruct, ephemeral_data={} ):
-    """
-    Build our data and options by calling our plugins then pass it to jsonlogic and see if the conditions pass
-
-    :param ruleset: JSON Logic rule dictionary.
-    :param extracted_data: Dictionary containing the facts.   <-- maybe the app struct+uuid
-    :return: Dictionary of plugin results.
-    """
-    from json_logic import jsonLogic
-
-    EXECUTE_DATA = {}
-    result = True
-    
-    ruleset_settings = application_datastruct['watching'].get(current_watch_uuid)
-
-    if ruleset_settings.get("conditions"):
-        logic_operator = "and" if ruleset_settings.get("conditions_match_logic", "ALL") == "ALL" else "or"
-        complete_rules = filter_complete_rules(ruleset_settings['conditions'])
-        if complete_rules:
-            # Give all plugins a chance to update the data dict again (that we will test the conditions against)
-            for plugin in plugin_manager.get_plugins():
-                new_execute_data = plugin.add_data(current_watch_uuid=current_watch_uuid,
-                                                   application_datastruct=application_datastruct,
-                                                   ephemeral_data=ephemeral_data)
-
-                if new_execute_data and isinstance(new_execute_data, dict):
-                    EXECUTE_DATA.update(new_execute_data)
-
-            # Create the ruleset
-            ruleset = convert_to_jsonlogic(logic_operator=logic_operator, rule_dict=complete_rules)
-            
-            # Pass the custom operations dictionary to jsonLogic
-            if not jsonLogic(logic=ruleset, data=EXECUTE_DATA, operations=CUSTOM_OPERATIONS):
-                result = False
-
-    return result
-
-
-# Load plugins dynamically
-for plugin in plugin_manager.get_plugins():
-    new_ops = plugin.register_operators()
-    if isinstance(new_ops, dict):
-        CUSTOM_OPERATIONS.update(new_ops)
-
-    new_operator_choices = plugin.register_operator_choices()
-    if isinstance(new_operator_choices, list):
-        operator_choices.extend(new_operator_choices)
-
-    new_field_choices = plugin.register_field_choices()
-    if isinstance(new_field_choices, list):
-        field_choices.extend(new_field_choices)
-

changedetectionio/conditions/blueprint.py

@@ -1,78 +0,0 @@
-# Flask Blueprint Definition
-import json
-
-from flask import Blueprint
-
-from changedetectionio.conditions import execute_ruleset_against_all_plugins
-
-
-def construct_blueprint(datastore):
-    from changedetectionio.flask_app import login_optionally_required
-
-    conditions_blueprint = Blueprint('conditions', __name__, template_folder="templates")
-
-    @conditions_blueprint.route("/<string:watch_uuid>/verify-condition-single-rule", methods=['POST'])
-    @login_optionally_required
-    def verify_condition_single_rule(watch_uuid):
-        """Verify a single condition rule against the current snapshot"""
-        from changedetectionio.processors.text_json_diff import prepare_filter_prevew
-        from flask import request, jsonify
-        from copy import deepcopy
-
-        ephemeral_data = {}
-
-        # Get the watch data
-        watch = datastore.data['watching'].get(watch_uuid)
-        if not watch:
-            return jsonify({'status': 'error', 'message': 'Watch not found'}), 404
-
-        # First use prepare_filter_prevew to process the form data
-        # This will return text_after_filter which is after all current form settings are applied
-        # Create ephemeral data with the text from the current snapshot
-
-        try:
-            # Call prepare_filter_prevew to get a processed version of the content with current form settings
-            # We'll ignore the returned response and just use the datastore which is modified by the function
-
-            # this should apply all filters etc so then we can run the CONDITIONS against the final output text
-            result = prepare_filter_prevew(datastore=datastore,
-                                           form_data=request.form,
-                                           watch_uuid=watch_uuid)
-
-            ephemeral_data['text'] = result.get('after_filter', '')
-            # Create a temporary watch data structure with this single rule
-            tmp_watch_data = deepcopy(datastore.data['watching'].get(watch_uuid))
-
-            # Override the conditions in the temporary watch
-            rule_json = request.args.get("rule")
-            rule = json.loads(rule_json) if rule_json else None
-            tmp_watch_data['conditions'] = [rule]
-            tmp_watch_data['conditions_match_logic'] = "ALL"  # Single rule, so use ALL
-
-            # Create a temporary application data structure for the rule check
-            temp_app_data = {
-                'watching': {
-                    watch_uuid: tmp_watch_data
-                }
-            }
-
-            # Execute the rule against the current snapshot with form data
-            result = execute_ruleset_against_all_plugins(
-                current_watch_uuid=watch_uuid,
-                application_datastruct=temp_app_data,
-                ephemeral_data=ephemeral_data
-            )
-
-            return jsonify({
-                'status': 'success',
-                'result': result,
-                'message': 'Condition passes' if result else 'Condition does not pass'
-            })
-
-        except Exception as e:
-            return jsonify({
-                'status': 'error',
-                'message': f'Error verifying condition: {str(e)}'
-            }), 500
-
-    return conditions_blueprint

changedetectionio/conditions/default_plugin.py

@@ -1,78 +0,0 @@
-import re
-
-import pluggy
-from price_parser import Price
-from loguru import logger
-
-hookimpl = pluggy.HookimplMarker("changedetectionio_conditions")
-
-
-@hookimpl
-def register_operators():
-    def starts_with(_, text, prefix):
-        return text.lower().strip().startswith(str(prefix).strip().lower())
-
-    def ends_with(_, text, suffix):
-        return text.lower().strip().endswith(str(suffix).strip().lower())
-
-    def length_min(_, text, strlen):
-        return len(text) >= int(strlen)
-
-    def length_max(_, text, strlen):
-        return len(text) <= int(strlen)
-
-    # ✅ Custom function for case-insensitive regex matching
-    def contains_regex(_, text, pattern):
-        """Returns True if `text` contains `pattern` (case-insensitive regex match)."""
-        return bool(re.search(pattern, str(text), re.IGNORECASE))
-
-    # ✅ Custom function for NOT matching case-insensitive regex
-    def not_contains_regex(_, text, pattern):
-        """Returns True if `text` does NOT contain `pattern` (case-insensitive regex match)."""
-        return not bool(re.search(pattern, str(text), re.IGNORECASE))
-
-    return {
-        "!contains_regex": not_contains_regex,
-        "contains_regex": contains_regex,
-        "ends_with": ends_with,
-        "length_max": length_max,
-        "length_min": length_min,
-        "starts_with": starts_with,
-    }
-
-@hookimpl
-def register_operator_choices():
-    return [
-        ("starts_with", "Text Starts With"),
-        ("ends_with", "Text Ends With"),
-        ("length_min", "Length minimum"),
-        ("length_max", "Length maximum"),
-        ("contains_regex", "Text Matches Regex"),
-        ("!contains_regex", "Text Does NOT Match Regex"),
-    ]
-
-@hookimpl
-def register_field_choices():
-    return [
-        ("extracted_number", "Extracted number after 'Filters & Triggers'"),
-#        ("meta_description", "Meta Description"),
-#        ("meta_keywords", "Meta Keywords"),
-        ("page_filtered_text", "Page text after 'Filters & Triggers'"),
-        #("page_title", "Page <title>"), # actual page title <title>
-    ]
-
-@hookimpl
-def add_data(current_watch_uuid, application_datastruct, ephemeral_data):
-
-    res = {}
-    if 'text' in ephemeral_data:
-        res['page_filtered_text'] = ephemeral_data['text']
-
-        # Better to not wrap this in try/except so that the UI can see any errors
-        price = Price.fromstring(ephemeral_data.get('text'))
-        if price and price.amount != None:
-            # This is slightly misleading, it's extracting a PRICE not a Number..
-            res['extracted_number'] = float(price.amount)
-            logger.debug(f"Extracted number result: '{price}' - returning float({res['extracted_number']})")
-
-    return res

changedetectionio/conditions/exceptions.py

@@ -1,6 +0,0 @@
-class EmptyConditionRuleRowNotUsable(Exception):
-    def __init__(self):
-        super().__init__("One of the 'conditions' rulesets is incomplete, cannot run.")
-
-    def __str__(self):
-        return self.args[0]

changedetectionio/conditions/form.py

@@ -1,44 +0,0 @@
-# Condition Rule Form (for each rule row)
-from wtforms import Form, SelectField, StringField, validators
-from wtforms import validators
-
-class ConditionFormRow(Form):
-
-    # ✅ Ensure Plugins Are Loaded BEFORE Importing Choices
-    from changedetectionio.conditions import plugin_manager
-    from changedetectionio.conditions import operator_choices, field_choices
-    field = SelectField(
-        "Field",
-        choices=field_choices,
-        validators=[validators.Optional()]
-    )
-
-    operator = SelectField(
-        "Operator",
-        choices=operator_choices,
-        validators=[validators.Optional()]
-    )
-
-    value = StringField("Value", validators=[validators.Optional()])
-
-    def validate(self, extra_validators=None):
-        # First, run the default validators
-        if not super().validate(extra_validators):
-            return False
-
-        # Custom validation logic
-        # If any of the operator/field/value is set, then they must be all set
-        if any(value not in ("", False, "None", None) for value in [self.operator.data, self.field.data, self.value.data]):
-            if not self.operator.data or self.operator.data == 'None':
-                self.operator.errors.append("Operator is required.")
-                return False
-
-            if not self.field.data or self.field.data == 'None':
-                self.field.errors.append("Field is required.")
-                return False
-
-            if not self.value.data:
-                self.value.errors.append("Value is required.")
-                return False
-
-        return True  # Only return True if all conditions pass

changedetectionio/conditions/pluggy_interface.py

@@ -1,44 +0,0 @@
-import pluggy
-from . import default_plugin  # Import the default plugin
-
-# ✅ Ensure that the namespace in HookspecMarker matches PluginManager
-PLUGIN_NAMESPACE = "changedetectionio_conditions"
-
-hookspec = pluggy.HookspecMarker(PLUGIN_NAMESPACE)
-hookimpl = pluggy.HookimplMarker(PLUGIN_NAMESPACE)
-
-
-class ConditionsSpec:
-    """Hook specifications for extending JSON Logic conditions."""
-
-    @hookspec
-    def register_operators():
-        """Return a dictionary of new JSON Logic operators."""
-        pass
-
-    @hookspec
-    def register_operator_choices():
-        """Return a list of new operator choices."""
-        pass
-
-    @hookspec
-    def register_field_choices():
-        """Return a list of new field choices."""
-        pass
-
-    @hookspec
-    def add_data(current_watch_uuid, application_datastruct, ephemeral_data):
-        """Add to the datadict"""
-        pass
-
-# ✅ Set up Pluggy Plugin Manager
-plugin_manager = pluggy.PluginManager(PLUGIN_NAMESPACE)
-
-# ✅ Register hookspecs (Ensures they are detected)
-plugin_manager.add_hookspecs(ConditionsSpec)
-
-# ✅ Register built-in plugins manually
-plugin_manager.register(default_plugin, "default_plugin")
-
-# ✅ Discover installed plugins from external packages (if any)
-plugin_manager.load_setuptools_entrypoints(PLUGIN_NAMESPACE)

changedetectionio/content_fetchers/__init__.py

@@ -1,12 +1,10 @@
 import sys
-from changedetectionio.strtobool import strtobool
+from distutils.util import strtobool
 from loguru import logger
 from changedetectionio.content_fetchers.exceptions import BrowserStepsStepException
 import os
 
-# Visual Selector scraper - 'Button' is there because some sites have <button>OUT OF STOCK</button>.
-visualselector_xpath_selectors = 'div,span,form,table,tbody,tr,td,a,p,ul,li,h1,h2,h3,h4,header,footer,section,article,aside,details,main,nav,section,summary,button'
-
+visualselector_xpath_selectors = 'div,span,form,table,tbody,tr,td,a,p,ul,li,h1,h2,h3,h4,header,footer,section,article,aside,details,main,nav,section,summary'
 
 # available_fetchers() will scan this implementation looking for anything starting with html_
 # this information is used in the form selections

changedetectionio/content_fetchers/base.py

@@ -28,7 +28,7 @@ def manage_user_agent(headers, current_ua=''):
     :return:
     """
     # Ask it what the user agent is, if its obviously ChromeHeadless, switch it to the default
-    ua_in_custom_headers = headers.get('User-Agent')
+    ua_in_custom_headers = next((v for k, v in headers.items() if k.lower() == "user-agent"), None)
     if ua_in_custom_headers:
         return ua_in_custom_headers
 
@@ -64,9 +64,10 @@ class Fetcher():
     render_extract_delay = 0
 
     def __init__(self):
-        import importlib.resources
-        self.xpath_element_js = importlib.resources.files("changedetectionio.content_fetchers.res").joinpath('xpath_element_scraper.js').read_text(encoding='utf-8')
-        self.instock_data_js = importlib.resources.files("changedetectionio.content_fetchers.res").joinpath('stock-not-in-stock.js').read_text(encoding='utf-8')
+        from pkg_resources import resource_string
+        # The code that scrapes elements and makes a list of elements/size/position to click on in the VisualSelector
+        self.xpath_element_js = resource_string(__name__, "res/xpath_element_scraper.js").decode('utf-8')
+        self.instock_data_js = resource_string(__name__, "res/stock-not-in-stock.js").decode('utf-8')
 
     @abstractmethod
     def get_error(self):
@@ -81,8 +82,7 @@ class Fetcher():
             request_method,
             ignore_status_codes=False,
             current_include_filters=None,
-            is_binary=False,
-            empty_pages_are_a_change=False):
+            is_binary=False):
         # Should set self.error, self.status_code and self.content
         pass
 
@@ -96,9 +96,6 @@ class Fetcher():
 
     @abstractmethod
     def screenshot_step(self, step_n):
-        if self.browser_steps_screenshot_path and not os.path.isdir(self.browser_steps_screenshot_path):
-            logger.debug(f"> Creating data dir {self.browser_steps_screenshot_path}")
-            os.mkdir(self.browser_steps_screenshot_path)
         return None
 
     @abstractmethod
@@ -115,26 +112,24 @@ class Fetcher():
 
     def browser_steps_get_valid_steps(self):
         if self.browser_steps is not None and len(self.browser_steps):
-            valid_steps = list(filter(
-                lambda s: (s['operation'] and len(s['operation']) and s['operation'] != 'Choose one'),
-                self.browser_steps))
-
-            # Just incase they selected Goto site by accident with older JS
-            if valid_steps and valid_steps[0]['operation'] == 'Goto site':
-                del(valid_steps[0])
+            valid_steps = filter(
+                lambda s: (s['operation'] and len(s['operation']) and s['operation'] != 'Choose one' and s['operation'] != 'Goto site'),
+                self.browser_steps)
 
             return valid_steps
 
         return None
 
-    def iterate_browser_steps(self, start_url=None):
+    def iterate_browser_steps(self):
         from changedetectionio.blueprint.browser_steps.browser_steps import steppable_browser_interface
         from playwright._impl._errors import TimeoutError, Error
-        from changedetectionio.safe_jinja import render as jinja_render
+        from jinja2 import Environment
+        jinja2_env = Environment(extensions=['jinja2_time.TimeExtension'])
+
         step_n = 0
 
         if self.browser_steps is not None and len(self.browser_steps):
-            interface = steppable_browser_interface(start_url=start_url)
+            interface = steppable_browser_interface()
             interface.page = self.page
             valid_steps = self.browser_steps_get_valid_steps()
 
@@ -148,9 +143,9 @@ class Fetcher():
                     selector = step['selector']
                     # Support for jinja2 template in step values, with date module added
                     if '{%' in step['optional_value'] or '{{' in step['optional_value']:
-                        optional_value = jinja_render(template_str=step['optional_value'])
+                        optional_value = str(jinja2_env.from_string(step['optional_value']).render())
                     if '{%' in step['selector'] or '{{' in step['selector']:
-                        selector = jinja_render(template_str=step['selector'])
+                        selector = str(jinja2_env.from_string(step['selector']).render())
 
                     getattr(interface, "call_action")(action_name=step['operation'],
                                                       selector=selector,
@@ -172,8 +167,5 @@ class Fetcher():
                 if os.path.isfile(f):
                     os.unlink(f)
 
-    def save_step_html(self, step_n):
-        if self.browser_steps_screenshot_path and not os.path.isdir(self.browser_steps_screenshot_path):
-            logger.debug(f"> Creating data dir {self.browser_steps_screenshot_path}")
-            os.mkdir(self.browser_steps_screenshot_path)
+    def save_step_html(self, param):
         pass

changedetectionio/content_fetchers/exceptions/__init__.py

@@ -1,5 +1,6 @@
 from loguru import logger
 
+
 class Non200ErrorCodeReceived(Exception):
     def __init__(self, status_code, url, screenshot=None, xpath_data=None, page_html=None):
         # Set this so we can use it in other parts of the app
@@ -80,18 +81,17 @@ class ScreenshotUnavailable(Exception):
         self.status_code = status_code
         self.url = url
         if page_html:
-            from changedetectionio.html_tools import html_to_text
+            from html_tools import html_to_text
             self.page_text = html_to_text(page_html)
         return
 
 
 class ReplyWithContentButNoText(Exception):
-    def __init__(self, status_code, url, screenshot=None, has_filters=False, html_content='', xpath_data=None):
+    def __init__(self, status_code, url, screenshot=None, has_filters=False, html_content=''):
         # Set this so we can use it in other parts of the app
         self.status_code = status_code
         self.url = url
         self.screenshot = screenshot
         self.has_filters = has_filters
         self.html_content = html_content
-        self.xpath_data = xpath_data
         return

changedetectionio/content_fetchers/helpers.py

@@ -1,104 +0,0 @@
-
-# Pages with a vertical height longer than this will use the 'stitch together' method.
-
-# - Many GPUs have a max texture size of 16384x16384px (or lower on older devices).
-# - If a page is taller than ~8000–10000px, it risks exceeding GPU memory limits.
-# - This is especially important on headless Chromium, where Playwright may fail to allocate a massive full-page buffer.
-
-
-# The size at which we will switch to stitching method
-SCREENSHOT_SIZE_STITCH_THRESHOLD=8000
-
-from loguru import logger
-
-def capture_stitched_together_full_page(page):
-    import io
-    import os
-    import time
-    from PIL import Image, ImageDraw, ImageFont
-
-    MAX_TOTAL_HEIGHT = SCREENSHOT_SIZE_STITCH_THRESHOLD*4  # Maximum total height for the final image (When in stitch mode)
-    MAX_CHUNK_HEIGHT = 4000  # Height per screenshot chunk
-    WARNING_TEXT_HEIGHT = 20  # Height of the warning text overlay
-
-    # Save the original viewport size
-    original_viewport = page.viewport_size
-    now = time.time()
-
-    try:
-        viewport = page.viewport_size
-        page_height = page.evaluate("document.documentElement.scrollHeight")
-
-        # Limit the total capture height
-        capture_height = min(page_height, MAX_TOTAL_HEIGHT)
-
-        images = []
-        total_captured_height = 0
-
-        for offset in range(0, capture_height, MAX_CHUNK_HEIGHT):
-            # Ensure we do not exceed the total height limit
-            chunk_height = min(MAX_CHUNK_HEIGHT, MAX_TOTAL_HEIGHT - total_captured_height)
-
-            # Adjust viewport size for this chunk
-            page.set_viewport_size({"width": viewport["width"], "height": chunk_height})
-
-            # Scroll to the correct position
-            page.evaluate(f"window.scrollTo(0, {offset})")
-
-            # Capture screenshot chunk
-            screenshot_bytes = page.screenshot(type='jpeg', quality=int(os.getenv("SCREENSHOT_QUALITY", 30)))
-            images.append(Image.open(io.BytesIO(screenshot_bytes)))
-
-            total_captured_height += chunk_height
-
-            # Stop if we reached the maximum total height
-            if total_captured_height >= MAX_TOTAL_HEIGHT:
-                break
-
-        # Create the final stitched image
-        stitched_image = Image.new('RGB', (viewport["width"], total_captured_height))
-        y_offset = 0
-
-        # Stitch the screenshot chunks together
-        for img in images:
-            stitched_image.paste(img, (0, y_offset))
-            y_offset += img.height
-
-        logger.debug(f"Screenshot stitched together in {time.time()-now:.2f}s")
-
-        # Overlay warning text if the screenshot was trimmed
-        if page_height > MAX_TOTAL_HEIGHT:
-            draw = ImageDraw.Draw(stitched_image)
-            warning_text = f"WARNING: Screenshot was {page_height}px but trimmed to {MAX_TOTAL_HEIGHT}px because it was too long"
-
-            # Load font (default system font if Arial is unavailable)
-            try:
-                font = ImageFont.truetype("arial.ttf", WARNING_TEXT_HEIGHT)  # Arial (Windows/Mac)
-            except IOError:
-                font = ImageFont.load_default()  # Default font if Arial not found
-
-            # Get text bounding box (correct method for newer Pillow versions)
-            text_bbox = draw.textbbox((0, 0), warning_text, font=font)
-            text_width = text_bbox[2] - text_bbox[0]  # Calculate text width
-            text_height = text_bbox[3] - text_bbox[1]  # Calculate text height
-
-            # Define background rectangle (top of the image)
-            draw.rectangle([(0, 0), (viewport["width"], WARNING_TEXT_HEIGHT)], fill="white")
-
-            # Center text horizontally within the warning area
-            text_x = (viewport["width"] - text_width) // 2
-            text_y = (WARNING_TEXT_HEIGHT - text_height) // 2
-
-            # Draw the warning text in red
-            draw.text((text_x, text_y), warning_text, fill="red", font=font)
-
-        # Save or return the final image
-        output = io.BytesIO()
-        stitched_image.save(output, format="JPEG", quality=int(os.getenv("SCREENSHOT_QUALITY", 30)))
-        screenshot = output.getvalue()
-
-    finally:
-        # Restore the original viewport size
-        page.set_viewport_size(original_viewport)
-
-    return screenshot

changedetectionio/content_fetchers/playwright.py

@@ -4,7 +4,6 @@ from urllib.parse import urlparse
 
 from loguru import logger
 
-from changedetectionio.content_fetchers.helpers import capture_stitched_together_full_page, SCREENSHOT_SIZE_STITCH_THRESHOLD
 from changedetectionio.content_fetchers.base import Fetcher, manage_user_agent
 from changedetectionio.content_fetchers.exceptions import PageUnloadable, Non200ErrorCodeReceived, EmptyReply, ScreenshotUnavailable
 
@@ -59,7 +58,6 @@ class fetcher(Fetcher):
                 self.proxy['password'] = parsed.password
 
     def screenshot_step(self, step_n=''):
-        super().screenshot_step(step_n=step_n)
         screenshot = self.page.screenshot(type='jpeg', full_page=True, quality=int(os.getenv("SCREENSHOT_QUALITY", 72)))
 
         if self.browser_steps_screenshot_path is not None:
@@ -69,7 +67,6 @@ class fetcher(Fetcher):
                 f.write(screenshot)
 
     def save_step_html(self, step_n):
-        super().save_step_html(step_n=step_n)
         content = self.page.content()
         destination = os.path.join(self.browser_steps_screenshot_path, 'step_{}.html'.format(step_n))
         logger.debug(f"Saving step HTML to {destination}")
@@ -84,13 +81,11 @@ class fetcher(Fetcher):
             request_method,
             ignore_status_codes=False,
             current_include_filters=None,
-            is_binary=False,
-            empty_pages_are_a_change=False):
+            is_binary=False):
 
         from playwright.sync_api import sync_playwright
         import playwright._impl._errors
         from changedetectionio.content_fetchers import visualselector_xpath_selectors
-        import time
         self.delete_browser_steps_screenshots()
         response = None
 
@@ -124,7 +119,7 @@ class fetcher(Fetcher):
 
             # Re-use as much code from browser steps as possible so its the same
             from changedetectionio.blueprint.browser_steps.browser_steps import steppable_browser_interface
-            browsersteps_interface = steppable_browser_interface(start_url=url)
+            browsersteps_interface = steppable_browser_interface()
             browsersteps_interface.page = self.page
 
             response = browsersteps_interface.action_goto_url(value=url)
@@ -133,7 +128,7 @@ class fetcher(Fetcher):
             if response is None:
                 context.close()
                 browser.close()
-                logger.debug("Content Fetcher > Response object from the browser communication was none")
+                logger.debug("Content Fetcher > Response object was none")
                 raise EmptyReply(url=url, status_code=None)
 
             try:
@@ -169,19 +164,18 @@ class fetcher(Fetcher):
 
                 raise Non200ErrorCodeReceived(url=url, status_code=self.status_code, screenshot=screenshot)
 
-            if not empty_pages_are_a_change and len(self.page.content().strip()) == 0:
-                logger.debug("Content Fetcher > Content was empty, empty_pages_are_a_change = False")
+            if len(self.page.content().strip()) == 0:
                 context.close()
                 browser.close()
+                logger.debug("Content Fetcher > Content was empty")
                 raise EmptyReply(url=url, status_code=response.status)
 
             # Run Browser Steps here
             if self.browser_steps_get_valid_steps():
-                self.iterate_browser_steps(start_url=url)
+                self.iterate_browser_steps()
 
             self.page.wait_for_timeout(extra_wait * 1000)
 
-            now = time.time()
             # So we can find an element on the page where its selector was entered manually (maybe not xPath etc)
             if current_include_filters is not None:
                 self.page.evaluate("var include_filters={}".format(json.dumps(current_include_filters)))
@@ -193,8 +187,6 @@ class fetcher(Fetcher):
             self.instock_data = self.page.evaluate("async () => {" + self.instock_data_js + "}")
 
             self.content = self.page.content()
-            logger.debug(f"Time to scrape xpath element data in browser {time.time() - now:.2f}s")
-
             # Bug 3 in Playwright screenshot handling
             # Some bug where it gives the wrong screenshot size, but making a request with the clip set first seems to solve it
             # JPEG is better here because the screenshots can be very very large
@@ -204,15 +196,10 @@ class fetcher(Fetcher):
             # acceptable screenshot quality here
             try:
                 # The actual screenshot - this always base64 and needs decoding! horrible! huge CPU usage
-                full_height = self.page.evaluate("document.documentElement.scrollHeight")
-
-                if full_height >= SCREENSHOT_SIZE_STITCH_THRESHOLD:
-                    logger.warning(
-                        f"Page full Height: {full_height}px longer than {SCREENSHOT_SIZE_STITCH_THRESHOLD}px, using 'stitched screenshot method'.")
-                    self.screenshot = capture_stitched_together_full_page(self.page)
-                else:
-                    self.screenshot = self.page.screenshot(type='jpeg', full_page=True, quality=int(os.getenv("SCREENSHOT_QUALITY", 30)))
-
+                self.screenshot = self.page.screenshot(type='jpeg',
+                                                       full_page=True,
+                                                       quality=int(os.getenv("SCREENSHOT_QUALITY", 72)),
+                                                       )
             except Exception as e:
                 # It's likely the screenshot was too long/big and something crashed
                 raise ScreenshotUnavailable(url=url, status_code=self.status_code)

changedetectionio/content_fetchers/puppeteer.py

@@ -9,6 +9,7 @@ from loguru import logger
 from changedetectionio.content_fetchers.base import Fetcher, manage_user_agent
 from changedetectionio.content_fetchers.exceptions import PageUnloadable, Non200ErrorCodeReceived, EmptyReply, BrowserFetchTimedOut, BrowserConnectError
 
+
 class fetcher(Fetcher):
     fetcher_description = "Puppeteer/direct {}/Javascript".format(
         os.getenv("PLAYWRIGHT_BROWSER_TYPE", 'chromium').capitalize()
@@ -75,8 +76,7 @@ class fetcher(Fetcher):
                          request_method,
                          ignore_status_codes,
                          current_include_filters,
-                         is_binary,
-                         empty_pages_are_a_change
+                         is_binary
                          ):
 
         from changedetectionio.content_fetchers import visualselector_xpath_selectors
@@ -93,38 +93,15 @@ class fetcher(Fetcher):
                                                        ignoreHTTPSErrors=True
                                                        )
         except websockets.exceptions.InvalidStatusCode as e:
-            raise BrowserConnectError(msg=f"Error while trying to connect the browser, Code {e.status_code} (check your access, whitelist IP, password etc)")
+            raise BrowserConnectError(msg=f"Error while trying to connect the browser, Code {e.status_code} (check your access)")
         except websockets.exceptions.InvalidURI:
             raise BrowserConnectError(msg=f"Error connecting to the browser, check your browser connection address (should be ws:// or wss://")
         except Exception as e:
             raise BrowserConnectError(msg=f"Error connecting to the browser {str(e)}")
-
-        # Better is to launch chrome with the URL as arg
-        # non-headless - newPage() will launch an extra tab/window, .browser should already contain 1 page/tab
-        # headless - ask a new page
-        self.page = (pages := await browser.pages) and len(pages) or await browser.newPage()
-
-        try:
-            from pyppeteerstealth import inject_evasions_into_page
-        except ImportError:
-            logger.debug("pyppeteerstealth module not available, skipping")
-            pass
         else:
-            # I tried hooking events via self.page.on(Events.Page.DOMContentLoaded, inject_evasions_requiring_obj_to_page)
-            # But I could never get it to fire reliably, so we just inject it straight after
-            await inject_evasions_into_page(self.page)
+            self.page = await browser.newPage()
 
-        # This user agent is similar to what was used when tweaking the evasions in inject_evasions_into_page(..)
-        user_agent = None
-        if request_headers and request_headers.get('User-Agent'):
-            # Request_headers should now be CaaseInsensitiveDict
-            # Remove it so it's not sent again with headers after
-            user_agent = request_headers.pop('User-Agent').strip()
-            await self.page.setUserAgent(user_agent)
-
-        if not user_agent:
-            # Attempt to strip 'HeadlessChrome' etc
-            await self.page.setUserAgent(manage_user_agent(headers=request_headers, current_ua=await self.page.evaluate('navigator.userAgent')))
+        await self.page.setUserAgent(manage_user_agent(headers=request_headers, current_ua=await self.page.evaluate('navigator.userAgent')))
 
         await self.page.setBypassCSP(True)
         if request_headers:
@@ -154,7 +131,7 @@ class fetcher(Fetcher):
         if response is None:
             await self.page.close()
             await browser.close()
-            logger.warning("Content Fetcher > Response object was none (as in, the response from the browser was empty, not just the content)")
+            logger.warning("Content Fetcher > Response object was none")
             raise EmptyReply(url=url, status_code=None)
 
         self.headers = response.headers
@@ -187,11 +164,10 @@ class fetcher(Fetcher):
 
             raise Non200ErrorCodeReceived(url=url, status_code=self.status_code, screenshot=screenshot)
         content = await self.page.content
-
-        if not empty_pages_are_a_change and len(content.strip()) == 0:
-            logger.error("Content Fetcher > Content was empty (empty_pages_are_a_change is False), closing browsers")
+        if len(content.strip()) == 0:
             await self.page.close()
             await browser.close()
+            logger.error("Content Fetcher > Content was empty")
             raise EmptyReply(url=url, status_code=response.status)
 
         # Run Browser Steps here
@@ -249,7 +225,7 @@ class fetcher(Fetcher):
         await self.fetch_page(**kwargs)
 
     def run(self, url, timeout, request_headers, request_body, request_method, ignore_status_codes=False,
-            current_include_filters=None, is_binary=False, empty_pages_are_a_change=False):
+            current_include_filters=None, is_binary=False):
 
         #@todo make update_worker async which could run any of these content_fetchers within memory and time constraints
         max_time = os.getenv('PUPPETEER_MAX_PROCESSING_TIMEOUT_SECONDS', 180)
@@ -264,8 +240,7 @@ class fetcher(Fetcher):
                 request_method=request_method,
                 ignore_status_codes=ignore_status_codes,
                 current_include_filters=current_include_filters,
-                is_binary=is_binary,
-                empty_pages_are_a_change=empty_pages_are_a_change
+                is_binary=is_binary
             ), timeout=max_time))
         except asyncio.TimeoutError:
             raise(BrowserFetchTimedOut(msg=f"Browser connected but was unable to process the page in {max_time} seconds."))

changedetectionio/content_fetchers/requests.py

@@ -1,7 +1,9 @@
-from loguru import logger
 import hashlib
 import os
-from changedetectionio import strtobool
+
+import chardet
+import requests
+
 from changedetectionio.content_fetchers.exceptions import BrowserStepsInUnsupportedFetcher, EmptyReply, Non200ErrorCodeReceived
 from changedetectionio.content_fetchers.base import Fetcher
 
@@ -23,15 +25,16 @@ class fetcher(Fetcher):
             request_method,
             ignore_status_codes=False,
             current_include_filters=None,
-            is_binary=False,
-            empty_pages_are_a_change=False):
-
-        import chardet
-        import requests
+            is_binary=False):
 
         if self.browser_steps_get_valid_steps():
             raise BrowserStepsInUnsupportedFetcher(url=url)
 
+        # Make requests use a more modern looking user-agent
+        if not {k.lower(): v for k, v in request_headers.items()}.get('user-agent', None):
+            request_headers['User-Agent'] = os.getenv("DEFAULT_SETTINGS_HEADERS_USERAGENT",
+                                                      'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/87.0.4280.66 Safari/537.36')
+
         proxies = {}
 
         # Allows override the proxy on a per-request basis
@@ -47,19 +50,13 @@ class fetcher(Fetcher):
             if self.system_https_proxy:
                 proxies['https'] = self.system_https_proxy
 
-        session = requests.Session()
-
-        if strtobool(os.getenv('ALLOW_FILE_URI', 'false')) and url.startswith('file://'):
-            from requests_file import FileAdapter
-            session.mount('file://', FileAdapter())
-
-        r = session.request(method=request_method,
-                            data=request_body.encode('utf-8') if type(request_body) is str else request_body,
-                            url=url,
-                            headers=request_headers,
-                            timeout=timeout,
-                            proxies=proxies,
-                            verify=False)
+        r = requests.request(method=request_method,
+                             data=request_body,
+                             url=url,
+                             headers=request_headers,
+                             timeout=timeout,
+                             proxies=proxies,
+                             verify=False)
 
         # If the response did not tell us what encoding format to expect, Then use chardet to override what `requests` thinks.
         # For example - some sites don't tell us it's utf-8, but return utf-8 content
@@ -75,11 +72,7 @@ class fetcher(Fetcher):
         self.headers = r.headers
 
         if not r.content or not len(r.content):
-            logger.debug(f"Requests returned empty content for '{url}'")
-            if not empty_pages_are_a_change:
-                raise EmptyReply(url=url, status_code=r.status_code)
-            else:
-                logger.debug(f"URL {url} gave zero byte content reply with Status Code {r.status_code}, but empty_pages_are_a_change = True")
+            raise EmptyReply(url=url, status_code=r.status_code)
 
         # @todo test this
         # @todo maybe you really want to test zero-byte return pages?

changedetectionio/content_fetchers/res/__init__.py

@@ -1 +0,0 @@
-# resources for browser injection/scraping

changedetectionio/content_fetchers/res/stock-not-in-stock.js

@@ -29,71 +29,45 @@ function isItemInStock() {
         'currently unavailable',
         'dieser artikel ist bald wieder verfügbar',
         'dostępne wkrótce',
-        'en rupture',
         'en rupture de stock',
-        'épuisé',
-        'esgotado',
-        'indisponible',
-        'indisponível',
-        'isn\'t in stock right now',
-        'isnt in stock right now',
-        'isn’t in stock right now',
+        'ist derzeit nicht auf lager',
         'item is no longer available',
         'let me know when it\'s available',
-        'mail me when available',
         'message if back in stock',
-        'mevcut değil',
         'nachricht bei',
         'nicht auf lager',
-        'nicht lagernd',
         'nicht lieferbar',
-        'nicht verfügbar',
-        'nicht vorrätig',
         'nicht zur verfügung',
-        'nie znaleziono produktów',
         'niet beschikbaar',
         'niet leverbaar',
         'niet op voorraad',
-        'no disponible',
-        'non disponibile',
-        'non disponible',
+        'no disponible temporalmente',
         'no longer in stock',
         'no tickets available',
         'not available',
         'not currently available',
         'not in stock',
         'notify me when available',
-        'notify me',
         'notify when available',
-        'não disponível',
         'não estamos a aceitar encomendas',
         'out of stock',
         'out-of-stock',
-        'plus disponible',
         'prodotto esaurito',
         'produkt niedostępny',
-        'rupture',
         'sold out',
         'sold-out',
-        'stokta yok',
         'temporarily out of stock',
         'temporarily unavailable',
         'there were no search results for',
         'this item is currently unavailable',
         'tickets unavailable',
         'tijdelijk uitverkocht',
-        'tükendi',
-        'unavailable nearby',
         'unavailable tickets',
-        'vergriffen',
-        'vorbestellen',
         'vorbestellung ist bald möglich',
-        'we don\'t currently have any',
         'we couldn\'t find any products that match',
         'we do not currently have an estimate of when this product will be back in stock.',
         'we don\'t know when or if this item will be back in stock.',
         'we were not able to find a match',
-        'when this arrives in stock',
         'zur zeit nicht an lager',
         '品切れ',
         '已售',
@@ -167,14 +141,10 @@ function isItemInStock() {
         }
 
         elementText = "";
-        try {
-            if (element.tagName.toLowerCase() === "input") {
-                elementText = element.value.toLowerCase().trim();
-            } else {
-                elementText = getElementBaseText(element);
-            }
-        } catch (e) {
-            console.warn('stock-not-in-stock.js scraper - handling element for gettext failed', e);
+        if (element.tagName.toLowerCase() === "input") {
+            elementText = element.value.toLowerCase().trim();
+        } else {
+            elementText = getElementBaseText(element);
         }
 
         if (elementText.length) {
@@ -191,8 +161,7 @@ function isItemInStock() {
         const element = elementsToScan[i];
         // outside the 'fold' or some weird text in the heading area
         // .getBoundingClientRect() was causing a crash in chrome 119, can only be run on contentVisibility != hidden
-        // Note: theres also an automated test that places the 'out of stock' text fairly low down
-        if (element.getBoundingClientRect().top + window.scrollY >= vh + 250 || element.getBoundingClientRect().top + window.scrollY <= 100) {
+        if (element.getBoundingClientRect().top + window.scrollY >= vh + 150 || element.getBoundingClientRect().top + window.scrollY <= 100) {
             continue
         }
         elementText = "";
@@ -206,7 +175,7 @@ function isItemInStock() {
             // and these mean its out of stock
             for (const outOfStockText of outOfStockTexts) {
                 if (elementText.includes(outOfStockText)) {
-                    console.log(`Selected 'Out of Stock' - found text "${outOfStockText}" - "${elementText}" - offset top ${element.getBoundingClientRect().top}, page height is ${vh}`)
+                    console.log(`Selected 'Out of Stock' - found text "${outOfStockText}" - "${elementText}"`)
                     return outOfStockText; // item is out of stock
                 }
             }

changedetectionio/content_fetchers/res/xpath_element_scraper.js

@@ -41,7 +41,7 @@ const findUpTag = (el) => {
 
     //  Strategy 1: If it's an input, with name, and there's only one, prefer that
     if (el.name !== undefined && el.name.length) {
-        var proposed = el.tagName + "[name=\"" + CSS.escape(el.name) + "\"]";
+        var proposed = el.tagName + "[name=" + el.name + "]";
         var proposed_element = window.document.querySelectorAll(proposed);
         if (proposed_element.length) {
             if (proposed_element.length === 1) {
@@ -102,15 +102,13 @@ function collectVisibleElements(parent, visibleElements) {
     const children = parent.children;
     for (let i = 0; i < children.length; i++) {
         const child = children[i];
-        const computedStyle = window.getComputedStyle(child);
-
         if (
             child.nodeType === Node.ELEMENT_NODE &&
-            computedStyle.display !== 'none' &&
-            computedStyle.visibility !== 'hidden' &&
+            window.getComputedStyle(child).display !== 'none' &&
+            window.getComputedStyle(child).visibility !== 'hidden' &&
             child.offsetWidth >= 0 &&
             child.offsetHeight >= 0 &&
-            computedStyle.contentVisibility !== 'hidden'
+            window.getComputedStyle(child).contentVisibility !== 'hidden'
         ) {
             // If the child is an element and is visible, recursively collect visible elements
             collectVisibleElements(child, visibleElements);
@@ -166,16 +164,6 @@ visibleElementsArray.forEach(function (element) {
         }
     }
 
-    let label = "not-interesting" // A placeholder, the actual labels for training are done by hand for now
-
-    let text = element.textContent.trim().slice(0, 30).trim();
-    while (/\n{2,}|\t{2,}/.test(text)) {
-        text = text.replace(/\n{2,}/g, '\n').replace(/\t{2,}/g, '\t')
-    }
-
-    // Try to identify any possible currency amounts "Sale: 4000" or "Sale now 3000 Kc", can help with the training.
-    const hasDigitCurrency = (/\d/.test(text.slice(0, 6)) || /\d/.test(text.slice(-6)) ) &&  /([€£$¥₩₹]|USD|AUD|EUR|Kč|kr|SEK|,–)/.test(text) ;
-    const computedStyle = window.getComputedStyle(element);
 
     size_pos.push({
         xpath: xpath_result,
@@ -183,16 +171,9 @@ visibleElementsArray.forEach(function (element) {
         height: Math.round(bbox['height']),
         left: Math.floor(bbox['left']),
         top: Math.floor(bbox['top']) + scroll_y,
-        // tagName used by Browser Steps
         tagName: (element.tagName) ? element.tagName.toLowerCase() : '',
-        // tagtype used by Browser Steps
         tagtype: (element.tagName.toLowerCase() === 'input' && element.type) ? element.type.toLowerCase() : '',
-        isClickable: computedStyle.cursor === "pointer",
-        // Used by the keras trainer
-        fontSize: computedStyle.getPropertyValue('font-size'),
-        fontWeight: computedStyle.getPropertyValue('font-weight'),
-        hasDigitCurrency: hasDigitCurrency,
-        label: label,
+        isClickable: window.getComputedStyle(element).cursor == "pointer"
     });
 
 });
@@ -201,7 +182,6 @@ visibleElementsArray.forEach(function (element) {
 // Inject the current one set in the include_filters, which may be a CSS rule
 // used for displaying the current one in VisualSelector, where its not one we generated.
 if (include_filters.length) {
-    let results;
     // Foreach filter, go and find it on the page and add it to the results so we can visualise it again
     for (const f of include_filters) {
         bbox = false;
@@ -217,15 +197,10 @@ if (include_filters.length) {
             if (f.startsWith('/') || f.startsWith('xpath')) {
                 var qry_f = f.replace(/xpath(:|\d:)/, '')
                 console.log("[xpath] Scanning for included filter " + qry_f)
-                let xpathResult = document.evaluate(qry_f, document, null, XPathResult.ORDERED_NODE_SNAPSHOT_TYPE, null);
-                results = [];
-                for (let i = 0; i < xpathResult.snapshotLength; i++) {
-                    results.push(xpathResult.snapshotItem(i));
-                }
+                q = document.evaluate(qry_f, document, null, XPathResult.FIRST_ORDERED_NODE_TYPE, null).singleNodeValue;
             } else {
                 console.log("[css] Scanning for included filter " + f)
-                console.log("[css] Scanning for included filter " + f);
-                results = document.querySelectorAll(f);
+                q = document.querySelector(f);
             }
         } catch (e) {
             // Maybe catch DOMException and alert?
@@ -233,45 +208,44 @@ if (include_filters.length) {
             console.log(e);
         }
 
-        if (results != null && results.length) {
+        if (q) {
+            // Try to resolve //something/text() back to its /something so we can atleast get the bounding box
+            try {
+                if (typeof q.nodeName == 'string' && q.nodeName === '#text') {
+                    q = q.parentElement
+                }
+            } catch (e) {
+                console.log(e)
+                console.log("xpath_element_scraper: #text resolver")
+            }
 
-            // Iterate over the results
-            results.forEach(node => {
-                // Try to resolve //something/text() back to its /something so we can atleast get the bounding box
+            // #1231 - IN the case XPath attribute filter is applied, we will have to traverse up and find the element.
+            if (typeof q.getBoundingClientRect == 'function') {
+                bbox = q.getBoundingClientRect();
+                console.log("xpath_element_scraper: Got filter element, scroll from top was " + scroll_y)
+            } else {
                 try {
-                    if (typeof node.nodeName == 'string' && node.nodeName === '#text') {
-                        node = node.parentElement
-                    }
+                    // Try and see we can find its ownerElement
+                    bbox = q.ownerElement.getBoundingClientRect();
+                    console.log("xpath_element_scraper: Got filter by ownerElement element, scroll from top was " + scroll_y)
                 } catch (e) {
                     console.log(e)
-                    console.log("xpath_element_scraper: #text resolver")
+                    console.log("xpath_element_scraper: error looking up q.ownerElement")
                 }
+            }
+        }
 
-                // #1231 - IN the case XPath attribute filter is applied, we will have to traverse up and find the element.
-                if (typeof node.getBoundingClientRect == 'function') {
-                    bbox = node.getBoundingClientRect();
-                    console.log("xpath_element_scraper: Got filter element, scroll from top was " + scroll_y)
-                } else {
-                    try {
-                        // Try and see we can find its ownerElement
-                        bbox = node.ownerElement.getBoundingClientRect();
-                        console.log("xpath_element_scraper: Got filter by ownerElement element, scroll from top was " + scroll_y)
-                    } catch (e) {
-                        console.log(e)
-                        console.log("xpath_element_scraper: error looking up q.ownerElement")
-                    }
-                }
+        if (!q) {
+            console.log("xpath_element_scraper: filter element " + f + " was not found");
+        }
 
-                if (bbox && bbox['width'] > 0 && bbox['height'] > 0) {
-                    size_pos.push({
-                        xpath: f,
-                        width: parseInt(bbox['width']),
-                        height: parseInt(bbox['height']),
-                        left: parseInt(bbox['left']),
-                        top: parseInt(bbox['top']) + scroll_y,
-                        highlight_as_custom_filter: true
-                    });
-                }
+        if (bbox && bbox['width'] > 0 && bbox['height'] > 0) {
+            size_pos.push({
+                xpath: f,
+                width: parseInt(bbox['width']),
+                height: parseInt(bbox['height']),
+                left: parseInt(bbox['left']),
+                top: parseInt(bbox['top']) + scroll_y
             });
         }
     }

changedetectionio/content_fetchers/webdriver_selenium.py

@@ -56,8 +56,7 @@ class fetcher(Fetcher):
             request_method,
             ignore_status_codes=False,
             current_include_filters=None,
-            is_binary=False,
-            empty_pages_are_a_change=False):
+            is_binary=False):
 
         from selenium import webdriver
         from selenium.webdriver.chrome.options import Options as ChromeOptions

changedetectionio/diff.py

@@ -1,113 +1,62 @@
+# used for the notifications, the front-end is using a JS library
+
 import difflib
-from typing import List, Iterator, Union
-
-REMOVED_STYLE = "background-color: #fadad7; color: #b30000;"
-ADDED_STYLE = "background-color: #eaf2c2; color: #406619;"
-
-def same_slicer(lst: List[str], start: int, end: int) -> List[str]:
-    """Return a slice of the list, or a single element if start == end."""
-    return lst[start:end] if start != end else [lst[start]]
-
-def customSequenceMatcher(
-    before: List[str],
-    after: List[str],
-    include_equal: bool = False,
-    include_removed: bool = True,
-    include_added: bool = True,
-    include_replaced: bool = True,
-    include_change_type_prefix: bool = True,
-    html_colour: bool = False
-) -> Iterator[List[str]]:
-    """
-    Compare two sequences and yield differences based on specified parameters.
-
-    Args:
-        before (List[str]): Original sequence
-        after (List[str]): Modified sequence
-        include_equal (bool): Include unchanged parts
-        include_removed (bool): Include removed parts
-        include_added (bool): Include added parts
-        include_replaced (bool): Include replaced parts
-        include_change_type_prefix (bool): Add prefixes to indicate change types
-        html_colour (bool): Use HTML background colors for differences
-
-    Yields:
-        List[str]: Differences between sequences
-    """
-    cruncher = difflib.SequenceMatcher(isjunk=lambda x: x in " \t", a=before, b=after)
 
 
+def same_slicer(l, a, b):
+    if a == b:
+        return [l[a]]
+    else:
+        return l[a:b]
 
+# like .compare but a little different output
+def customSequenceMatcher(before, after, include_equal=False, include_removed=True, include_added=True, include_replaced=True, include_change_type_prefix=True):
+    cruncher = difflib.SequenceMatcher(isjunk=lambda x: x in " \\t", a=before, b=after)
+
+    # @todo Line-by-line mode instead of buncghed, including `after` that is not in `before` (maybe unset?)
     for tag, alo, ahi, blo, bhi in cruncher.get_opcodes():
         if include_equal and tag == 'equal':
-            yield before[alo:ahi]
+            g = before[alo:ahi]
+            yield g
         elif include_removed and tag == 'delete':
-            if html_colour:
-                yield [f'<span style="{REMOVED_STYLE}">{line}</span>' for line in same_slicer(before, alo, ahi)]
-            else:
-                yield [f"(removed) {line}" for line in same_slicer(before, alo, ahi)] if include_change_type_prefix else same_slicer(before, alo, ahi)
+            row_prefix = "(removed) " if include_change_type_prefix else ''
+            g = [ row_prefix + i for i in same_slicer(before, alo, ahi)]
+            yield g
         elif include_replaced and tag == 'replace':
-            if html_colour:
-                yield [f'<span style="{REMOVED_STYLE}">{line}</span>' for line in same_slicer(before, alo, ahi)] + \
-                      [f'<span style="{ADDED_STYLE}">{line}</span>' for line in same_slicer(after, blo, bhi)]
-            else:
-                yield [f"(changed) {line}" for line in same_slicer(before, alo, ahi)] + \
-                      [f"(into) {line}" for line in same_slicer(after, blo, bhi)] if include_change_type_prefix else same_slicer(before, alo, ahi) + same_slicer(after, blo, bhi)
+            row_prefix = "(changed) " if include_change_type_prefix else ''
+            g = [row_prefix + i for i in same_slicer(before, alo, ahi)]
+            row_prefix = "(into) " if include_change_type_prefix else ''
+            g += [row_prefix + i for i in same_slicer(after, blo, bhi)]
+            yield g
         elif include_added and tag == 'insert':
-            if html_colour:
-                yield [f'<span style="{ADDED_STYLE}">{line}</span>' for line in same_slicer(after, blo, bhi)]
-            else:
-                yield [f"(added) {line}" for line in same_slicer(after, blo, bhi)] if include_change_type_prefix else same_slicer(after, blo, bhi)
+            row_prefix = "(added) " if include_change_type_prefix else ''
+            g = [row_prefix + i for i in same_slicer(after, blo, bhi)]
+            yield g
 
-def render_diff(
-    previous_version_file_contents: str,
-    newest_version_file_contents: str,
-    include_equal: bool = False,
-    include_removed: bool = True,
-    include_added: bool = True,
-    include_replaced: bool = True,
-    line_feed_sep: str = "\n",
-    include_change_type_prefix: bool = True,
-    patch_format: bool = False,
-    html_colour: bool = False
-) -> str:
-    """
-    Render the difference between two file contents.
+# only_differences - only return info about the differences, no context
+# line_feed_sep could be "<br>" or "<li>" or "\n" etc
+def render_diff(previous_version_file_contents, newest_version_file_contents, include_equal=False, include_removed=True, include_added=True, include_replaced=True, line_feed_sep="\n", include_change_type_prefix=True, patch_format=False):
 
-    Args:
-        previous_version_file_contents (str): Original file contents
-        newest_version_file_contents (str): Modified file contents
-        include_equal (bool): Include unchanged parts
-        include_removed (bool): Include removed parts
-        include_added (bool): Include added parts
-        include_replaced (bool): Include replaced parts
-        line_feed_sep (str): Separator for lines in output
-        include_change_type_prefix (bool): Add prefixes to indicate change types
-        patch_format (bool): Use patch format for output
-        html_colour (bool): Use HTML background colors for differences
+    newest_version_file_contents = [line.rstrip() for line in newest_version_file_contents.splitlines()]
 
-    Returns:
-        str: Rendered difference
-    """
-    newest_lines = [line.rstrip() for line in newest_version_file_contents.splitlines()]
-    previous_lines = [line.rstrip() for line in previous_version_file_contents.splitlines()] if previous_version_file_contents else []
+    if previous_version_file_contents:
+        previous_version_file_contents = [line.rstrip() for line in previous_version_file_contents.splitlines()]
+    else:
+        previous_version_file_contents = ""
 
     if patch_format:
-        patch = difflib.unified_diff(previous_lines, newest_lines)
+        patch = difflib.unified_diff(previous_version_file_contents, newest_version_file_contents)
         return line_feed_sep.join(patch)
 
-    rendered_diff = customSequenceMatcher(
-        before=previous_lines,
-        after=newest_lines,
-        include_equal=include_equal,
-        include_removed=include_removed,
-        include_added=include_added,
-        include_replaced=include_replaced,
-        include_change_type_prefix=include_change_type_prefix,
-        html_colour=html_colour
-    )
+    rendered_diff = customSequenceMatcher(before=previous_version_file_contents,
+                                          after=newest_version_file_contents,
+                                          include_equal=include_equal,
+                                          include_removed=include_removed,
+                                          include_added=include_added,
+                                          include_replaced=include_replaced,
+                                          include_change_type_prefix=include_change_type_prefix)
 
-    def flatten(lst: List[Union[str, List[str]]]) -> str:
-        return line_feed_sep.join(flatten(x) if isinstance(x, list) else x for x in lst)
-
-    return flatten(rendered_diff)
+    # Recursively join lists
+    f = lambda L: line_feed_sep.join([f(x) if type(x) is list else x for x in L])
+    p= f(rendered_diff)
+    return p

changedetectionio/forms.py

@@ -1,15 +1,10 @@
 import os
 import re
-from loguru import logger
-from wtforms.widgets.core import TimeInput
-
-from changedetectionio.conditions.form import ConditionFormRow
-from changedetectionio.strtobool import strtobool
+from distutils.util import strtobool
 
 from wtforms import (
     BooleanField,
     Form,
-    Field,
     IntegerField,
     RadioField,
     SelectField,
@@ -128,87 +123,6 @@ class StringTagUUID(StringField):
 
         return 'error'
 
-class TimeDurationForm(Form):
-    hours = SelectField(choices=[(f"{i}", f"{i}") for i in range(0, 25)], default="24",  validators=[validators.Optional()])
-    minutes = SelectField(choices=[(f"{i}", f"{i}") for i in range(0, 60)], default="00", validators=[validators.Optional()])
-
-class TimeStringField(Field):
-    """
-    A WTForms field for time inputs (HH:MM) that stores the value as a string.
-    """
-    widget = TimeInput()  # Use the built-in time input widget
-
-    def _value(self):
-        """
-        Returns the value for rendering in the form.
-        """
-        return self.data if self.data is not None else ""
-
-    def process_formdata(self, valuelist):
-        """
-        Processes the raw input from the form and stores it as a string.
-        """
-        if valuelist:
-            time_str = valuelist[0]
-            # Simple validation for HH:MM format
-            if not time_str or len(time_str.split(":")) != 2:
-                raise ValidationError("Invalid time format. Use HH:MM.")
-            self.data = time_str
-
-
-class validateTimeZoneName(object):
-    """
-       Flask wtform validators wont work with basic auth
-    """
-
-    def __init__(self, message=None):
-        self.message = message
-
-    def __call__(self, form, field):
-        from zoneinfo import available_timezones
-        python_timezones = available_timezones()
-        if field.data and field.data not in python_timezones:
-            raise ValidationError("Not a valid timezone name")
-
-class ScheduleLimitDaySubForm(Form):
-    enabled = BooleanField("not set", default=True)
-    start_time = TimeStringField("Start At", default="00:00", validators=[validators.Optional()])
-    duration = FormField(TimeDurationForm, label="Run duration")
-
-class ScheduleLimitForm(Form):
-    enabled = BooleanField("Use time scheduler", default=False)
-    # Because the label for=""" doesnt line up/work with the actual checkbox
-    monday = FormField(ScheduleLimitDaySubForm, label="")
-    tuesday = FormField(ScheduleLimitDaySubForm, label="")
-    wednesday = FormField(ScheduleLimitDaySubForm, label="")
-    thursday = FormField(ScheduleLimitDaySubForm, label="")
-    friday = FormField(ScheduleLimitDaySubForm, label="")
-    saturday = FormField(ScheduleLimitDaySubForm, label="")
-    sunday = FormField(ScheduleLimitDaySubForm, label="")
-
-    timezone = StringField("Optional timezone to run in",
-                                  render_kw={"list": "timezones"},
-                                  validators=[validateTimeZoneName()]
-                                  )
-    def __init__(
-        self,
-        formdata=None,
-        obj=None,
-        prefix="",
-        data=None,
-        meta=None,
-        **kwargs,
-    ):
-        super().__init__(formdata, obj, prefix, data, meta, **kwargs)
-        self.monday.form.enabled.label.text="Monday"
-        self.tuesday.form.enabled.label.text = "Tuesday"
-        self.wednesday.form.enabled.label.text = "Wednesday"
-        self.thursday.form.enabled.label.text = "Thursday"
-        self.friday.form.enabled.label.text = "Friday"
-        self.saturday.form.enabled.label.text = "Saturday"
-        self.sunday.form.enabled.label.text = "Sunday"
-
-
 class TimeBetweenCheckForm(Form):
     weeks = IntegerField('Weeks', validators=[validators.Optional(), validators.NumberRange(min=0, message="Should contain zero or more seconds")])
     days = IntegerField('Days', validators=[validators.Optional(), validators.NumberRange(min=0, message="Should contain zero or more seconds")])
@@ -307,49 +221,36 @@ class ValidateAppRiseServers(object):
         import apprise
         apobj = apprise.Apprise()
 
-        # so that the custom endpoints are registered
-        from .apprise_asset import asset
-
         for server_url in field.data:
-            url = server_url.strip()
-            if url.startswith("#"):
-                continue
-
-            if not apobj.add(url):
-                message = field.gettext('\'%s\' is not a valid AppRise URL.' % (url))
+            if not apobj.add(server_url):
+                message = field.gettext('\'%s\' is not a valid AppRise URL.' % (server_url))
                 raise ValidationError(message)
 
 class ValidateJinja2Template(object):
     """
     Validates that a {token} is from a valid set
     """
+    def __init__(self, message=None):
+        self.message = message
+
     def __call__(self, form, field):
         from changedetectionio import notification
 
-        from jinja2 import BaseLoader, TemplateSyntaxError, UndefinedError
-        from jinja2.sandbox import ImmutableSandboxedEnvironment
+        from jinja2 import Environment, BaseLoader, TemplateSyntaxError, UndefinedError
         from jinja2.meta import find_undeclared_variables
-        import jinja2.exceptions
 
-        # Might be a list of text, or might be just text (like from the apprise url list)
-        joined_data = ' '.join(map(str, field.data)) if isinstance(field.data, list) else f"{field.data}"
 
         try:
-            jinja2_env = ImmutableSandboxedEnvironment(loader=BaseLoader)
+            jinja2_env = Environment(loader=BaseLoader)
             jinja2_env.globals.update(notification.valid_tokens)
-            # Extra validation tokens provided on the form_class(... extra_tokens={}) setup
-            if hasattr(field, 'extra_notification_tokens'):
-                jinja2_env.globals.update(field.extra_notification_tokens)
 
-            jinja2_env.from_string(joined_data).render()
+            rendered = jinja2_env.from_string(field.data).render()
         except TemplateSyntaxError as e:
             raise ValidationError(f"This is not a valid Jinja2 template: {e}") from e
         except UndefinedError as e:
             raise ValidationError(f"A variable or function is not defined: {e}") from e
-        except jinja2.exceptions.SecurityError as e:
-            raise ValidationError(f"This is not a valid Jinja2 template: {e}") from e
 
-        ast = jinja2_env.parse(joined_data)
+        ast = jinja2_env.parse(field.data)
         undefined = ", ".join(find_undeclared_variables(ast))
         if undefined:
             raise ValidationError(
@@ -369,7 +270,6 @@ class validateURL(object):
         # This should raise a ValidationError() or not
         validate_url(field.data)
 
-
 def validate_url(test_url):
     # If hosts that only contain alphanumerics are allowed ("localhost" for example)
     try:
@@ -512,28 +412,17 @@ class quickWatchForm(Form):
     edit_and_watch_submit_button = SubmitField('Edit > Watch', render_kw={"class": "pure-button pure-button-primary"})
 
 
-
 # Common to a single watch and the global settings
 class commonSettingsForm(Form):
-    from . import processors
 
-    def __init__(self, formdata=None, obj=None, prefix="", data=None, meta=None, **kwargs):
-        super().__init__(formdata, obj, prefix, data, meta, **kwargs)
-        self.notification_body.extra_notification_tokens = kwargs.get('extra_notification_tokens', {})
-        self.notification_title.extra_notification_tokens = kwargs.get('extra_notification_tokens', {})
-        self.notification_urls.extra_notification_tokens = kwargs.get('extra_notification_tokens', {})
-
-    extract_title_as_title = BooleanField('Extract <title> from document and use as watch title', default=False)
-    fetch_backend = RadioField(u'Fetch Method', choices=content_fetchers.available_fetchers(), validators=[ValidateContentFetcherIsReady()])
+    notification_urls = StringListField('Notification URL List', validators=[validators.Optional(), ValidateAppRiseServers()])
+    notification_title = StringField('Notification Title', default='ChangeDetection.io Notification - {{ watch_url }}', validators=[validators.Optional(), ValidateJinja2Template()])
     notification_body = TextAreaField('Notification Body', default='{{ watch_url }} had a change.', validators=[validators.Optional(), ValidateJinja2Template()])
     notification_format = SelectField('Notification format', choices=valid_notification_formats.keys())
-    notification_title = StringField('Notification Title', default='ChangeDetection.io Notification - {{ watch_url }}', validators=[validators.Optional(), ValidateJinja2Template()])
-    notification_urls = StringListField('Notification URL List', validators=[validators.Optional(), ValidateAppRiseServers(), ValidateJinja2Template()])
-    processor = RadioField( label=u"Processor - What do you want to achieve?", choices=processors.available_processors(), default="text_json_diff")
-    timezone = StringField("Timezone for watch schedule", render_kw={"list": "timezones"}, validators=[validateTimeZoneName()])
-    webdriver_delay = IntegerField('Wait seconds before extracting text', validators=[validators.Optional(), validators.NumberRange(min=1, message="Should contain one or more seconds")])
-
-
+    fetch_backend = RadioField(u'Fetch Method', choices=content_fetchers.available_fetchers(), validators=[ValidateContentFetcherIsReady()])
+    extract_title_as_title = BooleanField('Extract <title> from document and use as watch title', default=False)
+    webdriver_delay = IntegerField('Wait seconds before extracting text', validators=[validators.Optional(), validators.NumberRange(min=1,
+                                                                                                                                    message="Should contain one or more seconds")])
 class importForm(Form):
     from . import processors
     processor = RadioField(u'Processor', choices=processors.available_processors(), default="text_json_diff")
@@ -541,6 +430,7 @@ class importForm(Form):
     xlsx_file = FileField('Upload .xlsx file', validators=[FileAllowed(['xlsx'], 'Must be .xlsx file!')])
     file_mapping = SelectField('File mapping', [validators.DataRequired()], choices={('wachete', 'Wachete mapping'), ('custom','Custom mapping')})
 
+
 class SingleBrowserStep(Form):
 
     operation = SelectField('Operation', [validators.Optional()], choices=browser_step_ui_config.keys())
@@ -552,46 +442,43 @@ class SingleBrowserStep(Form):
 #    remove_button = SubmitField('-', render_kw={"type": "button", "class": "pure-button pure-button-primary", 'title': 'Remove'})
 #    add_button = SubmitField('+', render_kw={"type": "button", "class": "pure-button pure-button-primary", 'title': 'Add new step after'})
 
-class processor_text_json_diff_form(commonSettingsForm):
+class watchForm(commonSettingsForm):
 
     url = fields.URLField('URL', validators=[validateURL()])
     tags = StringTagUUID('Group tag', [validators.Optional()], default='')
 
     time_between_check = FormField(TimeBetweenCheckForm)
 
-    time_schedule_limit = FormField(ScheduleLimitForm)
-
-    time_between_check_use_default = BooleanField('Use global settings for time between check', default=False)
-
     include_filters = StringListField('CSS/JSONPath/JQ/XPath Filters', [ValidateCSSJSONXPATHInput()], default='')
 
-    subtractive_selectors = StringListField('Remove elements', [ValidateCSSJSONXPATHInput(allow_json=False)])
+    subtractive_selectors = StringListField('Remove elements', [ValidateCSSJSONXPATHInput(allow_xpath=False, allow_json=False)])
 
     extract_text = StringListField('Extract text', [ValidateListRegex()])
 
     title = StringField('Title', default='')
 
-    ignore_text = StringListField('Ignore lines containing', [ValidateListRegex()])
+    ignore_text = StringListField('Ignore text', [ValidateListRegex()])
     headers = StringDictKeyValue('Request headers')
     body = TextAreaField('Request body', [validators.Optional()])
     method = SelectField('Request method', choices=valid_method, default=default_method)
     ignore_status_codes = BooleanField('Ignore status codes (process non-2xx status codes as normal)', default=False)
-    check_unique_lines = BooleanField('Only trigger when unique lines appear in all history', default=False)
-    remove_duplicate_lines = BooleanField('Remove duplicate lines of text', default=False)
+    check_unique_lines = BooleanField('Only trigger when unique lines appear', default=False)
     sort_text_alphabetically =  BooleanField('Sort text alphabetically', default=False)
-    trim_text_whitespace = BooleanField('Trim whitespace before and after text', default=False)
 
     filter_text_added = BooleanField('Added lines', default=True)
     filter_text_replaced = BooleanField('Replaced/changed lines', default=True)
     filter_text_removed = BooleanField('Removed lines', default=True)
 
+    # @todo this class could be moved to its own text_json_diff_watchForm and this goes to restock_diff_Watchform perhaps
+    in_stock_only = BooleanField('Only trigger when product goes BACK to in-stock', default=True)
+
     trigger_text = StringListField('Trigger/wait for text', [validators.Optional(), ValidateListRegex()])
     if os.getenv("PLAYWRIGHT_DRIVER_URL"):
         browser_steps = FieldList(FormField(SingleBrowserStep), min_entries=10)
     text_should_not_be_present = StringListField('Block change-detection while text matches', [validators.Optional(), ValidateListRegex()])
     webdriver_js_execute_code = TextAreaField('Execute JavaScript before change detection', render_kw={"rows": "5"}, validators=[validators.Optional()])
 
-    save_button = SubmitField('Save', render_kw={"class": "pure-button button-small pure-button-primary"})
+    save_button = SubmitField('Save', render_kw={"class": "pure-button pure-button-primary"})
 
     proxy = RadioField('Proxy')
     filter_failure_notification_send = BooleanField(
@@ -600,21 +487,10 @@ class processor_text_json_diff_form(commonSettingsForm):
     notification_muted = BooleanField('Notifications Muted / Off', default=False)
     notification_screenshot = BooleanField('Attach screenshot to notification (where possible)', default=False)
 
-    conditions_match_logic = RadioField(u'Match', choices=[('ALL', 'Match all of the following'),('ANY', 'Match any of the following')], default='ALL')
-    conditions = FieldList(FormField(ConditionFormRow), min_entries=1)  # Add rule logic here
-
-
-    def extra_tab_content(self):
-        return None
-
-    def extra_form_content(self):
-        return None
-
     def validate(self, **kwargs):
         if not super().validate():
             return False
 
-        from changedetectionio.safe_jinja import render as jinja_render
         result = True
 
         # Fail form validation when a body is set for a GET
@@ -623,65 +499,16 @@ class processor_text_json_diff_form(commonSettingsForm):
             result = False
 
         # Attempt to validate jinja2 templates in the URL
+        from jinja2 import Environment
+        # Jinja2 available in URLs along with https://pypi.org/project/jinja2-time/
+        jinja2_env = Environment(extensions=['jinja2_time.TimeExtension'])
         try:
-            jinja_render(template_str=self.url.data)
-        except ModuleNotFoundError as e:
-            # incase jinja2_time or others is missing
-            logger.error(e)
-            self.url.errors.append(f'Invalid template syntax configuration: {e}')
-            result = False
+            ready_url = str(jinja2_env.from_string(self.url.data).render())
         except Exception as e:
-            logger.error(e)
-            self.url.errors.append(f'Invalid template syntax: {e}')
+            self.url.errors.append('Invalid template syntax')
             result = False
-
-        # Attempt to validate jinja2 templates in the body
-        if self.body.data and self.body.data.strip():
-            try:
-                jinja_render(template_str=self.body.data)
-            except ModuleNotFoundError as e:
-                # incase jinja2_time or others is missing
-                logger.error(e)
-                self.body.errors.append(f'Invalid template syntax configuration: {e}')
-                result = False
-            except Exception as e:
-                logger.error(e)
-                self.body.errors.append(f'Invalid template syntax: {e}')
-                result = False
-
-        # Attempt to validate jinja2 templates in the headers
-        if len(self.headers.data) > 0:
-            try:
-                for header, value in self.headers.data.items():
-                    jinja_render(template_str=value)
-            except ModuleNotFoundError as e:
-                # incase jinja2_time or others is missing
-                logger.error(e)
-                self.headers.errors.append(f'Invalid template syntax configuration: {e}')
-                result = False
-            except Exception as e:
-                logger.error(e)
-                self.headers.errors.append(f'Invalid template syntax in "{header}" header: {e}')
-                result = False
-
         return result
 
-    def __init__(
-            self,
-            formdata=None,
-            obj=None,
-            prefix="",
-            data=None,
-            meta=None,
-            **kwargs,
-    ):
-        super().__init__(formdata, obj, prefix, data, meta, **kwargs)
-        if kwargs and kwargs.get('default_system_settings'):
-            default_tz = kwargs.get('default_system_settings').get('application', {}).get('timezone')
-            if default_tz:
-                self.time_schedule_limit.form.timezone.render_kw['placeholder'] = default_tz
-
-
 
 class SingleExtraProxy(Form):
 
@@ -695,15 +522,10 @@ class SingleExtraBrowser(Form):
     browser_connection_url = StringField('Browser connection URL', [validators.Optional()], render_kw={"placeholder": "wss://brightdata... wss://oxylabs etc", "size":50})
     # @todo do the validation here instead
 
-class DefaultUAInputForm(Form):
-    html_requests = StringField('Plaintext requests', validators=[validators.Optional()], render_kw={"placeholder": "<default>"})
-    if os.getenv("PLAYWRIGHT_DRIVER_URL") or os.getenv("WEBDRIVER_URL"):
-        html_webdriver = StringField('Chrome requests', validators=[validators.Optional()], render_kw={"placeholder": "<default>"})
 
 # datastore.data['settings']['requests']..
 class globalSettingsRequestForm(Form):
     time_between_check = FormField(TimeBetweenCheckForm)
-    time_schedule_limit = FormField(ScheduleLimitForm)
     proxy = RadioField('Proxy')
     jitter_seconds = IntegerField('Random jitter seconds ± check',
                                   render_kw={"style": "width: 5em;"},
@@ -711,8 +533,6 @@ class globalSettingsRequestForm(Form):
     extra_proxies = FieldList(FormField(SingleExtraProxy), min_entries=5)
     extra_browsers = FieldList(FormField(SingleExtraBrowser), min_entries=5)
 
-    default_ua = FormField(DefaultUAInputForm, label="Default User-Agent overrides")
-
     def validate_extra_proxies(self, extra_validators=None):
         for e in self.data['extra_proxies']:
             if e.get('proxy_name') or e.get('proxy_url'):
@@ -732,7 +552,7 @@ class globalSettingsApplicationForm(commonSettingsForm):
     empty_pages_are_a_change =  BooleanField('Treat empty pages as a change?', default=False)
     fetch_backend = RadioField('Fetch Method', default="html_requests", choices=content_fetchers.available_fetchers(), validators=[ValidateContentFetcherIsReady()])
     global_ignore_text = StringListField('Ignore Text', [ValidateListRegex()])
-    global_subtractive_selectors = StringListField('Remove elements', [ValidateCSSJSONXPATHInput(allow_json=False)])
+    global_subtractive_selectors = StringListField('Remove elements', [ValidateCSSJSONXPATHInput(allow_xpath=False, allow_json=False)])
     ignore_whitespace = BooleanField('Ignore whitespace')
     password = SaltyPasswordField()
     pager_size = IntegerField('Pager size',
@@ -742,8 +562,6 @@ class globalSettingsApplicationForm(commonSettingsForm):
     removepassword_button = SubmitField('Remove password', render_kw={"class": "pure-button pure-button-primary"})
     render_anchor_tag_content = BooleanField('Render anchor tag content', default=False)
     shared_diff_access = BooleanField('Allow access to view diff page when password is enabled', default=False, validators=[validators.Optional()])
-    rss_hide_muted_watches = BooleanField('Hide muted watches from RSS feed', default=True,
-                                      validators=[validators.Optional()])
     filter_failure_notification_threshold_attempts = IntegerField('Number of times the filter can be missing before sending a notification',
                                                                   render_kw={"style": "width: 5em;"},
                                                                   validators=[validators.NumberRange(min=0,
@@ -754,15 +572,10 @@ class globalSettingsForm(Form):
     # Define these as FormFields/"sub forms", this way it matches the JSON storage
     # datastore.data['settings']['application']..
     # datastore.data['settings']['requests']..
-    def __init__(self, formdata=None, obj=None, prefix="", data=None, meta=None, **kwargs):
-        super().__init__(formdata, obj, prefix, data, meta, **kwargs)
-        self.application.notification_body.extra_notification_tokens = kwargs.get('extra_notification_tokens', {})
-        self.application.notification_title.extra_notification_tokens = kwargs.get('extra_notification_tokens', {})
-        self.application.notification_urls.extra_notification_tokens = kwargs.get('extra_notification_tokens', {})
 
     requests = FormField(globalSettingsRequestForm)
     application = FormField(globalSettingsApplicationForm)
-    save_button = SubmitField('Save', render_kw={"class": "pure-button button-small pure-button-primary"})
+    save_button = SubmitField('Save', render_kw={"class": "pure-button pure-button-primary"})
 
 
 class extractDataForm(Form):

changedetectionio/html_tools.py

@@ -1,14 +1,20 @@
-from loguru import logger
-from lxml import etree
+
+from bs4 import BeautifulSoup
+from inscriptis import get_text
+from jsonpath_ng.ext import parse
 from typing import List
+from inscriptis.css_profiles import CSS_PROFILES, HtmlElement
+from inscriptis.html_properties import Display
+from inscriptis.model.config import ParserConfig
+from xml.sax.saxutils import escape as xml_escape
 import json
 import re
 
+
 # HTML added to be sure each result matching a filter (.example) gets converted to a new line by Inscriptis
 TEXT_FILTER_LIST_LINE_SUFFIX = "<br>"
-TRANSLATE_WHITESPACE_TABLE = str.maketrans('', '', '\r\n\t ')
-PERL_STYLE_REGEX = r'^/(.*?)/([a-z]*)?$'
 
+PERL_STYLE_REGEX = r'^/(.*?)/([a-z]*)?$'
 # 'price' , 'lowPrice', 'highPrice' are usually under here
 # All of those may or may not appear on different websites - I didnt find a way todo case-insensitive searching here
 LD_JSON_PRODUCT_OFFER_SELECTORS = ["json:$..offers", "json:$..Offers"]
@@ -35,7 +41,6 @@ def perl_style_slash_enclosed_regex_to_options(regex):
 
 # Given a CSS Rule, and a blob of HTML, return the blob of HTML that matches
 def include_filters(include_filters, html_content, append_pretty_line_formatting=False):
-    from bs4 import BeautifulSoup
     soup = BeautifulSoup(html_content, "html.parser")
     html_block = ""
     r = soup.select(include_filters, separator="")
@@ -53,67 +58,16 @@ def include_filters(include_filters, html_content, append_pretty_line_formatting
     return html_block
 
 def subtractive_css_selector(css_selector, html_content):
-    from bs4 import BeautifulSoup
     soup = BeautifulSoup(html_content, "html.parser")
-
-    # So that the elements dont shift their index, build a list of elements here which will be pointers to their place in the DOM
-    elements_to_remove = soup.select(css_selector)
-
-    # Then, remove them in a separate loop
-    for item in elements_to_remove:
+    for item in soup.select(css_selector):
         item.decompose()
-
     return str(soup)
 
-def subtractive_xpath_selector(selectors: List[str], html_content: str) -> str:
-    # Parse the HTML content using lxml
-    html_tree = etree.HTML(html_content)
-
-    # First, collect all elements to remove
-    elements_to_remove = []
-
-    # Iterate over the list of XPath selectors
-    for selector in selectors:
-        # Collect elements for each selector
-        elements_to_remove.extend(html_tree.xpath(selector))
-
-    # Then, remove them in a separate loop
-    for element in elements_to_remove:
-        if element.getparent() is not None:  # Ensure the element has a parent before removing
-            element.getparent().remove(element)
-
-    # Convert the modified HTML tree back to a string
-    modified_html = etree.tostring(html_tree, method="html").decode("utf-8")
-    return modified_html
-
 
 def element_removal(selectors: List[str], html_content):
-    """Removes elements that match a list of CSS or XPath selectors."""
-    modified_html = html_content
-    css_selectors = []
-    xpath_selectors = []
-
-    for selector in selectors:
-        if selector.startswith(('xpath:', 'xpath1:', '//')):
-            # Handle XPath selectors separately
-            xpath_selector = selector.removeprefix('xpath:').removeprefix('xpath1:')
-            xpath_selectors.append(xpath_selector)
-        else:
-            # Collect CSS selectors as one "hit", see comment in subtractive_css_selector
-            css_selectors.append(selector.strip().strip(","))
-
-    if xpath_selectors:
-        modified_html = subtractive_xpath_selector(xpath_selectors, modified_html)
-
-    if css_selectors:
-        # Remove duplicates, then combine all CSS selectors into one string, separated by commas
-        # This stops the elements index shifting
-        unique_selectors = list(set(css_selectors))  # Ensure uniqueness
-        combined_css_selector = " , ".join(unique_selectors)
-        modified_html = subtractive_css_selector(combined_css_selector, modified_html)
-
-
-    return modified_html
+    """Joins individual filters into one css filter."""
+    selector = ",".join(selectors)
+    return subtractive_css_selector(selector, html_content)
 
 def elementpath_tostring(obj):
     """
@@ -215,21 +169,20 @@ def xpath1_filter(xpath_filter, html_content, append_pretty_line_formatting=Fals
         # And where the matched result doesn't include something that will cause Inscriptis to add a newline
         # (This way each 'match' reliably has a new-line in the diff)
         # Divs are converted to 4 whitespaces by inscriptis
-        if append_pretty_line_formatting and len(html_block) and (not hasattr(element, 'tag') or not element.tag in (['br', 'hr', 'div', 'p'])):
+        if append_pretty_line_formatting and len(html_block) and (not hasattr( element, 'tag' ) or not element.tag in (['br', 'hr', 'div', 'p'])):
             html_block += TEXT_FILTER_LIST_LINE_SUFFIX
 
-        # Some kind of text, UTF-8 or other
-        if isinstance(element, (str, bytes)):
-            html_block += element
+        if type(element) == etree._ElementStringResult:
+            html_block += str(element)
+        elif type(element) == etree._ElementUnicodeResult:
+            html_block += str(element)
         else:
-            # Return the HTML which will get parsed as text
             html_block += etree.tostring(element, pretty_print=True).decode('utf-8')
 
     return html_block
 
 # Extract/find element
 def extract_element(find='title', html_content=''):
-    from bs4 import BeautifulSoup
 
     #Re #106, be sure to handle when its not found
     element_text = None
@@ -243,14 +196,12 @@ def extract_element(find='title', html_content=''):
 
 #
 def _parse_json(json_data, json_filter):
-    from jsonpath_ng.ext import parse
-
-    if json_filter.startswith("json:"):
+    if 'json:' in json_filter:
         jsonpath_expression = parse(json_filter.replace('json:', ''))
         match = jsonpath_expression.find(json_data)
         return _get_stripped_text_from_json_match(match)
 
-    if json_filter.startswith("jq:") or json_filter.startswith("jqraw:"):
+    if 'jq:' in json_filter:
 
         try:
             import jq
@@ -258,15 +209,10 @@ def _parse_json(json_data, json_filter):
             # `jq` requires full compilation in windows and so isn't generally available
             raise Exception("jq not support not found")
 
-        if json_filter.startswith("jq:"):
-            jq_expression = jq.compile(json_filter.removeprefix("jq:"))
-            match = jq_expression.input(json_data).all()
-            return _get_stripped_text_from_json_match(match)
+        jq_expression = jq.compile(json_filter.replace('jq:', ''))
+        match = jq_expression.input(json_data).all()
 
-        if json_filter.startswith("jqraw:"):
-            jq_expression = jq.compile(json_filter.removeprefix("jqraw:"))
-            match = jq_expression.input(json_data).all()
-            return '\n'.join(str(item) for item in match)
+        return _get_stripped_text_from_json_match(match)
 
 def _get_stripped_text_from_json_match(match):
     s = []
@@ -293,16 +239,12 @@ def _get_stripped_text_from_json_match(match):
 # json_filter - ie json:$..price
 # ensure_is_ldjson_info_type - str "product", optional, "@type == product" (I dont know how to do that as a json selector)
 def extract_json_as_string(content, json_filter, ensure_is_ldjson_info_type=None):
-    from bs4 import BeautifulSoup
-
     stripped_text_from_html = False
-# https://github.com/dgtlmoon/changedetection.io/pull/2041#issuecomment-1848397161w
+
     # Try to parse/filter out the JSON, if we get some parser error, then maybe it's embedded within HTML tags
     try:
-        # .lstrip("\ufeff") strings ByteOrderMark from UTF8 and still lets the UTF work
-        stripped_text_from_html = _parse_json(json.loads(content.lstrip("\ufeff") ), json_filter)
-    except json.JSONDecodeError as e:
-        logger.warning(str(e))
+        stripped_text_from_html = _parse_json(json.loads(content), json_filter)
+    except json.JSONDecodeError:
 
         # Foreach <script json></script> blob.. just return the first that matches json_filter
         # As a last resort, try to parse the whole <body>
@@ -337,19 +279,17 @@ def extract_json_as_string(content, json_filter, ensure_is_ldjson_info_type=None
                 if isinstance(json_data, dict):
                     # If it has LD JSON 'key' @type, and @type is 'product', and something was found for the search
                     # (Some sites have multiple of the same ld+json @type='product', but some have the review part, some have the 'price' part)
-                    # @type could also be a list although non-standard ("@type": ["Product", "SubType"],)
+                    # @type could also be a list (Product, SubType)
                     # LD_JSON auto-extract also requires some content PLUS the ldjson to be present
                     # 1833 - could be either str or dict, should not be anything else
-
-                    t = json_data.get('@type')
-                    if t and stripped_text_from_html:
-
-                        if isinstance(t, str) and t.lower() == ensure_is_ldjson_info_type.lower():
-                            break
-                        # The non-standard part, some have a list
-                        elif isinstance(t, list):
-                            if ensure_is_ldjson_info_type.lower() in [x.lower().strip() for x in t]:
-                                break
+                    if json_data.get('@type') and stripped_text_from_html:
+                        try:
+                            if json_data.get('@type') == str or json_data.get('@type') == dict:
+                                types = [json_data.get('@type')] if isinstance(json_data.get('@type'), str) else json_data.get('@type')
+                                if ensure_is_ldjson_info_type.lower() in [x.lower().strip() for x in types]:
+                                    break
+                        except:
+                            continue
 
             elif stripped_text_from_html:
                 break
@@ -364,7 +304,6 @@ def extract_json_as_string(content, json_filter, ensure_is_ldjson_info_type=None
 #          - "line numbers" return a list of line numbers that match (int list)
 #
 # wordlist - list of regex's (str) or words (str)
-# Preserves all linefeeds and other whitespacing, its not the job of this to remove that
 def strip_ignore_text(content, wordlist, mode="content"):
     i = 0
     output = []
@@ -380,33 +319,34 @@ def strip_ignore_text(content, wordlist, mode="content"):
         else:
             ignore_text.append(k.strip())
 
-    for line in content.splitlines(keepends=True):
+    for line in content.splitlines():
         i += 1
         # Always ignore blank lines in this mode. (when this function gets called)
         got_match = False
-        for l in ignore_text:
-            if l.lower() in line.lower():
-                got_match = True
-
-        if not got_match:
-            for r in ignore_regex:
-                if r.search(line):
+        if len(line.strip()):
+            for l in ignore_text:
+                if l.lower() in line.lower():
                     got_match = True
 
-        if not got_match:
-            # Not ignored, and should preserve "keepends"
-            output.append(line)
-        else:
-            ignored_line_numbers.append(i)
+            if not got_match:
+                for r in ignore_regex:
+                    if r.search(line):
+                        got_match = True
+
+            if not got_match:
+                # Not ignored
+                output.append(line.encode('utf8'))
+            else:
+                ignored_line_numbers.append(i)
+
 
     # Used for finding out what to highlight
     if mode == "line numbers":
         return ignored_line_numbers
 
-    return ''.join(output)
+    return "\n".encode('utf8').join(output)
 
 def cdata_in_document_to_text(html_content: str, render_anchor_tag_content=False) -> str:
-    from xml.sax.saxutils import escape as xml_escape
     pattern = '<!\[CDATA\[(\s*(?:.(?<!\]\]>)\s*)*)\]\]>'
     def repl(m):
         text = m.group(1)
@@ -415,9 +355,6 @@ def cdata_in_document_to_text(html_content: str, render_anchor_tag_content=False
     return re.sub(pattern, repl, html_content)
 
 def html_to_text(html_content: str, render_anchor_tag_content=False, is_rss=False) -> str:
-    from inscriptis import get_text
-    from inscriptis.model.config import ParserConfig
-
     """Converts html string to a string with just the text. If ignoring
     rendering anchor tag content is enable, anchor tag content are also
     included in the text
@@ -455,23 +392,22 @@ def html_to_text(html_content: str, render_anchor_tag_content=False, is_rss=Fals
 
 # Does LD+JSON exist with a @type=='product' and a .price set anywhere?
 def has_ldjson_product_info(content):
+    pricing_data = ''
+
     try:
-        lc = content.lower()
-        if 'application/ld+json' in lc and lc.count('"price"') == 1 and '"pricecurrency"' in lc:
-            return True
+        if not 'application/ld+json' in content:
+            return False
+
+        for filter in LD_JSON_PRODUCT_OFFER_SELECTORS:
+            pricing_data += extract_json_as_string(content=content,
+                                                  json_filter=filter,
+                                                  ensure_is_ldjson_info_type="product")
 
-#       On some pages this is really terribly expensive when they dont really need it
-#       (For example you never want price monitoring, but this runs on every watch to suggest it)
-#        for filter in LD_JSON_PRODUCT_OFFER_SELECTORS:
-#            pricing_data += extract_json_as_string(content=content,
-#                                                  json_filter=filter,
-#                                                  ensure_is_ldjson_info_type="product")
     except Exception as e:
-        # OK too
+        # Totally fine
         return False
-
-    return False
-
+    x=bool(pricing_data)
+    return x
 
 
 def workarounds_for_obfuscations(content):

changedetectionio/model/App.py

@@ -5,9 +5,7 @@ from changedetectionio.notification import (
     default_notification_title,
 )
 
-# Equal to or greater than this number of FilterNotFoundInResponse exceptions will trigger a filter-not-found notification
 _FILTER_FAILURE_THRESHOLD_ATTEMPTS_DEFAULT = 6
-DEFAULT_SETTINGS_HEADERS_USERAGENT='Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/87.0.4280.66 Safari/537.36'
 
 class model(dict):
     base_config = {
@@ -24,10 +22,6 @@ class model(dict):
                     'time_between_check': {'weeks': None, 'days': None, 'hours': 3, 'minutes': None, 'seconds': None},
                     'timeout': int(getenv("DEFAULT_SETTINGS_REQUESTS_TIMEOUT", "45")),  # Default 45 seconds
                     'workers': int(getenv("DEFAULT_SETTINGS_REQUESTS_WORKERS", "10")),  # Number of threads, lower is better for slow connections
-                    'default_ua': {
-                        'html_requests': getenv("DEFAULT_SETTINGS_HEADERS_USERAGENT", DEFAULT_SETTINGS_HEADERS_USERAGENT),
-                        'html_webdriver': None,
-                    }
                 },
                 'application': {
                     # Custom notification content
@@ -47,13 +41,10 @@ class model(dict):
                     'pager_size': 50,
                     'password': False,
                     'render_anchor_tag_content': False,
-                    'rss_access_token': None,
-                    'rss_hide_muted_watches': True,
                     'schema_version' : 0,
                     'shared_diff_access': False,
                     'webdriver_delay': None , # Extra delay in seconds before extracting text
-                    'tags': {}, #@todo use Tag.model initialisers
-                    'timezone': None, # Default IANA timezone name
+                    'tags': {} #@todo use Tag.model initialisers
                 }
             }
         }
@@ -69,7 +60,7 @@ def parse_headers_from_text_file(filepath):
         for l in f.readlines():
             l = l.strip()
             if not l.startswith('#') and ':' in l:
-                (k, v) = l.split(':', 1)  # Split only on the first colon
+                (k, v) = l.split(':')
                 headers[k.strip()] = v.strip()
 
     return headers

changedetectionio/model/Tag.py

@@ -1,14 +1,19 @@
+from .Watch import base_config
+import uuid
 
-from changedetectionio.model import watch_base
-
-
-class model(watch_base):
+class model(dict):
 
     def __init__(self, *arg, **kw):
-        super(model, self).__init__(*arg, **kw)
 
-        self['overrides_watch'] = kw.get('default', {}).get('overrides_watch')
+        self.update(base_config)
+
+        self['uuid'] = str(uuid.uuid4())
 
         if kw.get('default'):
             self.update(kw['default'])
             del kw['default']
+
+
+        # Goes at the end so we update the default object with the initialiser
+        super(model, self).__init__(*arg, **kw)
+

changedetectionio/model/Watch.py

@@ -1,20 +1,80 @@
-from changedetectionio.strtobool import strtobool
-from changedetectionio.safe_jinja import render as jinja_render
-from . import watch_base
+from distutils.util import strtobool
 import os
 import re
+import time
+import uuid
 from pathlib import Path
 from loguru import logger
 
-from ..html_tools import TRANSLATE_WHITESPACE_TABLE
-
 # Allowable protocols, protects against javascript: etc
 # file:// is further checked by ALLOW_FILE_URI
 SAFE_PROTOCOL_REGEX='^(http|https|ftp|file):'
 
-minimum_seconds_recheck_time = int(os.getenv('MINIMUM_SECONDS_RECHECK_TIME', 3))
+minimum_seconds_recheck_time = int(os.getenv('MINIMUM_SECONDS_RECHECK_TIME', 60))
 mtable = {'seconds': 1, 'minutes': 60, 'hours': 3600, 'days': 86400, 'weeks': 86400 * 7}
 
+from changedetectionio.notification import (
+    default_notification_format_for_watch
+)
+
+base_config = {
+    'body': None,
+    'browser_steps': [],
+    'browser_steps_last_error_step': None,
+    'check_unique_lines': False,  # On change-detected, compare against all history if its something new
+    'check_count': 0,
+    'date_created': None,
+    'consecutive_filter_failures': 0,  # Every time the CSS/xPath filter cannot be located, reset when all is fine.
+    'extract_text': [],  # Extract text by regex after filters
+    'extract_title_as_title': False,
+    'fetch_backend': 'system', # plaintext, playwright etc
+    'fetch_time': 0.0,
+    'processor': 'text_json_diff', # could be restock_diff or others from .processors
+    'filter_failure_notification_send': strtobool(os.getenv('FILTER_FAILURE_NOTIFICATION_SEND_DEFAULT', 'True')),
+    'filter_text_added': True,
+    'filter_text_replaced': True,
+    'filter_text_removed': True,
+    'has_ldjson_price_data': None,
+    'track_ldjson_price_data': None,
+    'headers': {},  # Extra headers to send
+    'ignore_text': [],  # List of text to ignore when calculating the comparison checksum
+    'in_stock' : None,
+    'in_stock_only' : True, # Only trigger change on going to instock from out-of-stock
+    'include_filters': [],
+    'last_checked': 0,
+    'last_error': False,
+    'last_viewed': 0,  # history key value of the last viewed via the [diff] link
+    'method': 'GET',
+    'notification_alert_count': 0,
+    # Custom notification content
+    'notification_body': None,
+    'notification_format': default_notification_format_for_watch,
+    'notification_muted': False,
+    'notification_title': None,
+    'notification_screenshot': False,  # Include the latest screenshot if available and supported by the apprise URL
+    'notification_urls': [],  # List of URLs to add to the notification Queue (Usually AppRise)
+    'paused': False,
+    'previous_md5': False,
+    'previous_md5_before_filters': False,  # Used for skipping changedetection entirely
+    'proxy': None,  # Preferred proxy connection
+    'remote_server_reply': None, # From 'server' reply header
+    'sort_text_alphabetically': False,
+    'subtractive_selectors': [],
+    'tag': '', # Old system of text name for a tag, to be removed
+    'tags': [], # list of UUIDs to App.Tags
+    'text_should_not_be_present': [],  # Text that should not present
+    # Re #110, so then if this is set to None, we know to use the default value instead
+    # Requires setting to None on submit if it's the same as the default
+    # Should be all None by default, so we use the system default in this case.
+    'time_between_check': {'weeks': None, 'days': None, 'hours': None, 'minutes': None, 'seconds': None},
+    'title': None,
+    'trigger_text': [],  # List of text or regex to wait for until a change is detected
+    'url': '',
+    'uuid': str(uuid.uuid4()),
+    'webdriver_delay': None,
+    'webdriver_js_execute_code': None,  # Run before change-detection
+}
+
 
 def is_safe_url(test_url):
     # See https://github.com/dgtlmoon/changedetection.io/issues/1358
@@ -31,27 +91,30 @@ def is_safe_url(test_url):
 
     return True
 
-
-class model(watch_base):
+class model(dict):
     __newest_history_key = None
     __history_n = 0
     jitter_seconds = 0
 
     def __init__(self, *arg, **kw):
-        self.__datastore_path = kw.get('datastore_path')
-        if kw.get('datastore_path'):
-            del kw['datastore_path']
-        super(model, self).__init__(*arg, **kw)
+
+        self.update(base_config)
+        self.__datastore_path = kw['datastore_path']
+
+        self['uuid'] = str(uuid.uuid4())
+
+        del kw['datastore_path']
+
         if kw.get('default'):
             self.update(kw['default'])
             del kw['default']
 
-        if self.get('default'):
-            del self['default']
-
         # Be sure the cached timestamp is ready
         bump = self.history
 
+        # Goes at the end so we update the default object with the initialiser
+        super(model, self).__init__(*arg, **kw)
+
     @property
     def viewed(self):
         # Don't return viewed when last_viewed is 0 and newest_key is 0
@@ -67,17 +130,19 @@ class model(watch_base):
 
     @property
     def link(self):
+
         url = self.get('url', '')
         if not is_safe_url(url):
             return 'DISABLED'
 
         ready_url = url
         if '{%' in url or '{{' in url:
+            from jinja2 import Environment
             # Jinja2 available in URLs along with https://pypi.org/project/jinja2-time/
+            jinja2_env = Environment(extensions=['jinja2_time.TimeExtension'])
             try:
-                ready_url = jinja_render(template_str=url)
+                ready_url = str(jinja2_env.from_string(url).render())
             except Exception as e:
-                logger.critical(f"Invalid URL template for: '{url}' - {str(e)}")
                 from flask import (
                     flash, Markup, url_for
                 )
@@ -88,52 +153,8 @@ class model(watch_base):
 
         if ready_url.startswith('source:'):
             ready_url=ready_url.replace('source:', '')
-
-        # Also double check it after any Jinja2 formatting just incase
-        if not is_safe_url(ready_url):
-            return 'DISABLED'
-            
-        # Check if a processor wants to customize the display link
-        processor_name = self.get('processor')
-        if processor_name:
-            try:
-                # Import here to avoid circular imports
-                from changedetectionio.processors.processor_registry import get_display_link
-                custom_link = get_display_link(url=ready_url, processor_name=processor_name)
-                if custom_link:
-                    return custom_link
-            except Exception as e:
-                logger.error(f"Error getting custom display link for processor {processor_name}: {str(e)}")
-                
         return ready_url
 
-    def clear_watch(self):
-        import pathlib
-
-        # JSON Data, Screenshots, Textfiles (history index and snapshots), HTML in the future etc
-        for item in pathlib.Path(str(self.watch_data_dir)).rglob("*.*"):
-            os.unlink(item)
-
-        # Force the attr to recalculate
-        bump = self.history
-
-        # Do this last because it will trigger a recheck due to last_checked being zero
-        self.update({
-            'browser_steps_last_error_step': None,
-            'check_count': 0,
-            'fetch_time': 0.0,
-            'has_ldjson_price_data': None,
-            'last_checked': 0,
-            'last_error': False,
-            'last_notification_error': False,
-            'last_viewed': 0,
-            'previous_md5': False,
-            'previous_md5_before_filters': False,
-            'remote_server_reply': None,
-            'track_ldjson_price_data': None
-        })
-        return
-
     @property
     def is_source_type_url(self):
         return self.get('url', '').startswith('source:')
@@ -190,10 +211,6 @@ class model(watch_base):
         """
         tmp_history = {}
 
-        # In the case we are only using the watch for processing without history
-        if not self.watch_data_dir:
-            return []
-
         # Read the history file as a dict
         fname = os.path.join(self.watch_data_dir, "history.txt")
         if os.path.isfile(fname):
@@ -219,8 +236,6 @@ class model(watch_base):
 
         if len(tmp_history):
             self.__newest_history_key = list(tmp_history.keys())[-1]
-        else:
-            self.__newest_history_key = None
 
         self.__history_n = len(tmp_history)
 
@@ -239,13 +254,6 @@ class model(watch_base):
 
         return has_browser_steps
 
-    @property
-    def has_restock_info(self):
-        if self.get('restock') and self['restock'].get('in_stock') != None:
-                return True
-
-        return False
-
     # Returns the newest key, but if theres only 1 record, then it's counted as not being new, so return 0.
     @property
     def newest_history_key(self):
@@ -259,32 +267,37 @@ class model(watch_base):
         bump = self.history
         return self.__newest_history_key
 
-    # Given an arbitrary timestamp, find the best history key for the [diff] button so it can preset a smarter from_version
+    # Given an arbitrary timestamp, find the closest next key
+    # For example, last_viewed = 1000 so it should return the next 1001 timestamp
+    #
+    # used for the [diff] button so it can preset a smarter from_version
     @property
-    def get_from_version_based_on_last_viewed(self):
+    def get_next_snapshot_key_to_last_viewed(self):
 
         """Unfortunately for now timestamp is stored as string key"""
         keys = list(self.history.keys())
         if not keys:
             return None
-        if len(keys) == 1:
-            return keys[0]
 
         last_viewed = int(self.get('last_viewed'))
+        prev_k = keys[0]
         sorted_keys = sorted(keys, key=lambda x: int(x))
         sorted_keys.reverse()
 
-        # When the 'last viewed' timestamp is greater than or equal the newest snapshot, return second newest
-        if last_viewed >= int(sorted_keys[0]):
+        # When the 'last viewed' timestamp is greater than the newest snapshot, return second last
+        if last_viewed > int(sorted_keys[0]):
             return sorted_keys[1]
-        
-        # When the 'last viewed' timestamp is between snapshots, return the older snapshot
-        for newer, older in list(zip(sorted_keys[0:], sorted_keys[1:])):
-            if last_viewed < int(newer) and last_viewed >= int(older):
-                return older
 
-        # When the 'last viewed' timestamp is less than the oldest snapshot, return oldest
-        return sorted_keys[-1]
+        for k in sorted_keys:
+            if int(k) < last_viewed:
+                if prev_k == sorted_keys[0]:
+                    # Return the second last one so we dont recommend the same version compares itself
+                    return sorted_keys[1]
+
+                return prev_k
+            prev_k = k
+
+        return keys[0]
 
     def get_history_snapshot(self, timestamp):
         import brotli
@@ -313,10 +326,13 @@ class model(watch_base):
     def save_history_text(self, contents, timestamp, snapshot_id):
         import brotli
 
-        logger.trace(f"{self.get('uuid')} - Updating history.txt with timestamp {timestamp}")
-
         self.ensure_data_dir_exists()
 
+        # Small hack so that we sleep just enough to allow 1 second  between history snapshots
+        # this is because history.txt indexes/keys snapshots by epoch seconds and we dont want dupe keys
+        if self.__newest_history_key and int(timestamp) == int(self.__newest_history_key):
+            time.sleep(timestamp - self.__newest_history_key)
+
         threshold = int(os.getenv('SNAPSHOT_BROTLI_COMPRESSION_THRESHOLD', 1024))
         skip_brotli = strtobool(os.getenv('DISABLE_BROTLI_TEXT_SNAPSHOT', 'False'))
 
@@ -325,13 +341,13 @@ class model(watch_base):
             dest = os.path.join(self.watch_data_dir, snapshot_fname)
             if not os.path.exists(dest):
                 with open(dest, 'wb') as f:
-                    f.write(brotli.compress(contents.encode('utf-8'), mode=brotli.MODE_TEXT))
+                    f.write(brotli.compress(contents, mode=brotli.MODE_TEXT))
         else:
             snapshot_fname = f"{snapshot_id}.txt"
             dest = os.path.join(self.watch_data_dir, snapshot_fname)
             if not os.path.exists(dest):
                 with open(dest, 'wb') as f:
-                    f.write(contents.encode('utf-8'))
+                    f.write(contents)
 
         # Append to index
         # @todo check last char was \n
@@ -362,32 +378,14 @@ class model(watch_base):
         return seconds
 
     # Iterate over all history texts and see if something new exists
-    # Always applying .strip() to start/end but optionally replace any other whitespace
-    def lines_contain_something_unique_compared_to_history(self, lines: list, ignore_whitespace=False):
-        local_lines = set([])
-        if lines:
-            if ignore_whitespace:
-                if isinstance(lines[0], str): # Can be either str or bytes depending on what was on the disk
-                    local_lines = set([l.translate(TRANSLATE_WHITESPACE_TABLE).lower() for l in lines])
-                else:
-                    local_lines = set([l.decode('utf-8').translate(TRANSLATE_WHITESPACE_TABLE).lower() for l in lines])
-            else:
-                if isinstance(lines[0], str): # Can be either str or bytes depending on what was on the disk
-                    local_lines = set([l.strip().lower() for l in lines])
-                else:
-                    local_lines = set([l.decode('utf-8').strip().lower() for l in lines])
-
+    def lines_contain_something_unique_compared_to_history(self, lines: list):
+        local_lines = set([l.decode('utf-8').strip().lower() for l in lines])
 
         # Compare each lines (set) against each history text file (set) looking for something new..
         existing_history = set({})
         for k, v in self.history.items():
             content = self.get_history_snapshot(k)
-
-            if ignore_whitespace:
-                alist = set([line.translate(TRANSLATE_WHITESPACE_TABLE).lower() for line in content.splitlines()])
-            else:
-                alist = set([line.strip().lower() for line in content.splitlines()])
-
+            alist = set([line.strip().lower() for line in content.splitlines()])
             existing_history = existing_history.union(alist)
 
         # Check that everything in local_lines(new stuff) already exists in existing_history - it should
@@ -431,8 +429,8 @@ class model(watch_base):
     @property
     def watch_data_dir(self):
         # The base dir of the watch data
-        return os.path.join(self.__datastore_path, self['uuid']) if self.__datastore_path else None
-
+        return os.path.join(self.__datastore_path, self['uuid'])
+    
     def get_error_text(self):
         """Return the text saved from a previous request that resulted in a non-200 error"""
         fname = os.path.join(self.watch_data_dir, "last-error.txt")
@@ -467,17 +465,6 @@ class model(watch_base):
     def toggle_mute(self):
         self['notification_muted'] ^= True
 
-    def extra_notification_token_values(self):
-        # Used for providing extra tokens
-        # return {'widget': 555}
-        return {}
-
-    def extra_notification_token_placeholder_info(self):
-        # Used for providing extra tokens
-        # return [('widget', "Get widget amounts")]
-        return []
-
-
     def extract_regex_from_all_history(self, regex):
         import csv
         import re
@@ -536,43 +523,8 @@ class model(watch_base):
         # None is set
         return False
 
-    def save_error_text(self, contents):
-        self.ensure_data_dir_exists()
-        target_path = os.path.join(self.watch_data_dir, "last-error.txt")
-        with open(target_path, 'w', encoding='utf-8') as f:
-            f.write(contents)
 
-    def save_xpath_data(self, data, as_error=False):
-        import json
-        import zlib
-
-        if as_error:
-            target_path = os.path.join(str(self.watch_data_dir), "elements-error.deflate")
-        else:
-            target_path = os.path.join(str(self.watch_data_dir), "elements.deflate")
-
-        self.ensure_data_dir_exists()
-
-        with open(target_path, 'wb') as f:
-            f.write(zlib.compress(json.dumps(data).encode()))
-            f.close()
-
-    # Save as PNG, PNG is larger but better for doing visual diff in the future
-    def save_screenshot(self, screenshot: bytes, as_error=False):
-
-        if as_error:
-            target_path = os.path.join(self.watch_data_dir, "last-error-screenshot.png")
-        else:
-            target_path = os.path.join(self.watch_data_dir, "last-screenshot.png")
-
-        self.ensure_data_dir_exists()
-
-        with open(target_path, 'wb') as f:
-            f.write(screenshot)
-            f.close()
-
-
-    def get_last_fetched_text_before_filters(self):
+    def get_last_fetched_before_filters(self):
         import brotli
         filepath = os.path.join(self.watch_data_dir, 'last-fetched.br')
 
@@ -587,56 +539,12 @@ class model(watch_base):
         with open(filepath, 'rb') as f:
             return(brotli.decompress(f.read()).decode('utf-8'))
 
-    def save_last_text_fetched_before_filters(self, contents):
+    def save_last_fetched_before_filters(self, contents):
         import brotli
         filepath = os.path.join(self.watch_data_dir, 'last-fetched.br')
         with open(filepath, 'wb') as f:
             f.write(brotli.compress(contents, mode=brotli.MODE_TEXT))
 
-    def save_last_fetched_html(self, timestamp, contents):
-        import brotli
-
-        self.ensure_data_dir_exists()
-        snapshot_fname = f"{timestamp}.html.br"
-        filepath = os.path.join(self.watch_data_dir, snapshot_fname)
-
-        with open(filepath, 'wb') as f:
-            contents = contents.encode('utf-8') if isinstance(contents, str) else contents
-            try:
-                f.write(brotli.compress(contents))
-            except Exception as e:
-                logger.warning(f"{self.get('uuid')} - Unable to compress snapshot, saving as raw data to {filepath}")
-                logger.warning(e)
-                f.write(contents)
-
-        self._prune_last_fetched_html_snapshots()
-
-    def get_fetched_html(self, timestamp):
-        import brotli
-
-        snapshot_fname = f"{timestamp}.html.br"
-        filepath = os.path.join(self.watch_data_dir, snapshot_fname)
-        if os.path.isfile(filepath):
-            with open(filepath, 'rb') as f:
-                return (brotli.decompress(f.read()).decode('utf-8'))
-
-        return False
-
-
-    def _prune_last_fetched_html_snapshots(self):
-
-        dates = list(self.history.keys())
-        dates.reverse()
-
-        for index, timestamp in enumerate(dates):
-            snapshot_fname = f"{timestamp}.html.br"
-            filepath = os.path.join(self.watch_data_dir, snapshot_fname)
-
-            # Keep only the first 2
-            if index > 1 and os.path.isfile(filepath):
-                os.remove(filepath)
-
-
     @property
     def get_browsersteps_available_screenshots(self):
         "For knowing which screenshots are available to show the user in BrowserSteps UI"

changedetectionio/model/__init__.py

@@ -1,135 +0,0 @@
-import os
-import uuid
-
-from changedetectionio import strtobool
-from changedetectionio.notification import default_notification_format_for_watch
-
-class watch_base(dict):
-
-    def __init__(self, *arg, **kw):
-        self.update({
-            # Custom notification content
-            # Re #110, so then if this is set to None, we know to use the default value instead
-            # Requires setting to None on submit if it's the same as the default
-            # Should be all None by default, so we use the system default in this case.
-            'body': None,
-            'browser_steps': [],
-            'browser_steps_last_error_step': None,
-            'check_count': 0,
-            'check_unique_lines': False,  # On change-detected, compare against all history if its something new
-            'consecutive_filter_failures': 0,  # Every time the CSS/xPath filter cannot be located, reset when all is fine.
-            'content-type': None,
-            'date_created': None,
-            'extract_text': [],  # Extract text by regex after filters
-            'extract_title_as_title': False,
-            'fetch_backend': 'system',  # plaintext, playwright etc
-            'fetch_time': 0.0,
-            'filter_failure_notification_send': strtobool(os.getenv('FILTER_FAILURE_NOTIFICATION_SEND_DEFAULT', 'True')),
-            'filter_text_added': True,
-            'filter_text_removed': True,
-            'filter_text_replaced': True,
-            'follow_price_changes': True,
-            'has_ldjson_price_data': None,
-            'headers': {},  # Extra headers to send
-            'ignore_text': [],  # List of text to ignore when calculating the comparison checksum
-            'in_stock_only': True,  # Only trigger change on going to instock from out-of-stock
-            'include_filters': [],
-            'last_checked': 0,
-            'last_error': False,
-            'last_viewed': 0,  # history key value of the last viewed via the [diff] link
-            'method': 'GET',
-            'notification_alert_count': 0,
-            'notification_body': None,
-            'notification_format': default_notification_format_for_watch,
-            'notification_muted': False,
-            'notification_screenshot': False,  # Include the latest screenshot if available and supported by the apprise URL
-            'notification_title': None,
-            'notification_urls': [],  # List of URLs to add to the notification Queue (Usually AppRise)
-            'paused': False,
-            'previous_md5': False,
-            'previous_md5_before_filters': False,  # Used for skipping changedetection entirely
-            'processor': 'text_json_diff',  # could be restock_diff or others from .processors
-            'price_change_threshold_percent': None,
-            'proxy': None,  # Preferred proxy connection
-            'remote_server_reply': None,  # From 'server' reply header
-            'sort_text_alphabetically': False,
-            'subtractive_selectors': [],
-            'tag': '',  # Old system of text name for a tag, to be removed
-            'tags': [],  # list of UUIDs to App.Tags
-            'text_should_not_be_present': [],  # Text that should not present
-            'time_between_check': {'weeks': None, 'days': None, 'hours': None, 'minutes': None, 'seconds': None},
-            'time_between_check_use_default': True,
-            "time_schedule_limit": {
-                "enabled": False,
-                "monday": {
-                    "enabled": True,
-                    "start_time": "00:00",
-                    "duration": {
-                        "hours": "24",
-                        "minutes": "00"
-                    }
-                },
-                "tuesday": {
-                    "enabled": True,
-                    "start_time": "00:00",
-                    "duration": {
-                        "hours": "24",
-                        "minutes": "00"
-                    }
-                },
-                "wednesday": {
-                    "enabled": True,
-                    "start_time": "00:00",
-                    "duration": {
-                        "hours": "24",
-                        "minutes": "00"
-                    }
-                },
-                "thursday": {
-                    "enabled": True,
-                    "start_time": "00:00",
-                    "duration": {
-                        "hours": "24",
-                        "minutes": "00"
-                    }
-                },
-                "friday": {
-                    "enabled": True,
-                    "start_time": "00:00",
-                    "duration": {
-                        "hours": "24",
-                        "minutes": "00"
-                    }
-                },
-                "saturday": {
-                    "enabled": True,
-                    "start_time": "00:00",
-                    "duration": {
-                        "hours": "24",
-                        "minutes": "00"
-                    }
-                },
-                "sunday": {
-                    "enabled": True,
-                    "start_time": "00:00",
-                    "duration": {
-                        "hours": "24",
-                        "minutes": "00"
-                    }
-                },
-            },
-            'title': None,
-            'track_ldjson_price_data': None,
-            'trim_text_whitespace': False,
-            'remove_duplicate_lines': False,
-            'trigger_text': [],  # List of text or regex to wait for until a change is detected
-            'url': '',
-            'uuid': str(uuid.uuid4()),
-            'webdriver_delay': None,
-            'webdriver_js_execute_code': None,  # Run before change-detection
-        })
-
-        super(watch_base, self).__init__(*arg, **kw)
-
-        if self.get('default'):
-            del self['default']

changedetectionio/notification.py

@@ -1,10 +1,10 @@
-
-import time
-from apprise import NotifyFormat
 import apprise
+import time
+from jinja2 import Environment, BaseLoader
+from apprise import NotifyFormat
+import json
 from loguru import logger
 
-
 valid_tokens = {
     'base_url': '',
     'current_snapshot': '',
@@ -23,7 +23,7 @@ valid_tokens = {
 }
 
 default_notification_format_for_watch = 'System default'
-default_notification_format = 'HTML Color'
+default_notification_format = 'Text'
 default_notification_body = '{{watch_url}} had a change.\n---\n{{diff}}\n---\n'
 default_notification_title = 'ChangeDetection.io Notification - {{watch_url}}'
 
@@ -31,24 +31,101 @@ valid_notification_formats = {
     'Text': NotifyFormat.TEXT,
     'Markdown': NotifyFormat.MARKDOWN,
     'HTML': NotifyFormat.HTML,
-    'HTML Color': 'htmlcolor',
     # Used only for editing a watch (not for global)
     default_notification_format_for_watch: default_notification_format_for_watch
 }
 
+# include the decorator
+from apprise.decorators import notify
+
+@notify(on="delete")
+@notify(on="deletes")
+@notify(on="get")
+@notify(on="gets")
+@notify(on="post")
+@notify(on="posts")
+@notify(on="put")
+@notify(on="puts")
+def apprise_custom_api_call_wrapper(body, title, notify_type, *args, **kwargs):
+    import requests
+    from apprise.utils import parse_url as apprise_parse_url
+    from apprise.URLBase import URLBase
+
+    url = kwargs['meta'].get('url')
+
+    if url.startswith('post'):
+        r = requests.post
+    elif url.startswith('get'):
+        r = requests.get
+    elif url.startswith('put'):
+        r = requests.put
+    elif url.startswith('delete'):
+        r = requests.delete
+
+    url = url.replace('post://', 'http://')
+    url = url.replace('posts://', 'https://')
+    url = url.replace('put://', 'http://')
+    url = url.replace('puts://', 'https://')
+    url = url.replace('get://', 'http://')
+    url = url.replace('gets://', 'https://')
+    url = url.replace('put://', 'http://')
+    url = url.replace('puts://', 'https://')
+    url = url.replace('delete://', 'http://')
+    url = url.replace('deletes://', 'https://')
+
+    headers = {}
+    params = {}
+    auth = None
+
+    # Convert /foobar?+some-header=hello to proper header dictionary
+    results = apprise_parse_url(url)
+    if results:
+        # Add our headers that the user can potentially over-ride if they wish
+        # to to our returned result set and tidy entries by unquoting them
+        headers = {URLBase.unquote(x): URLBase.unquote(y)
+                   for x, y in results['qsd+'].items()}
+
+        # https://github.com/caronc/apprise/wiki/Notify_Custom_JSON#get-parameter-manipulation
+        # In Apprise, it relies on prefixing each request arg with "-", because it uses say &method=update as a flag for apprise
+        # but here we are making straight requests, so we need todo convert this against apprise's logic
+        for k, v in results['qsd'].items():
+            if not k.strip('+-') in results['qsd+'].keys():
+                params[URLBase.unquote(k)] = URLBase.unquote(v)
+
+        # Determine Authentication
+        auth = ''
+        if results.get('user') and results.get('password'):
+            auth = (URLBase.unquote(results.get('user')), URLBase.unquote(results.get('user')))
+        elif results.get('user'):
+            auth = (URLBase.unquote(results.get('user')))
+
+    # Try to auto-guess if it's JSON
+    try:
+        json.loads(body)
+        headers['Content-Type'] = 'application/json; charset=utf-8'
+    except ValueError as e:
+        pass
+
+    r(results.get('url'),
+      auth=auth,
+      data=body,
+      headers=headers,
+      params=params
+      )
 
 
 def process_notification(n_object, datastore):
-    # so that the custom endpoints are registered
-    from changedetectionio.apprise_plugin import apprise_custom_api_call_wrapper
 
-    from .safe_jinja import render as jinja_render
     now = time.time()
     if n_object.get('notification_timestamp'):
         logger.trace(f"Time since queued {now-n_object['notification_timestamp']:.3f}s")
     # Insert variables into the notification content
     notification_parameters = create_notification_parameters(n_object, datastore)
 
+    # Get the notification body from datastore
+    jinja2_env = Environment(loader=BaseLoader)
+    n_body = jinja2_env.from_string(n_object.get('notification_body', '')).render(**notification_parameters)
+    n_title = jinja2_env.from_string(n_object.get('notification_title', '')).render(**notification_parameters)
     n_format = valid_notification_formats.get(
         n_object.get('notification_format', default_notification_format),
         valid_notification_formats[default_notification_format],
@@ -67,10 +144,6 @@ def process_notification(n_object, datastore):
 
     sent_objs = []
     from .apprise_asset import asset
-
-    if 'as_async' in n_object:
-        asset.async_mode = n_object.get('as_async')
-
     apobj = apprise.Apprise(debug=True, asset=asset)
 
     if not n_object.get('notification_urls'):
@@ -78,25 +151,13 @@ def process_notification(n_object, datastore):
 
     with apprise.LogCapture(level=apprise.logging.DEBUG) as logs:
         for url in n_object['notification_urls']:
-
-            # Get the notification body from datastore
-            n_body = jinja_render(template_str=n_object.get('notification_body', ''), **notification_parameters)
-            if n_object.get('notification_format', '').startswith('HTML'):
-                n_body = n_body.replace("\n", '<br>')
-
-            n_title = jinja_render(template_str=n_object.get('notification_title', ''), **notification_parameters)
-
             url = url.strip()
-            if url.startswith('#'):
-                logger.trace(f"Skipping commented out notification URL - {url}")
-                continue
-
             if not url:
                 logger.warning(f"Process Notification: skipping empty notification URL.")
                 continue
 
-            logger.info(f">> Process Notification: AppRise notifying {url}")
-            url = jinja_render(template_str=url, **notification_parameters)
+            logger.info(">> Process Notification: AppRise notifying {}".format(url))
+            url = jinja2_env.from_string(url).render(**notification_parameters)
 
             # Re 323 - Limit discord length to their 2000 char limit total or it wont send.
             # Because different notifications may require different pre-processing, run each sequentially :(
@@ -161,12 +222,13 @@ def process_notification(n_object, datastore):
             attach=n_object.get('screenshot', None)
         )
 
+        # Give apprise time to register an error
+        time.sleep(3)
 
         # Returns empty string if nothing found, multi-line string otherwise
         log_value = logs.getvalue()
 
         if log_value and 'WARNING' in log_value or 'ERROR' in log_value:
-            logger.critical(log_value)
             raise Exception(log_value)
 
     # Return what was sent for better logging - after the for loop
@@ -209,18 +271,19 @@ def create_notification_parameters(n_object, datastore):
     tokens.update(
         {
             'base_url': base_url,
+            'current_snapshot': n_object.get('current_snapshot', ''),
+            'diff': n_object.get('diff', ''),  # Null default in the case we use a test
+            'diff_added': n_object.get('diff_added', ''),  # Null default in the case we use a test
+            'diff_full': n_object.get('diff_full', ''),  # Null default in the case we use a test
+            'diff_patch': n_object.get('diff_patch', ''),  # Null default in the case we use a test
+            'diff_removed': n_object.get('diff_removed', ''),  # Null default in the case we use a test
             'diff_url': diff_url,
             'preview_url': preview_url,
+            'triggered_text': n_object.get('triggered_text', ''),
             'watch_tag': watch_tag if watch_tag is not None else '',
             'watch_title': watch_title if watch_title is not None else '',
             'watch_url': watch_url,
             'watch_uuid': uuid,
         })
 
-    # n_object will contain diff, diff_added etc etc
-    tokens.update(n_object)
-
-    if uuid:
-        tokens.update(datastore.data['watching'].get(uuid).extra_notification_token_values())
-
     return tokens

changedetectionio/processors/README.md

@@ -8,8 +8,4 @@ The concept here is to be able to switch between different domain specific probl
 Some suggestions for the future
 
 - `graphical` 
-
-## Todo
-
-- Make each processor return a extra list of sub-processed (so you could configure a single processor in different ways)
-- move restock_diff to its own pip/github repo
+- `restock_and_price` - extract price AND stock text

changedetectionio/processors/__init__.py

@@ -1,19 +1,10 @@
 from abc import abstractmethod
-from changedetectionio.content_fetchers.base import Fetcher
-from changedetectionio.strtobool import strtobool
-from copy import deepcopy
-from loguru import logger
-
-import hashlib
-import importlib
-import inspect
 import os
-import pkgutil
+import hashlib
 import re
-
-# Import the plugin manager
-from .pluggy_interface import plugin_manager
-
+from copy import deepcopy
+from distutils.util import strtobool
+from loguru import logger
 
 class difference_detection_processor():
 
@@ -23,119 +14,28 @@ class difference_detection_processor():
     screenshot = None
     watch = None
     xpath_data = None
-    preferred_proxy = None
 
     def __init__(self, *args, datastore, watch_uuid, **kwargs):
         super().__init__(*args, **kwargs)
         self.datastore = datastore
         self.watch = deepcopy(self.datastore.data['watching'].get(watch_uuid))
-        # Generic fetcher that should be extended (requests, playwright etc)
-        self.fetcher = Fetcher()
-        
-    def _get_proxy_for_watch(self, preferred_proxy_id=None):
-        """Get proxy configuration based on watch settings and preferred proxy ID
-        
-        Args:
-            preferred_proxy_id: Optional explicit proxy ID to use
-            
-        Returns:
-            dict: Proxy configuration or None if no proxy should be used
-            str: Proxy URL or None if no proxy should be used
-        """
-        # Default to no proxy config
-        proxy_config = None
-        proxy_url = None
-        
-        # Check if datastore is available and has get_preferred_proxy_for_watch method
-        if hasattr(self, 'datastore') and self.datastore:
-            try:
-                # Get preferred proxy ID if not provided
-                if not preferred_proxy_id and hasattr(self.datastore, 'get_preferred_proxy_for_watch'):
-                    # Get the watch UUID if available
-                    watch_uuid = None
-                    if hasattr(self.watch, 'get'):
-                        watch_uuid = self.watch.get('uuid')
-                    elif hasattr(self.watch, 'uuid'):
-                        watch_uuid = self.watch.uuid
-                    
-                    if watch_uuid:
-                        preferred_proxy_id = self.datastore.get_preferred_proxy_for_watch(uuid=watch_uuid)
-                
-                # Check if we have a proxy list and a valid proxy ID
-                if preferred_proxy_id and hasattr(self.datastore, 'proxy_list') and self.datastore.proxy_list:
-                    proxy_info = self.datastore.proxy_list.get(preferred_proxy_id)
-                    
-                    if proxy_info and 'url' in proxy_info:
-                        proxy_url = proxy_info.get('url')
-                        logger.debug(f"Selected proxy key '{preferred_proxy_id}' as proxy URL '{proxy_url}'")
-                        
-                        # Parse the proxy URL to build a proxy dict for requests
-                        import urllib.parse
-                        parsed_proxy = urllib.parse.urlparse(proxy_url)
-                        proxy_type = parsed_proxy.scheme
-                        
-                        # Extract credentials if present
-                        username = None
-                        password = None
-                        if parsed_proxy.username:
-                            username = parsed_proxy.username
-                            if parsed_proxy.password:
-                                password = parsed_proxy.password
-                        
-                        # Build the proxy URL without credentials for the proxy dict
-                        netloc = parsed_proxy.netloc
-                        if '@' in netloc:
-                            netloc = netloc.split('@')[1]
-                        
-                        proxy_addr = f"{proxy_type}://{netloc}"
-                        
-                        # Create the proxy configuration
-                        proxy_config = {
-                            'http': proxy_addr,
-                            'https': proxy_addr
-                        }
-                        
-                        # Add credentials if present
-                        if username:
-                            proxy_config['username'] = username
-                            if password:
-                                proxy_config['password'] = password
-            except Exception as e:
-                # Log the error but continue without a proxy
-                logger.error(f"Error setting up proxy: {str(e)}")
-                proxy_config = None
-                proxy_url = None
-                
-        return proxy_config, proxy_url
 
-    def call_browser(self, preferred_proxy_id=None):
-        """Fetch content using the appropriate browser/fetcher
-        
-        This method will:
-        1. Determine the appropriate fetcher to use based on watch settings
-        2. Set up proxy configuration if needed
-        3. Initialize the fetcher with the correct parameters
-        4. Configure any browser steps if needed
-        
-        Args:
-            preferred_proxy_id: Optional explicit proxy ID to use
-        """
-        from requests.structures import CaseInsensitiveDict
+    def call_browser(self):
 
-        url = self.watch.link
-
-        # Protect against file:, file:/, file:// access, check the real "link" without any meta "source:" etc prepended.
-        if re.search(r'^file:', url.strip(), re.IGNORECASE):
+        # Protect against file:// access
+        if re.search(r'^file://', self.watch.get('url', '').strip(), re.IGNORECASE):
             if not strtobool(os.getenv('ALLOW_FILE_URI', 'false')):
                 raise Exception(
                     "file:// type access is denied for security reasons."
                 )
 
+        url = self.watch.link
+
         # Requests, playwright, other browser via wss:// etc, fetch_extra_something
         prefer_fetch_backend = self.watch.get('fetch_backend', 'system')
 
-        # Get proxy configuration
-        proxy_config, proxy_url = self._get_proxy_for_watch(preferred_proxy_id)
+        # Proxy ID "key"
+        preferred_proxy_id = self.datastore.get_preferred_proxy_for_watch(uuid=self.watch.get('uuid'))
 
         # Pluggable content self.fetcher
         if not prefer_fetch_backend or prefer_fetch_backend == 'system':
@@ -173,10 +73,14 @@ class difference_detection_processor():
             # What it referenced doesnt exist, Just use a default
             fetcher_obj = getattr(content_fetchers, "html_requests")
 
-        # Custom browser endpoints should NOT have a proxy added
-        if proxy_url and prefer_fetch_backend.startswith('extra_browser_'):
-            logger.debug(f"Skipping adding proxy data when custom Browser endpoint is specified.")
-            proxy_url = None
+        proxy_url = None
+        if preferred_proxy_id:
+            # Custom browser endpoints should NOT have a proxy added
+            if not prefer_fetch_backend.startswith('extra_browser_'):
+                proxy_url = self.datastore.proxy_list.get(preferred_proxy_id).get('url')
+                logger.debug(f"Selected proxy key '{preferred_proxy_id}' as proxy URL '{proxy_url}' for {url}")
+            else:
+                logger.debug(f"Skipping adding proxy data when custom Browser endpoint is specified. ")
 
         # Now call the fetcher (playwright/requests/etc) with arguments that only a fetcher would need.
         # When browser_connection_url is None, it method should default to working out whats the best defaults (os env vars etc)
@@ -189,14 +93,7 @@ class difference_detection_processor():
             self.fetcher.browser_steps_screenshot_path = os.path.join(self.datastore.datastore_path, self.watch.get('uuid'))
 
         # Tweak the base config with the per-watch ones
-        from changedetectionio.safe_jinja import render as jinja_render
-        request_headers = CaseInsensitiveDict()
-
-        ua = self.datastore.data['settings']['requests'].get('default_ua')
-        if ua and ua.get(prefer_fetch_backend):
-            request_headers.update({'User-Agent': ua.get(prefer_fetch_backend)})
-
-        request_headers.update(self.watch.get('headers', {}))
+        request_headers = self.watch.get('headers', [])
         request_headers.update(self.datastore.get_all_base_headers())
         request_headers.update(self.datastore.get_all_headers_in_textfile_for_watch(uuid=self.watch.get('uuid')))
 
@@ -206,15 +103,9 @@ class difference_detection_processor():
         if 'Accept-Encoding' in request_headers and "br" in request_headers['Accept-Encoding']:
             request_headers['Accept-Encoding'] = request_headers['Accept-Encoding'].replace(', br', '')
 
-        for header_name in request_headers:
-            request_headers.update({header_name: jinja_render(template_str=request_headers.get(header_name))})
-
         timeout = self.datastore.data['settings']['requests'].get('timeout')
 
         request_body = self.watch.get('body')
-        if request_body:
-            request_body = jinja_render(template_str=self.watch.get('body'))
-        
         request_method = self.watch.get('method')
         ignore_status_codes = self.watch.get('ignore_status_codes', False)
 
@@ -232,18 +123,8 @@ class difference_detection_processor():
         is_binary = self.watch.is_pdf
 
         # And here we go! call the right browser with browser-specific settings
-        empty_pages_are_a_change = self.datastore.data['settings']['application'].get('empty_pages_are_a_change', False)
-
-        self.fetcher.run(url=url,
-                         timeout=timeout,
-                         request_headers=request_headers,
-                         request_body=request_body,
-                         request_method=request_method,
-                         ignore_status_codes=ignore_status_codes,
-                         current_include_filters=self.watch.get('include_filters'),
-                         is_binary=is_binary,
-                         empty_pages_are_a_change=empty_pages_are_a_change
-                         )
+        self.fetcher.run(url, timeout, request_headers, request_body, request_method, ignore_status_codes, self.watch.get('include_filters'),
+                    is_binary=is_binary)
 
         #@todo .quit here could go on close object, so we can run JS if change-detected
         self.fetcher.quit()
@@ -251,7 +132,7 @@ class difference_detection_processor():
         # After init, call run_changedetection() which will do the actual change-detection
 
     @abstractmethod
-    def run_changedetection(self, watch):
+    def run_changedetection(self, uuid, skip_when_checksum_same=True):
         update_obj = {'last_notification_error': False, 'last_error': False}
         some_data = 'xxxxx'
         update_obj["previous_md5"] = hashlib.md5(some_data.encode('utf-8')).hexdigest()
@@ -259,140 +140,8 @@ class difference_detection_processor():
         return changed_detected, update_obj, ''.encode('utf-8')
 
 
-def find_sub_packages(package_name):
-    """
-    Find all sub-packages within the given package.
-
-    :param package_name: The name of the base package to scan for sub-packages.
-    :return: A list of sub-package names.
-    """
-    package = importlib.import_module(package_name)
-    return [name for _, name, is_pkg in pkgutil.iter_modules(package.__path__) if is_pkg]
-
-
-def find_processors():
-    """
-    Find all subclasses of DifferenceDetectionProcessor in the specified package
-    and also include processors from the plugin system.
-
-    :return: A list of (module, class) tuples.
-    """
-    package_name = "changedetectionio.processors"  # Name of the current package/module
-
-    processors = []
-    sub_packages = find_sub_packages(package_name)
-
-    # Find traditional processors
-    for sub_package in sub_packages:
-        module_name = f"{package_name}.{sub_package}.processor"
-        try:
-            module = importlib.import_module(module_name)
-
-            # Iterate through all classes in the module
-            for name, obj in inspect.getmembers(module, inspect.isclass):
-                if issubclass(obj, difference_detection_processor) and obj is not difference_detection_processor:
-                    processors.append((module, sub_package))
-        except (ModuleNotFoundError, ImportError) as e:
-            logger.warning(f"Failed to import module {module_name}: {e} (find_processors())")
-
-    # Also include processors from the plugin system
-    try:
-        from .processor_registry import get_plugin_processor_modules
-        plugin_modules = get_plugin_processor_modules()
-        if plugin_modules:
-            processors.extend(plugin_modules)
-    except (ImportError, ModuleNotFoundError) as e:
-        logger.warning(f"Failed to import plugin modules: {e} (find_processors())")
-
-    return processors
-
-
-def get_parent_module(module):
-    module_name = module.__name__
-    if '.' not in module_name:
-        return None  # Top-level module has no parent
-    parent_module_name = module_name.rsplit('.', 1)[0]
-    try:
-        return importlib.import_module(parent_module_name)
-    except Exception as e:
-        pass
-
-    return False
-
-
-def get_custom_watch_obj_for_processor(processor_name):
-    """
-    Get the custom watch object for a processor
-    :param processor_name: Name of the processor
-    :return: Watch class or None
-    """
-    # First, try to get the watch model from the pluggy system
-    try:
-        from .processor_registry import get_processor_watch_model
-        watch_model = get_processor_watch_model(processor_name)
-        if watch_model:
-            return watch_model
-    except Exception as e:
-        logger.warning(f"Error getting processor watch model from pluggy: {e}")
-
-    # Fall back to the traditional approach
-    from changedetectionio.model import Watch
-    watch_class = Watch.model
-    processor_classes = find_processors()
-    custom_watch_obj = next((tpl for tpl in processor_classes if tpl[1] == processor_name), None)
-    if custom_watch_obj:
-        # Parent of .processor.py COULD have its own Watch implementation
-        parent_module = get_parent_module(custom_watch_obj[0])
-        if hasattr(parent_module, 'Watch'):
-            watch_class = parent_module.Watch
-
-    return watch_class
-
-
 def available_processors():
-    """
-    Get a list of processors by name and description for the UI elements
-    :return: A list of tuples (processor_name, description)
-    """
-    # Get processors from the pluggy system
-    pluggy_processors = []
-    try:
-        from .processor_registry import get_all_processors
-        pluggy_processors = get_all_processors()
-    except Exception as e:
-        logger.error(f"Error getting processors from pluggy: {str(e)}")
-    
-    # Get processors from the traditional file-based system
-    traditional_processors = []
-    try:
-        # Let's not use find_processors() directly since it now also includes pluggy processors
-        package_name = "changedetectionio.processors"
-        sub_packages = find_sub_packages(package_name)
-        
-        for sub_package in sub_packages:
-            module_name = f"{package_name}.{sub_package}.processor"
-            try:
-                module = importlib.import_module(module_name)
-                # Get the name and description from the module if available
-                name = getattr(module, 'name', f"Traditional processor: {sub_package}")
-                description = getattr(module, 'description', sub_package)
-                traditional_processors.append((sub_package, name))
-            except (ModuleNotFoundError, ImportError, AttributeError) as e:
-                logger.warning(f"Failed to import module {module_name}: {e} (available_processors())")
-    except Exception as e:
-        logger.error(f"Error getting traditional processors: {str(e)}")
-    
-    # Combine the lists, ensuring no duplicates
-    # Pluggy processors take precedence
-    all_processors = []
-    
-    # Add all pluggy processors
-    all_processors.extend(pluggy_processors)
-    
-    # Add traditional processors that aren't already registered via pluggy
-    pluggy_processor_names = [name for name, _ in pluggy_processors]
-    for processor_class, name in traditional_processors:
-        if processor_class not in pluggy_processor_names:
-            all_processors.append((processor_class, name))
-    
-    return all_processors
+    from . import restock_diff, text_json_diff
+    x=[('text_json_diff', text_json_diff.name), ('restock_diff', restock_diff.name)]
+    # @todo Make this smarter with introspection of sorts.
+    return x

changedetectionio/processors/exceptions.py

@@ -1,10 +0,0 @@
-class ProcessorException(Exception):
-    def __init__(self, message=None, status_code=None, url=None, screenshot=None, has_filters=False, html_content='', xpath_data=None):
-        self.message = message
-        self.status_code = status_code
-        self.url = url
-        self.screenshot = screenshot
-        self.has_filters = has_filters
-        self.html_content = html_content
-        self.xpath_data = xpath_data
-        return

changedetectionio/processors/form.py

@@ -1,17 +0,0 @@
-from wtforms import (
-    BooleanField,
-    validators,
-    RadioField
-)
-from wtforms.fields.choices import SelectField
-from wtforms.fields.form import FormField
-from wtforms.form import Form
-
-class BaseProcessorForm(Form):
-    """Base class for processor forms"""
-    
-    def extra_tab_content(self):
-        return None
-
-    def extra_form_content(self):
-        return None

changedetectionio/processors/forms.py

@@ -1,4 +0,0 @@
-"""
-Forms for processors
-"""
-from changedetectionio.forms import processor_text_json_diff_form

changedetectionio/processors/pluggy_interface.py

@@ -1,69 +0,0 @@
-import pluggy
-
-# Define the plugin namespace for processors
-PLUGIN_NAMESPACE = "changedetectionio_processors"
-
-hookspec = pluggy.HookspecMarker(PLUGIN_NAMESPACE)
-hookimpl = pluggy.HookimplMarker(PLUGIN_NAMESPACE)
-
-
-class ProcessorSpec:
-    """Hook specifications for processor plugins."""
-
-    @hookspec
-    def get_processor_name():
-        """Return the name of the processor."""
-        pass
-
-    @hookspec
-    def get_processor_description():
-        """Return the description of the processor."""
-        pass
-    
-    @hookspec
-    def get_processor_class():
-        """Return the processor class."""
-        pass
-    
-    @hookspec
-    def get_processor_form():
-        """Return the processor form class."""
-        pass
-    
-    @hookspec
-    def get_processor_watch_model():
-        """Return the watch model class for this processor (if any)."""
-        pass
-    
-    @hookspec
-    def get_display_link(url, processor_name):
-        """Return a custom display link for the given processor.
-        
-        Args:
-            url: The original URL from the watch
-            processor_name: The name of the processor
-            
-        Returns:
-            A string with the custom display link or None to use the default
-        """
-        pass
-    
-    @hookspec
-    def perform_site_check(datastore, watch_uuid):
-        """Create and return a processor instance ready to perform site check.
-        
-        Args:
-            datastore: The application datastore
-            watch_uuid: The UUID of the watch to check
-            
-        Returns:
-            A processor instance ready to perform site check
-        """
-        pass
-
-
-# Set up the plugin manager
-plugin_manager = pluggy.PluginManager(PLUGIN_NAMESPACE)
-
-# Register hook specifications
-plugin_manager.add_hookspecs(ProcessorSpec)

changedetectionio/processors/processor_registry.py

@@ -1,222 +0,0 @@
-from loguru import logger
-from changedetectionio.model import Watch
-from .pluggy_interface import plugin_manager
-from typing import Dict, Any, List, Tuple, Optional, TypeVar, Type
-import functools
-
-# Import and register internal plugins
-from . import whois_plugin
-from . import test_plugin
-
-# Register plugins
-plugin_manager.register(whois_plugin)
-plugin_manager.register(test_plugin)
-
-# Load any setuptools entrypoints
-plugin_manager.load_setuptools_entrypoints("changedetectionio_processors")
-
-# Type definitions for better type hinting
-T = TypeVar('T')
-ProcessorClass = TypeVar('ProcessorClass')
-ProcessorForm = TypeVar('ProcessorForm')
-ProcessorWatchModel = TypeVar('ProcessorWatchModel')
-ProcessorInstance = TypeVar('ProcessorInstance')
-
-# Cache for plugin name mapping to improve performance
-_plugin_name_map: Dict[str, Any] = {}
-
-def register_plugin(plugin_module):
-    """Register a processor plugin"""
-    plugin_manager.register(plugin_module)
-    # Clear the plugin name map cache when a new plugin is registered
-    global _plugin_name_map
-    _plugin_name_map = {}
-
-def _get_plugin_name_map() -> Dict[str, Any]:
-    """Get a mapping of processor names to plugins
-    :return: Dictionary mapping processor names to plugins
-    """
-    global _plugin_name_map
-    
-    # Return cached map if available
-    if _plugin_name_map:
-        return _plugin_name_map
-    
-    # Build the map
-    result = {}
-    
-    # Get all plugins from the plugin manager
-    all_plugins = list(plugin_manager.get_plugins())
-    
-    # First register known internal plugins by name for reliability
-    known_plugins = {
-        'whois': whois_plugin,
-        'test': test_plugin
-    }
-    
-    for name, plugin in known_plugins.items():
-        if plugin in all_plugins:
-            result[name] = plugin
-    
-    # Then process remaining plugins through the hook system
-    for plugin in all_plugins:
-        if plugin in known_plugins.values():
-            continue  # Skip plugins we've already registered
-            
-        try:
-            # Get the processor name from this plugin
-            name_results = plugin_manager.hook.get_processor_name(plugin=plugin)
-            
-            if name_results:
-                plugin_name = name_results[0]
-                
-                # Check for name collisions
-                if plugin_name in result:
-                    logger.warning(f"Plugin name collision: '{plugin_name}' is already registered")
-                    continue
-                    
-                result[plugin_name] = plugin
-        except Exception as e:
-            logger.error(f"Error getting processor name from plugin: {str(e)}")
-    
-    # Cache the map
-    _plugin_name_map = result
-    return result
-
-def _get_plugin_by_name(processor_name: str) -> Optional[Any]:
-    """Get a plugin by its processor name
-    :param processor_name: Name of the processor
-    :return: Plugin object or None
-    """
-    return _get_plugin_name_map().get(processor_name)
-
-def _call_hook_for_plugin(plugin: Any, hook_name: str, default_value: T = None, **kwargs) -> Optional[T]:
-    """Call a hook for a specific plugin and handle exceptions
-    :param plugin: The plugin to call the hook for
-    :param hook_name: Name of the hook to call
-    :param default_value: Default value to return if the hook call fails
-    :param kwargs: Additional arguments to pass to the hook
-    :return: Result of the hook call or default value
-    """
-    if not plugin:
-        return default_value
-    
-    try:
-        hook = getattr(plugin_manager.hook, hook_name)
-        results = hook(plugin=plugin, **kwargs)
-        
-        if results:
-            return results[0]
-    except Exception as e:
-        logger.error(f"Error calling {hook_name} for plugin: {str(e)}")
-    
-    return default_value
-
-def get_all_processors() -> List[Tuple[str, str]]:
-    """Get all processors
-    :return: List of tuples (processor_name, processor_description)
-    """
-    processors = []
-    
-    for processor_name, plugin in _get_plugin_name_map().items():
-        description = _call_hook_for_plugin(plugin, 'get_processor_description')
-        if description:
-            processors.append((processor_name, description))
-    
-    return processors
-
-def get_processor_class(processor_name: str) -> Optional[Type[ProcessorClass]]:
-    """Get processor class by name
-    :param processor_name: Name of the processor
-    :return: Processor class or None
-    """
-    plugin = _get_plugin_by_name(processor_name)
-    return _call_hook_for_plugin(plugin, 'get_processor_class')
-
-def get_processor_form(processor_name: str) -> Optional[Type[ProcessorForm]]:
-    """Get processor form by name
-    :param processor_name: Name of the processor
-    :return: Processor form class or None
-    """
-    plugin = _get_plugin_by_name(processor_name)
-    return _call_hook_for_plugin(plugin, 'get_processor_form')
-
-def get_processor_watch_model(processor_name: str) -> Type[ProcessorWatchModel]:
-    """Get processor watch model by name
-    :param processor_name: Name of the processor
-    :return: Watch model class or default Watch model
-    """
-    plugin = _get_plugin_by_name(processor_name)
-    return _call_hook_for_plugin(plugin, 'get_processor_watch_model', default_value=Watch.model)
-
-def get_processor_site_check(processor_name: str, datastore: Any, watch_uuid: str) -> Optional[ProcessorInstance]:
-    """Get a processor instance ready to perform site check
-    :param processor_name: Name of the processor
-    :param datastore: The application datastore
-    :param watch_uuid: The UUID of the watch to check
-    :return: A processor instance ready to perform site check, or None
-    """
-    plugin = _get_plugin_by_name(processor_name)
-    if not plugin:
-        return None
-    
-    # Try to get the perform_site_check implementation
-    try:
-        processor = _call_hook_for_plugin(
-            plugin, 
-            'perform_site_check', 
-            datastore=datastore, 
-            watch_uuid=watch_uuid
-        )
-        if processor:
-            return processor
-        
-        # If no perform_site_check hook implementation, try getting the class and instantiating it
-        processor_class = _call_hook_for_plugin(plugin, 'get_processor_class')
-        if processor_class:
-            return processor_class(datastore=datastore, watch_uuid=watch_uuid)
-    except Exception as e:
-        logger.error(f"Error getting processor site check for {processor_name}: {str(e)}")
-    
-    return None
-
-def get_display_link(url: str, processor_name: str) -> Optional[str]:
-    """Get a custom display link for the given processor
-    :param url: The original URL from the watch
-    :param processor_name: Name of the processor
-    :return: A string with the custom display link or None to use the default
-    """
-    plugin = _get_plugin_by_name(processor_name)
-    return _call_hook_for_plugin(
-        plugin, 
-        'get_display_link', 
-        url=url, 
-        processor_name=processor_name
-    )
-
-def get_plugin_processor_modules() -> List[Tuple[Any, str]]:
-    """Get processor modules for all plugins that can be used with the find_processors function
-    
-    This function adapts pluggy plugins to be compatible with the traditional find_processors system
-    
-    :return: A list of (module, processor_name) tuples
-    """
-    result = []
-    
-    # Import base modules once to avoid repeated imports
-    from changedetectionio.processors.text_json_diff import processor as text_json_diff_processor
-
-    # For each plugin, map to a suitable module for find_processors
-    for processor_name, plugin in _get_plugin_name_map().items():
-        try:
-            processor_class = _call_hook_for_plugin(plugin, 'get_processor_class')
-            
-            if processor_class:
-                # Check if this processor extends the text_json_diff processor
-                base_class_name = str(processor_class.__bases__[0].__name__)
-                if base_class_name == 'perform_site_check' or 'TextJsonDiffProcessor' in base_class_name:
-                    result.append((text_json_diff_processor, processor_name))
-        except Exception as e:
-            logger.error(f"Error mapping processor module for {processor_name}: {str(e)}")
-    
-    return result

changedetectionio/processors/restock_diff.py

@@ -0,0 +1,66 @@
+
+from . import difference_detection_processor
+from copy import deepcopy
+from loguru import logger
+import hashlib
+import urllib3
+
+urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)
+
+name = 'Re-stock detection for single product pages'
+description = 'Detects if the product goes back to in-stock'
+
+class UnableToExtractRestockData(Exception):
+    def __init__(self, status_code):
+        # Set this so we can use it in other parts of the app
+        self.status_code = status_code
+        return
+
+class perform_site_check(difference_detection_processor):
+    screenshot = None
+    xpath_data = None
+
+    def run_changedetection(self, uuid, skip_when_checksum_same=True):
+
+        # DeepCopy so we can be sure we don't accidently change anything by reference
+        watch = deepcopy(self.datastore.data['watching'].get(uuid))
+
+        if not watch:
+            raise Exception("Watch no longer exists.")
+
+        # Unset any existing notification error
+        update_obj = {'last_notification_error': False, 'last_error': False}
+
+        self.screenshot = self.fetcher.screenshot
+        self.xpath_data = self.fetcher.xpath_data
+
+        # Track the content type
+        update_obj['content_type'] = self.fetcher.headers.get('Content-Type', '')
+        update_obj["last_check_status"] = self.fetcher.get_last_status_code()
+
+        # Main detection method
+        fetched_md5 = None
+        if self.fetcher.instock_data:
+            fetched_md5 = hashlib.md5(self.fetcher.instock_data.encode('utf-8')).hexdigest()
+            # 'Possibly in stock' comes from stock-not-in-stock.js when no string found above the fold.
+            update_obj["in_stock"] = True if self.fetcher.instock_data == 'Possibly in stock' else False
+            logger.debug(f"Watch UUID {uuid} restock check returned '{self.fetcher.instock_data}' from JS scraper.")
+        else:
+            raise UnableToExtractRestockData(status_code=self.fetcher.status_code)
+
+        # The main thing that all this at the moment comes down to :)
+        changed_detected = False
+        logger.debug(f"Watch UUID {uuid} restock check - Previous MD5: {watch.get('previous_md5')}, Fetched MD5 {fetched_md5}")
+
+        if watch.get('previous_md5') and watch.get('previous_md5') != fetched_md5:
+            # Yes if we only care about it going to instock, AND we are in stock
+            if watch.get('in_stock_only') and update_obj["in_stock"]:
+                changed_detected = True
+
+            if not watch.get('in_stock_only'):
+                # All cases
+                changed_detected = True
+
+        # Always record the new checksum
+        update_obj["previous_md5"] = fetched_md5
+        return changed_detected, update_obj, self.fetcher.instock_data.encode('utf-8').strip()

changedetectionio/processors/restock_diff/__init__.py

@@ -1,84 +0,0 @@
-
-from babel.numbers import parse_decimal
-from changedetectionio.model.Watch import model as BaseWatch
-from typing import Union
-import re
-
-class Restock(dict):
-
-    def parse_currency(self, raw_value: str) -> Union[float, None]:
-        # Clean and standardize the value (ie 1,400.00 should be 1400.00), even better would be store the whole thing as an integer.
-        standardized_value = raw_value
-
-        if ',' in standardized_value and '.' in standardized_value:
-            # Identify the correct decimal separator
-            if standardized_value.rfind('.') > standardized_value.rfind(','):
-                standardized_value = standardized_value.replace(',', '')
-            else:
-                standardized_value = standardized_value.replace('.', '').replace(',', '.')
-        else:
-            standardized_value = standardized_value.replace(',', '.')
-
-        # Remove any non-numeric characters except for the decimal point
-        standardized_value = re.sub(r'[^\d.-]', '', standardized_value)
-
-        if standardized_value:
-            # Convert to float
-            return float(parse_decimal(standardized_value, locale='en'))
-
-        return None
-
-    def __init__(self, *args, **kwargs):
-        # Define default values
-        default_values = {
-            'in_stock': None,
-            'price': None,
-            'currency': None,
-            'original_price': None
-        }
-
-        # Initialize the dictionary with default values
-        super().__init__(default_values)
-
-        # Update with any provided positional arguments (dictionaries)
-        if args:
-            if len(args) == 1 and isinstance(args[0], dict):
-                self.update(args[0])
-            else:
-                raise ValueError("Only one positional argument of type 'dict' is allowed")
-
-    def __setitem__(self, key, value):
-        # Custom logic to handle setting price and original_price
-        if key == 'price' or key == 'original_price':
-            if isinstance(value, str):
-                value = self.parse_currency(raw_value=value)
-
-        super().__setitem__(key, value)
-
-class Watch(BaseWatch):
-    def __init__(self, *arg, **kw):
-        super().__init__(*arg, **kw)
-        self['restock'] = Restock(kw['default']['restock']) if kw.get('default') and kw['default'].get('restock') else Restock()
-
-        self['restock_settings'] = kw['default']['restock_settings'] if kw.get('default',{}).get('restock_settings') else {
-            'follow_price_changes': True,
-            'in_stock_processing' : 'in_stock_only'
-        } #@todo update
-
-    def clear_watch(self):
-        super().clear_watch()
-        self.update({'restock': Restock()})
-
-    def extra_notification_token_values(self):
-        values = super().extra_notification_token_values()
-        values['restock'] = self.get('restock', {})
-        return values
-
-    def extra_notification_token_placeholder_info(self):
-        values = super().extra_notification_token_placeholder_info()
-
-        values.append(('restock.price', "Price detected"))
-        values.append(('restock.original_price', "Original price at first check"))
-
-        return values
-

changedetectionio/processors/restock_diff/forms.py

@@ -1,81 +0,0 @@
-from wtforms import (
-    BooleanField,
-    validators,
-    FloatField
-)
-from wtforms.fields.choices import RadioField
-from wtforms.fields.form import FormField
-from wtforms.form import Form
-
-from changedetectionio.forms import processor_text_json_diff_form
-
-
-class RestockSettingsForm(Form):
-    in_stock_processing = RadioField(label='Re-stock detection', choices=[
-        ('in_stock_only', "In Stock only (Out Of Stock -> In Stock only)"),
-        ('all_changes', "Any availability changes"),
-        ('off', "Off, don't follow availability/restock"),
-    ], default="in_stock_only")
-
-    price_change_min = FloatField('Below price to trigger notification', [validators.Optional()],
-                                  render_kw={"placeholder": "No limit", "size": "10"})
-    price_change_max = FloatField('Above price to trigger notification', [validators.Optional()],
-                                  render_kw={"placeholder": "No limit", "size": "10"})
-    price_change_threshold_percent = FloatField('Threshold in % for price changes since the original price', validators=[
-
-        validators.Optional(),
-        validators.NumberRange(min=0, max=100, message="Should be between 0 and 100"),
-    ], render_kw={"placeholder": "0%", "size": "5"})
-
-    follow_price_changes = BooleanField('Follow price changes', default=True)
-
-class processor_settings_form(processor_text_json_diff_form):
-    restock_settings = FormField(RestockSettingsForm)
-
-    def extra_tab_content(self):
-        return 'Restock & Price Detection'
-
-    def extra_form_content(self):
-        output = ""
-
-        if getattr(self, 'watch', None) and getattr(self, 'datastore'):
-            for tag_uuid in self.watch.get('tags'):
-                tag = self.datastore.data['settings']['application']['tags'].get(tag_uuid, {})
-                if tag.get('overrides_watch'):
-                    # @todo - Quick and dirty, cant access 'url_for' here because its out of scope somehow
-                    output = f"""<p><strong>Note! A Group tag overrides the restock and price detection here.</strong></p><style>#restock-fieldset-price-group {{ opacity: 0.6; }}</style>"""
-
-        output += """
-        {% from '_helpers.html' import render_field, render_checkbox_field, render_button %}
-        <script>        
-            $(document).ready(function () {
-                toggleOpacity('#restock_settings-follow_price_changes', '.price-change-minmax', true);
-            });
-        </script>
-
-        <fieldset id="restock-fieldset-price-group">
-            <div class="pure-control-group">
-                <fieldset class="pure-group inline-radio">
-                    {{ render_field(form.restock_settings.in_stock_processing) }}
-                </fieldset>
-                <fieldset class="pure-group">
-                    {{ render_checkbox_field(form.restock_settings.follow_price_changes) }}
-                    <span class="pure-form-message-inline">Changes in price should trigger a notification</span>
-                </fieldset>
-                <fieldset class="pure-group price-change-minmax">               
-                    {{ render_field(form.restock_settings.price_change_min, placeholder=watch.get('restock', {}).get('price')) }}
-                    <span class="pure-form-message-inline">Minimum amount, Trigger a change/notification when the price drops <i>below</i> this value.</span>
-                </fieldset>
-                <fieldset class="pure-group price-change-minmax">
-                    {{ render_field(form.restock_settings.price_change_max, placeholder=watch.get('restock', {}).get('price')) }}
-                    <span class="pure-form-message-inline">Maximum amount, Trigger a change/notification when the price rises <i>above</i> this value.</span>
-                </fieldset>
-                <fieldset class="pure-group price-change-minmax">
-                    {{ render_field(form.restock_settings.price_change_threshold_percent) }}
-                    <span class="pure-form-message-inline">Price must change more than this % to trigger a change since the first check.</span><br>
-                    <span class="pure-form-message-inline">For example, If the product is $1,000 USD originally, <strong>2%</strong> would mean it has to change more than $20 since the first check.</span><br>
-                </fieldset>                
-            </div>
-        </fieldset>
-        """
-        return output

changedetectionio/processors/restock_diff/processor.py

@@ -1,314 +0,0 @@
-from .. import difference_detection_processor
-from ..exceptions import ProcessorException
-from . import Restock
-from loguru import logger
-
-import urllib3
-import time
-
-urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)
-name = 'Re-stock & Price detection for single product pages'
-description = 'Detects if the product goes back to in-stock'
-
-class UnableToExtractRestockData(Exception):
-    def __init__(self, status_code):
-        # Set this so we can use it in other parts of the app
-        self.status_code = status_code
-        return
-
-class MoreThanOnePriceFound(Exception):
-    def __init__(self):
-        return
-
-def _search_prop_by_value(matches, value):
-    for properties in matches:
-        for prop in properties:
-            if value in prop[0]:
-                return prop[1]  # Yield the desired value and exit the function
-
-def _deduplicate_prices(data):
-    import re
-
-    '''
-    Some price data has multiple entries, OR it has a single entry with ['$159', '159', 159, "$ 159"] or just "159"
-    Get all the values, clean it and add it to a set then return the unique values
-    '''
-    unique_data = set()
-
-    # Return the complete 'datum' where its price was not seen before
-    for datum in data:
-
-        if isinstance(datum.value, list):
-            # Process each item in the list
-            normalized_value = set([float(re.sub(r'[^\d.]', '', str(item))) for item in datum.value if str(item).strip()])
-            unique_data.update(normalized_value)
-        else:
-            # Process single value
-            v = float(re.sub(r'[^\d.]', '', str(datum.value)))
-            unique_data.add(v)
-
-    return list(unique_data)
-
-
-# should return Restock()
-# add casting?
-def get_itemprop_availability(html_content) -> Restock:
-    """
-    Kind of funny/cool way to find price/availability in one many different possibilities.
-    Use 'extruct' to find any possible RDFa/microdata/json-ld data, make a JSON string from the output then search it.
-    """
-    from jsonpath_ng import parse
-
-    import re
-    now = time.time()
-    import extruct
-    logger.trace(f"Imported extruct module in {time.time() - now:.3f}s")
-
-    now = time.time()
-
-    # Extruct is very slow, I'm wondering if some ML is going to be faster (800ms on my i7), 'rdfa' seems to be the heaviest.
-    syntaxes = ['dublincore', 'json-ld', 'microdata', 'microformat', 'opengraph']
-    try:
-        data = extruct.extract(html_content, syntaxes=syntaxes)
-    except Exception as e:
-        logger.warning(f"Unable to extract data, document parsing with extruct failed with {type(e).__name__} - {str(e)}")
-        return Restock()
-
-    logger.trace(f"Extruct basic extract of all metadata done in {time.time() - now:.3f}s")
-
-    # First phase, dead simple scanning of anything that looks useful
-    value = Restock()
-    if data:
-        logger.debug(f"Using jsonpath to find price/availability/etc")
-        price_parse = parse('$..(price|Price)')
-        pricecurrency_parse = parse('$..(pricecurrency|currency|priceCurrency )')
-        availability_parse = parse('$..(availability|Availability)')
-
-        price_result = _deduplicate_prices(price_parse.find(data))
-        if price_result:
-            # Right now, we just support single product items, maybe we will store the whole actual metadata seperately in teh future and
-            # parse that for the UI?
-            if len(price_result) > 1 and len(price_result) > 1:
-                # See of all prices are different, in the case that one product has many embedded data types with the same price
-                # One might have $121.95 and another 121.95 etc
-                logger.warning(f"More than one price found {price_result}, throwing exception, cant use this plugin.")
-                raise MoreThanOnePriceFound()
-
-            value['price'] = price_result[0]
-
-        pricecurrency_result = pricecurrency_parse.find(data)
-        if pricecurrency_result:
-            value['currency'] = pricecurrency_result[0].value
-
-        availability_result = availability_parse.find(data)
-        if availability_result:
-            value['availability'] = availability_result[0].value
-
-        if value.get('availability'):
-            value['availability'] = re.sub(r'(?i)^(https|http)://schema.org/', '',
-                                           value.get('availability').strip(' "\'').lower()) if value.get('availability') else None
-
-        # Second, go dig OpenGraph which is something that jsonpath_ng cant do because of the tuples and double-dots (:)
-        if not value.get('price') or value.get('availability'):
-            logger.debug(f"Alternatively digging through OpenGraph properties for restock/price info..")
-            jsonpath_expr = parse('$..properties')
-
-            for match in jsonpath_expr.find(data):
-                if not value.get('price'):
-                    value['price'] = _search_prop_by_value([match.value], "price:amount")
-                if not value.get('availability'):
-                    value['availability'] = _search_prop_by_value([match.value], "product:availability")
-                if not value.get('currency'):
-                    value['currency'] = _search_prop_by_value([match.value], "price:currency")
-    logger.trace(f"Processed with Extruct in {time.time()-now:.3f}s")
-
-    return value
-
-
-def is_between(number, lower=None, upper=None):
-    """
-    Check if a number is between two values.
-
-    Parameters:
-    number (float): The number to check.
-    lower (float or None): The lower bound (inclusive). If None, no lower bound.
-    upper (float or None): The upper bound (inclusive). If None, no upper bound.
-
-    Returns:
-    bool: True if the number is between the lower and upper bounds, False otherwise.
-    """
-    return (lower is None or lower <= number) and (upper is None or number <= upper)
-
-
-class perform_site_check(difference_detection_processor):
-    screenshot = None
-    xpath_data = None
-
-    def run_changedetection(self, watch):
-        import hashlib
-
-        if not watch:
-            raise Exception("Watch no longer exists.")
-
-        # Unset any existing notification error
-        update_obj = {'last_notification_error': False, 'last_error': False, 'restock':  Restock()}
-
-        self.screenshot = self.fetcher.screenshot
-        self.xpath_data = self.fetcher.xpath_data
-
-        # Track the content type
-        update_obj['content_type'] = self.fetcher.headers.get('Content-Type', '')
-        update_obj["last_check_status"] = self.fetcher.get_last_status_code()
-
-        # Only try to process restock information (like scraping for keywords) if the page was actually rendered correctly.
-        # Otherwise it will assume "in stock" because nothing suggesting the opposite was found
-        from ...html_tools import html_to_text
-        text = html_to_text(self.fetcher.content)
-        logger.debug(f"Length of text after conversion: {len(text)}")
-        if not len(text):
-            from ...content_fetchers.exceptions import ReplyWithContentButNoText
-            raise ReplyWithContentButNoText(url=watch.link,
-                                            status_code=self.fetcher.get_last_status_code(),
-                                            screenshot=self.fetcher.screenshot,
-                                            html_content=self.fetcher.content,
-                                            xpath_data=self.fetcher.xpath_data
-                                            )
-
-        # Which restock settings to compare against?
-        restock_settings = watch.get('restock_settings', {})
-
-        # See if any tags have 'activate for individual watches in this tag/group?' enabled and use the first we find
-        for tag_uuid in watch.get('tags'):
-            tag = self.datastore.data['settings']['application']['tags'].get(tag_uuid, {})
-            if tag.get('overrides_watch'):
-                restock_settings = tag.get('restock_settings', {})
-                logger.info(f"Watch {watch.get('uuid')} - Tag '{tag.get('title')}' selected for restock settings override")
-                break
-
-
-        itemprop_availability = {}
-        try:
-            itemprop_availability = get_itemprop_availability(self.fetcher.content)
-        except MoreThanOnePriceFound as e:
-            # Add the real data
-            raise ProcessorException(message="Cannot run, more than one price detected, this plugin is only for product pages with ONE product, try the content-change detection mode.",
-                                     url=watch.get('url'),
-                                     status_code=self.fetcher.get_last_status_code(),
-                                     screenshot=self.fetcher.screenshot,
-                                     xpath_data=self.fetcher.xpath_data
-                                     )
-
-        # Something valid in get_itemprop_availability() by scraping metadata ?
-        if itemprop_availability.get('price') or itemprop_availability.get('availability'):
-            # Store for other usage
-            update_obj['restock'] = itemprop_availability
-
-            if itemprop_availability.get('availability'):
-                # @todo: Configurable?
-                if any(substring.lower() in itemprop_availability['availability'].lower() for substring in [
-                    'instock',
-                    'instoreonly',
-                    'limitedavailability',
-                    'onlineonly',
-                    'presale']
-                       ):
-                    update_obj['restock']['in_stock'] = True
-                else:
-                    update_obj['restock']['in_stock'] = False
-
-        # Main detection method
-        fetched_md5 = None
-
-        # store original price if not set
-        if itemprop_availability and itemprop_availability.get('price') and not itemprop_availability.get('original_price'):
-            itemprop_availability['original_price'] = itemprop_availability.get('price')
-            update_obj['restock']["original_price"] = itemprop_availability.get('price')
-
-        if not self.fetcher.instock_data and not itemprop_availability.get('availability') and not itemprop_availability.get('price'):
-            raise ProcessorException(
-                message=f"Unable to extract restock data for this page unfortunately. (Got code {self.fetcher.get_last_status_code()} from server), no embedded stock information was found and nothing interesting in the text, try using this watch with Chrome.",
-                url=watch.get('url'),
-                status_code=self.fetcher.get_last_status_code(),
-                screenshot=self.fetcher.screenshot,
-                xpath_data=self.fetcher.xpath_data
-                )
-
-        logger.debug(f"self.fetcher.instock_data is - '{self.fetcher.instock_data}' and itemprop_availability.get('availability') is {itemprop_availability.get('availability')}")
-        # Nothing automatic in microdata found, revert to scraping the page
-        if self.fetcher.instock_data and itemprop_availability.get('availability') is None:
-            # 'Possibly in stock' comes from stock-not-in-stock.js when no string found above the fold.
-            # Careful! this does not really come from chrome/js when the watch is set to plaintext
-            update_obj['restock']["in_stock"] = True if self.fetcher.instock_data == 'Possibly in stock' else False
-            logger.debug(f"Watch UUID {watch.get('uuid')} restock check returned instock_data - '{self.fetcher.instock_data}' from JS scraper.")
-
-        # Very often websites will lie about the 'availability' in the metadata, so if the scraped version says its NOT in stock, use that.
-        if self.fetcher.instock_data and self.fetcher.instock_data != 'Possibly in stock':
-            if update_obj['restock'].get('in_stock'):
-                logger.warning(
-                    f"Lie detected in the availability machine data!! when scraping said its not in stock!! itemprop was '{itemprop_availability}' and scraped from browser was '{self.fetcher.instock_data}' update obj was {update_obj['restock']} ")
-                logger.warning(f"Setting instock to FALSE, scraper found '{self.fetcher.instock_data}' in the body but metadata reported not-in-stock")
-                update_obj['restock']["in_stock"] = False
-
-        # What we store in the snapshot
-        price = update_obj.get('restock').get('price') if update_obj.get('restock').get('price') else ""
-        snapshot_content = f"In Stock: {update_obj.get('restock').get('in_stock')} - Price: {price}"
-
-        # Main detection method
-        fetched_md5 = hashlib.md5(snapshot_content.encode('utf-8')).hexdigest()
-
-        # The main thing that all this at the moment comes down to :)
-        changed_detected = False
-        logger.debug(f"Watch UUID {watch.get('uuid')} restock check - Previous MD5: {watch.get('previous_md5')}, Fetched MD5 {fetched_md5}")
-
-        # out of stock -> back in stock only?
-        if watch.get('restock') and watch['restock'].get('in_stock') != update_obj['restock'].get('in_stock'):
-            # Yes if we only care about it going to instock, AND we are in stock
-            if restock_settings.get('in_stock_processing') == 'in_stock_only' and update_obj['restock']['in_stock']:
-                changed_detected = True
-
-            if restock_settings.get('in_stock_processing') == 'all_changes':
-                # All cases
-                changed_detected = True
-
-        if restock_settings.get('follow_price_changes') and watch.get('restock') and update_obj.get('restock') and update_obj['restock'].get('price'):
-            price = float(update_obj['restock'].get('price'))
-            # Default to current price if no previous price found
-            if watch['restock'].get('original_price'):
-                previous_price = float(watch['restock'].get('original_price'))
-                # It was different, but negate it further down
-                if price != previous_price:
-                    changed_detected = True
-
-            # Minimum/maximum price limit
-            if update_obj.get('restock') and update_obj['restock'].get('price'):
-                logger.debug(
-                    f"{watch.get('uuid')} - Change was detected, 'price_change_max' is '{restock_settings.get('price_change_max', '')}' 'price_change_min' is '{restock_settings.get('price_change_min', '')}', price from website is '{update_obj['restock'].get('price', '')}'.")
-                if update_obj['restock'].get('price'):
-                    min_limit = float(restock_settings.get('price_change_min')) if restock_settings.get('price_change_min') else None
-                    max_limit = float(restock_settings.get('price_change_max')) if restock_settings.get('price_change_max') else None
-
-                    price = float(update_obj['restock'].get('price'))
-                    logger.debug(f"{watch.get('uuid')} after float conversion - Min limit: '{min_limit}' Max limit: '{max_limit}' Price: '{price}'")
-                    if min_limit or max_limit:
-                        if is_between(number=price, lower=min_limit, upper=max_limit):
-                            # Price was between min/max limit, so there was nothing todo in any case
-                            logger.trace(f"{watch.get('uuid')} {price} is between {min_limit} and {max_limit}, nothing to check, forcing changed_detected = False (was {changed_detected})")
-                            changed_detected = False
-                        else:
-                            logger.trace(f"{watch.get('uuid')} {price} is between {min_limit} and {max_limit}, continuing normal comparison")
-
-                    # Price comparison by %
-                    if watch['restock'].get('original_price') and changed_detected and restock_settings.get('price_change_threshold_percent'):
-                        previous_price = float(watch['restock'].get('original_price'))
-                        pc = float(restock_settings.get('price_change_threshold_percent'))
-                        change = abs((price - previous_price) / previous_price * 100)
-                        if change and change <= pc:
-                            logger.debug(f"{watch.get('uuid')} Override change-detected to FALSE because % threshold ({pc}%) was {change:.3f}%")
-                            changed_detected = False
-                        else:
-                            logger.debug(f"{watch.get('uuid')} Price change was {change:.3f}% , (threshold {pc}%)")
-
-        # Always record the new checksum
-        update_obj["previous_md5"] = fetched_md5
-
-        return changed_detected, update_obj, snapshot_content.strip()

changedetectionio/processors/text_json_diff.py

@@ -6,24 +6,22 @@ import os
 import re
 import urllib3
 
-from changedetectionio.conditions import execute_ruleset_against_all_plugins
-from changedetectionio.processors import difference_detection_processor
-from changedetectionio.html_tools import PERL_STYLE_REGEX, cdata_in_document_to_text, TRANSLATE_WHITESPACE_TABLE
+from . import difference_detection_processor
+from ..html_tools import PERL_STYLE_REGEX, cdata_in_document_to_text
 from changedetectionio import html_tools, content_fetchers
 from changedetectionio.blueprint.price_data_follower import PRICE_DATA_TRACK_ACCEPT, PRICE_DATA_TRACK_REJECT
+import changedetectionio.content_fetchers
+from copy import deepcopy
 from loguru import logger
 
 urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)
 
 name = 'Webpage Text/HTML, JSON and PDF changes'
 description = 'Detects all text changes where possible'
-
-json_filter_prefixes = ['json:', 'jq:', 'jqraw:']
+json_filter_prefixes = ['json:', 'jq:']
 
 class FilterNotFoundInResponse(ValueError):
-    def __init__(self, msg, screenshot=None, xpath_data=None):
-        self.screenshot = screenshot
-        self.xpath_data = xpath_data
+    def __init__(self, msg):
         ValueError.__init__(self, msg)
 
 
@@ -36,12 +34,14 @@ class PDFToHTMLToolNotFound(ValueError):
 # (set_proxy_from_list)
 class perform_site_check(difference_detection_processor):
 
-    def run_changedetection(self, watch):
+    def run_changedetection(self, uuid, skip_when_checksum_same=True):
         changed_detected = False
         html_content = ""
         screenshot = False  # as bytes
         stripped_text_from_html = ""
 
+        # DeepCopy so we can be sure we don't accidently change anything by reference
+        watch = deepcopy(self.datastore.data['watching'].get(uuid))
         if not watch:
             raise Exception("Watch no longer exists.")
 
@@ -59,6 +59,9 @@ class perform_site_check(difference_detection_processor):
         # Watches added automatically in the queue manager will skip if its the same checksum as the previous run
         # Saves a lot of CPU
         update_obj['previous_md5_before_filters'] = hashlib.md5(self.fetcher.content.encode('utf-8')).hexdigest()
+        if skip_when_checksum_same:
+            if update_obj['previous_md5_before_filters'] == watch.get('previous_md5_before_filters'):
+                raise content_fetchers.exceptions.checksumFromPreviousCheckWasTheSame()
 
         # Fetching complete, now filters
 
@@ -113,12 +116,12 @@ class perform_site_check(difference_detection_processor):
         # Better would be if Watch.model could access the global data also
         # and then use getattr https://docs.python.org/3/reference/datamodel.html#object.__getitem__
         # https://realpython.com/inherit-python-dict/ instead of doing it procedurely
-        include_filters_from_tags = self.datastore.get_tag_overrides_for_watch(uuid=watch.get('uuid'), attr='include_filters')
+        include_filters_from_tags = self.datastore.get_tag_overrides_for_watch(uuid=uuid, attr='include_filters')
 
         # 1845 - remove duplicated filters in both group and watch include filter
         include_filters_rule = list(dict.fromkeys(watch.get('include_filters', []) + include_filters_from_tags))
 
-        subtractive_selectors = [*self.datastore.get_tag_overrides_for_watch(uuid=watch.get('uuid'), attr='subtractive_selectors'),
+        subtractive_selectors = [*self.datastore.get_tag_overrides_for_watch(uuid=uuid, attr='subtractive_selectors'),
                                  *watch.get("subtractive_selectors", []),
                                  *self.datastore.data["settings"]["application"].get("global_subtractive_selectors", [])
                                  ]
@@ -173,19 +176,19 @@ class perform_site_check(difference_detection_processor):
                                                                     html_content=self.fetcher.content,
                                                                     append_pretty_line_formatting=not watch.is_source_type_url,
                                                                     is_rss=is_rss)
-
                         elif filter_rule.startswith('xpath1:'):
                             html_content += html_tools.xpath1_filter(xpath_filter=filter_rule.replace('xpath1:', ''),
-                                                                     html_content=self.fetcher.content,
-                                                                     append_pretty_line_formatting=not watch.is_source_type_url,
-                                                                     is_rss=is_rss)
+                                                                    html_content=self.fetcher.content,
+                                                                    append_pretty_line_formatting=not watch.is_source_type_url,
+                                                                    is_rss=is_rss)
                         else:
+                            # CSS Filter, extract the HTML that matches and feed that into the existing inscriptis::get_text
                             html_content += html_tools.include_filters(include_filters=filter_rule,
                                                                        html_content=self.fetcher.content,
                                                                        append_pretty_line_formatting=not watch.is_source_type_url)
 
                     if not html_content.strip():
-                        raise FilterNotFoundInResponse(msg=include_filters_rule, screenshot=self.fetcher.screenshot, xpath_data=self.fetcher.xpath_data)
+                        raise FilterNotFoundInResponse(include_filters_rule)
 
                 if has_subtractive_selectors:
                     html_content = html_tools.element_removal(subtractive_selectors, html_content)
@@ -195,27 +198,31 @@ class perform_site_check(difference_detection_processor):
                 else:
                     # extract text
                     do_anchor = self.datastore.data["settings"]["application"].get("render_anchor_tag_content", False)
-                    stripped_text_from_html = html_tools.html_to_text(html_content=html_content,
-                                                                      render_anchor_tag_content=do_anchor,
-                                                                      is_rss=is_rss)  # 1874 activate the <title workaround hack
+                    stripped_text_from_html = \
+                        html_tools.html_to_text(
+                            html_content=html_content,
+                            render_anchor_tag_content=do_anchor,
+                            is_rss=is_rss # #1874 activate the <title workaround hack
+                        )
 
-        if watch.get('trim_text_whitespace'):
-            stripped_text_from_html = '\n'.join(line.strip() for line in stripped_text_from_html.replace("\n\n", "\n").splitlines())
+        if watch.get('sort_text_alphabetically') and stripped_text_from_html:
+            # Note: Because a <p>something</p> will add an extra line feed to signify the paragraph gap
+            # we end up with 'Some text\n\n', sorting will add all those extra \n at the start, so we remove them here.
+            stripped_text_from_html = stripped_text_from_html.replace('\n\n', '\n')
+            stripped_text_from_html = '\n'.join( sorted(stripped_text_from_html.splitlines(), key=lambda x: x.lower() ))
 
         # Re #340 - return the content before the 'ignore text' was applied
-        # Also used to calculate/show what was removed
-        text_content_before_ignored_filter = stripped_text_from_html
+        text_content_before_ignored_filter = stripped_text_from_html.encode('utf-8')
 
         # @todo whitespace coming from missing rtrim()?
         # stripped_text_from_html could be based on their preferences, replace the processed text with only that which they want to know about.
         # Rewrite's the processing text based on only what diff result they want to see
-
         if watch.has_special_diff_filter_options_set() and len(watch.history.keys()):
             # Now the content comes from the diff-parser and not the returned HTTP traffic, so could be some differences
-            from changedetectionio import diff
+            from .. import diff
             # needs to not include (added) etc or it may get used twice
             # Replace the processed text with the preferred result
-            rendered_diff = diff.render_diff(previous_version_file_contents=watch.get_last_fetched_text_before_filters(),
+            rendered_diff = diff.render_diff(previous_version_file_contents=watch.get_last_fetched_before_filters(),
                                              newest_version_file_contents=stripped_text_from_html,
                                              include_equal=False,  # not the same lines
                                              include_added=watch.get('filter_text_added', True),
@@ -224,12 +231,12 @@ class perform_site_check(difference_detection_processor):
                                              line_feed_sep="\n",
                                              include_change_type_prefix=False)
 
-            watch.save_last_text_fetched_before_filters(text_content_before_ignored_filter.encode('utf-8'))
+            watch.save_last_fetched_before_filters(text_content_before_ignored_filter)
 
             if not rendered_diff and stripped_text_from_html:
                 # We had some content, but no differences were found
                 # Store our new file as the MD5 so it will trigger in the future
-                c = hashlib.md5(stripped_text_from_html.translate(TRANSLATE_WHITESPACE_TABLE).encode('utf-8')).hexdigest()
+                c = hashlib.md5(text_content_before_ignored_filter.translate(None, b'\r\n\t ')).hexdigest()
                 return False, {'previous_md5': c}, stripped_text_from_html.encode('utf-8')
             else:
                 stripped_text_from_html = rendered_diff
@@ -239,10 +246,9 @@ class perform_site_check(difference_detection_processor):
         if not is_json and not empty_pages_are_a_change and len(stripped_text_from_html.strip()) == 0:
             raise content_fetchers.exceptions.ReplyWithContentButNoText(url=url,
                                                             status_code=self.fetcher.get_last_status_code(),
-                                                            screenshot=self.fetcher.screenshot,
+                                                            screenshot=screenshot,
                                                             has_filters=has_filter_rule,
-                                                            html_content=html_content,
-                                                            xpath_data=self.fetcher.xpath_data
+                                                            html_content=html_content
                                                             )
 
         # We rely on the actual text in the html output.. many sites have random script vars etc,
@@ -250,6 +256,14 @@ class perform_site_check(difference_detection_processor):
 
         update_obj["last_check_status"] = self.fetcher.get_last_status_code()
 
+        # If there's text to skip
+        # @todo we could abstract out the get_text() to handle this cleaner
+        text_to_ignore = watch.get('ignore_text', []) + self.datastore.data['settings']['application'].get('global_ignore_text', [])
+        if len(text_to_ignore):
+            stripped_text_from_html = html_tools.strip_ignore_text(stripped_text_from_html, text_to_ignore)
+        else:
+            stripped_text_from_html = stripped_text_from_html.encode('utf8')
+
         # 615 Extract text by regex
         extract_text = watch.get('extract_text', [])
         if len(extract_text) > 0:
@@ -258,53 +272,37 @@ class perform_site_check(difference_detection_processor):
                 # incase they specified something in '/.../x'
                 if re.search(PERL_STYLE_REGEX, s_re, re.IGNORECASE):
                     regex = html_tools.perl_style_slash_enclosed_regex_to_options(s_re)
-                    result = re.findall(regex, stripped_text_from_html)
+                    result = re.findall(regex.encode('utf-8'), stripped_text_from_html)
 
                     for l in result:
                         if type(l) is tuple:
                             # @todo - some formatter option default (between groups)
-                            regex_matched_output += list(l) + ['\n']
+                            regex_matched_output += list(l) + [b'\n']
                         else:
                             # @todo - some formatter option default (between each ungrouped result)
-                            regex_matched_output += [l] + ['\n']
+                            regex_matched_output += [l] + [b'\n']
                 else:
                     # Doesnt look like regex, just hunt for plaintext and return that which matches
                     # `stripped_text_from_html` will be bytes, so we must encode s_re also to bytes
-                    r = re.compile(re.escape(s_re), re.IGNORECASE)
+                    r = re.compile(re.escape(s_re.encode('utf-8')), re.IGNORECASE)
                     res = r.findall(stripped_text_from_html)
                     if res:
                         for match in res:
-                            regex_matched_output += [match] + ['\n']
-
-            ##########################################################
-            stripped_text_from_html = ''
+                            regex_matched_output += [match] + [b'\n']
 
+            # Now we will only show what the regex matched
+            stripped_text_from_html = b''
+            text_content_before_ignored_filter = b''
             if regex_matched_output:
                 # @todo some formatter for presentation?
-                stripped_text_from_html = ''.join(regex_matched_output)
-
-        if watch.get('remove_duplicate_lines'):
-            stripped_text_from_html = '\n'.join(dict.fromkeys(line for line in stripped_text_from_html.replace("\n\n", "\n").splitlines()))
-
-
-        if watch.get('sort_text_alphabetically'):
-            # Note: Because a <p>something</p> will add an extra line feed to signify the paragraph gap
-            # we end up with 'Some text\n\n', sorting will add all those extra \n at the start, so we remove them here.
-            stripped_text_from_html = stripped_text_from_html.replace("\n\n", "\n")
-            stripped_text_from_html = '\n'.join(sorted(stripped_text_from_html.splitlines(), key=lambda x: x.lower()))
-
-### CALCULATE MD5
-        # If there's text to ignore
-        text_to_ignore = watch.get('ignore_text', []) + self.datastore.data['settings']['application'].get('global_ignore_text', [])
-        text_for_checksuming = stripped_text_from_html
-        if text_to_ignore:
-            text_for_checksuming = html_tools.strip_ignore_text(stripped_text_from_html, text_to_ignore)
+                stripped_text_from_html = b''.join(regex_matched_output)
+                text_content_before_ignored_filter = stripped_text_from_html
 
         # Re #133 - if we should strip whitespaces from triggering the change detected comparison
-        if text_for_checksuming and self.datastore.data['settings']['application'].get('ignore_whitespace', False):
-            fetched_md5 = hashlib.md5(text_for_checksuming.translate(TRANSLATE_WHITESPACE_TABLE).encode('utf-8')).hexdigest()
+        if self.datastore.data['settings']['application'].get('ignore_whitespace', False):
+            fetched_md5 = hashlib.md5(stripped_text_from_html.translate(None, b'\r\n\t ')).hexdigest()
         else:
-            fetched_md5 = hashlib.md5(text_for_checksuming.encode('utf-8')).hexdigest()
+            fetched_md5 = hashlib.md5(stripped_text_from_html).hexdigest()
 
         ############ Blocking rules, after checksum #################
         blocked = False
@@ -332,50 +330,37 @@ class perform_site_check(difference_detection_processor):
             if result:
                 blocked = True
 
-        # And check if 'conditions' will let this pass through
-        if watch.get('conditions') and watch.get('conditions_match_logic'):
-            if not execute_ruleset_against_all_plugins(current_watch_uuid=watch.get('uuid'),
-                                                application_datastruct=self.datastore.data,
-                                                ephemeral_data={
-                                                    'text': stripped_text_from_html
-                                                }
-                                                ):
-                # Conditions say "Condition not met" so we block it.
-                blocked = True
+        # The main thing that all this at the moment comes down to :)
+        if watch.get('previous_md5') != fetched_md5:
+            changed_detected = True
 
         # Looks like something changed, but did it match all the rules?
         if blocked:
             changed_detected = False
-        else:
-            # The main thing that all this at the moment comes down to :)
-            if watch.get('previous_md5') != fetched_md5:
-                changed_detected = True
 
-            # Always record the new checksum
-            update_obj["previous_md5"] = fetched_md5
+        # Extract title as title
+        if is_html:
+            if self.datastore.data['settings']['application'].get('extract_title_as_title') or watch['extract_title_as_title']:
+                if not watch['title'] or not len(watch['title']):
+                    update_obj['title'] = html_tools.extract_element(find='title', html_content=self.fetcher.content)
 
-            # On the first run of a site, watch['previous_md5'] will be None, set it the current one.
-            if not watch.get('previous_md5'):
-                watch['previous_md5'] = fetched_md5
-
-        logger.debug(f"Watch UUID {watch.get('uuid')} content check - Previous MD5: {watch.get('previous_md5')}, Fetched MD5 {fetched_md5}")
+        logger.debug(f"Watch UUID {uuid} content check - Previous MD5: {watch.get('previous_md5')}, Fetched MD5 {fetched_md5}")
 
         if changed_detected:
             if watch.get('check_unique_lines', False):
-                ignore_whitespace = self.datastore.data['settings']['application'].get('ignore_whitespace')
-
-                has_unique_lines = watch.lines_contain_something_unique_compared_to_history(
-                    lines=stripped_text_from_html.splitlines(),
-                    ignore_whitespace=ignore_whitespace
-                )
-
+                has_unique_lines = watch.lines_contain_something_unique_compared_to_history(lines=stripped_text_from_html.splitlines())
                 # One or more lines? unsure?
                 if not has_unique_lines:
-                    logger.debug(f"check_unique_lines: UUID {watch.get('uuid')} didnt have anything new setting change_detected=False")
+                    logger.debug(f"check_unique_lines: UUID {uuid} didnt have anything new setting change_detected=False")
                     changed_detected = False
                 else:
-                    logger.debug(f"check_unique_lines: UUID {watch.get('uuid')} had unique content")
+                    logger.debug(f"check_unique_lines: UUID {uuid} had unique content")
 
+        # Always record the new checksum
+        update_obj["previous_md5"] = fetched_md5
 
-        # stripped_text_from_html - Everything after filters and NO 'ignored' content
-        return changed_detected, update_obj, stripped_text_from_html
+        # On the first run of a site, watch['previous_md5'] will be None, set it the current one.
+        if not watch.get('previous_md5'):
+            watch['previous_md5'] = fetched_md5
+
+        return changed_detected, update_obj, text_content_before_ignored_filter

changedetectionio/processors/text_json_diff/__init__.py

@@ -1,114 +0,0 @@
-
-from loguru import logger
-
-
-
-def _task(watch, update_handler):
-    from changedetectionio.content_fetchers.exceptions import ReplyWithContentButNoText
-    from changedetectionio.processors.text_json_diff.processor import FilterNotFoundInResponse
-
-    text_after_filter = ''
-
-    try:
-        # The slow process (we run 2 of these in parallel)
-        changed_detected, update_obj, text_after_filter = update_handler.run_changedetection(watch=watch)
-    except FilterNotFoundInResponse as e:
-        text_after_filter = f"Filter not found in HTML: {str(e)}"
-    except ReplyWithContentButNoText as e:
-        text_after_filter = f"Filter found but no text (empty result)"
-    except Exception as e:
-        text_after_filter = f"Error: {str(e)}"
-
-    if not text_after_filter.strip():
-        text_after_filter = 'Empty content'
-
-    # because run_changedetection always returns bytes due to saving the snapshots etc
-    text_after_filter = text_after_filter.decode('utf-8') if isinstance(text_after_filter, bytes) else text_after_filter
-
-    return text_after_filter
-
-
-def prepare_filter_prevew(datastore, watch_uuid, form_data):
-    '''Used by @app.route("/edit/<string:uuid>/preview-rendered", methods=['POST'])'''
-    from changedetectionio import forms, html_tools
-    from changedetectionio.model.Watch import model as watch_model
-    from concurrent.futures import ProcessPoolExecutor
-    from copy import deepcopy
-    from flask import request
-    import brotli
-    import importlib
-    import os
-    import time
-    now = time.time()
-
-    text_after_filter = ''
-    text_before_filter = ''
-    trigger_line_numbers = []
-    ignore_line_numbers = []
-
-    tmp_watch = deepcopy(datastore.data['watching'].get(watch_uuid))
-
-    if tmp_watch and tmp_watch.history and os.path.isdir(tmp_watch.watch_data_dir):
-        # Splice in the temporary stuff from the form
-        form = forms.processor_text_json_diff_form(formdata=form_data if request.method == 'POST' else None,
-                                                   data=form_data
-                                                   )
-
-        # Only update vars that came in via the AJAX post
-        p = {k: v for k, v in form.data.items() if k in form_data.keys()}
-        tmp_watch.update(p)
-        blank_watch_no_filters = watch_model()
-        blank_watch_no_filters['url'] = tmp_watch.get('url')
-
-        latest_filename = next(reversed(tmp_watch.history))
-        html_fname = os.path.join(tmp_watch.watch_data_dir, f"{latest_filename}.html.br")
-        with open(html_fname, 'rb') as f:
-            decompressed_data = brotli.decompress(f.read()).decode('utf-8') if html_fname.endswith('.br') else f.read().decode('utf-8')
-
-            # Just like a normal change detection except provide a fake "watch" object and dont call .call_browser()
-            processor_module = importlib.import_module("changedetectionio.processors.text_json_diff.processor")
-            update_handler = processor_module.perform_site_check(datastore=datastore,
-                                                                 watch_uuid=tmp_watch.get('uuid')  # probably not needed anymore anyway?
-                                                                 )
-            # Use the last loaded HTML as the input
-            update_handler.datastore = datastore
-            update_handler.fetcher.content = str(decompressed_data) # str() because playwright/puppeteer/requests return string
-            update_handler.fetcher.headers['content-type'] = tmp_watch.get('content-type')
-
-            # Process our watch with filters and the HTML from disk, and also a blank watch with no filters but also with the same HTML from disk
-            # Do this as a parallel process because it could take some time
-            with ProcessPoolExecutor(max_workers=2) as executor:
-                future1 = executor.submit(_task, tmp_watch, update_handler)
-                future2 = executor.submit(_task, blank_watch_no_filters, update_handler)
-
-                text_after_filter = future1.result()
-                text_before_filter = future2.result()
-
-    try:
-        trigger_line_numbers = html_tools.strip_ignore_text(content=text_after_filter,
-                                                            wordlist=tmp_watch['trigger_text'],
-                                                            mode='line numbers'
-                                                            )
-    except Exception as e:
-        text_before_filter = f"Error: {str(e)}"
-
-    try:
-        text_to_ignore = tmp_watch.get('ignore_text', []) + datastore.data['settings']['application'].get('global_ignore_text', [])
-        ignore_line_numbers = html_tools.strip_ignore_text(content=text_after_filter,
-                                                           wordlist=text_to_ignore,
-                                                           mode='line numbers'
-                                                           )
-    except Exception as e:
-        text_before_filter = f"Error: {str(e)}"
-
-    logger.trace(f"Parsed in {time.time() - now:.3f}s")
-
-    return ({
-            'after_filter': text_after_filter,
-            'before_filter': text_before_filter.decode('utf-8') if isinstance(text_before_filter, bytes) else text_before_filter,
-            'duration': time.time() - now,
-            'trigger_line_numbers': trigger_line_numbers,
-            'ignore_line_numbers': ignore_line_numbers,
-        })
-
-

changedetectionio/processors/whois_plugin.py

@@ -1,169 +0,0 @@
-from loguru import logger
-import re
-import urllib.parse
-from .pluggy_interface import hookimpl
-from requests.structures import CaseInsensitiveDict
-from changedetectionio.content_fetchers.base import Fetcher
-
-# Import the text_json_diff processor
-from changedetectionio.processors.text_json_diff.processor import perform_site_check as TextJsonDiffProcessor
-
-# WHOIS Processor implementation that extends TextJsonDiffProcessor
-class WhoisProcessor(TextJsonDiffProcessor):
-    
-    def _extract_domain_from_url(self, url):
-        """Extract domain from URL, removing www. prefix if present"""
-        parsed_url = urllib.parse.urlparse(url)
-        domain = parsed_url.netloc
-        
-        # Remove www. prefix if present
-        domain = re.sub(r'^www\.', '', domain)
-        
-        return domain
-    
-    def call_browser(self, preferred_proxy_id=None):
-        """Override call_browser to perform WHOIS lookup instead of using a browser
-        
-        Note: The python-whois library doesn't directly support proxies. For real proxy support,
-        we would need to implement a custom socket connection that routes through the proxy.
-        This is a TODO for a future enhancement.
-        """
-        # Initialize a basic fetcher - this is used by the parent class
-        self.fetcher = Fetcher()
-        
-        # Extract URL from watch
-        url = self.watch.link
-        
-        # Check for file:// access
-        if re.search(r'^file:', url.strip(), re.IGNORECASE):
-            if not self.datastore.data.get('settings', {}).get('application', {}).get('allow_file_uri', False):
-                raise Exception("file:// type access is denied for security reasons.")
-        
-        # Extract domain from URL
-        domain = self._extract_domain_from_url(url)
-        
-        # Ensure we have a valid domain
-        if not domain:
-            error_msg = f"Could not extract domain from URL: '{url}'"
-            self.fetcher.content = error_msg
-            self.fetcher.status_code = 400
-            logger.error(error_msg)
-            return
-        
-        # Get proxy configuration using the common method from parent class
-        proxy_config, proxy_url = super()._get_proxy_for_watch(preferred_proxy_id)
-        
-        try:
-            # Use python-whois to get domain information
-            import whois
-            
-            # If we have proxy config, use it for the WHOIS lookup
-            # Note: The python-whois library doesn't directly support proxies,
-            # but we can implement proxy support if necessary using custom socket code
-            if proxy_config:
-                # For now, just log that we would use a proxy
-                logger.info(f"Using proxy for WHOIS lookup: {proxy_config}")
-            
-            # Perform the WHOIS lookup
-            whois_info = whois.whois(domain)
-            
-            # Convert whois_info object to text
-            if hasattr(whois_info, 'text'):
-                # Some whois implementations store raw text in .text attribute
-                whois_text = whois_info.text
-            else:
-                # Otherwise, format it nicely as key-value pairs
-                whois_text = f"WHOIS Information for domain: {domain}\n\n"
-                for key, value in whois_info.items():
-                    if value:
-                        whois_text += f"{key}: {value}\n"
-            
-            # Set the content and status for the fetcher
-            self.fetcher.content = whois_text
-            self.fetcher.status_code = 200
-            
-            # Setup headers dictionary for the fetcher
-            self.fetcher.headers = CaseInsensitiveDict({
-                'content-type': 'text/plain',
-                'server': 'whois-processor'
-            })
-            
-            # Add getters for headers
-            self.fetcher.get_all_headers = lambda: self.fetcher.headers
-            self.fetcher.get_last_status_code = lambda: self.fetcher.status_code
-            
-            # Implement necessary methods
-            self.fetcher.quit = lambda: None
-            
-        except Exception as e:
-            error_msg = f"Error fetching WHOIS data for domain {domain}: {str(e)}"
-            self.fetcher.content = error_msg
-            self.fetcher.status_code = 500
-            self.fetcher.headers = CaseInsensitiveDict({
-                'content-type': 'text/plain',
-                'server': 'whois-processor'
-            })
-            self.fetcher.get_all_headers = lambda: self.fetcher.headers
-            self.fetcher.get_last_status_code = lambda: self.fetcher.status_code
-            self.fetcher.quit = lambda: None
-            logger.error(error_msg)
-
-        return
-    
-    def run_changedetection(self, watch):
-        """Use the parent's run_changedetection which will use our overridden call_browser method"""
-        try:
-            # Let the parent class handle everything now that we've overridden call_browser
-            changed_detected, update_obj, filtered_text = super().run_changedetection(watch)
-            return changed_detected, update_obj, filtered_text
-            
-        except Exception as e:
-            error_msg = f"Error in WHOIS processor: {str(e)}"
-            update_obj = {'last_notification_error': False, 'last_error': error_msg}
-            logger.error(error_msg)
-            return False, update_obj, error_msg.encode('utf-8')
-
-    @staticmethod
-    def perform_site_check(datastore, watch_uuid):
-        """Factory method to create a WhoisProcessor instance - for compatibility with legacy code"""
-        processor = WhoisProcessor(datastore=datastore, watch_uuid=watch_uuid)
-        return processor
-
-@hookimpl
-def perform_site_check(datastore, watch_uuid):
-    """Create and return a processor instance ready to perform site check"""
-    return WhoisProcessor(datastore=datastore, watch_uuid=watch_uuid)
-
-@hookimpl(trylast=True)  # Use trylast to ensure this runs last in case of conflicts
-def get_processor_name():
-    """Return the name of this processor"""
-    from loguru import logger
-    logger.debug("whois_plugin.get_processor_name() called")
-    return "whois"
-
-@hookimpl
-def get_processor_description():
-    """Return the description of this processor"""
-    return "WHOIS Domain Information Changes Detector"
-
-@hookimpl
-def get_processor_class():
-    """Return the processor class"""
-    return WhoisProcessor
-
-@hookimpl
-def get_processor_form():
-    """Return the processor form class"""
-    # Import here to avoid circular imports
-    try:
-        from changedetectionio.forms import processor_text_json_diff_form
-        return processor_text_json_diff_form
-    except Exception as e:
-        from loguru import logger
-        logger.error(f"Error importing form for whois plugin: {str(e)}")
-        return None
-
-@hookimpl
-def get_processor_watch_model():
-    """Return the watch model class for this processor"""
-    return None  # Use default watch model

changedetectionio/run_basic_tests.sh

@@ -35,8 +35,4 @@ pytest tests/test_access_control.py
 pytest tests/test_notification.py
 pytest tests/test_backend.py
 pytest tests/test_rss.py
-pytest tests/test_unique_lines.py
-
-# Check file:// will pickup a file when enabled
-echo "Hello world" > /tmp/test-file.txt
-ALLOW_FILE_URI=yes pytest tests/test_security.py
+pytest tests/test_unique_lines.py

changedetectionio/run_socks_proxy_tests.sh

@@ -16,31 +16,25 @@ echo "---------------------------------- SOCKS5 -------------------"
 docker run --network changedet-network \
   -v `pwd`/tests/proxy_socks5/proxies.json-example:/app/changedetectionio/test-datastore/proxies.json \
   --rm \
-  -e "FLASK_SERVER_NAME=cdio" \
-  --hostname cdio \
   -e "SOCKSTEST=proxiesjson" \
   test-changedetectionio \
-  bash -c 'cd changedetectionio && pytest --live-server-host=0.0.0.0 --live-server-port=5004  -s tests/proxy_socks5/test_socks5_proxy_sources.py'
+  bash -c 'cd changedetectionio && pytest tests/proxy_socks5/test_socks5_proxy_sources.py'
 
 # SOCKS5 related - by manually entering in UI
 docker run --network changedet-network \
   --rm \
-  -e "FLASK_SERVER_NAME=cdio" \
-  --hostname cdio \
   -e "SOCKSTEST=manual" \
   test-changedetectionio \
-  bash -c 'cd changedetectionio && pytest --live-server-host=0.0.0.0 --live-server-port=5004  -s tests/proxy_socks5/test_socks5_proxy.py'
+  bash -c 'cd changedetectionio && pytest tests/proxy_socks5/test_socks5_proxy.py'
 
 # SOCKS5 related - test from proxies.json via playwright - NOTE- PLAYWRIGHT DOESNT SUPPORT AUTHENTICATING PROXY
 docker run --network changedet-network \
   -e "SOCKSTEST=manual-playwright" \
-  --hostname cdio \
-  -e "FLASK_SERVER_NAME=cdio" \
   -v `pwd`/tests/proxy_socks5/proxies.json-example-noauth:/app/changedetectionio/test-datastore/proxies.json \
   -e "PLAYWRIGHT_DRIVER_URL=ws://sockpuppetbrowser:3000" \
   --rm \
   test-changedetectionio \
-  bash -c 'cd changedetectionio && pytest --live-server-host=0.0.0.0 --live-server-port=5004  -s tests/proxy_socks5/test_socks5_proxy_sources.py'
+  bash -c 'cd changedetectionio && pytest tests/proxy_socks5/test_socks5_proxy_sources.py'
 
 echo "socks5 server logs"
 docker logs socks5proxy

changedetectionio/safe_jinja.py

@@ -1,18 +0,0 @@
-"""
-Safe Jinja2 render with max payload sizes
-
-See https://jinja.palletsprojects.com/en/3.1.x/sandbox/#security-considerations
-"""
-
-import jinja2.sandbox
-import typing as t
-import os
-
-JINJA2_MAX_RETURN_PAYLOAD_SIZE = 1024 * int(os.getenv("JINJA2_MAX_RETURN_PAYLOAD_SIZE_KB", 1024 * 10))
-
-
-def render(template_str, **args: t.Any) -> str:
-    jinja2_env = jinja2.sandbox.ImmutableSandboxedEnvironment(extensions=['jinja2_time.TimeExtension'])
-    output = jinja2_env.from_string(template_str).render(args)
-    return output[:JINJA2_MAX_RETURN_PAYLOAD_SIZE]
-

changedetectionio/static/images/copy.svg

@@ -1,7 +1,7 @@
 <?xml version="1.0" encoding="UTF-8" standalone="no"?>
 <svg
    version="1.1"
-   id="copy"
+   id="Layer_1"
    x="0px"
    y="0px"
    viewBox="0 0 115.77 122.88"

changedetectionio/static/images/email.svg

@@ -6,7 +6,7 @@
    height="7.5005589"
    width="11.248507"
    version="1.1"
-   id="email"
+   id="Layer_1"
    viewBox="0 0 7.1975545 4.7993639"
    xml:space="preserve"
    xmlns="http://www.w3.org/2000/svg"

changedetectionio/static/images/schedule.svg

@@ -1,225 +0,0 @@
-<?xml version="1.0" encoding="UTF-8" standalone="no"?>
-<svg
-   version="1.1"
-   id="schedule"
-   x="0px"
-   y="0px"
-   viewBox="0 0 661.20001 665.40002"
-   xml:space="preserve"
-   width="661.20001"
-   height="665.40002"
-   inkscape:version="1.1.2 (0a00cf5339, 2022-02-04)"
-   sodipodi:docname="schedule.svg"
-   xmlns:inkscape="http://www.inkscape.org/namespaces/inkscape"
-   xmlns:sodipodi="http://sodipodi.sourceforge.net/DTD/sodipodi-0.dtd"
-   xmlns="http://www.w3.org/2000/svg"
-   xmlns:svg="http://www.w3.org/2000/svg"><defs
-   id="defs77" /><sodipodi:namedview
-   id="namedview75"
-   pagecolor="#ffffff"
-   bordercolor="#666666"
-   borderopacity="1.0"
-   inkscape:pageshadow="2"
-   inkscape:pageopacity="0.0"
-   inkscape:pagecheckerboard="0"
-   showgrid="false"
-   fit-margin-top="0"
-   fit-margin-left="0"
-   fit-margin-right="0"
-   fit-margin-bottom="0"
-   inkscape:zoom="1.2458671"
-   inkscape:cx="300.59386"
-   inkscape:cy="332.29869"
-   inkscape:window-width="1920"
-   inkscape:window-height="1051"
-   inkscape:window-x="1920"
-   inkscape:window-y="0"
-   inkscape:window-maximized="1"
-   inkscape:current-layer="g72" /> <style
-   type="text/css"
-   id="style2"> .st0{fill:#FFFFFF;} .st1{fill:#C1272D;} .st2{fill:#991D26;} .st3{fill:#CCCCCC;} .st4{fill:#E6E6E6;} .st5{fill:#F7931E;} .st6{fill:#F2F2F2;} .st7{fill:none;stroke:#999999;stroke-width:17.9763;stroke-linecap:round;stroke-miterlimit:10;} .st8{fill:none;stroke:#333333;stroke-width:8.9882;stroke-linecap:round;stroke-miterlimit:10;} .st9{fill:none;stroke:#C1272D;stroke-width:5.9921;stroke-linecap:round;stroke-miterlimit:10;} .st10{fill:#245F7F;} </style> <g
-   id="g72"
-   transform="translate(-149.4,-147.3)"> <path
-   class="st0"
-   d="M 601.2,699.8 H 205 c -30.7,0 -55.6,-24.9 -55.6,-55.6 V 248 c 0,-30.7 24.9,-55.6 55.6,-55.6 h 396.2 c 30.7,0 55.6,24.9 55.6,55.6 v 396.2 c 0,30.7 -24.9,55.6 -55.6,55.6 z"
-   id="path4"
-   style="fill:#dfdfdf;fill-opacity:1" /> <path
-   class="st1"
-   d="M 601.2,192.4 H 205 c -30.7,0 -55.6,24.9 -55.6,55.6 v 88.5 H 656.8 V 248 c 0,-30.7 -24.9,-55.6 -55.6,-55.6 z"
-   id="path6"
-   style="fill:#d62128;fill-opacity:1" /> <circle
-   class="st2"
-   cx="253.3"
-   cy="264.5"
-   r="36.700001"
-   id="circle8" /> <circle
-   class="st2"
-   cx="551.59998"
-   cy="264.5"
-   r="36.700001"
-   id="circle10" /> <path
-   class="st3"
-   d="m 253.3,275.7 v 0 c -11.8,0 -21.3,-9.6 -21.3,-21.3 v -85.8 c 0,-11.8 9.6,-21.3 21.3,-21.3 v 0 c 11.8,0 21.3,9.6 21.3,21.3 v 85.8 c 0,11.8 -9.5,21.3 -21.3,21.3 z"
-   id="path12" /> <path
-   class="st3"
-   d="m 551.6,275.7 v 0 c -11.8,0 -21.3,-9.6 -21.3,-21.3 v -85.8 c 0,-11.8 9.6,-21.3 21.3,-21.3 v 0 c 11.8,0 21.3,9.6 21.3,21.3 v 85.8 c 0.1,11.8 -9.5,21.3 -21.3,21.3 z"
-   id="path14" /> <rect
-   x="215.7"
-   y="370.89999"
-   class="st4"
-   width="75.199997"
-   height="75.199997"
-   id="rect16"
-   style="fill:#ffffff;fill-opacity:1" /> <rect
-   x="313"
-   y="370.89999"
-   class="st4"
-   width="75.199997"
-   height="75.199997"
-   id="rect18"
-   style="fill:#ffffff;fill-opacity:1" /> <rect
-   x="410.20001"
-   y="370.89999"
-   class="st4"
-   width="75.199997"
-   height="75.199997"
-   id="rect20"
-   style="fill:#ffffff;fill-opacity:1" /> <rect
-   x="507.5"
-   y="370.89999"
-   class="st4"
-   width="75.199997"
-   height="75.199997"
-   id="rect22"
-   style="fill:#ffffff;fill-opacity:1" /> <rect
-   x="215.7"
-   y="465"
-   class="st4"
-   width="75.199997"
-   height="75.199997"
-   id="rect24"
-   style="fill:#ffffff;fill-opacity:1" /> <rect
-   x="313"
-   y="465"
-   class="st1"
-   width="75.199997"
-   height="75.199997"
-   id="rect26"
-   style="fill:#27c12b;fill-opacity:1" /> <rect
-   x="410.20001"
-   y="465"
-   class="st4"
-   width="75.199997"
-   height="75.199997"
-   id="rect28"
-   style="fill:#ffffff;fill-opacity:1" /> <rect
-   x="507.5"
-   y="465"
-   class="st4"
-   width="75.199997"
-   height="75.199997"
-   id="rect30" /> <rect
-   x="215.7"
-   y="559.09998"
-   class="st4"
-   width="75.199997"
-   height="75.199997"
-   id="rect32"
-   style="fill:#ffffff;fill-opacity:1" /> <rect
-   x="313"
-   y="559.09998"
-   class="st4"
-   width="75.199997"
-   height="75.199997"
-   id="rect34"
-   style="fill:#ffffff;fill-opacity:1" /> <rect
-   x="410.20001"
-   y="559.09998"
-   class="st4"
-   width="75.199997"
-   height="75.199997"
-   id="rect36"
-   style="fill:#ffffff;fill-opacity:1" /> <rect
-   x="507.5"
-   y="559.09998"
-   class="st4"
-   width="75.199997"
-   height="75.199997"
-   id="rect38" /> <g
-   id="g70"> <circle
-   class="st5"
-   cx="621.90002"
-   cy="624"
-   r="188.7"
-   id="circle40" /> <circle
-   class="st0"
-   cx="621.90002"
-   cy="624"
-   r="148"
-   id="circle42" /> <path
-   class="st6"
-   d="m 486.6,636.8 c 0,-81.7 66.3,-148 148,-148 37.6,0 72,14.1 98.1,37.2 -27.1,-30.6 -66.7,-49.9 -110.8,-49.9 -81.7,0 -148,66.3 -148,148 0,44.1 19.3,83.7 49.9,110.8 -23.1,-26.2 -37.2,-60.5 -37.2,-98.1 z"
-   id="path44" /> <polyline
-   class="st7"
-   points="621.9,530.4 621.9,624 559,624  "
-   id="polyline46" /> <g
-   id="g64"> <line
-   class="st8"
-   x1="621.90002"
-   y1="508.29999"
-   x2="621.90002"
-   y2="497.10001"
-   id="line48" /> <line
-   class="st8"
-   x1="621.90002"
-   y1="756.29999"
-   x2="621.90002"
-   y2="745.09998"
-   id="line50" /> <line
-   class="st8"
-   x1="740.29999"
-   y1="626.70001"
-   x2="751.5"
-   y2="626.70001"
-   id="line52" /> <line
-   class="st8"
-   x1="492.29999"
-   y1="626.70001"
-   x2="503.5"
-   y2="626.70001"
-   id="line54" /> <line
-   class="st8"
-   x1="705.59998"
-   y1="710.40002"
-   x2="713.5"
-   y2="718.29999"
-   id="line56" /> <line
-   class="st8"
-   x1="530.29999"
-   y1="535.09998"
-   x2="538.20001"
-   y2="543"
-   id="line58" /> <line
-   class="st8"
-   x1="538.20001"
-   y1="710.40002"
-   x2="530.29999"
-   y2="718.29999"
-   id="line60" /> <line
-   class="st8"
-   x1="713.5"
-   y1="535.09998"
-   x2="705.59998"
-   y2="543"
-   id="line62" /> </g> <line
-   class="st9"
-   x1="604.40002"
-   y1="606.29999"
-   x2="684.5"
-   y2="687.40002"
-   id="line66" /> <circle
-   class="st10"
-   cx="621.90002"
-   cy="624"
-   r="16.1"
-   id="circle68" /> </g> </g> </svg>

changedetectionio/static/js/browser-steps.js

@@ -1,5 +1,14 @@
 $(document).ready(function () {
 
+    // duplicate
+    var csrftoken = $('input[name=csrf_token]').val();
+    $.ajaxSetup({
+        beforeSend: function (xhr, settings) {
+            if (!/^(GET|HEAD|OPTIONS|TRACE)$/i.test(settings.type) && !this.crossDomain) {
+                xhr.setRequestHeader("X-CSRFToken", csrftoken)
+            }
+        }
+    })
     var browsersteps_session_id;
     var browser_interface_seconds_remaining = 0;
     var apply_buttons_disabled = false;
@@ -17,8 +26,7 @@ $(document).ready(function () {
         set_scale();
     });
     // Should always be disabled
-    $('#browser_steps-0-operation option[value="Goto site"]').prop("selected", "selected");
-    $('#browser_steps-0-operation').attr('disabled', 'disabled');
+    $('#browser_steps >li:first-child select').val('Goto site').attr('disabled', 'disabled');
 
     $('#browsersteps-click-start').click(function () {
         $("#browsersteps-click-start").fadeOut();
@@ -221,7 +229,7 @@ $(document).ready(function () {
                     // If you switch to "Click X,y" after an element here is setup, it will give the last co-ords anyway
                     //if (x['isClickable'] || x['tagName'].startsWith('h') || x['tagName'] === 'a' || x['tagName'] === 'button' || x['tagtype'] === 'submit' || x['tagtype'] === 'checkbox' || x['tagtype'] === 'radio' || x['tagtype'] === 'li') {
                         $('select', first_available).val('Click element').change();
-                        $('input[type=text]', first_available).first().val(x['xpath']).focus();
+                        $('input[type=text]', first_available).first().val(x['xpath']);
                         found_something = true;
                     //}
                 }
@@ -305,7 +313,7 @@ $(document).ready(function () {
 
         if ($(this).val() === 'Click X,Y' && last_click_xy['x'] > 0 && $(elem_value).val().length === 0) {
             // @todo handle scale
-            $(elem_value).val(last_click_xy['x'] + ',' + last_click_xy['y']).focus();
+            $(elem_value).val(last_click_xy['x'] + ',' + last_click_xy['y']);
         }
     }).change();
 

changedetectionio/static/js/conditions.js

@@ -1,150 +0,0 @@
-$(document).ready(function () {
-    // Function to set up button event handlers
-    function setupButtonHandlers() {
-        // Unbind existing handlers first to prevent duplicates
-        $(".addRuleRow, .removeRuleRow, .verifyRuleRow").off("click");
-        
-        // Add row button handler
-        $(".addRuleRow").on("click", function(e) {
-            e.preventDefault();
-            
-            let currentRow = $(this).closest("tr");
-            
-            // Clone without events
-            let newRow = currentRow.clone(false);
-            
-            // Reset input values in the cloned row
-            newRow.find("input").val("");
-            newRow.find("select").prop("selectedIndex", 0);
-            
-            // Insert the new row after the current one
-            currentRow.after(newRow);
-            
-            // Reindex all rows
-            reindexRules();
-        });
-        
-        // Remove row button handler
-        $(".removeRuleRow").on("click", function(e) {
-            e.preventDefault();
-            
-            // Only remove if there's more than one row
-            if ($("#rulesTable tbody tr").length > 1) {
-                $(this).closest("tr").remove();
-                reindexRules();
-            }
-        });
-        
-        // Verify rule button handler
-        $(".verifyRuleRow").on("click", function(e) {
-            e.preventDefault();
-            
-            let row = $(this).closest("tr");
-            let field = row.find("select[name$='field']").val();
-            let operator = row.find("select[name$='operator']").val();
-            let value = row.find("input[name$='value']").val();
-            
-            // Validate that all fields are filled
-            if (!field || field === "None" || !operator || operator === "None" || !value) {
-                alert("Please fill in all fields (Field, Operator, and Value) before verifying.");
-                return;
-            }
-
-            
-            // Create a rule object
-            const rule = {
-                field: field,
-                operator: operator,
-                value: value
-            };
-            
-            // Show a spinner or some indication that verification is in progress
-            const $button = $(this);
-            const originalHTML = $button.html();
-            $button.html("⌛").prop("disabled", true);
-            
-            // Collect form data - similar to request_textpreview_update() in watch-settings.js
-            let formData = new FormData();
-            $('#edit-text-filter textarea, #edit-text-filter input').each(function() {
-                const $element = $(this);
-                const name = $element.attr('name');
-                if (name) {
-                    if ($element.is(':checkbox')) {
-                        formData.append(name, $element.is(':checked') ? $element.val() : false);
-                    } else {
-                        formData.append(name, $element.val());
-                    }
-                }
-            });
-            
-            // Also collect select values
-            $('#edit-text-filter select').each(function() {
-                const $element = $(this);
-                const name = $element.attr('name');
-                if (name) {
-                    formData.append(name, $element.val());
-                }
-            });
-
-
-            // Send the request to verify the rule
-            $.ajax({
-                url: verify_condition_rule_url+"?"+ new URLSearchParams({ rule: JSON.stringify(rule) }).toString(),
-                type: "POST",
-                data: formData,
-                processData: false, // Prevent jQuery from converting FormData to a string
-                contentType: false, // Let the browser set the correct content type
-                success: function (response) {
-                    if (response.status === "success") {
-                        if (response.result) {
-                            alert("✅ Condition PASSES verification against current snapshot!");
-                        } else {
-                            alert("❌ Condition FAILS verification against current snapshot.");
-                        }
-                    } else {
-                        alert("Error: " + response.message);
-                    }
-                    $button.html(originalHTML).prop("disabled", false);
-                },
-                error: function (xhr) {
-                    let errorMsg = "Error verifying condition.";
-                    if (xhr.responseJSON && xhr.responseJSON.message) {
-                        errorMsg = xhr.responseJSON.message;
-                    }
-                    alert(errorMsg);
-                    $button.html(originalHTML).prop("disabled", false);
-                }
-            });
-        });
-    }
-
-    // Function to reindex form elements and re-setup event handlers
-    function reindexRules() {
-        // Unbind all button handlers first
-        $(".addRuleRow, .removeRuleRow, .verifyRuleRow").off("click");
-        
-        // Reindex all form elements
-        $("#rulesTable tbody tr").each(function(index) {
-            $(this).find("select, input").each(function() {
-                let oldName = $(this).attr("name");
-                let oldId = $(this).attr("id");
-
-                if (oldName) {
-                    let newName = oldName.replace(/\d+/, index);
-                    $(this).attr("name", newName);
-                }
-
-                if (oldId) {
-                    let newId = oldId.replace(/\d+/, index);
-                    $(this).attr("id", newId);
-                }
-            });
-        });
-        
-        // Reattach event handlers after reindexing
-        setupButtonHandlers();
-    }
-
-    // Initial setup of button handlers
-    setupButtonHandlers();
-});

changedetectionio/static/js/csrf.js

@@ -1,10 +0,0 @@
-$(document).ready(function () {
-    $.ajaxSetup({
-        beforeSend: function (xhr, settings) {
-            if (!/^(GET|HEAD|OPTIONS|TRACE)$/i.test(settings.type) && !this.crossDomain) {
-                xhr.setRequestHeader("X-CSRFToken", csrftoken)
-            }
-        }
-    })
-});
-

changedetectionio/static/js/diff-overview.js

@@ -1,10 +1,12 @@
 $(document).ready(function () {
-    $('.needs-localtime').each(function () {
-        for (var option of this.options) {
-            var dateObject = new Date(option.value * 1000);
-            option.label = dateObject.toLocaleString(undefined, {dateStyle: "full", timeStyle: "medium"});
+    var csrftoken = $('input[name=csrf_token]').val();
+    $.ajaxSetup({
+        beforeSend: function (xhr, settings) {
+            if (!/^(GET|HEAD|OPTIONS|TRACE)$/i.test(settings.type) && !this.crossDomain) {
+                xhr.setRequestHeader("X-CSRFToken", csrftoken)
+            }
         }
-    });
+    })
 
     // Load it when the #screenshot tab is in use, so we dont give a slow experience when waiting for the text diff to load
     window.addEventListener('hashchange', function (e) {
@@ -39,12 +41,6 @@ $(document).ready(function () {
       $("#highlightSnippet").remove();
     }
 
-    // Listen for Escape key press
-    window.addEventListener('keydown', function (e) {
-        if (e.key === 'Escape') {
-            clean();
-        }
-    }, false);
 
     function dragTextHandler(event) {
         console.log('mouseupped');

changedetectionio/static/js/diff-render.js

@@ -79,7 +79,12 @@ $(document).ready(function () {
         $('#jump-next-diff').click();
     }
 
-
+    $('.needs-localtime').each(function () {
+        for (var option of this.options) {
+            var dateObject = new Date(option.value * 1000);
+            option.label = dateObject.toLocaleString(undefined, {dateStyle: "full", timeStyle: "medium"});
+        }
+    })
     onDiffTypeChange(
         document.querySelector('#settings [name="diff_type"]:checked'),
     );

changedetectionio/static/js/global-settings.js

@@ -18,25 +18,9 @@ $(document).ready(function () {
 
     });
 
-    $(".toggle-show").click(function (e) {
+    $("#notification-token-toggle").click(function (e) {
         e.preventDefault();
-        let target = $(this).data('target');
-        $(target).toggle();
+        $('#notification-tokens-info').toggle();
     });
-
-    // Time zone config related
-    $(".local-time").each(function (e) {
-        $(this).text(new Date($(this).data("utc")).toLocaleString());
-    })
-
-    const timezoneInput = $('#application-timezone');
-    if(timezoneInput.length) {
-        const timezone = Intl.DateTimeFormat().resolvedOptions().timeZone;
-        if (!timezoneInput.val().trim()) {
-            timezoneInput.val(timezone);
-            timezoneInput.after('<div class="timezone-message">The timezone was set from your browser, <strong>be sure to press save!</strong></div>');
-        }
-    }
-
 });
 

changedetectionio/static/js/limit.js

@@ -0,0 +1,56 @@
+/**
+ * debounce
+ * @param {integer} milliseconds This param indicates the number of milliseconds
+ *     to wait after the last call before calling the original function.
+ * @param {object} What "this" refers to in the returned function.
+ * @return {function} This returns a function that when called will wait the
+ *     indicated number of milliseconds after the last call before
+ *     calling the original function.
+ */
+Function.prototype.debounce = function (milliseconds, context) {
+    var baseFunction = this,
+        timer = null,
+        wait = milliseconds;
+
+    return function () {
+        var self = context || this,
+            args = arguments;
+
+        function complete() {
+            baseFunction.apply(self, args);
+            timer = null;
+        }
+
+        if (timer) {
+            clearTimeout(timer);
+        }
+
+        timer = setTimeout(complete, wait);
+    };
+};
+
+/**
+* throttle
+* @param {integer} milliseconds This param indicates the number of milliseconds
+*     to wait between calls before calling the original function.
+* @param {object} What "this" refers to in the returned function.
+* @return {function} This returns a function that when called will wait the
+*     indicated number of milliseconds between calls before
+*     calling the original function.
+*/
+Function.prototype.throttle = function (milliseconds, context) {
+    var baseFunction = this,
+        lastEventTimestamp = null,
+        limit = milliseconds;
+
+    return function () {
+        var self = context || this,
+            args = arguments,
+            now = Date.now();
+
+        if (!lastEventTimestamp || now - lastEventTimestamp >= limit) {
+            lastEventTimestamp = now;
+            baseFunction.apply(self, args);
+        }
+    };
+};

changedetectionio/static/js/notifications.js

@@ -1,52 +1,59 @@
-$(document).ready(function () {
+$(document).ready(function() {
 
-    $('#add-email-helper').click(function (e) {
-        e.preventDefault();
-        email = prompt("Destination email");
-        if (email) {
-            var n = $(".notification-urls");
-            var p = email_notification_prefix;
-            $(n).val($.trim($(n).val()) + "\n" + email_notification_prefix + email);
-        }
-    });
+  $('#add-email-helper').click(function (e) {
+    e.preventDefault();
+    email = prompt("Destination email");
+    if(email) {
+      var n = $(".notification-urls");
+      var p=email_notification_prefix;
+      $(n).val( $.trim( $(n).val() )+"\n"+email_notification_prefix+email );
+    }
+  });
 
-    $('#send-test-notification').click(function (e) {
-        e.preventDefault();
+  $('#send-test-notification').click(function (e) {
+    e.preventDefault();
 
-        data = {
-            notification_body: $('#notification_body').val(),
-            notification_format: $('#notification_format').val(),
-            notification_title: $('#notification_title').val(),
-            notification_urls: $('.notification-urls').val(),
-            tags: $('#tags').val(),
-            window_url: window.location.href,
-        }
-
-        $('.notifications-wrapper .spinner').fadeIn();
-        $('#notification-test-log').show();
-        $.ajax({
-            type: "POST",
-            url: notification_base_url,
-            data: data,
-            statusCode: {
-                400: function (data) {
-                    $("#notification-test-log>span").text(data.responseText);
-                },
+    // this can be global
+    var csrftoken = $('input[name=csrf_token]').val();
+    $.ajaxSetup({
+        beforeSend: function(xhr, settings) {
+            if (!/^(GET|HEAD|OPTIONS|TRACE)$/i.test(settings.type) && !this.crossDomain) {
+                xhr.setRequestHeader("X-CSRFToken", csrftoken)
             }
-        }).done(function (data) {
-            $("#notification-test-log>span").text(data);
-        }).fail(function (jqXHR, textStatus, errorThrown) {
-            // Handle connection refused or other errors
-            if (textStatus === "error" && errorThrown === "") {
-                console.error("Connection refused or server unreachable");
-                $("#notification-test-log>span").text("Error: Connection refused or server is unreachable.");
-            } else {
-                console.error("Error:", textStatus, errorThrown);
-                $("#notification-test-log>span").text("An error occurred: " + textStatus);
-            }
-        }).always(function () {
-            $('.notifications-wrapper .spinner').hide();
-        })
-    });
+        }
+    })
+
+    data = {
+      notification_body: $('#notification_body').val(),
+      notification_format: $('#notification_format').val(),
+      notification_title: $('#notification_title').val(),
+      notification_urls: $('.notification-urls').val(),
+      window_url: window.location.href,
+    }
+
+
+    if (!data['notification_urls'].length) {
+      alert("Notification URL list is empty, cannot send test.")
+      return;
+    }
+
+    $.ajax({
+      type: "POST",
+      url: notification_base_url,
+      data : data,
+        statusCode: {
+        400: function() {
+            // More than likely the CSRF token was lost when the server restarted
+          alert("There was a problem processing the request, please reload the page.");
+        }
+      }
+    }).done(function(data){
+      console.log(data);
+      alert('Sent');
+    }).fail(function(data){
+      console.log(data);
+      alert('There was an error communicating with the server.');
+    })
+  });
 });
 

changedetectionio/static/js/plugins.js

@@ -1,196 +0,0 @@
-(function ($) {
-    /**
-     * debounce
-     * @param {integer} milliseconds This param indicates the number of milliseconds
-     *     to wait after the last call before calling the original function.
-     * @param {object} What "this" refers to in the returned function.
-     * @return {function} This returns a function that when called will wait the
-     *     indicated number of milliseconds after the last call before
-     *     calling the original function.
-     */
-    Function.prototype.debounce = function (milliseconds, context) {
-        var baseFunction = this,
-            timer = null,
-            wait = milliseconds;
-
-        return function () {
-            var self = context || this,
-                args = arguments;
-
-            function complete() {
-                baseFunction.apply(self, args);
-                timer = null;
-            }
-
-            if (timer) {
-                clearTimeout(timer);
-            }
-
-            timer = setTimeout(complete, wait);
-        };
-    };
-
-    /**
-     * throttle
-     * @param {integer} milliseconds This param indicates the number of milliseconds
-     *     to wait between calls before calling the original function.
-     * @param {object} What "this" refers to in the returned function.
-     * @return {function} This returns a function that when called will wait the
-     *     indicated number of milliseconds between calls before
-     *     calling the original function.
-     */
-    Function.prototype.throttle = function (milliseconds, context) {
-        var baseFunction = this,
-            lastEventTimestamp = null,
-            limit = milliseconds;
-
-        return function () {
-            var self = context || this,
-                args = arguments,
-                now = Date.now();
-
-            if (!lastEventTimestamp || now - lastEventTimestamp >= limit) {
-                lastEventTimestamp = now;
-                baseFunction.apply(self, args);
-            }
-        };
-    };
-
-    $.fn.highlightLines = function (configurations) {
-        return this.each(function () {
-            const $pre = $(this);
-            const textContent = $pre.text();
-            const lines = textContent.split(/\r?\n/); // Handles both \n and \r\n line endings
-
-            // Build a map of line numbers to styles
-            const lineStyles = {};
-
-            configurations.forEach(config => {
-                const {color, lines: lineNumbers} = config;
-                lineNumbers.forEach(lineNumber => {
-                    lineStyles[lineNumber] = color;
-                });
-            });
-
-            // Function to escape HTML characters
-            function escapeHtml(text) {
-                return text.replace(/[&<>"'`=\/]/g, function (s) {
-                    return "&#" + s.charCodeAt(0) + ";";
-                });
-            }
-
-            // Process each line
-            const processedLines = lines.map((line, index) => {
-                const lineNumber = index + 1; // Line numbers start at 1
-                const escapedLine = escapeHtml(line);
-                const color = lineStyles[lineNumber];
-
-                if (color) {
-                    // Wrap the line in a span with inline style
-                    return `<span style="background-color: ${color}">${escapedLine}</span>`;
-                } else {
-                    return escapedLine;
-                }
-            });
-
-            // Join the lines back together
-            const newContent = processedLines.join('\n');
-
-            // Set the new content as HTML
-            $pre.html(newContent);
-        });
-    };
-    $.fn.miniTabs = function (tabsConfig, options) {
-        const settings = {
-            tabClass: 'minitab',
-            tabsContainerClass: 'minitabs',
-            activeClass: 'active',
-            ...(options || {})
-        };
-
-        return this.each(function () {
-            const $wrapper = $(this);
-            const $contents = $wrapper.find('div[id]').hide();
-            const $tabsContainer = $('<div>', {class: settings.tabsContainerClass}).prependTo($wrapper);
-
-            // Generate tabs
-            Object.entries(tabsConfig).forEach(([tabTitle, contentSelector], index) => {
-                const $content = $wrapper.find(contentSelector);
-                if (index === 0) $content.show(); // Show first content by default
-
-                $('<a>', {
-                    class: `${settings.tabClass}${index === 0 ? ` ${settings.activeClass}` : ''}`,
-                    text: tabTitle,
-                    'data-target': contentSelector
-                }).appendTo($tabsContainer);
-            });
-
-            // Tab click event
-            $tabsContainer.on('click', `.${settings.tabClass}`, function (e) {
-                e.preventDefault();
-                const $tab = $(this);
-                const target = $tab.data('target');
-
-                // Update active tab
-                $tabsContainer.find(`.${settings.tabClass}`).removeClass(settings.activeClass);
-                $tab.addClass(settings.activeClass);
-
-                // Show/hide content
-                $contents.hide();
-                $wrapper.find(target).show();
-            });
-        });
-    };
-
-    // Object to store ongoing requests by namespace
-    const requests = {};
-
-    $.abortiveSingularAjax = function (options) {
-        const namespace = options.namespace || 'default';
-
-        // Abort the current request in this namespace if it's still ongoing
-        if (requests[namespace]) {
-            requests[namespace].abort();
-        }
-
-        // Start a new AJAX request and store its reference in the correct namespace
-        requests[namespace] = $.ajax(options);
-
-        // Return the current request in case it's needed
-        return requests[namespace];
-    };
-})(jQuery);
-
-
-
-function toggleOpacity(checkboxSelector, fieldSelector, inverted) {
-    const checkbox = document.querySelector(checkboxSelector);
-    const fields = document.querySelectorAll(fieldSelector);
-
-    function updateOpacity() {
-        const opacityValue = !checkbox.checked ? (inverted ? 0.6 : 1) : (inverted ? 1 : 0.6);
-        fields.forEach(field => {
-            field.style.opacity = opacityValue;
-        });
-    }
-
-    // Initial setup
-    updateOpacity();
-    checkbox.addEventListener('change', updateOpacity);
-}
-
-function toggleVisibility(checkboxSelector, fieldSelector, inverted) {
-    const checkbox = document.querySelector(checkboxSelector);
-    const fields = document.querySelectorAll(fieldSelector);
-
-    function updateOpacity() {
-        const opacityValue = !checkbox.checked ? (inverted ? 'none' : 'block') : (inverted ? 'block' : 'none');
-        fields.forEach(field => {
-            field.style.display = opacityValue;
-        });
-    }
-
-    // Initial setup
-    updateOpacity();
-    checkbox.addEventListener('change', updateOpacity);
-}

changedetectionio/static/js/preview.js

@@ -1,63 +0,0 @@
-function redirectToVersion(version) {
-    var currentUrl = window.location.href.split('?')[0]; // Base URL without query parameters
-    var anchor = '';
-
-    // Check if there is an anchor
-    if (currentUrl.indexOf('#') !== -1) {
-        anchor = currentUrl.substring(currentUrl.indexOf('#'));
-        currentUrl = currentUrl.substring(0, currentUrl.indexOf('#'));
-    }
-
-    window.location.href = currentUrl + '?version=' + version + anchor;
-}
-
-function setupDateWidget() {
-    $(document).on('keydown', function (event) {
-        var $selectElement = $('#preview-version');
-        var $selectedOption = $selectElement.find('option:selected');
-
-        if ($selectedOption.length) {
-            if (event.key === 'ArrowLeft' && $selectedOption.prev().length) {
-                redirectToVersion($selectedOption.prev().val());
-            } else if (event.key === 'ArrowRight' && $selectedOption.next().length) {
-                redirectToVersion($selectedOption.next().val());
-            }
-        }
-    });
-
-    $('#preview-version').on('change', function () {
-        redirectToVersion($(this).val());
-    });
-
-    var $selectedOption = $('#preview-version option:selected');
-
-    if ($selectedOption.length) {
-        var $prevOption = $selectedOption.prev();
-        var $nextOption = $selectedOption.next();
-
-        if ($prevOption.length) {
-            $('#btn-previous').attr('href', '?version=' + $prevOption.val());
-        } else {
-            $('#btn-previous').remove();
-        }
-
-        if ($nextOption.length) {
-            $('#btn-next').attr('href', '?version=' + $nextOption.val());
-        } else {
-            $('#btn-next').remove();
-        }
-    }
-}
-
-$(document).ready(function () {
-    if ($('#preview-version').length) {
-        setupDateWidget();
-    }
-
-    $('#diff-col > pre').highlightLines([
-        {
-            'color': '#ee0000',
-            'lines': triggered_line_numbers
-        }
-    ]);
-});

changedetectionio/static/js/recheck-proxy.js

@@ -1,14 +1,14 @@
 $(function () {
     /* add container before each proxy location to show status */
-    var isActive = false;
 
-    function setup_html_widget() {
-        var option_li = $('.fetch-backend-proxy li').filter(function () {
-            return $("input", this)[0].value.length > 0;
-        });
-        $(option_li).prepend('<div class="proxy-status"></div>');
-        $(option_li).append('<div class="proxy-timing"></div><div class="proxy-check-details"></div>');
-    }
+    var option_li = $('.fetch-backend-proxy li').filter(function() {
+        return $("input",this)[0].value.length >0;
+    });
+
+    //var option_li = $('.fetch-backend-proxy li');
+    var isActive = false;
+    $(option_li).prepend('<div class="proxy-status"></div>');
+    $(option_li).append('<div class="proxy-timing"></div><div class="proxy-check-details"></div>');
 
     function set_proxy_check_status(proxy_key, state) {
         // select input by value name
@@ -59,14 +59,8 @@ $(function () {
     }
 
     $('#check-all-proxies').click(function (e) {
-
         e.preventDefault()
-
-        if (!$('body').hasClass('proxy-check-active')) {
-            setup_html_widget();
-            $('body').addClass('proxy-check-active');
-        }
-
+        $('body').addClass('proxy-check-active');
         $('.proxy-check-details').html('');
         $('.proxy-status').html('<span class="spinner"></span>').fadeIn();
         $('.proxy-timing').html('');

changedetectionio/static/js/scheduler.js

@@ -1,109 +0,0 @@
-function getTimeInTimezone(timezone) {
-    const now = new Date();
-    const options = {
-        timeZone: timezone,
-        weekday: 'long',
-        year: 'numeric',
-        hour12: false,
-        month: '2-digit',
-        day: '2-digit',
-        hour: '2-digit',
-        minute: '2-digit',
-        second: '2-digit',
-    };
-
-    const formatter = new Intl.DateTimeFormat('en-US', options);
-    return formatter.format(now);
-}
-
-$(document).ready(function () {
-
-    let exceedsLimit = false;
-    const warning_text = $("#timespan-warning")
-    const timezone_text_widget = $("input[id*='time_schedule_limit-timezone']")
-
-    toggleVisibility('#time_schedule_limit-enabled, #requests-time_schedule_limit-enabled', '#schedule-day-limits-wrapper', true)
-
-    setInterval(() => {
-        let success = true;
-        try {
-            // Show the current local time according to either placeholder or entered TZ name
-            if (timezone_text_widget.val().length) {
-                $('#local-time-in-tz').text(getTimeInTimezone(timezone_text_widget.val()));
-            } else {
-                // So maybe use what is in the placeholder (which will be the default settings)
-                $('#local-time-in-tz').text(getTimeInTimezone(timezone_text_widget.attr('placeholder')));
-            }
-        } catch (error) {
-            success = false;
-            $('#local-time-in-tz').text("");
-            console.error(timezone_text_widget.val())
-        }
-
-        $(timezone_text_widget).toggleClass('error', !success);
-
-    }, 500);
-
-    $('#schedule-day-limits-wrapper').on('change click blur', 'input, checkbox, select', function() {
-
-        let allOk = true;
-
-        // Controls setting the warning that the time could overlap into the next day
-        $("li.day-schedule").each(function () {
-            const $schedule = $(this);
-            const $checkbox = $schedule.find("input[type='checkbox']");
-
-            if ($checkbox.is(":checked")) {
-                const timeValue = $schedule.find("input[type='time']").val();
-                const durationHours = parseInt($schedule.find("select[name*='-duration-hours']").val(), 10) || 0;
-                const durationMinutes = parseInt($schedule.find("select[name*='-duration-minutes']").val(), 10) || 0;
-
-                if (timeValue) {
-                    const [startHours, startMinutes] = timeValue.split(":").map(Number);
-                    const totalMinutes = (startHours * 60 + startMinutes) + (durationHours * 60 + durationMinutes);
-
-                    exceedsLimit = totalMinutes > 1440
-                    if (exceedsLimit) {
-                        allOk = false
-                    }
-                    // Set the row/day-of-week highlight
-                    $schedule.toggleClass("warning", exceedsLimit);
-                }
-            } else {
-                $schedule.toggleClass("warning", false);
-            }
-        });
-
-        warning_text.toggle(!allOk)
-    });
-
-    $('table[id*="time_schedule_limit-saturday"], table[id*="time_schedule_limit-sunday"]').addClass("weekend-day")
-
-    // Presets [weekend] [business hours] etc
-    $(document).on('click', '[data-template].set-schedule', function () {
-        // Get the value of the 'data-template' attribute
-        switch ($(this).attr('data-template')) {
-            case 'business-hours':
-                $('.day-schedule table:not(.weekend-day) input[type="time"]').val('09:00')
-                $('.day-schedule table:not(.weekend-day) select[id*="-duration-hours"]').val('8');
-                $('.day-schedule table:not(.weekend-day) select[id*="-duration-minutes"]').val('0');
-                $('.day-schedule input[id*="-enabled"]').prop('checked', true);
-                $('.day-schedule .weekend-day input[id*="-enabled"]').prop('checked', false);
-                break;
-            case 'weekend':
-                $('.day-schedule .weekend-day input[type="time"][id$="start-time"]').val('00:00')
-                $('.day-schedule .weekend-day select[id*="-duration-hours"]').val('24');
-                $('.day-schedule .weekend-day select[id*="-duration-minutes"]').val('0');
-                $('.day-schedule input[id*="-enabled"]').prop('checked', false);
-                $('.day-schedule .weekend-day input[id*="-enabled"]').prop('checked', true);
-                break;
-            case 'reset':
-
-                $('.day-schedule input[type="time"]').val('00:00')
-                $('.day-schedule select[id*="-duration-hours"]').val('24');
-                $('.day-schedule select[id*="-duration-minutes"]').val('0');
-                $('.day-schedule input[id*="-enabled"]').prop('checked', true);
-                break;
-        }
-    });
-});

changedetectionio/static/js/tabs.js

@@ -26,6 +26,8 @@ function set_active_tab() {
     if (tab.length) {
         tab[0].parentElement.className = "active";
     }
+    // hash could move the page down
+    window.scrollTo(0, 0);
 }
 
 function focus_error_tab() {

changedetectionio/static/js/toggle-theme.js

@@ -49,9 +49,4 @@ $(document).ready(function () {
         $("#overlay").toggleClass('visible');
         heartpath.style.fill = document.getElementById("overlay").classList.contains("visible") ? '#ff0000' : 'var(--color-background)';
     });
-
-    setInterval(function () {
-        $('body').toggleClass('spinner-active', $.active > 0);
-    }, 2000);
-
 });

changedetectionio/static/js/visual-selector.js

@@ -2,259 +2,250 @@
 // All rights reserved.
 // yes - this is really a hack, if you are a front-ender and want to help, please get in touch!
 
-let runInClearMode = false;
+$(document).ready(function () {
 
-$(document).ready(() => {
-    let currentSelections = [];
-    let currentSelection = null;
-    let appendToList = false;
-    let c, xctx, ctx;
-    let xScale = 1, yScale = 1;
-    let selectorImage, selectorImageRect, selectorData;
+    var current_selected_i;
+    var state_clicked = false;
 
+    var c;
 
-    // Global jQuery selectors with "Elem" appended
-    const $selectorCanvasElem = $('#selector-canvas');
-    const $includeFiltersElem = $("#include_filters");
-    const $selectorBackgroundElem = $("img#selector-background");
-    const $selectorCurrentXpathElem = $("#selector-current-xpath span");
-    const $fetchingUpdateNoticeElem = $('.fetching-update-notice');
-    const $selectorWrapperElem = $("#selector-wrapper");
+    // greyed out fill context
+    var xctx;
+    // redline highlight context
+    var ctx;
 
-    // Color constants
-    const FILL_STYLE_HIGHLIGHT = 'rgba(205,0,0,0.35)';
-    const FILL_STYLE_GREYED_OUT = 'rgba(205,205,205,0.95)';
-    const STROKE_STYLE_HIGHLIGHT = 'rgba(255,0,0, 0.9)';
-    const FILL_STYLE_REDLINE = 'rgba(255,0,0, 0.1)';
-    const STROKE_STYLE_REDLINE = 'rgba(225,0,0,0.9)';
+    var current_default_xpath = [];
+    var x_scale = 1;
+    var y_scale = 1;
+    var selector_image;
+    var selector_image_rect;
+    var selector_data;
 
-    $('#visualselector-tab').click(() => {
-        $selectorBackgroundElem.off('load');
-        currentSelections = [];
-        bootstrapVisualSelector();
+    $('#visualselector-tab').click(function () {
+        $("img#selector-background").off('load');
+        state_clicked = false;
+        current_selected_i = false;
+        bootstrap_visualselector();
     });
 
-    function clearReset() {
-        ctx.clearRect(0, 0, c.width, c.height);
-
-        if ($includeFiltersElem.val().length) {
-            alert("Existing filters under the 'Filters & Triggers' tab were cleared.");
-        }
-        $includeFiltersElem.val('');
-
-        currentSelections = [];
-
-        // Means we ignore the xpaths from the scraper marked as sel.highlight_as_custom_filter (it matched a previous selector)
-        runInClearMode = true;
-
-        highlightCurrentSelected();
-    }
-
-    function splitToList(v) {
-        return v.split('\n').map(line => line.trim()).filter(line => line.length > 0);
-    }
-
-    function sortScrapedElementsBySize() {
-        // Sort the currentSelections array by area (width * height) in descending order
-        selectorData['size_pos'].sort((a, b) => {
-            const areaA = a.width * a.height;
-            const areaB = b.width * b.height;
-            return areaB - areaA;
-        });
-    }
-
-    $(document).on('keydown keyup', (event) => {
-        if (event.code === 'ShiftLeft' || event.code === 'ShiftRight') {
-            appendToList = event.type === 'keydown';
-        }
-
-        if (event.type === 'keydown') {
-            if ($selectorBackgroundElem.is(":visible") && event.key === "Escape") {
-                clearReset();
+    $(document).on('keydown', function (event) {
+        if ($("img#selector-background").is(":visible")) {
+            if (event.key == "Escape") {
+                state_clicked = false;
+                ctx.clearRect(0, 0, c.width, c.height);
             }
         }
     });
 
-    $('#clear-selector').on('click', () => {
-        clearReset();
-    });
-    // So if they start switching between visualSelector and manual filters, stop it from rendering old filters
-    $('li.tab a').on('click', () => {
-        runInClearMode = true;
-    });
-
-    if (!window.location.hash || window.location.hash !== '#visualselector') {
-        $selectorBackgroundElem.attr('src', '');
+    // For when the page loads
+    if (!window.location.hash || window.location.hash != '#visualselector') {
+        $("img#selector-background").attr('src', '');
         return;
     }
 
-    bootstrapVisualSelector();
+    // Handle clearing button/link
+    $('#clear-selector').on('click', function (event) {
+        if (!state_clicked) {
+            alert('Oops, Nothing selected!');
+        }
+        state_clicked = false;
+        ctx.clearRect(0, 0, c.width, c.height);
+        xctx.clearRect(0, 0, c.width, c.height);
+        $("#include_filters").val('');
+    });
 
-    function bootstrapVisualSelector() {
-        $selectorBackgroundElem
-            .on("error", () => {
-                $fetchingUpdateNoticeElem.html("<strong>Ooops!</strong> The VisualSelector tool needs at least one fetched page, please unpause the watch and/or wait for the watch to complete fetching and then reload this page.")
-                    .css('color', '#bb0000');
-                $('#selector-current-xpath, #clear-selector').hide();
-            })
-            .on('load', () => {
+
+    bootstrap_visualselector();
+
+
+    function bootstrap_visualselector() {
+        if (1) {
+            // bootstrap it, this will trigger everything else
+            $("img#selector-background").on("error", function () {
+                $('.fetching-update-notice').html("<strong>Ooops!</strong> The VisualSelector tool needs atleast one fetched page, please unpause the watch and/or wait for the watch to complete fetching and then reload this page.");
+                $('.fetching-update-notice').css('color','#bb0000');
+                $('#selector-current-xpath').hide();
+                $('#clear-selector').hide();
+            }).bind('load', function () {
                 console.log("Loaded background...");
                 c = document.getElementById("selector-canvas");
+                // greyed out fill context
                 xctx = c.getContext("2d");
+                // redline highlight context
                 ctx = c.getContext("2d");
-                fetchData();
-                $selectorCanvasElem.off("mousemove mousedown");
-            })
-            .attr("src", screenshot_url);
-
-        let s = `${$selectorBackgroundElem.attr('src')}?${new Date().getTime()}`;
-        $selectorBackgroundElem.attr('src', s);
-    }
-
-    function alertIfFilterNotFound() {
-        let existingFilters = splitToList($includeFiltersElem.val());
-        let sizePosXpaths = selectorData['size_pos'].map(sel => sel.xpath);
-
-        for (let filter of existingFilters) {
-            if (!sizePosXpaths.includes(filter)) {
-                alert(`One or more of your existing filters was not found and will be removed when a new filter is selected.`);
-                break;
-            }
+                if ($("#include_filters").val().trim().length) {
+                    current_default_xpath = $("#include_filters").val().split(/\r?\n/g);
+                } else {
+                    current_default_xpath = [];
+                }
+                fetch_data();
+                $('#selector-canvas').off("mousemove mousedown");
+                // screenshot_url defined in the edit.html template
+            }).attr("src", screenshot_url);
         }
+        // Tell visualSelector that the image should update
+        var s = $("img#selector-background").attr('src') + "?" + new Date().getTime();
+        $("img#selector-background").attr('src', s)
     }
 
-    function fetchData() {
-        $fetchingUpdateNoticeElem.html("Fetching element data..");
+    // This is fired once the img src is loaded in bootstrap_visualselector()
+    function fetch_data() {
+        // Image is ready
+        $('.fetching-update-notice').html("Fetching element data..");
 
         $.ajax({
             url: watch_visual_selector_data_url,
             context: document.body
-        }).done((data) => {
-            $fetchingUpdateNoticeElem.html("Rendering..");
-            selectorData = data;
-
-            sortScrapedElementsBySize();
-            console.log(`Reported browser width from backend: ${data['browser_width']}`);
-
-            // Little sanity check for the user, alert them if something missing
-            alertIfFilterNotFound();
-
-            setScale();
-            reflowSelector();
-            $fetchingUpdateNoticeElem.fadeOut();
+        }).done(function (data) {
+            $('.fetching-update-notice').html("Rendering..");
+            selector_data = data;
+            console.log("Reported browser width from backend: " + data['browser_width']);
+            state_clicked = false;
+            set_scale();
+            reflow_selector();
+            $('.fetching-update-notice').fadeOut();
         });
+
     }
 
-    function updateFiltersText() {
-        // Assuming currentSelections is already defined and contains the selections
-        let uniqueSelections = new Set(currentSelections.map(sel => (sel[0] === '/' ? `xpath:${sel.xpath}` : sel.xpath)));
 
-        if (currentSelections.length > 0) {
-            // Convert the Set back to an array and join with newline characters
-            let textboxFilterText = Array.from(uniqueSelections).join("\n");
-            $includeFiltersElem.val(textboxFilterText);
-        }
-    }
+    function set_scale() {
 
-    function setScale() {
-        $selectorWrapperElem.show();
-        selectorImage = $selectorBackgroundElem[0];
-        selectorImageRect = selectorImage.getBoundingClientRect();
+        // some things to check if the scaling doesnt work
+        // - that the widths/sizes really are about the actual screen size cat elements.json |grep -o width......|sort|uniq
+        $("#selector-wrapper").show();
+        selector_image = $("img#selector-background")[0];
+        selector_image_rect = selector_image.getBoundingClientRect();
 
-        $selectorCanvasElem.attr({
-            'height': selectorImageRect.height,
-            'width': selectorImageRect.width
-        });
-        $selectorWrapperElem.attr('width', selectorImageRect.width);
-        $('#visual-selector-heading').css('max-width', selectorImageRect.width + "px")
-
-        xScale = selectorImageRect.width / selectorImage.naturalWidth;
-        yScale = selectorImageRect.height / selectorImage.naturalHeight;
-
-        ctx.strokeStyle = STROKE_STYLE_HIGHLIGHT;
-        ctx.fillStyle = FILL_STYLE_REDLINE;
+        // make the canvas the same size as the image
+        $('#selector-canvas').attr('height', selector_image_rect.height);
+        $('#selector-canvas').attr('width', selector_image_rect.width);
+        $('#selector-wrapper').attr('width', selector_image_rect.width);
+        x_scale = selector_image_rect.width / selector_data['browser_width'];
+        y_scale = selector_image_rect.height / selector_image.naturalHeight;
+        ctx.strokeStyle = 'rgba(255,0,0, 0.9)';
+        ctx.fillStyle = 'rgba(255,0,0, 0.1)';
         ctx.lineWidth = 3;
-        console.log("Scaling set  x: " + xScale + " by y:" + yScale);
-        $("#selector-current-xpath").css('max-width', selectorImageRect.width);
+        console.log("scaling set  x: " + x_scale + " by y:" + y_scale);
+        $("#selector-current-xpath").css('max-width', selector_image_rect.width);
     }
 
-    function reflowSelector() {
-        $(window).resize(() => {
-            setScale();
-            highlightCurrentSelected();
+    function reflow_selector() {
+        $(window).resize(function () {
+            set_scale();
+            highlight_current_selected_i();
         });
+        var selector_currnt_xpath_text = $("#selector-current-xpath span");
 
-        setScale();
+        set_scale();
 
-        console.log(selectorData['size_pos'].length + " selectors found");
+        console.log(selector_data['size_pos'].length + " selectors found");
 
-        let existingFilters = splitToList($includeFiltersElem.val());
-
-        selectorData['size_pos'].forEach(sel => {
-            if ((!runInClearMode && sel.highlight_as_custom_filter) || existingFilters.includes(sel.xpath)) {
-                console.log("highlighting " + c);
-                currentSelections.push(sel);
+        // highlight the default one if we can find it in the xPath list
+        // or the xpath matches the default one
+        found = false;
+        if (current_default_xpath.length) {
+            // Find the first one that matches
+            // @todo In the future paint all that match
+            for (const c of current_default_xpath) {
+                for (var i = selector_data['size_pos'].length; i !== 0; i--) {
+                    if (selector_data['size_pos'][i - 1].xpath.trim() === c.trim()) {
+                        console.log("highlighting " + c);
+                        current_selected_i = i - 1;
+                        highlight_current_selected_i();
+                        found = true;
+                        break;
+                    }
+                }
+                if (found) {
+                    break;
+                }
             }
-        });
+            if (!found) {
+                alert("Unfortunately your existing CSS/xPath Filter was no longer found!");
+            }
+        }
 
 
-        highlightCurrentSelected();
-        updateFiltersText();
+        $('#selector-canvas').bind('mousemove', function (e) {
+            if (state_clicked) {
+                return;
+            }
+            ctx.clearRect(0, 0, c.width, c.height);
+            current_selected_i = null;
 
-        $selectorCanvasElem.bind('mousemove', handleMouseMove.debounce(5));
-        $selectorCanvasElem.bind('mousedown', handleMouseDown.debounce(5));
-        $selectorCanvasElem.bind('mouseleave', highlightCurrentSelected.debounce(5));
-
-        function handleMouseMove(e) {
-            if (!e.offsetX && !e.offsetY) {
-                const targetOffset = $(e.target).offset();
+            // Add in offset
+            if ((typeof e.offsetX === "undefined" || typeof e.offsetY === "undefined") || (e.offsetX === 0 && e.offsetY === 0)) {
+                var targetOffset = $(e.target).offset();
                 e.offsetX = e.pageX - targetOffset.left;
                 e.offsetY = e.pageY - targetOffset.top;
             }
 
-            ctx.fillStyle = FILL_STYLE_HIGHLIGHT;
+            // Reverse order - the most specific one should be deeper/"laster"
+            // Basically, find the most 'deepest'
+            var found = 0;
+            ctx.fillStyle = 'rgba(205,0,0,0.35)';
+            // Will be sorted by smallest width*height first
+            for (var i = 0; i <= selector_data['size_pos'].length; i++) {
+                // draw all of them? let them choose somehow?
+                var sel = selector_data['size_pos'][i];
+                // If we are in a bounding-box
+                if (e.offsetY > sel.top * y_scale && e.offsetY < sel.top * y_scale + sel.height * y_scale
+                    &&
+                    e.offsetX > sel.left * y_scale && e.offsetX < sel.left * y_scale + sel.width * y_scale
 
-            selectorData['size_pos'].forEach(sel => {
-                if (e.offsetY > sel.top * yScale && e.offsetY < sel.top * yScale + sel.height * yScale &&
-                    e.offsetX > sel.left * yScale && e.offsetX < sel.left * yScale + sel.width * yScale) {
-                    setCurrentSelectedText(sel.xpath);
-                    drawHighlight(sel);
-                    currentSelections.push(sel);
-                    currentSelection = sel;
-                    highlightCurrentSelected();
-                    currentSelections.pop();
+                ) {
+
+                    // FOUND ONE
+                    set_current_selected_text(sel.xpath);
+                    ctx.strokeRect(sel.left * x_scale, sel.top * y_scale, sel.width * x_scale, sel.height * y_scale);
+                    ctx.fillRect(sel.left * x_scale, sel.top * y_scale, sel.width * x_scale, sel.height * y_scale);
+
+                    // no need to keep digging
+                    // @todo or, O to go out/up, I to go in
+                    // or double click to go up/out the selector?
+                    current_selected_i = i;
+                    found += 1;
+                    break;
                 }
-            })
+            }
+
+        }.debounce(5));
+
+        function set_current_selected_text(s) {
+            selector_currnt_xpath_text[0].innerHTML = s;
+        }
+
+        function highlight_current_selected_i() {
+            if (state_clicked) {
+                state_clicked = false;
+                xctx.clearRect(0, 0, c.width, c.height);
+                return;
+            }
+
+            var sel = selector_data['size_pos'][current_selected_i];
+            if (sel[0] == '/') {
+                // @todo - not sure just checking / is right
+                $("#include_filters").val('xpath:' + sel.xpath);
+            } else {
+                $("#include_filters").val(sel.xpath);
+            }
+            xctx.fillStyle = 'rgba(205,205,205,0.95)';
+            xctx.strokeStyle = 'rgba(225,0,0,0.9)';
+            xctx.lineWidth = 3;
+            xctx.fillRect(0, 0, c.width, c.height);
+            // Clear out what only should be seen (make a clear/clean spot)
+            xctx.clearRect(sel.left * x_scale, sel.top * y_scale, sel.width * x_scale, sel.height * y_scale);
+            xctx.strokeRect(sel.left * x_scale, sel.top * y_scale, sel.width * x_scale, sel.height * y_scale);
+            state_clicked = true;
+            set_current_selected_text(sel.xpath);
+
         }
 
 
-        function setCurrentSelectedText(s) {
-            $selectorCurrentXpathElem[0].innerHTML = s;
-        }
-
-        function drawHighlight(sel) {
-            ctx.strokeRect(sel.left * xScale, sel.top * yScale, sel.width * xScale, sel.height * yScale);
-            ctx.fillRect(sel.left * xScale, sel.top * yScale, sel.width * xScale, sel.height * yScale);
-        }
-
-        function handleMouseDown() {
-            // If we are in 'appendToList' mode, grow the list, if not, just 1
-            currentSelections = appendToList ? [...currentSelections, currentSelection] : [currentSelection];
-            highlightCurrentSelected();
-            updateFiltersText();
-        }
-
-    }
-
-    function highlightCurrentSelected() {
-        xctx.fillStyle = FILL_STYLE_GREYED_OUT;
-        xctx.strokeStyle = STROKE_STYLE_REDLINE;
-        xctx.lineWidth = 3;
-        xctx.clearRect(0, 0, c.width, c.height);
-
-        currentSelections.forEach(sel => {
-            //xctx.clearRect(sel.left * xScale, sel.top * yScale, sel.width * xScale, sel.height * yScale);
-            xctx.strokeRect(sel.left * xScale, sel.top * yScale, sel.width * xScale, sel.height * yScale);
+        $('#selector-canvas').bind('mousedown', function (e) {
+            highlight_current_selected_i();
         });
     }
+
 });

changedetectionio/static/js/watch-settings.js

@@ -1,51 +1,4 @@
-
-function request_textpreview_update() {
-    if (!$('body').hasClass('preview-text-enabled')) {
-        console.error("Preview text was requested but body tag was not setup")
-        return
-    }
-
-    const data = {};
-    $('textarea:visible, input:visible').each(function () {
-        const $element = $(this); // Cache the jQuery object for the current element
-        const name = $element.attr('name'); // Get the name attribute of the element
-        data[name] = $element.is(':checkbox') ? ($element.is(':checked') ? $element.val() : false) : $element.val();
-    });
-
-    $('body').toggleClass('spinner-active', 1);
-
-    $.abortiveSingularAjax({
-        type: "POST",
-        url: preview_text_edit_filters_url,
-        data: data,
-        namespace: 'watchEdit'
-    }).done(function (data) {
-        console.debug(data['duration'])
-        $('#filters-and-triggers #text-preview-before-inner').text(data['before_filter']);
-        $('#filters-and-triggers #text-preview-inner')
-            .text(data['after_filter'])
-            .highlightLines([
-                {
-                    'color': '#ee0000',
-                    'lines': data['trigger_line_numbers']
-                },
-                {
-                    'color': '#757575',
-                    'lines': data['ignore_line_numbers']
-                }
-            ])
-    }).fail(function (error) {
-        if (error.statusText === 'abort') {
-            console.log('Request was aborted due to a new request being fired.');
-        } else {
-            $('#filters-and-triggers #text-preview-inner').text('There was an error communicating with the server.');
-        }
-    })
-}
-
-
 $(document).ready(function () {
-
     $('#notification-setting-reset-to-default').click(function (e) {
         $('#notification_title').val('');
         $('#notification_body').val('');
@@ -57,25 +10,4 @@ $(document).ready(function () {
         e.preventDefault();
         $('#notification-tokens-info').toggle();
     });
-
-    toggleOpacity('#time_between_check_use_default', '#time-check-widget-wrapper, #time-between-check-schedule', false);
-
-
-    const vh = Math.max(document.documentElement.clientHeight || 0, window.innerHeight || 0);
-    $("#text-preview-inner").css('max-height', (vh - 300) + "px");
-    $("#text-preview-before-inner").css('max-height', (vh - 300) + "px");
-
-    $("#activate-text-preview").click(function (e) {
-        $('body').toggleClass('preview-text-enabled')
-        request_textpreview_update();
-        const method = $('body').hasClass('preview-text-enabled') ? 'on' : 'off';
-        $('#filters-and-triggers textarea')[method]('blur', request_textpreview_update.throttle(1000));
-        $('#filters-and-triggers input')[method]('change', request_textpreview_update.throttle(1000));
-        $("#filters-and-triggers-tab")[method]('click', request_textpreview_update.throttle(1000));
-    });
-    $('.minitabs-wrapper').miniTabs({
-        "Content after filters": "#text-preview-inner",
-        "Content raw/before filters": "#text-preview-before-inner"
-    });
 });
-

changedetectionio/static/styles/diff.css

@@ -68,7 +68,7 @@
   --color-last-checked: #bbb;
   --color-text-footer: #444;
   --color-border-watch-table-cell: #eee;
-  --color-text-watch-tag-list: rgba(231, 0, 105, 0.4);
+  --color-text-watch-tag-list: #e70069;
   --color-background-new-watch-form: rgba(0, 0, 0, 0.05);
   --color-background-new-watch-input: var(--color-white);
   --color-text-new-watch-input: var(--color-text);
@@ -111,7 +111,7 @@ html[data-darkmode="true"] {
   --color-background-input: var(--color-grey-350);
   --color-text-input-description: var(--color-grey-600);
   --color-text-input-placeholder: var(--color-grey-600);
-  --color-text-watch-tag-list: rgba(250, 62, 146, 0.4);
+  --color-text-watch-tag-list: #fa3e92;
   --color-background-code: var(--color-grey-200);
   --color-background-tab: rgba(0, 0, 0, 0.2);
   --color-background-tab-hover: rgba(0, 0, 0, 0.5);
@@ -153,8 +153,7 @@ html[data-darkmode="true"] {
     border: 1px solid transparent;
     vertical-align: top;
     font: 1em monospace;
-    text-align: left;
-    overflow: clip; }
+    text-align: left; }
   #diff-ui pre {
     white-space: pre-wrap; }
 
@@ -173,9 +172,7 @@ ins {
   text-decoration: none; }
 
 #result {
-  white-space: pre-wrap;
-  word-break: break-word;
-  overflow-wrap: break-word; }
+  white-space: pre-wrap; }
 
 #settings {
   background: rgba(0, 0, 0, 0.05);
@@ -234,12 +231,3 @@ td#diff-col div {
   border-radius: 5px;
   background: var(--color-background);
   box-shadow: 1px 1px 4px var(--color-shadow-jump); }
-
-.pure-form button.reset-margin {
-  margin: 0px; }
-
-.diff-fieldset {
-  display: flex;
-  align-items: center;
-  gap: 4px;
-  flex-wrap: wrap; }

changedetectionio/static/styles/scss/diff.scss

@@ -24,7 +24,6 @@
     vertical-align: top;
     font: 1em monospace;
     text-align: left;
-    overflow: clip; // clip overflowing contents to cell boundariess
   }
 
   pre {
@@ -51,8 +50,6 @@ ins {
 
 #result {
   white-space: pre-wrap;
-  word-break: break-word;
-  overflow-wrap: break-word;
 
   .change {
     span {}
@@ -137,15 +134,3 @@ td#diff-col div {
   background: var(--color-background);
   box-shadow: 1px 1px 4px var(--color-shadow-jump);
 }
-
-// resets button margin to 0px
-.pure-form button.reset-margin {
-  margin: 0px;
-}
-
-.diff-fieldset {
-  display: flex;
-  align-items: center;
-  gap: 4px;
-  flex-wrap: wrap;
-}

changedetectionio/static/styles/scss/parts/_browser-steps.scss

@@ -40,47 +40,24 @@
   }
 }
 
-@media only screen and (min-width: 760px) {
+#browser-steps-fieldlist {
+  height: 100%;
+  overflow-y: scroll;
+}
 
-  #browser-steps .flex-wrapper {
-    display: flex;
-    flex-flow: row;
-    height: 70vh;
-    font-size: 80%;
-
-    #browser-steps-ui {
-      flex-grow: 1; /* Allow it to grow and fill the available space */
-      flex-shrink: 1; /* Allow it to shrink if needed */
-      flex-basis: 0; /* Start with 0 base width so it stretches as much as possible */
-      background-color: #eee;
-      border-radius: 5px;
-
-    }
-  }
-
-  #browser-steps-fieldlist {
-    flex-grow: 0;      /* Don't allow it to grow */
-    flex-shrink: 0;    /* Don't allow it to shrink */
-    flex-basis: auto;  /* Base width is determined by the content */
-    max-width: 400px;  /* Set a max width to prevent overflow */
-    padding-left: 1rem;
-    overflow-y: scroll;
-  }
-
-  /*  this is duplicate :( */
-  #browsersteps-selector-wrapper {
-    height: 100% !important;
-  }
+#browser-steps .flex-wrapper {
+  display: flex;
+  flex-flow: row;
+  height: 70vh;
 }
 
 /*  this is duplicate :( */
 #browsersteps-selector-wrapper {
-
+  height: 100%;
   width: 100%;
   overflow-y: scroll;
   position: relative;
-  height: 80vh;
-
+  //width: 100%;
   > img {
     position: absolute;
     max-width: 100%;
@@ -100,6 +77,7 @@
     left: 50%;
     top: 50%;
     transform: translate(-50%, -50%);
+    margin-left: -40px;
     z-index: 100;
     max-width: 350px;
     text-align: center;

changedetectionio/static/styles/scss/parts/_edit.scss

@@ -1,9 +0,0 @@
-ul#conditions_match_logic {
-    list-style: none;
-  input, label, li {
-    display: inline-block;
-  }
-  li {
-    padding-right: 1em;
-  }
-}

changedetectionio/static/styles/scss/parts/_extra_browsers.scss

@@ -11,22 +11,7 @@ ul#requests-extra_browsers {
   /* each proxy entry is a `table` */
   table {
     tr {
-      display: table-row; // default display for small screens
-      input[type=text] {
-        width: 100%;
-      }
-    }
-  }
-  
-  // apply inline display for larger screens
-  @media only screen and (min-width: 1280px) {
-    table {
-      tr {
-        display: inline;
-        input[type=text] {
-          width: 100%;
-        }
-      }
+      display: inline;
     }
   }
 }

changedetectionio/static/styles/scss/parts/_extra_proxies.scss

@@ -11,19 +11,7 @@ ul#requests-extra_proxies {
   /* each proxy entry is a `table` */
   table {
     tr {
-      display: table-row; // default display for small screens
-      input[type=text] {
-        width: 100%;
-      }
-    }
-  }
-  
-  // apply inline display for large screens
-  @media only screen and (min-width: 1024px) {
-    table {
-      tr {
-        display: inline;
-      }
+      display: inline;
     }
   }
 }
@@ -37,19 +25,15 @@ ul#requests-extra_proxies {
 
 body.proxy-check-active {
   #request {
-    // Padding set by flex layout
-    /*
     .proxy-status {
       width: 2em;
     }
-    */
 
     .proxy-check-details {
       font-size: 80%;
       color: #555;
       display: block;
-      padding-left: 2em;
-      max-width: 500px;
+      padding-left: 4em;
     }
 
     .proxy-timing {