Maintain the pool of workers in its own thread for more reliability and flexibility

.github/dependabot.yml

@@ -4,10 +4,6 @@ updates:
     directory: /
     schedule:
       interval: "weekly"
-    "caronc/apprise":
-      versioning-strategy: "increase"
-      schedule:
-        interval: "daily"
     groups:
       all:
         patterns:

.github/workflows/containers.yml

@@ -88,7 +88,7 @@ jobs:
       - name: Build and push :dev
         id: docker_build
         if: ${{ github.ref }} == "refs/heads/master"
-        uses: docker/build-push-action@v6
+        uses: docker/build-push-action@v5
         with:
           context: ./
           file: ./Dockerfile
@@ -106,7 +106,7 @@ jobs:
       - name: Build and push :tag
         id: docker_build_tag_release
         if: github.event_name == 'release' && startsWith(github.event.release.tag_name, '0.')
-        uses: docker/build-push-action@v6
+        uses: docker/build-push-action@v5
         with:
           context: ./
           file: ./Dockerfile

.github/workflows/test-container-build.yml

@@ -51,7 +51,7 @@ jobs:
         # Check we can still build under alpine/musl
         - name: Test that the docker containers can build (musl via alpine check)
           id: docker_build_musl
-          uses: docker/build-push-action@v6
+          uses: docker/build-push-action@v5
           with:
             context: ./
             file: ./.github/test/Dockerfile-alpine
@@ -59,7 +59,7 @@ jobs:
 
         - name: Test that the docker containers can build
           id: docker_build
-          uses: docker/build-push-action@v6
+          uses: docker/build-push-action@v5
           # https://github.com/docker/build-push-action#customizing
           with:
             context: ./

.github/workflows/test-only.yml

@@ -4,10 +4,17 @@ name: ChangeDetection.io App Test
 on: [push, pull_request]
 
 jobs:
-  lint-code:
+  test-application:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v4
+
+      # Mainly just for link/flake8
+      - name: Set up Python 3.11
+        uses: actions/setup-python@v5
+        with:
+          python-version: '3.11'
+
       - name: Lint with flake8
         run: |
           pip3 install flake8
@@ -16,24 +23,194 @@ jobs:
           # exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide
           flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics
 
-  test-application-3-10:
-    needs: lint-code
-    uses: ./.github/workflows/test-stack-reusable-workflow.yml
-    with:
-      python-version: '3.10'
+      - name: Spin up ancillary testable services
+        run: |
+          
+          docker network create changedet-network
+          
+          # Selenium
+          docker run --network changedet-network -d --hostname selenium  -p 4444:4444 --rm --shm-size="2g"  selenium/standalone-chrome:4
+          
+          # SocketPuppetBrowser + Extra for custom browser test
+          docker run --network changedet-network -d -e "LOG_LEVEL=TRACE" --cap-add=SYS_ADMIN --name sockpuppetbrowser --hostname sockpuppetbrowser --rm -p 3000:3000 dgtlmoon/sockpuppetbrowser:latest                    
+          docker run --network changedet-network -d -e "LOG_LEVEL=TRACE" --cap-add=SYS_ADMIN --name sockpuppetbrowser-custom-url --hostname sockpuppetbrowser-custom-url  -p 3001:3000 --rm dgtlmoon/sockpuppetbrowser:latest
 
+      - name: Build changedetection.io container for testing
+        run: |         
+          # Build a changedetection.io container and start testing inside
+          docker build --build-arg LOGGER_LEVEL=TRACE -t test-changedetectionio .
+          # Debug info
+          docker run test-changedetectionio  bash -c 'pip list'
 
-  test-application-3-11:
-    needs: lint-code
-    uses: ./.github/workflows/test-stack-reusable-workflow.yml
-    with:
-      python-version: '3.11'
-      skip-pypuppeteer: true
+      - name: Spin up ancillary SMTP+Echo message test server
+        run: |
+          # Debug SMTP server/echo message back server
+          docker run --network changedet-network -d -p 11025:11025 -p 11080:11080  --hostname mailserver test-changedetectionio  bash -c 'python changedetectionio/tests/smtp/smtp-test-server.py' 
 
-  test-application-3-12:
-    needs: lint-code
-    uses: ./.github/workflows/test-stack-reusable-workflow.yml
-    with:
-      python-version: '3.12'
-      skip-pypuppeteer: true
+      - name: Show docker container state and other debug info
+        run: |
+          set -x
+          echo "Running processes in docker..."
+          docker ps
 
+      - name: Test built container with Pytest (generally as requests/plaintext fetching)
+        run: |
+          # Unit tests
+          echo "run test with unittest"
+          docker run test-changedetectionio  bash -c 'python3 -m unittest changedetectionio.tests.unit.test_notification_diff'
+          docker run test-changedetectionio  bash -c 'python3 -m unittest changedetectionio.tests.unit.test_watch_model'
+          
+          # All tests
+          echo "run test with pytest"
+          # The default pytest logger_level is TRACE
+          # To change logger_level for pytest(test/conftest.py),
+          # append the docker option. e.g. '-e LOGGER_LEVEL=DEBUG'
+          docker run --name test-cdio-basic-tests --network changedet-network  test-changedetectionio  bash -c 'cd changedetectionio && ./run_basic_tests.sh'
+
+# PLAYWRIGHT/NODE-> CDP
+      - name: Playwright and SocketPuppetBrowser - Specific tests in built container
+        run: |
+          # Playwright via Sockpuppetbrowser fetch
+          # tests/visualselector/test_fetch_data.py will do browser steps  
+          docker run --rm -e "PLAYWRIGHT_DRIVER_URL=ws://sockpuppetbrowser:3000" --network changedet-network test-changedetectionio  bash -c 'cd changedetectionio;pytest tests/fetchers/test_content.py && pytest tests/test_errorhandling.py && pytest tests/visualselector/test_fetch_data.py'
+
+      - name: Playwright and SocketPuppetBrowser - Headers and requests
+        run: |       
+          # Settings headers playwright tests - Call back in from Sockpuppetbrowser, check headers
+          docker run --name "changedet" --hostname changedet --rm -e "FLASK_SERVER_NAME=changedet" -e "PLAYWRIGHT_DRIVER_URL=ws://sockpuppetbrowser:3000?dumpio=true" --network changedet-network test-changedetectionio  bash -c 'cd changedetectionio; pytest --live-server-host=0.0.0.0  --live-server-port=5004 tests/test_request.py'
+
+      - name: Playwright and SocketPuppetBrowser - Restock detection
+        run: |                            
+          # restock detection via playwright - added name=changedet here so that playwright and sockpuppetbrowser can connect to it
+          docker run --rm --name "changedet" -e "FLASK_SERVER_NAME=changedet" -e "PLAYWRIGHT_DRIVER_URL=ws://sockpuppetbrowser:3000" --network changedet-network test-changedetectionio  bash -c 'cd changedetectionio;pytest --live-server-port=5004 --live-server-host=0.0.0.0 tests/restock/test_restock.py'
+
+# STRAIGHT TO CDP
+      - name: Pyppeteer and SocketPuppetBrowser - Specific tests in built container
+        run: |
+          # Playwright via Sockpuppetbrowser fetch
+          docker run --rm -e "PLAYWRIGHT_DRIVER_URL=ws://sockpuppetbrowser:3000" -e "FAST_PUPPETEER_CHROME_FETCHER=True" --network changedet-network test-changedetectionio  bash -c 'cd changedetectionio;pytest tests/fetchers/test_content.py && pytest tests/test_errorhandling.py && pytest tests/visualselector/test_fetch_data.py'
+
+      - name: Pyppeteer and SocketPuppetBrowser - Headers and requests checks
+        run: |       
+          # Settings headers playwright tests - Call back in from Sockpuppetbrowser, check headers
+          docker run --name "changedet" --hostname changedet --rm  -e "FAST_PUPPETEER_CHROME_FETCHER=True" -e "FLASK_SERVER_NAME=changedet" -e "PLAYWRIGHT_DRIVER_URL=ws://sockpuppetbrowser:3000?dumpio=true" --network changedet-network test-changedetectionio  bash -c 'cd changedetectionio; pytest --live-server-host=0.0.0.0  --live-server-port=5004 tests/test_request.py'
+
+      - name: Pyppeteer and SocketPuppetBrowser - Restock detection
+        run: |                            
+          # restock detection via playwright - added name=changedet here so that playwright and sockpuppetbrowser can connect to it
+          docker run --rm --name "changedet" -e "FLASK_SERVER_NAME=changedet"  -e "FAST_PUPPETEER_CHROME_FETCHER=True"  -e "PLAYWRIGHT_DRIVER_URL=ws://sockpuppetbrowser:3000" --network changedet-network test-changedetectionio  bash -c 'cd changedetectionio;pytest --live-server-port=5004 --live-server-host=0.0.0.0 tests/restock/test_restock.py'
+
+# SELENIUM
+      - name: Specific tests in built container for Selenium
+        run: |
+          # Selenium fetch
+          docker run --rm -e "WEBDRIVER_URL=http://selenium:4444/wd/hub" --network changedet-network test-changedetectionio  bash -c 'cd changedetectionio;pytest tests/fetchers/test_content.py && pytest tests/test_errorhandling.py'
+
+      - name: Specific tests in built container for headers and requests checks with Selenium
+        run: |
+          docker run --name "changedet" --hostname changedet --rm -e "FLASK_SERVER_NAME=changedet" -e "WEBDRIVER_URL=http://selenium:4444/wd/hub" --network changedet-network test-changedetectionio  bash -c 'cd changedetectionio; pytest --live-server-host=0.0.0.0  --live-server-port=5004 tests/test_request.py'
+
+# OTHER STUFF
+      - name: Test SMTP notification mime types
+        run: |
+          # SMTP content types - needs the 'Debug SMTP server/echo message back server' container from above
+          docker run --rm  --network changedet-network test-changedetectionio bash -c 'cd changedetectionio;pytest tests/smtp/test_notification_smtp.py'
+
+      # @todo Add a test via playwright/puppeteer
+      # squid with auth is tested in run_proxy_tests.sh -> tests/proxy_list/test_select_custom_proxy.py
+      - name: Test proxy squid style interaction
+        run: |
+          cd changedetectionio
+          ./run_proxy_tests.sh
+          cd ..
+
+      - name: Test proxy SOCKS5 style interaction
+        run: |
+          cd changedetectionio
+          ./run_socks_proxy_tests.sh
+          cd ..
+
+      - name: Test custom browser URL
+        run: |
+          cd changedetectionio
+          ./run_custom_browser_url_tests.sh
+          cd ..
+
+      - name: Test changedetection.io container starts+runs basically without error
+        run: |
+          docker run --name test-changedetectionio -p 5556:5000  -d test-changedetectionio
+          sleep 3
+          # Should return 0 (no error) when grep finds it
+          curl --retry-connrefused --retry 6  -s http://localhost:5556 |grep -q checkbox-uuid
+          
+          # and IPv6
+          curl --retry-connrefused --retry 6  -s -g -6 "http://[::1]:5556"|grep -q checkbox-uuid
+
+          # Check whether TRACE log is enabled.
+          # Also, check whether TRACE is came from STDERR
+          docker logs test-changedetectionio 2>&1 1>/dev/null | grep 'TRACE log is enabled' || exit 1
+          # Check whether DEBUG is came from STDOUT
+          docker logs test-changedetectionio 2>/dev/null | grep 'DEBUG' || exit 1
+
+          docker kill test-changedetectionio
+
+      - name: Test changedetection.io SIGTERM and SIGINT signal shutdown
+        run: |
+          
+          echo SIGINT Shutdown request test
+          docker run --name sig-test -d test-changedetectionio
+          sleep 3
+          echo ">>> Sending SIGINT to sig-test container"
+          docker kill --signal=SIGINT sig-test
+          sleep 3
+          # invert the check (it should be not 0/not running)
+          docker ps
+          # check signal catch(STDERR) log. Because of
+          # changedetectionio/__init__.py: logger.add(sys.stderr, level=logger_level)
+          docker logs sig-test 2>&1 | grep 'Shutdown: Got Signal - SIGINT' || exit 1
+          test -z "`docker ps|grep sig-test`"
+          if [ $? -ne 0 ]
+          then
+            echo "Looks like container was running when it shouldnt be"
+            docker ps
+            exit 1
+          fi
+          
+          # @todo - scan the container log to see the right "graceful shutdown" text exists 
+          docker rm sig-test
+          
+          echo SIGTERM Shutdown request test
+          docker run --name sig-test -d test-changedetectionio
+          sleep 3
+          echo ">>> Sending SIGTERM to sig-test container"
+          docker kill --signal=SIGTERM sig-test
+          sleep 3
+          # invert the check (it should be not 0/not running)
+          docker ps
+          # check signal catch(STDERR) log. Because of
+          # changedetectionio/__init__.py: logger.add(sys.stderr, level=logger_level)
+          docker logs sig-test 2>&1 | grep 'Shutdown: Got Signal - SIGTERM' || exit 1
+          test -z "`docker ps|grep sig-test`"
+          if [ $? -ne 0 ]
+          then
+            echo "Looks like container was running when it shouldnt be"
+            docker ps
+            exit 1
+          fi
+          
+          # @todo - scan the container log to see the right "graceful shutdown" text exists           
+          docker rm sig-test
+
+      - name: Dump container log
+        if: always()
+        run: |
+          mkdir output-logs
+          docker logs test-cdio-basic-tests > output-logs/test-cdio-basic-tests-stdout.txt
+          docker logs test-cdio-basic-tests 2> output-logs/test-cdio-basic-tests-stderr.txt
+
+      - name: Store container log
+        if: always()
+        uses: actions/upload-artifact@v4
+        with:
+          name: test-cdio-basic-tests-output
+          path: output-logs

.github/workflows/test-stack-reusable-workflow.yml

@@ -1,239 +0,0 @@
-name: ChangeDetection.io App Test
-
-on:
-  workflow_call:
-    inputs:
-      python-version:
-        description: 'Python version to use'
-        required: true
-        type: string
-        default: '3.10'
-      skip-pypuppeteer:
-        description: 'Skip PyPuppeteer (not supported in 3.11/3.12)'
-        required: false
-        type: boolean
-        default: false
-
-jobs:
-  test-application:
-    runs-on: ubuntu-latest
-    env:
-      PYTHON_VERSION: ${{ inputs.python-version }}
-    steps:
-      - uses: actions/checkout@v4
-
-      # Mainly just for link/flake8
-      - name: Set up Python ${{ env.PYTHON_VERSION }}
-        uses: actions/setup-python@v5
-        with:
-          python-version: ${{ env.PYTHON_VERSION }}
-
-      - name: Build changedetection.io container for testing under Python ${{ env.PYTHON_VERSION }}
-        run: |
-          echo "---- Building for Python ${{ env.PYTHON_VERSION }} -----"
-          # Build a changedetection.io container and start testing inside
-          docker build --build-arg PYTHON_VERSION=${{ env.PYTHON_VERSION }} --build-arg LOGGER_LEVEL=TRACE -t test-changedetectionio .
-          # Debug info
-          docker run test-changedetectionio  bash -c 'pip list'         
-
-      - name: We should be Python ${{ env.PYTHON_VERSION }} ...
-        run: |         
-          docker run test-changedetectionio  bash -c 'python3 --version'
-
-      - name: Spin up ancillary testable services
-        run: |
-          
-          docker network create changedet-network
-          
-          # Selenium
-          docker run --network changedet-network -d --hostname selenium  -p 4444:4444 --rm --shm-size="2g"  selenium/standalone-chrome:4
-          
-          # SocketPuppetBrowser + Extra for custom browser test
-          docker run --network changedet-network -d -e "LOG_LEVEL=TRACE" --cap-add=SYS_ADMIN --name sockpuppetbrowser --hostname sockpuppetbrowser --rm -p 3000:3000 dgtlmoon/sockpuppetbrowser:latest                    
-          docker run --network changedet-network -d -e "LOG_LEVEL=TRACE" --cap-add=SYS_ADMIN --name sockpuppetbrowser-custom-url --hostname sockpuppetbrowser-custom-url  -p 3001:3000 --rm dgtlmoon/sockpuppetbrowser:latest
-
-      - name: Spin up ancillary SMTP+Echo message test server
-        run: |
-          # Debug SMTP server/echo message back server
-          docker run --network changedet-network -d -p 11025:11025 -p 11080:11080  --hostname mailserver test-changedetectionio  bash -c 'pip3 install aiosmtpd && python changedetectionio/tests/smtp/smtp-test-server.py'
-          docker ps
-
-      - name: Show docker container state and other debug info
-        run: |
-          set -x
-          echo "Running processes in docker..."
-          docker ps
-
-      - name: Test built container with Pytest (generally as requests/plaintext fetching)
-        run: |
-          # Unit tests
-          echo "run test with unittest"
-          docker run test-changedetectionio  bash -c 'python3 -m unittest changedetectionio.tests.unit.test_notification_diff'
-          docker run test-changedetectionio  bash -c 'python3 -m unittest changedetectionio.tests.unit.test_watch_model'
-          docker run test-changedetectionio  bash -c 'python3 -m unittest changedetectionio.tests.unit.test_jinja2_security'
-          
-          # All tests
-          echo "run test with pytest"
-          # The default pytest logger_level is TRACE
-          # To change logger_level for pytest(test/conftest.py),
-          # append the docker option. e.g. '-e LOGGER_LEVEL=DEBUG'
-          docker run --name test-cdio-basic-tests --network changedet-network  test-changedetectionio  bash -c 'cd changedetectionio && ./run_basic_tests.sh'
-
-# PLAYWRIGHT/NODE-> CDP
-      - name: Playwright and SocketPuppetBrowser - Specific tests in built container
-        run: |
-          # Playwright via Sockpuppetbrowser fetch
-          # tests/visualselector/test_fetch_data.py will do browser steps  
-          docker run --rm -e "FLASK_SERVER_NAME=cdio" -e "PLAYWRIGHT_DRIVER_URL=ws://sockpuppetbrowser:3000" --network changedet-network --hostname=cdio test-changedetectionio  bash -c 'cd changedetectionio;pytest --live-server-host=0.0.0.0 --live-server-port=5004 tests/fetchers/test_content.py'
-          docker run --rm -e "FLASK_SERVER_NAME=cdio" -e "PLAYWRIGHT_DRIVER_URL=ws://sockpuppetbrowser:3000" --network changedet-network --hostname=cdio test-changedetectionio  bash -c 'cd changedetectionio;pytest --live-server-host=0.0.0.0 --live-server-port=5004 tests/test_errorhandling.py'
-          docker run --rm -e "FLASK_SERVER_NAME=cdio" -e "PLAYWRIGHT_DRIVER_URL=ws://sockpuppetbrowser:3000" --network changedet-network --hostname=cdio test-changedetectionio  bash -c 'cd changedetectionio;pytest --live-server-host=0.0.0.0 --live-server-port=5004 tests/visualselector/test_fetch_data.py'
-          docker run --rm -e "FLASK_SERVER_NAME=cdio" -e "PLAYWRIGHT_DRIVER_URL=ws://sockpuppetbrowser:3000" --network changedet-network --hostname=cdio test-changedetectionio  bash -c 'cd changedetectionio;pytest --live-server-host=0.0.0.0 --live-server-port=5004 tests/fetchers/test_custom_js_before_content.py'
-
-
-      - name: Playwright and SocketPuppetBrowser - Headers and requests
-        run: |       
-          # Settings headers playwright tests - Call back in from Sockpuppetbrowser, check headers
-          docker run --name "changedet" --hostname changedet --rm -e "FLASK_SERVER_NAME=changedet" -e "PLAYWRIGHT_DRIVER_URL=ws://sockpuppetbrowser:3000?dumpio=true" --network changedet-network test-changedetectionio  bash -c 'cd changedetectionio; pytest --live-server-host=0.0.0.0  --live-server-port=5004 tests/test_request.py'
-
-      - name: Playwright and SocketPuppetBrowser - Restock detection
-        run: |                            
-          # restock detection via playwright - added name=changedet here so that playwright and sockpuppetbrowser can connect to it
-          docker run --rm --name "changedet" -e "FLASK_SERVER_NAME=changedet" -e "PLAYWRIGHT_DRIVER_URL=ws://sockpuppetbrowser:3000" --network changedet-network test-changedetectionio  bash -c 'cd changedetectionio;pytest --live-server-port=5004 --live-server-host=0.0.0.0 tests/restock/test_restock.py'
-
-# STRAIGHT TO CDP
-      - name: Pyppeteer and SocketPuppetBrowser - Specific tests in built container
-        if: ${{ inputs.skip-pypuppeteer == false }}
-        run: |
-          # Playwright via Sockpuppetbrowser fetch 
-          docker run --rm -e "FLASK_SERVER_NAME=cdio" -e "FAST_PUPPETEER_CHROME_FETCHER=True" -e "PLAYWRIGHT_DRIVER_URL=ws://sockpuppetbrowser:3000" --network changedet-network --hostname=cdio test-changedetectionio  bash -c 'cd changedetectionio;pytest --live-server-host=0.0.0.0 --live-server-port=5004 tests/fetchers/test_content.py'
-          docker run --rm -e "FLASK_SERVER_NAME=cdio" -e "FAST_PUPPETEER_CHROME_FETCHER=True" -e "PLAYWRIGHT_DRIVER_URL=ws://sockpuppetbrowser:3000" --network changedet-network --hostname=cdio test-changedetectionio  bash -c 'cd changedetectionio;pytest --live-server-host=0.0.0.0 --live-server-port=5004 tests/test_errorhandling.py'
-          docker run --rm -e "FLASK_SERVER_NAME=cdio" -e "FAST_PUPPETEER_CHROME_FETCHER=True" -e "PLAYWRIGHT_DRIVER_URL=ws://sockpuppetbrowser:3000" --network changedet-network --hostname=cdio test-changedetectionio  bash -c 'cd changedetectionio;pytest --live-server-host=0.0.0.0 --live-server-port=5004 tests/visualselector/test_fetch_data.py'
-          docker run --rm -e "FLASK_SERVER_NAME=cdio" -e "FAST_PUPPETEER_CHROME_FETCHER=True" -e "PLAYWRIGHT_DRIVER_URL=ws://sockpuppetbrowser:3000" --network changedet-network --hostname=cdio test-changedetectionio  bash -c 'cd changedetectionio;pytest --live-server-host=0.0.0.0 --live-server-port=5004 tests/fetchers/test_custom_js_before_content.py'
-
-      - name: Pyppeteer and SocketPuppetBrowser - Headers and requests checks
-        if: ${{ inputs.skip-pypuppeteer == false }}
-        run: |
-          # Settings headers playwright tests - Call back in from Sockpuppetbrowser, check headers
-          docker run --name "changedet" --hostname changedet --rm  -e "FAST_PUPPETEER_CHROME_FETCHER=True" -e "FLASK_SERVER_NAME=changedet" -e "PLAYWRIGHT_DRIVER_URL=ws://sockpuppetbrowser:3000?dumpio=true" --network changedet-network test-changedetectionio  bash -c 'cd changedetectionio; pytest --live-server-host=0.0.0.0  --live-server-port=5004 tests/test_request.py'
-
-      - name: Pyppeteer and SocketPuppetBrowser - Restock detection
-        if: ${{ inputs.skip-pypuppeteer == false }}
-        run: |                            
-          # restock detection via playwright - added name=changedet here so that playwright and sockpuppetbrowser can connect to it
-          docker run --rm --name "changedet" -e "FLASK_SERVER_NAME=changedet"  -e "FAST_PUPPETEER_CHROME_FETCHER=True"  -e "PLAYWRIGHT_DRIVER_URL=ws://sockpuppetbrowser:3000" --network changedet-network test-changedetectionio  bash -c 'cd changedetectionio;pytest --live-server-port=5004 --live-server-host=0.0.0.0 tests/restock/test_restock.py'
-
-# SELENIUM
-      - name: Specific tests in built container for Selenium
-        run: |
-          # Selenium fetch
-          docker run --rm -e "WEBDRIVER_URL=http://selenium:4444/wd/hub" --network changedet-network test-changedetectionio  bash -c 'cd changedetectionio;pytest tests/fetchers/test_content.py && pytest tests/test_errorhandling.py'
-
-      - name: Specific tests in built container for headers and requests checks with Selenium
-        run: |
-          docker run --name "changedet" --hostname changedet --rm -e "FLASK_SERVER_NAME=changedet" -e "WEBDRIVER_URL=http://selenium:4444/wd/hub" --network changedet-network test-changedetectionio  bash -c 'cd changedetectionio; pytest --live-server-host=0.0.0.0  --live-server-port=5004 tests/test_request.py'
-
-# OTHER STUFF
-      - name: Test SMTP notification mime types
-        run: |
-          # SMTP content types - needs the 'Debug SMTP server/echo message back server' container from above
-          # "mailserver" hostname defined above
-          docker run --rm --network changedet-network test-changedetectionio bash -c 'cd changedetectionio;pytest tests/smtp/test_notification_smtp.py'
-
-      # @todo Add a test via playwright/puppeteer
-      # squid with auth is tested in run_proxy_tests.sh -> tests/proxy_list/test_select_custom_proxy.py
-      - name: Test proxy squid style interaction
-        run: |
-          cd changedetectionio
-          ./run_proxy_tests.sh
-          cd ..
-
-      - name: Test proxy SOCKS5 style interaction
-        run: |
-          cd changedetectionio
-          ./run_socks_proxy_tests.sh
-          cd ..
-
-      - name: Test custom browser URL
-        run: |
-          cd changedetectionio
-          ./run_custom_browser_url_tests.sh
-          cd ..
-
-      - name: Test changedetection.io container starts+runs basically without error
-        run: |
-          docker run --name test-changedetectionio -p 5556:5000  -d test-changedetectionio
-          sleep 3
-          # Should return 0 (no error) when grep finds it
-          curl --retry-connrefused --retry 6  -s http://localhost:5556 |grep -q checkbox-uuid
-          
-          # and IPv6
-          curl --retry-connrefused --retry 6  -s -g -6 "http://[::1]:5556"|grep -q checkbox-uuid
-
-          # Check whether TRACE log is enabled.
-          # Also, check whether TRACE is came from STDERR
-          docker logs test-changedetectionio 2>&1 1>/dev/null | grep 'TRACE log is enabled' || exit 1
-          # Check whether DEBUG is came from STDOUT
-          docker logs test-changedetectionio 2>/dev/null | grep 'DEBUG' || exit 1
-
-          docker kill test-changedetectionio
-
-      - name: Test changedetection.io SIGTERM and SIGINT signal shutdown
-        run: |
-          
-          echo SIGINT Shutdown request test
-          docker run --name sig-test -d test-changedetectionio
-          sleep 3
-          echo ">>> Sending SIGINT to sig-test container"
-          docker kill --signal=SIGINT sig-test
-          sleep 3
-          # invert the check (it should be not 0/not running)
-          docker ps
-          # check signal catch(STDERR) log. Because of
-          # changedetectionio/__init__.py: logger.add(sys.stderr, level=logger_level)
-          docker logs sig-test 2>&1 | grep 'Shutdown: Got Signal - SIGINT' || exit 1
-          test -z "`docker ps|grep sig-test`"
-          if [ $? -ne 0 ]
-          then
-            echo "Looks like container was running when it shouldnt be"
-            docker ps
-            exit 1
-          fi
-          
-          # @todo - scan the container log to see the right "graceful shutdown" text exists 
-          docker rm sig-test
-          
-          echo SIGTERM Shutdown request test
-          docker run --name sig-test -d test-changedetectionio
-          sleep 3
-          echo ">>> Sending SIGTERM to sig-test container"
-          docker kill --signal=SIGTERM sig-test
-          sleep 3
-          # invert the check (it should be not 0/not running)
-          docker ps
-          # check signal catch(STDERR) log. Because of
-          # changedetectionio/__init__.py: logger.add(sys.stderr, level=logger_level)
-          docker logs sig-test 2>&1 | grep 'Shutdown: Got Signal - SIGTERM' || exit 1
-          test -z "`docker ps|grep sig-test`"
-          if [ $? -ne 0 ]
-          then
-            echo "Looks like container was running when it shouldnt be"
-            docker ps
-            exit 1
-          fi
-          
-          # @todo - scan the container log to see the right "graceful shutdown" text exists           
-          docker rm sig-test
-
-      - name: Dump container log
-        if: always()
-        run: |
-          mkdir output-logs
-          docker logs test-cdio-basic-tests > output-logs/test-cdio-basic-tests-stdout-${{ env.PYTHON_VERSION }}.txt
-          docker logs test-cdio-basic-tests 2> output-logs/test-cdio-basic-tests-stderr-${{ env.PYTHON_VERSION }}.txt
-
-      - name: Store container log
-        if: always()
-        uses: actions/upload-artifact@v4
-        with:
-          name: test-cdio-basic-tests-output-py${{ env.PYTHON_VERSION }}
-          path: output-logs

CONTRIBUTING.md

@@ -2,7 +2,7 @@ Contributing is always welcome!
 
 I am no professional flask developer, if you know a better way that something can be done, please let me know!
 
-Otherwise, it's always best to PR into the `master` branch.
+Otherwise, it's always best to PR into the `dev` branch.
 
 Please be sure that all new functionality has a matching test!
 

Dockerfile

@@ -2,10 +2,7 @@
 
 # @NOTE! I would love to move to 3.11 but it breaks the async handler in changedetectionio/content_fetchers/puppeteer.py
 #        If you know how to fix it, please do! and test it for both 3.10 and 3.11
-
-ARG PYTHON_VERSION=3.11
-
-FROM python:${PYTHON_VERSION}-slim-bookworm AS builder
+FROM python:3.10-slim-bookworm as builder
 
 # See `cryptography` pin comment in requirements.txt
 ARG CRYPTOGRAPHY_DONT_BUILD_RUST=1
@@ -26,8 +23,7 @@ WORKDIR /install
 
 COPY requirements.txt /requirements.txt
 
-# --extra-index-url https://www.piwheels.org/simple  is for cryptography module to be prebuilt (or rustc etc needs to be installed)
-RUN pip install --extra-index-url https://www.piwheels.org/simple  --target=/dependencies -r /requirements.txt
+RUN pip install --target=/dependencies -r /requirements.txt
 
 # Playwright is an alternative to Selenium
 # Excluded this package from requirements.txt to prevent arm/v6 and arm/v7 builds from failing
@@ -36,7 +32,7 @@ RUN pip install --target=/dependencies playwright~=1.41.2 \
     || echo "WARN: Failed to install Playwright. The application can still run, but the Playwright option will be disabled."
 
 # Final image stage
-FROM python:${PYTHON_VERSION}-slim-bookworm
+FROM python:3.10-slim-bookworm
 
 RUN apt-get update && apt-get install -y --no-install-recommends \
     libxslt1.1 \

MANIFEST.in

@@ -1,8 +1,8 @@
 recursive-include changedetectionio/api *
 recursive-include changedetectionio/blueprint *
-recursive-include changedetectionio/content_fetchers *
 recursive-include changedetectionio/model *
 recursive-include changedetectionio/processors *
+recursive-include changedetectionio/res *
 recursive-include changedetectionio/static *
 recursive-include changedetectionio/templates *
 recursive-include changedetectionio/tests *

README.md

@@ -91,14 +91,6 @@ We [recommend and use Bright Data](https://brightdata.grsm.io/n0r16zf7eivq) glob
 
 Please :star: star :star: this project and help it grow! https://github.com/dgtlmoon/changedetection.io/
 
-### We have a Chrome extension!
-
-Easily add the current web page to your changedetection.io tool, simply install the extension and click "Sync" to connect it to your existing changedetection.io install.
-
-[<img src="./docs/chrome-extension-screenshot.png" style="max-width:80%;" alt="Chrome Extension to easily add the current web-page to detect a change."  title="Chrome Extension to easily add the current web-page to detect a change."  />](https://chromewebstore.google.com/detail/changedetectionio-website/kefcfmgmlhmankjmnbijimhofdjekbop)
-
-[Goto the Chrome Webstore to download the extension.](https://chromewebstore.google.com/detail/changedetectionio-website/kefcfmgmlhmankjmnbijimhofdjekbop)
-
 ## Installation
 
 ### Docker
@@ -257,7 +249,13 @@ Supports managing the website watch list [via our API](https://changedetection.i
 Do you use changedetection.io to make money? does it save you time or money? Does it make your life easier? less stressful? Remember, we write this software when we should be doing actual paid work, we have to buy food and pay rent just like you.
 
 
-Consider taking out an officially supported [website change detection subscription](https://changedetection.io?src=github) , even if you don't use it, you still get the warm fuzzy feeling of helping out the project. (And who knows, you might just use it!)
+Firstly, consider taking out an officially supported [website change detection subscription](https://changedetection.io?src=github) , even if you don't use it, you still get the warm fuzzy feeling of helping out the project. (And who knows, you might just use it!)
+
+Or directly donate an amount PayPal [![Donate](https://img.shields.io/badge/Donate-PayPal-green.svg)](https://www.paypal.com/donate/?hosted_button_id=7CP6HR9ZCNDYJ)
+
+Or BTC `1PLFN327GyUarpJd7nVe7Reqg9qHx5frNn`
+
+<img src="https://raw.githubusercontent.com/dgtlmoon/changedetection.io/master/docs/btc-support.png" style="max-width:50%;" alt="Support us!"  />
 
 ## Commercial Support
 

changedetectionio/__init__.py

@@ -2,12 +2,12 @@
 
 # Read more https://github.com/dgtlmoon/changedetection.io/wiki
 
-__version__ = '0.45.25'
+__version__ = '0.45.14'
 
-from changedetectionio.strtobool import strtobool
+from distutils.util import strtobool
 from json.decoder import JSONDecodeError
 import os
-os.environ['EVENTLET_NO_GREENDNS'] = 'yes'
+#os.environ['EVENTLET_NO_GREENDNS'] = 'yes'
 import eventlet
 import eventlet.wsgi
 import getopt
@@ -175,7 +175,6 @@ def main():
     #         proxy_set_header Host "localhost";
     #         proxy_set_header X-Forwarded-Prefix /app;
 
-
     if os.getenv('USE_X_SETTINGS'):
         logger.info("USE_X_SETTINGS is ENABLED")
         from werkzeug.middleware.proxy_fix import ProxyFix

changedetectionio/api/api_v1.py

@@ -1,5 +1,5 @@
 import os
-from changedetectionio.strtobool import strtobool
+from distutils.util import strtobool
 
 from flask_expects_json import expects_json
 from changedetectionio import queuedWatchMetaData
@@ -170,33 +170,23 @@ class WatchSingleHistory(Resource):
             curl http://localhost:5000/api/v1/watch/cc0cfffa-f449-477b-83ea-0caafd1dc091/history/1677092977 -H"x-api-key:813031b16330fe25e3780cf0325daa45" -H "Content-Type: application/json"
         @apiName Get single snapshot content
         @apiGroup Watch History
-        @apiParam {String} [html]       Optional Set to =1 to return the last HTML (only stores last 2 snapshots, use `latest` as timestamp)
         @apiSuccess (200) {String} OK
         @apiSuccess (404) {String} ERR Not found
         """
         watch = self.datastore.data['watching'].get(uuid)
         if not watch:
-            abort(404, message=f"No watch exists with the UUID of {uuid}")
+            abort(404, message='No watch exists with the UUID of {}'.format(uuid))
 
         if not len(watch.history):
-            abort(404, message=f"Watch found but no history exists for the UUID {uuid}")
+            abort(404, message='Watch found but no history exists for the UUID {}'.format(uuid))
 
         if timestamp == 'latest':
             timestamp = list(watch.history.keys())[-1]
 
-        if request.args.get('html'):
-            content = watch.get_fetched_html(timestamp)
-            if content:
-                response = make_response(content, 200)
-                response.mimetype = "text/html"
-            else:
-                response = make_response("No content found", 404)
-                response.mimetype = "text/plain"
-        else:
-            content = watch.get_history_snapshot(timestamp)
-            response = make_response(content, 200)
-            response.mimetype = "text/plain"
+        content = watch.get_history_snapshot(timestamp)
 
+        response = make_response(content, 200)
+        response.mimetype = "text/plain"
         return response
 
 

changedetectionio/blueprint/browser_steps/__init__.py

@@ -12,7 +12,7 @@
 #
 #
 
-from changedetectionio.strtobool import strtobool
+from distutils.util import strtobool
 from flask import Blueprint, request, make_response
 import os
 
@@ -84,9 +84,7 @@ def construct_blueprint(datastore: ChangeDetectionStore):
         # Tell Playwright to connect to Chrome and setup a new session via our stepper interface
         browsersteps_start_session['browserstepper'] = browser_steps.browsersteps_live_ui(
             playwright_browser=browsersteps_start_session['browser'],
-            proxy=proxy,
-            start_url=datastore.data['watching'][watch_uuid].get('url')
-        )
+            proxy=proxy)
 
         # For test
         #browsersteps_start_session['browserstepper'].action_goto_url(value="http://example.com?time="+str(time.time()))
@@ -169,6 +167,11 @@ def construct_blueprint(datastore: ChangeDetectionStore):
             step_n = int(request.form.get('step_n'))
             is_last_step = strtobool(request.form.get('is_last_step'))
 
+            if step_operation == 'Goto site':
+                step_operation = 'goto_url'
+                step_optional_value = datastore.data['watching'][uuid].get('url')
+                step_selector = None
+
             # @todo try.. accept.. nice errors not popups..
             try:
 
@@ -187,10 +190,8 @@ def construct_blueprint(datastore: ChangeDetectionStore):
             u = browsersteps_sessions[browsersteps_session_id]['browserstepper'].page.url
             if is_last_step and u:
                 (screenshot, xpath_data) = browsersteps_sessions[browsersteps_session_id]['browserstepper'].request_visualselector_data()
-                watch = datastore.data['watching'].get(uuid)
-                if watch:
-                    watch.save_screenshot(screenshot=screenshot)
-                    watch.save_xpath_data(data=xpath_data)
+                datastore.save_screenshot(watch_uuid=uuid, screenshot=screenshot)
+                datastore.save_xpath_data(watch_uuid=uuid, data=xpath_data)
 
 #        if not this_session.page:
 #            cleanup_playwright_session()

changedetectionio/blueprint/browser_steps/browser_steps.py

@@ -6,9 +6,6 @@ import re
 from random import randint
 from loguru import logger
 
-from changedetectionio.content_fetchers.base import manage_user_agent
-from changedetectionio.safe_jinja import render as jinja_render
-
 # Two flags, tell the JS which of the "Selector" or "Value" field should be enabled in the front end
 # 0- off, 1- on
 browser_step_ui_config = {'Choose one': '0 0',
@@ -49,10 +46,6 @@ browser_step_ui_config = {'Choose one': '0 0',
 # ONLY Works in Playwright because we need the fullscreen screenshot
 class steppable_browser_interface():
     page = None
-    start_url = None
-
-    def __init__(self, start_url):
-        self.start_url = start_url
 
     # Convert and perform "Click Button" for example
     def call_action(self, action_name, selector=None, optional_value=None):
@@ -69,12 +62,14 @@ class steppable_browser_interface():
         action_handler = getattr(self, "action_" + call_action_name)
 
         # Support for Jinja2 variables in the value and selector
+        from jinja2 import Environment
+        jinja2_env = Environment(extensions=['jinja2_time.TimeExtension'])
 
         if selector and ('{%' in selector or '{{' in selector):
-            selector = jinja_render(template_str=selector)
+            selector = str(jinja2_env.from_string(selector).render())
 
         if optional_value and ('{%' in optional_value or '{{' in optional_value):
-            optional_value = jinja_render(template_str=optional_value)
+            optional_value = str(jinja2_env.from_string(optional_value).render())
 
         action_handler(selector, optional_value)
         self.page.wait_for_timeout(1.5 * 1000)
@@ -91,10 +86,6 @@ class steppable_browser_interface():
         logger.debug(f"Time to goto URL {time.time()-now:.2f}s")
         return response
 
-    # Incase they request to go back to the start
-    def action_goto_site(self, selector=None, value=None):
-        return self.action_goto_url(value=self.start_url)
-
     def action_click_element_containing_text(self, selector=None, value=''):
         if not len(value.strip()):
             return
@@ -187,7 +178,6 @@ class browsersteps_live_ui(steppable_browser_interface):
     stale = False
     # bump and kill this if idle after X sec
     age_start = 0
-    headers = {}
 
     # use a special driver, maybe locally etc
     command_executor = os.getenv(
@@ -202,11 +192,9 @@ class browsersteps_live_ui(steppable_browser_interface):
 
     browser_type = os.getenv("PLAYWRIGHT_BROWSER_TYPE", 'chromium').strip('"')
 
-    def __init__(self, playwright_browser, proxy=None, headers=None, start_url=None):
-        self.headers = headers or {}
+    def __init__(self, playwright_browser, proxy=None):
         self.age_start = time.time()
         self.playwright_browser = playwright_browser
-        self.start_url = start_url
         if self.context is None:
             self.connect(proxy=proxy)
 
@@ -218,17 +206,16 @@ class browsersteps_live_ui(steppable_browser_interface):
 
         # @todo handle multiple contexts, bind a unique id from the browser on each req?
         self.context = self.playwright_browser.new_context(
-            accept_downloads=False,  # Should never be needed
-            bypass_csp=True,  # This is needed to enable JavaScript execution on GitHub and others
-            extra_http_headers=self.headers,
-            ignore_https_errors=True,
-            proxy=proxy,
-            service_workers=os.getenv('PLAYWRIGHT_SERVICE_WORKERS', 'allow'),
-            # Should be `allow` or `block` - sites like YouTube can transmit large amounts of data via Service Workers
-            user_agent=manage_user_agent(headers=self.headers),
+            # @todo
+            #                user_agent=request_headers['User-Agent'] if request_headers.get('User-Agent') else 'Mozilla/5.0',
+            #               proxy=self.proxy,
+            # This is needed to enable JavaScript execution on GitHub and others
+            bypass_csp=True,
+            # Should never be needed
+            accept_downloads=False,
+            proxy=proxy
         )
 
-
         self.page = self.context.new_page()
 
         # self.page.set_default_navigation_timeout(keep_open)
@@ -255,9 +242,8 @@ class browsersteps_live_ui(steppable_browser_interface):
 
     def get_current_state(self):
         """Return the screenshot and interactive elements mapping, generally always called after action_()"""
-        import importlib.resources
-        xpath_element_js = importlib.resources.files("changedetectionio.content_fetchers.res").joinpath('xpath_element_scraper.js').read_text()
-
+        from pkg_resources import resource_string
+        xpath_element_js = resource_string(__name__, "../../content_fetchers/res/xpath_element_scraper.js").decode('utf-8')
         now = time.time()
         self.page.wait_for_timeout(1 * 1000)
 
@@ -288,9 +274,11 @@ class browsersteps_live_ui(steppable_browser_interface):
         :param current_include_filters:
         :return:
         """
-        import importlib.resources
+
         self.page.evaluate("var include_filters=''")
-        xpath_element_js = importlib.resources.files("changedetectionio.content_fetchers.res").joinpath('xpath_element_scraper.js').read_text()
+        from pkg_resources import resource_string
+        # The code that scrapes elements and makes a list of elements/size/position to click on in the VisualSelector
+        xpath_element_js = resource_string(__name__, "../../content_fetchers/res/xpath_element_scraper.js").decode('utf-8')
         from changedetectionio.content_fetchers import visualselector_xpath_selectors
         xpath_element_js = xpath_element_js.replace('%ELEMENTS%', visualselector_xpath_selectors)
         xpath_data = self.page.evaluate("async () => {" + xpath_element_js + "}")

changedetectionio/blueprint/browser_steps/nonContext.py

@@ -1,4 +1,5 @@
 from playwright.sync_api import PlaywrightContextManager
+import asyncio
 
 # So playwright wants to run as a context manager, but we do something horrible and hacky
 # we are holding the session open for as long as possible, then shutting it down, and opening a new one

changedetectionio/blueprint/check_proxies/__init__.py

@@ -31,9 +31,9 @@ def construct_blueprint(datastore: ChangeDetectionStore):
         import time
         from changedetectionio.content_fetchers import exceptions as content_fetcher_exceptions
         from changedetectionio.processors import text_json_diff
-        from changedetectionio.safe_jinja import render as jinja_render
 
         status = {'status': '', 'length': 0, 'text': ''}
+        from jinja2 import Environment, BaseLoader
 
         contents = ''
         now = time.time()
@@ -64,9 +64,7 @@ def construct_blueprint(datastore: ChangeDetectionStore):
             status.update({'status': 'OK', 'length': len(contents), 'text': ''})
 
         if status.get('text'):
-            # parse 'text' as text for safety
-            v = {'text': status['text']}
-            status['text'] = jinja_render(template_str='{{text|e}}', **v)
+            status['text'] = Environment(loader=BaseLoader()).from_string('{{text|e}}').render({'text': status['text']})
 
         status['time'] = "{:.2f}s".format(time.time() - now)
 

changedetectionio/blueprint/price_data_follower/__init__.py

@@ -1,5 +1,5 @@
 
-from changedetectionio.strtobool import strtobool
+from distutils.util import strtobool
 from flask import Blueprint, flash, redirect, url_for
 from flask_login import login_required
 from changedetectionio.store import ChangeDetectionStore

changedetectionio/blueprint/tags/__init__.py

@@ -11,16 +11,9 @@ def construct_blueprint(datastore: ChangeDetectionStore):
     def tags_overview_page():
         from .form import SingleTag
         add_form = SingleTag(request.form)
-        sorted_tags = sorted(datastore.data['settings']['application'].get('tags').items(), key=lambda x: x[1]['title'])
-
-        from collections import Counter
-
-        tag_count = Counter(tag for watch in datastore.data['watching'].values() if watch.get('tags') for tag in watch['tags'])
-
         output = render_template("groups-overview.html",
-                                 available_tags=sorted_tags,
                                  form=add_form,
-                                 tag_count=tag_count
+                                 available_tags=datastore.data['settings']['application'].get('tags', {}),
                                  )
 
         return output

changedetectionio/blueprint/tags/templates/edit-tag.html

@@ -1,9 +1,9 @@
 {% extends 'base.html' %}
 {% block content %}
-{% from '_helpers.html' import render_field, render_checkbox_field, render_button %}
-{% from '_common_fields.html' import render_common_settings_form %}
+{% from '_helpers.jinja' import render_field, render_checkbox_field, render_button %}
+{% from '_common_fields.jinja' import render_common_settings_form %}
 <script>
-    const notification_base_url="{{url_for('ajax_callback_send_notification_test', mode="group-settings")}}";
+    const notification_base_url="{{url_for('ajax_callback_send_notification_test', watch_uuid=uuid)}}";
 </script>
 
 <script src="{{url_for('static_content', group='js', filename='tabs.js')}}" defer></script>
@@ -63,7 +63,7 @@ xpath://body/div/span[contains(@class, 'example-class')]",
                             <ul>
                                 <li>JSONPath: Prefix with <code>json:</code>, use <code>json:$</code> to force re-formatting if required,  <a href="https://jsonpath.com/" target="new">test your JSONPath here</a>.</li>
                                 {% if jq_support %}
-                                <li>jq: Prefix with <code>jq:</code> and <a href="https://jqplay.org/" target="new">test your jq here</a>. Using <a href="https://stedolan.github.io/jq/" target="new">jq</a> allows for complex filtering and processing of JSON data with built-in functions, regex, filtering, and more. See examples and documentation <a href="https://stedolan.github.io/jq/manual/" target="new">here</a>. Prefix <code>jqraw:</code> outputs the results as text instead of a JSON list.</li>
+                                <li>jq: Prefix with <code>jq:</code> and <a href="https://jqplay.org/" target="new">test your jq here</a>. Using <a href="https://stedolan.github.io/jq/" target="new">jq</a> allows for complex filtering and processing of JSON data with built-in functions, regex, filtering, and more. See examples and documentation <a href="https://stedolan.github.io/jq/manual/" target="new">here</a>.</li>
                                 {% else %}
                                 <li>jq support not installed</li>
                                 {% endif %}

changedetectionio/blueprint/tags/templates/groups-overview.html

@@ -1,6 +1,6 @@
 {% extends 'base.html' %}
 {% block content %}
-{% from '_helpers.html' import render_simple_field, render_field %}
+{% from '_helpers.jinja' import render_simple_field, render_field %}
 <script src="{{url_for('static_content', group='js', filename='jquery-3.6.0.min.js')}}"></script>
 
 <div class="box">
@@ -27,7 +27,6 @@
             <thead>
             <tr>
                 <th></th>
-                <th># Watches</th>
                 <th>Tag / Label name</th>
                 <th></th>
             </tr>
@@ -41,13 +40,12 @@
                 <td colspan="3">No website organisational tags/groups configured</td>
             </tr>
             {% endif %}
-            {% for uuid, tag in available_tags  %}
+            {% for uuid, tag in available_tags.items()  %}
             <tr id="{{ uuid }}" class="{{ loop.cycle('pure-table-odd', 'pure-table-even') }}">
                 <td class="watch-controls">
                     <a class="link-mute state-{{'on' if tag.notification_muted else 'off'}}" href="{{url_for('tags.mute', uuid=tag.uuid)}}"><img src="{{url_for('static_content', group='images', filename='bell-off.svg')}}" alt="Mute notifications" title="Mute notifications" class="icon icon-mute" ></a>
                 </td>
-                <td>{{ "{:,}".format(tag_count[uuid]) if uuid in tag_count else 0 }}</td>
-                <td class="title-col inline"> <a href="{{url_for('index', tag=uuid) }}">{{ tag.title }}</a></td>
+                <td class="title-col inline">{{tag.title}}</td>
                 <td>
                     <a class="pure-button pure-button-primary" href="{{ url_for('tags.form_tag_edit', uuid=uuid) }}">Edit</a>&nbsp;
                     <a class="pure-button pure-button-primary" href="{{ url_for('tags.delete', uuid=uuid) }}" title="Deletes and removes tag">Delete</a>

changedetectionio/content_fetchers/__init__.py

@@ -1,10 +1,10 @@
 import sys
-from changedetectionio.strtobool import strtobool
-from loguru import logger
+from distutils.util import strtobool
+
 from changedetectionio.content_fetchers.exceptions import BrowserStepsStepException
 import os
 
-visualselector_xpath_selectors = 'div,span,form,table,tbody,tr,td,a,p,ul,li,h1,h2,h3,h4,header,footer,section,article,aside,details,main,nav,section,summary'
+visualselector_xpath_selectors = 'div,span,form,table,tbody,tr,td,a,p,ul,li,h1,h2,h3,h4, header, footer, section, article, aside, details, main, nav, section, summary'
 
 # available_fetchers() will scan this implementation looking for anything starting with html_
 # this information is used in the form selections
@@ -29,15 +29,10 @@ def available_fetchers():
 # rather than site-specific.
 use_playwright_as_chrome_fetcher = os.getenv('PLAYWRIGHT_DRIVER_URL', False)
 if use_playwright_as_chrome_fetcher:
-    # @note - For now, browser steps always uses playwright
     if not strtobool(os.getenv('FAST_PUPPETEER_CHROME_FETCHER', 'False')):
-        logger.debug('Using Playwright library as fetcher')
         from .playwright import fetcher as html_webdriver
     else:
-        logger.debug('Using direct Python Puppeteer library as fetcher')
         from .puppeteer import fetcher as html_webdriver
 
 else:
-    logger.debug("Falling back to selenium as fetcher")
     from .webdriver_selenium import fetcher as html_webdriver
-

changedetectionio/content_fetchers/base.py

@@ -5,40 +5,6 @@ from loguru import logger
 from changedetectionio.content_fetchers import BrowserStepsStepException
 
 
-def manage_user_agent(headers, current_ua=''):
-    """
-    Basic setting of user-agent
-
-    NOTE!!!!!! The service that does the actual Chrome fetching should handle any anti-robot techniques
-    THERE ARE MANY WAYS THAT IT CAN BE DETECTED AS A ROBOT!!
-    This does not take care of
-    - Scraping of 'navigator' (platform, productSub, vendor, oscpu etc etc) browser object (navigator.appVersion) etc
-    - TCP/IP fingerprint JA3 etc
-    - Graphic rendering fingerprinting
-    - Your IP being obviously in a pool of bad actors
-    - Too many requests
-    - Scraping of SCH-UA browser replies (thanks google!!)
-    - Scraping of ServiceWorker, new window calls etc
-
-    See https://filipvitas.medium.com/how-to-set-user-agent-header-with-puppeteer-js-and-not-fail-28c7a02165da
-    Puppeteer requests https://github.com/dgtlmoon/pyppeteerstealth
-
-    :param page:
-    :param headers:
-    :return:
-    """
-    # Ask it what the user agent is, if its obviously ChromeHeadless, switch it to the default
-    ua_in_custom_headers = headers.get('User-Agent')
-    if ua_in_custom_headers:
-        return ua_in_custom_headers
-
-    if not ua_in_custom_headers and current_ua:
-        current_ua = current_ua.replace('HeadlessChrome', 'Chrome')
-        return current_ua
-
-    return None
-
-
 class Fetcher():
     browser_connection_is_custom = None
     browser_connection_url = None
@@ -64,9 +30,10 @@ class Fetcher():
     render_extract_delay = 0
 
     def __init__(self):
-        import importlib.resources
-        self.xpath_element_js = importlib.resources.files("changedetectionio.content_fetchers.res").joinpath('xpath_element_scraper.js').read_text()
-        self.instock_data_js = importlib.resources.files("changedetectionio.content_fetchers.res").joinpath('stock-not-in-stock.js').read_text()
+        from pkg_resources import resource_string
+        # The code that scrapes elements and makes a list of elements/size/position to click on in the VisualSelector
+        self.xpath_element_js = resource_string(__name__, "res/xpath_element_scraper.js").decode('utf-8')
+        self.instock_data_js = resource_string(__name__, "res/stock-not-in-stock.js").decode('utf-8')
 
     @abstractmethod
     def get_error(self):
@@ -111,26 +78,24 @@ class Fetcher():
 
     def browser_steps_get_valid_steps(self):
         if self.browser_steps is not None and len(self.browser_steps):
-            valid_steps = list(filter(
-                lambda s: (s['operation'] and len(s['operation']) and s['operation'] != 'Choose one'),
-                self.browser_steps))
-
-            # Just incase they selected Goto site by accident with older JS
-            if valid_steps and valid_steps[0]['operation'] == 'Goto site':
-                del(valid_steps[0])
+            valid_steps = filter(
+                lambda s: (s['operation'] and len(s['operation']) and s['operation'] != 'Choose one' and s['operation'] != 'Goto site'),
+                self.browser_steps)
 
             return valid_steps
 
         return None
 
-    def iterate_browser_steps(self, start_url=None):
+    def iterate_browser_steps(self):
         from changedetectionio.blueprint.browser_steps.browser_steps import steppable_browser_interface
         from playwright._impl._errors import TimeoutError, Error
-        from changedetectionio.safe_jinja import render as jinja_render
+        from jinja2 import Environment
+        jinja2_env = Environment(extensions=['jinja2_time.TimeExtension'])
+
         step_n = 0
 
         if self.browser_steps is not None and len(self.browser_steps):
-            interface = steppable_browser_interface(start_url=start_url)
+            interface = steppable_browser_interface()
             interface.page = self.page
             valid_steps = self.browser_steps_get_valid_steps()
 
@@ -144,9 +109,9 @@ class Fetcher():
                     selector = step['selector']
                     # Support for jinja2 template in step values, with date module added
                     if '{%' in step['optional_value'] or '{{' in step['optional_value']:
-                        optional_value = jinja_render(template_str=step['optional_value'])
+                        optional_value = str(jinja2_env.from_string(step['optional_value']).render())
                     if '{%' in step['selector'] or '{{' in step['selector']:
-                        selector = jinja_render(template_str=step['selector'])
+                        selector = str(jinja2_env.from_string(step['selector']).render())
 
                     getattr(interface, "call_action")(action_name=step['operation'],
                                                       selector=selector,

changedetectionio/content_fetchers/exceptions/__init__.py

@@ -87,12 +87,11 @@ class ScreenshotUnavailable(Exception):
 
 
 class ReplyWithContentButNoText(Exception):
-    def __init__(self, status_code, url, screenshot=None, has_filters=False, html_content='', xpath_data=None):
+    def __init__(self, status_code, url, screenshot=None, has_filters=False, html_content=''):
         # Set this so we can use it in other parts of the app
         self.status_code = status_code
         self.url = url
         self.screenshot = screenshot
         self.has_filters = has_filters
         self.html_content = html_content
-        self.xpath_data = xpath_data
         return

changedetectionio/content_fetchers/playwright.py

@@ -3,8 +3,7 @@ import os
 from urllib.parse import urlparse
 
 from loguru import logger
-
-from changedetectionio.content_fetchers.base import Fetcher, manage_user_agent
+from changedetectionio.content_fetchers.base import Fetcher
 from changedetectionio.content_fetchers.exceptions import PageUnloadable, Non200ErrorCodeReceived, EmptyReply, ScreenshotUnavailable
 
 class fetcher(Fetcher):
@@ -103,23 +102,26 @@ class fetcher(Fetcher):
             # Set user agent to prevent Cloudflare from blocking the browser
             # Use the default one configured in the App.py model that's passed from fetch_site_status.py
             context = browser.new_context(
-                accept_downloads=False,  # Should never be needed
-                bypass_csp=True,  # This is needed to enable JavaScript execution on GitHub and others
-                extra_http_headers=request_headers,
-                ignore_https_errors=True,
+                user_agent={k.lower(): v for k, v in request_headers.items()}.get('user-agent', None),
                 proxy=self.proxy,
-                service_workers=os.getenv('PLAYWRIGHT_SERVICE_WORKERS', 'allow'), # Should be `allow` or `block` - sites like YouTube can transmit large amounts of data via Service Workers
-                user_agent=manage_user_agent(headers=request_headers),
+                # This is needed to enable JavaScript execution on GitHub and others
+                bypass_csp=True,
+                # Should be `allow` or `block` - sites like YouTube can transmit large amounts of data via Service Workers
+                service_workers=os.getenv('PLAYWRIGHT_SERVICE_WORKERS', 'allow'),
+                # Should never be needed
+                accept_downloads=False
             )
 
             self.page = context.new_page()
+            if len(request_headers):
+                context.set_extra_http_headers(request_headers)
 
             # Listen for all console events and handle errors
             self.page.on("console", lambda msg: print(f"Playwright console: Watch URL: {url} {msg.type}: {msg.text} {msg.args}"))
 
             # Re-use as much code from browser steps as possible so its the same
             from changedetectionio.blueprint.browser_steps.browser_steps import steppable_browser_interface
-            browsersteps_interface = steppable_browser_interface(start_url=url)
+            browsersteps_interface = steppable_browser_interface()
             browsersteps_interface.page = self.page
 
             response = browsersteps_interface.action_goto_url(value=url)
@@ -172,7 +174,7 @@ class fetcher(Fetcher):
 
             # Run Browser Steps here
             if self.browser_steps_get_valid_steps():
-                self.iterate_browser_steps(start_url=url)
+                self.iterate_browser_steps()
 
             self.page.wait_for_timeout(extra_wait * 1000)
 

changedetectionio/content_fetchers/puppeteer.py

@@ -5,10 +5,10 @@ import websockets.exceptions
 from urllib.parse import urlparse
 
 from loguru import logger
-
-from changedetectionio.content_fetchers.base import Fetcher, manage_user_agent
+from changedetectionio.content_fetchers.base import Fetcher
 from changedetectionio.content_fetchers.exceptions import PageUnloadable, Non200ErrorCodeReceived, EmptyReply, BrowserFetchTimedOut, BrowserConnectError
 
+
 class fetcher(Fetcher):
     fetcher_description = "Puppeteer/direct {}/Javascript".format(
         os.getenv("PLAYWRIGHT_BROWSER_TYPE", 'chromium').capitalize()
@@ -92,42 +92,18 @@ class fetcher(Fetcher):
                                                        ignoreHTTPSErrors=True
                                                        )
         except websockets.exceptions.InvalidStatusCode as e:
-            raise BrowserConnectError(msg=f"Error while trying to connect the browser, Code {e.status_code} (check your access, whitelist IP, password etc)")
+            raise BrowserConnectError(msg=f"Error while trying to connect the browser, Code {e.status_code} (check your access)")
         except websockets.exceptions.InvalidURI:
             raise BrowserConnectError(msg=f"Error connecting to the browser, check your browser connection address (should be ws:// or wss://")
         except Exception as e:
             raise BrowserConnectError(msg=f"Error connecting to the browser {str(e)}")
-
-        # Better is to launch chrome with the URL as arg
-        # non-headless - newPage() will launch an extra tab/window, .browser should already contain 1 page/tab
-        # headless - ask a new page
-        self.page = (pages := await browser.pages) and len(pages) or await browser.newPage()
-
-        try:
-            from pyppeteerstealth import inject_evasions_into_page
-        except ImportError:
-            logger.debug("pyppeteerstealth module not available, skipping")
-            pass
         else:
-            # I tried hooking events via self.page.on(Events.Page.DOMContentLoaded, inject_evasions_requiring_obj_to_page)
-            # But I could never get it to fire reliably, so we just inject it straight after
-            await inject_evasions_into_page(self.page)
-
-        # This user agent is similar to what was used when tweaking the evasions in inject_evasions_into_page(..)
-        user_agent = None
-        if request_headers and request_headers.get('User-Agent'):
-            # Request_headers should now be CaaseInsensitiveDict
-            # Remove it so it's not sent again with headers after
-            user_agent = request_headers.pop('User-Agent').strip()
-            await self.page.setUserAgent(user_agent)
-
-        if not user_agent:
-            # Attempt to strip 'HeadlessChrome' etc
-            await self.page.setUserAgent(manage_user_agent(headers=request_headers, current_ua=await self.page.evaluate('navigator.userAgent')))
+            self.page = await browser.newPage()
 
         await self.page.setBypassCSP(True)
         if request_headers:
             await self.page.setExtraHTTPHeaders(request_headers)
+            # @todo check user-agent worked
 
         # SOCKS5 with authentication is not supported (yet)
         # https://github.com/microsoft/playwright/issues/10567
@@ -236,12 +212,8 @@ class fetcher(Fetcher):
                 logger.error('ERROR: Failed to get viewport-only reduced screenshot :(')
                 pass
         finally:
-            # It's good to log here in the case that the browser crashes on shutting down but we still get the data we need
-            logger.success(f"Fetching '{url}' complete, closing page")
             await self.page.close()
-            logger.success(f"Fetching '{url}' complete, closing browser")
             await browser.close()
-        logger.success(f"Fetching '{url}' complete, exiting puppeteer fetch.")
 
     async def main(self, **kwargs):
         await self.fetch_page(**kwargs)

changedetectionio/content_fetchers/requests.py

@@ -30,6 +30,11 @@ class fetcher(Fetcher):
         if self.browser_steps_get_valid_steps():
             raise BrowserStepsInUnsupportedFetcher(url=url)
 
+        # Make requests use a more modern looking user-agent
+        if not {k.lower(): v for k, v in request_headers.items()}.get('user-agent', None):
+            request_headers['User-Agent'] = os.getenv("DEFAULT_SETTINGS_HEADERS_USERAGENT",
+                                                      'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/87.0.4280.66 Safari/537.36')
+
         proxies = {}
 
         # Allows override the proxy on a per-request basis

changedetectionio/content_fetchers/res/__init__.py

@@ -1 +0,0 @@
-# resources for browser injection/scraping

changedetectionio/content_fetchers/res/stock-not-in-stock.js

@@ -10,15 +10,12 @@ function isItemInStock() {
     const outOfStockTexts = [
         ' أخبرني عندما يتوفر',
         '0 in stock',
-        'actuellement indisponible',
         'agotado',
         'article épuisé',
         'artikel zurzeit vergriffen',
         'as soon as stock is available',
         'ausverkauft', // sold out
         'available for back order',
-        'awaiting stock',
-        'back in stock soon',
         'back-order or out of stock',
         'backordered',
         'benachrichtigt mich', // notify me
@@ -27,24 +24,16 @@ function isItemInStock() {
         'coming soon',
         'currently have any tickets for this',
         'currently unavailable',
-        'dieser artikel ist bald wieder verfügbar',
         'dostępne wkrótce',
         'en rupture de stock',
-        'isn\'t in stock right now',
-        'isnt in stock right now',
-        'isn’t in stock right now',
+        'ist derzeit nicht auf lager',
         'item is no longer available',
         'let me know when it\'s available',
-        'mail me when available',
         'message if back in stock',
         'nachricht bei',
         'nicht auf lager',
-        'nicht lagernd',
         'nicht lieferbar',
-        'nicht verfügbar',
-        'nicht vorrätig',
         'nicht zur verfügung',
-        'nie znaleziono produktów',
         'niet beschikbaar',
         'niet leverbaar',
         'niet op voorraad',
@@ -53,10 +42,9 @@ function isItemInStock() {
         'no tickets available',
         'not available',
         'not currently available',
-        'not in stock',
+        'not in stock',        
         'notify me when available',
-        'notify me',
-        'notify when available',
+        'notify when available',            
         'não estamos a aceitar encomendas',
         'out of stock',
         'out-of-stock',
@@ -66,30 +54,18 @@ function isItemInStock() {
         'sold-out',
         'temporarily out of stock',
         'temporarily unavailable',
-        'there were no search results for',
-        'this item is currently unavailable',
         'tickets unavailable',
         'tijdelijk uitverkocht',
-        'unavailable nearby',
         'unavailable tickets',
-        'vergriffen',
-        'vorbestellen',
-        'vorbestellung ist bald möglich',
-        'we couldn\'t find any products that match',
         'we do not currently have an estimate of when this product will be back in stock.',
         'we don\'t know when or if this item will be back in stock.',
-        'we were not able to find a match',
-        'when this arrives in stock',
         'zur zeit nicht an lager',
         '品切れ',
-        '已售',
         '已售完',
         '품절'
     ];
 
-
     const vh = Math.max(document.documentElement.clientHeight || 0, window.innerHeight || 0);
-
     function getElementBaseText(element) {
         // .textContent can include text from children which may give the wrong results
         // scan only immediate TEXT_NODEs, which will be a child of the element
@@ -100,69 +76,29 @@ function isItemInStock() {
         return text.toLowerCase().trim();
     }
 
-    const negateOutOfStockRegex = new RegExp('^([0-9] in stock|add to cart|in stock)', 'ig');
+    const negateOutOfStockRegex = new RegExp('([0-9] in stock|add to cart)', 'ig');
 
     // The out-of-stock or in-stock-text is generally always above-the-fold
     // and often below-the-fold is a list of related products that may or may not contain trigger text
     // so it's good to filter to just the 'above the fold' elements
     // and it should be atleast 100px from the top to ignore items in the toolbar, sometimes menu items like "Coming soon" exist
-
-
-// @todo - if it's SVG or IMG, go into image diff mode
-// %ELEMENTS% replaced at injection time because different interfaces use it with different settings
-
-    console.log("Scanning %ELEMENTS%");
-
-    function collectVisibleElements(parent, visibleElements) {
-        if (!parent) return; // Base case: if parent is null or undefined, return
-
-        // Add the parent itself to the visible elements array if it's of the specified types
-        visibleElements.push(parent);
-
-        // Iterate over the parent's children
-        const children = parent.children;
-        for (let i = 0; i < children.length; i++) {
-            const child = children[i];
-            if (
-                child.nodeType === Node.ELEMENT_NODE &&
-                window.getComputedStyle(child).display !== 'none' &&
-                window.getComputedStyle(child).visibility !== 'hidden' &&
-                child.offsetWidth >= 0 &&
-                child.offsetHeight >= 0 &&
-                window.getComputedStyle(child).contentVisibility !== 'hidden'
-            ) {
-                // If the child is an element and is visible, recursively collect visible elements
-                collectVisibleElements(child, visibleElements);
-            }
-        }
-    }
-
-    const elementsToScan = [];
-    collectVisibleElements(document.body, elementsToScan);
+    const elementsToScan = Array.from(document.getElementsByTagName('*')).filter(element => element.getBoundingClientRect().top + window.scrollY <= vh && element.getBoundingClientRect().top + window.scrollY >= 100);
 
     var elementText = "";
 
     // REGEXS THAT REALLY MEAN IT'S IN STOCK
     for (let i = elementsToScan.length - 1; i >= 0; i--) {
         const element = elementsToScan[i];
-
-        // outside the 'fold' or some weird text in the heading area
-        // .getBoundingClientRect() was causing a crash in chrome 119, can only be run on contentVisibility != hidden
-        if (element.getBoundingClientRect().top + window.scrollY >= vh || element.getBoundingClientRect().top + window.scrollY <= 100) {
-            continue
-        }
-
         elementText = "";
         if (element.tagName.toLowerCase() === "input") {
-            elementText = element.value.toLowerCase().trim();
+            elementText = element.value.toLowerCase();
         } else {
             elementText = getElementBaseText(element);
         }
 
         if (elementText.length) {
             // try which ones could mean its in stock
-            if (negateOutOfStockRegex.test(elementText) && !elementText.includes('(0 products)')) {
-                console.log(`Negating/overriding 'Out of Stock' back to "Possibly in stock" found "${elementText}"`)
+            if (negateOutOfStockRegex.test(elementText)) {
                 return 'Possibly in stock';
             }
         }
@@ -171,34 +107,28 @@ function isItemInStock() {
     // OTHER STUFF THAT COULD BE THAT IT'S OUT OF STOCK
     for (let i = elementsToScan.length - 1; i >= 0; i--) {
         const element = elementsToScan[i];
-        // outside the 'fold' or some weird text in the heading area
-        // .getBoundingClientRect() was causing a crash in chrome 119, can only be run on contentVisibility != hidden
-        if (element.getBoundingClientRect().top + window.scrollY >= vh + 150 || element.getBoundingClientRect().top + window.scrollY <= 100) {
-            continue
-        }
-        elementText = "";
-        if (element.tagName.toLowerCase() === "input") {
-            elementText = element.value.toLowerCase().trim();
-        } else {
-            elementText = getElementBaseText(element);
-        }
+        if (element.offsetWidth > 0 || element.offsetHeight > 0 || element.getClientRects().length > 0) {
+            elementText = "";
+            if (element.tagName.toLowerCase() === "input") {
+                elementText = element.value.toLowerCase();
+            } else {
+                elementText = getElementBaseText(element);
+            }
 
-        if (elementText.length) {
-            // and these mean its out of stock
-            for (const outOfStockText of outOfStockTexts) {
-                if (elementText.includes(outOfStockText)) {
-                    console.log(`Selected 'Out of Stock' - found text "${outOfStockText}" - "${elementText}"`)
-                    return outOfStockText; // item is out of stock
+            if (elementText.length) {
+                // and these mean its out of stock
+                for (const outOfStockText of outOfStockTexts) {
+                    if (elementText.includes(outOfStockText)) {
+                        return outOfStockText; // item is out of stock
+                    }
                 }
             }
         }
     }
 
-    console.log(`Returning 'Possibly in stock' - cant' find any useful matching text`)
     return 'Possibly in stock'; // possibly in stock, cant decide otherwise.
 }
 
 // returns the element text that makes it think it's out of stock
 return isItemInStock().trim()
 
-

changedetectionio/content_fetchers/res/xpath_element_scraper.js

@@ -16,23 +16,24 @@ try {
 }
 
 
+
 // Include the getXpath script directly, easier than fetching
 function getxpath(e) {
-    var n = e;
-    if (n && n.id) return '//*[@id="' + n.id + '"]';
-    for (var o = []; n && Node.ELEMENT_NODE === n.nodeType;) {
-        for (var i = 0, r = !1, d = n.previousSibling; d;) d.nodeType !== Node.DOCUMENT_TYPE_NODE && d.nodeName === n.nodeName && i++, d = d.previousSibling;
-        for (d = n.nextSibling; d;) {
-            if (d.nodeName === n.nodeName) {
-                r = !0;
-                break
+        var n = e;
+        if (n && n.id) return '//*[@id="' + n.id + '"]';
+        for (var o = []; n && Node.ELEMENT_NODE === n.nodeType;) {
+            for (var i = 0, r = !1, d = n.previousSibling; d;) d.nodeType !== Node.DOCUMENT_TYPE_NODE && d.nodeName === n.nodeName && i++, d = d.previousSibling;
+            for (d = n.nextSibling; d;) {
+                if (d.nodeName === n.nodeName) {
+                    r = !0;
+                    break
+                }
+                d = d.nextSibling
             }
-            d = d.nextSibling
+            o.push((n.prefix ? n.prefix + ":" : "") + n.localName + (i || r ? "[" + (i + 1) + "]" : "")), n = n.parentNode
         }
-        o.push((n.prefix ? n.prefix + ":" : "") + n.localName + (i || r ? "[" + (i + 1) + "]" : "")), n = n.parentNode
+        return o.length ? "/" + o.reverse().join("/") : ""
     }
-    return o.length ? "/" + o.reverse().join("/") : ""
-}
 
 const findUpTag = (el) => {
     let r = el
@@ -58,14 +59,14 @@ const findUpTag = (el) => {
 
     // Strategy 2: Keep going up until we hit an ID tag, imagine it's like  #list-widget div h4
     while (r.parentNode) {
-        if (depth === 5) {
+        if (depth == 5) {
             break;
         }
         if ('' !== r.id) {
             chained_css.unshift("#" + CSS.escape(r.id));
             final_selector = chained_css.join(' > ');
             // Be sure theres only one, some sites have multiples of the same ID tag :-(
-            if (window.document.querySelectorAll(final_selector).length === 1) {
+            if (window.document.querySelectorAll(final_selector).length == 1) {
                 return final_selector;
             }
             return null;
@@ -81,60 +82,30 @@ const findUpTag = (el) => {
 
 // @todo - if it's SVG or IMG, go into image diff mode
 // %ELEMENTS% replaced at injection time because different interfaces use it with different settings
-
+var elements = window.document.querySelectorAll("%ELEMENTS%");
 var size_pos = [];
 // after page fetch, inject this JS
 // build a map of all elements and their positions (maybe that only include text?)
 var bbox;
-console.log("Scanning %ELEMENTS%");
+for (var i = 0; i < elements.length; i++) {
+    bbox = elements[i].getBoundingClientRect();
 
-function collectVisibleElements(parent, visibleElements) {
-    if (!parent) return; // Base case: if parent is null or undefined, return
-
-
-    // Add the parent itself to the visible elements array if it's of the specified types
-    const tagName = parent.tagName.toLowerCase();
-    if ("%ELEMENTS%".split(',').includes(tagName)) {
-        visibleElements.push(parent);
+    // Exclude items that are not interactable or visible
+    if(elements[i].style.opacity === "0") {
+        continue
     }
-
-    // Iterate over the parent's children
-    const children = parent.children;
-    for (let i = 0; i < children.length; i++) {
-        const child = children[i];
-        if (
-            child.nodeType === Node.ELEMENT_NODE &&
-            window.getComputedStyle(child).display !== 'none' &&
-            window.getComputedStyle(child).visibility !== 'hidden' &&
-            child.offsetWidth >= 0 &&
-            child.offsetHeight >= 0 &&
-            window.getComputedStyle(child).contentVisibility !== 'hidden'
-        ) {
-            // If the child is an element and is visible, recursively collect visible elements
-            collectVisibleElements(child, visibleElements);
-        }
+    if(elements[i].style.display === "none" || elements[i].style.pointerEvents === "none" ) {
+        continue
     }
-}
-
-// Create an array to hold the visible elements
-const visibleElementsArray = [];
-
-// Call collectVisibleElements with the starting parent element
-collectVisibleElements(document.body, visibleElementsArray);
-
-
-visibleElementsArray.forEach(function (element) {
-
-    bbox = element.getBoundingClientRect();
 
     // Skip really small ones, and where width or height ==0
-    if (bbox['width'] * bbox['height'] < 10) {
-        return
+    if (bbox['width'] * bbox['height'] < 100) {
+        continue;
     }
 
     // Don't include elements that are offset from canvas
-    if (bbox['top'] + scroll_y < 0 || bbox['left'] < 0) {
-        return
+    if (bbox['top']+scroll_y < 0 || bbox['left'] < 0) {
+        continue;
     }
 
     // @todo the getXpath kind of sucks, it doesnt know when there is for example just one ID sometimes
@@ -143,46 +114,50 @@ visibleElementsArray.forEach(function (element) {
 
     // 1st primitive - if it has class, try joining it all and select, if theres only one.. well thats us.
     xpath_result = false;
+
     try {
-        var d = findUpTag(element);
+        var d = findUpTag(elements[i]);
         if (d) {
             xpath_result = d;
         }
     } catch (e) {
         console.log(e);
     }
+
     // You could swap it and default to getXpath and then try the smarter one
     // default back to the less intelligent one
     if (!xpath_result) {
         try {
             // I've seen on FB and eBay that this doesnt work
             // ReferenceError: getXPath is not defined at eval (eval at evaluate (:152:29), <anonymous>:67:20) at UtilityScript.evaluate (<anonymous>:159:18) at UtilityScript.<anonymous> (<anonymous>:1:44)
-            xpath_result = getxpath(element);
+            xpath_result = getxpath(elements[i]);
         } catch (e) {
             console.log(e);
-            return
+            continue;
         }
     }
 
+    if (window.getComputedStyle(elements[i]).visibility === "hidden") {
+        continue;
+    }
 
+    // @todo Possible to ONLY list where it's clickable to save JSON xfer size
     size_pos.push({
         xpath: xpath_result,
         width: Math.round(bbox['width']),
         height: Math.round(bbox['height']),
         left: Math.floor(bbox['left']),
-        top: Math.floor(bbox['top']) + scroll_y,
-        tagName: (element.tagName) ? element.tagName.toLowerCase() : '',
-        tagtype: (element.tagName.toLowerCase() === 'input' && element.type) ? element.type.toLowerCase() : '',
-        isClickable: window.getComputedStyle(element).cursor == "pointer"
+        top: Math.floor(bbox['top'])+scroll_y,
+        tagName: (elements[i].tagName) ? elements[i].tagName.toLowerCase() : '',
+        tagtype: (elements[i].tagName == 'INPUT' && elements[i].type) ? elements[i].type.toLowerCase() : '',
+        isClickable: (elements[i].onclick) || window.getComputedStyle(elements[i]).cursor == "pointer"
     });
 
-});
-
+}
 
 // Inject the current one set in the include_filters, which may be a CSS rule
 // used for displaying the current one in VisualSelector, where its not one we generated.
 if (include_filters.length) {
-    let results;
     // Foreach filter, go and find it on the page and add it to the results so we can visualise it again
     for (const f of include_filters) {
         bbox = false;
@@ -198,61 +173,55 @@ if (include_filters.length) {
             if (f.startsWith('/') || f.startsWith('xpath')) {
                 var qry_f = f.replace(/xpath(:|\d:)/, '')
                 console.log("[xpath] Scanning for included filter " + qry_f)
-                let xpathResult = document.evaluate(qry_f, document, null, XPathResult.ORDERED_NODE_SNAPSHOT_TYPE, null);
-                results = [];
-                for (let i = 0; i < xpathResult.snapshotLength; i++) {
-                    results.push(xpathResult.snapshotItem(i));
-                }
+                q = document.evaluate(qry_f, document, null, XPathResult.FIRST_ORDERED_NODE_TYPE, null).singleNodeValue;
             } else {
                 console.log("[css] Scanning for included filter " + f)
-                console.log("[css] Scanning for included filter " + f);
-                results = document.querySelectorAll(f);
+                q = document.querySelector(f);
             }
         } catch (e) {
             // Maybe catch DOMException and alert?
-            console.log("xpath_element_scraper: Exception selecting element from filter " + f);
+            console.log("xpath_element_scraper: Exception selecting element from filter "+f);
             console.log(e);
         }
 
-        if (results.length) {
+        if (q) {
+            // Try to resolve //something/text() back to its /something so we can atleast get the bounding box
+            try {
+                if (typeof q.nodeName == 'string' && q.nodeName === '#text') {
+                    q = q.parentElement
+                }
+            } catch (e) {
+                console.log(e)
+                console.log("xpath_element_scraper: #text resolver")
+            }
 
-            // Iterate over the results
-            results.forEach(node => {
-                // Try to resolve //something/text() back to its /something so we can atleast get the bounding box
+            // #1231 - IN the case XPath attribute filter is applied, we will have to traverse up and find the element.
+            if (typeof q.getBoundingClientRect == 'function') {
+                bbox = q.getBoundingClientRect();
+                console.log("xpath_element_scraper: Got filter element, scroll from top was " + scroll_y)
+            } else {
                 try {
-                    if (typeof node.nodeName == 'string' && node.nodeName === '#text') {
-                        node = node.parentElement
-                    }
+                    // Try and see we can find its ownerElement
+                    bbox = q.ownerElement.getBoundingClientRect();
+                    console.log("xpath_element_scraper: Got filter by ownerElement element, scroll from top was " + scroll_y)
                 } catch (e) {
                     console.log(e)
-                    console.log("xpath_element_scraper: #text resolver")
+                    console.log("xpath_element_scraper: error looking up q.ownerElement")
                 }
+            }
+        }
+        
+        if(!q) {
+            console.log("xpath_element_scraper: filter element " + f + " was not found");
+        }
 
-                // #1231 - IN the case XPath attribute filter is applied, we will have to traverse up and find the element.
-                if (typeof node.getBoundingClientRect == 'function') {
-                    bbox = node.getBoundingClientRect();
-                    console.log("xpath_element_scraper: Got filter element, scroll from top was " + scroll_y)
-                } else {
-                    try {
-                        // Try and see we can find its ownerElement
-                        bbox = node.ownerElement.getBoundingClientRect();
-                        console.log("xpath_element_scraper: Got filter by ownerElement element, scroll from top was " + scroll_y)
-                    } catch (e) {
-                        console.log(e)
-                        console.log("xpath_element_scraper: error looking up q.ownerElement")
-                    }
-                }
-
-                if (bbox && bbox['width'] > 0 && bbox['height'] > 0) {
-                    size_pos.push({
-                        xpath: f,
-                        width: parseInt(bbox['width']),
-                        height: parseInt(bbox['height']),
-                        left: parseInt(bbox['left']),
-                        top: parseInt(bbox['top']) + scroll_y,
-                        highlight_as_custom_filter: true
-                    });
-                }
+        if (bbox && bbox['width'] > 0 && bbox['height'] > 0) {
+            size_pos.push({
+                xpath: f,
+                width: parseInt(bbox['width']),
+                height: parseInt(bbox['height']),
+                left: parseInt(bbox['left']),
+                top: parseInt(bbox['top'])+scroll_y
             });
         }
     }
@@ -260,7 +229,7 @@ if (include_filters.length) {
 
 // Sort the elements so we find the smallest one first, in other words, we find the smallest one matching in that area
 // so that we dont select the wrapping element by mistake and be unable to select what we want
-size_pos.sort((a, b) => (a.width * a.height > b.width * b.height) ? 1 : -1)
+size_pos.sort((a, b) => (a.width*a.height > b.width*b.height) ? 1 : -1)
 
 // Window.width required for proper scaling in the frontend
 return {'size_pos': size_pos, 'browser_width': window.innerWidth};

changedetectionio/diff.py

@@ -1,97 +1,62 @@
+# used for the notifications, the front-end is using a JS library
+
 import difflib
-from typing import List, Iterator, Union
 
-def same_slicer(lst: List[str], start: int, end: int) -> List[str]:
-    """Return a slice of the list, or a single element if start == end."""
-    return lst[start:end] if start != end else [lst[start]]
 
-def customSequenceMatcher(
-    before: List[str],
-    after: List[str],
-    include_equal: bool = False,
-    include_removed: bool = True,
-    include_added: bool = True,
-    include_replaced: bool = True,
-    include_change_type_prefix: bool = True
-) -> Iterator[List[str]]:
-    """
-    Compare two sequences and yield differences based on specified parameters.
-    
-    Args:
-        before (List[str]): Original sequence
-        after (List[str]): Modified sequence
-        include_equal (bool): Include unchanged parts
-        include_removed (bool): Include removed parts
-        include_added (bool): Include added parts
-        include_replaced (bool): Include replaced parts
-        include_change_type_prefix (bool): Add prefixes to indicate change types
-    
-    Yields:
-        List[str]: Differences between sequences
-    """
-    cruncher = difflib.SequenceMatcher(isjunk=lambda x: x in " \t", a=before, b=after)
-    
+def same_slicer(l, a, b):
+    if a == b:
+        return [l[a]]
+    else:
+        return l[a:b]
+
+# like .compare but a little different output
+def customSequenceMatcher(before, after, include_equal=False, include_removed=True, include_added=True, include_replaced=True, include_change_type_prefix=True):
+    cruncher = difflib.SequenceMatcher(isjunk=lambda x: x in " \\t", a=before, b=after)
+
+    # @todo Line-by-line mode instead of buncghed, including `after` that is not in `before` (maybe unset?)
     for tag, alo, ahi, blo, bhi in cruncher.get_opcodes():
         if include_equal and tag == 'equal':
-            yield before[alo:ahi]
+            g = before[alo:ahi]
+            yield g
         elif include_removed and tag == 'delete':
-            prefix = "(removed) " if include_change_type_prefix else ''
-            yield [f"{prefix}{line}" for line in same_slicer(before, alo, ahi)]
+            row_prefix = "(removed) " if include_change_type_prefix else ''
+            g = [ row_prefix + i for i in same_slicer(before, alo, ahi)]
+            yield g
         elif include_replaced and tag == 'replace':
-            prefix_changed = "(changed) " if include_change_type_prefix else ''
-            prefix_into = "(into) " if include_change_type_prefix else ''
-            yield [f"{prefix_changed}{line}" for line in same_slicer(before, alo, ahi)] + \
-                  [f"{prefix_into}{line}" for line in same_slicer(after, blo, bhi)]
+            row_prefix = "(changed) " if include_change_type_prefix else ''
+            g = [row_prefix + i for i in same_slicer(before, alo, ahi)]
+            row_prefix = "(into) " if include_change_type_prefix else ''
+            g += [row_prefix + i for i in same_slicer(after, blo, bhi)]
+            yield g
         elif include_added and tag == 'insert':
-            prefix = "(added) " if include_change_type_prefix else ''
-            yield [f"{prefix}{line}" for line in same_slicer(after, blo, bhi)]
+            row_prefix = "(added) " if include_change_type_prefix else ''
+            g = [row_prefix + i for i in same_slicer(after, blo, bhi)]
+            yield g
 
-def render_diff(
-    previous_version_file_contents: str,
-    newest_version_file_contents: str,
-    include_equal: bool = False,
-    include_removed: bool = True,
-    include_added: bool = True,
-    include_replaced: bool = True,
-    line_feed_sep: str = "\n",
-    include_change_type_prefix: bool = True,
-    patch_format: bool = False
-) -> str:
-    """
-    Render the difference between two file contents.
-    
-    Args:
-        previous_version_file_contents (str): Original file contents
-        newest_version_file_contents (str): Modified file contents
-        include_equal (bool): Include unchanged parts
-        include_removed (bool): Include removed parts
-        include_added (bool): Include added parts
-        include_replaced (bool): Include replaced parts
-        line_feed_sep (str): Separator for lines in output
-        include_change_type_prefix (bool): Add prefixes to indicate change types
-        patch_format (bool): Use patch format for output
-    
-    Returns:
-        str: Rendered difference
-    """
-    newest_lines = [line.rstrip() for line in newest_version_file_contents.splitlines()]
-    previous_lines = [line.rstrip() for line in previous_version_file_contents.splitlines()] if previous_version_file_contents else []
+# only_differences - only return info about the differences, no context
+# line_feed_sep could be "<br>" or "<li>" or "\n" etc
+def render_diff(previous_version_file_contents, newest_version_file_contents, include_equal=False, include_removed=True, include_added=True, include_replaced=True, line_feed_sep="\n", include_change_type_prefix=True, patch_format=False):
+
+    newest_version_file_contents = [line.rstrip() for line in newest_version_file_contents.splitlines()]
+
+    if previous_version_file_contents:
+        previous_version_file_contents = [line.rstrip() for line in previous_version_file_contents.splitlines()]
+    else:
+        previous_version_file_contents = ""
 
     if patch_format:
-        patch = difflib.unified_diff(previous_lines, newest_lines)
+        patch = difflib.unified_diff(previous_version_file_contents, newest_version_file_contents)
         return line_feed_sep.join(patch)
 
-    rendered_diff = customSequenceMatcher(
-        before=previous_lines,
-        after=newest_lines,
-        include_equal=include_equal,
-        include_removed=include_removed,
-        include_added=include_added,
-        include_replaced=include_replaced,
-        include_change_type_prefix=include_change_type_prefix
-    )
+    rendered_diff = customSequenceMatcher(before=previous_version_file_contents,
+                                          after=newest_version_file_contents,
+                                          include_equal=include_equal,
+                                          include_removed=include_removed,
+                                          include_added=include_added,
+                                          include_replaced=include_replaced,
+                                          include_change_type_prefix=include_change_type_prefix)
 
-    def flatten(lst: List[Union[str, List[str]]]) -> str:
-        return line_feed_sep.join(flatten(x) if isinstance(x, list) else x for x in lst)
-
-    return flatten(rendered_diff)
+    # Recursively join lists
+    f = lambda L: line_feed_sep.join([f(x) if type(x) is list else x for x in L])
+    p= f(rendered_diff)
+    return p

changedetectionio/flask_app.py

@@ -5,11 +5,11 @@ import os
 import queue
 import threading
 import time
-from .safe_jinja import render as jinja_render
-from changedetectionio.strtobool import strtobool
 from copy import deepcopy
+from distutils.util import strtobool
 from functools import wraps
 from threading import Event
+
 import flask_login
 import pytz
 import timeago
@@ -30,7 +30,6 @@ from flask_compress import Compress as FlaskCompress
 from flask_login import current_user
 from flask_paginate import Pagination, get_page_parameter
 from flask_restful import abort, Api
-from flask_cors import CORS
 from flask_wtf import CSRFProtect
 from loguru import logger
 
@@ -54,9 +53,6 @@ app = Flask(__name__,
             static_folder="static",
             template_folder="templates")
 
-# Enable CORS, especially useful for the Chrome extension to operate from anywhere
-CORS(app)
-
 # Super handy for compressing large BrowserSteps responses and others
 FlaskCompress(app)
 
@@ -124,10 +120,10 @@ def _jinja2_filter_datetime(watch_obj, format="%Y-%m-%d %H:%M:%S"):
 
 @app.template_filter('format_timestamp_timeago')
 def _jinja2_filter_datetimestamp(timestamp, format="%Y-%m-%d %H:%M:%S"):
-    if not timestamp:
+    if timestamp == False:
         return 'Not yet'
 
-    return timeago.format(int(timestamp), time.time())
+    return timeago.format(timestamp, time.time())
 
 
 @app.template_filter('pagination_slice')
@@ -319,6 +315,8 @@ def changedetection_app(config=None, datastore_o=None):
 
     @app.route("/rss", methods=['GET'])
     def rss():
+        from jinja2 import Environment, BaseLoader
+        jinja2_env = Environment(loader=BaseLoader)
         now = time.time()
         # Always requires token set
         app_rss_token = datastore.data['settings']['application'].get('rss_access_token')
@@ -338,11 +336,8 @@ def changedetection_app(config=None, datastore_o=None):
 
         # @todo needs a .itemsWithTag() or something - then we can use that in Jinaj2 and throw this away
         for uuid, watch in datastore.data['watching'].items():
-            # @todo tag notification_muted skip also (improve Watch model)
-            if datastore.data['settings']['application'].get('rss_hide_muted_watches') and watch.get('notification_muted'):
-                continue
             if limit_tag and not limit_tag in watch['tags']:
-                continue
+                    continue
             watch['uuid'] = uuid
             sorted_watches.append(watch)
 
@@ -389,7 +384,7 @@ def changedetection_app(config=None, datastore_o=None):
                 # @todo Make this configurable and also consider html-colored markup
                 # @todo User could decide if <link> goes to the diff page, or to the watch link
                 rss_template = "<html><body>\n<h4><a href=\"{{watch_url}}\">{{watch_title}}</a></h4>\n<p>{{html_diff}}</p>\n</body></html>\n"
-                content = jinja_render(template_str=rss_template, watch_title=watch_title, html_diff=html_diff, watch_url=watch.link)
+                content = jinja2_env.from_string(rss_template).render(watch_title=watch_title, html_diff=html_diff, watch_url=watch.link)
 
                 fe.content(content=content, type='CDATA')
 
@@ -409,21 +404,17 @@ def changedetection_app(config=None, datastore_o=None):
         global datastore
         from changedetectionio import forms
 
-        active_tag_req = request.args.get('tag', '').lower().strip()
-        active_tag_uuid = active_tag = None
+        limit_tag = request.args.get('tag', '').lower().strip()
 
         # Be sure limit_tag is a uuid
-        if active_tag_req:
-            for uuid, tag in datastore.data['settings']['application'].get('tags', {}).items():
-                if active_tag_req == tag.get('title', '').lower().strip() or active_tag_req == uuid:
-                    active_tag = tag
-                    active_tag_uuid = uuid
-                    break
+        for uuid, tag in datastore.data['settings']['application'].get('tags', {}).items():
+            if limit_tag == tag.get('title', '').lower().strip():
+                limit_tag = uuid
 
 
         # Redirect for the old rss path which used the /?rss=true
         if request.args.get('rss'):
-            return redirect(url_for('rss', tag=active_tag_uuid))
+            return redirect(url_for('rss', tag=limit_tag))
 
         op = request.args.get('op')
         if op:
@@ -434,7 +425,7 @@ def changedetection_app(config=None, datastore_o=None):
                 datastore.data['watching'][uuid].toggle_mute()
 
             datastore.needs_write = True
-            return redirect(url_for('index', tag = active_tag_uuid))
+            return redirect(url_for('index', tag = limit_tag))
 
         # Sort by last_changed and add the uuid which is usually the key..
         sorted_watches = []
@@ -445,7 +436,7 @@ def changedetection_app(config=None, datastore_o=None):
             if with_errors and not watch.get('last_error'):
                 continue
 
-            if active_tag_uuid and not active_tag_uuid in watch['tags']:
+            if limit_tag and not limit_tag in watch['tags']:
                     continue
             if watch.get('last_error'):
                 errored_count += 1
@@ -453,8 +444,6 @@ def changedetection_app(config=None, datastore_o=None):
             if search_q:
                 if (watch.get('title') and search_q in watch.get('title').lower()) or search_q in watch.get('url', '').lower():
                     sorted_watches.append(watch)
-                elif watch.get('last_error') and search_q in watch.get('last_error').lower():
-                    sorted_watches.append(watch)
             else:
                 sorted_watches.append(watch)
 
@@ -466,13 +455,12 @@ def changedetection_app(config=None, datastore_o=None):
                                 total=total_count,
                                 per_page=datastore.data['settings']['application'].get('pager_size', 50), css_framework="semantic")
 
-        sorted_tags = sorted(datastore.data['settings']['application'].get('tags').items(), key=lambda x: x[1]['title'])
+
         output = render_template(
             "watch-overview.html",
                                  # Don't link to hosting when we're on the hosting environment
-                                 active_tag=active_tag,
-                                 active_tag_uuid=active_tag_uuid,
-                                 app_rss_token=datastore.data['settings']['application'].get('rss_access_token'),
+                                 active_tag=limit_tag,
+                                 app_rss_token=datastore.data['settings']['application']['rss_access_token'],
                                  datastore=datastore,
                                  errored_count=errored_count,
                                  form=form,
@@ -486,7 +474,7 @@ def changedetection_app(config=None, datastore_o=None):
                                  sort_attribute=request.args.get('sort') if request.args.get('sort') else request.cookies.get('sort'),
                                  sort_order=request.args.get('order') if request.args.get('order') else request.cookies.get('order'),
                                  system_default_fetcher=datastore.data['settings']['application'].get('fetch_backend'),
-                                 tags=sorted_tags,
+                                 tags=datastore.data['settings']['application'].get('tags'),
                                  watches=sorted_watches
                                  )
 
@@ -519,38 +507,21 @@ def changedetection_app(config=None, datastore_o=None):
 
         watch = datastore.data['watching'].get(watch_uuid) if watch_uuid else None
 
-        notification_urls = request.form['notification_urls'].strip().splitlines()
+        # validate URLS
+        if not len(request.form['notification_urls'].strip()):
+            return make_response({'error': 'No Notification URLs set'}, 400)
 
-        if not notification_urls:
-            logger.debug("Test notification - Trying by group/tag in the edit form if available")
-            # On an edit page, we should also fire off to the tags if they have notifications
-            if request.form.get('tags') and request.form['tags'].strip():
-                for k in request.form['tags'].split(','):
-                    tag = datastore.tag_exists_by_name(k.strip())
-                    notification_urls = tag.get('notifications_urls') if tag and tag.get('notifications_urls') else None
-
-        is_global_settings_form = request.args.get('mode', '') == 'global-settings'
-        is_group_settings_form = request.args.get('mode', '') == 'group-settings'
-        if not notification_urls and not is_global_settings_form and not is_group_settings_form:
-            # In the global settings, use only what is typed currently in the text box
-            logger.debug("Test notification - Trying by global system settings notifications")
-            if datastore.data['settings']['application'].get('notification_urls'):
-                notification_urls = datastore.data['settings']['application']['notification_urls']
-
-
-        if not notification_urls:
-            return 'No Notification URLs set/found'
-
-        for n_url in notification_urls:
-            if len(n_url.strip()):
-                if not apobj.add(n_url):
-                    return f'Error - {n_url} is not a valid AppRise URL.'
+        for server_url in request.form['notification_urls'].splitlines():
+            if len(server_url.strip()):
+                if not apobj.add(server_url):
+                    message = '{} is not a valid AppRise URL.'.format(server_url)
+                    return make_response({'error': message}, 400)
 
         try:
             # use the same as when it is triggered, but then override it with the form test values
             n_object = {
                 'watch_url': request.form['window_url'],
-                'notification_urls': notification_urls
+                'notification_urls': request.form['notification_urls'].splitlines()
             }
 
             # Only use if present, if not set in n_object it should use the default system value
@@ -569,7 +540,7 @@ def changedetection_app(config=None, datastore_o=None):
         except Exception as e:
             return make_response({'error': str(e)}, 400)
 
-        return 'OK - Sent test notifications'
+        return 'OK'
 
 
     @app.route("/clear_history/<string:uuid>", methods=['GET'])
@@ -606,12 +577,6 @@ def changedetection_app(config=None, datastore_o=None):
         output = render_template("clear_all_history.html")
         return output
 
-    def _watch_has_tag_options_set(watch):
-        """This should be fixed better so that Tag is some proper Model, a tag is just a Watch also"""
-        for tag_uuid, tag in datastore.data['settings']['application'].get('tags', {}).items():
-            if tag_uuid in watch.get('tags', []) and (tag.get('include_filters') or tag.get('subtractive_selectors')):
-                return True
-
     @app.route("/edit/<string:uuid>", methods=['GET', 'POST'])
     @login_optionally_required
     # https://stackoverflow.com/questions/42984453/wtforms-populate-form-with-data-if-data-exists
@@ -622,6 +587,7 @@ def changedetection_app(config=None, datastore_o=None):
         from .blueprint.browser_steps.browser_steps import browser_step_ui_config
         from . import processors
 
+        using_default_check_time = True
         # More for testing, possible to return the first/only
         if not datastore.data['watching'].keys():
             flash("No watches to edit", "error")
@@ -646,6 +612,10 @@ def changedetection_app(config=None, datastore_o=None):
         # be sure we update with a copy instead of accidently editing the live object by reference
         default = deepcopy(datastore.data['watching'][uuid])
 
+        # Show system wide default if nothing configured
+        if all(value == 0 or value == None for value in datastore.data['watching'][uuid]['time_between_check'].values()):
+            default['time_between_check'] = deepcopy(datastore.data['settings']['requests']['time_between_check'])
+
         # Defaults for proxy choice
         if datastore.proxy_list is not None:  # When enabled
             # @todo
@@ -679,15 +649,22 @@ def changedetection_app(config=None, datastore_o=None):
 
         if request.method == 'POST' and form.validate():
 
-            extra_update_obj = {
-                'consecutive_filter_failures': 0,
-                'last_error' : False
-            }
+            extra_update_obj = {}
 
             if request.args.get('unpause_on_save'):
                 extra_update_obj['paused'] = False
+            # Re #110, if they submit the same as the default value, set it to None, so we continue to follow the default
+            # Assume we use the default value, unless something relevant is different, then use the form value
+            # values could be None, 0 etc.
+            # Set to None unless the next for: says that something is different
+            extra_update_obj['time_between_check'] = dict.fromkeys(form.time_between_check.data)
+            for k, v in form.time_between_check.data.items():
+                if v and v != datastore.data['settings']['requests']['time_between_check'][k]:
+                    extra_update_obj['time_between_check'] = form.time_between_check.data
+                    using_default_check_time = False
+                    break
+
 
-            extra_update_obj['time_between_check'] = form.time_between_check.data
 
              # Ignore text
             form_ignore_text = form.ignore_text.data
@@ -721,7 +698,7 @@ def changedetection_app(config=None, datastore_o=None):
             datastore.data['watching'][uuid].update(extra_update_obj)
 
             if request.args.get('unpause_on_save'):
-                flash("Updated watch - unpaused!")
+                flash("Updated watch - unpaused!.")
             else:
                 flash("Updated watch.")
 
@@ -768,13 +745,13 @@ def changedetection_app(config=None, datastore_o=None):
                                      extra_title=f" - Edit - {watch.label}",
                                      form=form,
                                      has_default_notification_urls=True if len(datastore.data['settings']['application']['notification_urls']) else False,
+                                     has_empty_checktime=using_default_check_time,
                                      has_extra_headers_file=len(datastore.get_all_headers_in_textfile_for_watch(uuid=uuid)) > 0,
-                                     has_special_tag_options=_watch_has_tag_options_set(watch=watch),
                                      is_html_webdriver=is_html_webdriver,
                                      jq_support=jq_support,
                                      playwright_enabled=os.getenv('PLAYWRIGHT_DRIVER_URL', False),
                                      settings_application=datastore.data['settings']['application'],
-                                     using_global_webdriver_wait=not default['webdriver_delay'],
+                                     using_global_webdriver_wait=default['webdriver_delay'] is None,
                                      uuid=uuid,
                                      visualselector_enabled=visualselector_enabled,
                                      watch=watch
@@ -853,13 +830,11 @@ def changedetection_app(config=None, datastore_o=None):
                 flash("An error occurred, please see below.", "error")
 
         output = render_template("settings.html",
-                                 api_key=datastore.data['settings']['application'].get('api_access_token'),
-                                 emailprefix=os.getenv('NOTIFICATION_MAIL_BUTTON_PREFIX', False),
                                  form=form,
                                  hide_remove_pass=os.getenv("SALTED_PASS", False),
-                                 min_system_recheck_seconds=int(os.getenv('MINIMUM_SECONDS_RECHECK_TIME', 3)),
-                                 settings_application=datastore.data['settings']['application']
-                                 )
+                                 api_key=datastore.data['settings']['application'].get('api_access_token'),
+                                 emailprefix=os.getenv('NOTIFICATION_MAIL_BUTTON_PREFIX', False),
+                                 settings_application=datastore.data['settings']['application'])
 
         return output
 
@@ -1069,8 +1044,6 @@ def changedetection_app(config=None, datastore_o=None):
         content = []
         ignored_line_numbers = []
         trigger_line_numbers = []
-        versions = []
-        timestamp = None
 
         # More for testing, possible to return the first/only
         if uuid == 'first':
@@ -1090,53 +1063,57 @@ def changedetection_app(config=None, datastore_o=None):
         if (watch.get('fetch_backend') == 'system' and system_uses_webdriver) or watch.get('fetch_backend') == 'html_webdriver' or watch.get('fetch_backend', '').startswith('extra_browser_'):
             is_html_webdriver = True
 
+        # Never requested successfully, but we detected a fetch error
         if datastore.data['watching'][uuid].history_n == 0 and (watch.get_error_text() or watch.get_error_snapshot()):
             flash("Preview unavailable - No fetch/check completed or triggers not reached", "error")
-        else:
-            # So prepare the latest preview or not
-            preferred_version = request.args.get('version')
-            versions = list(watch.history.keys())
-            timestamp = versions[-1]
-            if preferred_version and preferred_version in versions:
-                timestamp = preferred_version
+            output = render_template("preview.html",
+                                     content=content,
+                                     history_n=watch.history_n,
+                                     extra_stylesheets=extra_stylesheets,
+#                                     current_diff_url=watch['url'],
+                                     watch=watch,
+                                     uuid=uuid,
+                                     is_html_webdriver=is_html_webdriver,
+                                     last_error=watch['last_error'],
+                                     last_error_text=watch.get_error_text(),
+                                     last_error_screenshot=watch.get_error_snapshot())
+            return output
 
-            try:
-                versions = list(watch.history.keys())
-                tmp = watch.get_history_snapshot(timestamp).splitlines()
+        timestamp = list(watch.history.keys())[-1]
+        try:
+            tmp = watch.get_history_snapshot(timestamp).splitlines()
 
-                # Get what needs to be highlighted
-                ignore_rules = watch.get('ignore_text', []) + datastore.data['settings']['application']['global_ignore_text']
+            # Get what needs to be highlighted
+            ignore_rules = watch.get('ignore_text', []) + datastore.data['settings']['application']['global_ignore_text']
 
-                # .readlines will keep the \n, but we will parse it here again, in the future tidy this up
-                ignored_line_numbers = html_tools.strip_ignore_text(content="\n".join(tmp),
-                                                                    wordlist=ignore_rules,
-                                                                    mode='line numbers'
-                                                                    )
+            # .readlines will keep the \n, but we will parse it here again, in the future tidy this up
+            ignored_line_numbers = html_tools.strip_ignore_text(content="\n".join(tmp),
+                                                                wordlist=ignore_rules,
+                                                                mode='line numbers'
+                                                                )
 
-                trigger_line_numbers = html_tools.strip_ignore_text(content="\n".join(tmp),
-                                                                    wordlist=watch['trigger_text'],
-                                                                    mode='line numbers'
-                                                                    )
-                # Prepare the classes and lines used in the template
-                i=0
-                for l in tmp:
-                    classes=[]
-                    i+=1
-                    if i in ignored_line_numbers:
-                        classes.append('ignored')
-                    if i in trigger_line_numbers:
-                        classes.append('triggered')
-                    content.append({'line': l, 'classes': ' '.join(classes)})
+            trigger_line_numbers = html_tools.strip_ignore_text(content="\n".join(tmp),
+                                                                wordlist=watch['trigger_text'],
+                                                                mode='line numbers'
+                                                                )
+            # Prepare the classes and lines used in the template
+            i=0
+            for l in tmp:
+                classes=[]
+                i+=1
+                if i in ignored_line_numbers:
+                    classes.append('ignored')
+                if i in trigger_line_numbers:
+                    classes.append('triggered')
+                content.append({'line': l, 'classes': ' '.join(classes)})
 
-            except Exception as e:
-                content.append({'line': f"File doesnt exist or unable to read timestamp {timestamp}", 'classes': ''})
+        except Exception as e:
+            content.append({'line': f"File doesnt exist or unable to read timestamp {timestamp}", 'classes': ''})
 
         output = render_template("preview.html",
                                  content=content,
-                                 current_version=timestamp,
                                  history_n=watch.history_n,
                                  extra_stylesheets=extra_stylesheets,
-                                 extra_title=f" - Diff - {watch.label} @ {timestamp}",
                                  ignored_line_numbers=ignored_line_numbers,
                                  triggered_line_numbers=trigger_line_numbers,
                                  current_diff_url=watch['url'],
@@ -1146,10 +1123,7 @@ def changedetection_app(config=None, datastore_o=None):
                                  is_html_webdriver=is_html_webdriver,
                                  last_error=watch['last_error'],
                                  last_error_text=watch.get_error_text(),
-                                 last_error_screenshot=watch.get_error_snapshot(),
-                                 versions=versions
-                                )
-
+                                 last_error_screenshot=watch.get_error_snapshot())
 
         return output
 
@@ -1296,8 +1270,9 @@ def changedetection_app(config=None, datastore_o=None):
 
         url = request.form.get('url').strip()
         if datastore.url_exists(url):
-            flash(f'Warning, URL {url} already exists', "notice")
-            
+            flash('The URL {} already exists'.format(url), "error")
+            return redirect(url_for('index'))
+
         add_paused = request.form.get('edit_and_watch_submit_button') != None
         processor = request.form.get('processor', 'text_json_diff')
         new_uuid = datastore.add_watch(url=url, tag=request.form.get('tags').strip(), extras={'paused': add_paused, 'processor': processor})
@@ -1447,13 +1422,6 @@ def changedetection_app(config=None, datastore_o=None):
                     update_q.put(queuedWatchMetaData.PrioritizedItem(priority=1, item={'uuid': uuid, 'skip_when_checksum_same': False}))
             flash("{} watches queued for rechecking".format(len(uuids)))
 
-        elif (op == 'clear-errors'):
-            for uuid in uuids:
-                uuid = uuid.strip()
-                if datastore.data['watching'].get(uuid):
-                    datastore.data['watching'][uuid]["last_error"] = False
-            flash(f"{len(uuids)} watches errors cleared")
-
         elif (op == 'clear-history'):
             for uuid in uuids:
                 uuid = uuid.strip()
@@ -1579,6 +1547,7 @@ def changedetection_app(config=None, datastore_o=None):
     # @todo handle ctrl break
     ticker_thread = threading.Thread(target=ticker_thread_check_time_launch_checks).start()
     threading.Thread(target=notification_runner).start()
+    threading.Thread(target=thread_maintain_worker_thread_pool).start()
 
     # Check for new release version, but not when running in test/build or pytest
     if not os.getenv("GITHUB_REF", False) and not config.get('disable_checkver') == True:
@@ -1661,23 +1630,73 @@ def notification_runner():
             # Trim the log length
             notification_debug_log = notification_debug_log[-100:]
 
-# Threaded runner, look for new watches to feed into the Queue.
+
+def thread_maintain_worker_thread_pool():
+    from changedetectionio import update_worker
+
+    n_workers = int(os.getenv("FETCH_WORKERS", datastore.data['settings']['requests']['workers']))
+
+    while not app.config.exit.is_set():
+        needed_threads = n_workers if not running_update_threads else 0
+        how_many_running_now = 0
+        dead_threads = []
+
+        for i, t in enumerate(running_update_threads):
+            if t.is_alive():
+                how_many_running_now += 1
+            else:
+                dead_threads.append(i)
+
+        for i in dead_threads:
+            del running_update_threads[i]
+
+        for _ in range(needed_threads - how_many_running_now):
+            logger.info("Adding new worker thread")
+            new_worker = update_worker.update_worker(update_q, notification_q, app, datastore)
+            running_update_threads.append(new_worker)
+            new_worker.start()
+
+        app.config.exit.wait(2)
+
+
+
+def thread_maintain_worker_thread_pool():
+    from changedetectionio import update_worker
+
+    logger.info("Starting thread pool worker maintainer thread")
+    n_workers = int(os.getenv("FETCH_WORKERS", datastore.data['settings']['requests']['workers']))
+
+    while not app.config.exit.is_set():
+        needed_threads = n_workers if not running_update_threads else 0
+        how_many_running_now = 0
+        dead_threads = []
+
+        for i, t in enumerate(running_update_threads):
+            if t.is_alive():
+                how_many_running_now += 1
+            else:
+                dead_threads.append(i)
+
+        for i in dead_threads:
+            del running_update_threads[i]
+
+        for _ in range(needed_threads - how_many_running_now):
+            logger.info("Adding new worker thread")
+            new_worker = update_worker.update_worker(update_q, notification_q, app, datastore)
+            running_update_threads.append(new_worker)
+            new_worker.start()
+
+        app.config.exit.wait(2)
+
+# Thread runner to check every minute, look for new watches to feed into the Queue.
 def ticker_thread_check_time_launch_checks():
     import random
-    from changedetectionio import update_worker
 
     proxy_last_called_time = {}
 
-    recheck_time_minimum_seconds = int(os.getenv('MINIMUM_SECONDS_RECHECK_TIME', 3))
+    recheck_time_minimum_seconds = int(os.getenv('MINIMUM_SECONDS_RECHECK_TIME', 20))
     logger.debug(f"System env MINIMUM_SECONDS_RECHECK_TIME {recheck_time_minimum_seconds}")
 
-    # Spin up Workers that do the fetching
-    # Can be overriden by ENV or use the default settings
-    n_workers = int(os.getenv("FETCH_WORKERS", datastore.data['settings']['requests']['workers']))
-    for _ in range(n_workers):
-        new_worker = update_worker.update_worker(update_q, notification_q, app, datastore)
-        running_update_threads.append(new_worker)
-        new_worker.start()
 
     while not app.config.exit.is_set():
 
@@ -1722,7 +1741,9 @@ def ticker_thread_check_time_launch_checks():
                 continue
 
             # If they supplied an individual entry minutes to threshold.
-            threshold = recheck_time_system_seconds if watch.get('time_between_check_use_default') else watch.threshold_seconds()
+
+            watch_threshold_seconds = watch.threshold_seconds()
+            threshold = watch_threshold_seconds if watch_threshold_seconds > 0 else recheck_time_system_seconds
 
             # #580 - Jitter plus/minus amount of time to make the check seem more random to the server
             jitter = datastore.data['settings']['requests'].get('jitter_seconds', 0)
@@ -1758,7 +1779,7 @@ def ticker_thread_check_time_launch_checks():
                     priority = int(time.time())
                     logger.debug(
                         f"> Queued watch UUID {uuid} "
-                        f"last checked at {watch['last_checked']} "
+                        f"last checked at {watch['last_checked']} ({seconds_since_last_recheck} seconds ago!) recheck min was :{recheck_time_minimum_seconds} "
                         f"queued at {now:0.2f} priority {priority} "
                         f"jitter {watch.jitter_seconds:0.2f}s, "
                         f"{now - watch['last_checked']:0.2f}s since last checked")

changedetectionio/forms.py

@@ -1,6 +1,6 @@
 import os
 import re
-from changedetectionio.strtobool import strtobool
+from distutils.util import strtobool
 
 from wtforms import (
     BooleanField,
@@ -236,26 +236,21 @@ class ValidateJinja2Template(object):
     def __call__(self, form, field):
         from changedetectionio import notification
 
-        from jinja2 import BaseLoader, TemplateSyntaxError, UndefinedError
-        from jinja2.sandbox import ImmutableSandboxedEnvironment
+        from jinja2 import Environment, BaseLoader, TemplateSyntaxError, UndefinedError
         from jinja2.meta import find_undeclared_variables
-        import jinja2.exceptions
 
-        # Might be a list of text, or might be just text (like from the apprise url list)
-        joined_data = ' '.join(map(str, field.data)) if isinstance(field.data, list) else f"{field.data}"
 
         try:
-            jinja2_env = ImmutableSandboxedEnvironment(loader=BaseLoader)
+            jinja2_env = Environment(loader=BaseLoader)
             jinja2_env.globals.update(notification.valid_tokens)
-            jinja2_env.from_string(joined_data).render()
+
+            rendered = jinja2_env.from_string(field.data).render()
         except TemplateSyntaxError as e:
             raise ValidationError(f"This is not a valid Jinja2 template: {e}") from e
         except UndefinedError as e:
             raise ValidationError(f"A variable or function is not defined: {e}") from e
-        except jinja2.exceptions.SecurityError as e:
-            raise ValidationError(f"This is not a valid Jinja2 template: {e}") from e
 
-        ast = jinja2_env.parse(joined_data)
+        ast = jinja2_env.parse(field.data)
         undefined = ", ".join(find_undeclared_variables(ast))
         if undefined:
             raise ValidationError(
@@ -420,7 +415,7 @@ class quickWatchForm(Form):
 # Common to a single watch and the global settings
 class commonSettingsForm(Form):
 
-    notification_urls = StringListField('Notification URL List', validators=[validators.Optional(), ValidateAppRiseServers(), ValidateJinja2Template()])
+    notification_urls = StringListField('Notification URL List', validators=[validators.Optional(), ValidateAppRiseServers()])
     notification_title = StringField('Notification Title', default='ChangeDetection.io Notification - {{ watch_url }}', validators=[validators.Optional(), ValidateJinja2Template()])
     notification_body = TextAreaField('Notification Body', default='{{ watch_url }} had a change.', validators=[validators.Optional(), ValidateJinja2Template()])
     notification_format = SelectField('Notification format', choices=valid_notification_formats.keys())
@@ -453,7 +448,6 @@ class watchForm(commonSettingsForm):
     tags = StringTagUUID('Group tag', [validators.Optional()], default='')
 
     time_between_check = FormField(TimeBetweenCheckForm)
-    time_between_check_use_default = BooleanField('Use global settings for time between check', default=False)
 
     include_filters = StringListField('CSS/JSONPath/JQ/XPath Filters', [ValidateCSSJSONXPATHInput()], default='')
 
@@ -505,9 +499,11 @@ class watchForm(commonSettingsForm):
             result = False
 
         # Attempt to validate jinja2 templates in the URL
+        from jinja2 import Environment
+        # Jinja2 available in URLs along with https://pypi.org/project/jinja2-time/
+        jinja2_env = Environment(extensions=['jinja2_time.TimeExtension'])
         try:
-            from changedetectionio.safe_jinja import render as jinja_render
-            jinja_render(template_str=self.url.data)
+            ready_url = str(jinja2_env.from_string(self.url.data).render())
         except Exception as e:
             self.url.errors.append('Invalid template syntax')
             result = False
@@ -526,10 +522,6 @@ class SingleExtraBrowser(Form):
     browser_connection_url = StringField('Browser connection URL', [validators.Optional()], render_kw={"placeholder": "wss://brightdata... wss://oxylabs etc", "size":50})
     # @todo do the validation here instead
 
-class DefaultUAInputForm(Form):
-    html_requests = StringField('Plaintext requests', validators=[validators.Optional()], render_kw={"placeholder": "<default>"})
-    if os.getenv("PLAYWRIGHT_DRIVER_URL") or os.getenv("WEBDRIVER_URL"):
-        html_webdriver = StringField('Chrome requests', validators=[validators.Optional()], render_kw={"placeholder": "<default>"})
 
 # datastore.data['settings']['requests']..
 class globalSettingsRequestForm(Form):
@@ -541,8 +533,6 @@ class globalSettingsRequestForm(Form):
     extra_proxies = FieldList(FormField(SingleExtraProxy), min_entries=5)
     extra_browsers = FieldList(FormField(SingleExtraBrowser), min_entries=5)
 
-    default_ua = FormField(DefaultUAInputForm, label="Default User-Agent overrides")
-
     def validate_extra_proxies(self, extra_validators=None):
         for e in self.data['extra_proxies']:
             if e.get('proxy_name') or e.get('proxy_url'):
@@ -572,8 +562,6 @@ class globalSettingsApplicationForm(commonSettingsForm):
     removepassword_button = SubmitField('Remove password', render_kw={"class": "pure-button pure-button-primary"})
     render_anchor_tag_content = BooleanField('Render anchor tag content', default=False)
     shared_diff_access = BooleanField('Allow access to view diff page when password is enabled', default=False, validators=[validators.Optional()])
-    rss_hide_muted_watches = BooleanField('Hide muted watches from RSS feed', default=True,
-                                      validators=[validators.Optional()])
     filter_failure_notification_threshold_attempts = IntegerField('Number of times the filter can be missing before sending a notification',
                                                                   render_kw={"style": "width: 5em;"},
                                                                   validators=[validators.NumberRange(min=0,

changedetectionio/html_tools.py

@@ -3,6 +3,8 @@ from bs4 import BeautifulSoup
 from inscriptis import get_text
 from jsonpath_ng.ext import parse
 from typing import List
+from inscriptis.css_profiles import CSS_PROFILES, HtmlElement
+from inscriptis.html_properties import Display
 from inscriptis.model.config import ParserConfig
 from xml.sax.saxutils import escape as xml_escape
 import json
@@ -167,14 +169,14 @@ def xpath1_filter(xpath_filter, html_content, append_pretty_line_formatting=Fals
         # And where the matched result doesn't include something that will cause Inscriptis to add a newline
         # (This way each 'match' reliably has a new-line in the diff)
         # Divs are converted to 4 whitespaces by inscriptis
-        if append_pretty_line_formatting and len(html_block) and (not hasattr(element, 'tag') or not element.tag in (['br', 'hr', 'div', 'p'])):
+        if append_pretty_line_formatting and len(html_block) and (not hasattr( element, 'tag' ) or not element.tag in (['br', 'hr', 'div', 'p'])):
             html_block += TEXT_FILTER_LIST_LINE_SUFFIX
 
-        # Some kind of text, UTF-8 or other
-        if isinstance(element, (str, bytes)):
-            html_block += element
+        if type(element) == etree._ElementStringResult:
+            html_block += str(element)
+        elif type(element) == etree._ElementUnicodeResult:
+            html_block += str(element)
         else:
-            # Return the HTML which will get parsed as text
             html_block += etree.tostring(element, pretty_print=True).decode('utf-8')
 
     return html_block
@@ -194,12 +196,12 @@ def extract_element(find='title', html_content=''):
 
 #
 def _parse_json(json_data, json_filter):
-    if json_filter.startswith("json:"):
+    if 'json:' in json_filter:
         jsonpath_expression = parse(json_filter.replace('json:', ''))
         match = jsonpath_expression.find(json_data)
         return _get_stripped_text_from_json_match(match)
 
-    if json_filter.startswith("jq:") or json_filter.startswith("jqraw:"):
+    if 'jq:' in json_filter:
 
         try:
             import jq
@@ -207,15 +209,10 @@ def _parse_json(json_data, json_filter):
             # `jq` requires full compilation in windows and so isn't generally available
             raise Exception("jq not support not found")
 
-        if json_filter.startswith("jq:"):
-            jq_expression = jq.compile(json_filter.removeprefix("jq:"))
-            match = jq_expression.input(json_data).all()
-            return _get_stripped_text_from_json_match(match)
+        jq_expression = jq.compile(json_filter.replace('jq:', ''))
+        match = jq_expression.input(json_data).all()
 
-        if json_filter.startswith("jqraw:"):
-            jq_expression = jq.compile(json_filter.removeprefix("jqraw:"))
-            match = jq_expression.input(json_data).all()
-            return '\n'.join(str(item) for item in match)
+        return _get_stripped_text_from_json_match(match)
 
 def _get_stripped_text_from_json_match(match):
     s = []

changedetectionio/model/App.py

@@ -5,9 +5,7 @@ from changedetectionio.notification import (
     default_notification_title,
 )
 
-# Equal to or greater than this number of FilterNotFoundInResponse exceptions will trigger a filter-not-found notification
 _FILTER_FAILURE_THRESHOLD_ATTEMPTS_DEFAULT = 6
-DEFAULT_SETTINGS_HEADERS_USERAGENT='Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/87.0.4280.66 Safari/537.36'
 
 class model(dict):
     base_config = {
@@ -24,10 +22,6 @@ class model(dict):
                     'time_between_check': {'weeks': None, 'days': None, 'hours': 3, 'minutes': None, 'seconds': None},
                     'timeout': int(getenv("DEFAULT_SETTINGS_REQUESTS_TIMEOUT", "45")),  # Default 45 seconds
                     'workers': int(getenv("DEFAULT_SETTINGS_REQUESTS_WORKERS", "10")),  # Number of threads, lower is better for slow connections
-                    'default_ua': {
-                        'html_requests': getenv("DEFAULT_SETTINGS_HEADERS_USERAGENT", DEFAULT_SETTINGS_HEADERS_USERAGENT),
-                        'html_webdriver': None,
-                    }
                 },
                 'application': {
                     # Custom notification content
@@ -47,8 +41,6 @@ class model(dict):
                     'pager_size': 50,
                     'password': False,
                     'render_anchor_tag_content': False,
-                    'rss_access_token': None,
-                    'rss_hide_muted_watches': True,
                     'schema_version' : 0,
                     'shared_diff_access': False,
                     'webdriver_delay': None , # Extra delay in seconds before extracting text

changedetectionio/model/Watch.py

@@ -1,6 +1,4 @@
-from changedetectionio.strtobool import strtobool
-from changedetectionio.safe_jinja import render as jinja_render
-
+from distutils.util import strtobool
 import os
 import re
 import time
@@ -12,7 +10,7 @@ from loguru import logger
 # file:// is further checked by ALLOW_FILE_URI
 SAFE_PROTOCOL_REGEX='^(http|https|ftp|file):'
 
-minimum_seconds_recheck_time = int(os.getenv('MINIMUM_SECONDS_RECHECK_TIME', 3))
+minimum_seconds_recheck_time = int(os.getenv('MINIMUM_SECONDS_RECHECK_TIME', 60))
 mtable = {'seconds': 1, 'minutes': 60, 'hours': 3600, 'days': 86400, 'weeks': 86400 * 7}
 
 from changedetectionio.notification import (
@@ -69,7 +67,6 @@ base_config = {
     # Requires setting to None on submit if it's the same as the default
     # Should be all None by default, so we use the system default in this case.
     'time_between_check': {'weeks': None, 'days': None, 'hours': None, 'minutes': None, 'seconds': None},
-    'time_between_check_use_default': True,
     'title': None,
     'trigger_text': [],  # List of text or regex to wait for until a change is detected
     'url': '',
@@ -140,11 +137,12 @@ class model(dict):
 
         ready_url = url
         if '{%' in url or '{{' in url:
+            from jinja2 import Environment
             # Jinja2 available in URLs along with https://pypi.org/project/jinja2-time/
+            jinja2_env = Environment(extensions=['jinja2_time.TimeExtension'])
             try:
-                ready_url = jinja_render(template_str=url)
+                ready_url = str(jinja2_env.from_string(url).render())
             except Exception as e:
-                logger.critical(f"Invalid URL template for: '{url}' - {str(e)}")
                 from flask import (
                     flash, Markup, url_for
                 )
@@ -238,8 +236,6 @@ class model(dict):
 
         if len(tmp_history):
             self.__newest_history_key = list(tmp_history.keys())[-1]
-        else:
-            self.__newest_history_key = None
 
         self.__history_n = len(tmp_history)
 
@@ -330,10 +326,13 @@ class model(dict):
     def save_history_text(self, contents, timestamp, snapshot_id):
         import brotli
 
-        logger.trace(f"{self.get('uuid')} - Updating history.txt with timestamp {timestamp}")
-
         self.ensure_data_dir_exists()
 
+        # Small hack so that we sleep just enough to allow 1 second  between history snapshots
+        # this is because history.txt indexes/keys snapshots by epoch seconds and we dont want dupe keys
+        if self.__newest_history_key and int(timestamp) == int(self.__newest_history_key):
+            time.sleep(timestamp - self.__newest_history_key)
+
         threshold = int(os.getenv('SNAPSHOT_BROTLI_COMPRESSION_THRESHOLD', 1024))
         skip_brotli = strtobool(os.getenv('DISABLE_BROTLI_TEXT_SNAPSHOT', 'False'))
 
@@ -363,7 +362,6 @@ class model(dict):
         # @todo bump static cache of the last timestamp so we dont need to examine the file to set a proper ''viewed'' status
         return snapshot_fname
 
-    @property
     @property
     def has_empty_checktime(self):
         # using all() + dictionary comprehension
@@ -525,42 +523,8 @@ class model(dict):
         # None is set
         return False
 
-    def save_error_text(self, contents):
-        self.ensure_data_dir_exists()
-        target_path = os.path.join(self.watch_data_dir, "last-error.txt")
-        with open(target_path, 'w') as f:
-            f.write(contents)
 
-    def save_xpath_data(self, data, as_error=False):
-        import json
-
-        if as_error:
-            target_path = os.path.join(self.watch_data_dir, "elements-error.json")
-        else:
-            target_path = os.path.join(self.watch_data_dir, "elements.json")
-
-        self.ensure_data_dir_exists()
-
-        with open(target_path, 'w') as f:
-            f.write(json.dumps(data))
-            f.close()
-
-    # Save as PNG, PNG is larger but better for doing visual diff in the future
-    def save_screenshot(self, screenshot: bytes, as_error=False):
-
-        if as_error:
-            target_path = os.path.join(self.watch_data_dir, "last-error-screenshot.png")
-        else:
-            target_path = os.path.join(self.watch_data_dir, "last-screenshot.png")
-
-        self.ensure_data_dir_exists()
-
-        with open(target_path, 'wb') as f:
-            f.write(screenshot)
-            f.close()
-
-
-    def get_last_fetched_text_before_filters(self):
+    def get_last_fetched_before_filters(self):
         import brotli
         filepath = os.path.join(self.watch_data_dir, 'last-fetched.br')
 
@@ -575,56 +539,12 @@ class model(dict):
         with open(filepath, 'rb') as f:
             return(brotli.decompress(f.read()).decode('utf-8'))
 
-    def save_last_text_fetched_before_filters(self, contents):
+    def save_last_fetched_before_filters(self, contents):
         import brotli
         filepath = os.path.join(self.watch_data_dir, 'last-fetched.br')
         with open(filepath, 'wb') as f:
             f.write(brotli.compress(contents, mode=brotli.MODE_TEXT))
 
-    def save_last_fetched_html(self, timestamp, contents):
-        import brotli
-
-        self.ensure_data_dir_exists()
-        snapshot_fname = f"{timestamp}.html.br"
-        filepath = os.path.join(self.watch_data_dir, snapshot_fname)
-
-        with open(filepath, 'wb') as f:
-            contents = contents.encode('utf-8') if isinstance(contents, str) else contents
-            try:
-                f.write(brotli.compress(contents))
-            except Exception as e:
-                logger.warning(f"{self.get('uuid')} - Unable to compress snapshot, saving as raw data to {filepath}")
-                logger.warning(e)
-                f.write(contents)
-
-        self._prune_last_fetched_html_snapshots()
-
-    def get_fetched_html(self, timestamp):
-        import brotli
-
-        snapshot_fname = f"{timestamp}.html.br"
-        filepath = os.path.join(self.watch_data_dir, snapshot_fname)
-        if os.path.isfile(filepath):
-            with open(filepath, 'rb') as f:
-                return (brotli.decompress(f.read()).decode('utf-8'))
-
-        return False
-
-
-    def _prune_last_fetched_html_snapshots(self):
-
-        dates = list(self.history.keys())
-        dates.reverse()
-
-        for index, timestamp in enumerate(dates):
-            snapshot_fname = f"{timestamp}.html.br"
-            filepath = os.path.join(self.watch_data_dir, snapshot_fname)
-
-            # Keep only the first 2
-            if index > 1 and os.path.isfile(filepath):
-                os.remove(filepath)
-
-
     @property
     def get_browsersteps_available_screenshots(self):
         "For knowing which screenshots are available to show the user in BrowserSteps UI"

changedetectionio/notification.py

@@ -1,5 +1,6 @@
 import apprise
 import time
+from jinja2 import Environment, BaseLoader
 from apprise import NotifyFormat
 import json
 from loguru import logger
@@ -48,7 +49,7 @@ from apprise.decorators import notify
 def apprise_custom_api_call_wrapper(body, title, notify_type, *args, **kwargs):
     import requests
     from apprise.utils import parse_url as apprise_parse_url
-    from apprise import URLBase
+    from apprise.URLBase import URLBase
 
     url = kwargs['meta'].get('url')
 
@@ -115,13 +116,16 @@ def apprise_custom_api_call_wrapper(body, title, notify_type, *args, **kwargs):
 
 def process_notification(n_object, datastore):
 
-    from .safe_jinja import render as jinja_render
     now = time.time()
     if n_object.get('notification_timestamp'):
         logger.trace(f"Time since queued {now-n_object['notification_timestamp']:.3f}s")
     # Insert variables into the notification content
     notification_parameters = create_notification_parameters(n_object, datastore)
 
+    # Get the notification body from datastore
+    jinja2_env = Environment(loader=BaseLoader)
+    n_body = jinja2_env.from_string(n_object.get('notification_body', '')).render(**notification_parameters)
+    n_title = jinja2_env.from_string(n_object.get('notification_title', '')).render(**notification_parameters)
     n_format = valid_notification_formats.get(
         n_object.get('notification_format', default_notification_format),
         valid_notification_formats[default_notification_format],
@@ -147,18 +151,13 @@ def process_notification(n_object, datastore):
 
     with apprise.LogCapture(level=apprise.logging.DEBUG) as logs:
         for url in n_object['notification_urls']:
-
-            # Get the notification body from datastore
-            n_body = jinja_render(template_str=n_object.get('notification_body', ''), **notification_parameters)
-            n_title = jinja_render(template_str=n_object.get('notification_title', ''), **notification_parameters)
-
             url = url.strip()
             if not url:
                 logger.warning(f"Process Notification: skipping empty notification URL.")
                 continue
 
             logger.info(">> Process Notification: AppRise notifying {}".format(url))
-            url = jinja_render(template_str=url, **notification_parameters)
+            url = jinja2_env.from_string(url).render(**notification_parameters)
 
             # Re 323 - Limit discord length to their 2000 char limit total or it wont send.
             # Because different notifications may require different pre-processing, run each sequentially :(

changedetectionio/processors/__init__.py

@@ -1,11 +1,10 @@
 from abc import abstractmethod
-from changedetectionio.strtobool import strtobool
-from changedetectionio.model import Watch
-from copy import deepcopy
-from loguru import logger
-import hashlib
 import os
+import hashlib
 import re
+from copy import deepcopy
+from distutils.util import strtobool
+from loguru import logger
 
 class difference_detection_processor():
 
@@ -22,7 +21,7 @@ class difference_detection_processor():
         self.watch = deepcopy(self.datastore.data['watching'].get(watch_uuid))
 
     def call_browser(self):
-        from requests.structures import CaseInsensitiveDict
+
         # Protect against file:// access
         if re.search(r'^file://', self.watch.get('url', '').strip(), re.IGNORECASE):
             if not strtobool(os.getenv('ALLOW_FILE_URI', 'false')):
@@ -76,12 +75,12 @@ class difference_detection_processor():
 
         proxy_url = None
         if preferred_proxy_id:
-            # Custom browser endpoints should NOT have a proxy added
-            if not prefer_fetch_backend.startswith('extra_browser_'):
+            # Custom browser endpoints should not have a proxy added
+            if not preferred_proxy_id.startswith('ui-'):
                 proxy_url = self.datastore.proxy_list.get(preferred_proxy_id).get('url')
                 logger.debug(f"Selected proxy key '{preferred_proxy_id}' as proxy URL '{proxy_url}' for {url}")
             else:
-                logger.debug(f"Skipping adding proxy data when custom Browser endpoint is specified. ")
+                logger.debug(f"Skipping adding proxy data when custom Browser endpoint is specified.")
 
         # Now call the fetcher (playwright/requests/etc) with arguments that only a fetcher would need.
         # When browser_connection_url is None, it method should default to working out whats the best defaults (os env vars etc)
@@ -94,13 +93,7 @@ class difference_detection_processor():
             self.fetcher.browser_steps_screenshot_path = os.path.join(self.datastore.datastore_path, self.watch.get('uuid'))
 
         # Tweak the base config with the per-watch ones
-        request_headers = CaseInsensitiveDict()
-
-        ua = self.datastore.data['settings']['requests'].get('default_ua')
-        if ua and ua.get(prefer_fetch_backend):
-            request_headers.update({'User-Agent': ua.get(prefer_fetch_backend)})
-
-        request_headers.update(self.watch.get('headers', {}))
+        request_headers = self.watch.get('headers', [])
         request_headers.update(self.datastore.get_all_base_headers())
         request_headers.update(self.datastore.get_all_headers_in_textfile_for_watch(uuid=self.watch.get('uuid')))
 
@@ -139,7 +132,7 @@ class difference_detection_processor():
         # After init, call run_changedetection() which will do the actual change-detection
 
     @abstractmethod
-    def run_changedetection(self, watch: Watch, skip_when_checksum_same=True):
+    def run_changedetection(self, uuid, skip_when_checksum_same=True):
         update_obj = {'last_notification_error': False, 'last_error': False}
         some_data = 'xxxxx'
         update_obj["previous_md5"] = hashlib.md5(some_data.encode('utf-8')).hexdigest()

changedetectionio/processors/restock_diff.py

@@ -1,5 +1,6 @@
 
 from . import difference_detection_processor
+from copy import deepcopy
 from loguru import logger
 import hashlib
 import urllib3
@@ -19,7 +20,10 @@ class perform_site_check(difference_detection_processor):
     screenshot = None
     xpath_data = None
 
-    def run_changedetection(self, watch, skip_when_checksum_same=True):
+    def run_changedetection(self, uuid, skip_when_checksum_same=True):
+
+        # DeepCopy so we can be sure we don't accidently change anything by reference
+        watch = deepcopy(self.datastore.data['watching'].get(uuid))
 
         if not watch:
             raise Exception("Watch no longer exists.")
@@ -40,13 +44,13 @@ class perform_site_check(difference_detection_processor):
             fetched_md5 = hashlib.md5(self.fetcher.instock_data.encode('utf-8')).hexdigest()
             # 'Possibly in stock' comes from stock-not-in-stock.js when no string found above the fold.
             update_obj["in_stock"] = True if self.fetcher.instock_data == 'Possibly in stock' else False
-            logger.debug(f"Watch UUID {watch.get('uuid')} restock check returned '{self.fetcher.instock_data}' from JS scraper.")
+            logger.debug(f"Watch UUID {uuid} restock check returned '{self.fetcher.instock_data}' from JS scraper.")
         else:
             raise UnableToExtractRestockData(status_code=self.fetcher.status_code)
 
         # The main thing that all this at the moment comes down to :)
         changed_detected = False
-        logger.debug(f"Watch UUID {watch.get('uuid')} restock check - Previous MD5: {watch.get('previous_md5')}, Fetched MD5 {fetched_md5}")
+        logger.debug(f"Watch UUID {uuid} restock check - Previous MD5: {watch.get('previous_md5')}, Fetched MD5 {fetched_md5}")
 
         if watch.get('previous_md5') and watch.get('previous_md5') != fetched_md5:
             # Yes if we only care about it going to instock, AND we are in stock

changedetectionio/processors/text_json_diff.py

@@ -10,18 +10,18 @@ from . import difference_detection_processor
 from ..html_tools import PERL_STYLE_REGEX, cdata_in_document_to_text
 from changedetectionio import html_tools, content_fetchers
 from changedetectionio.blueprint.price_data_follower import PRICE_DATA_TRACK_ACCEPT, PRICE_DATA_TRACK_REJECT
+import changedetectionio.content_fetchers
+from copy import deepcopy
 from loguru import logger
 
 urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)
 
 name = 'Webpage Text/HTML, JSON and PDF changes'
 description = 'Detects all text changes where possible'
-json_filter_prefixes = ['json:', 'jq:', 'jqraw:']
+json_filter_prefixes = ['json:', 'jq:']
 
 class FilterNotFoundInResponse(ValueError):
-    def __init__(self, msg, screenshot=None, xpath_data=None):
-        self.screenshot = screenshot
-        self.xpath_data = xpath_data
+    def __init__(self, msg):
         ValueError.__init__(self, msg)
 
 
@@ -34,12 +34,14 @@ class PDFToHTMLToolNotFound(ValueError):
 # (set_proxy_from_list)
 class perform_site_check(difference_detection_processor):
 
-    def run_changedetection(self, watch, skip_when_checksum_same=True):
+    def run_changedetection(self, uuid, skip_when_checksum_same=True):
         changed_detected = False
         html_content = ""
         screenshot = False  # as bytes
         stripped_text_from_html = ""
 
+        # DeepCopy so we can be sure we don't accidently change anything by reference
+        watch = deepcopy(self.datastore.data['watching'].get(uuid))
         if not watch:
             raise Exception("Watch no longer exists.")
 
@@ -114,12 +116,12 @@ class perform_site_check(difference_detection_processor):
         # Better would be if Watch.model could access the global data also
         # and then use getattr https://docs.python.org/3/reference/datamodel.html#object.__getitem__
         # https://realpython.com/inherit-python-dict/ instead of doing it procedurely
-        include_filters_from_tags = self.datastore.get_tag_overrides_for_watch(uuid=watch.get('uuid'), attr='include_filters')
+        include_filters_from_tags = self.datastore.get_tag_overrides_for_watch(uuid=uuid, attr='include_filters')
 
         # 1845 - remove duplicated filters in both group and watch include filter
-        include_filters_rule = list(dict.fromkeys(watch.get('include_filters', []) + include_filters_from_tags))
+        include_filters_rule = list({*watch.get('include_filters', []), *include_filters_from_tags})
 
-        subtractive_selectors = [*self.datastore.get_tag_overrides_for_watch(uuid=watch.get('uuid'), attr='subtractive_selectors'),
+        subtractive_selectors = [*self.datastore.get_tag_overrides_for_watch(uuid=uuid, attr='subtractive_selectors'),
                                  *watch.get("subtractive_selectors", []),
                                  *self.datastore.data["settings"]["application"].get("global_subtractive_selectors", [])
                                  ]
@@ -186,7 +188,7 @@ class perform_site_check(difference_detection_processor):
                                                                        append_pretty_line_formatting=not watch.is_source_type_url)
 
                     if not html_content.strip():
-                        raise FilterNotFoundInResponse(msg=include_filters_rule, screenshot=self.fetcher.screenshot, xpath_data=self.fetcher.xpath_data)
+                        raise FilterNotFoundInResponse(include_filters_rule)
 
                 if has_subtractive_selectors:
                     html_content = html_tools.element_removal(subtractive_selectors, html_content)
@@ -220,7 +222,7 @@ class perform_site_check(difference_detection_processor):
             from .. import diff
             # needs to not include (added) etc or it may get used twice
             # Replace the processed text with the preferred result
-            rendered_diff = diff.render_diff(previous_version_file_contents=watch.get_last_fetched_text_before_filters(),
+            rendered_diff = diff.render_diff(previous_version_file_contents=watch.get_last_fetched_before_filters(),
                                              newest_version_file_contents=stripped_text_from_html,
                                              include_equal=False,  # not the same lines
                                              include_added=watch.get('filter_text_added', True),
@@ -229,7 +231,7 @@ class perform_site_check(difference_detection_processor):
                                              line_feed_sep="\n",
                                              include_change_type_prefix=False)
 
-            watch.save_last_text_fetched_before_filters(text_content_before_ignored_filter)
+            watch.save_last_fetched_before_filters(text_content_before_ignored_filter)
 
             if not rendered_diff and stripped_text_from_html:
                 # We had some content, but no differences were found
@@ -244,10 +246,9 @@ class perform_site_check(difference_detection_processor):
         if not is_json and not empty_pages_are_a_change and len(stripped_text_from_html.strip()) == 0:
             raise content_fetchers.exceptions.ReplyWithContentButNoText(url=url,
                                                             status_code=self.fetcher.get_last_status_code(),
-                                                            screenshot=self.fetcher.screenshot,
+                                                            screenshot=screenshot,
                                                             has_filters=has_filter_rule,
-                                                            html_content=html_content,
-                                                            xpath_data=self.fetcher.xpath_data
+                                                            html_content=html_content
                                                             )
 
         # We rely on the actual text in the html output.. many sites have random script vars etc,
@@ -343,17 +344,17 @@ class perform_site_check(difference_detection_processor):
                 if not watch['title'] or not len(watch['title']):
                     update_obj['title'] = html_tools.extract_element(find='title', html_content=self.fetcher.content)
 
-        logger.debug(f"Watch UUID {watch.get('uuid')} content check - Previous MD5: {watch.get('previous_md5')}, Fetched MD5 {fetched_md5}")
+        logger.debug(f"Watch UUID {uuid} content check - Previous MD5: {watch.get('previous_md5')}, Fetched MD5 {fetched_md5}")
 
         if changed_detected:
             if watch.get('check_unique_lines', False):
                 has_unique_lines = watch.lines_contain_something_unique_compared_to_history(lines=stripped_text_from_html.splitlines())
                 # One or more lines? unsure?
                 if not has_unique_lines:
-                    logger.debug(f"check_unique_lines: UUID {watch.get('uuid')} didnt have anything new setting change_detected=False")
+                    logger.debug(f"check_unique_lines: UUID {uuid} didnt have anything new setting change_detected=False")
                     changed_detected = False
                 else:
-                    logger.debug(f"check_unique_lines: UUID {watch.get('uuid')} had unique content")
+                    logger.debug(f"check_unique_lines: UUID {uuid} had unique content")
 
         # Always record the new checksum
         update_obj["previous_md5"] = fetched_md5

changedetectionio/safe_jinja.py

@@ -1,18 +0,0 @@
-"""
-Safe Jinja2 render with max payload sizes
-
-See https://jinja.palletsprojects.com/en/3.1.x/sandbox/#security-considerations
-"""
-
-import jinja2.sandbox
-import typing as t
-import os
-
-JINJA2_MAX_RETURN_PAYLOAD_SIZE = 1024 * int(os.getenv("JINJA2_MAX_RETURN_PAYLOAD_SIZE_KB", 1024 * 10))
-
-
-def render(template_str, **args: t.Any) -> str:
-    jinja2_env = jinja2.sandbox.ImmutableSandboxedEnvironment(extensions=['jinja2_time.TimeExtension'])
-    output = jinja2_env.from_string(template_str).render(args)
-    return output[:JINJA2_MAX_RETURN_PAYLOAD_SIZE]
-

changedetectionio/static/js/browser-steps.js

@@ -1,5 +1,14 @@
 $(document).ready(function () {
 
+    // duplicate
+    var csrftoken = $('input[name=csrf_token]').val();
+    $.ajaxSetup({
+        beforeSend: function (xhr, settings) {
+            if (!/^(GET|HEAD|OPTIONS|TRACE)$/i.test(settings.type) && !this.crossDomain) {
+                xhr.setRequestHeader("X-CSRFToken", csrftoken)
+            }
+        }
+    })
     var browsersteps_session_id;
     var browser_interface_seconds_remaining = 0;
     var apply_buttons_disabled = false;
@@ -17,8 +26,7 @@ $(document).ready(function () {
         set_scale();
     });
     // Should always be disabled
-    $('#browser_steps-0-operation option[value="Goto site"]').prop("selected", "selected");
-    $('#browser_steps-0-operation').attr('disabled', 'disabled');
+    $('#browser_steps >li:first-child select').val('Goto site').attr('disabled', 'disabled');
 
     $('#browsersteps-click-start').click(function () {
         $("#browsersteps-click-start").fadeOut();
@@ -152,12 +160,6 @@ $(document).ready(function () {
                     e.offsetX > item.left * y_scale && e.offsetX < item.left * y_scale + item.width * y_scale
 
                 ) {
-                    // Ignore really large ones, because we are scraping 'div' also from xpath_element_scraper but
-                    // that div or whatever could be some wrapper and would generally make you select the whole page
-                    if (item.width > 800 && item.height > 400) {
-                        return
-                    }
-
                     // There could be many elements here, record them all and then we'll find out which is the most 'useful'
                     // (input, textarea, button, A etc)
                     if (item.width < xpath_data['browser_width']) {

changedetectionio/static/js/csrf.js

@@ -1,10 +0,0 @@
-$(document).ready(function () {
-    $.ajaxSetup({
-        beforeSend: function (xhr, settings) {
-            if (!/^(GET|HEAD|OPTIONS|TRACE)$/i.test(settings.type) && !this.crossDomain) {
-                xhr.setRequestHeader("X-CSRFToken", csrftoken)
-            }
-        }
-    })
-});
-

changedetectionio/static/js/diff-overview.js

@@ -1,10 +1,12 @@
 $(document).ready(function () {
-    $('.needs-localtime').each(function () {
-        for (var option of this.options) {
-            var dateObject = new Date(option.value * 1000);
-            option.label = dateObject.toLocaleString(undefined, {dateStyle: "full", timeStyle: "medium"});
+    var csrftoken = $('input[name=csrf_token]').val();
+    $.ajaxSetup({
+        beforeSend: function (xhr, settings) {
+            if (!/^(GET|HEAD|OPTIONS|TRACE)$/i.test(settings.type) && !this.crossDomain) {
+                xhr.setRequestHeader("X-CSRFToken", csrftoken)
+            }
         }
-    });
+    })
 
     // Load it when the #screenshot tab is in use, so we dont give a slow experience when waiting for the text diff to load
     window.addEventListener('hashchange', function (e) {
@@ -39,12 +41,6 @@ $(document).ready(function () {
       $("#highlightSnippet").remove();
     }
 
-    // Listen for Escape key press
-    window.addEventListener('keydown', function (e) {
-        if (e.key === 'Escape') {
-            clean();
-        }
-    }, false);
 
     function dragTextHandler(event) {
         console.log('mouseupped');

changedetectionio/static/js/diff-render.js

@@ -79,7 +79,12 @@ $(document).ready(function () {
         $('#jump-next-diff').click();
     }
 
-
+    $('.needs-localtime').each(function () {
+        for (var option of this.options) {
+            var dateObject = new Date(option.value * 1000);
+            option.label = dateObject.toLocaleString(undefined, {dateStyle: "full", timeStyle: "medium"});
+        }
+    })
     onDiffTypeChange(
         document.querySelector('#settings [name="diff_type"]:checked'),
     );

changedetectionio/static/js/notifications.js

@@ -13,16 +13,30 @@ $(document).ready(function() {
   $('#send-test-notification').click(function (e) {
     e.preventDefault();
 
+    // this can be global
+    var csrftoken = $('input[name=csrf_token]').val();
+    $.ajaxSetup({
+        beforeSend: function(xhr, settings) {
+            if (!/^(GET|HEAD|OPTIONS|TRACE)$/i.test(settings.type) && !this.crossDomain) {
+                xhr.setRequestHeader("X-CSRFToken", csrftoken)
+            }
+        }
+    })
+
     data = {
       notification_body: $('#notification_body').val(),
       notification_format: $('#notification_format').val(),
       notification_title: $('#notification_title').val(),
       notification_urls: $('.notification-urls').val(),
-      tags: $('#tags').val(),
       window_url: window.location.href,
     }
 
 
+    if (!data['notification_urls'].length) {
+      alert("Notification URL list is empty, cannot send test.")
+      return;
+    }
+
     $.ajax({
       type: "POST",
       url: notification_base_url,
@@ -35,7 +49,7 @@ $(document).ready(function() {
       }
     }).done(function(data){
       console.log(data);
-      alert(data);
+      alert('Sent');
     }).fail(function(data){
       console.log(data);
       alert('There was an error communicating with the server.');

changedetectionio/static/js/preview.js

@@ -1,53 +0,0 @@
-function redirect_to_version(version) {
-    var currentUrl = window.location.href;
-    var baseUrl = currentUrl.split('?')[0]; // Base URL without query parameters
-    var anchor = '';
-
-    // Check if there is an anchor
-    if (baseUrl.indexOf('#') !== -1) {
-        anchor = baseUrl.substring(baseUrl.indexOf('#'));
-        baseUrl = baseUrl.substring(0, baseUrl.indexOf('#'));
-    }
-    window.location.href = baseUrl + '?version=' + version + anchor;
-}
-
-document.addEventListener('keydown', function (event) {
-    var selectElement = document.getElementById('preview-version');
-    if (selectElement) {
-        var selectedOption = selectElement.querySelector('option:checked');
-        if (selectedOption) {
-            if (event.key === 'ArrowLeft') {
-                if (selectedOption.previousElementSibling) {
-                    redirect_to_version(selectedOption.previousElementSibling.value);
-                }
-            } else if (event.key === 'ArrowRight') {
-                if (selectedOption.nextElementSibling) {
-                    redirect_to_version(selectedOption.nextElementSibling.value);
-                }
-            }
-        }
-    }
-});
-
-
-document.getElementById('preview-version').addEventListener('change', function () {
-    redirect_to_version(this.value);
-});
-
-var selectElement = document.getElementById('preview-version');
-if (selectElement) {
-    var selectedOption = selectElement.querySelector('option:checked');
-    if (selectedOption) {
-        if (selectedOption.previousElementSibling) {
-            document.getElementById('btn-previous').href = "?version=" + selectedOption.previousElementSibling.value;
-        } else {
-            document.getElementById('btn-previous').remove()
-        }
-        if (selectedOption.nextElementSibling) {
-            document.getElementById('btn-next').href = "?version=" + selectedOption.nextElementSibling.value;
-        } else {
-            document.getElementById('btn-next').remove()
-        }
-
-    }
-}

changedetectionio/static/js/visual-selector.js

@@ -2,258 +2,250 @@
 // All rights reserved.
 // yes - this is really a hack, if you are a front-ender and want to help, please get in touch!
 
-let runInClearMode = false;
+$(document).ready(function () {
 
-$(document).ready(() => {
-    let currentSelections = [];
-    let currentSelection = null;
-    let appendToList = false;
-    let c, xctx, ctx;
-    let xScale = 1, yScale = 1;
-    let selectorImage, selectorImageRect, selectorData;
+    var current_selected_i;
+    var state_clicked = false;
 
+    var c;
 
-    // Global jQuery selectors with "Elem" appended
-    const $selectorCanvasElem = $('#selector-canvas');
-    const $includeFiltersElem = $("#include_filters");
-    const $selectorBackgroundElem = $("img#selector-background");
-    const $selectorCurrentXpathElem = $("#selector-current-xpath span");
-    const $fetchingUpdateNoticeElem = $('.fetching-update-notice');
-    const $selectorWrapperElem = $("#selector-wrapper");
+    // greyed out fill context
+    var xctx;
+    // redline highlight context
+    var ctx;
 
-    // Color constants
-    const FILL_STYLE_HIGHLIGHT = 'rgba(205,0,0,0.35)';
-    const FILL_STYLE_GREYED_OUT = 'rgba(205,205,205,0.95)';
-    const STROKE_STYLE_HIGHLIGHT = 'rgba(255,0,0, 0.9)';
-    const FILL_STYLE_REDLINE = 'rgba(255,0,0, 0.1)';
-    const STROKE_STYLE_REDLINE = 'rgba(225,0,0,0.9)';
+    var current_default_xpath = [];
+    var x_scale = 1;
+    var y_scale = 1;
+    var selector_image;
+    var selector_image_rect;
+    var selector_data;
 
-    $('#visualselector-tab').click(() => {
-        $selectorBackgroundElem.off('load');
-        currentSelections = [];
-        bootstrapVisualSelector();
+    $('#visualselector-tab').click(function () {
+        $("img#selector-background").off('load');
+        state_clicked = false;
+        current_selected_i = false;
+        bootstrap_visualselector();
     });
 
-    function clearReset() {
-        ctx.clearRect(0, 0, c.width, c.height);
-
-        if ($includeFiltersElem.val().length) {
-            alert("Existing filters under the 'Filters & Triggers' tab were cleared.");
-        }
-        $includeFiltersElem.val('');
-
-        currentSelections = [];
-
-        // Means we ignore the xpaths from the scraper marked as sel.highlight_as_custom_filter (it matched a previous selector)
-        runInClearMode = true;
-
-        highlightCurrentSelected();
-    }
-
-    function splitToList(v) {
-        return v.split('\n').map(line => line.trim()).filter(line => line.length > 0);
-    }
-
-    function sortScrapedElementsBySize() {
-        // Sort the currentSelections array by area (width * height) in descending order
-        selectorData['size_pos'].sort((a, b) => {
-            const areaA = a.width * a.height;
-            const areaB = b.width * b.height;
-            return areaB - areaA;
-        });
-    }
-
-    $(document).on('keydown keyup', (event) => {
-        if (event.code === 'ShiftLeft' || event.code === 'ShiftRight') {
-            appendToList = event.type === 'keydown';
-        }
-
-        if (event.type === 'keydown') {
-            if ($selectorBackgroundElem.is(":visible") && event.key === "Escape") {
-                clearReset();
+    $(document).on('keydown', function (event) {
+        if ($("img#selector-background").is(":visible")) {
+            if (event.key == "Escape") {
+                state_clicked = false;
+                ctx.clearRect(0, 0, c.width, c.height);
             }
         }
     });
 
-    $('#clear-selector').on('click', () => {
-        clearReset();
-    });
-    // So if they start switching between visualSelector and manual filters, stop it from rendering old filters
-    $('li.tab a').on('click', () => {
-        runInClearMode = true;
-    });
-
-    if (!window.location.hash || window.location.hash !== '#visualselector') {
-        $selectorBackgroundElem.attr('src', '');
+    // For when the page loads
+    if (!window.location.hash || window.location.hash != '#visualselector') {
+        $("img#selector-background").attr('src', '');
         return;
     }
 
-    bootstrapVisualSelector();
+    // Handle clearing button/link
+    $('#clear-selector').on('click', function (event) {
+        if (!state_clicked) {
+            alert('Oops, Nothing selected!');
+        }
+        state_clicked = false;
+        ctx.clearRect(0, 0, c.width, c.height);
+        xctx.clearRect(0, 0, c.width, c.height);
+        $("#include_filters").val('');
+    });
 
-    function bootstrapVisualSelector() {
-        $selectorBackgroundElem
-            .on("error", () => {
-                $fetchingUpdateNoticeElem.html("<strong>Ooops!</strong> The VisualSelector tool needs at least one fetched page, please unpause the watch and/or wait for the watch to complete fetching and then reload this page.")
-                    .css('color', '#bb0000');
-                $('#selector-current-xpath, #clear-selector').hide();
-            })
-            .on('load', () => {
+
+    bootstrap_visualselector();
+
+
+    function bootstrap_visualselector() {
+        if (1) {
+            // bootstrap it, this will trigger everything else
+            $("img#selector-background").on("error", function () {
+                $('.fetching-update-notice').html("<strong>Ooops!</strong> The VisualSelector tool needs atleast one fetched page, please unpause the watch and/or wait for the watch to complete fetching and then reload this page.");
+                $('.fetching-update-notice').css('color','#bb0000');
+                $('#selector-current-xpath').hide();
+                $('#clear-selector').hide();
+            }).bind('load', function () {
                 console.log("Loaded background...");
                 c = document.getElementById("selector-canvas");
+                // greyed out fill context
                 xctx = c.getContext("2d");
+                // redline highlight context
                 ctx = c.getContext("2d");
-                fetchData();
-                $selectorCanvasElem.off("mousemove mousedown");
-            })
-            .attr("src", screenshot_url);
-
-        let s = `${$selectorBackgroundElem.attr('src')}?${new Date().getTime()}`;
-        $selectorBackgroundElem.attr('src', s);
-    }
-
-    function alertIfFilterNotFound() {
-        let existingFilters = splitToList($includeFiltersElem.val());
-        let sizePosXpaths = selectorData['size_pos'].map(sel => sel.xpath);
-
-        for (let filter of existingFilters) {
-            if (!sizePosXpaths.includes(filter)) {
-                alert(`One or more of your existing filters was not found and will be removed when a new filter is selected.`);
-                break;
-            }
+                if ($("#include_filters").val().trim().length) {
+                    current_default_xpath = $("#include_filters").val().split(/\r?\n/g);
+                } else {
+                    current_default_xpath = [];
+                }
+                fetch_data();
+                $('#selector-canvas').off("mousemove mousedown");
+                // screenshot_url defined in the edit.html template
+            }).attr("src", screenshot_url);
         }
+        // Tell visualSelector that the image should update
+        var s = $("img#selector-background").attr('src') + "?" + new Date().getTime();
+        $("img#selector-background").attr('src', s)
     }
 
-    function fetchData() {
-        $fetchingUpdateNoticeElem.html("Fetching element data..");
+    // This is fired once the img src is loaded in bootstrap_visualselector()
+    function fetch_data() {
+        // Image is ready
+        $('.fetching-update-notice').html("Fetching element data..");
 
         $.ajax({
             url: watch_visual_selector_data_url,
             context: document.body
-        }).done((data) => {
-            $fetchingUpdateNoticeElem.html("Rendering..");
-            selectorData = data;
-            sortScrapedElementsBySize();
-            console.log(`Reported browser width from backend: ${data['browser_width']}`);
-
-            // Little sanity check for the user, alert them if something missing
-            alertIfFilterNotFound();
-
-            setScale();
-            reflowSelector();
-            $fetchingUpdateNoticeElem.fadeOut();
+        }).done(function (data) {
+            $('.fetching-update-notice').html("Rendering..");
+            selector_data = data;
+            console.log("Reported browser width from backend: " + data['browser_width']);
+            state_clicked = false;
+            set_scale();
+            reflow_selector();
+            $('.fetching-update-notice').fadeOut();
         });
+
     }
 
-    function updateFiltersText() {
-        // Assuming currentSelections is already defined and contains the selections
-        let uniqueSelections = new Set(currentSelections.map(sel => (sel[0] === '/' ? `xpath:${sel.xpath}` : sel.xpath)));
 
-        if (currentSelections.length > 0) {
-            // Convert the Set back to an array and join with newline characters
-            let textboxFilterText = Array.from(uniqueSelections).join("\n");
-            $includeFiltersElem.val(textboxFilterText);
-        }
-    }
+    function set_scale() {
 
-    function setScale() {
-        $selectorWrapperElem.show();
-        selectorImage = $selectorBackgroundElem[0];
-        selectorImageRect = selectorImage.getBoundingClientRect();
+        // some things to check if the scaling doesnt work
+        // - that the widths/sizes really are about the actual screen size cat elements.json |grep -o width......|sort|uniq
+        $("#selector-wrapper").show();
+        selector_image = $("img#selector-background")[0];
+        selector_image_rect = selector_image.getBoundingClientRect();
 
-        $selectorCanvasElem.attr({
-            'height': selectorImageRect.height,
-            'width': selectorImageRect.width
-        });
-        $selectorWrapperElem.attr('width', selectorImageRect.width);
-        $('#visual-selector-heading').css('max-width', selectorImageRect.width + "px")
-
-        xScale = selectorImageRect.width / selectorImage.naturalWidth;
-        yScale = selectorImageRect.height / selectorImage.naturalHeight;
-
-        ctx.strokeStyle = STROKE_STYLE_HIGHLIGHT;
-        ctx.fillStyle = FILL_STYLE_REDLINE;
+        // make the canvas the same size as the image
+        $('#selector-canvas').attr('height', selector_image_rect.height);
+        $('#selector-canvas').attr('width', selector_image_rect.width);
+        $('#selector-wrapper').attr('width', selector_image_rect.width);
+        x_scale = selector_image_rect.width / selector_data['browser_width'];
+        y_scale = selector_image_rect.height / selector_image.naturalHeight;
+        ctx.strokeStyle = 'rgba(255,0,0, 0.9)';
+        ctx.fillStyle = 'rgba(255,0,0, 0.1)';
         ctx.lineWidth = 3;
-        console.log("Scaling set  x: " + xScale + " by y:" + yScale);
-        $("#selector-current-xpath").css('max-width', selectorImageRect.width);
+        console.log("scaling set  x: " + x_scale + " by y:" + y_scale);
+        $("#selector-current-xpath").css('max-width', selector_image_rect.width);
     }
 
-    function reflowSelector() {
-        $(window).resize(() => {
-            setScale();
-            highlightCurrentSelected();
+    function reflow_selector() {
+        $(window).resize(function () {
+            set_scale();
+            highlight_current_selected_i();
         });
+        var selector_currnt_xpath_text = $("#selector-current-xpath span");
 
-        setScale();
+        set_scale();
 
-        console.log(selectorData['size_pos'].length + " selectors found");
+        console.log(selector_data['size_pos'].length + " selectors found");
 
-        let existingFilters = splitToList($includeFiltersElem.val());
-
-        selectorData['size_pos'].forEach(sel => {
-            if ((!runInClearMode && sel.highlight_as_custom_filter) || existingFilters.includes(sel.xpath)) {
-                console.log("highlighting " + c);
-                currentSelections.push(sel);
+        // highlight the default one if we can find it in the xPath list
+        // or the xpath matches the default one
+        found = false;
+        if (current_default_xpath.length) {
+            // Find the first one that matches
+            // @todo In the future paint all that match
+            for (const c of current_default_xpath) {
+                for (var i = selector_data['size_pos'].length; i !== 0; i--) {
+                    if (selector_data['size_pos'][i - 1].xpath.trim() === c.trim()) {
+                        console.log("highlighting " + c);
+                        current_selected_i = i - 1;
+                        highlight_current_selected_i();
+                        found = true;
+                        break;
+                    }
+                }
+                if (found) {
+                    break;
+                }
             }
-        });
+            if (!found) {
+                alert("Unfortunately your existing CSS/xPath Filter was no longer found!");
+            }
+        }
 
 
-        highlightCurrentSelected();
-        updateFiltersText();
+        $('#selector-canvas').bind('mousemove', function (e) {
+            if (state_clicked) {
+                return;
+            }
+            ctx.clearRect(0, 0, c.width, c.height);
+            current_selected_i = null;
 
-        $selectorCanvasElem.bind('mousemove', handleMouseMove.debounce(5));
-        $selectorCanvasElem.bind('mousedown', handleMouseDown.debounce(5));
-        $selectorCanvasElem.bind('mouseleave', highlightCurrentSelected.debounce(5));
-
-        function handleMouseMove(e) {
-            if (!e.offsetX && !e.offsetY) {
-                const targetOffset = $(e.target).offset();
+            // Add in offset
+            if ((typeof e.offsetX === "undefined" || typeof e.offsetY === "undefined") || (e.offsetX === 0 && e.offsetY === 0)) {
+                var targetOffset = $(e.target).offset();
                 e.offsetX = e.pageX - targetOffset.left;
                 e.offsetY = e.pageY - targetOffset.top;
             }
 
-            ctx.fillStyle = FILL_STYLE_HIGHLIGHT;
+            // Reverse order - the most specific one should be deeper/"laster"
+            // Basically, find the most 'deepest'
+            var found = 0;
+            ctx.fillStyle = 'rgba(205,0,0,0.35)';
+            // Will be sorted by smallest width*height first
+            for (var i = 0; i <= selector_data['size_pos'].length; i++) {
+                // draw all of them? let them choose somehow?
+                var sel = selector_data['size_pos'][i];
+                // If we are in a bounding-box
+                if (e.offsetY > sel.top * y_scale && e.offsetY < sel.top * y_scale + sel.height * y_scale
+                    &&
+                    e.offsetX > sel.left * y_scale && e.offsetX < sel.left * y_scale + sel.width * y_scale
 
-            selectorData['size_pos'].forEach(sel => {
-                if (e.offsetY > sel.top * yScale && e.offsetY < sel.top * yScale + sel.height * yScale &&
-                    e.offsetX > sel.left * yScale && e.offsetX < sel.left * yScale + sel.width * yScale) {
-                    setCurrentSelectedText(sel.xpath);
-                    drawHighlight(sel);
-                    currentSelections.push(sel);
-                    currentSelection = sel;
-                    highlightCurrentSelected();
-                    currentSelections.pop();
+                ) {
+
+                    // FOUND ONE
+                    set_current_selected_text(sel.xpath);
+                    ctx.strokeRect(sel.left * x_scale, sel.top * y_scale, sel.width * x_scale, sel.height * y_scale);
+                    ctx.fillRect(sel.left * x_scale, sel.top * y_scale, sel.width * x_scale, sel.height * y_scale);
+
+                    // no need to keep digging
+                    // @todo or, O to go out/up, I to go in
+                    // or double click to go up/out the selector?
+                    current_selected_i = i;
+                    found += 1;
+                    break;
                 }
-            })
+            }
+
+        }.debounce(5));
+
+        function set_current_selected_text(s) {
+            selector_currnt_xpath_text[0].innerHTML = s;
+        }
+
+        function highlight_current_selected_i() {
+            if (state_clicked) {
+                state_clicked = false;
+                xctx.clearRect(0, 0, c.width, c.height);
+                return;
+            }
+
+            var sel = selector_data['size_pos'][current_selected_i];
+            if (sel[0] == '/') {
+                // @todo - not sure just checking / is right
+                $("#include_filters").val('xpath:' + sel.xpath);
+            } else {
+                $("#include_filters").val(sel.xpath);
+            }
+            xctx.fillStyle = 'rgba(205,205,205,0.95)';
+            xctx.strokeStyle = 'rgba(225,0,0,0.9)';
+            xctx.lineWidth = 3;
+            xctx.fillRect(0, 0, c.width, c.height);
+            // Clear out what only should be seen (make a clear/clean spot)
+            xctx.clearRect(sel.left * x_scale, sel.top * y_scale, sel.width * x_scale, sel.height * y_scale);
+            xctx.strokeRect(sel.left * x_scale, sel.top * y_scale, sel.width * x_scale, sel.height * y_scale);
+            state_clicked = true;
+            set_current_selected_text(sel.xpath);
+
         }
 
 
-        function setCurrentSelectedText(s) {
-            $selectorCurrentXpathElem[0].innerHTML = s;
-        }
-
-        function drawHighlight(sel) {
-            ctx.strokeRect(sel.left * xScale, sel.top * yScale, sel.width * xScale, sel.height * yScale);
-            ctx.fillRect(sel.left * xScale, sel.top * yScale, sel.width * xScale, sel.height * yScale);
-        }
-
-        function handleMouseDown() {
-            // If we are in 'appendToList' mode, grow the list, if not, just 1
-            currentSelections = appendToList ? [...currentSelections, currentSelection] : [currentSelection];
-            highlightCurrentSelected();
-            updateFiltersText();
-        }
-
-    }
-
-    function highlightCurrentSelected() {
-        xctx.fillStyle = FILL_STYLE_GREYED_OUT;
-        xctx.strokeStyle = STROKE_STYLE_REDLINE;
-        xctx.lineWidth = 3;
-        xctx.clearRect(0, 0, c.width, c.height);
-
-        currentSelections.forEach(sel => {
-            //xctx.clearRect(sel.left * xScale, sel.top * yScale, sel.width * xScale, sel.height * yScale);
-            xctx.strokeRect(sel.left * xScale, sel.top * yScale, sel.width * xScale, sel.height * yScale);
+        $('#selector-canvas').bind('mousedown', function (e) {
+            highlight_current_selected_i();
         });
     }
+
 });

changedetectionio/static/js/watch-settings.js

@@ -1,17 +1,3 @@
-function toggleOpacity(checkboxSelector, fieldSelector) {
-    const checkbox = document.querySelector(checkboxSelector);
-    const fields = document.querySelectorAll(fieldSelector);
-    function updateOpacity() {
-        const opacityValue = checkbox.checked ? 0.6 : 1;
-        fields.forEach(field => {
-            field.style.opacity = opacityValue;
-        });
-    }
-    // Initial setup
-    updateOpacity();
-    checkbox.addEventListener('change', updateOpacity);
-}
-
 $(document).ready(function () {
     $('#notification-setting-reset-to-default').click(function (e) {
         $('#notification_title').val('');
@@ -24,7 +10,4 @@ $(document).ready(function () {
         e.preventDefault();
         $('#notification-tokens-info').toggle();
     });
-
-    toggleOpacity('#time_between_check_use_default', '#time_between_check');
 });
-

changedetectionio/static/styles/diff.css

@@ -68,7 +68,7 @@
   --color-last-checked: #bbb;
   --color-text-footer: #444;
   --color-border-watch-table-cell: #eee;
-  --color-text-watch-tag-list: rgba(231, 0, 105, 0.4);
+  --color-text-watch-tag-list: #e70069;
   --color-background-new-watch-form: rgba(0, 0, 0, 0.05);
   --color-background-new-watch-input: var(--color-white);
   --color-text-new-watch-input: var(--color-text);
@@ -111,7 +111,7 @@ html[data-darkmode="true"] {
   --color-background-input: var(--color-grey-350);
   --color-text-input-description: var(--color-grey-600);
   --color-text-input-placeholder: var(--color-grey-600);
-  --color-text-watch-tag-list: rgba(250, 62, 146, 0.4);
+  --color-text-watch-tag-list: #fa3e92;
   --color-background-code: var(--color-grey-200);
   --color-background-tab: rgba(0, 0, 0, 0.2);
   --color-background-tab-hover: rgba(0, 0, 0, 0.5);

changedetectionio/static/styles/scss/parts/_variables.scss

@@ -75,7 +75,7 @@
   --color-text-footer: #444;
   --color-border-watch-table-cell: #eee;
 
-  --color-text-watch-tag-list: rgba(231, 0, 105, 0.4);
+  --color-text-watch-tag-list: #e70069;
   --color-background-new-watch-form: rgba(0, 0, 0, 0.05);
   --color-background-new-watch-input: var(--color-white);
   --color-text-new-watch-input: var(--color-text);
@@ -127,7 +127,7 @@ html[data-darkmode="true"] {
   --color-background-input: var(--color-grey-350);
   --color-text-input-description: var(--color-grey-600);
   --color-text-input-placeholder: var(--color-grey-600);
-  --color-text-watch-tag-list: rgba(250, 62, 146, 0.4);
+  --color-text-watch-tag-list: #fa3e92;
   --color-background-code: var(--color-grey-200);
 
   --color-background-tab: rgba(0, 0, 0, 0.2);

changedetectionio/static/styles/scss/parts/_visualselector.scss

@@ -1,8 +1,6 @@
 
 #selector-wrapper {
   height: 100%;
-  text-align: center;
-  
   max-height: 70vh;
   overflow-y: scroll;
   position: relative;

changedetectionio/static/styles/scss/styles.scss

@@ -187,11 +187,8 @@ code {
 }
 
 .watch-tag-list {
-  color: var(--color-white);
+  color: var(--color-text-watch-tag-list);
   white-space: nowrap;
-  background: var(--color-text-watch-tag-list);
-  border-radius: 5px;
-  padding: 2px 5px;
 }
 
 .box {
@@ -243,6 +240,7 @@ body::after {
 body::before {
   // background-image set in base.html so it works with reverse proxies etc
   content: "";
+  background-size: cover
 }
 
 body:after,
@@ -671,25 +669,14 @@ footer {
   and also iPads specifically.
   */
   .watch-table {
-    /* make headings work on mobile */
-    thead {
-      display: block;
-      tr {
-        th {
-          display: inline-block;
-        }
-      }
-      .empty-cell {
-        display: none;
-      }
-    }
 
     /* Force table to not be like tables anymore */
-    tbody {
-      td,
-      tr {
-        display: block;
-      }
+    thead,
+    tbody,
+    th,
+    td,
+    tr {
+      display: block;
     }
 
     .last-checked {
@@ -713,6 +700,13 @@ footer {
       display: inline-block;
     }
 
+    /* Hide table headers (but not display: none;, for accessibility) */
+    thead tr {
+      position: absolute;
+      top: -9999px;
+      left: -9999px;
+    }
+
     .pure-table td,
     .pure-table th {
       border: none;
@@ -757,7 +751,6 @@ footer {
   thead {
     background-color: var(--color-background-table-thead);
     color: var(--color-text);
-    border-bottom: 1px solid var(--color-background-table-thead);
   }
 
   td,
@@ -932,26 +925,23 @@ body.full-width {
       font-size: .875em;
     }
   }
-}
-
-.border-fieldset {
-  h3 {
-    margin-top: 0;
-  }
-  border: 1px solid #ccc;
-  padding: 1rem;
-  border-radius: 5px;
-  margin-bottom: 1rem;
-  fieldset:last-of-type {
-    padding-bottom: 0;
-    .pure-control-group {
+  .text-filtering {
+    h3 {
+      margin-top: 0;
+    }
+    border: 1px solid #ccc;
+    padding: 1rem;
+    border-radius: 5px;
+    margin-bottom: 1rem;
+    fieldset:last-of-type {
       padding-bottom: 0;
+      .pure-control-group {
+        padding-bottom: 0;
+      }
     }
   }
 }
 
-
-
 ul {
   padding-left: 1em;
   padding-top: 0px;
@@ -1026,11 +1016,6 @@ ul {
   border-radius: 10px;
   margin-bottom: 1em;
   display: none;
-  button {
-    /* some space if they wrap the page */
-    margin-bottom: 3px;
-    margin-top: 3px;
-  }
 }
 
 .checkbox-uuid {
@@ -1092,9 +1077,6 @@ ul {
     li {
       list-style: none;
       font-size: 0.8rem;
-      > * {
-        display: inline-block;
-      }
     }
   }
 
@@ -1114,16 +1096,3 @@ ul {
   white-space: nowrap;
 }
 
-#chrome-extension-link {
-  img {
-    height: 21px;
-    padding: 2px;
-    vertical-align: middle;
-  }
-
-  padding: 9px;
-  border: 1px solid var(--color-grey-800);
-  border-radius: 10px;
-  vertical-align: middle;
-}
-

changedetectionio/static/styles/styles.css

@@ -284,7 +284,7 @@ ul#requests-extra_browsers {
   --color-last-checked: #bbb;
   --color-text-footer: #444;
   --color-border-watch-table-cell: #eee;
-  --color-text-watch-tag-list: rgba(231, 0, 105, 0.4);
+  --color-text-watch-tag-list: #e70069;
   --color-background-new-watch-form: rgba(0, 0, 0, 0.05);
   --color-background-new-watch-input: var(--color-white);
   --color-text-new-watch-input: var(--color-text);
@@ -327,7 +327,7 @@ html[data-darkmode="true"] {
   --color-background-input: var(--color-grey-350);
   --color-text-input-description: var(--color-grey-600);
   --color-text-input-placeholder: var(--color-grey-600);
-  --color-text-watch-tag-list: rgba(250, 62, 146, 0.4);
+  --color-text-watch-tag-list: #fa3e92;
   --color-background-code: var(--color-grey-200);
   --color-background-tab: rgba(0, 0, 0, 0.2);
   --color-background-tab-hover: rgba(0, 0, 0, 0.5);
@@ -532,11 +532,8 @@ code {
     margin: 0 3px 0 5px; }
 
 .watch-tag-list {
-  color: var(--color-white);
-  white-space: nowrap;
-  background: var(--color-text-watch-tag-list);
-  border-radius: 5px;
-  padding: 2px 5px; }
+  color: var(--color-text-watch-tag-list);
+  white-space: nowrap; }
 
 .box {
   max-width: 80%;
@@ -574,7 +571,8 @@ body::after {
   opacity: 0.91; }
 
 body::before {
-  content: ""; }
+  content: "";
+  background-size: cover; }
 
 body:after,
 body:before {
@@ -863,17 +861,14 @@ footer {
   and also iPads specifically.
   */
   .watch-table {
-    /* make headings work on mobile */
     /* Force table to not be like tables anymore */
-    /* Force table to not be like tables anymore */ }
-    .watch-table thead {
-      display: block; }
-      .watch-table thead tr th {
-        display: inline-block; }
-      .watch-table thead .empty-cell {
-        display: none; }
-    .watch-table tbody td,
-    .watch-table tbody tr {
+    /* Force table to not be like tables anymore */
+    /* Hide table headers (but not display: none;, for accessibility) */ }
+    .watch-table thead,
+    .watch-table tbody,
+    .watch-table th,
+    .watch-table td,
+    .watch-table tr {
       display: block; }
     .watch-table .last-checked > span {
       vertical-align: middle; }
@@ -885,6 +880,10 @@ footer {
       content: "Last Changed "; }
     .watch-table td.inline {
       display: inline-block; }
+    .watch-table thead tr {
+      position: absolute;
+      top: -9999px;
+      left: -9999px; }
     .watch-table .pure-table td,
     .watch-table .pure-table th {
       border: none; }
@@ -911,8 +910,7 @@ footer {
   border-color: var(--color-border-table-cell); }
   .pure-table thead {
     background-color: var(--color-background-table-thead);
-    color: var(--color-text);
-    border-bottom: 1px solid var(--color-background-table-thead); }
+    color: var(--color-text); }
   .pure-table td,
   .pure-table th {
     border-left-color: var(--color-border-table-cell); }
@@ -1040,18 +1038,17 @@ body.full-width .edit-form {
     color: var(--color-text-input-description); }
     .edit-form .pure-form-message-inline code {
       font-size: .875em; }
-
-.border-fieldset {
-  border: 1px solid #ccc;
-  padding: 1rem;
-  border-radius: 5px;
-  margin-bottom: 1rem; }
-  .border-fieldset h3 {
-    margin-top: 0; }
-  .border-fieldset fieldset:last-of-type {
-    padding-bottom: 0; }
-    .border-fieldset fieldset:last-of-type .pure-control-group {
+  .edit-form .text-filtering {
+    border: 1px solid #ccc;
+    padding: 1rem;
+    border-radius: 5px;
+    margin-bottom: 1rem; }
+    .edit-form .text-filtering h3 {
+      margin-top: 0; }
+    .edit-form .text-filtering fieldset:last-of-type {
       padding-bottom: 0; }
+      .edit-form .text-filtering fieldset:last-of-type .pure-control-group {
+        padding-bottom: 0; }
 
 ul {
   padding-left: 1em;
@@ -1065,7 +1062,6 @@ ul {
 
 #selector-wrapper {
   height: 100%;
-  text-align: center;
   max-height: 70vh;
   overflow-y: scroll;
   position: relative; }
@@ -1128,10 +1124,6 @@ ul {
   border-radius: 10px;
   margin-bottom: 1em;
   display: none; }
-  #checkbox-operations button {
-    /* some space if they wrap the page */
-    margin-bottom: 3px;
-    margin-top: 3px; }
 
 .checkbox-uuid > * {
   vertical-align: middle; }
@@ -1177,8 +1169,6 @@ ul {
     #quick-watch-processor-type ul li {
       list-style: none;
       font-size: 0.8rem; }
-      #quick-watch-processor-type ul li > * {
-        display: inline-block; }
 
 .restock-label {
   padding: 3px;
@@ -1190,13 +1180,3 @@ ul {
   .restock-label.not-in-stock {
     background-color: var(--color-background-button-cancel);
     color: #777; }
-
-#chrome-extension-link {
-  padding: 9px;
-  border: 1px solid var(--color-grey-800);
-  border-radius: 10px;
-  vertical-align: middle; }
-  #chrome-extension-link img {
-    height: 21px;
-    padding: 2px;
-    vertical-align: middle; }

changedetectionio/store.py

@@ -1,4 +1,4 @@
-from changedetectionio.strtobool import strtobool
+from distutils.util import strtobool
 
 from flask import (
     flash
@@ -124,12 +124,12 @@ class ChangeDetectionStore:
                 self.__data['app_guid'] = str(uuid_builder.uuid4())
 
         # Generate the URL access token for RSS feeds
-        if not self.__data['settings']['application'].get('rss_access_token'):
+        if not 'rss_access_token' in self.__data['settings']['application']:
             secret = secrets.token_hex(16)
             self.__data['settings']['application']['rss_access_token'] = secret
 
         # Generate the API access token
-        if not self.__data['settings']['application'].get('api_access_token'):
+        if not 'api_access_token' in self.__data['settings']['application']:
             secret = secrets.token_hex(16)
             self.__data['settings']['application']['api_access_token'] = secret
 
@@ -163,6 +163,7 @@ class ChangeDetectionStore:
                         del (update_obj[dict_key])
 
             self.__data['watching'][uuid].update(update_obj)
+
         self.needs_write = True
 
     @property
@@ -177,7 +178,7 @@ class ChangeDetectionStore:
     @property
     def has_unviewed(self):
         for uuid, watch in self.__data['watching'].items():
-            if watch.history_n >= 2 and watch.viewed == False:
+            if watch.viewed == False:
                 return True
         return False
 
@@ -242,14 +243,6 @@ class ChangeDetectionStore:
     def clear_watch_history(self, uuid):
         import pathlib
 
-        # JSON Data, Screenshots, Textfiles (history index and snapshots), HTML in the future etc
-        for item in pathlib.Path(os.path.join(self.datastore_path, uuid)).rglob("*.*"):
-            unlink(item)
-
-        # Force the attr to recalculate
-        bump = self.__data['watching'][uuid].history
-
-        # Do this last because it will trigger a recheck due to last_checked being zero
         self.__data['watching'][uuid].update({
                 'browser_steps_last_error_step' : None,
                 'check_count': 0,
@@ -266,6 +259,13 @@ class ChangeDetectionStore:
                 'track_ldjson_price_data': None,
             })
 
+        # JSON Data, Screenshots, Textfiles (history index and snapshots), HTML in the future etc
+        for item in pathlib.Path(os.path.join(self.datastore_path, uuid)).rglob("*.*"):
+            unlink(item)
+
+        # Force the attr to recalculate
+        bump = self.__data['watching'][uuid].history
+
         self.needs_write_urgent = True
 
     def add_watch(self, url, tag='', extras=None, tag_uuids=None, write_to_disk_now=True):
@@ -376,6 +376,46 @@ class ChangeDetectionStore:
 
         return False
 
+    # Save as PNG, PNG is larger but better for doing visual diff in the future
+    def save_screenshot(self, watch_uuid, screenshot: bytes, as_error=False):
+        if not self.data['watching'].get(watch_uuid):
+            return
+
+        if as_error:
+            target_path = os.path.join(self.datastore_path, watch_uuid, "last-error-screenshot.png")
+        else:
+            target_path = os.path.join(self.datastore_path, watch_uuid, "last-screenshot.png")
+
+        self.data['watching'][watch_uuid].ensure_data_dir_exists()
+
+        with open(target_path, 'wb') as f:
+            f.write(screenshot)
+            f.close()
+
+
+    def save_error_text(self, watch_uuid, contents):
+        if not self.data['watching'].get(watch_uuid):
+            return
+
+        self.data['watching'][watch_uuid].ensure_data_dir_exists()
+        target_path = os.path.join(self.datastore_path, watch_uuid, "last-error.txt")
+        with open(target_path, 'w') as f:
+            f.write(contents)
+
+    def save_xpath_data(self, watch_uuid, data, as_error=False):
+
+        if not self.data['watching'].get(watch_uuid):
+            return
+        if as_error:
+            target_path = os.path.join(self.datastore_path, watch_uuid, "elements-error.json")
+        else:
+            target_path = os.path.join(self.datastore_path, watch_uuid, "elements.json")
+        self.data['watching'][watch_uuid].ensure_data_dir_exists()
+        with open(target_path, 'w') as f:
+            f.write(json.dumps(data))
+            f.close()
+
+
     def sync_to_json(self):
         logger.info("Saving JSON..")
         try:
@@ -514,6 +554,7 @@ class ChangeDetectionStore:
         return os.path.isfile(filepath)
 
     def get_all_base_headers(self):
+        from .model.App import parse_headers_from_text_file
         headers = {}
         # Global app settings
         headers.update(self.data['settings'].get('headers', {}))
@@ -616,10 +657,7 @@ class ChangeDetectionStore:
         return res
 
     def tag_exists_by_name(self, tag_name):
-        # Check if any tag dictionary has a 'title' attribute matching the provided tag_name
-        tags = self.__data['settings']['application']['tags'].values()
-        return next((v for v in tags if v.get('title', '').lower() == tag_name.lower()),
-                    None)
+        return any(v.get('title', '').lower() == tag_name.lower() for k, v in self.__data['settings']['application']['tags'].items())
 
     def get_updates_available(self):
         import inspect
@@ -831,21 +869,3 @@ class ChangeDetectionStore:
                         self.__data["watching"][awatch]['include_filters'][num] = 'xpath1:' + selector
                     if selector.startswith('xpath:'):
                         self.__data["watching"][awatch]['include_filters'][num] = selector.replace('xpath:', 'xpath1:', 1)
-
-    # Use more obvious default time setting
-    def update_15(self):
-        for uuid in self.__data["watching"]:
-            if self.__data["watching"][uuid]['time_between_check'] == self.__data['settings']['requests']['time_between_check']:
-                # What the old logic was, which was pretty confusing
-                self.__data["watching"][uuid]['time_between_check_use_default'] = True
-            elif all(value is None or value == 0 for value in self.__data["watching"][uuid]['time_between_check'].values()):
-                self.__data["watching"][uuid]['time_between_check_use_default'] = True
-            else:
-                # Something custom here
-                self.__data["watching"][uuid]['time_between_check_use_default'] = False
-
-    # Correctly set datatype for older installs where 'tag' was string and update_12 did not catch it
-    def update_16(self):
-        for uuid, watch in self.data['watching'].items():
-            if isinstance(watch.get('tags'), str):
-                self.data['watching'][uuid]['tags'] = []

changedetectionio/strtobool.py

@@ -1,23 +0,0 @@
-# Because strtobool was removed in python 3.12 distutils
-
-_MAP = {
-    'y': True,
-    'yes': True,
-    't': True,
-    'true': True,
-    'on': True,
-    '1': True,
-    'n': False,
-    'no': False,
-    'f': False,
-    'false': False,
-    'off': False,
-    '0': False
-}
-
-
-def strtobool(value):
-    try:
-        return _MAP[str(value).lower()]
-    except KeyError:
-        raise ValueError('"{}" is not a valid bool value'.format(value))

changedetectionio/templates/IMPORTANT.md

@@ -1,6 +0,0 @@
-# Important notes about templates
-
-Template names should always end in ".html", ".htm", ".xml", ".xhtml", ".svg", even the `import`'ed templates.
-
-Jinja2's `def select_jinja_autoescape(self, filename: str) -> bool:` will check the filename extension and enable autoescaping
-

changedetectionio/templates/_common_fields.jinja

@@ -1,5 +1,5 @@
 
-{% from '_helpers.html' import render_field %}
+{% from '_helpers.jinja' import render_field %}
 
 {% macro render_common_settings_form(form, emailprefix, settings_application) %}
                         <div class="pure-control-group">

changedetectionio/templates/base.html

@@ -6,9 +6,7 @@
     <meta name="viewport" content="width=device-width, initial-scale=1.0" >
     <meta name="description" content="Self hosted website change detection." >
     <title>Change Detection{{extra_title}}</title>
-    {% if app_rss_token %}
-      <link rel="alternate" type="application/rss+xml" title="Changedetection.io » Feed{% if active_tag_uuid %}- {{active_tag.title}}{% endif %}" href="{{ url_for('rss', tag=active_tag_uuid , token=app_rss_token)}}" >
-    {% endif %}
+    <link rel="alternate" type="application/rss+xml" title="Changedetection.io » Feed{% if active_tag %}- {{active_tag}}{% endif %}" href="{{ url_for('rss', tag=active_tag , token=app_rss_token)}}" >
     <link rel="stylesheet" href="{{url_for('static_content', group='styles', filename='pure-min.css')}}" >
     <link rel="stylesheet" href="{{url_for('static_content', group='styles', filename='styles.css')}}?v={{ get_css_version() }}" >
     {% if extra_stylesheets %}
@@ -26,11 +24,13 @@
     <meta name="msapplication-TileColor" content="#da532c">
     <meta name="msapplication-config" content="favicons/browserconfig.xml">
     <meta name="theme-color" content="#ffffff">
-    <script>
-        const csrftoken="{{ csrf_token() }}";
-    </script>
+
+    <style>
+      body::before {
+        background-image: url({{url_for('static_content', group='images', filename='gradient-border.png') }});
+      }
+    </style>
     <script src="{{url_for('static_content', group='js', filename='jquery-3.6.0.min.js')}}"></script>
-    <script src="{{url_for('static_content', group='js', filename='csrf.js')}}" defer></script>
   </head>
 
   <body>
@@ -89,8 +89,8 @@
           <li class="pure-menu-item pure-form" id="search-menu-item">
             <!-- We use GET here so it offers people a chance to set bookmarks etc -->
             <form name="searchForm" action="" method="GET">
-              <input id="search-q" class="" name="q" placeholder="URL or Title {% if active_tag_uuid %}in '{{ active_tag.title }}'{% endif %}" required="" type="text" value="">
-              <input name="tags" type="hidden" value="{% if active_tag_uuid %}{{active_tag_uuid}}{% endif %}">
+              <input id="search-q" class="" name="q" placeholder="URL or Title {% if active_tag %}in '{{ active_tag }}'{% endif %}" required="" type="text" value="">
+              <input name="tags" type="hidden" value="{% if active_tag %}{{active_tag}}{% endif %}">
               <button class="toggle-button " id="toggle-search" type="button" title="Search, or Use Alt+S Key" >
                 {% include "svgs/search-icon.svg" %}
               </button>
@@ -147,19 +147,7 @@
     <section class="content">
         <div id="overlay">
             <div class="content">
-                <h4>Try our Chrome extension</h4>
-                <p>
-                    <a id="chrome-extension-link"
-                       title="Try our new Chrome Extension!"
-                       href="https://chromewebstore.google.com/detail/changedetectionio-website/kefcfmgmlhmankjmnbijimhofdjekbop">
-                        <img src="{{url_for('static_content', group='images', filename='Google-Chrome-icon.png')}}">
-                        Chrome Webstore
-                    </a>
-                </p>
-
-                Easily add the current web-page from your browser directly into your changedetection.io tool, more great features coming soon!
-
-                <h4>Changedetection.io needs your support!</h4>
+                <strong>changedetection.io needs your support!</strong><br>
                 <p>
                     You can help us by supporting changedetection.io on these platforms;
                 </p>

changedetectionio/templates/diff.html

@@ -1,5 +1,5 @@
 {% extends 'base.html' %}
-{% from '_helpers.html' import render_field, render_checkbox_field, render_button %}
+{% from '_helpers.jinja' import render_field, render_checkbox_field, render_button %}
 {% block content %}
 <script>
     const screenshot_url="{{url_for('static_content', group='screenshot', filename=uuid)}}";

changedetectionio/templates/edit.html

@@ -1,14 +1,13 @@
 {% extends 'base.html' %}
 {% block content %}
-{% from '_helpers.html' import render_field, render_checkbox_field, render_button %}
-{% from '_common_fields.html' import render_common_settings_form %}
+{% from '_helpers.jinja' import render_field, render_checkbox_field, render_button %}
+{% from '_common_fields.jinja' import render_common_settings_form %}
 <script src="{{url_for('static_content', group='js', filename='tabs.js')}}" defer></script>
 <script src="{{url_for('static_content', group='js', filename='vis.js')}}" defer></script>
 <script>
     const browser_steps_available_screenshots=JSON.parse('{{ watch.get_browsersteps_available_screenshots|tojson }}');
     const browser_steps_config=JSON.parse('{{ browser_steps_config|tojson }}');
-    <!-- Should be _external so that firefox and others load it more reliably -->
-    const browser_steps_fetch_screenshot_image_url="{{url_for('browser_steps.browser_steps_fetch_screenshot_image', uuid=uuid, _external=True)}}";
+    const browser_steps_fetch_screenshot_image_url="{{url_for('browser_steps.browser_steps_fetch_screenshot_image', uuid=uuid)}}";
     const browser_steps_last_error_step={{ watch.browser_steps_last_error_step|tojson }};
     const browser_steps_start_url="{{url_for('browser_steps.browsersteps_start_session', uuid=uuid)}}";
     const browser_steps_sync_url="{{url_for('browser_steps.browsersteps_ui_update', uuid=uuid)}}";
@@ -32,7 +31,6 @@
 <script src="{{url_for('static_content', group='js', filename='browser-steps.js')}}" defer></script>
 {% endif %}
 
-{% set has_tag_filters_extra="WARNING: Watch has tag/groups set with special filters\n" if has_special_tag_options else '' %}
 <script src="{{url_for('static_content', group='js', filename='recheck-proxy.js')}}" defer></script>
 
 <div class="edit-form monospaced-textarea">
@@ -87,9 +85,15 @@
                         {{ render_field(form.tags) }}
                         <span class="pure-form-message-inline">Organisational tag/group name used in the main listing page</span>
                     </div>
-                    <div class="pure-control-group time-between-check border-fieldset">
+                    <div class="pure-control-group">
                         {{ render_field(form.time_between_check, class="time-check-widget") }}
-                        {{ render_checkbox_field(form.time_between_check_use_default, class="use-default-timecheck") }}
+                        {% if has_empty_checktime %}
+                        <span class="pure-form-message-inline">Currently using the <a
+                                href="{{ url_for('settings_page', uuid=uuid) }}">default global settings</a>, change to another value if you want to be specific.</span>
+                        {% else %}
+                        <span class="pure-form-message-inline">Set to blank to use the <a
+                                href="{{ url_for('settings_page', uuid=uuid) }}">default global settings</a>.</span>
+                        {% endif %}
                     </div>
                     <div class="pure-control-group">
                         {{ render_checkbox_field(form.extract_title_as_title) }}
@@ -276,7 +280,7 @@ User-Agent: wonderbra 1.0") }}
                     <div class="pure-control-group">
                         {% set field = render_field(form.include_filters,
                             rows=5,
-                            placeholder=has_tag_filters_extra+"#example
+                            placeholder="#example
 xpath://body/div/span[contains(@class, 'example-class')]",
                             class="m-d")
                         %}
@@ -292,7 +296,7 @@ xpath://body/div/span[contains(@class, 'example-class')]",
                             <ul>
                                 <li>JSONPath: Prefix with <code>json:</code>, use <code>json:$</code> to force re-formatting if required,  <a href="https://jsonpath.com/" target="new">test your JSONPath here</a>.</li>
                                 {% if jq_support %}
-                                <li>jq: Prefix with <code>jq:</code> and <a href="https://jqplay.org/" target="new">test your jq here</a>. Using <a href="https://stedolan.github.io/jq/" target="new">jq</a> allows for complex filtering and processing of JSON data with built-in functions, regex, filtering, and more. See examples and documentation <a href="https://stedolan.github.io/jq/manual/" target="new">here</a>. Prefix <code>jqraw:</code> outputs the results as text instead of a JSON list.</li>
+                                <li>jq: Prefix with <code>jq:</code> and <a href="https://jqplay.org/" target="new">test your jq here</a>. Using <a href="https://stedolan.github.io/jq/" target="new">jq</a> allows for complex filtering and processing of JSON data with built-in functions, regex, filtering, and more. See examples and documentation <a href="https://stedolan.github.io/jq/manual/" target="new">here</a>.</li>
                                 {% else %}
                                 <li>jq support not installed</li>
                                 {% endif %}
@@ -312,19 +316,18 @@ xpath://body/div/span[contains(@class, 'example-class')]",
                 </span>
                     </div>
                 <fieldset class="pure-control-group">
-                    {{ render_field(form.subtractive_selectors, rows=5, placeholder=has_tag_filters_extra+"header
+                    {{ render_field(form.subtractive_selectors, rows=5, placeholder="header
 footer
 nav
 .stockticker") }}
                     <span class="pure-form-message-inline">
                         <ul>
                           <li> Remove HTML element(s) by CSS selector before text conversion. </li>
-                          <li> Don't paste HTML here, use only CSS selectors </li>
                           <li> Add multiple elements or CSS selectors per line to ignore multiple parts of the HTML. </li>
                         </ul>
                       </span>
                 </fieldset>
-                <div class="text-filtering border-fieldset">
+                <div class="text-filtering">
                 <fieldset class="pure-group" id="text-filtering-type-options">
                     <h3>Text filtering</h3>
                         Limit trigger/ignore/block/extract to;<br>
@@ -432,8 +435,8 @@ Unavailable") }}
                 <fieldset>
                     <div class="pure-control-group">
                         {% if visualselector_enabled %}
-                            <span class="pure-form-message-inline" id="visual-selector-heading">
-                                The Visual Selector tool lets you select the <i>text</i> elements that will be used for the change detection. It automatically fills-in the filters in the "CSS/JSONPath/JQ/XPath Filters" box of the <a href="#filters-and-triggers">Filters & Triggers</a> tab. Use <strong>Shift+Click</strong> to select multiple items.
+                            <span class="pure-form-message-inline">
+                                The Visual Selector tool lets you select the <i>text</i> elements that will be used for the change detection &dash; after the <i>Browser Steps</i> has completed.<br><br>
                             </span>
 
                             <div id="selector-header">
@@ -481,17 +484,13 @@ Unavailable") }}
                             <td>{{ "{:,}".format(watch.history|length) }}</td>
                         </tr>
                         <tr>
-                            <td>Last fetch duration</td>
+                            <td>Last fetch time</td>
                             <td>{{ watch.fetch_time }}s</td>
                         </tr>
                         <tr>
                             <td>Notification alert count</td>
                             <td>{{ watch.notification_alert_count }}</td>
                         </tr>
-                        <tr>
-                            <td>Server type reply</td>
-                            <td>{{ watch.get('remote_server_reply') }}</td>
-                        </tr>
                         </tbody>
                     </table>
                 </div>

changedetectionio/templates/import.html

@@ -1,6 +1,6 @@
 {% extends 'base.html' %}
 {% block content %}
-{% from '_helpers.html' import render_field %}
+{% from '_helpers.jinja' import render_field %}
 <script src="{{url_for('static_content', group='js', filename='tabs.js')}}" defer></script>
 <div class="edit-form monospaced-textarea">
 
@@ -107,7 +107,7 @@
                                     <option value="" style="color: #aaa"> -- none --</option>
                                     <option value="url">URL</option>
                                     <option value="title">Title</option>
-                                    <option value="include_filters">CSS/xPath filter</option>
+                                    <option value="include_filter">CSS/xPath filter</option>
                                     <option value="tag">Group / Tag name(s)</option>
                                     <option value="interval_minutes">Recheck time (minutes)</option>
                                 </select></td>

changedetectionio/templates/preview.html

@@ -1,103 +1,72 @@
 {% extends 'base.html' %}
 
 {% block content %}
-    <script>
-        const screenshot_url = "{{url_for('static_content', group='screenshot', filename=uuid)}}";
-        {% if last_error_screenshot %}
-            const error_screenshot_url = "{{url_for('static_content', group='screenshot', filename=uuid, error_screenshot=1) }}";
-        {% endif %}
-        const highlight_submit_ignore_url = "{{url_for('highlight_submit_ignore_url', uuid=uuid)}}";
-    </script>
-    <script src="{{ url_for('static_content', group='js', filename='diff-overview.js') }}" defer></script>
-    <script src="{{ url_for('static_content', group='js', filename='preview.js') }}" defer></script>
-    <script src="{{ url_for('static_content', group='js', filename='tabs.js') }}" defer></script>
-    {% if versions|length >= 2 %}
-        <div id="settings" style="text-align: center;">
-            <form class="pure-form " action="" method="POST">
-                <fieldset>
-                    <label for="preview-version">Select timestamp</label> <select id="preview-version"
-                                                                                 name="from_version"
-                                                                                 class="needs-localtime">
-                    {% for version in versions|reverse %}
-                        <option value="{{ version }}" {% if version == current_version %} selected="" {% endif %}>
-                            {{ version }}
-                        </option>
-                    {% endfor %}
-                </select>
-                    <button type="submit" class="pure-button pure-button-primary">Go</button>
-
-                </fieldset>
-            </form>
-                <br>
-                <strong>Keyboard: </strong><a href="" class="pure-button pure-button-primary" id="btn-previous">
-                &larr; Previous</a> &nbsp; <a class="pure-button pure-button-primary" id="btn-next" href="">
-                &rarr; Next</a>
-        </div>
+<script>
+    const screenshot_url="{{url_for('static_content', group='screenshot', filename=uuid)}}";
+    {% if last_error_screenshot %}
+    const error_screenshot_url="{{url_for('static_content', group='screenshot', filename=uuid, error_screenshot=1) }}";
     {% endif %}
+    const highlight_submit_ignore_url="{{url_for('highlight_submit_ignore_url', uuid=uuid)}}";
+</script>
+<script src="{{url_for('static_content', group='js', filename='diff-overview.js')}}" defer></script>
 
-    <div class="tabs">
-        <ul>
-            {% if last_error_text %}
-                <li class="tab" id="error-text-tab"><a href="#error-text">Error Text</a></li> {% endif %}
-            {% if last_error_screenshot %}
-                <li class="tab" id="error-screenshot-tab"><a href="#error-screenshot">Error Screenshot</a>
-                </li> {% endif %}
-            {% if history_n > 0 %}
-                <li class="tab" id="text-tab"><a href="#text">Text</a></li>
-                <li class="tab" id="screenshot-tab"><a href="#screenshot">Screenshot</a></li>
-            {% endif %}
-        </ul>
-    </div>
-
-
-    <div id="diff-ui">
-        <div class="tab-pane-inner" id="error-text">
-            <div class="snapshot-age error">{{ watch.error_text_ctime|format_seconds_ago }} seconds ago</div>
-            <pre>
+<script src="{{url_for('static_content', group='js', filename='tabs.js')}}" defer></script>
+<div class="tabs">
+    <ul>
+        {% if last_error_text %}<li class="tab" id="error-text-tab"><a href="#error-text">Error Text</a></li> {% endif %}
+        {% if last_error_screenshot %}<li class="tab" id="error-screenshot-tab"><a href="#error-screenshot">Error Screenshot</a></li> {% endif %}
+        {% if history_n > 0 %}
+        <li class="tab" id="text-tab"><a href="#text">Text</a></li>
+        <li class="tab" id="screenshot-tab"><a href="#screenshot">Screenshot</a></li>
+        {% endif %}
+    </ul>
+</div>
+<form><input type="hidden" name="csrf_token" value="{{ csrf_token() }}"></form>
+<div id="diff-ui">
+    <div class="tab-pane-inner" id="error-text">
+        <div class="snapshot-age error">{{watch.error_text_ctime|format_seconds_ago}} seconds ago</div>
+        <pre>
             {{ last_error_text }}
         </pre>
-        </div>
-
-        <div class="tab-pane-inner" id="error-screenshot">
-            <div class="snapshot-age error">{{ watch.snapshot_error_screenshot_ctime|format_seconds_ago }} seconds ago
-            </div>
-            <img id="error-screenshot-img" style="max-width: 80%"
-                 alt="Current erroring screenshot from most recent request">
-        </div>
-
-        <div class="tab-pane-inner" id="text">
-            <div class="snapshot-age">{{ current_version|format_timestamp_timeago }}</div>
-            <span class="ignored">Grey lines are ignored</span> <span class="triggered">Blue lines are triggers</span>
-            <span class="tip"><strong>Pro-tip</strong>: Highlight text to add to ignore filters</span>
-
-            <table>
-                <tbody>
-                <tr>
-                    <td id="diff-col" class="highlightable-filter">
-                        {% for row in content %}
-                            <div class="{{ row.classes }}">{{ row.line }}</div>
-                        {% endfor %}
-                    </td>
-                </tr>
-                </tbody>
-            </table>
-        </div>
-
-        <div class="tab-pane-inner" id="screenshot">
-            <div class="tip">
-                For now, Differences are performed on text, not graphically, only the latest screenshot is available.
-            </div>
-            <br>
-            {% if is_html_webdriver %}
-                {% if screenshot %}
-                    <div class="snapshot-age">{{ watch.snapshot_screenshot_ctime|format_timestamp_timeago }}</div>
-                    <img style="max-width: 80%" id="screenshot-img" alt="Current screenshot from most recent request">
-                {% else %}
-                    No screenshot available just yet! Try rechecking the page.
-                {% endif %}
-            {% else %}
-                <strong>Screenshot requires Playwright/WebDriver enabled</strong>
-            {% endif %}
-        </div>
     </div>
+
+    <div class="tab-pane-inner" id="error-screenshot">
+        <div class="snapshot-age error">{{watch.snapshot_error_screenshot_ctime|format_seconds_ago}} seconds ago</div>
+        <img id="error-screenshot-img"  style="max-width: 80%" alt="Current erroring screenshot from most recent request" >
+    </div>
+
+    <div class="tab-pane-inner" id="text">
+        <div class="snapshot-age">{{watch.snapshot_text_ctime|format_timestamp_timeago}}</div>
+        <span class="ignored">Grey lines are ignored</span> <span class="triggered">Blue lines are triggers</span> <span class="tip"><strong>Pro-tip</strong>: Highlight text to add to ignore filters</span>
+
+        <table>
+            <tbody>
+            <tr>
+                <td id="diff-col" class="highlightable-filter">
+                    {% for row in content %}
+                    <div class="{{row.classes}}">{{row.line}}</div>
+                    {% endfor %}
+                </td>
+            </tr>
+            </tbody>
+        </table>
+    </div>
+
+     <div class="tab-pane-inner" id="screenshot">
+         <div class="tip">
+             For now, Differences are performed on text, not graphically, only the latest screenshot is available.
+         </div>
+         <br>
+         {% if is_html_webdriver %}
+           {% if screenshot %}
+             <div class="snapshot-age">{{watch.snapshot_screenshot_ctime|format_timestamp_timeago}}</div>
+             <img style="max-width: 80%" id="screenshot-img" alt="Current screenshot from most recent request" >
+           {% else %}
+              No screenshot available just yet! Try rechecking the page.
+           {% endif %}
+         {% else %}
+           <strong>Screenshot requires Playwright/WebDriver enabled</strong>
+         {% endif %}
+     </div>
+</div>
 {% endblock %}

changedetectionio/templates/settings.html

@@ -1,10 +1,10 @@
 {% extends 'base.html' %}
 
 {% block content %}
-{% from '_helpers.html' import render_field, render_checkbox_field, render_button %}
-{% from '_common_fields.html' import render_common_settings_form %}
+{% from '_helpers.jinja' import render_field, render_checkbox_field, render_button %}
+{% from '_common_fields.jinja' import render_common_settings_form %}
 <script>
-    const notification_base_url="{{url_for('ajax_callback_send_notification_test', mode="global-settings")}}";
+    const notification_base_url="{{url_for('ajax_callback_send_notification_test', watch_uuid=uuid)}}";
 {% if emailprefix %}
     const email_notification_prefix=JSON.parse('{{emailprefix|tojson}}');
 {% endif %}
@@ -31,7 +31,7 @@
                 <fieldset>
                     <div class="pure-control-group">
                         {{ render_field(form.requests.form.time_between_check, class="time-check-widget") }}
-                        <span class="pure-form-message-inline">Default recheck time for all watches, current system minimum is <i>{{min_system_recheck_seconds}}</i> seconds (<a href="https://github.com/dgtlmoon/changedetection.io/wiki/Misc-system-settings#enviroment-variables">more info</a>).</span>
+                        <span class="pure-form-message-inline">Default time for all watches, when the watch does not have a specific time setting.</span>
                     </div>
                     <div class="pure-control-group">
                         {{ render_field(form.requests.form.jitter_seconds, class="jitter_seconds") }}
@@ -62,9 +62,6 @@
                         <span class="pure-form-message-inline">Allow access to view watch diff page when password is enabled (Good for sharing the diff page)
                         </span>
                     </div>
-                    <div class="pure-control-group">
-                        {{ render_checkbox_field(form.application.form.rss_hide_muted_watches) }}
-                    </div>
                     <div class="pure-control-group">
                         {{ render_field(form.application.form.pager_size) }}
                         <span class="pure-form-message-inline">Number of items per page in the watch overview list, 0 to disable.</span>
@@ -111,6 +108,8 @@
                         <p>Use the <strong>Basic</strong> method (default) where your watched sites don't need Javascript to render.</p>
                         <p>The <strong>Chrome/Javascript</strong> method requires a network connection to a running WebDriver+Chrome server, set by the ENV var 'WEBDRIVER_URL'. </p>
                     </span>
+                    <br>
+                    Tip: <a href="https://github.com/dgtlmoon/changedetection.io/wiki/Proxy-configuration#brightdata-proxy-support">Connect using Bright Data and Oxylabs Proxies, find out more here.</a>
                 </div>
                 <fieldset class="pure-group" id="webdriver-override-options" data-visible-for="application-fetch_backend=html_webdriver">
                     <div class="pure-form-message-inline">
@@ -122,18 +121,6 @@
                         {{ render_field(form.application.form.webdriver_delay) }}
                     </div>
                 </fieldset>
-                <div class="pure-control-group inline-radio">
-                    {{ render_field(form.requests.form.default_ua) }}
-                    <span class="pure-form-message-inline">
-                        Applied to all requests.<br><br>
-                        Note: Simply changing the User-Agent often does not defeat anti-robot technologies, it's important to consider <a href="https://changedetection.io/tutorial/what-are-main-types-anti-robot-mechanisms">all of the ways that the browser is detected</a>.
-                    </span>
-                </div>
-                <div class="pure-control-group">
-                                        <br>
-                    Tip: <a href="https://github.com/dgtlmoon/changedetection.io/wiki/Proxy-configuration#brightdata-proxy-support">Connect using Bright Data and Oxylabs Proxies, find out more here.</a>
-
-                </div>
             </div>
 
             <div class="tab-pane-inner" id="filters">
@@ -181,12 +168,12 @@ nav
            </div>
 
             <div class="tab-pane-inner" id="api">
-                <h4>API Access</h4>
+
                 <p>Drive your changedetection.io via API, More about <a href="https://github.com/dgtlmoon/changedetection.io/wiki/API-Reference">API access here</a></p>
 
                 <div class="pure-control-group">
                     {{ render_checkbox_field(form.application.form.api_access_token_enabled) }}
-                    <div class="pure-form-message-inline">Restrict API access limit by using <code>x-api-key</code> header - required for the Chrome Extension to work</div><br>
+                    <div class="pure-form-message-inline">Restrict API access limit by using <code>x-api-key</code> header</div><br>
                     <div class="pure-form-message-inline"><br>API Key <span id="api-key">{{api_key}}</span>
                         <span style="display:none;" id="api-key-copy" >copy</span>
                     </div>
@@ -194,20 +181,6 @@ nav
                 <div class="pure-control-group">
                     <a href="{{url_for('settings_reset_api_key')}}" class="pure-button button-small button-cancel">Regenerate API key</a>
                 </div>
-                <div class="pure-control-group">
-                    <h4>Chrome Extension</h4>
-                    <p>Easily add any web-page to your changedetection.io installation from within Chrome.</p>
-                    <strong>Step 1</strong> Install the extension, <strong>Step 2</strong> Navigate to this page,
-                    <strong>Step 3</strong> Open the extension from the toolbar and click "<i>Sync API Access</i>"
-                    <p>
-                        <a id="chrome-extension-link"
-                           title="Try our new Chrome Extension!"
-                           href="https://chromewebstore.google.com/detail/changedetectionio-website/kefcfmgmlhmankjmnbijimhofdjekbop">
-                            <img src="{{ url_for('static_content', group='images', filename='Google-Chrome-icon.png') }}" alt="Chrome">
-                            Chrome Webstore
-                        </a>
-                    </p>
-                </div>
             </div>
             <div class="tab-pane-inner" id="proxies">
                 <div id="recommended-proxy">

changedetectionio/templates/watch-overview.html

@@ -1,6 +1,6 @@
 {% extends 'base.html' %}
 {% block content %}
-{% from '_helpers.html' import render_simple_field, render_field, render_nolabel_field, sort_by_title %}
+{% from '_helpers.jinja' import render_simple_field, render_field, render_nolabel_field %}
 <script src="{{url_for('static_content', group='js', filename='jquery-3.6.0.min.js')}}"></script>
 <script src="{{url_for('static_content', group='js', filename='watch-overview.js')}}" defer></script>
 
@@ -13,7 +13,7 @@
             <div id="watch-add-wrapper-zone">
 
                     {{ render_nolabel_field(form.url, placeholder="https://...", required=true) }}
-                    {{ render_nolabel_field(form.tags, value=active_tag.title if active_tag_uuid else '', placeholder="watch label / tag") }}
+                    {{ render_nolabel_field(form.tags, value=tags[active_tag].title if active_tag else '', placeholder="watch label / tag") }}
                     {{ render_nolabel_field(form.watch_submit_button, title="Watch this URL!" ) }}
                     {{ render_nolabel_field(form.edit_and_watch_submit_button, title="Edit first then Watch") }}
             </div>
@@ -37,7 +37,6 @@
         <button class="pure-button button-secondary button-xsmall" name="op" value="assign-tag" id="checkbox-assign-tag">Tag</button>
         <button class="pure-button button-secondary button-xsmall" name="op" value="mark-viewed">Mark viewed</button>
         <button class="pure-button button-secondary button-xsmall" name="op" value="notification-default">Use default notification</button>
-        <button class="pure-button button-secondary button-xsmall" name="op" value="clear-errors">Clear errors</button>
         <button class="pure-button button-secondary button-xsmall" style="background: #dd4242;" name="op" value="clear-history">Clear/reset history</button>
         <button class="pure-button button-secondary button-xsmall" style="background: #dd4242;" name="op" value="delete">Delete</button>
     </div>
@@ -46,14 +45,12 @@
     {% endif %}
     {% if search_q %}<div id="search-result-info">Searching "<strong><i>{{search_q}}</i></strong>"</div>{% endif %}
     <div>
-        <a href="{{url_for('index')}}" class="pure-button button-tag {{'active' if not active_tag_uuid }}">All</a>
-
-    <!-- tag list -->
-    {% for uuid, tag in tags %}
-        {% if tag != "" %}
-            <a href="{{url_for('index', tag=uuid) }}" class="pure-button button-tag {{'active' if active_tag_uuid == uuid }}">{{ tag.title }}</a>
-        {% endif %}
-    {% endfor %}
+        <a href="{{url_for('index')}}" class="pure-button button-tag {{'active' if not active_tag }}">All</a>
+        {% for uuid, tag in tags.items() %}
+            {% if tag != "" %}
+                <a href="{{url_for('index', tag=uuid) }}" class="pure-button button-tag {{'active' if active_tag == uuid }}">{{ tag.title }}</a>
+            {% endif %}
+        {% endfor %}
     </div>
 
     {% set sort_order = sort_order or 'asc' %}
@@ -67,18 +64,18 @@
             <tr>
                 {% set link_order = "desc" if sort_order  == 'asc' else "asc" %}
                 {% set arrow_span = "" %}
-                <th><input style="vertical-align: middle" type="checkbox" id="check-all" > <a class="{{ 'active '+link_order if sort_attribute == 'date_created' else 'inactive' }}"  href="{{url_for('index', sort='date_created', order=link_order, tag=active_tag_uuid)}}"># <span class='arrow {{link_order}}'></span></a></th>
-                <th class="empty-cell"></th>
-                <th><a class="{{ 'active '+link_order if sort_attribute == 'label' else 'inactive' }}" href="{{url_for('index', sort='label', order=link_order, tag=active_tag_uuid)}}">Website <span class='arrow {{link_order}}'></span></a></th>
-                <th><a class="{{ 'active '+link_order if sort_attribute == 'last_checked' else 'inactive' }}" href="{{url_for('index', sort='last_checked', order=link_order, tag=active_tag_uuid)}}">Last Checked <span class='arrow {{link_order}}'></span></a></th>
-                <th><a class="{{ 'active '+link_order if sort_attribute == 'last_changed' else 'inactive' }}" href="{{url_for('index', sort='last_changed', order=link_order, tag=active_tag_uuid)}}">Last Changed <span class='arrow {{link_order}}'></span></a></th>
-                <th class="empty-cell"></th>
+                <th><input style="vertical-align: middle" type="checkbox" id="check-all" > <a class="{{ 'active '+link_order if sort_attribute == 'date_created' else 'inactive' }}"  href="{{url_for('index', sort='date_created', order=link_order, tag=active_tag)}}"># <span class='arrow {{link_order}}'></span></a></th>
+                <th></th>
+                <th><a class="{{ 'active '+link_order if sort_attribute == 'label' else 'inactive' }}" href="{{url_for('index', sort='label', order=link_order, tag=active_tag)}}">Website <span class='arrow {{link_order}}'></span></a></th>
+                <th><a class="{{ 'active '+link_order if sort_attribute == 'last_checked' else 'inactive' }}" href="{{url_for('index', sort='last_checked', order=link_order, tag=active_tag)}}">Last Checked <span class='arrow {{link_order}}'></span></a></th>
+                <th><a class="{{ 'active '+link_order if sort_attribute == 'last_changed' else 'inactive' }}" href="{{url_for('index', sort='last_changed', order=link_order, tag=active_tag)}}">Last Changed <span class='arrow {{link_order}}'></span></a></th>
+                <th></th>
             </tr>
             </thead>
             <tbody>
             {% if not watches|length %}
             <tr>
-                <td colspan="6" style="text-wrap: wrap;">No website watches configured, please add a URL in the box above, or <a href="{{ url_for('import_page')}}" >import a list</a>.</td>
+                <td colspan="6">No website watches configured, please add a URL in the box above, or <a href="{{ url_for('import_page')}}" >import a list</a>.</td>
             </tr>
             {% endif %}
             {% for watch in (watches|sort(attribute=sort_attribute, reverse=sort_order == 'asc'))|pagination_slice(skip=pagination.skip) %}
@@ -95,11 +92,11 @@
                 <td class="inline checkbox-uuid" ><input name="uuids"  type="checkbox" value="{{ watch.uuid}} " > <span>{{ loop.index+pagination.skip }}</span></td>
                 <td class="inline watch-controls">
                     {% if not watch.paused %}
-                    <a class="state-off" href="{{url_for('index', op='pause', uuid=watch.uuid, tag=active_tag_uuid)}}"><img src="{{url_for('static_content', group='images', filename='pause.svg')}}" alt="Pause checks" title="Pause checks" class="icon icon-pause" ></a>
+                    <a class="state-off" href="{{url_for('index', op='pause', uuid=watch.uuid, tag=active_tag)}}"><img src="{{url_for('static_content', group='images', filename='pause.svg')}}" alt="Pause checks" title="Pause checks" class="icon icon-pause" ></a>
                     {% else %}
-                    <a class="state-on" href="{{url_for('index', op='pause', uuid=watch.uuid, tag=active_tag_uuid)}}"><img src="{{url_for('static_content', group='images', filename='play.svg')}}" alt="UnPause checks" title="UnPause checks" class="icon icon-unpause" ></a>
+                    <a class="state-on" href="{{url_for('index', op='pause', uuid=watch.uuid, tag=active_tag)}}"><img src="{{url_for('static_content', group='images', filename='play.svg')}}" alt="UnPause checks" title="UnPause checks" class="icon icon-unpause" ></a>
                     {% endif %}
-                    <a class="link-mute state-{{'on' if watch.notification_muted else 'off'}}" href="{{url_for('index', op='mute', uuid=watch.uuid, tag=active_tag_uuid)}}"><img src="{{url_for('static_content', group='images', filename='bell-off.svg')}}" alt="Mute notifications" title="Mute notifications" class="icon icon-mute" ></a>
+                    <a class="link-mute state-{{'on' if watch.notification_muted else 'off'}}" href="{{url_for('index', op='mute', uuid=watch.uuid, tag=active_tag)}}"><img src="{{url_for('static_content', group='images', filename='bell-off.svg')}}" alt="Mute notifications" title="Mute notifications" class="icon icon-mute" ></a>
                 </td>
                 <td class="title-col inline">{{watch.title if watch.title is not none and watch.title|length > 0 else watch.url}}
                     <a class="external" target="_blank" rel="noopener" href="{{ watch.link.replace('source:','') }}"></a>
@@ -169,7 +166,7 @@
                 <td>
                     <a {% if watch.uuid in queued_uuids %}disabled="true"{% endif %} href="{{ url_for('form_watch_checknow', uuid=watch.uuid, tag=request.args.get('tag')) }}"
                        class="recheck pure-button pure-button-primary">{% if watch.uuid in queued_uuids %}Queued{% else %}Recheck{% endif %}</a>
-                    <a href="{{ url_for('edit_page', uuid=watch.uuid)}}#general" class="pure-button pure-button-primary">Edit</a>
+                    <a href="{{ url_for('edit_page', uuid=watch.uuid)}}" class="pure-button pure-button-primary">Edit</a>
                     {% if watch.history_n >= 2 %}
 
                         {%  if is_unviewed %}
@@ -200,11 +197,11 @@
             </li>
             {% endif %}
             <li>
-               <a href="{{ url_for('form_watch_checknow', tag=active_tag_uuid, with_errors=request.args.get('with_errors',0)) }}" class="pure-button button-tag ">Recheck
-                all {% if active_tag_uuid %} in "{{active_tag.title}}"{%endif%}</a>
+               <a href="{{ url_for('form_watch_checknow', tag=active_tag, with_errors=request.args.get('with_errors',0)) }}" class="pure-button button-tag ">Recheck
+                all {% if active_tag%} in "{{tags[active_tag].title}}"{%endif%}</a>
             </li>
             <li>
-                <a href="{{ url_for('rss', tag=active_tag_uuid, token=app_rss_token)}}"><img alt="RSS Feed" id="feed-icon" src="{{url_for('static_content', group='images', filename='Generic_Feed-icon.svg')}}" height="15"></a>
+                <a href="{{ url_for('rss', tag=active_tag , token=app_rss_token)}}"><img alt="RSS Feed" id="feed-icon" src="{{url_for('static_content', group='images', filename='Generic_Feed-icon.svg')}}" height="15"></a>
             </li>
         </ul>
         {{ pagination.links }}

changedetectionio/tests/fetchers/test_custom_js_before_content.py

@@ -1,56 +0,0 @@
-import os
-from flask import url_for
-from ..util import live_server_setup, wait_for_all_checks, extract_UUID_from_client
-
-
-def test_execute_custom_js(client, live_server):
-
-    live_server_setup(live_server)
-    assert os.getenv('PLAYWRIGHT_DRIVER_URL'), "Needs PLAYWRIGHT_DRIVER_URL set for this test"
-
-    test_url = url_for('test_interactive_html_endpoint', _external=True)
-    test_url = test_url.replace('localhost.localdomain', 'cdio')
-    test_url = test_url.replace('localhost', 'cdio')
-
-    res = client.post(
-        url_for("form_quick_watch_add"),
-        data={"url": test_url, "tags": '', 'edit_and_watch_submit_button': 'Edit > Watch'},
-        follow_redirects=True
-    )
-
-    assert b"Watch added in Paused state, saving will unpause" in res.data
-
-    res = client.post(
-        url_for("edit_page", uuid="first", unpause_on_save=1),
-        data={
-            "url": test_url,
-            "tags": "",
-            'fetch_backend': "html_webdriver",
-            'webdriver_js_execute_code': 'document.querySelector("button[name=test-button]").click();',
-            'headers': "testheader: yes\buser-agent: MyCustomAgent",
-        },
-        follow_redirects=True
-    )
-    assert b"unpaused" in res.data
-    wait_for_all_checks(client)
-
-    uuid = extract_UUID_from_client(client)
-    assert live_server.app.config['DATASTORE'].data['watching'][uuid].history_n >= 1, "Watch history had atleast 1 (everything fetched OK)"
-
-    assert b"This text should be removed" not in res.data
-
-    # Check HTML conversion detected and workd
-    res = client.get(
-        url_for("preview_page", uuid=uuid),
-        follow_redirects=True
-    )
-    assert b"This text should be removed" not in res.data
-    assert b"I smell JavaScript because the button was pressed" in res.data
-
-    assert b"testheader: yes" in res.data
-    assert b"user-agent: mycustomagent" in res.data
-
-    client.get(
-        url_for("form_delete", uuid="all"),
-        follow_redirects=True
-    )

changedetectionio/tests/restock/test_restock.py

@@ -95,7 +95,7 @@ def test_restock_detection(client, live_server):
 
     # We should have a notification
     time.sleep(2)
-    assert os.path.isfile("test-datastore/notification.txt"), "Notification received"
+    assert os.path.isfile("test-datastore/notification.txt")
     os.unlink("test-datastore/notification.txt")
 
     # Default behaviour is to only fire notification when it goes OUT OF STOCK -> IN STOCK
@@ -103,9 +103,4 @@ def test_restock_detection(client, live_server):
     set_original_response()
     client.get(url_for("form_watch_checknow"), follow_redirects=True)
     wait_for_all_checks(client)
-    assert not os.path.isfile("test-datastore/notification.txt"), "No notification should have fired when it went OUT OF STOCK by default"
-
-    # BUT we should see that it correctly shows "not in stock"
-    res = client.get(url_for("index"))
-    assert b'not-in-stock' in res.data, "Correctly showing NOT IN STOCK in the list after it changed from IN STOCK"
-
+    assert not os.path.isfile("test-datastore/notification.txt")

changedetectionio/tests/smtp/smtp-test-server.py

@@ -1,51 +1,42 @@
 #!/usr/bin/python3
-import asyncio
-from aiosmtpd.controller import Controller
-from aiosmtpd.smtp import SMTP
+import smtpd
+import asyncore
 
 # Accept a SMTP message and offer a way to retrieve the last message via TCP Socket
 
 last_received_message = b"Nothing"
 
 
-class CustomSMTPHandler:
-    async def handle_DATA(self, server, session, envelope):
+class CustomSMTPServer(smtpd.SMTPServer):
+
+    def process_message(self, peer, mailfrom, rcpttos, data, **kwargs):
         global last_received_message
-        last_received_message = envelope.content
-        print('Receiving message from:', session.peer)
-        print('Message addressed from:', envelope.mail_from)
-        print('Message addressed to  :', envelope.rcpt_tos)
-        print('Message length        :', len(envelope.content))
-        print(envelope.content.decode('utf8'))
-        return '250 Message accepted for delivery'
+        last_received_message = data
+        print('Receiving message from:', peer)
+        print('Message addressed from:', mailfrom)
+        print('Message addressed to  :', rcpttos)
+        print('Message length        :', len(data))
+        print(data.decode('utf8'))
+        return
 
 
-class EchoServerProtocol(asyncio.Protocol):
-    def connection_made(self, transport):
+# Just print out the last message received on plain TCP socket server
+class EchoServer(asyncore.dispatcher):
+
+    def __init__(self, host, port):
+        asyncore.dispatcher.__init__(self)
+        self.create_socket()
+        self.set_reuse_addr()
+        self.bind((host, port))
+        self.listen(5)
+
+    def handle_accepted(self, sock, addr):
         global last_received_message
-        self.transport = transport
-        peername = transport.get_extra_info('peername')
-        print('Incoming connection from {}'.format(peername))
-        self.transport.write(last_received_message)
-
+        print('Incoming connection from %s' % repr(addr))
+        sock.send(last_received_message)
         last_received_message = b''
-        self.transport.close()
 
 
-async def main():
-    # Start the SMTP server
-    controller = Controller(CustomSMTPHandler(), hostname='0.0.0.0', port=11025)
-    controller.start()
-
-    # Start the TCP Echo server
-    loop = asyncio.get_running_loop()
-    server = await loop.create_server(
-        lambda: EchoServerProtocol(),
-        '0.0.0.0', 11080
-    )
-    async with server:
-        await server.serve_forever()
-
-
-if __name__ == "__main__":
-    asyncio.run(main())
+server = CustomSMTPServer(('0.0.0.0', 11025), None)  # SMTP mail goes here
+server2 = EchoServer('0.0.0.0', 11080)  # Echo back last message received
+asyncore.loop()

changedetectionio/tests/smtp/test_notification_smtp.py

@@ -32,8 +32,6 @@ def get_last_message_from_smtp_server():
     client_socket.connect((smtp_test_server, port))  # connect to the server
 
     data = client_socket.recv(50024).decode()  # receive response
-    logging.info("get_last_message_from_smtp_server..")
-    logging.info(data)
     client_socket.close()  # close the connection
     return data
 
@@ -73,8 +71,6 @@ def test_check_notification_email_formats_default_HTML(client, live_server):
 
     wait_for_all_checks(client)
     set_longer_modified_response()
-    time.sleep(2)
-
     client.get(url_for("form_watch_checknow"), follow_redirects=True)
     wait_for_all_checks(client)
 
@@ -85,7 +81,7 @@ def test_check_notification_email_formats_default_HTML(client, live_server):
 
     # The email should have two bodies, and the text/html part should be <br>
     assert 'Content-Type: text/plain' in msg
-    assert '(added) So let\'s see what happens.\r\n' in msg  # The plaintext part with \r\n
+    assert '(added) So let\'s see what happens.\n' in msg  # The plaintext part with \n
     assert 'Content-Type: text/html' in msg
     assert '(added) So let\'s see what happens.<br>' in msg  # the html part
     res = client.get(url_for("form_delete", uuid="all"), follow_redirects=True)
@@ -139,7 +135,6 @@ def test_check_notification_email_formats_default_Text_override_HTML(client, liv
 
     wait_for_all_checks(client)
     set_longer_modified_response()
-    time.sleep(2)
     client.get(url_for("form_watch_checknow"), follow_redirects=True)
     wait_for_all_checks(client)
 
@@ -152,7 +147,7 @@ def test_check_notification_email_formats_default_Text_override_HTML(client, liv
     # The email should not have two bodies, should be TEXT only
 
     assert 'Content-Type: text/plain' in msg
-    assert '(added) So let\'s see what happens.\r\n' in msg  # The plaintext part with \r\n
+    assert '(added) So let\'s see what happens.\n' in msg  # The plaintext part with \n
 
     set_original_response()
     # Now override as HTML format
@@ -173,7 +168,7 @@ def test_check_notification_email_formats_default_Text_override_HTML(client, liv
 
     # The email should have two bodies, and the text/html part should be <br>
     assert 'Content-Type: text/plain' in msg
-    assert '(removed) So let\'s see what happens.\r\n' in msg  # The plaintext part with \n
+    assert '(removed) So let\'s see what happens.\n' in msg  # The plaintext part with \n
     assert 'Content-Type: text/html' in msg
     assert '(removed) So let\'s see what happens.<br>' in msg  # the html part
 

changedetectionio/tests/test_add_replace_remove_filter.py

@@ -1,5 +1,5 @@
 #!/usr/bin/python3
-import os.path
+
 import time
 from flask import url_for
 from .util import live_server_setup, wait_for_all_checks
@@ -107,6 +107,7 @@ def test_check_add_line_contains_trigger(client, live_server):
     #live_server_setup(live_server)
 
     # Give the endpoint time to spin up
+    time.sleep(1)
     test_notification_url = url_for('test_notification_endpoint', _external=True).replace('http://', 'post://') + "?xxx={{ watch_url }}"
 
     res = client.post(
@@ -165,7 +166,6 @@ def test_check_add_line_contains_trigger(client, live_server):
 
     # Takes a moment for apprise to fire
     time.sleep(3)
-    assert os.path.isfile("test-datastore/notification.txt"), "Notification fired because I can see the output file"
     with open("test-datastore/notification.txt", 'r') as f:
         response= f.read()
         assert '-Oh yes please-' in response

changedetectionio/tests/test_api.py

@@ -149,15 +149,6 @@ def test_api_simple(client, live_server):
         headers={'x-api-key': api_key},
     )
     assert b'which has this one new line' in res.data
-    assert b'<div id' not in res.data
-
-    # Fetch the HTML of the latest one
-    res = client.get(
-        url_for("watchsinglehistory", uuid=watch_uuid, timestamp='latest')+"?html=1",
-        headers={'x-api-key': api_key},
-    )
-    assert b'which has this one new line' in res.data
-    assert b'<div id' in res.data
 
     # Fetch the whole watch
     res = client.get(

changedetectionio/tests/test_auth.py

@@ -2,12 +2,13 @@
 
 import time
 from flask import url_for
-from .util import live_server_setup, wait_for_all_checks
-
+from . util import live_server_setup
 
 def test_basic_auth(client, live_server):
 
     live_server_setup(live_server)
+    # Give the endpoint time to spin up
+    time.sleep(1)
 
     # Add our URL to the import page
     test_url = url_for('test_basicauth_method', _external=True).replace("//","//myuser:mypass@")
@@ -18,8 +19,8 @@ def test_basic_auth(client, live_server):
         follow_redirects=True
     )
     assert b"1 Imported" in res.data
-    wait_for_all_checks(client)
     time.sleep(1)
+
     # Check form validation
     res = client.post(
         url_for("edit_page", uuid="first"),
@@ -28,7 +29,7 @@ def test_basic_auth(client, live_server):
     )
     assert b"Updated watch." in res.data
 
-    wait_for_all_checks(client)
+    time.sleep(1)
     res = client.get(
         url_for("preview_page", uuid="first"),
         follow_redirects=True

changedetectionio/tests/test_automatic_follow_ldjson_price.py

@@ -100,12 +100,11 @@ def test_check_ldjson_price_autodetect(client, live_server):
 
     # Accept it
     uuid = extract_UUID_from_client(client)
-    time.sleep(1)
+
     client.get(url_for('price_data_follower.accept', uuid=uuid, follow_redirects=True))
     wait_for_all_checks(client)
 
     # Trigger a check
-    time.sleep(1)
     client.get(url_for("form_watch_checknow"), follow_redirects=True)
     wait_for_all_checks(client)
     # Offer should be gone

changedetectionio/tests/test_backend.py

@@ -3,8 +3,7 @@
 import time
 from flask import url_for
 from urllib.request import urlopen
-from .util import set_original_response, set_modified_response, live_server_setup, wait_for_all_checks, extract_rss_token_from_UI, \
-    extract_UUID_from_client
+from .util import set_original_response, set_modified_response, live_server_setup, wait_for_all_checks, extract_rss_token_from_UI
 
 sleep_time_for_fetch_thread = 3
 
@@ -63,6 +62,9 @@ def test_check_basic_change_detection_functionality(client, live_server):
     # Make a change
     set_modified_response()
 
+    res = urlopen(url_for('test_endpoint', _external=True))
+    assert b'which has this one new line' in res.read()
+
     # Force recheck
     res = client.get(url_for("form_watch_checknow"), follow_redirects=True)
     assert b'1 watches queued for rechecking.' in res.data
@@ -133,23 +135,12 @@ def test_check_basic_change_detection_functionality(client, live_server):
     # It should have picked up the <title>
     assert b'head title' in res.data
 
-    # Be sure the last_viewed is going to be greater than the last snapshot
-    time.sleep(1)
-
     # hit the mark all viewed link
     res = client.get(url_for("mark_all_viewed"), follow_redirects=True)
 
     assert b'Mark all viewed' not in res.data
     assert b'unviewed' not in res.data
 
-    # #2458 "clear history" should make the Watch object update its status correctly when the first snapshot lands again
-    uuid = extract_UUID_from_client(client)
-    client.get(url_for("clear_watch_history", uuid=uuid))
-    client.get(url_for("form_watch_checknow"), follow_redirects=True)
-    wait_for_all_checks(client)
-    res = client.get(url_for("index"))
-    assert b'preview/' in res.data
-
     #
     # Cleanup everything
     res = client.get(url_for("form_delete", uuid="all"), follow_redirects=True)

changedetectionio/tests/test_encoding.py

@@ -3,7 +3,7 @@
 
 import time
 from flask import url_for
-from .util import live_server_setup, wait_for_all_checks
+from .util import live_server_setup
 import pytest
 
 
@@ -27,6 +27,9 @@ def set_html_response():
 def test_check_encoding_detection(client, live_server):
     set_html_response()
 
+    # Give the endpoint time to spin up
+    time.sleep(1)
+
     # Add our URL to the import page
     test_url = url_for('test_endpoint', content_type="text/html", _external=True)
     client.post(
@@ -36,7 +39,7 @@ def test_check_encoding_detection(client, live_server):
     )
 
     # Give the thread time to pick it up
-    wait_for_all_checks(client)
+    time.sleep(2)
 
     res = client.get(
         url_for("preview_page", uuid="first"),
@@ -53,6 +56,9 @@ def test_check_encoding_detection(client, live_server):
 def test_check_encoding_detection_missing_content_type_header(client, live_server):
     set_html_response()
 
+    # Give the endpoint time to spin up
+    time.sleep(1)
+
     # Add our URL to the import page
     test_url = url_for('test_endpoint', _external=True)
     client.post(
@@ -61,7 +67,8 @@ def test_check_encoding_detection_missing_content_type_header(client, live_serve
         follow_redirects=True
     )
 
-    wait_for_all_checks(client)
+    # Give the thread time to pick it up
+    time.sleep(2)
 
     res = client.get(
         url_for("preview_page", uuid="first"),

changedetectionio/tests/test_extract_csv.py

@@ -29,7 +29,6 @@ def test_check_extract_text_from_diff(client, live_server):
     # Load in 5 different numbers/changes
     last_date=""
     for n in range(5):
-        time.sleep(1)
         # Give the thread time to pick it up
         print("Bumping snapshot and checking.. ", n)
         last_date = str(time.time())

changedetectionio/tests/test_filter_failure_notification.py

@@ -21,11 +21,10 @@ def set_response_with_filter():
         f.write(test_return_data)
     return None
 
-def run_filter_test(client, live_server, content_filter):
-
-    # Response WITHOUT the filter ID element
-    set_original_response()
+def run_filter_test(client, content_filter):
 
+    # Give the endpoint time to spin up
+    time.sleep(1)
     # cleanup for the next
     client.get(
         url_for("form_delete", uuid="all"),
@@ -80,7 +79,6 @@ def run_filter_test(client, live_server, content_filter):
         "include_filters": content_filter,
         "fetch_backend": "html_requests"})
 
-    # A POST here will also reset the filter failure counter (filter_failure_notification_threshold_attempts)
     res = client.post(
         url_for("edit_page", uuid="first"),
         data=notification_form_data,
@@ -93,21 +91,20 @@ def run_filter_test(client, live_server, content_filter):
     # Now the notification should not exist, because we didnt reach the threshold
     assert not os.path.isfile("test-datastore/notification.txt")
 
-    # recheck it up to just before the threshold, including the fact that in the previous POST it would have rechecked (and incremented)
+    # -2 because we would have checked twice above (on adding and on edit)
     for i in range(0, App._FILTER_FAILURE_THRESHOLD_ATTEMPTS_DEFAULT-2):
-        client.get(url_for("form_watch_checknow"), follow_redirects=True)
+        res = client.get(url_for("form_watch_checknow"), follow_redirects=True)
         wait_for_all_checks(client)
-        time.sleep(2) # delay for apprise to fire
-        assert not os.path.isfile("test-datastore/notification.txt"), f"test-datastore/notification.txt should not exist - Attempt {i} when threshold is {App._FILTER_FAILURE_THRESHOLD_ATTEMPTS_DEFAULT}"
+        assert not os.path.isfile("test-datastore/notification.txt"), f"test-datastore/notification.txt should not exist - Attempt {i}"
 
     # We should see something in the frontend
-    res = client.get(url_for("index"))
     assert b'Warning, no filters were found' in res.data
 
-    # One more check should trigger the _FILTER_FAILURE_THRESHOLD_ATTEMPTS_DEFAULT threshold
+    # One more check should trigger it (see -2 above)
+    client.get(url_for("form_watch_checknow"), follow_redirects=True)
+    wait_for_all_checks(client)
     client.get(url_for("form_watch_checknow"), follow_redirects=True)
     wait_for_all_checks(client)
-    time.sleep(2)  # delay for apprise to fire
     # Now it should exist and contain our "filter not found" alert
     assert os.path.isfile("test-datastore/notification.txt")
 
@@ -152,9 +149,13 @@ def test_setup(live_server):
     live_server_setup(live_server)
 
 def test_check_include_filters_failure_notification(client, live_server):
-    run_filter_test(client, live_server,'#nope-doesnt-exist')
+    set_original_response()
+    wait_for_all_checks(client)
+    run_filter_test(client, '#nope-doesnt-exist')
 
 def test_check_xpath_filter_failure_notification(client, live_server):
-    run_filter_test(client, live_server, '//*[@id="nope-doesnt-exist"]')
+    set_original_response()
+    time.sleep(1)
+    run_filter_test(client, '//*[@id="nope-doesnt-exist"]')
 
 # Test that notification is never sent

changedetectionio/tests/test_group.py

@@ -100,12 +100,6 @@ def test_setup_group_tag(client, live_server):
     assert b'Should be only this' in res.data
     assert b'And never this' not in res.data
 
-    res = client.get(
-        url_for("edit_page", uuid="first"),
-        follow_redirects=True
-    )
-    # 2307 the UI notice should appear in the placeholder
-    assert b'WARNING: Watch has tag/groups set with special filters' in res.data
 
     # RSS Group tag filter
     # An extra one that should be excluded
@@ -327,154 +321,3 @@ def test_clone_tag_on_quickwatchform_add(client, live_server):
 
     res = client.get(url_for("tags.delete_all"), follow_redirects=True)
     assert b'All tags deleted' in res.data
-
-def test_order_of_filters_tag_filter_and_watch_filter(client, live_server):
-
-    # Add a tag with some config, import a tag and it should roughly work
-    res = client.post(
-        url_for("tags.form_tag_add"),
-        data={"name": "test-tag-keep-order"},
-        follow_redirects=True
-    )
-    assert b"Tag added" in res.data
-    assert b"test-tag-keep-order" in res.data
-    tag_filters = [
-            '#only-this', # duplicated filters
-            '#only-this',
-            '#only-this',
-            '#only-this',
-            ]
-
-    res = client.post(
-        url_for("tags.form_tag_edit_submit", uuid="first"),
-        data={"name": "test-tag-keep-order",
-              "include_filters": '\n'.join(tag_filters) },
-        follow_redirects=True
-    )
-    assert b"Updated" in res.data
-    tag_uuid = get_UUID_for_tag_name(client, name="test-tag-keep-order")
-    res = client.get(
-        url_for("tags.form_tag_edit", uuid="first")
-    )
-    assert b"#only-this" in res.data
-
-
-    d = """<html>
-       <body>
-     Some initial text<br>
-     <p id="only-this">And 1 this</p>
-     <br>
-     <p id="not-this">And 2 this</p>
-     <p id="">And 3 this</p><!--/html/body/p[3]/-->
-     <p id="">And 4 this</p><!--/html/body/p[4]/-->
-     <p id="">And 5 this</p><!--/html/body/p[5]/-->
-     <p id="">And 6 this</p><!--/html/body/p[6]/-->
-     <p id="">And 7 this</p><!--/html/body/p[7]/-->
-     <p id="">And 8 this</p><!--/html/body/p[8]/-->
-     <p id="">And 9 this</p><!--/html/body/p[9]/-->
-     <p id="">And 10 this</p><!--/html/body/p[10]/-->
-     <p id="">And 11 this</p><!--/html/body/p[11]/-->
-     <p id="">And 12 this</p><!--/html/body/p[12]/-->
-     <p id="">And 13 this</p><!--/html/body/p[13]/-->
-     <p id="">And 14 this</p><!--/html/body/p[14]/-->
-     <p id="not-this">And 15 this</p><!--/html/body/p[15]/-->
-     </body>
-     </html>
-    """
-
-    with open("test-datastore/endpoint-content.txt", "w") as f:
-        f.write(d)
-
-    test_url = url_for('test_endpoint', _external=True)
-    res = client.post(
-        url_for("import_page"),
-        data={"urls": test_url},
-        follow_redirects=True
-    )
-    assert b"1 Imported" in res.data
-    wait_for_all_checks(client)
-
-    filters = [
-            '/html/body/p[3]',
-            '/html/body/p[4]',
-            '/html/body/p[5]',
-            '/html/body/p[6]',
-            '/html/body/p[7]',
-            '/html/body/p[8]',
-            '/html/body/p[9]',
-            '/html/body/p[10]',
-            '/html/body/p[11]',
-            '/html/body/p[12]',
-            '/html/body/p[13]', # duplicated tags
-            '/html/body/p[13]',
-            '/html/body/p[13]',
-            '/html/body/p[13]',
-            '/html/body/p[13]',
-            '/html/body/p[14]',
-            ]
-
-    res = client.post(
-        url_for("edit_page", uuid="first"),
-        data={"include_filters": '\n'.join(filters),
-            "url": test_url,
-            "tags": "test-tag-keep-order",
-            "headers": "",
-            'fetch_backend': "html_requests"},
-        follow_redirects=True
-    )
-    assert b"Updated watch." in res.data
-    wait_for_all_checks(client)
-
-    res = client.get(
-        url_for("preview_page", uuid="first"),
-        follow_redirects=True
-    )
-
-    assert b"And 1 this" in res.data  # test-tag-keep-order
-
-    a_tag_filter_check = b'And 1 this' #'#only-this' of tag_filters
-    # check there is no duplication of tag_filters
-    assert res.data.count(a_tag_filter_check) == 1, f"duplicated filters didn't removed {res.data.count(a_tag_filter_check)} of {a_tag_filter_check} in {res.data=}"
-
-    a_filter_check = b"And 13 this" # '/html/body/p[13]'
-    # check there is no duplication of filters
-    assert res.data.count(a_filter_check) == 1, f"duplicated filters didn't removed. {res.data.count(a_filter_check)} of {a_filter_check} in {res.data=}"
-
-    a_filter_check_not_include = b"And 2 this" # '/html/body/p[2]'
-    assert a_filter_check_not_include not in res.data
-
-    checklist = [
-            b"And 3 this",
-            b"And 4 this",
-            b"And 5 this",
-            b"And 6 this",
-            b"And 7 this",
-            b"And 8 this",
-            b"And 9 this",
-            b"And 10 this",
-            b"And 11 this",
-            b"And 12 this",
-            b"And 13 this",
-            b"And 14 this",
-            b"And 1 this", # result of filter from tag.
-            ]
-    # check whether everything a user requested is there
-    for test in checklist:
-        assert test in res.data
-
-    # check whether everything a user requested is in order of filters.
-    n = 0
-    for test in checklist:
-        t_index = res.data[n:].find(test)
-        # if the text is not searched, return -1.
-        assert t_index >= 0, f"""failed because {test=} not in {res.data[n:]=}
-#####################
-Looks like some feature changed the order of result of filters.
-#####################
-the {test} appeared before. {test in res.data[:n]=}
-{res.data[:n]=}
-        """
-        n += t_index + len(test)
-
-    res = client.get(url_for("form_delete", uuid="all"), follow_redirects=True)
-    assert b'Deleted' in res.data

changedetectionio/tests/test_history_consistency.py

@@ -5,13 +5,15 @@ import os
 import json
 import logging
 from flask import url_for
-from .util import live_server_setup, wait_for_all_checks
+from .util import live_server_setup
 from urllib.parse import urlparse, parse_qs
 
 def test_consistent_history(client, live_server):
     live_server_setup(live_server)
 
-    r = range(1, 30)
+    # Give the endpoint time to spin up
+    time.sleep(1)
+    r = range(1, 50)
 
     for one in r:
         test_url = url_for('test_endpoint', content_type="text/html", content=str(one), _external=True)
@@ -23,8 +25,15 @@ def test_consistent_history(client, live_server):
 
         assert b"1 Imported" in res.data
 
-    wait_for_all_checks(client)
+    time.sleep(3)
+    while True:
+        res = client.get(url_for("index"))
+        logging.debug("Waiting for 'Checking now' to go away..")
+        if b'Checking now' not in res.data:
+            break
+        time.sleep(0.5)
 
+    time.sleep(3)
     # Essentially just triggers the DB write/update
     res = client.post(
         url_for("settings_page"),
@@ -35,9 +44,8 @@ def test_consistent_history(client, live_server):
     )
     assert b"Settings updated." in res.data
 
-
-    time.sleep(2)
-
+    # Give it time to write it out
+    time.sleep(3)
     json_db_file = os.path.join(live_server.app.config['DATASTORE'].datastore_path, 'url-watches.json')
 
     json_obj = None
@@ -50,7 +58,7 @@ def test_consistent_history(client, live_server):
     # each one should have a history.txt containing just one line
     for w in json_obj['watching'].keys():
         history_txt_index_file = os.path.join(live_server.app.config['DATASTORE'].datastore_path, w, 'history.txt')
-        assert os.path.isfile(history_txt_index_file), f"History.txt should exist where I expect it at {history_txt_index_file}"
+        assert os.path.isfile(history_txt_index_file), "History.txt should exist where I expect it - {}".format(history_txt_index_file)
 
         # Same like in model.Watch
         with open(history_txt_index_file, "r") as f:
@@ -62,15 +70,15 @@ def test_consistent_history(client, live_server):
                                                      w))
         # Find the snapshot one
         for fname in files_in_watch_dir:
-            if fname != 'history.txt' and 'html' not in fname:
+            if fname != 'history.txt':
                 # contents should match what we requested as content returned from the test url
                 with open(os.path.join(live_server.app.config['DATASTORE'].datastore_path, w, fname), 'r') as snapshot_f:
                     contents = snapshot_f.read()
                     watch_url = json_obj['watching'][w]['url']
                     u = urlparse(watch_url)
                     q = parse_qs(u[4])
-                    assert q['content'][0] == contents.strip(), f"Snapshot file {fname} should contain {q['content'][0]}"
+                    assert q['content'][0] == contents.strip(), "Snapshot file {} should contain {}".format(fname, q['content'][0])
 
 
 
-        assert len(files_in_watch_dir) == 3, "Should be just three files in the dir, html.br snapshot, history.txt and the extracted text snapshot"
+        assert len(files_in_watch_dir) == 2, "Should be just two files in the dir, history.txt and the snapshot"

changedetectionio/tests/test_ignorehighlighter.py

@@ -45,6 +45,7 @@ def test_highlight_ignore(client, live_server):
     )
 
     res = client.get(url_for("edit_page", uuid=uuid))
+
     # should be a regex now
     assert b'/oh\ yeah\ \d+/' in res.data
 
@@ -54,7 +55,3 @@ def test_highlight_ignore(client, live_server):
     # And it should register in the preview page
     res = client.get(url_for("preview_page", uuid=uuid))
     assert b'<div class="ignored">oh yeah 456' in res.data
-
-    # Should be in base.html
-    assert b'csrftoken' in res.data
-

changedetectionio/tests/test_jinja2.py

@@ -2,15 +2,15 @@
 
 import time
 from flask import url_for
-from .util import live_server_setup, wait_for_all_checks
+from .util import live_server_setup
 
 
-def test_setup(client, live_server):
-    live_server_setup(live_server)
-
 # If there was only a change in the whitespacing, then we shouldnt have a change detected
 def test_jinja2_in_url_query(client, live_server):
-    #live_server_setup(live_server)
+    live_server_setup(live_server)
+
+    # Give the endpoint time to spin up
+    time.sleep(1)
 
     # Add our URL to the import page
     test_url = url_for('test_return_query', _external=True)
@@ -24,35 +24,10 @@ def test_jinja2_in_url_query(client, live_server):
         follow_redirects=True
     )
     assert b"Watch added" in res.data
-    wait_for_all_checks(client)
-
+    time.sleep(3)
     # It should report nothing found (no new 'unviewed' class)
     res = client.get(
         url_for("preview_page", uuid="first"),
         follow_redirects=True
     )
     assert b'date=2' in res.data
-
-# https://techtonics.medium.com/secure-templating-with-jinja2-understanding-ssti-and-jinja2-sandbox-environment-b956edd60456
-def test_jinja2_security_url_query(client, live_server):
-    #live_server_setup(live_server)
-
-    # Add our URL to the import page
-    test_url = url_for('test_return_query', _external=True)
-
-    # because url_for() will URL-encode the var, but we dont here
-    full_url = "{}?{}".format(test_url,
-                              "date={{ ''.__class__.__mro__[1].__subclasses__()}}", )
-    res = client.post(
-        url_for("form_quick_watch_add"),
-        data={"url": full_url, "tags": "test"},
-        follow_redirects=True
-    )
-    assert b"Watch added" in res.data
-    wait_for_all_checks(client)
-
-    # It should report nothing found (no new 'unviewed' class)
-    res = client.get(url_for("index"))
-    assert b'is invalid and cannot be used' in res.data
-    # Some of the spewed output from the subclasses
-    assert b'dict_values' not in res.data

changedetectionio/tests/test_jsonpath_jq_selector.py

@@ -41,26 +41,19 @@ and it can also be repeated
     from .. import html_tools
 
     # See that we can find the second <script> one, which is not broken, and matches our filter
-    text = html_tools.extract_json_as_string(content, "json:$.offers.priceCurrency")
-    assert text == '"AUD"'
-
-    text = html_tools.extract_json_as_string('{"id":5}', "json:$.id")
-    assert text == "5"
+    text = html_tools.extract_json_as_string(content, "json:$.offers.price")
+    assert text == "23.5"
 
     # also check for jq
     if jq_support:
-        text = html_tools.extract_json_as_string(content, "jq:.offers.priceCurrency")
-        assert text == '"AUD"'
+        text = html_tools.extract_json_as_string(content, "jq:.offers.price")
+        assert text == "23.5"
 
         text = html_tools.extract_json_as_string('{"id":5}', "jq:.id")
         assert text == "5"
 
-        text = html_tools.extract_json_as_string(content, "jqraw:.offers.priceCurrency")
-        assert text == "AUD"
-
-        text = html_tools.extract_json_as_string('{"id":5}', "jqraw:.id")
-        assert text == "5"
-
+    text = html_tools.extract_json_as_string('{"id":5}', "json:$.id")
+    assert text == "5"
 
     # When nothing at all is found, it should throw JSONNOTFound
     # Which is caught and shown to the user in the watch-overview table
@@ -71,9 +64,6 @@ and it can also be repeated
         with pytest.raises(html_tools.JSONNotFound) as e_info:
             html_tools.extract_json_as_string('COMPLETE GIBBERISH, NO JSON!', "jq:.id")
 
-        with pytest.raises(html_tools.JSONNotFound) as e_info:
-            html_tools.extract_json_as_string('COMPLETE GIBBERISH, NO JSON!', "jqraw:.id")
-
 
 def test_unittest_inline_extract_body():
     content = """
@@ -301,10 +291,6 @@ def test_check_jq_filter(client, live_server):
     if jq_support:
         check_json_filter('jq:.boss.name', client, live_server)
 
-def test_check_jqraw_filter(client, live_server):
-    if jq_support:
-        check_json_filter('jqraw:.boss.name', client, live_server)
-
 def check_json_filter_bool_val(json_filter, client, live_server):
     set_original_response()
 
@@ -359,10 +345,6 @@ def test_check_jq_filter_bool_val(client, live_server):
     if jq_support:
         check_json_filter_bool_val("jq:.available", client, live_server)
 
-def test_check_jqraw_filter_bool_val(client, live_server):
-    if jq_support:
-        check_json_filter_bool_val("jq:.available", client, live_server)
-
 # Re #265 - Extended JSON selector test
 # Stuff to consider here
 # - Selector should be allowed to return empty when it doesnt match (people might wait for some condition)
@@ -497,9 +479,8 @@ def test_correct_header_detect(client, live_server):
         url_for("preview_page", uuid="first"),
         follow_redirects=True
     )
-
-    assert b'&#34;hello&#34;: 123,' in res.data
-    assert b'&#34;world&#34;: 123</div>' in res.data
+    assert b'&#34;world&#34;:' in res.data
+    assert res.data.count(b'{') >= 2
 
     res = client.get(url_for("form_delete", uuid="all"), follow_redirects=True)
     assert b'Deleted' in res.data
@@ -509,8 +490,4 @@ def test_check_jsonpath_ext_filter(client, live_server):
 
 def test_check_jq_ext_filter(client, live_server):
     if jq_support:
-        check_json_ext_filter('jq:.[] | select(.status | contains("Sold"))', client, live_server)
-
-def test_check_jqraw_ext_filter(client, live_server):
-    if jq_support:
-        check_json_ext_filter('jq:.[] | select(.status | contains("Sold"))', client, live_server)
+        check_json_ext_filter('jq:.[] | select(.status | contains("Sold"))', client, live_server)

changedetectionio/tests/test_notification_errors.py

@@ -9,6 +9,9 @@ def test_check_notification_error_handling(client, live_server):
     live_server_setup(live_server)
     set_original_response()
 
+    # Give the endpoint time to spin up
+    time.sleep(1)
+
     # Set a URL and fetch it, then set a notification URL which is going to give errors
     test_url = url_for('test_endpoint', _external=True)
     res = client.post(

changedetectionio/tests/test_pdf.py

@@ -29,8 +29,7 @@ def test_fetch_pdf(client, live_server):
         follow_redirects=True
     )
 
-    # PDF header should not be there (it was converted to text)
-    assert b'PDF' not in res.data[:10]
+    assert b'PDF-1.5' not in res.data
     assert b'hello world' in res.data
 
     # So we know if the file changes in other ways

changedetectionio/tests/test_request.py

@@ -253,99 +253,15 @@ def test_method_in_request(client, live_server):
     res = client.get(url_for("form_delete", uuid="all"), follow_redirects=True)
     assert b'Deleted' in res.data
 
-# Re #2408 - user-agent override test, also should handle case-insensitive header deduplication
-def test_ua_global_override(client, live_server):
-    # live_server_setup(live_server)
-    test_url = url_for('test_headers', _external=True)
-
-    res = client.post(
-        url_for("settings_page"),
-        data={
-            "application-fetch_backend": "html_requests",
-            "application-minutes_between_check": 180,
-            "requests-default_ua-html_requests": "html-requests-user-agent"
-        },
-        follow_redirects=True
-    )
-    assert b'Settings updated' in res.data
-
-    res = client.post(
-        url_for("import_page"),
-        data={"urls": test_url},
-        follow_redirects=True
-    )
-    assert b"1 Imported" in res.data
-
-    wait_for_all_checks(client)
-    res = client.get(
-        url_for("preview_page", uuid="first"),
-        follow_redirects=True
-    )
-
-    assert b"html-requests-user-agent" in res.data
-    # default user-agent should have shown by now
-    # now add a custom one in the headers
-
-
-    # Add some headers to a request
-    res = client.post(
-        url_for("edit_page", uuid="first"),
-        data={
-            "url": test_url,
-            "tags": "testtag",
-            "fetch_backend": 'html_requests',
-            # Important - also test case-insensitive
-            "headers": "User-AGent: agent-from-watch"},
-        follow_redirects=True
-    )
-    assert b"Updated watch." in res.data
-    wait_for_all_checks(client)
-    res = client.get(
-        url_for("preview_page", uuid="first"),
-        follow_redirects=True
-    )
-    assert b"agent-from-watch" in res.data
-    assert b"html-requests-user-agent" not in res.data
-    res = client.get(url_for("form_delete", uuid="all"), follow_redirects=True)
-    assert b'Deleted' in res.data
-
 def test_headers_textfile_in_request(client, live_server):
     #live_server_setup(live_server)
     # Add our URL to the import page
-
-    webdriver_ua = "Hello fancy webdriver UA 1.0"
-    requests_ua = "Hello basic requests UA 1.1"
-
     test_url = url_for('test_headers', _external=True)
     if os.getenv('PLAYWRIGHT_DRIVER_URL'):
         # Because its no longer calling back to localhost but from the browser container, set in test-only.yml
         test_url = test_url.replace('localhost', 'cdio')
 
-    form_data = {
-        "application-fetch_backend": "html_requests",
-        "application-minutes_between_check": 180,
-        "requests-default_ua-html_requests": requests_ua
-    }
-
-    if os.getenv('PLAYWRIGHT_DRIVER_URL'):
-        form_data["requests-default_ua-html_webdriver"] = webdriver_ua
-
-    res = client.post(
-        url_for("settings_page"),
-        data=form_data,
-        follow_redirects=True
-    )
-    assert b'Settings updated' in res.data
-
-    res = client.get(url_for("settings_page"))
-
-    # Only when some kind of real browser is setup
-    if os.getenv('PLAYWRIGHT_DRIVER_URL'):
-        assert b'requests-default_ua-html_webdriver' in res.data
-
-    # Field should always be there
-    assert b"requests-default_ua-html_requests" in res.data
-
+    print ("TEST URL IS ",test_url)
     # Add the test URL twice, we will check
     res = client.post(
         url_for("import_page"),
@@ -356,14 +272,15 @@ def test_headers_textfile_in_request(client, live_server):
 
     wait_for_all_checks(client)
 
+
     # Add some headers to a request
     res = client.post(
         url_for("edit_page", uuid="first"),
         data={
-            "url": test_url,
-            "tags": "testtag",
-            "fetch_backend": 'html_webdriver' if os.getenv('PLAYWRIGHT_DRIVER_URL') else 'html_requests',
-            "headers": "xxx:ooo\ncool:yeah\r\n"},
+              "url": test_url,
+              "tags": "testtag",
+              "fetch_backend": 'html_webdriver' if os.getenv('PLAYWRIGHT_DRIVER_URL') else 'html_requests',
+              "headers": "xxx:ooo\ncool:yeah\r\n"},
         follow_redirects=True
     )
     assert b"Updated watch." in res.data
@@ -375,7 +292,7 @@ def test_headers_textfile_in_request(client, live_server):
     with open('test-datastore/headers.txt', 'w') as f:
         f.write("global-header: nice\r\nnext-global-header: nice")
 
-    with open('test-datastore/' + extract_UUID_from_client(client) + '/headers.txt', 'w') as f:
+    with open('test-datastore/'+extract_UUID_from_client(client)+'/headers.txt', 'w') as f:
         f.write("watch-header: nice")
 
     client.get(url_for("form_watch_checknow"), follow_redirects=True)
@@ -389,7 +306,7 @@ def test_headers_textfile_in_request(client, live_server):
     # Not needed anymore
     os.unlink('test-datastore/headers.txt')
     os.unlink('test-datastore/headers-testtag.txt')
-
+    os.unlink('test-datastore/'+extract_UUID_from_client(client)+'/headers.txt')
     # The service should echo back the request verb
     res = client.get(
         url_for("preview_page", uuid="first"),
@@ -402,12 +319,7 @@ def test_headers_textfile_in_request(client, live_server):
     assert b"Watch-Header:nice" in res.data
     assert b"Tag-Header:test" in res.data
 
-    # Check the custom UA from system settings page made it through
-    if os.getenv('PLAYWRIGHT_DRIVER_URL'):
-        assert "User-Agent:".encode('utf-8') + webdriver_ua.encode('utf-8') in res.data
-    else:
-        assert "User-Agent:".encode('utf-8') + requests_ua.encode('utf-8') in res.data
 
-    # unlink headers.txt on start/stop
+    #unlink headers.txt on start/stop
     res = client.get(url_for("form_delete", uuid="all"), follow_redirects=True)
-    assert b'Deleted' in res.data
+    assert b'Deleted' in res.data

changedetectionio/tests/test_rss.py

@@ -69,7 +69,6 @@ def test_rss_and_token(client, live_server):
 
     wait_for_all_checks(client)
     set_modified_response()
-    time.sleep(1)
     client.get(url_for("form_watch_checknow"), follow_redirects=True)
     wait_for_all_checks(client)
 
@@ -88,7 +87,7 @@ def test_rss_and_token(client, live_server):
     assert b"Access denied, bad token" not in res.data
     assert b"Random content" in res.data
 
-    client.get(url_for("form_delete", uuid="all"), follow_redirects=True)
+    res = client.get(url_for("form_delete", uuid="all"), follow_redirects=True)
 
 def test_basic_cdata_rss_markup(client, live_server):
     #live_server_setup(live_server)

changedetectionio/tests/test_security.py

@@ -2,11 +2,9 @@ from flask import url_for
 from .util import set_original_response, set_modified_response, live_server_setup, wait_for_all_checks
 import time
 
-def test_setup(client, live_server):
-    live_server_setup(live_server)
 
 def test_bad_access(client, live_server):
-    #live_server_setup(live_server)
+    live_server_setup(live_server)
     res = client.post(
         url_for("import_page"),
         data={"urls": 'https://localhost'},
@@ -65,25 +63,4 @@ def test_bad_access(client, live_server):
     wait_for_all_checks(client)
     res = client.get(url_for("index"))
 
-    assert b'file:// type access is denied for security reasons.' in res.data
-
-def test_xss(client, live_server):
-    #live_server_setup(live_server)
-    from changedetectionio.notification import (
-        default_notification_format
-    )
-    # the template helpers were named .jinja which meant they were not having jinja2 autoescape enabled.
-    res = client.post(
-        url_for("settings_page"),
-        data={"application-notification_urls": '"><img src=x onerror=alert(document.domain)>',
-              "application-notification_title": '"><img src=x onerror=alert(document.domain)>',
-              "application-notification_body": '"><img src=x onerror=alert(document.domain)>',
-              "application-notification_format": default_notification_format,
-              "requests-time_between_check-minutes": 180,
-              'application-fetch_backend': "html_requests"},
-        follow_redirects=True
-    )
-
-    assert b"<img src=x onerror=alert(" not in res.data
-    assert b"&lt;img" in res.data
-
+    assert b'file:// type access is denied for security reasons.' in res.data

changedetectionio/tests/test_watch_fields_storage.py

@@ -54,3 +54,102 @@ def test_check_watch_field_storage(client, live_server):
     assert b"woohoo" in res.data
     assert b"curl: foo" in res.data
 
+
+
+# Re https://github.com/dgtlmoon/changedetection.io/issues/110
+def test_check_recheck_global_setting(client, live_server):
+
+    res = client.post(
+        url_for("settings_page"),
+        data={
+               "requests-time_between_check-minutes": 1566,
+               'application-fetch_backend': "html_requests"
+               },
+        follow_redirects=True
+    )
+    assert b"Settings updated." in res.data
+
+    # Now add a record
+
+    test_url = "http://somerandomsitewewatch.com"
+
+    res = client.post(
+        url_for("import_page"),
+        data={"urls": test_url},
+        follow_redirects=True
+    )
+    assert b"1 Imported" in res.data
+
+    # Now visit the edit page, it should have the default minutes
+
+    res = client.get(
+        url_for("edit_page", uuid="first"),
+        follow_redirects=True
+    )
+
+    # Should show the default minutes
+    assert b"change to another value if you want to be specific" in res.data
+    assert b"1566" in res.data
+
+    res = client.post(
+        url_for("settings_page"),
+        data={
+               "requests-time_between_check-minutes": 222,
+                'application-fetch_backend': "html_requests"
+               },
+        follow_redirects=True
+    )
+    assert b"Settings updated." in res.data
+
+    res = client.get(
+        url_for("edit_page", uuid="first"),
+        follow_redirects=True
+    )
+
+    # Should show the default minutes
+    assert b"change to another value if you want to be specific" in res.data
+    assert b"222" in res.data
+
+    # Now change it specifically, it should show the new minutes
+    res = client.post(
+        url_for("edit_page", uuid="first"),
+        data={"url": test_url,
+              "time_between_check-minutes": 55,
+              'fetch_backend': "html_requests"
+              },
+        follow_redirects=True
+    )
+
+    res = client.get(
+        url_for("edit_page", uuid="first"),
+        follow_redirects=True
+    )
+    assert b"55" in res.data
+
+    # Now submit an empty field, it should give back the default global minutes
+    res = client.post(
+        url_for("settings_page"),
+        data={
+               "requests-time_between_check-minutes": 666,
+                "application-fetch_backend": "html_requests"
+               },
+        follow_redirects=True
+    )
+    assert b"Settings updated." in res.data
+
+    res = client.post(
+        url_for("edit_page", uuid="first"),
+        data={"url": test_url,
+              "time_between_check-minutes": "",
+              'fetch_backend': "html_requests"
+              },
+        follow_redirects=True
+    )
+
+    assert b"Updated watch." in res.data
+
+    res = client.get(
+        url_for("edit_page", uuid="first"),
+        follow_redirects=True
+    )
+    assert b"666" in res.data

changedetectionio/tests/test_xpath_selector.py

@@ -1,4 +1,4 @@
-# -*- coding: utf-8 -*-
+#!/usr/bin/python3
 
 import time
 from flask import url_for
@@ -255,69 +255,6 @@ def test_xpath23_prefix_validation(client, live_server):
     res = client.get(url_for("form_delete", uuid="all"), follow_redirects=True)
     assert b'Deleted' in res.data
 
-def test_xpath1_lxml(client, live_server):
-    #live_server_setup(live_server)
-
-    d = '''<?xml version="1.0" encoding="UTF-8"?>
-    <rss xmlns:taxo="http://purl.org/rss/1.0/modules/taxonomy/" xmlns:rdf="http://www.w3.org/1999/02/22-rdf-syntax-ns#" xmlns:itunes="http://www.itunes.com/dtds/podcast-1.0.dtd" xmlns:dc="http://purl.org/dc/elements/1.1/" version="2.0">
-    	<channel>
-    		<title>rpilocator.com</title>
-    		<link>https://rpilocator.com</link>
-    		<description>Find Raspberry Pi Computers in Stock</description>
-    		<lastBuildDate>Thu, 19 May 2022 23:27:30 GMT</lastBuildDate>
-    		<image>
-    			<url>https://rpilocator.com/favicon.png</url>
-    			<title>rpilocator.com</title>
-    			<link>https://rpilocator.com/</link>
-    			<width>32</width>
-    			<height>32</height>
-    		</image>
-    		<item>
-    			<title>Stock Alert (UK): RPi CM4</title>
-    			<foo>something else unrelated</foo>
-    		</item>
-    		<item>
-    			<title>Stock Alert (UK): Big monitorěěěě</title>
-    			<foo>something else unrelated</foo>
-    		</item>		
-    	</channel>
-    </rss>'''.encode('utf-8')
-
-    with open("test-datastore/endpoint-content.txt", "wb") as f:
-        f.write(d)
-
-
-    test_url = url_for('test_endpoint', _external=True)
-    res = client.post(
-        url_for("import_page"),
-        data={"urls": test_url},
-        follow_redirects=True
-    )
-    assert b"1 Imported" in res.data
-    wait_for_all_checks(client)
-
-    res = client.post(
-        url_for("edit_page", uuid="first"),
-        data={"include_filters": "xpath1://title/text()", "url": test_url, "tags": "", "headers": "",
-              'fetch_backend': "html_requests"},
-        follow_redirects=True
-    )
-
-    ##### #2312
-    wait_for_all_checks(client)
-    res = client.get(url_for("index"))
-    assert b'_ElementStringResult' not in res.data # tested with 5.1.1 when it was removed and 5.1.0
-    assert b'Exception' not in res.data
-    res = client.get(
-        url_for("preview_page", uuid="first"),
-        follow_redirects=True
-    )
-
-    assert b"rpilocator.com" in res.data  # in selector
-    assert "Stock Alert (UK): Big monitorěěěě".encode('utf-8') in res.data  # not in selector
-
-    #####
-
 
 def test_xpath1_validation(client, live_server):
     # Add our URL to the import page

changedetectionio/tests/unit/test_jinja2_security.py

@@ -1,57 +0,0 @@
-#!/usr/bin/python3
-
-# run from dir above changedetectionio/ dir
-# python3 -m unittest changedetectionio.tests.unit.test_jinja2_security
-
-import unittest
-from changedetectionio import safe_jinja
-
-
-# mostly
-class TestJinja2SSTI(unittest.TestCase):
-
-    def test_exception(self):
-        import jinja2
-
-        # Where sandbox should kick in
-        attempt_list = [
-            "My name is {{ self.__init__.__globals__.__builtins__.__import__('os').system('id') }}",
-            "{{ self._TemplateReference__context.cycler.__init__.__globals__.os }}",
-            "{{ self.__init__.__globals__.__builtins__.__import__('os').popen('id').read() }}",
-            "{{cycler.__init__.__globals__.os.popen('id').read()}}",
-            "{{joiner.__init__.__globals__.os.popen('id').read()}}",
-            "{{namespace.__init__.__globals__.os.popen('id').read()}}",
-            "{{ ''.__class__.__mro__[2].__subclasses__()[40]('/tmp/hello.txt', 'w').write('Hello here !') }}",
-            "My name is {{ self.__init__.__globals__ }}",
-            "{{ dict.__base__.__subclasses__() }}"
-        ]
-        for attempt in attempt_list:
-            with self.assertRaises(jinja2.exceptions.SecurityError):
-                safe_jinja.render(attempt)
-
-    def test_exception_debug_calls(self):
-        import jinja2
-        # Where sandbox should kick in - configs and debug calls
-        attempt_list = [
-            "{% debug %}",
-        ]
-        for attempt in attempt_list:
-            # Usually should be something like 'Encountered unknown tag 'debug'.'
-            with self.assertRaises(jinja2.exceptions.TemplateSyntaxError):
-                safe_jinja.render(attempt)
-
-    # https://book.hacktricks.xyz/pentesting-web/ssti-server-side-template-injection/jinja2-ssti#accessing-global-objects
-    def test_exception_empty_calls(self):
-        import jinja2
-        attempt_list = [
-            "{{config}}",
-            "{{ debug }}"
-            "{{[].__class__}}",
-        ]
-        for attempt in attempt_list:
-            self.assertEqual(len(safe_jinja.render(attempt)), 0, f"string test '{attempt}' is correctly empty")
-
-
-
-if __name__ == '__main__':
-    unittest.main()

changedetectionio/tests/util.py

@@ -116,7 +116,7 @@ def extract_UUID_from_client(client):
     )
     # <span id="api-key">{{api_key}}</span>
 
-    m = re.search('edit/(.+?)[#"]', str(res.data))
+    m = re.search('edit/(.+?)"', str(res.data))
     uuid = m.group(1)
     return uuid.strip()
 
@@ -242,28 +242,5 @@ def live_server_setup(live_server):
             resp.headers['Content-Type'] = 'application/pdf'
             return resp
 
-    @live_server.app.route('/test-interactive-html-endpoint')
-    def test_interactive_html_endpoint():
-        header_text=""
-        for k,v in request.headers.items():
-            header_text += f"{k}: {v}<br>"
-
-        resp = make_response(f"""
-        <html>
-          <body>
-          Primitive JS check for <pre>changedetectionio/tests/visualselector/test_fetch_data.py</pre>
-            <p id="remove">This text should be removed</p>
-              <form onsubmit="event.preventDefault();">
-            <!-- obfuscated text so that we dont accidentally get a false positive due to conversion of the source :) --->
-                <button name="test-button" onclick="getElementById('remove').remove();getElementById('some-content').innerHTML = atob('SSBzbWVsbCBKYXZhU2NyaXB0IGJlY2F1c2UgdGhlIGJ1dHRvbiB3YXMgcHJlc3NlZCE=')">Click here</button>
-                <div id=some-content></div>
-                <pre>
-                {header_text.lower()}
-                </pre>
-              </body>
-         </html>""", 200)
-        resp.headers['Content-Type'] = 'text/html'
-        return resp
-
     live_server.start()
 

changedetectionio/tests/visualselector/test_fetch_data.py

@@ -7,19 +7,15 @@ from ..util import live_server_setup, wait_for_all_checks, extract_UUID_from_cli
 def test_setup(client, live_server):
     live_server_setup(live_server)
 
-
 # Add a site in paused mode, add an invalid filter, we should still have visual selector data ready
 def test_visual_selector_content_ready(client, live_server):
-
     import os
     import json
 
     assert os.getenv('PLAYWRIGHT_DRIVER_URL'), "Needs PLAYWRIGHT_DRIVER_URL set for this test"
 
     # Add our URL to the import page, because the docker container (playwright/selenium) wont be able to connect to our usual test url
-    test_url = url_for('test_interactive_html_endpoint', _external=True)
-    test_url = test_url.replace('localhost.localdomain', 'cdio')
-    test_url = test_url.replace('localhost', 'cdio')
+    test_url = "https://changedetection.io/ci-test/test-runjs.html"
 
     res = client.post(
         url_for("form_quick_watch_add"),
@@ -27,31 +23,28 @@ def test_visual_selector_content_ready(client, live_server):
         follow_redirects=True
     )
     assert b"Watch added in Paused state, saving will unpause" in res.data
-    uuid = extract_UUID_from_client(client)
+
     res = client.post(
-        url_for("edit_page", uuid=uuid, unpause_on_save=1),
+        url_for("edit_page", uuid="first", unpause_on_save=1),
         data={
-            "url": test_url,
-            "tags": "",
-            # For now, cookies doesnt work in headers because it must be a full cookiejar object
-            'headers': "testheader: yes\buser-agent: MyCustomAgent",
-            'fetch_backend': "html_webdriver",
+              "url": test_url,
+              "tags": "",
+              "headers": "",
+              'fetch_backend': "html_webdriver",
+              'webdriver_js_execute_code': 'document.querySelector("button[name=test-button]").click();'
         },
         follow_redirects=True
     )
     assert b"unpaused" in res.data
     wait_for_all_checks(client)
+    uuid = extract_UUID_from_client(client)
 
-
-    assert live_server.app.config['DATASTORE'].data['watching'][uuid].history_n >= 1, "Watch history had atleast 1 (everything fetched OK)"
-
+    # Check the JS execute code before extract worked
     res = client.get(
-        url_for("preview_page", uuid=uuid),
+        url_for("preview_page", uuid="first"),
         follow_redirects=True
     )
-    assert b"testheader: yes" in res.data
-    assert b"user-agent: mycustomagent" in res.data
-
+    assert b'I smell JavaScript' in res.data
 
     assert os.path.isfile(os.path.join('test-datastore', uuid, 'last-screenshot.png')), "last-screenshot.png should exist"
     assert os.path.isfile(os.path.join('test-datastore', uuid, 'elements.json')), "xpath elements.json data should exist"
@@ -81,32 +74,30 @@ def test_visual_selector_content_ready(client, live_server):
 
 def test_basic_browserstep(client, live_server):
 
-    #live_server_setup(live_server)
     assert os.getenv('PLAYWRIGHT_DRIVER_URL'), "Needs PLAYWRIGHT_DRIVER_URL set for this test"
+    #live_server_setup(live_server)
 
-    test_url = url_for('test_interactive_html_endpoint', _external=True)
-    test_url = test_url.replace('localhost.localdomain', 'cdio')
-    test_url = test_url.replace('localhost', 'cdio')
+    # Add our URL to the import page, because the docker container (playwright/selenium) wont be able to connect to our usual test url
+    test_url = "https://changedetection.io/ci-test/test-runjs.html"
 
     res = client.post(
         url_for("form_quick_watch_add"),
         data={"url": test_url, "tags": '', 'edit_and_watch_submit_button': 'Edit > Watch'},
         follow_redirects=True
     )
-
     assert b"Watch added in Paused state, saving will unpause" in res.data
 
     res = client.post(
         url_for("edit_page", uuid="first", unpause_on_save=1),
         data={
-            "url": test_url,
-            "tags": "",
-            'fetch_backend': "html_webdriver",
-            'browser_steps-0-operation': 'Click element',
-            'browser_steps-0-selector': 'button[name=test-button]',
-            'browser_steps-0-optional_value': '',
-            # For now, cookies doesnt work in headers because it must be a full cookiejar object
-            'headers': "testheader: yes\buser-agent: MyCustomAgent",
+              "url": test_url,
+              "tags": "",
+              "headers": "",
+              'fetch_backend': "html_webdriver",
+              'browser_steps-0-operation': 'Goto site',
+              'browser_steps-1-operation': 'Click element',
+              'browser_steps-1-selector': 'button[name=test-button]',
+              'browser_steps-1-optional_value': ''
         },
         follow_redirects=True
     )
@@ -114,9 +105,6 @@ def test_basic_browserstep(client, live_server):
     wait_for_all_checks(client)
 
     uuid = extract_UUID_from_client(client)
-    assert live_server.app.config['DATASTORE'].data['watching'][uuid].history_n >= 1, "Watch history had atleast 1 (everything fetched OK)"
-
-    assert b"This text should be removed" not in res.data
 
     # Check HTML conversion detected and workd
     res = client.get(
@@ -126,23 +114,18 @@ def test_basic_browserstep(client, live_server):
     assert b"This text should be removed" not in res.data
     assert b"I smell JavaScript because the button was pressed" in res.data
 
-    assert b"testheader: yes" in res.data
-    assert b"user-agent: mycustomagent" in res.data
-
-    four_o_four_url =  url_for('test_endpoint', status_code=404, _external=True)
-    four_o_four_url = four_o_four_url.replace('localhost.localdomain', 'cdio')
-    four_o_four_url = four_o_four_url.replace('localhost', 'cdio')
-
     # now test for 404 errors
     res = client.post(
         url_for("edit_page", uuid=uuid, unpause_on_save=1),
         data={
-              "url": four_o_four_url,
+              "url": "https://changedetection.io/404",
               "tags": "",
+              "headers": "",
               'fetch_backend': "html_webdriver",
-              'browser_steps-0-operation': 'Click element',
-              'browser_steps-0-selector': 'button[name=test-button]',
-              'browser_steps-0-optional_value': ''
+              'browser_steps-0-operation': 'Goto site',
+              'browser_steps-1-operation': 'Click element',
+              'browser_steps-1-selector': 'button[name=test-button]',
+              'browser_steps-1-optional_value': ''
         },
         follow_redirects=True
     )

changedetectionio/update_worker.py

@@ -1,12 +1,11 @@
-from . import content_fetchers
-from .processors.restock_diff import UnableToExtractRestockData
-from .processors.text_json_diff import FilterNotFoundInResponse
-from changedetectionio import html_tools
-from copy import deepcopy
 import os
-import queue
 import threading
+import queue
 import time
+from . import content_fetchers
+from changedetectionio import html_tools
+from .processors.text_json_diff import FilterNotFoundInResponse
+from .processors.restock_diff import UnableToExtractRestockData
 
 # A single update worker
 #
@@ -246,18 +245,14 @@ class update_worker(threading.Thread):
                     contents = b''
                     process_changedetection_results = True
                     update_obj = {}
-
-                    # Clear last errors (move to preflight func?)
-                    self.datastore.data['watching'][uuid]['browser_steps_last_error_step'] = None
-
-                    watch = self.datastore.data['watching'].get(uuid)
-
-                    logger.info(f"Processing watch UUID {uuid} Priority {queued_item_data.priority} URL {watch['url']}")
+                    logger.info(f"Processing watch UUID {uuid} "
+                            f"Priority {queued_item_data.priority} "
+                            f"URL {self.datastore.data['watching'][uuid]['url']}")
                     now = time.time()
 
                     try:
                         # Processor is what we are using for detecting the "Change"
-                        processor = watch.get('processor', 'text_json_diff')
+                        processor = self.datastore.data['watching'][uuid].get('processor', 'text_json_diff')
                         # if system...
 
                         # Abort processing when the content was the same as the last fetch
@@ -277,12 +272,14 @@ class update_worker(threading.Thread):
                                                                                watch_uuid=uuid
                                                                                )
 
+                        # Clear last errors (move to preflight func?)
+                        self.datastore.data['watching'][uuid]['browser_steps_last_error_step'] = None
+
                         update_handler.call_browser()
 
-                        changed_detected, update_obj, contents = update_handler.run_changedetection(
-                            watch=watch,
-                            skip_when_checksum_same=skip_when_same_checksum,
-                        )
+                        changed_detected, update_obj, contents = update_handler.run_changedetection(uuid,
+                                                                                    skip_when_checksum_same=skip_when_same_checksum,
+                                                                                    )
 
                         # Re #342
                         # In Python 3, all strings are sequences of Unicode characters. There is a bytes type that holds raw bytes.
@@ -312,11 +309,7 @@ class update_worker(threading.Thread):
                         })
 
                         if e.screenshot:
-                            watch.save_screenshot(screenshot=e.screenshot, as_error=True)
-
-                        if e.xpath_data:
-                            watch.save_xpath_data(data=e.xpath_data)
-                            
+                            self.datastore.save_screenshot(watch_uuid=uuid, screenshot=e.screenshot)
                         process_changedetection_results = False
 
                     except content_fetchers.exceptions.Non200ErrorCodeReceived as e:
@@ -332,11 +325,11 @@ class update_worker(threading.Thread):
                             err_text = "Error - Request returned a HTTP error code {}".format(str(e.status_code))
 
                         if e.screenshot:
-                            watch.save_screenshot(screenshot=e.screenshot, as_error=True)
+                            self.datastore.save_screenshot(watch_uuid=uuid, screenshot=e.screenshot, as_error=True)
                         if e.xpath_data:
-                            watch.save_xpath_data(data=e.xpath_data, as_error=True)
+                            self.datastore.save_xpath_data(watch_uuid=uuid, data=e.xpath_data, as_error=True)
                         if e.page_text:
-                            watch.save_error_text(contents=e.page_text)
+                            self.datastore.save_error_text(watch_uuid=uuid, contents=e.page_text)
 
                         self.datastore.update_watch(uuid=uuid, update_obj={'last_error': err_text})
                         process_changedetection_results = False
@@ -348,23 +341,16 @@ class update_worker(threading.Thread):
                         err_text = "Warning, no filters were found, no change detection ran - Did the page change layout? update your Visual Filter if necessary."
                         self.datastore.update_watch(uuid=uuid, update_obj={'last_error': err_text})
 
-                        # Filter wasnt found, but we should still update the visual selector so that they can have a chance to set it up again
-                        if e.screenshot:
-                            watch.save_screenshot(screenshot=e.screenshot)
-
-                        if e.xpath_data:
-                            watch.save_xpath_data(data=e.xpath_data)
-
                         # Only when enabled, send the notification
-                        if watch.get('filter_failure_notification_send', False):
-                            c = watch.get('consecutive_filter_failures', 5)
+                        if self.datastore.data['watching'][uuid].get('filter_failure_notification_send', False):
+                            c = self.datastore.data['watching'][uuid].get('consecutive_filter_failures', 5)
                             c += 1
                             # Send notification if we reached the threshold?
                             threshold = self.datastore.data['settings']['application'].get('filter_failure_notification_threshold_attempts',
                                                                                            0)
-                            logger.warning(f"Filter for {uuid} not found, consecutive_filter_failures: {c}")
+                            logger.error(f"Filter for {uuid} not found, consecutive_filter_failures: {c}")
                             if threshold > 0 and c >= threshold:
-                                if not watch.get('notification_muted'):
+                                if not self.datastore.data['watching'][uuid].get('notification_muted'):
                                     self.send_filter_failure_notification(uuid)
                                 c = 0
 
@@ -376,6 +362,7 @@ class update_worker(threading.Thread):
                         # Yes fine, so nothing todo, don't continue to process.
                         process_changedetection_results = False
                         changed_detected = False
+                        self.datastore.update_watch(uuid=uuid, update_obj={'last_error': False})
                     except content_fetchers.exceptions.BrowserConnectError as e:
                         self.datastore.update_watch(uuid=uuid,
                                                     update_obj={'last_error': e.msg
@@ -414,15 +401,15 @@ class update_worker(threading.Thread):
                                                                 }
                                                     )
 
-                        if watch.get('filter_failure_notification_send', False):
-                            c = watch.get('consecutive_filter_failures', 5)
+                        if self.datastore.data['watching'][uuid].get('filter_failure_notification_send', False):
+                            c = self.datastore.data['watching'][uuid].get('consecutive_filter_failures', 5)
                             c += 1
                             # Send notification if we reached the threshold?
                             threshold = self.datastore.data['settings']['application'].get('filter_failure_notification_threshold_attempts',
                                                                                            0)
                             logger.error(f"Step for {uuid} not found, consecutive_filter_failures: {c}")
                             if threshold > 0 and c >= threshold:
-                                if not watch.get('notification_muted'):
+                                if not self.datastore.data['watching'][uuid].get('notification_muted'):
                                     self.send_step_failure_notification(watch_uuid=uuid, step_n=e.step_n)
                                 c = 0
 
@@ -444,7 +431,7 @@ class update_worker(threading.Thread):
                     except content_fetchers.exceptions.JSActionExceptions as e:
                         err_text = "Error running JS Actions - Page request - "+e.message
                         if e.screenshot:
-                            watch.save_screenshot(screenshot=e.screenshot, as_error=True)
+                            self.datastore.save_screenshot(watch_uuid=uuid, screenshot=e.screenshot, as_error=True)
                         self.datastore.update_watch(uuid=uuid, update_obj={'last_error': err_text,
                                                                            'last_check_status': e.status_code})
                         process_changedetection_results = False
@@ -454,7 +441,7 @@ class update_worker(threading.Thread):
                             err_text = "{} - {}".format(err_text, e.message)
 
                         if e.screenshot:
-                            watch.save_screenshot(screenshot=e.screenshot, as_error=True)
+                            self.datastore.save_screenshot(watch_uuid=uuid, screenshot=e.screenshot, as_error=True)
 
                         self.datastore.update_watch(uuid=uuid, update_obj={'last_error': err_text,
                                                                            'last_check_status': e.status_code,
@@ -475,9 +462,11 @@ class update_worker(threading.Thread):
                     except Exception as e:
                         logger.error(f"Exception reached processing watch UUID: {uuid}")
                         logger.error(str(e))
-                        self.datastore.update_watch(uuid=uuid, update_obj={'last_error': "Exception: " + str(e)})
+                        self.datastore.update_watch(uuid=uuid, update_obj={'last_error': str(e)})
                         # Other serious error
                         process_changedetection_results = False
+#                        import traceback
+#                        print(traceback.format_exc())
 
                     else:
                         # Crash protection, the watch entry could have been removed by this point (during a slow chrome fetch etc)
@@ -485,7 +474,7 @@ class update_worker(threading.Thread):
                             continue
 
                         # Mark that we never had any failures
-                        if not watch.get('ignore_status_codes'):
+                        if not self.datastore.data['watching'][uuid].get('ignore_status_codes'):
                             update_obj['consecutive_filter_failures'] = 0
 
                         # Everything ran OK, clean off any previous error
@@ -493,48 +482,25 @@ class update_worker(threading.Thread):
 
                         self.cleanup_error_artifacts(uuid)
 
-                    if not self.datastore.data['watching'].get(uuid):
-                        continue
                     #
                     # Different exceptions mean that we may or may not want to bump the snapshot, trigger notifications etc
                     if process_changedetection_results:
-                        # Always save the screenshot if it's available
-
-                        if update_handler.screenshot:
-                            watch.save_screenshot(screenshot=update_handler.screenshot)
-
-                        if update_handler.xpath_data:
-                            watch.save_xpath_data(data=update_handler.xpath_data)
-
                         try:
+                            watch = self.datastore.data['watching'].get(uuid)
                             self.datastore.update_watch(uuid=uuid, update_obj=update_obj)
 
                             # Also save the snapshot on the first time checked
-                            if changed_detected or not watch.get('last_checked'):
-                                timestamp = round(time.time())
-
-                                # Small hack so that we sleep just enough to allow 1 second  between history snapshots
-                                # this is because history.txt indexes/keys snapshots by epoch seconds and we dont want dupe keys
-
-                                if watch.newest_history_key and int(timestamp) == int(watch.newest_history_key):
-                                    logger.warning(
-                                        f"Timestamp {timestamp} already exists, waiting 1 seconds so we have a unique key in history.txt")
-                                    timestamp = str(int(timestamp) + 1)
-                                    time.sleep(1)
-
+                            if changed_detected or not watch['last_checked']:
                                 watch.save_history_text(contents=contents,
-                                                        timestamp=timestamp,
+                                                        timestamp=str(round(time.time())),
                                                         snapshot_id=update_obj.get('previous_md5', 'none'))
 
-                                if update_handler.fetcher.content:
-                                    watch.save_last_fetched_html(contents=update_handler.fetcher.content, timestamp=timestamp)
-
                             # A change was detected
                             if changed_detected:
                                 # Notifications should only trigger on the second time (first time, we gather the initial snapshot)
                                 if watch.history_n >= 2:
                                     logger.info(f"Change detected in UUID {uuid} - {watch['url']}")
-                                    if not watch.get('notification_muted'):
+                                    if not self.datastore.data['watching'][uuid].get('notification_muted'):
                                         self.send_content_changed_notification(watch_uuid=uuid)
                                 else:
                                     logger.info(f"Change triggered in UUID {uuid} due to first history saving (no notifications sent) - {watch['url']}")
@@ -545,23 +511,29 @@ class update_worker(threading.Thread):
                             logger.critical(str(e))
                             self.datastore.update_watch(uuid=uuid, update_obj={'last_error': str(e)})
 
+                    if self.datastore.data['watching'].get(uuid):
+                        # Always record that we atleast tried
+                        count = self.datastore.data['watching'][uuid].get('check_count', 0) + 1
 
-                    # Always record that we atleast tried
-                    count = watch.get('check_count', 0) + 1
+                        # Record the 'server' header reply, can be used for actions in the future like cloudflare/akamai workarounds
+                        try:
+                            server_header = update_handler.fetcher.headers.get('server', '').strip().lower()[:255]
+                            self.datastore.update_watch(uuid=uuid,
+                                                        update_obj={'remote_server_reply': server_header}
+                                                        )
+                        except Exception as e:
+                            pass
 
-                    # Record the 'server' header reply, can be used for actions in the future like cloudflare/akamai workarounds
-                    try:
-                        server_header = update_handler.fetcher.headers.get('server', '').strip().lower()[:255]
-                        self.datastore.update_watch(uuid=uuid,
-                                                    update_obj={'remote_server_reply': server_header}
-                                                    )
-                    except Exception as e:
-                        pass
+                        self.datastore.update_watch(uuid=uuid, update_obj={'fetch_time': round(time.time() - now, 3),
+                                                                           'last_checked': round(time.time()),
+                                                                           'check_count': count
+                                                                           })
 
-                    self.datastore.update_watch(uuid=uuid, update_obj={'fetch_time': round(time.time() - now, 3),
-                                                                       'last_checked': round(time.time()),
-                                                                       'check_count': count
-                                                                       })
+                        # Always save the screenshot if it's available
+                        if update_handler.screenshot:
+                            self.datastore.save_screenshot(watch_uuid=uuid, screenshot=update_handler.screenshot)
+                        if update_handler.xpath_data:
+                            self.datastore.save_xpath_data(watch_uuid=uuid, data=update_handler.xpath_data)
 
 
                 self.current_uuid = None  # Done

docker-compose.yml

@@ -54,9 +54,7 @@ services:
   #        
   #        Default number of parallel/concurrent fetchers
   #      - FETCH_WORKERS=10
-  #
-  #        Absolute minimum seconds to recheck, overrides any watch minimum, change to 0 to disable
-  #      - MINIMUM_SECONDS_RECHECK_TIME=3
+
       # Comment out ports: when using behind a reverse proxy , enable networks: etc.
       ports:
         - 5000:5000
@@ -68,10 +66,9 @@ services:
      
      # If WEBDRIVER or PLAYWRIGHT are enabled, changedetection container depends on that
      # and must wait before starting (substitute "browser-chrome" with "playwright-chrome" if last one is used)
-#      depends_on:
-#          playwright-chrome:
-#              condition: service_started
-
+#    depends_on:
+#        browser-chrome:
+#            condition: service_started
 
      # Used for fetching pages via Playwright+Chrome where you need Javascript support.
      # RECOMMENDED FOR FETCHING PAGES WITH CHROME

requirements.txt

@@ -1,7 +1,7 @@
 # Used by Pyppeteer
 pyee
 
-eventlet>=0.36.1 # fixes SSL error on Python 3.12
+eventlet>=0.33.3 # related to dnspython fixes
 feedgen~=0.9
 flask-compress
 # 0.6.3 included compatibility fix for werkzeug 3.x (2.x had deprecation of url handlers)
@@ -9,7 +9,6 @@ flask-login>=0.6.3
 flask-paginate
 flask_expects_json~=1.7
 flask_restful
-flask_cors # For the Chrome extension to operate
 flask_wtf~=1.2
 flask~=2.3
 inscriptis~=2.2
@@ -23,35 +22,32 @@ validators~=0.21
 brotli~=1.0
 requests[socks]
 
-urllib3==1.26.19
+urllib3>1.26
 chardet>2.3.0
 
 wtforms~=3.0
 jsonpath-ng~=1.5.3
 
-dnspython==2.6.1 # related to eventlet fixes
+dnspython~=2.4 # related to eventlet fixes
 
 # jq not available on Windows so must be installed manually
 
 # Notification library
-apprise~=1.8.0
+apprise~=1.7.1
 
 # apprise mqtt https://github.com/dgtlmoon/changedetection.io/issues/315
-# and 2.0.0 https://github.com/dgtlmoon/changedetection.io/issues/2241 not yet compatible
-# use v1.x due to https://github.com/eclipse/paho.mqtt.python/issues/814
-paho-mqtt>=1.6.1,<2.0.0
+paho-mqtt
 
-# Requires extra wheel for rPi
-cryptography~=42.0.8
+# This mainly affects some ARM builds, which unlike the other builds ignores "ARG CRYPTOGRAPHY_DONT_BUILD_RUST=1"
+# so without this pinning, the newer versions on ARM will forcefully try to build rust, which results in "rust compiler not found"
+# (introduced once apprise became a dep)
+cryptography~=3.4
 
 # Used for CSS filtering
 beautifulsoup4
 
 # XPath filtering, lxml is required by bs4 anyway, but put it here to be safe.
-# #2328 - 5.2.0 and 5.2.1 had extra CPU flag CFLAGS set which was not compatible on older hardware
-#         It could be advantageous to run its own pypi package here with those performance flags set
-#         https://bugs.launchpad.net/lxml/+bug/2059910/comments/16
-lxml >=4.8.0,<6,!=5.2.0,!=5.2.1
+lxml
 
 # XPath 2.0-3.1 support - 4.2.0 broke something?
 elementpath==4.1.5
@@ -69,10 +65,12 @@ openpyxl
 jq~=1.3; python_version >= "3.8" and sys_platform == "darwin"
 jq~=1.3; python_version >= "3.8" and sys_platform == "linux"
 
+# Any current modern version, required so far for screenshot PNG->JPEG conversion but will be used more in the future
+pillow
 # playwright is installed at Dockerfile build time because it's not available on all platforms
 
-pyppeteer-ng==2.0.0rc5
-pyppeteerstealth>=0.0.4
+# experimental release
+pyppeteer-ng==2.0.0rc2
 
 # Include pytest, so if theres a support issue we can ask them to run these tests on their setup
 pytest ~=7.2
@@ -82,5 +80,3 @@ pytest-flask ~=1.2
 jsonschema==4.17.3
 
 loguru
-# Needed for > 3.10, https://github.com/microsoft/playwright-python/issues/2096
-greenlet >= 3.0.3